From 8203666e586b8304f4784bedec46f5e6e3a3dc3b Mon Sep 17 00:00:00 2001 From: Nataly Musilah <115026536+Musilah@users.noreply.github.com> Date: Thu, 22 Jan 2026 16:16:52 +0300 Subject: [PATCH] NOISSUE - Update READMEs (#380) * fix web url Signed-off-by: Musilah * add new readmes Signed-off-by: Musilah * add other services Signed-off-by: Musilah * fix docker link Signed-off-by: Musilah * fix reports examples Signed-off-by: Musilah * revert go.sum change Signed-off-by: Musilah --------- Signed-off-by: Musilah --- MAINTAINERS | 4 +- README.md | 52 ++-- alarms/README.md | 197 +++++++++++++ apidocs/asyncapi/mqtt.yaml | 2 +- apidocs/asyncapi/websocket.yaml | 2 +- apidocs/openapi/README.md | 2 +- apidocs/openapi/alarms.yaml | 2 +- apidocs/openapi/bootstrap.yaml | 2 +- apidocs/openapi/notifiers.yaml | 2 +- apidocs/openapi/readers.yaml | 2 +- apidocs/openapi/rules.yaml | 2 +- bootstrap/README.md | 2 +- bootstrap/tracing/doc.go | 2 +- consumers/README.md | 4 +- consumers/notifiers/README.md | 187 +++++++++++- consumers/notifiers/tracing/doc.go | 2 +- consumers/writers/README.md | 274 +++++++++++++++++- docker/README.md | 8 +- provision/README.md | 198 ++++++++----- re/README.md | 450 ++++++++++++++++++----------- readers/README.md | 251 +++++++++++++++- readers/postgres/README.md | 2 +- readers/timescale/README.md | 2 +- reports/README.md | 335 +++++++++++++++++++++ tools/e2e/README.md | 2 +- tools/e2e/cmd/main.go | 2 +- tools/mqtt-bench/README.md | 2 +- tools/mqtt-bench/cmd/main.go | 2 +- tools/provision/README.md | 2 +- tools/provision/cmd/main.go | 2 +- 30 files changed, 1670 insertions(+), 328 deletions(-) create mode 100644 alarms/README.md create mode 100644 reports/README.md diff --git a/MAINTAINERS b/MAINTAINERS index 8df02cf48..51ec57df8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7,7 +7,7 @@ [[drasko]] Name = "Drasko Draskovic" - Email = "draasko.draskovic@abstractmachines.fr" + Email = "draasko.draskovic@absmach.eu" GitHub = "drasko" # However, this role serves only in dead-lock events, or in a special and very rare cases @@ -26,5 +26,5 @@ [[dusan]] Name = "Dusan Borovcanin" - Email = "dusan.borovcanin@abstractmachines.fr" + Email = "dusan.borovcanin@absmach.eu" GitHub = "dborovcanin" diff --git a/README.md b/README.md index 38687c7f3..4f8e65cda 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,36 @@
- # Magistrala - - **A Modern IoT Platform Built on SuperMQ** - - **Scalable • Secure • Open-Source** - - [![Check License Header](https://github.com/absmach/magistrala/actions/workflows/check-license.yaml/badge.svg?branch=main)](https://github.com/absmach/magistrala/actions/workflows/check-license.yaml) - [![Continuous Delivery](https://github.com/absmach/magistrala/actions/workflows/build.yaml/badge.svg?branch=main)](https://github.com/absmach/magistrala/actions/workflows/build.yaml) - [![Go Report Card](https://goreportcard.com/badge/github.com/absmach/magistrala)](https://goreportcard.com/report/github.com/absmach/magistrala) - [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/absmach/magistrala) - [![Coverage](https://codecov.io/gh/absmach/magistrala/graph/badge.svg?token=SEMDAO3L09)](https://codecov.io/gh/absmach/magistrala) - [![License](https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square)](LICENSE) - [![Matrix](https://img.shields.io/matrix/magistrala:matrix.org?style=flat-square)](https://matrix.to/#/#magistrala:matrix.org) - - ### [Guide](https://docs.magistrala.absmach.eu) | [Contributing](CONTRIBUTING.md) | [Website](https://www.absmach.eu/magistrala) | [Chat](https://matrix.to/#/#magistrala:matrix.org) +# Magistrala - Made with ❤️ by [Abstract Machines](https://www.absmach.eu) +### A Modern IoT Platform Built on SuperMQ + +### Scalable • Secure • Open-Source + +[![Check License Header](https://github.com/absmach/magistrala/actions/workflows/check-license.yaml/badge.svg?branch=main)](https://github.com/absmach/magistrala/actions/workflows/check-license.yaml) +[![Continuous Delivery](https://github.com/absmach/magistrala/actions/workflows/build.yaml/badge.svg?branch=main)](https://github.com/absmach/magistrala/actions/workflows/build.yaml) +[![Go Report Card](https://goreportcard.com/badge/github.com/absmach/magistrala)](https://goreportcard.com/report/github.com/absmach/magistrala) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/absmach/magistrala) +[![Coverage](https://codecov.io/gh/absmach/magistrala/graph/badge.svg?token=SEMDAO3L09)](https://codecov.io/gh/absmach/magistrala) +[![License](https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square)](LICENSE) +[![Matrix](https://img.shields.io/matrix/magistrala:matrix.org?style=flat-square)](https://matrix.to/#/#magistrala:matrix.org) + +### [Guide](https://docs.magistrala.absmach.eu) | [Contributing](CONTRIBUTING.md) | [Website](https://www.absmach.eu/magistrala) | [Chat](https://matrix.to/#/#magistrala:matrix.org) + +Made with ❤️ by [Abstract Machines](https://www.absmach.eu)
- ## Introduction 🌍 Magistrala is a cutting-edge, open-source IoT cloud platform built on top of [SuperMQ](https://github.com/absmach/supermq). It serves as a robust middleware solution for building complex IoT applications. With Magistrala, you can connect and manage IoT devices seamlessly using multi-protocol support, all while ensuring security and scalability. -### Key Benefits: -- **Unified IoT Management**: Connect sensors, actuators, and applications over various network protocols. +### Key Benefits + +- **Unified IoT Management**: Connect sensors, actuators and applications over various network protocols. - **Scalability and Performance**: Designed to handle enterprise-grade IoT deployments. - **Secure by Design**: Features such as mutual TLS authentication and fine-grained access control. - **Open-Source Freedom**: Patent-free, community-driven, and designed for extensibility. - ## ✨ Features - 🏢 **Multi-Tenancy**: Support for managing multiple independent domains seamlessly. @@ -52,7 +51,6 @@ Magistrala is a cutting-edge, open-source IoT cloud platform built on top of [Su - 🛠️ **Developer Tools**: Comprehensive SDK and CLI for efficient development. - 🏗️ **Domain-Driven Design**: High-quality codebase and extensive test coverage. - ## Installation 🛠️ There are multiple ways to run Magistrala. @@ -91,7 +89,7 @@ git checkout main ## 📤 Usage -#### Using the CLI: +**Using the CLI :** Check the health of a specific service using the CLI: @@ -102,9 +100,9 @@ make cli Replace `` with the name of the service you want to check. -#### Using Curl: +**Using Curl :** -Alternatively, use a simple HTTP GET request to check the platform's health: +Alternatively, use a simple HTTP `GET` request to check the platform's health: ```bash curl -X GET http://localhost:8080/health @@ -112,13 +110,11 @@ curl -X GET http://localhost:8080/health For additional usage examples and advanced configurations, visit the [official documentation](https://docs.magistrala.absmach.eu). - ## 📚 Documentation Complete documentation is available at the [Magistrala official docs page](https://docs.magistrala.absmach.eu). -For CLI usage details, visit the [CLI Documentation](https://docs.magistrala.absmach.eu/cli). - +For CLI usage details, visit the [CLI Documentation](https://docs.magistrala.absmach.eu/dev-guide/cli/introduction-to-cli). ## 🌐 Community and Contributing @@ -128,12 +124,10 @@ Join the community and contribute to the future of IoT middleware: - [Contribution Guide](CONTRIBUTING.md) - [Matrix Chat](https://matrix.to/#/#magistrala:matrix.org) - ## 📜 License Magistrala is open-source software licensed under the [Apache-2.0](LICENSE) license. Contributions are welcome and encouraged! - ## 💼 Professional Support Need help deploying Magistrala or integrating it into your systems? Contact **[Abstract Machines](https://www.absmach.eu)** for expert guidance and support. diff --git a/alarms/README.md b/alarms/README.md new file mode 100644 index 000000000..7df866f74 --- /dev/null +++ b/alarms/README.md @@ -0,0 +1,197 @@ +# Alarms + +The Alarms service stores, manages and exposes alarms raised by rules and device activity. It consumes alarm events from the message broker, persists them to PostgreSQL, and provides an HTTP API for listing, viewing, updating, and deleting alarms with full authn/authz, metrics, and tracing support. + +## Configuration + +The service is configured using the following environment variables (values shown are from [docker/.env](https://github.com/absmach/magistrala/blob/main/docker/.env) as consumed by [docker/docker-compose.yaml](https://github.com/absmach/magistrala/blob/main/docker/docker-compose.yaml)): + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_ALARMS_LOG_LEVEL` | Log level for the service | `debug` | +| `MG_ALARMS_HTTP_HOST` | HTTP host to bind | `alarms` | +| `MG_ALARMS_HTTP_PORT` | HTTP port to bind | `8050` | +| `MG_ALARMS_HTTP_SERVER_CERT` | Path to PEM-encoded HTTPS server certificate | "" | +| `MG_ALARMS_HTTP_SERVER_KEY` | Path to PEM-encoded HTTPS server key | "" | +| `MG_ALARMS_DB_HOST` | PostgreSQL host | `alarms-db` | +| `MG_ALARMS_DB_PORT` | PostgreSQL port | `5432` | +| `MG_ALARMS_DB_USER` | PostgreSQL user | `magistrala` | +| `MG_ALARMS_DB_PASS` | PostgreSQL password | `magistrala` | +| `MG_ALARMS_DB_NAME` | PostgreSQL database name | `alarms` | +| `MG_ALARMS_DB_SSL_MODE` | PostgreSQL SSL mode | `disable` | +| `MG_ALARMS_DB_SSL_CERT` | PostgreSQL SSL client cert | "" | +| `MG_ALARMS_DB_SSL_KEY` | PostgreSQL SSL client key | "" | +| `MG_ALARMS_DB_SSL_ROOT_CERT` | PostgreSQL SSL root cert | "" | +| `MG_ALARMS_INSTANCE_ID` | Instance ID for tracing/health | "" | +| `SMQ_MESSAGE_BROKER_URL` | Message broker URL for alarm ingestion | `nats://nats:4222` | +| `SMQ_JAEGER_URL` | Jaeger collector endpoint | `http://jaeger:4318/v1/traces` | +| `SMQ_JAEGER_TRACE_RATIO` | Trace sampling ratio | `1.0` | +| `SMQ_AUTH_GRPC_URL` | Auth gRPC endpoint | `auth:7001` | +| `SMQ_AUTH_GRPC_TIMEOUT` | Auth gRPC timeout | `300s` | +| `SMQ_AUTH_GRPC_CLIENT_CERT` | Auth gRPC client cert path | `${GRPC_MTLS:+./ssl/certs/auth-grpc-client.crt}` | +| `SMQ_AUTH_GRPC_CLIENT_KEY` | Auth gRPC client key path | `${GRPC_MTLS:+./ssl/certs/auth-grpc-client.key}` | +| `SMQ_AUTH_GRPC_SERVER_CA_CERTS` | Auth gRPC server CA path | `${GRPC_MTLS:+./ssl/certs/ca.crt}` | +| `SMQ_DOMAINS_GRPC_URL` | Domains gRPC endpoint | `domains:7003` | +| `SMQ_DOMAINS_GRPC_TIMEOUT` | Domains gRPC timeout | `300s` | +| `SMQ_DOMAINS_GRPC_CLIENT_CERT` | Domains gRPC client cert path | `${GRPC_MTLS:+./ssl/certs/domains-grpc-client.crt}` | +| `SMQ_DOMAINS_GRPC_CLIENT_KEY` | Domains gRPC client key path | `${GRPC_MTLS:+./ssl/certs/domains-grpc-client.key}` | +| `SMQ_DOMAINS_GRPC_SERVER_CA_CERTS` | Domains gRPC server CA path | `${GRPC_MTLS:+./ssl/certs/ca.crt}` | +| `SMQ_ALLOW_UNVERIFIED_USER` | Allow unverified users to access | `true` | + +## Features + +- **Alarm ingestion**: Consumes alarms from the message broker and persists them to PostgreSQL. +- **Stateful updates**: Updates assignee, acknowledgment, resolution, and metadata fields. +- **Filtering and paging**: Lists alarms by domain, rule, channel, client, subtopic, status, severity, and time range. +- **Observability**: `/metrics` Prometheus endpoint and Jaeger tracing support. +- **Auth and authorization**: Authn/authz enforced via gRPC auth and domains services. + +## Architecture + +### Runtime flow + +1. The message broker publishes alarm events under the `alarms.>` subject. +2. The Alarms consumer decodes the event payload, enriches it with message metadata, validates it, and calls `CreateAlarm`. +3. The repository writes to PostgreSQL while deduplicating repeated active alarms with the same severity. +4. The HTTP API exposes list/view/update/delete operations with authn/authz, metrics, and tracing middleware. + +### Components + +- **HTTP API**: `alarms/api` exposes REST endpoints and health/metrics handlers. +- **Service layer**: `alarms/service.go` validates requests and coordinates repository operations. +- **Repository**: `alarms/postgres/alarms.go` implements persistence and filtering. +- **Consumer**: `alarms/consumer` processes broker messages and creates alarms. +- **Message broker**: `alarms/brokers` uses NATS JetStream with stream `alarms` and subject `alarms.>`. +- **Migrations**: `alarms/postgres/init.go` defines the alarms schema and indexes. + +### Alarms table + +Defined in `alarms/postgres/init.go`: + +| Column | Type | Description | +| --- | --- | --- | +| `id` | `VARCHAR(36)` | Alarm UUID (primary key) | +| `rule_id` | `VARCHAR(36)` | Rule ID that triggered the alarm | +| `domain_id` | `VARCHAR(36)` | Domain ID | +| `channel_id` | `VARCHAR(36)` | Channel ID | +| `subtopic` | `TEXT` | Subtopic associated with the alarm | +| `client_id` | `VARCHAR(36)` | Client ID | +| `measurement` | `TEXT` | Measurement name | +| `value` | `TEXT` | Measured value | +| `unit` | `TEXT` | Measurement unit | +| `threshold` | `TEXT` | Threshold value | +| `cause` | `TEXT` | Cause/description | +| `status` | `SMALLINT` | 0 = active, 1 = cleared | +| `severity` | `SMALLINT` | Severity (0-100) | +| `assignee_id` | `VARCHAR(36)` | Assignee ID | +| `created_at` | `TIMESTAMPTZ` | Creation timestamp | +| `updated_at` | `TIMESTAMPTZ` | Last update timestamp | +| `updated_by` | `VARCHAR(36)` | User who updated | +| `assigned_at` | `TIMESTAMPTZ` | When assigned | +| `assigned_by` | `VARCHAR(36)` | Who assigned | +| `acknowledged_at` | `TIMESTAMPTZ` | When acknowledged | +| `acknowledged_by` | `VARCHAR(36)` | Who acknowledged | +| `resolved_at` | `TIMESTAMPTZ` | When resolved | +| `resolved_by` | `VARCHAR(36)` | Who resolved | +| `metadata` | `JSONB` | Custom metadata | + +Index: `idx_alarms_state (domain_id, rule_id, channel_id, subtopic, client_id, measurement, created_at DESC)` + +## Deployment + +### Build and run locally + +```bash +make alarms + +MG_ALARMS_LOG_LEVEL=debug \ +MG_ALARMS_HTTP_PORT=8050 \ +MG_ALARMS_DB_HOST=localhost \ +MG_ALARMS_DB_PORT=5432 \ +MG_ALARMS_DB_USER=magistrala \ +MG_ALARMS_DB_PASS=magistrala \ +MG_ALARMS_DB_NAME=alarms \ +SMQ_MESSAGE_BROKER_URL=nats://localhost:4222 \ +SMQ_AUTH_GRPC_URL=localhost:7001 \ +SMQ_AUTH_GRPC_TIMEOUT=300s \ +SMQ_DOMAINS_GRPC_URL=localhost:7003 \ +SMQ_DOMAINS_GRPC_TIMEOUT=300s \ +./build/alarms +``` + +### Docker Compose + +The service is available as a Docker container. Refer to [docker/docker-compose.yaml](https://github.com/absmach/magistrala/blob/main/docker/docker-compose.yaml) for the `alarms` and `alarms-db` services and their environment variables. For a full local stack, make sure the auth, domains, and message broker services are also running. + +```bash +docker compose -f docker/docker-compose.yaml up alarms alarms-db +``` + +### Health check + +```bash +curl -X GET http://localhost:8050/health \ + -H "accept: application/health+json" +``` + +## Testing + +```bash +go test ./alarms/... +``` + +## Usage + +The Alarms service supports the following operations: + +| Operation | Method & Path | Description | +| --- | --- | --- | +| `listAlarms` | `GET /{domainID}/alarms` | List alarms with filters | +| `viewAlarm` | `GET /{domainID}/alarms/{alarmID}` | Retrieve a single alarm | +| `updateAlarm` | `PUT /{domainID}/alarms/{alarmID}` | Update alarm status/assignee/metadata | +| `deleteAlarm` | `DELETE /{domainID}/alarms/{alarmID}` | Delete an alarm | +| `health` | `GET /health` | Service health check | + +Alarm creation is driven by message broker events and is not exposed as an HTTP endpoint. + +### Example: List alarms + +```bash +curl -X GET "http://localhost:8050//alarms?limit=10&offset=0&status=active&severity=50" \ + -H "Authorization: Bearer " +``` + +### Example: View an alarm + +```bash +curl -X GET http://localhost:8050//alarms/ \ + -H "Authorization: Bearer " +``` + +### Example: Update an alarm + +```bash +curl -X PUT http://localhost:8050//alarms/ \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "status": "cleared", + "assignee_id": "", + "severity": 40, + "metadata": { "note": "cleared after inspection" } + }' +``` + +### Example: Delete an alarm + +```bash +curl -X DELETE http://localhost:8050//alarms/ \ + -H "Authorization: Bearer " +``` + +### Example: Health check + +```bash +curl -X GET http://localhost:8050/health \ + -H "accept: application/health+json" +``` diff --git a/apidocs/asyncapi/mqtt.yaml b/apidocs/asyncapi/mqtt.yaml index a8577701c..9e399c910 100644 --- a/apidocs/asyncapi/mqtt.yaml +++ b/apidocs/asyncapi/mqtt.yaml @@ -9,7 +9,7 @@ info: contact: name: Magistrala Team url: 'https://github.com/absmach/magistrala' - email: info@abstractmachines.fr + email: info@absmach.eu description: | MQTT adapter provides an MQTT API for sending messages through the platform. MQTT adapter uses [mProxy](https://github.com/absmach/mproxy) for proxying traffic between client and MQTT broker. Additionally, the MQTT adapter and the message broker are replicating the traffic between brokers. diff --git a/apidocs/asyncapi/websocket.yaml b/apidocs/asyncapi/websocket.yaml index 779b5c670..923c11e9b 100644 --- a/apidocs/asyncapi/websocket.yaml +++ b/apidocs/asyncapi/websocket.yaml @@ -10,7 +10,7 @@ info: contact: name: Magistrala Team url: 'https://github.com/absmach/magistrala' - email: info@abstractmachines.fr + email: info@absmach.eu license: name: Apache 2.0 url: 'https://github.com/absmach/magistrala/blob/main/LICENSE' diff --git a/apidocs/openapi/README.md b/apidocs/openapi/README.md index 09dbcfc03..a3f54979a 100644 --- a/apidocs/openapi/README.md +++ b/apidocs/openapi/README.md @@ -2,4 +2,4 @@ This folder contains an OpenAPI specifications for Magistrala API. -View specification in Swagger UI at [docs.api.magistrala.abstractmachines.fr](https://docs.api.magistrala.abstractmachines.fr) \ No newline at end of file +View specification in Swagger UI at [docs.api.magistrala.absmach.eu](https://docs.api.magistrala.absmach.eu) \ No newline at end of file diff --git a/apidocs/openapi/alarms.yaml b/apidocs/openapi/alarms.yaml index 2f2a26dac..1b0688a09 100644 --- a/apidocs/openapi/alarms.yaml +++ b/apidocs/openapi/alarms.yaml @@ -9,7 +9,7 @@ info: Some useful links: - [The Magistrala repository](https://github.com/absmach/magistrala) contact: - email: info@abstractmachines.fr + email: info@absmach.eu license: name: Apache 2.0 url: https://github.com/absmach/magistrala/blob/main/LICENSE diff --git a/apidocs/openapi/bootstrap.yaml b/apidocs/openapi/bootstrap.yaml index dd5c2fae7..56247f5ef 100644 --- a/apidocs/openapi/bootstrap.yaml +++ b/apidocs/openapi/bootstrap.yaml @@ -9,7 +9,7 @@ info: Some useful links: - [The Magistrala repository](https://github.com/absmach/magistrala) contact: - email: info@abstractmachines.fr + email: info@absmach.eu license: name: Apache 2.0 url: https://github.com/absmach/magistrala/blob/main/LICENSE diff --git a/apidocs/openapi/notifiers.yaml b/apidocs/openapi/notifiers.yaml index 85d6679a4..660f9cf3b 100644 --- a/apidocs/openapi/notifiers.yaml +++ b/apidocs/openapi/notifiers.yaml @@ -9,7 +9,7 @@ info: Some useful links: - [The Magistrala repository](https://github.com/absmach/magistrala) contact: - email: info@abstractmachines.fr + email: info@absmach.eu license: name: Apache 2.0 url: https://github.com/absmach/magistrala/blob/main/LICENSE diff --git a/apidocs/openapi/readers.yaml b/apidocs/openapi/readers.yaml index 3b86abf13..24a72bff1 100644 --- a/apidocs/openapi/readers.yaml +++ b/apidocs/openapi/readers.yaml @@ -9,7 +9,7 @@ info: Some useful links: - [The Magistrala repository](https://github.com/absmach/magistrala) contact: - email: info@abstractmachines.fr + email: info@absmach.eu license: name: Apache 2.0 url: https://github.com/absmach/magistrala/blob/main/LICENSE diff --git a/apidocs/openapi/rules.yaml b/apidocs/openapi/rules.yaml index 92e64a0a8..1df40dbb2 100644 --- a/apidocs/openapi/rules.yaml +++ b/apidocs/openapi/rules.yaml @@ -9,7 +9,7 @@ info: Some useful links: - [The Magistrala repository](https://github.com/absmach/magistrala) contact: - email: info@abstractmachines.fr + email: info@absmach.eu license: name: Apache 2.0 url: https://github.com/absmach/magistrala/blob/main/LICENSE diff --git a/bootstrap/README.md b/bootstrap/README.md index f0e2503ab..e3a6bb4bc 100644 --- a/bootstrap/README.md +++ b/bootstrap/README.md @@ -119,4 +119,4 @@ Setting `SMQ_AUTH_GRPC_CLIENT_CERT` and `SMQ_AUTH_GRPC_CLIENT_KEY` will enable T ## Usage -For more information about service capabilities and its usage, please check out the [API documentation](https://docs.api.magistrala.abstractmachines.fr/?urls.primaryName=bootstrap.yaml). +For more information about service capabilities and its usage, please check out the [API documentation](https://docs.api.magistrala.absmach.eu/?urls.primaryName=bootstrap.yaml). diff --git a/bootstrap/tracing/doc.go b/bootstrap/tracing/doc.go index aa43565e4..8a7079a42 100644 --- a/bootstrap/tracing/doc.go +++ b/bootstrap/tracing/doc.go @@ -8,5 +8,5 @@ // SuperMQ Users service. // // For more details about tracing instrumentation for SuperMQ messaging refer -// to the documentation at https://docs.supermq.abstractmachines.fr/tracing/. +// to the documentation at https://docs.supermq.absmach.eu/tracing/. package tracing diff --git a/consumers/README.md b/consumers/README.md index 28ea8b5de..9d9353855 100644 --- a/consumers/README.md +++ b/consumers/README.md @@ -13,6 +13,6 @@ For an in-depth explanation of the usage of `consumers`, as well as thorough understanding of SuperMQ, please check out the [official documentation][doc]. For more information about service capabilities and its usage, please check out -the [API documentation](https://docs.api.supermq.abstractmachines.fr/?urls.primaryName=consumers-notifiers-openapi.yaml). +the [API documentation](https://docs.api.supermq.absmach.eu/?urls.primaryName=consumers-notifiers-openapi.yaml). -[doc]: https://docs.supermq.abstractmachines.fr +[doc]: https://docs.supermq.absmach.eu diff --git a/consumers/notifiers/README.md b/consumers/notifiers/README.md index f64a2213e..ae5219af9 100644 --- a/consumers/notifiers/README.md +++ b/consumers/notifiers/README.md @@ -1,23 +1,182 @@ -# Notifiers service +# Notifiers -Notifiers service provides a service for sending notifications using Notifiers. -Notifiers service can be configured to use different types of Notifiers to send -different types of notifications such as SMS messages, emails, or push notifications. -Service is extensible so that new implementations of Notifiers can be easily added. -Notifiers **are not standalone services** but rather dependencies used by Notifiers service -for sending notifications over specific protocols. +The Notifiers service manages notification subscriptions and dispatches alerts for incoming messages. It stores subscription records (topic + contact), exposes an HTTP API for CRUD operations, and consumes SuperMQ messages to fan out notifications via notifier implementations (SMTP for email, SMPP for SMS). Notifiers are dependencies used by the service, not standalone services. ## Configuration -The service is configured using the environment variables. -The environment variables needed for service configuration depend on the underlying Notifier. -An example of the service configuration for SMTP Notifier can be found [in SMTP Notifier documentation](smtp/README.md). -Note that any unset variables will be replaced with their -default values. +The service is configured using environment variables. Values shown are from [docker/.env](https://github.com/absmach/magistrala/blob/main/docker/.env) when available; otherwise defaults come from code or notifier-specific docs. +### SMTP notifier (email) + +Used by `consumers/notifiers/smtp` via `internal/email`. + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_EMAIL_HOST` | SMTP host | `smtp.mailtrap.io` | +| `MG_EMAIL_PORT` | SMTP port | `2525` | +| `MG_EMAIL_USERNAME` | SMTP username | `18bf7f70705139` | +| `MG_EMAIL_PASSWORD` | SMTP password | `2b0d302e775b1e` | +| `MG_EMAIL_FROM_ADDRESS` | Default from address (used if `from` is empty) | `from@example.com` | +| `MG_EMAIL_FROM_NAME` | Default from name | `Example` | +| `MG_EMAIL_TEMPLATE` | Email template path | `email.tmpl` | + +### SMPP notifier (SMS) + +#### SMPP transport settings + +Defined in `consumers/notifiers/smpp/config.go`. + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_SMPP_ADDRESS` | SMPP address in `host:port` format | "" | +| `MG_SMPP_USERNAME` | SMPP username | "" | +| `MG_SMPP_PASSWORD` | SMPP password | "" | +| `MG_SMPP_SYSTEM_TYPE` | SMPP system type | "" | +| `MG_SMPP_SRC_ADDR_TON` | SMPP source address TON | `0` | +| `MG_SMPP_DST_ADDR_TON` | SMPP source address NPI | `0` | +| `MG_SMPP_SRC_ADDR_NPI` | SMPP destination address TON | `0` | +| `MG_SMPP_DST_ADDR_NPI` | SMPP destination address NPI | `0` | + +Note: The SMPP env tags are mapped exactly as defined in `consumers/notifiers/smpp/config.go`. + +#### SMPP notifier service settings + +Defined in `consumers/notifiers/smpp/README.md`. + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_SMPP_NOTIFIER_LOG_LEVEL` | Log level for SMPP notifier | `info` | +| `MG_SMPP_NOTIFIER_FROM_ADDRESS` | From address for SMS notifications | "" | +| `MG_SMPP_NOTIFIER_CONFIG_PATH` | Config file path for message broker subjects and payload type | `/config.toml` | +| `MG_SMPP_NOTIFIER_HTTP_HOST` | Service HTTP host | `localhost` | +| `MG_SMPP_NOTIFIER_HTTP_PORT` | Service HTTP port | `9014` | +| `MG_SMPP_NOTIFIER_HTTP_SERVER_CERT` | Service HTTP server certificate path | "" | +| `MG_SMPP_NOTIFIER_HTTP_SERVER_KEY` | Service HTTP server key path | "" | +| `MG_SMPP_NOTIFIER_DB_HOST` | Database host address | `localhost` | +| `MG_SMPP_NOTIFIER_DB_PORT` | Database host port | `5432` | +| `MG_SMPP_NOTIFIER_DB_USER` | Database user | `magistrala` | +| `MG_SMPP_NOTIFIER_DB_PASS` | Database password | `magistrala` | +| `MG_SMPP_NOTIFIER_DB_NAME` | Database name | `subscriptions` | +| `MG_SMPP_NOTIFIER_DB_SSL_MODE` | DB SSL mode (disable, require, verify-ca, verify-full) | `disable` | +| `MG_SMPP_NOTIFIER_DB_SSL_CERT` | DB SSL client cert path | "" | +| `MG_SMPP_NOTIFIER_DB_SSL_KEY` | DB SSL client key path | "" | +| `MG_SMPP_NOTIFIER_DB_SSL_ROOT_CERT` | DB SSL root cert path | "" | +| `SMQ_AUTH_GRPC_URL` | Auth gRPC URL | `localhost:7001` | +| `SMQ_AUTH_GRPC_TIMEOUT` | Auth gRPC timeout | `1s` | +| `MG_AUTH_GRPC_CLIENT_TLS` | Auth client TLS flag | `false` | +| `MG_AUTH_GRPC_CA_CERT` | Auth client CA certs path | "" | +| `SMQ_MESSAGE_BROKER_URL` | Message broker URL | `nats://127.0.0.1:4222` | +| `SMQ_JAEGER_URL` | Jaeger tracing URL | `http://jaeger:14268/api/traces` | +| `SMQ_SEND_TELEMETRY` | Send telemetry to Magistrala call-home server | `true` | +| `MG_SMPP_NOTIFIER_INSTANCE_ID` | SMPP notifier instance ID | "" | + +## Features + +- **Subscription management**: Create, view, list, and remove notification subscriptions. +- **Topic-based dispatch**: Matches subscriptions by topic and fan-outs to contacts. +- **Multiple notifier backends**: SMTP (email) and SMPP (SMS) implementations are available. +- **Observability**: Exposes `/metrics` and `/health` endpoints. +- **Uniqueness guardrails**: Prevents duplicate subscriptions for the same topic/contact pair. + +## Architecture + +### Runtime flow + +1. Clients register subscriptions through the HTTP API (`topic` + `contact`). +2. The service authenticates the token, assigns an owner ID, and persists the subscription. +3. When a message arrives, the service builds the topic as `channel` or `channel.subtopic`, retrieves matching subscriptions, and gathers contacts. +4. The notifier implementation sends notifications using the configured backend. + +### Components + +- **HTTP API**: `consumers/notifiers/api` exposes `/subscriptions`, `/health`, and `/metrics`. +- **Service layer**: `consumers/notifiers/service.go` handles authn, ID creation, and notification dispatch. +- **Repository**: `consumers/notifiers/postgres` persists subscriptions and supports filtering. +- **Notifier implementations**: `consumers/notifiers/smtp` (email) and `consumers/notifiers/smpp` (SMS). +- **Email agent**: `internal/email` manages SMTP connectivity and template rendering. + +### Subscriptions table + +Defined in `consumers/notifiers/postgres/init.go`: + +| Column | Type | Description | +| --- | --- | --- | +| `id` | `VARCHAR(254)` | Subscription identifier (primary key) | +| `owner_id` | `VARCHAR(254)` | Owner ID derived from the auth token | +| `contact` | `VARCHAR(254)` | Notification contact (email or phone) | +| `topic` | `TEXT` | Topic to match (`channel` or `channel.subtopic`) | + +Constraint: `UNIQUE(topic, contact)` + +## Deployment + +The Notifiers service is provided as a consumer package. It is typically wired into a notifier-specific binary that provides the HTTP server and message broker subscription. For the SMPP notifier runtime configuration, see `consumers/notifiers/smpp/README.md`. + +### Health check + +```bash +curl -X GET http://localhost:9014/health \ + -H "accept: application/health+json" +``` + +## Testing + +```bash +go test ./consumers/notifiers/... +``` ## Usage -Subscriptions service will start consuming messages and sending notifications when a message is received. +The Notifiers service supports the following operations (see `apidocs/openapi/notifiers.yaml`): -[doc]: https://docs.supermq.abstractmachines.fr +| Operation | Method & Path | Description | +| --- | --- | --- | +| `createSubscription` | `POST /subscriptions` | Create a new subscription | +| `listSubscriptions` | `GET /subscriptions` | List subscriptions with filters | +| `viewSubscription` | `GET /subscriptions/{id}` | Retrieve a subscription | +| `removeSubscription` | `DELETE /subscriptions/{id}` | Delete a subscription | +| `health` | `GET /health` | Service health check | + +### Example: Create a subscription + +```bash +curl -X POST http://localhost:9014/subscriptions \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "topic": "channel.subtopic", + "contact": "user@example.com" + }' +``` + +### Example: List subscriptions + +```bash +curl -X GET "http://localhost:9014/subscriptions?topic=channel.subtopic&contact=user@example.com&limit=20&offset=0" \ + -H "Authorization: Bearer " +``` + +### Example: View a subscription + +```bash +curl -X GET http://localhost:9014/subscriptions/ \ + -H "Authorization: Bearer " +``` + +### Example: Remove a subscription + +```bash +curl -X DELETE http://localhost:9014/subscriptions/ \ + -H "Authorization: Bearer " +``` + +### Example: Health check + +```bash +curl -X GET http://localhost:9014/health \ + -H "accept: application/health+json" +``` + +For an in-depth explanation of the Notifiers, see the [official documentation][doc]. + +[doc]: https://docs.magistrala.absmach.eu/dev-guide/consumers/#notifiers diff --git a/consumers/notifiers/tracing/doc.go b/consumers/notifiers/tracing/doc.go index aadb62fe7..da7ad43bb 100644 --- a/consumers/notifiers/tracing/doc.go +++ b/consumers/notifiers/tracing/doc.go @@ -8,5 +8,5 @@ // SuperMQ WebSocket adapter service. // // For more details about tracing instrumentation for SuperMQ messaging refer -// to the documentation at https://docs.supermq.abstractmachines.fr/tracing/. +// to the documentation at https://docs.supermq.absmach.eu/tracing/. package tracing diff --git a/consumers/writers/README.md b/consumers/writers/README.md index 82225c3cc..04beebaa3 100644 --- a/consumers/writers/README.md +++ b/consumers/writers/README.md @@ -1,16 +1,268 @@ # Writers -Writers provide an implementation of various `message writers`. -Message writers are services that normalize (in `SenML` format) -SuperMQ messages and store them in specific data store. +Writers consume messages from the message broker, normalize them (SenML or JSON), and persist them to a storage backend. Magistrala provides two writer services: -Writers are optional services and are treated as plugins. In order to -run writer services, core services must be up and running. For more info -on the platform core services with its dependencies, please check out -the [Docker Compose][compose] file. +- **Postgres writer**: Stores data in PostgreSQL. +- **Timescale writer**: Stores data in TimescaleDB and uses hypertables for time-series workloads. -For an in-depth explanation of the usage of `writers`, as well as thorough -understanding of SuperMQ, please check out the [official documentation][doc]. +Writers are optional services and are treated as plugins. Core services and the message broker must be running first. For platform dependencies, see [Docker Compose](https://github.com/absmach/magistrala/blob/main/docker/docker-compose.yaml). -[doc]: https://docs.supermq.abstractmachines.fr -[compose]: ../docker/docker-compose.yaml +## Configuration + +Values shown are from [docker/.env](https://github.com/absmach/magistrala/blob/main/docker/.env) and the add-on compose files in `docker/addons/*-writer/docker-compose.yaml`. + +### Postgres writer + +#### Postgres Service endpoints + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_POSTGRES_WRITER_LOG_LEVEL` | Service log level | `debug` | +| `MG_POSTGRES_WRITER_CONFIG_PATH` | Config file path (subjects/transformer) | `/config.toml` | +| `MG_POSTGRES_WRITER_HTTP_HOST` | HTTP host | `postgres-writer` | +| `MG_POSTGRES_WRITER_HTTP_PORT` | HTTP port | `9007` | +| `MG_POSTGRES_WRITER_HTTP_SERVER_CERT` | HTTPS server certificate path | "" | +| `MG_POSTGRES_WRITER_HTTP_SERVER_KEY` | HTTPS server key path | "" | +| `MG_POSTGRES_WRITER_INSTANCE_ID` | Instance ID | "" | + +#### Postgres Database + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_POSTGRES_HOST` | PostgreSQL host | `postgres` | +| `MG_POSTGRES_PORT` | PostgreSQL port | `5432` | +| `MG_POSTGRES_USER` | PostgreSQL user | `supermq` | +| `MG_POSTGRES_PASS` | PostgreSQL password | `supermq` | +| `MG_POSTGRES_NAME` | PostgreSQL database name | `messages` | +| `MG_POSTGRES_SSL_MODE` | PostgreSQL SSL mode | `disable` | +| `MG_POSTGRES_SSL_CERT` | PostgreSQL SSL client cert | "" | +| `MG_POSTGRES_SSL_KEY` | PostgreSQL SSL client key | "" | +| `MG_POSTGRES_SSL_ROOT_CERT` | PostgreSQL SSL root cert | "" | + +#### Postgres Message broker and observability + +| Variable | Description | Default | +| --- | --- | --- | +| `SMQ_MESSAGE_BROKER_URL` | Message broker URL | `nats://nats:4222` | +| `SMQ_JAEGER_URL` | Jaeger collector endpoint | `http://jaeger:4318/v1/traces` | +| `SMQ_JAEGER_TRACE_RATIO` | Trace sampling ratio | `1.0` | +| `SMQ_SEND_TELEMETRY` | Send telemetry to Magistrala call-home server | `true` | + +### Timescale writer + +#### Timescale Service endpoints + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_TIMESCALE_WRITER_LOG_LEVEL` | Service log level | `debug` | +| `MG_TIMESCALE_WRITER_CONFIG_PATH` | Config file path (subjects/transformer) | `/config.toml` | +| `MG_TIMESCALE_WRITER_HTTP_HOST` | HTTP host | `timescale-writer` | +| `MG_TIMESCALE_WRITER_HTTP_PORT` | HTTP port | `9012` | +| `MG_TIMESCALE_WRITER_HTTP_SERVER_CERT` | HTTPS server certificate path | "" | +| `MG_TIMESCALE_WRITER_HTTP_SERVER_KEY` | HTTPS server key path | "" | +| `MG_TIMESCALE_WRITER_INSTANCE_ID` | Instance ID | "" | + +#### Timescale Database + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_TIMESCALE_HOST` | TimescaleDB host | `timescale` | +| `MG_TIMESCALE_PORT` | TimescaleDB port | `5432` | +| `MG_TIMESCALE_USER` | TimescaleDB user | `supermq` | +| `MG_TIMESCALE_PASS` | TimescaleDB password | `supermq` | +| `MG_TIMESCALE_NAME` | TimescaleDB database name | `supermq` | +| `MG_TIMESCALE_SSL_MODE` | TimescaleDB SSL mode | `disable` | +| `MG_TIMESCALE_SSL_CERT` | TimescaleDB SSL client cert | "" | +| `MG_TIMESCALE_SSL_KEY` | TimescaleDB SSL client key | "" | +| `MG_TIMESCALE_SSL_ROOT_CERT` | TimescaleDB SSL root cert | "" | + +#### Timescale Message broker and observability + +Timescale writer uses the same broker and telemetry variables listed for Postgres writer. + +### Writer config file + +Both writers read a config file defined by `*_WRITER_CONFIG_PATH`. The default add-on config files are: + +- `docker/addons/postgres-writer/config.toml` +- `docker/addons/timescale-writer/config.toml` + +The config file controls subscription subjects and, for Postgres, optional transformer settings: + +```toml +["subscriber"] +subjects = ["writers.>"] + +[transformer] +format = "senml" +content_type = "application/senml+json" +time_fields = [ + { field_name = "seconds_key", field_format = "unix", location = "UTC" }, + { field_name = "millis_key", field_format = "unix_ms", location = "UTC" }, + { field_name = "micros_key", field_format = "unix_us", location = "UTC" }, + { field_name = "nanos_key", field_format = "unix_ns", location = "UTC" } +] +``` + +NATS uses subject `writers.>` and RabbitMQ uses routing key `writers.#` (both are handled by `consumers/writers/brokers`). + +## Features + +- **Message persistence**: Stores incoming SenML messages into PostgreSQL or TimescaleDB. +- **JSON payload support**: Saves JSON payloads into dynamically created tables. +- **Broker-backed ingestion**: Consumes from NATS JetStream or RabbitMQ topics. +- **Configurable subscription**: Limits ingestion to specific `writers.*` subjects. +- **Observability**: Exposes `/health` and `/metrics` endpoints, with Jaeger tracing. + +## Architecture + +### Runtime flow + +1. The message broker publishes messages under `writers.*`. +2. The writer loads `config.toml` to select subjects and transformer settings. +3. The consumer converts messages to SenML or JSON payloads. +4. The repository writes records to the target database. + +### Components + +- **Message broker adapter**: `consumers/writers/brokers` (NATS JetStream or RabbitMQ). +- **Writer services**: `consumers/writers/postgres` and `consumers/writers/timescale`. +- **HTTP API**: `consumers/writers/api` exposes `/health` and `/metrics`. +- **Migrations**: `consumers/writers/*/init.go` defines the schema and indexes. + +### PostgreSQL schema (SenML messages) + +Defined in `consumers/writers/postgres/init.go`: + +| Column | Type | Description | +| --- | --- | --- | +| `id` | `UUID` | Message ID | +| `channel` | `UUID` | Channel ID | +| `subtopic` | `VARCHAR(254)` | Subtopic | +| `publisher` | `UUID` | Publisher (client) ID | +| `protocol` | `TEXT` | Protocol name | +| `name` | `TEXT` | SenML name | +| `unit` | `TEXT` | SenML unit | +| `value` | `FLOAT` | Numeric value | +| `string_value` | `TEXT` | String value | +| `bool_value` | `BOOL` | Boolean value | +| `data_value` | `BYTEA` | Data value | +| `sum` | `FLOAT` | Sum value | +| `time` | `FLOAT` | Measurement time | +| `update_time` | `FLOAT` | Update time | + +Primary key: `(time, publisher, subtopic, name)` + +### TimescaleDB schema (SenML messages) + +Defined in `consumers/writers/timescale/init.go`: + +| Column | Type | Description | +| --- | --- | --- | +| `time` | `BIGINT` | Measurement time | +| `channel` | `UUID` | Channel ID | +| `subtopic` | `VARCHAR(254)` | Subtopic | +| `publisher` | `VARCHAR(254)` | Publisher (client) ID | +| `protocol` | `TEXT` | Protocol name | +| `name` | `VARCHAR(254)` | SenML name | +| `unit` | `TEXT` | SenML unit | +| `value` | `FLOAT` | Numeric value | +| `string_value` | `TEXT` | String value | +| `bool_value` | `BOOL` | Boolean value | +| `data_value` | `BYTEA` | Data value | +| `sum` | `FLOAT` | Sum value | +| `update_time` | `FLOAT` | Update time | + +Primary key: `(time, channel, subtopic, protocol, publisher, name)` + +Timescale writer creates a hypertable on `messages` and adds time-series indexes for common query paths. + +### JSON payload tables (dynamic) + +If the transformer emits JSON payloads, the writers create a table named after the payload format: + +Postgres JSON table: +`id UUID`, `created BIGINT`, `channel VARCHAR(254)`, `subtopic VARCHAR(254)`, `publisher VARCHAR(254)`, `protocol TEXT`, `payload JSONB` (PK: `id`) + +Timescale JSON table: +`created BIGINT`, `channel VARCHAR(254)`, `subtopic VARCHAR(254)`, `publisher VARCHAR(254)`, `protocol TEXT`, `payload JSONB` (PK: `created`, `publisher`, `subtopic`) + +## Deployment + +### Build and run locally + +Postgres writer: + +```bash +make postgres-writer + +MG_POSTGRES_WRITER_LOG_LEVEL=debug \ +MG_POSTGRES_WRITER_CONFIG_PATH=./docker/addons/postgres-writer/config.toml \ +MG_POSTGRES_WRITER_HTTP_PORT=9007 \ +MG_POSTGRES_HOST=localhost \ +MG_POSTGRES_PORT=5432 \ +MG_POSTGRES_USER=supermq \ +MG_POSTGRES_PASS=supermq \ +MG_POSTGRES_NAME=messages \ +SMQ_MESSAGE_BROKER_URL=nats://localhost:4222 \ +SMQ_JAEGER_URL=http://localhost:4318/v1/traces \ +./build/postgres-writer +``` + +Timescale writer: + +```bash +make timescale-writer + +MG_TIMESCALE_WRITER_LOG_LEVEL=debug \ +MG_TIMESCALE_WRITER_CONFIG_PATH=./docker/addons/timescale-writer/config.toml \ +MG_TIMESCALE_WRITER_HTTP_PORT=9012 \ +MG_TIMESCALE_HOST=localhost \ +MG_TIMESCALE_PORT=5432 \ +MG_TIMESCALE_USER=supermq \ +MG_TIMESCALE_PASS=supermq \ +MG_TIMESCALE_NAME=supermq \ +SMQ_MESSAGE_BROKER_URL=nats://localhost:4222 \ +SMQ_JAEGER_URL=http://localhost:4318/v1/traces \ +./build/timescale-writer +``` + +### Docker Compose + +Postgres writer add-on: + +```bash +docker compose -f docker/docker-compose.yaml -f docker/addons/postgres-writer/docker-compose.yaml up +``` + +Timescale writer add-on: + +```bash +docker compose -f docker/docker-compose.yaml -f docker/addons/timescale-writer/docker-compose.yaml up +``` + +### Health check + +```bash +curl -X GET http://localhost:9007/health \ + -H "accept: application/health+json" +``` + +## Testing + +```bash +go test ./consumers/writers/... +``` + +## Usage + +Writers do not expose a message ingestion API. Messages are written via the message broker. The HTTP API provides only health and metrics endpoints. + +| Endpoint | Description | +| --- | --- | +| `GET /health` | Service health check | +| `GET /metrics` | Prometheus metrics | + +For an in-depth explanation of Writers, see the [official documentation][doc]. + +[doc]: https://docs.magistrala.absmach.eu/dev-guide/consumers/ diff --git a/docker/README.md b/docker/README.md index 8e3f55882..a83357432 100644 --- a/docker/README.md +++ b/docker/README.md @@ -33,7 +33,7 @@ Events store: This is used by Magistrala services to store events for distribute This is the same as MESSAGE_BROKER. This can either be 'NATS' or 'RabbitMQ' or 'Redis'. If Redis is used as an events store, then RabbitMQ or NATS is used as a message broker. -The current deployment strategy for Magistrala in `docker/docker-compose.yaml` is to use VerneMQ as a MQTT_BROKER and NATS as a MESSAGE_BROKER and EVENTS_STORE. +The current deployment strategy for Magistrala in [docker/docker-compose.yaml](https://github.com/absmach/magistrala/blob/main/docker/docker-compose.yaml) is to use VerneMQ as a MQTT_BROKER and NATS as a MESSAGE_BROKER and EVENTS_STORE. Therefore, the following combinations are possible: @@ -46,7 +46,7 @@ Therefore, the following combinations are possible: - MQTT_BROKER: NATS, MESSAGE_BROKER: NATS, EVENTS_STORE: NATS - MQTT_BROKER: NATS, MESSAGE_BROKER: NATS, EVENTS_STORE: Redis -For Message brokers other than NATS, you would need to build the docker images with RabbitMQ as the build tag and change the `docker/.env`. For example, to use RabbitMQ as a message broker: +For Message brokers other than NATS, you would need to build the docker images with RabbitMQ as the build tag and change the [docker/.env](https://github.com/absmach/magistrala/blob/main/docker/.env). For example, to use RabbitMQ as a message broker: ```bash MG_MESSAGE_BROKER_TYPE=rabbitmq make dockers @@ -70,7 +70,7 @@ MG_ES_TYPE=redis MG_ES_URL=${MG_REDIS_URL} ``` -For MQTT broker other than VerneMQ, you would need to change the `docker/.env`. For example, to use NATS as a MQTT broker: +For MQTT broker other than VerneMQ, you would need to change the [docker/.env](https://github.com/absmach/magistrala/blob/main/docker/.env). For example, to use NATS as a MQTT broker: ```env MG_MQTT_BROKER_TYPE=nats @@ -121,7 +121,7 @@ services: ## Nginx Configuration Nginx is the entry point for all traffic to Magistrala. -By using environment variables file at `docker/.env` you can modify the below given Nginx directive. +By using environment variables file at [docker/.env](https://github.com/absmach/magistrala/blob/main/docker/.env) you can modify the below given Nginx directive. `SMQ_NGINX_SERVER_NAME` environmental variable is used to configure nginx directive `server_name`. If environmental variable `SMQ_NGINX_SERVER_NAME` is empty then default value `localhost` will set to `server_name`. diff --git a/provision/README.md b/provision/README.md index 805803c62..36878fa5f 100644 --- a/provision/README.md +++ b/provision/README.md @@ -1,58 +1,74 @@ # Provision service -Provision service provides an HTTP API to interact with [SuperMQ][supermq]. -Provision service is used to setup initial applications configuration i.e. clients, channels, connections and certificates that will be required for the specific use case especially useful for gateway provision. +Provision service provides an HTTP API to create initial SuperMQ resources for gateways or edge deployments. It can create clients and channels based on a configurable layout, optionally create bootstrap configurations, whitelist clients, and issue X.509 certificates for mTLS. -For gateways to communicate with [SuperMQ][supermq] configuration is required (mqtt host, client, channels, certificates...). To get the configuration gateway will send a request to [Bootstrap][bootstrap] service providing `` and `` in request. To make a request to [Bootstrap][bootstrap] service you can use [Agent][agent] service on a gateway. +For gateways to communicate with [SuperMQ][supermq], configuration is required (MQTT host, client, channels, certificates). A gateway can fetch bootstrap configuration from the [Bootstrap][bootstrap] service using its `` and ``. The [Agent][agent] service is typically used on gateways to retrieve that configuration. -To create bootstrap configuration you can use [Bootstrap][bootstrap] or `Provision` service. [SuperMQ UI][mgxui] uses [Bootstrap][bootstrap] service for creating gateway configurations. `Provision` service should provide an easy way of provisioning your gateways i.e creating bootstrap configuration and as many clients and channels that your setup requires. - -Also you may use provision service to create certificates for each client. Each service running on gateway may require more than one client and channel for communication. Let's say that you are using services [Agent][agent] and [Export][export] on a gateway you will need two channels for `Agent` (`data` and `control`) and one for `Export` and one client. Additionally if you enabled mtls each service will need its own client and certificate for access to [SuperMQ][supermq]. Your setup could require any number of clients and channels this kind of setup we can call `provision layout`. - -Provision service provides a way of specifying this `provision layout` and creating a setup according to that layout by serving requests on `/mapping` endpoint. Provision layout is configured in [config.toml](configs/config.toml). +You can create bootstrap configuration directly via [Bootstrap][bootstrap] or through Provision. [SuperMQ UI][mgxui] uses the Bootstrap service; Provision is intended to automate gateway setups where one physical gateway may require multiple clients and channels (for example, [Agent][agent] and [Export][export]). This setup is defined as a **provision layout**. ## Configuration -The service is configured using the environment variables presented in the -following table. Note that any unset variables will be replaced with their -default values. +The service is configured using environment variables and/or a TOML config file. Defaults below are from `provision/config.go`. Docker add-on examples are in `docker/addons/provision/docker-compose.yaml` and [docker/.env](https://github.com/absmach/magistrala/blob/main/docker/.env). The binary reads `SMQ_PROVISION_*` variables; the add-on compose file uses `MG_PROVISION_*`, so ensure the container receives the expected names. -| Variable | Description | Default | -| ------------------------------------ | -------------------------------------------------- | ----------------------- | -| SMQ_PROVISION_LOG_LEVEL | Service log level | debug | -| SMQ_PROVISION_USER | User (email) for accessing SuperMQ | | -| SMQ_PROVISION_PASS | SuperMQ password | user123 | -| SMQ_PROVISION_API_KEY | SuperMQ authentication token | | -| SMQ_PROVISION_CONFIG_FILE | Provision config file | config.toml | -| SMQ_PROVISION_HTTP_PORT | Provision service listening port | 9016 | -| SMQ_PROVISION_ENV_CLIENTS_TLS | SuperMQ SDK TLS verification | false | -| SMQ_PROVISION_SERVER_CERT | SuperMQ gRPC secure server cert | | -| SMQ_PROVISION_SERVER_KEY | SuperMQ gRPC secure server key | | -| SMQ_PROVISION_USERS_LOCATION | Users service URL | | -| SMQ_PROVISION_CLIENTS_LOCATION | Clients service URL | | -| SMQ_PROVISION_BS_SVC_URL | SuperMQ Bootstrap service URL | | -| SMQ_PROVISION_CERTS_SVC_URL | Certificates service URL | | -| SMQ_PROVISION_X509_PROVISIONING | Should X509 client cert be provisioned | false | -| SMQ_PROVISION_BS_CONFIG_PROVISIONING | Should client config be saved in Bootstrap service | true | -| SMQ_PROVISION_BS_AUTO_WHITELIST | Should client be auto whitelisted | true | -| SMQ_PROVISION_BS_CONTENT | Bootstrap service configs content, JSON format | {} | -| SMQ_PROVISION_CERTS_RSA_BITS | Certificate RSA bits parameter | 4096 | -| SMQ_PROVISION_CERTS_HOURS_VALID | Number of hours that certificate is valid | "2400h" | -| SMQ_SEND_TELEMETRY | Send telemetry to supermq call home server | true | +### Core service -By default, call to `/mapping` endpoint will create one client and two channels (`control` and `data`) and connect it. If there is a requirement for different provision layout we can use [config](docker/configs/config.toml) file in addition to environment variables. +| Variable | Description | Default | +| --- | --- | --- | +| `SMQ_PROVISION_HTTP_PORT` | Provision service listening port | `9016` | +| `SMQ_PROVISION_LOG_LEVEL` | Service log level | `info` | +| `SMQ_PROVISION_ENV_CLIENTS_TLS` | SDK TLS verification | `false` | +| `SMQ_PROVISION_SERVER_CERT` | HTTPS server certificate | "" | +| `SMQ_PROVISION_SERVER_KEY` | HTTPS server key | "" | +| `SMQ_SEND_TELEMETRY` | Send telemetry to SuperMQ call-home server | `true` | +| `SMQ_MQTT_ADAPTER_INSTANCE_ID` | Instance ID used in health output | "" | -For the purposes of running provision as an add-on in docker composition environment variables seems more suitable. Environment variables are set in [.env](.env). +### SuperMQ endpoints and credentials -Configuration can be specified in [config.toml](configs/config.toml). Config file can specify all the settings that environment variables can configure and in addition -`/mapping` endpoint provision layout can be configured. +| Variable | Description | Default | +| --- | --- | --- | +| `SMQ_PROVISION_USERS_LOCATION` | Users service URL | `http://localhost` | +| `SMQ_PROVISION_CLIENTS_LOCATION` | Clients service URL | `http://localhost` | +| `SMQ_PROVISION_CERTS_LOCATION` | Certs service URL (certs SDK) | `http://localhost` | +| `SMQ_PROVISION_BS_SVC_URL` | Bootstrap service URL | `http://localhost:9000` | +| `SMQ_PROVISION_CERTS_SVC_URL` | Certs service URL (Magistrala SDK) | `http://localhost:9019` | +| `SMQ_PROVISION_USERNAME` | SuperMQ username | `user` | +| `SMQ_PROVISION_PASS` | SuperMQ password | `test` | +| `SMQ_PROVISION_API_KEY` | SuperMQ authentication token | "" | +| `SMQ_PROVISION_EMAIL` | SuperMQ user email | `test@example.com` | +| `SMQ_PROVISION_DOMAIN_ID` | Default domain ID (unused by HTTP API) | "" | -In `config.toml` we can enlist array of clients and channels that we want to create and make connections between them which we call provision layout. +### Provisioning behavior -Metadata can be whatever suits your needs except that at least one client needs to have `external_id` (which is populated with value from [request](#example)). Client that has `external_id` will be used for creating bootstrap configuration which can be fetched with [Agent][agent]. -For channels metadata `type` is reserved for `control` and `data` which we use with [Agent][agent]. +| Variable | Description | Default | +| --- | --- | --- | +| `SMQ_PROVISION_CONFIG_FILE` | Provision config file | `config.toml` | +| `SMQ_PROVISION_X509_PROVISIONING` | Issue client certificates during provisioning | `false` | +| `SMQ_PROVISION_BS_CONFIG_PROVISIONING` | Save client config in Bootstrap | `true` | +| `SMQ_PROVISION_BS_AUTO_WHITELIST` | Auto-whitelist client | `true` | +| `SMQ_PROVISION_BS_CONTENT` | Bootstrap config content (JSON string) | "" | +| `SMQ_PROVISION_CERTS_HOURS_VALID` | Client cert validity period | `2400h` | -Example of provision layout below +## Features + +- **Layout-driven provisioning**: Create clients and channels from a predefined layout. +- **Bootstrap integration**: Create bootstrap configs and optionally whitelist clients. +- **X.509 certificates**: Issue client certificates during provisioning when enabled. +- **Gateway metadata**: Enrich gateway clients with control/data/export channel IDs. +- **Observability**: `/metrics` and `/health` endpoints. + +## Provision layout + +Provision layout is configured in a TOML file (see `provision/configs/config.toml` or `docker/addons/provision/configs/config.toml`). If the file exists, it is loaded and any missing fields are filled with env values. The layout defines which clients and channels will be created when calling `/mapping`. + +Default behavior (when no config file is loaded) creates one client and two channels: `control` and `data`. + +Notes: + +- At least one client must include `external_id` in metadata. This value is replaced with the `external_id` from the provisioning request and is used for bootstrap creation. +- Channel metadata `type` is reserved for `control`, `data`, and `export` and is used to enrich gateway metadata. +- Bootstrap content can be provided via `bootstrap.content` in the TOML file or as JSON through `SMQ_PROVISION_BS_CONTENT`. + +Example layout: ```toml [[clients]] @@ -61,7 +77,6 @@ Example of provision layout below [clients.metadata] external_id = "xxxxxx" - [[channels]] name = "control-channel" @@ -83,57 +98,77 @@ Example of provision layout below ## Authentication -In order to create necessary entities provision service needs to authenticate against SuperMQ. To provide authentication credentials to the provision service you can pass it in an environment variable or in a config file as SuperMQ user and password or as API token that can be issued on `/users/tokens/issue`. +Provision uses SuperMQ APIs and requires a valid token. There are three ways to provide it: -Additionally users or API token can be passed in Authorization header, this authentication takes precedence over others. +- `Authorization: Bearer ` on each request. +- `SMQ_PROVISION_API_KEY` in env or TOML (used when no header token is provided). +- `SMQ_PROVISION_USERNAME` and `SMQ_PROVISION_PASS` in env or TOML (used to create an access token when no header token is provided). -- `username`, `password` - (`SMQ_PROVISION_USER`, `SMQ_PROVISION_PASSWORD` in [.env](../.env), `mg_user`, `mg_pass` in [config.toml](../docker/addons/provision/configs/config.toml)) -- API Key - (`SMQ_PROVISION_API_KEY` in [.env](../.env) or [config.toml](../docker/addons/provision/configs/config.toml)) -- `Authorization: Bearer Token` - request authorization header containing either users token. +`POST /{domainID}/mapping` can create its own token using API key or username/password if no `Authorization` header is provided. The `Authorization` header takes precedence when present. `GET /{domainID}/mapping` always requires a bearer token. + +## Architecture + +### Runtime flow + +1. The service loads configuration from env and optionally merges a config file. +2. `POST /{domainID}/mapping` validates the request and ensures a token exists. +3. Clients are created from the configured layout (external ID is injected into metadata). +4. Channels are created with names prefixed by the request `name`. +5. If enabled, bootstrap configs are created and clients are whitelisted (connected to channels). +6. If X.509 provisioning is enabled, certificates are issued and returned in the response. ## Running -Provision service can be run as a standalone or in docker composition as addon to the core docker composition. +Provision service can be run standalone or via Docker Compose. Standalone: ```bash +make provision + SMQ_PROVISION_BS_SVC_URL=http://localhost:9013 \ -SMQ_PROVISION_CLIENTS_LOCATION=http://localhost:9000 \ +SMQ_PROVISION_CLIENTS_LOCATION=http://localhost:9006 \ SMQ_PROVISION_USERS_LOCATION=http://localhost:9002 \ -SMQ_PROVISION_CONFIG_FILE=docker/addons/provision/configs/config.toml \ -build/supermq-provision +SMQ_PROVISION_CONFIG_FILE=provision/configs/config.toml \ +./build/provision ``` -Docker composition: +Docker Compose (add-on): ```bash -docker compose -f docker/addons/provision/docker-compose.yaml up +docker compose -f docker/docker-compose.yaml -f docker/addons/provision/docker-compose.yaml up provision ``` -For the case that credentials or API token is passed in configuration file or environment variables, call to `/mapping` endpoint doesn't require `Authentication` header: +## Usage + +The Provision service exposes the following endpoints: + +| Operation | Method & Path | Description | +| --- | --- | --- | +| `provision` | `POST /{domainID}/mapping` | Create clients, channels, bootstrap config, and optional certs | +| `mapping` | `GET /{domainID}/mapping` | Return bootstrap content from config | +| `health` | `GET /health` | Service health check | + +### Example: Provision a gateway + +When credentials are available via env/config, you can omit the `Authorization` header. `Content-Type` must be exactly `application/json`. ```bash -curl -s -S -X POST http://localhost:/mapping -H 'Content-Type: application/json' -d '{"external_id": "33:52:77:99:43", "external_key": "223334fw2"}' +curl -s -S -X POST http://localhost://mapping \ + -H 'Content-Type: application/json' \ + -d '{"name": "gateway-a", "external_id": "33:52:77:99:43", "external_key": "223334fw2"}' ``` -In the case that provision service is not deployed with credentials or API key or you want to use user other than one being set in environment (or config file): +If you want to supply a token explicitly: ```bash -curl -s -S -X POST http://localhost:/mapping -H "Authorization: Bearer " -H 'Content-Type: application/json' -d '{"external_id": "", "external_key": ""}' +curl -s -S -X POST http://localhost://mapping \ + -H "Authorization: Bearer " \ + -H 'Content-Type: application/json' \ + -d '{"name": "gateway-a", "external_id": "", "external_key": ""}' ``` -Or if you want to specify a name for client different than in `config.toml` you can specify post data as: - -```json -{ - "name": "", - "external_id": "", - "external_key": "" -} -``` - -Response contains created clients, channels and certificates if any: +Response contains created clients, channels, and optional certificate data: ```json { @@ -169,24 +204,27 @@ Response contains created clients, channels and certificates if any: } ``` -## Certificates - -Provision service has `/certs` endpoint that can be used to generate certificates for clients when mTLS is required: - -- `users_token` - users authentication token or API token -- `client_id` - id of the client for which certificate is going to be generated +### Example: Read bootstrap mapping ```bash -curl -s -X POST http://localhost:8190/certs -H "Authorization: Bearer " -H 'Content-Type: application/json' -d '{"client_id": "", "ttl":"2400h" }' +curl -s -S -X GET http://localhost://mapping \ + -H "Authorization: Bearer " \ + -H 'Content-Type: application/json' ``` -```json -{ - "client_cert": "-----BEGIN CERTIFICATE-----\nMIIEmDCCA4CgAwIBAgIQCZ0NOq2oKLo+XftbAu0TfzANBgkqhkiG9w0BAQsFADBX\nMRIwEAYDVQQDDAlsb2NhbGhvc3QxETAPBgNVBAoMCE1haW5mbHV4MQwwCgYDVQQL\nDANJb1QxIDAeBgkqhkiG9w0BCQEWEWluZm9AbWFpbmZsdXguY29tMB4XDTIwMDYw\nNTEyMzc1M1oXDTIwMDkxMzEyMzc1M1owVTERMA8GA1UEChMITWFpbmZsdXgxETAP\nBgNVBAsTCG1haW5mbHV4MS0wKwYDVQQDEyQyYmZlYmZmMC05ODZhLTQ3ZTAtOGQ3\nYS00YTRiN2UyYjU3OGUwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCn\nWvTuOIdhqOLEREcEJqfQAtDoYu3rUDijOffXuWFZgNqfZTGmoD5ZqJXxwbZ4tCST\npdSteHtyr7JXnPJQN1dsslU+q3haKjFoZRc39/7u4/8XCTwlqbMl9YVcwqS+FLkM\niLSyyqzryP7Y8H8cidTKg56p5JALaEKfzZS6Km3G+CCinR6hNNW9ckWsy29a0/9E\nMAUtM+Lsk5OjsHzOnWruuqHsCx4ODI5aJQaMC1qntkbXkht0WDiwAt9SDQ3uLWru\nAoSJDK9a6EgR3a0Jf7ZiVPiwlZNjrB/I5OQyFDGqcmSAl2rdJqPkmaDXKKFyL1cG\nMIyHv62QzJoMdRoXu20lxyGxAvEjQNVHux4LA3dbf/85nEVTI2uP8crMf2Jnzbg5\n9zF+iTMJGpUlatCyK2RJS/mvHbbUIf5Ro3VbcPHbgFroJ7qMFz0Fc5kYY8IdwXjG\nlyG9MobKEO2CfBGRjPmCuTQq2HcuOy7F6KfQf3HToI8MmC5hBtCmTNbV8I3GIjWA\n/xJQLm2pVZ41QhrnNGtuqAYoe3Zt6OldxGRcoAj7KlIpYcPZ55PJ6mWcV6dB9Fnl\n5mYOwQL8jtfybbGWvqJldhTxUqm7/EbAaF0Qjmh4oOHMl2xADrmYzJHvf0llwr6g\noRQuzqxPi0aW3tkFNsm63NX1Ab5BXFQhMSj5+82blwIDAQABo2IwYDAOBgNVHQ8B\nAf8EBAMCB4AwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMA4GA1UdDgQH\nBAUBAgMEBjAfBgNVHSMEGDAWgBRs4xR91qEjNRGmw391xS7x6Tc+8jANBgkqhkiG\n9w0BAQsFAAOCAQEAphLT8PjawRRWswU1B5oWnnqeTllnvGB88sjDPLAG0UiBlDLX\nwoPiBVPWuYV+MMJuaREgheYF1Ahx4Jrfy9stFDU7B99ON1T58oM1aKEq4rKc+/Ke\nyxrAFTonclC0LNaaOvpZZjsPFWr2muTQO8XHiS8icw3BLxEzoF+5aJ8ihtxRtfKL\nUvtHDqC6IPAbSUcvqyjrFh3RrTUAyGOzW12IEWSXP9DLwoiLPwJ6kCVoXdG/asjz\nUpk/jj7AUn9oJNF8nUbyhdOnmeJ2z0x1ylgYrIAxvGzm8zs+NEVN67CrBYKwstlN\nvw7DRQsCvGJjZzWj28VV3FGLtXFgu52bFZNBww==\n-----END CERTIFICATE-----\n", - "client_cert_key": "-----BEGIN RSA PRIVATE KEY-----\nMIIJJwIBAAKCAgEAp1r07jiHYajixERHBCan0ALQ6GLt61A4ozn317lhWYDan2Ux\npqA+WaiV8cG2eLQkk6XUrXh7cq+yV5zyUDdXbLJVPqt4WioxaGUXN/f+7uP/Fwk8\nJamzJfWFXMKkvhS5DIi0ssqs68j+2PB/HInUyoOeqeSQC2hCn82Uuiptxvggop0e\noTTVvXJFrMtvWtP/RDAFLTPi7JOTo7B8zp1q7rqh7AseDgyOWiUGjAtap7ZG15Ib\ndFg4sALfUg0N7i1q7gKEiQyvWuhIEd2tCX+2YlT4sJWTY6wfyOTkMhQxqnJkgJdq\n3Saj5Jmg1yihci9XBjCMh7+tkMyaDHUaF7ttJcchsQLxI0DVR7seCwN3W3//OZxF\nUyNrj/HKzH9iZ824OfcxfokzCRqVJWrQsitkSUv5rx221CH+UaN1W3Dx24Ba6Ce6\njBc9BXOZGGPCHcF4xpchvTKGyhDtgnwRkYz5grk0Kth3Ljsuxein0H9x06CPDJgu\nYQbQpkzW1fCNxiI1gP8SUC5tqVWeNUIa5zRrbqgGKHt2bejpXcRkXKAI+ypSKWHD\n2eeTyeplnFenQfRZ5eZmDsEC/I7X8m2xlr6iZXYU8VKpu/xGwGhdEI5oeKDhzJds\nQA65mMyR739JZcK+oKEULs6sT4tGlt7ZBTbJutzV9QG+QVxUITEo+fvNm5cCAwEA\nAQKCAgAmCIfNc89gpG8Ux6eUC+zrWxh7F7CWX97fSZdH0XuMSbplqyvDgHtrCOM6\n1BlSCS6e13skCVOU1tUjECoJjOoza7vvyCxL4XblEMRcFeI8DFi2tYST0qNCJzAt\nypaCFFeRv6fBUkpGM6GnT9Czfad8drkiRy1tSj6J7sC0JlxYcZ+JFUgWvtksesHW\n6UzfSXqj1n32reoOdeOBueRDWIcqxgNyj3w/GR9o4S1BunrZzpT+/Nd8c2g+qAh0\nrz7ROEUq3iucseNQN6XZWZWvqPScGE+EYhni9wUqNMqfjvNSlzi7+K1yoQtyMm/Z\nNgSq3JNcdsAZQbiCRd1ko2BQsGm3ZBnbsAJ1Dxcn+i9nF5DT/ddWjUWin6LYWuUM\n/0Bqfv3etlrFuP6yxc8bPEMX0ucJg4yVxdkDrm1tYlJ+ANEQoOlZqhngvjz0f8uO\nOtEcDLmiG5VG6Yl72UtWIw+ALnKc5U7ib43Qve0bDAKR5zlHODcRetN9BCMvpekY\nOA4hohkllTP25xmMzLokBqY9n38zEt74kJOp67VKMvhoF7QkrLOfKWCRJjFL7/9I\nHDa6jb31INA9Wu+p/2LIa6I1SUYnMvCUqISgF2hBG9Q9S9TZvKnYUvfurhFS9jZv\n18sxW7IFYWmQyioo+gsAmfKLolJtLl9hCmTfYi7oqCh/EtZdIQKCAQEA0Umkp0Uu\nimVilLjgYGTWLcg8T3NWaELQzb2HYRXSzEq/M8GOtEr7TR7noJBm8fcgl55HEnPl\ni4cEJrr+VprzGbdMtXjHbCD+I945GA6vv3khg7mbqS9a1Uw6gjrQEZgZQU+/IVCu\n9Pbvx8Af32xaBWuN2cFzC7Z6iB815LPc2O5qyZ3+3nEUPah+Z+a9WEeTR6M0hy5c\nkkaRqhehugHDgqMRWGt8GfsFOmaR13kvfFfKadPRPkaGkftCSKBMWjrU4uX7aulm\nD7k4VDbnXIBMhI039+0znSkhZdcV1zk6qwBYn9TtZ11PTlspFPjtPxqS5M6IGflw\nsXkZGv4rZ5CkiQKCAQEAzLVdw2qw/8rWGsCV39EKp7hXLvp7+FuodPvX1L55lWB0\nvmSOldGcNvb2ZsK3RNvgteb8VfKRgaY6waeN5Qm1UXazsOX4F+GThPGHstdNuzkt\nJofRQQHQVR3npZbCngSkSZdahQ9SjiLIDKn8baPN8I8HfpJ4oHLUvkayavbch1kJ\nYWUfGtVKxHGX5m/nnxLdgbJEx9Q+3Qa7DDHuxTqsEqhkk0R0Ganred34HjpDNMs6\nV95HFNolW3yKfuHETKA1bLhej+XdMa11Ts5hBVGCMnnT07WcGhxtyK2dSa656SyT\ngT9+Hd1VWZ/KPpAkQmH9boOr2ihE+oAXiZ4D1t53HwKCAQAD0cA7fTu4Mtl1tVoC\n6FQwSbMwD/7HsFB3MLpDv041hDexDhs4lxW29pVrjLcUO1pQ6gaKA6twvGoK+uah\nVfqRwZKYzTd2dbOtm+SW183FRMSjzsNUdxTFR7rZnZEmgQwU8Quf5AUNW2RM1Oi/\n/w41gxz3mFwtHotl6IvnPJEPNGqme0enb5Da/zQvWTqjXcsGR6gxv1rZIIiP/hZp\nepbCz48FehCtuLMDudN3hzKipkd/Xuo2pLrX9ynigWpjSyePbHsGHHRMXSj2AHqA\naab71EftMlr6x0FgxmgToWu8qyjy4cPjWwSTfX5mb5SEzktX+ZzqPG8eDgOzRmgs\nX6thAoIBADL3kQG/hZQaL1Z3zpjsFggOKH7E1KrQP0/pCCKqzeC4JDjnFm0MxCUX\nNd/96N1XFUqU2QyZGUs7VPO0QOrekOtYb4LCrxNbEXyPGicX3f2YTbqDJEFYL0OR\n74PV1ly7cR/1dA8e8oH6/O3SQMwXdYXIRqhn1Wq1TGyXc4KYNe3o6CH8qFLo+fWR\nBq3T/MopS0coWGGcYY5sR5PQts8aPY9jp67W40UkfkFYV5dHEEaLttn7uJzjd1ug\n1Waj1VjypnqMKNcQ9xKQSl21mohVc+IXXPsgA16o51iIiVm4DAeXFp6ebUsIOWDY\nHOWYw75XYV7rn5TwY8Qusi2MTw5nUycCggEAB/45U0LW7ZGpks/aF/BeGaSWiLIG\nodBWUjRQ4w+Le/pTC8Ci9fiidxuCDH6TQbsUTGKOk7GsfncWHTQJogaMyO26IJ1N\nmYGgK2JJvs7PKyIkocPDVD/Yh0gIzQIE92ZdyXUT21pIYKDUB9e3p0fy/+E0pyeI\nsmsV8oaLr4tZRY1cMogI+pvtUUferbLQmZHhFd9X3m3RslR43Dl1qpYQyzE3x/a3\nWA2NJZbJhh+LiAKzqk7swXOqrTrmXuzLcjMG+T/3lizrbLLuKjQrf+eehlpw0db0\nHVVvkMLOP5ZH/ImkmvOZJY7xxup89VV7LD7TfMKwXafOrjMDdvTAYPtgxw==\n-----END RSA PRIVATE KEY-----\n" -} +## Certificates + +When `SMQ_PROVISION_X509_PROVISIONING=true`, the provisioning flow issues certificates for each client and returns them in the response as `client_cert`, `client_key`, and `ca_cert`. The certificate TTL is controlled by `SMQ_PROVISION_CERTS_HOURS_VALID`. + +## Testing + +```bash +go test ./provision/... ``` +For an in-depth explanation of our Provision Service, see the [official documentation][doc]. + +[doc]: https://docs.magistrala.absmach.eu/dev-guide/provision/ [supermq]: https://github.com/absmach/supermq [bootstrap]: https://github.com/absmach/supermq/tree/main/bootstrap [export]: https://github.com/absmach/export diff --git a/re/README.md b/re/README.md index de87ba7a9..cacb89e3e 100644 --- a/re/README.md +++ b/re/README.md @@ -1,201 +1,323 @@ -# Magistrala Rules Engine +# Rules Engine -The Magistrala Rules Engine (RE) is a service that enables real-time message processing and transformation through user-defined rules. It allows you to create rules that process incoming messages using Lua scripts and publish the results to output channels. +The Magistrala Rules Engine (RE) processes incoming messages using user-defined scripts (Lua or Go) and routes the results to outputs such as channels, alarms, email, SenML writers, PostgreSQL, or Slack. It also supports scheduled rule execution and publishes rule events to the event store. + +## Configuration + +The service is configured using the following environment variables (values shown are from [docker/.env](https://github.com/absmach/magistrala/blob/main/docker/.env) as consumed by [docker/docker-compose.yaml](https://github.com/absmach/magistrala/blob/main/docker/docker-compose.yaml)): + +### Core service + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_RE_LOG_LEVEL` | Log level for the service | `debug` | +| `MG_RE_HTTP_HOST` | HTTP host to bind | `re` | +| `MG_RE_HTTP_PORT` | HTTP port to bind | `9008` | +| `MG_RE_HTTP_SERVER_CERT` | Path to PEM-encoded HTTPS server certificate | "" | +| `MG_RE_HTTP_SERVER_KEY` | Path to PEM-encoded HTTPS server key | "" | +| `MG_RE_INSTANCE_ID` | Instance ID for tracing/health | "" | +| `SMQ_MESSAGE_BROKER_URL` | Internal message broker URL | `nats://nats:4222` | +| `SMQ_ES_URL` | Event store broker URL | `nats://nats:4222` | +| `SMQ_JAEGER_URL` | Jaeger collector endpoint | `http://jaeger:4318/v1/traces` | +| `SMQ_JAEGER_TRACE_RATIO` | Trace sampling ratio | `1.0` | +| `SMQ_SEND_TELEMETRY` | Send telemetry to Magistrala call-home server | `true` | + +### Database + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_RE_DB_HOST` | PostgreSQL host | `re-db` | +| `MG_RE_DB_PORT` | PostgreSQL port | `5432` | +| `MG_RE_DB_USER` | PostgreSQL user | `magistrala` | +| `MG_RE_DB_PASS` | PostgreSQL password | `magistrala` | +| `MG_RE_DB_NAME` | PostgreSQL database name | `rules_engine` | +| `MG_RE_DB_SSL_MODE` | PostgreSQL SSL mode | `disable` | +| `MG_RE_DB_SSL_CERT` | PostgreSQL SSL client cert | "" | +| `MG_RE_DB_SSL_KEY` | PostgreSQL SSL client key | "" | +| `MG_RE_DB_SSL_ROOT_CERT` | PostgreSQL SSL root cert | "" | + +### Auth and domains gRPC + +| Variable | Description | Default | +| --- | --- | --- | +| `SMQ_AUTH_GRPC_URL` | Auth gRPC endpoint | `auth:7001` | +| `SMQ_AUTH_GRPC_TIMEOUT` | Auth gRPC timeout | `300s` | +| `SMQ_AUTH_GRPC_CLIENT_CERT` | Auth gRPC client cert path | `${GRPC_MTLS:+./ssl/certs/auth-grpc-client.crt}` | +| `SMQ_AUTH_GRPC_CLIENT_KEY` | Auth gRPC client key path | `${GRPC_MTLS:+./ssl/certs/auth-grpc-client.key}` | +| `SMQ_AUTH_GRPC_SERVER_CA_CERTS` | Auth gRPC server CA path | `${GRPC_MTLS:+./ssl/certs/ca.crt}` | +| `SMQ_DOMAINS_GRPC_URL` | Domains gRPC endpoint | `domains:7003` | +| `SMQ_DOMAINS_GRPC_TIMEOUT` | Domains gRPC timeout | `300s` | +| `SMQ_DOMAINS_GRPC_CLIENT_CERT` | Domains gRPC client cert path | `${GRPC_MTLS:+./ssl/certs/domains-grpc-client.crt}` | +| `SMQ_DOMAINS_GRPC_CLIENT_KEY` | Domains gRPC client key path | `${GRPC_MTLS:+./ssl/certs/domains-grpc-client.key}` | +| `SMQ_DOMAINS_GRPC_SERVER_CA_CERTS` | Domains gRPC server CA path | `${GRPC_MTLS:+./ssl/certs/ca.crt}` | +| `SMQ_ALLOW_UNVERIFIED_USER` | Allow unverified users to access | `true` | + +### Readers gRPC + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_TIMESCALE_READER_GRPC_URL` | Readers gRPC endpoint | `timescale-reader:7011` | +| `MG_TIMESCALE_READER_GRPC_TIMEOUT` | Readers gRPC timeout | `300s` | +| `MG_TIMESCALE_READER_GRPC_CLIENT_CERT` | Readers gRPC client cert path | `${GRPC_MTLS:+./ssl/certs/reader-grpc-client.crt}` | +| `MG_TIMESCALE_READER_GRPC_CLIENT_CA_CERTS` | Readers gRPC server CA path | `${GRPC_MTLS:+./ssl/certs/ca.crt}` | +| `MG_TIMESCALE_READER_GRPC_CLIENT_KEY` | Readers gRPC client key path | `${GRPC_MTLS:+./ssl/certs/readers-grpc-client.key}` | + +### Email + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_EMAIL_HOST` | SMTP host | `smtp.mailtrap.io` | +| `MG_EMAIL_PORT` | SMTP port | `2525` | +| `MG_EMAIL_USERNAME` | SMTP username | `18bf7f70705139` | +| `MG_EMAIL_PASSWORD` | SMTP password | `2b0d302e775b1e` | +| `MG_EMAIL_FROM_ADDRESS` | Sender email address | `from@example.com` | +| `MG_EMAIL_FROM_NAME` | Sender display name | `Example` | +| `MG_EMAIL_TEMPLATE` | Email template path | `email.tmpl` | +| `MG_RE_EMAIL_TEMPLATE` | Template file mounted by Docker Compose | `re.tmpl` | + +### Callout + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_RE_CALLOUT_URLS` | Callout target URLs | "" | +| `MG_RE_CALLOUT_METHOD` | Callout HTTP method | `POST` | +| `MG_RE_CALLOUT_TLS_VERIFICATION` | TLS verification for callout | `false` | +| `MG_RE_CALLOUT_TIMEOUT` | Callout timeout | `10s` | +| `MG_RE_CALLOUT_CA_CERT` | Callout CA cert path | "" | +| `MG_RE_CALLOUT_CERT` | Callout client cert path | "" | +| `MG_RE_CALLOUT_KEY` | Callout client key path | "" | +| `MG_RE_CALLOUT_OPERATIONS` | Callout operations filter | "" | + +### Optional cache defaults (from code) + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_RE_CACHE_URL` | Cache URL | `redis://localhost:6379/0` | +| `MG_RE_CACHE_KEY_DURATION` | Cache key TTL | `10m` | + +## Features + +- **Rule execution**: Runs Lua or Go scripts for incoming messages. +- **Multiple outputs**: Channels, alarms, email, SenML writers, remote PostgreSQL, and Slack outputs. +- **Scheduling**: Runs rules at specific times with recurring intervals. +- **Filtering and matching**: Input channel filtering and NATS-style topic matching (`*`, `>`). +- **Observability**: `/metrics` Prometheus endpoint and Jaeger tracing support. +- **Payload limit**: Messages over 100 kB are rejected for processing. ## Architecture -The Rules Engine operates by: -1. Listening for messages on configured input channels -2. Processing these messages through Lua scripts -3. Optionally publishing results to output channels -4. Supporting scheduled rule execution based on various recurring patterns +### Runtime flow -## Core Concepts +1. The service subscribes to all internal broker messages. +2. For each message, it lists enabled rules for the same domain and input channel. +3. It matches the rule `input_topic` against the message subtopic using NATS-style wildcards. +4. The rule logic (Lua or Go) is executed and the result is passed to configured outputs. -### Rules +### Message payloads -A rule consists of: +In Lua, the engine injects a global `message` object: -- `id` - Unique identifier -- `name` - Human-readable name -- `domain` - Domain the rule belongs to -- `input_channel` - Channel to listen for incoming messages -- `input_topic` - Specific topic within the input channel -- `logic` - Lua script that processes the message -- `output_channel` - (Optional) Channel to publish results to -- `output_topic` - (Optional) Topic within the output channel -- `schedule` - (Optional) Scheduling configuration -- `status` - Rule state (enabled/disabled/deleted) -- `metadata` - Additional rule metadata +```lua +message = { + domain = "domain_id", + channel = "channel_id", + subtopic = "subtopic", + publisher = "client_id", + protocol = "nats", + created = timestamp, + payload = { ... } -- JSON object/array or a byte array if payload is not JSON +} +``` -A rule can be in one of these states: -- `enabled` - Rule is active and processing messages -- `disabled` - Rule is inactive and won't process messages -- `deleted` - Rule is marked for deletion +For Go scripts, the message is exposed as `messaging/m.message` and `main.logicFunction` must return a value. -### Message Processing +In rule definitions, `logic.type` uses numeric values: `0` = Lua, `1` = Go. -When a message arrives on a rule's input channel, the Rules Engine: - -1. Creates a Lua environment -2. Injects the message as a global variable with the following structure: - ```lua - message = { - channel = "channel_name", - subtopic = "subtopic_name", - publisher = "publisher_id", - protocol = "protocol_name", - created = timestamp, - payload = [byte_array] - } - ``` -3. Executes the rule's Lua script -4. If the script returns a non-nil value and an output channel is configured, publishes the result +If a script returns `false`, outputs are skipped. ### Scheduling -Rules can be scheduled to run at specific times with various recurring patterns. The scheduler works through several key components: +The scheduler runs on a 30-second ticker and selects enabled rules with a due time (`time`) earlier than now. It updates the next due time using `Schedule.NextDue()` and executes each rule with a synthetic message containing the scheduled timestamp. -#### Schedule Structure -```go -type Schedule struct { - StartDateTime time.Time // When the schedule becomes active - Time time.Time // Specific time for the rule to run - Recurring Recurring // None, Daily, Weekly, Monthly - RecurringPeriod uint // Interval between executions: 1 = every interval, 2 = every second interval, etc. -} -``` +Recurring types are: `none`, `hourly`, `daily`, `weekly`, `monthly`. The `recurring_period` controls the interval (1 = every interval, 2 = every second interval, etc.). -#### How Scheduling Works +### Outputs -1. **Initialization**: - - The scheduler starts when the service begins running via `StartScheduler()` - - It uses a ticker to check for rules that need to be executed at regular intervals +Supported output types (`outputs.OutputType`) and their fields: -2. **Rule Evaluation**: - - For each tick, the scheduler: - - Gets all enabled rules scheduled before the current time - - For each rule, checks if it should run using `shouldRunRule()` - - If a rule should run, processes it asynchronously +| Output type | Fields | Notes | +| --- | --- | --- | +| `channels` | `channel`, `topic` | Republish result to another channel/topic. | +| `alarms` | none | Emits alarms from the script result. | +| `save_senml` | none | Forwards SenML to writers. | +| `email` | `to`, `subject`, `content` | `content` is a Go template. | +| `save_remote_pg` | `host`, `port`, `user`, `password`, `database`, `table`, `mapping` | `mapping` is a Go template that must render a JSON object. | +| `slack` | `token`, `channel_id`, `message` | `message` is a Go template. | -3. **Execution Timing**: - The `shouldRunRule()` function determines if a rule should run by checking: - - If the rule's start time has been reached - - If the current time matches the scheduled execution time - - For recurring rules: - - **Daily**: Checks if the correct number of days have passed since start - - **Weekly**: Checks if the correct number of weeks have passed since start - - **Monthly**: Checks if the correct number of months have passed since start +Templates receive a `Message` (the incoming message) and a `Result` (the script output) value. -4. **Recurring Patterns**: - - `None`: Rule runs once at the specified time - - `Daily`: Rule runs every N days where N is the RecurringPeriod - - `Weekly`: Rule runs every N weeks - - `Monthly`: Rule runs every N months +## Data model -For example, to run a rule: -- Every day at 9 AM: Set recurring to "daily" with recurring_period = 1 -- Every other week: Set recurring to "weekly" with recurring_period = 2 -- Monthly on the 1st: Set recurring to "monthly" with recurring_period = 1 +### Rules table -## API Operations +Defined in `re/postgres/init.go`: -The Rules Engine service provides the following operations: +| Column | Type | Description | +| --- | --- | --- | +| `id` | `VARCHAR(36)` | Rule UUID (primary key) | +| `name` | `VARCHAR(1024)` | Rule name | +| `domain_id` | `VARCHAR(36)` | Domain ID | +| `metadata` | `JSONB` | Custom metadata | +| `tags` | `TEXT[]` | Rule tags | +| `created_by` | `VARCHAR(254)` | Creator user ID | +| `created_at` | `TIMESTAMP` | Creation timestamp | +| `updated_at` | `TIMESTAMP` | Last update timestamp | +| `updated_by` | `VARCHAR(254)` | Last updater user ID | +| `input_channel` | `VARCHAR(36)` | Input channel ID | +| `input_topic` | `TEXT` | Input topic (supports wildcards) | +| `outputs` | `JSONB` | Output definitions | +| `status` | `SMALLINT` | 0 = enabled, 1 = disabled, 2 = deleted | +| `logic_type` | `SMALLINT` | 0 = Lua, 1 = Go | +| `logic_value` | `BYTEA` | Script body | +| `start_datetime` | `TIMESTAMP` | Schedule start time | +| `time` | `TIMESTAMP` | Next scheduled execution time | +| `recurring` | `SMALLINT` | Recurring type | +| `recurring_period` | `SMALLINT` | Recurring period | -- `AddRule` - Create a new rule -- `ViewRule` - Retrieve a specific rule -- `UpdateRule` - Modify an existing rule -- `ListRules` - Query rules with filtering options -- `RemoveRule` - Delete a rule -- `EnableRule` - Activate a rule -- `DisableRule` - Deactivate a rule +## Deployment -## Using the API - -### Adding a Rule - -You can create a new rule using the Rules Engine API. Here's an example using curl: +### Build and run locally ```bash -curl --location 'http://localhost:9008/8353542f-d8f1-4dce-b787-4af3712f117e/rules' \ ---header 'Content-Type: application/json' \ ---header 'Authorization: Bearer ' \ ---data '{ - "name": "High Temperature Alert", - "input_channel": "sensors", - "input_topic": "temperature", - "logic": { - "type": 0, - "value": "if message.payload > 30 then return '\''Temperature too high!'\'' end" - }, - "output_channel": "alerts", - "output_topic": "temperature", - "schedule": { - "start_datetime": "2024-01-01T00:00", - "time": "2024-01-01T09:00", - "recurring": "daily", - "recurring_period": 1 - } -}' +make re + +MG_RE_LOG_LEVEL=debug \ +MG_RE_HTTP_PORT=9008 \ +MG_RE_DB_HOST=localhost \ +MG_RE_DB_PORT=5432 \ +MG_RE_DB_USER=magistrala \ +MG_RE_DB_PASS=magistrala \ +MG_RE_DB_NAME=rules_engine \ +SMQ_MESSAGE_BROKER_URL=nats://localhost:4222 \ +SMQ_ES_URL=nats://localhost:4222 \ +SMQ_AUTH_GRPC_URL=localhost:7001 \ +SMQ_AUTH_GRPC_TIMEOUT=300s \ +SMQ_DOMAINS_GRPC_URL=localhost:7003 \ +SMQ_DOMAINS_GRPC_TIMEOUT=300s \ +MG_TIMESCALE_READER_GRPC_URL=localhost:7011 \ +MG_TIMESCALE_READER_GRPC_TIMEOUT=300s \ +./build/re ``` -This request: -- Creates a temperature monitoring rule -- Processes messages from the "sensors" channel -- Checks for temperatures above 30 degrees -- Publishes alerts to the "alerts" channel -- Runs daily at 9 AM +### Docker Compose -The API endpoint follows the format: `http://localhost:9008/{domain_id}/rules` - -Required headers: -- `Content-Type: application/json` - Specifies the request body format -- `Authorization: Bearer ` - Your authentication token - -### Example Rule Structure - -Here's a breakdown of the rule structure: - -```json -{ - "name": "High Temperature Alert", - "input_channel": "sensors", - "input_topic": "temperature", - "logic": { - "type": 0, - "value": "if message.payload > 30 then return 'Temperature too high!' end" - }, - "output_channel": "alerts", - "output_topic": "temperature", - "schedule": { - "start_datetime": "2024-01-01T00:00", - "time": "2024-01-01T09:00", - "recurring": "daily", - "recurring_period": 1 - } -} -``` - -This rule: -1. Listens on the "sensors" channel, "temperature" topic -2. Checks if temperature exceeds 30 degrees -3. If true, publishes an alert message -4. Runs daily at 9 AM - -## Running the Service - -To start the Rules Engine service, run: +The service is available as a Docker container. Refer to [docker/docker-compose.yaml](https://github.com/absmach/magistrala/blob/main/docker/docker-compose.yaml) for the `re` and `re-db` services and their environment variables. For a full local stack, ensure auth, domains, readers, and the message broker are running. ```bash -make run_addons re +docker compose -f docker/docker-compose.yaml up re re-db ``` -This command starts the Rules Engine service using Docker Compose with the configuration defined in [docker-compose.yaml][compose]. +### Health check -## For More Information +```bash +curl -X GET http://localhost:9008/health \ + -H "accept: application/health+json" +``` -- [Magistrala Documentation][doc] -- [Docker Compose][compose] +## Testing -[doc]: https://docs.magistrala.absmach.eu -[compose]: ../docker/docker-compose.yaml +```bash +go test ./re/... +``` + +## Usage + +The Rules Engine service supports the following operations: + +| Operation | Method & Path | Description | +| --- | --- | --- | +| `createRule` | `POST /{domainID}/rules` | Create a new rule | +| `listRules` | `GET /{domainID}/rules` | List rules with filters | +| `viewRule` | `GET /{domainID}/rules/{ruleID}` | Retrieve a rule | +| `updateRule` | `PATCH /{domainID}/rules/{ruleID}` | Update a rule | +| `updateRuleTags` | `PATCH /{domainID}/rules/{ruleID}/tags` | Update rule tags | +| `updateRuleSchedule` | `PATCH /{domainID}/rules/{ruleID}/schedule` | Update rule schedule | +| `enableRule` | `POST /{domainID}/rules/{ruleID}/enable` | Enable a rule | +| `disableRule` | `POST /{domainID}/rules/{ruleID}/disable` | Disable a rule | +| `removeRule` | `DELETE /{domainID}/rules/{ruleID}` | Delete a rule | +| `health` | `GET /health` | Service health check | + +List filters: `offset`, `limit`, `name`, `input_channel`, `status`, `order` (`name`, `created_at`, `updated_at`), `dir` (`asc`, `desc`), and `tag`. + +### Example: Create a rule (Lua + alarms + channels) + +```bash +curl -X POST http://localhost:9008//rules \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "name": "High Temperature Alert", + "input_channel": "sensors", + "input_topic": "temperature.*", + "logic": { + "type": 0, + "value": "if message.payload.t > 30 then return {measurement=\"temperature\", value=tostring(message.payload.t), unit=\"C\", threshold=\"30\", cause=\"temp high\", severity=90} end" + }, + "outputs": [ + { "type": "alarms" }, + { "type": "channels", "channel": "alerts", "topic": "temperature" } + ], + "tags": ["temp", "alerts"], + "metadata": { "site": "lab" } + }' +``` + +### Example: List rules + +```bash +curl -X GET "http://localhost:9008//rules?status=enabled&input_channel=sensors&order=updated_at&dir=desc&tag=temp" \ + -H "Authorization: Bearer " +``` + +### Example: Update rule tags + +```bash +curl -X PATCH http://localhost:9008//rules//tags \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ "tags": ["temp", "critical"] }' +``` + +### Example: Update rule schedule + +```bash +curl -X PATCH http://localhost:9008//rules//schedule \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "schedule": { + "start_datetime": "2025-01-01T00:00:00Z", + "time": "2025-01-01T00:00:00Z", + "recurring": "hourly", + "recurring_period": 1 + } + }' +``` + +### Example: Enable a rule + +```bash +curl -X POST http://localhost:9008//rules//enable \ + -H "Authorization: Bearer " +``` + +### Example: Delete a rule + +```bash +curl -X DELETE http://localhost:9008//rules/ \ + -H "Authorization: Bearer " +``` + +For an in-depth explanation of our Rules Engine Service, see the [official documentation][doc]. + +[doc]: https://docs.magistrala.absmach.eu/dev-guide/rules-engine/ diff --git a/readers/README.md b/readers/README.md index 4f22131e4..88e2b9ac5 100644 --- a/readers/README.md +++ b/readers/README.md @@ -1,7 +1,252 @@ # Readers -Readers provide implementations of various `message readers`. Message readers are services that consume normalized (in `SenML` format) SuperMQ messages from data storage and expose HTTP API for message consumption. +Readers expose HTTP and gRPC APIs for retrieving stored messages. They read normalized SenML records (and optional JSON payloads) from storage backends and apply authn/authz checks via the Auth, Clients, and Channels services. Magistrala provides two reader services: `postgres-reader` and `timescale-reader`. -For an in-depth explanation of the usage of `reader`, as well as thorough understanding of SuperMQ, please check out the [official documentation][doc]. +## Configuration -[doc]: https://docs.supermq.abstractmachines.fr +Readers are optional services. Values shown are from [docker/.env](https://github.com/absmach/magistrala/blob/main/docker/.env) and the corresponding add-on compose files in `docker/addons/*-reader/docker-compose.yaml`. + +### Postgres reader + +#### Service endpoints + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_POSTGRES_READER_LOG_LEVEL` | Service log level | `debug` | +| `MG_POSTGRES_READER_HTTP_HOST` | HTTP host | `postgres-reader` | +| `MG_POSTGRES_READER_HTTP_PORT` | HTTP port | `9009` | +| `MG_POSTGRES_READER_HTTP_SERVER_CERT` | HTTPS server certificate path | "" | +| `MG_POSTGRES_READER_HTTP_SERVER_KEY` | HTTPS server key path | "" | +| `MG_POSTGRES_READER_GRPC_HOST` | gRPC host | `postgres-reader` | +| `MG_POSTGRES_READER_GRPC_PORT` | gRPC port | `7009` | +| `MG_POSTGRES_READER_GRPC_SERVER_CERT` | gRPC server cert path | `${GRPC_MTLS:+./ssl/certs/readers-grpc-server.crt}${GRPC_TLS:+./ssl/certs/readers-grpc-server.crt}` | +| `MG_POSTGRES_READER_GRPC_SERVER_KEY` | gRPC server key path | `${GRPC_MTLS:+./ssl/certs/readers-grpc-server.key}${GRPC_TLS:+./ssl/certs/readers-grpc-server.key}` | +| `MG_POSTGRES_READER_GRPC_SERVER_CA_CERTS` | gRPC server CA certs path | `${GRPC_MTLS:+./ssl/certs/ca.crt}${GRPC_TLS:+./ssl/certs/ca.crt}` | +| `MG_POSTGRES_READER_INSTANCE_ID` | Instance ID | "" | + +#### Database + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_POSTGRES_HOST` | PostgreSQL host | `postgres` | +| `MG_POSTGRES_PORT` | PostgreSQL port | `5432` | +| `MG_POSTGRES_USER` | PostgreSQL user | `supermq` | +| `MG_POSTGRES_PASS` | PostgreSQL password | `supermq` | +| `MG_POSTGRES_NAME` | PostgreSQL database name | `messages` | +| `MG_POSTGRES_SSL_MODE` | PostgreSQL SSL mode | `disable` | +| `MG_POSTGRES_SSL_CERT` | PostgreSQL SSL client cert | "" | +| `MG_POSTGRES_SSL_KEY` | PostgreSQL SSL client key | "" | +| `MG_POSTGRES_SSL_ROOT_CERT` | PostgreSQL SSL root cert | "" | + +#### Dependencies + +| Variable | Description | Default | +| --- | --- | --- | +| `SMQ_AUTH_GRPC_URL` | Auth gRPC URL | `auth:7001` | +| `SMQ_AUTH_GRPC_TIMEOUT` | Auth gRPC timeout | `300s` | +| `SMQ_AUTH_GRPC_CLIENT_CERT` | Auth gRPC client cert | `${GRPC_MTLS:+./ssl/certs/auth-grpc-client.crt}` | +| `SMQ_AUTH_GRPC_CLIENT_KEY` | Auth gRPC client key | `${GRPC_MTLS:+./ssl/certs/auth-grpc-client.key}` | +| `SMQ_AUTH_GRPC_CLIENT_CA_CERTS` | Auth gRPC CA certs | `${GRPC_MTLS:+./ssl/certs/ca.crt}` | +| `SMQ_CLIENTS_GRPC_URL` | Clients gRPC URL | `clients:7006` | +| `SMQ_CLIENTS_GRPC_TIMEOUT` | Clients gRPC timeout | `300s` | +| `SMQ_CLIENTS_GRPC_CLIENT_CERT` | Clients gRPC client cert | `${GRPC_MTLS:+./ssl/certs/clients-grpc-client.crt}` | +| `SMQ_CLIENTS_GRPC_CLIENT_KEY` | Clients gRPC client key | `${GRPC_MTLS:+./ssl/certs/clients-grpc-client.key}` | +| `SMQ_CLIENTS_GRPC_CLIENT_CA_CERTS` | Clients gRPC CA certs | `${GRPC_MTLS:+./ssl/certs/ca.crt}` | +| `SMQ_CHANNELS_GRPC_URL` | Channels gRPC URL | `channels:7005` | +| `SMQ_CHANNELS_GRPC_TIMEOUT` | Channels gRPC timeout | `300s` | +| `SMQ_CHANNELS_GRPC_CLIENT_CERT` | Channels gRPC client cert | `${GRPC_MTLS:+./ssl/certs/channels-grpc-client.crt}` | +| `SMQ_CHANNELS_GRPC_CLIENT_KEY` | Channels gRPC client key | `${GRPC_MTLS:+./ssl/certs/channels-grpc-client.key}` | +| `SMQ_CHANNELS_GRPC_CLIENT_CA_CERTS` | Channels gRPC CA certs | `${GRPC_MTLS:+./ssl/certs/ca.crt}` | +| `SMQ_SEND_TELEMETRY` | Send telemetry to call-home server | `true` | + +Note: When running the postgres reader binary directly, configuration is read from `SMQ_POSTGRES_*` and `SMQ_POSTGRES_READER_HTTP_*` prefixes (see `cmd/postgres-reader/main.go` and `readers/postgres/README.md`). The Docker add-on uses `MG_POSTGRES_*`/`MG_POSTGRES_READER_*` values. + +### Timescale reader + +#### Service endpoints + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_TIMESCALE_READER_LOG_LEVEL` | Service log level | `debug` | +| `MG_TIMESCALE_READER_HTTP_HOST` | HTTP host | `timescale-reader` | +| `MG_TIMESCALE_READER_HTTP_PORT` | HTTP port | `9011` | +| `MG_TIMESCALE_READER_HTTP_SERVER_CERT` | HTTPS server certificate path | "" | +| `MG_TIMESCALE_READER_HTTP_SERVER_KEY` | HTTPS server key path | "" | +| `MG_TIMESCALE_READER_GRPC_HOST` | gRPC host | `timescale-reader` | +| `MG_TIMESCALE_READER_GRPC_PORT` | gRPC port | `7011` | +| `MG_TIMESCALE_READER_GRPC_SERVER_CERT` | gRPC server cert path | `${GRPC_MTLS:+./ssl/certs/readers-grpc-server.crt}${GRPC_TLS:+./ssl/certs/readers-grpc-server.crt}` | +| `MG_TIMESCALE_READER_GRPC_SERVER_KEY` | gRPC server key path | `${GRPC_MTLS:+./ssl/certs/readers-grpc-server.key}${GRPC_TLS:+./ssl/certs/readers-grpc-server.key}` | +| `MG_TIMESCALE_READER_GRPC_SERVER_CA_CERTS` | gRPC server CA certs path | `${GRPC_MTLS:+./ssl/certs/ca.crt}${GRPC_TLS:+./ssl/certs/ca.crt}` | +| `MG_TIMESCALE_READER_INSTANCE_ID` | Instance ID | "" | + +#### Database + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_TIMESCALE_HOST` | TimescaleDB host | `timescale` | +| `MG_TIMESCALE_PORT` | TimescaleDB port | `5432` | +| `MG_TIMESCALE_USER` | TimescaleDB user | `supermq` | +| `MG_TIMESCALE_PASS` | TimescaleDB password | `supermq` | +| `MG_TIMESCALE_NAME` | TimescaleDB database name | `supermq` | +| `MG_TIMESCALE_SSL_MODE` | TimescaleDB SSL mode | `disable` | +| `MG_TIMESCALE_SSL_CERT` | TimescaleDB SSL client cert | "" | +| `MG_TIMESCALE_SSL_KEY` | TimescaleDB SSL client key | "" | +| `MG_TIMESCALE_SSL_ROOT_CERT` | TimescaleDB SSL root cert | "" | + +#### Dependencies + +Timescale reader uses the same gRPC dependency variables listed for the Postgres reader (`SMQ_AUTH_GRPC_*`, `SMQ_CLIENTS_GRPC_*`, `SMQ_CHANNELS_GRPC_*`) and `SMQ_SEND_TELEMETRY`. + +## Features + +- **Message retrieval**: Read SenML messages by channel with paging and filters. +- **Flexible filters**: Subtopic, publisher, protocol, name, numeric/string/data/bool values, and time range. +- **Aggregation**: Timescale reader supports `aggregation` + `interval` (requires `from` and `to`). +- **Multiple formats**: Use `format=messages` for SenML or another table name for JSON payloads. +- **Authn/authz**: Supports user tokens or thing keys; enforces channel access. +- **Observability**: `/metrics` Prometheus endpoint and `/health` checks. + +## Architecture + +### Runtime flow + +1. Client calls `GET /{domainID}/channels/{chanID}/messages` with a user token or thing key. +2. The service authenticates the caller and authorizes channel access via gRPC (Auth, Clients, Channels). +3. The reader repository builds a filtered SQL query and reads from storage. +4. Results are returned as a paged list of SenML messages or JSON payloads. + +### Components + +- **HTTP API**: `readers/api/http` exposes the messages endpoint, health, and metrics. +- **gRPC API**: `readers/api/grpc` exposes the readers service for internal use. +- **Repositories**: `readers/postgres` and `readers/timescale` implement storage access. +- **Middleware**: `readers/middleware` adds logging and metrics. + +### Messages table (SenML) + +Defined in `readers/postgres/init.go` and consumed by both readers: + +| Column | Type | Description | +| --- | --- | --- | +| `id` | `UUID` | Message ID (primary key) | +| `channel` | `UUID` | Channel ID | +| `subtopic` | `VARCHAR(254)` | Subtopic | +| `publisher` | `UUID` | Publisher (client) ID | +| `protocol` | `TEXT` | Protocol name | +| `name` | `TEXT` | SenML name | +| `unit` | `TEXT` | SenML unit | +| `value` | `FLOAT` | Numeric value | +| `string_value` | `TEXT` | String value | +| `bool_value` | `BOOL` | Boolean value | +| `data_value` | `TEXT` | Data value (base64 or raw string) | +| `sum` | `FLOAT` | Sum value | +| `time` | `FLOAT` | Measurement time | +| `update_time` | `FLOAT` | Update time | + +## Deployment + +### Build and run locally + +Postgres reader: + +```bash +make postgres-reader + +SMQ_POSTGRES_READER_LOG_LEVEL=info \ +SMQ_POSTGRES_READER_HTTP_PORT=9009 \ +SMQ_POSTGRES_HOST=localhost \ +SMQ_POSTGRES_PORT=5432 \ +SMQ_POSTGRES_USER=supermq \ +SMQ_POSTGRES_PASS=supermq \ +SMQ_POSTGRES_NAME=messages \ +SMQ_AUTH_GRPC_URL=localhost:7001 \ +SMQ_CLIENTS_GRPC_URL=localhost:7006 \ +SMQ_CHANNELS_GRPC_URL=localhost:7005 \ +./build/postgres-reader +``` + +Timescale reader: + +```bash +make timescale-reader + +MG_TIMESCALE_READER_LOG_LEVEL=info \ +MG_TIMESCALE_READER_HTTP_PORT=9011 \ +MG_TIMESCALE_HOST=localhost \ +MG_TIMESCALE_PORT=5432 \ +MG_TIMESCALE_USER=supermq \ +MG_TIMESCALE_PASS=supermq \ +MG_TIMESCALE_NAME=supermq \ +SMQ_AUTH_GRPC_URL=localhost:7001 \ +SMQ_CLIENTS_GRPC_URL=localhost:7006 \ +SMQ_CHANNELS_GRPC_URL=localhost:7005 \ +./build/timescale-reader +``` + +### Docker Compose + +Postgres reader add-on: + +```bash +docker compose -f docker/docker-compose.yaml -f docker/addons/postgres-reader/docker-compose.yaml up +``` + +Timescale reader add-on: + +```bash +docker compose -f docker/docker-compose.yaml -f docker/addons/timescale-reader/docker-compose.yaml up +``` + +## Testing + +```bash +go test ./readers/... +``` + +## Usage + +The Readers service supports the following operations (see `apidocs/openapi/readers.yaml`): + +| Operation | Method & Path | Description | +| --- | --- | --- | +| `getMessages` | `GET /{domainID}/channels/{chanID}/messages` | List messages with filters | +| `health` | `GET /health` | Service health check | + +Supported query parameters include: + +`limit`, `offset`, `format`, `subtopic`, `publisher`, `protocol`, `name`, `v`, `comparator`, `vs`, `vd`, `vb`, `from`, `to`, `aggregation`, `interval`, `order`, `dir`. + +Comparator usage (for `vs`/`vd`): + +| Comparator | Usage | Example | +| --- | --- | --- | +| `eq` | Equal to the query | `eq["active"] -> "active"` | +| `ge` | Substrings of the query | `ge["tiv"] -> "active", "tiv"` | +| `gt` | Substrings excluding exact match | `gt["tiv"] -> "active"` | +| `le` | Superstrings of the query | `le["active"] -> "tiv"` | +| `lt` | Superstrings excluding exact match | `lt["active"] -> "active", "tiv"` | + +### Example: List messages + +```bash +curl -X GET "http://localhost:9009//channels//messages?limit=10&offset=0&subtopic=s1&name=temp&v=21.5&comparator=ge" \ + -H "Authorization: Bearer " +``` + +### Example: Aggregate messages (Timescale) + +```bash +curl -X GET "http://localhost:9011//channels//messages?aggregation=avg&interval=10s&from=1709218556069&to=1709218757503" \ + -H "Authorization: Thing " +``` + +### Example: Health check + +```bash +curl -X GET http://localhost:9009/health \ + -H "accept: application/health+json" +``` + +For an in-depth explanation of Readers Service, see the [official documentation][doc]. + +[doc]: https://docs.magistrala.absmach.eu/dev-guide/readers/ diff --git a/readers/postgres/README.md b/readers/postgres/README.md index cb7190d59..72eeb25fc 100644 --- a/readers/postgres/README.md +++ b/readers/postgres/README.md @@ -98,4 +98,4 @@ Comparator Usage Guide: | le | Return values that are superstrings of the query | le["active"] -> "tiv" | | lt | Return values that are superstrings of the query and not equal to the query | lt["active"] -> "active" and "tiv" | -Official docs can be found [here](https://docs.supermq.abstractmachines.fr). +Official docs can be found [here](https://docs.supermq.absmach.eu). diff --git a/readers/timescale/README.md b/readers/timescale/README.md index 6a3534fc1..a144b3b31 100644 --- a/readers/timescale/README.md +++ b/readers/timescale/README.md @@ -96,4 +96,4 @@ Comparator Usage Guide: | le | Return values that are superstrings of the query | le["active"] -> "tiv" | | lt | Return values that are superstrings of the query and not equal to the query | lt["active"] -> "active" and "tiv" | -Official docs can be found [here](https://docs.supermq.abstractmachines.fr). +Official docs can be found [here](https://docs.supermq.absmach.eu). diff --git a/reports/README.md b/reports/README.md new file mode 100644 index 000000000..c51e45625 --- /dev/null +++ b/reports/README.md @@ -0,0 +1,335 @@ +# Reports + +The Reports service generates time-series reports from stored messages. It fetches data from the readers gRPC service, formats results as JSON, CSV, or PDF, optionally emails the report, and supports scheduled report delivery. + +## Configuration + +The service is configured using the following environment variables (values shown are from [docker/.env](https://github.com/absmach/magistrala/blob/main/docker/.env) where available, otherwise from service defaults): + +### Core service + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_REPORTS_LOG_LEVEL` | Log level for the service | `debug` | +| `MG_REPORTS_HTTP_HOST` | HTTP host to bind | `reports` | +| `MG_REPORTS_HTTP_PORT` | HTTP port to bind | `9017` | +| `MG_REPORTS_HTTP_SERVER_CERT` | Path to PEM-encoded HTTPS server certificate | "" | +| `MG_REPORTS_HTTP_SERVER_KEY` | Path to PEM-encoded HTTPS server key | "" | +| `MG_REPORTS_INSTANCE_ID` | Instance ID for tracing/health | "" | +| `SMQ_JAEGER_URL` | Jaeger collector endpoint | `http://jaeger:4318/v1/traces` | +| `SMQ_JAEGER_TRACE_RATIO` | Trace sampling ratio | `1.0` | +| `SMQ_SEND_TELEMETRY` | Send telemetry to Magistrala call-home server | `true` | +| `SMQ_MESSAGE_BROKER_URL` | Message broker URL (parsed, currently unused by reports) | `nats://nats:4222` | +| `SMQ_ES_URL` | Event store URL (parsed, currently unused by reports) | `nats://nats:4222` | + +### Database + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_REPORTS_DB_HOST` | PostgreSQL host | `reports-db` | +| `MG_REPORTS_DB_PORT` | PostgreSQL port | `5432` | +| `MG_REPORTS_DB_USER` | PostgreSQL user | `magistrala` | +| `MG_REPORTS_DB_PASS` | PostgreSQL password | `magistrala` | +| `MG_REPORTS_DB_NAME` | PostgreSQL database name | `reports` | +| `MG_REPORTS_DB_SSL_MODE` | PostgreSQL SSL mode | `disable` | +| `MG_REPORTS_DB_SSL_CERT` | PostgreSQL SSL client cert | "" | +| `MG_REPORTS_DB_SSL_KEY` | PostgreSQL SSL client key | "" | +| `MG_REPORTS_DB_SSL_ROOT_CERT` | PostgreSQL SSL root cert | "" | + +### Auth and domains gRPC + +| Variable | Description | Default | +| --- | --- | --- | +| `SMQ_AUTH_GRPC_URL` | Auth gRPC endpoint | `auth:7001` | +| `SMQ_AUTH_GRPC_TIMEOUT` | Auth gRPC timeout | `300s` | +| `SMQ_AUTH_GRPC_CLIENT_CERT` | Auth gRPC client cert path | `${GRPC_MTLS:+./ssl/certs/auth-grpc-client.crt}` | +| `SMQ_AUTH_GRPC_CLIENT_KEY` | Auth gRPC client key path | `${GRPC_MTLS:+./ssl/certs/auth-grpc-client.key}` | +| `SMQ_AUTH_GRPC_SERVER_CA_CERTS` | Auth gRPC server CA path | `${GRPC_MTLS:+./ssl/certs/ca.crt}` | +| `SMQ_DOMAINS_GRPC_URL` | Domains gRPC endpoint | `domains:7003` | +| `SMQ_DOMAINS_GRPC_TIMEOUT` | Domains gRPC timeout | `300s` | +| `SMQ_DOMAINS_GRPC_CLIENT_CERT` | Domains gRPC client cert path | `${GRPC_MTLS:+./ssl/certs/domains-grpc-client.crt}` | +| `SMQ_DOMAINS_GRPC_CLIENT_KEY` | Domains gRPC client key path | `${GRPC_MTLS:+./ssl/certs/domains-grpc-client.key}` | +| `SMQ_DOMAINS_GRPC_SERVER_CA_CERTS` | Domains gRPC server CA path | `${GRPC_MTLS:+./ssl/certs/ca.crt}` | +| `SMQ_ALLOW_UNVERIFIED_USER` | Allow unverified users to access | `true` | +| `SMQ_SPICEDB_PRE_SHARED_KEY` | SpiceDB pre-shared key | `12345678` | +| `SMQ_SPICEDB_HOST` | SpiceDB host | `supermq-spicedb` | +| `SMQ_SPICEDB_PORT` | SpiceDB gRPC port | `50051` | + +### Readers gRPC + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_TIMESCALE_READER_GRPC_URL` | Readers gRPC endpoint | `timescale-reader:7011` | +| `MG_TIMESCALE_READER_GRPC_TIMEOUT` | Readers gRPC timeout | `300s` | +| `MG_TIMESCALE_READER_GRPC_CLIENT_CERT` | Readers gRPC client cert path | `${GRPC_MTLS:+./ssl/certs/reader-grpc-client.crt}` | +| `MG_TIMESCALE_READER_GRPC_CLIENT_CA_CERTS` | Readers gRPC server CA path | `${GRPC_MTLS:+./ssl/certs/ca.crt}` | +| `MG_TIMESCALE_READER_GRPC_CLIENT_KEY` | Readers gRPC client key path | `${GRPC_MTLS:+./ssl/certs/readers-grpc-client.key}` | + +### Email + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_EMAIL_HOST` | SMTP host | `smtp.mailtrap.io` | +| `MG_EMAIL_PORT` | SMTP port | `2525` | +| `MG_EMAIL_USERNAME` | SMTP username | `18bf7f70705139` | +| `MG_EMAIL_PASSWORD` | SMTP password | `2b0d302e775b1e` | +| `MG_EMAIL_FROM_ADDRESS` | Sender email address | `from@example.com` | +| `MG_EMAIL_FROM_NAME` | Sender display name | `Example` | +| `MG_EMAIL_TEMPLATE` | Email template path | `email.tmpl` | +| `MG_REPORTS_EMAIL_TEMPLATE` | Template file mounted by Docker Compose | `reports.tmpl` | + +### Templates and PDF conversion + +| Variable | Description | Default | +| --- | --- | --- | +| `MG_REPORTS_DEFAULT_TEMPLATE` | Use on-disk HTML template when non-empty | "" | +| `MG_PDF_CONVERTER_URL` | HTML-to-PDF conversion endpoint | `http://pdf-generator:3000/forms/chromium/convert/html` | + +## Features + +- **Report generation**: Build report data from time-series messages. +- **Multiple formats**: JSON responses, CSV exports, and PDF rendering. +- **Scheduling**: Periodic report delivery via email. +- **Template support**: Custom HTML templates for PDF reports. +- **Observability**: `/metrics` Prometheus endpoint and Jaeger tracing support. + +## Architecture + +### Runtime flow + +1. The Reports API receives a report request or a scheduled run triggers report generation. +2. The service expands requested metrics and fetches messages via the readers gRPC API in batches of 1000. +3. Results are grouped by publisher when `client_ids` are not specified. +4. Output is returned as JSON, rendered to CSV, or converted to PDF via `MG_PDF_CONVERTER_URL`. +5. For scheduled/email actions, the report is sent as an email attachment. + +### Scheduling + +The scheduler runs on a 30-second ticker and selects enabled report configs with `due` time earlier than now. It updates `due` using `Schedule.NextDue()` and generates a report with the `email` action. + +Recurring types are: `none`, `hourly`, `daily`, `weekly`, `monthly`. The `recurring_period` controls the interval (1 = every interval, 2 = every second interval, etc.). + +### Templates + +PDF templates are Go `html/template` documents. A template must include: + +- `{{$.Title}}` +- `{{range .Messages}}` or `{{range .Reports}}` +- `{{formatTime .Time}}` +- `{{formatValue .}}` +- `{{end}}` + +Helper functions include `formatTime`, `formatValue`, `add`, `sub`, `div`, `mod`, `iterate`, `eq`, `ge`, `lt`, `getStartRow`, and `getEndRow`. + +## Data model + +### report_config table + +Defined in `reports/postgres/init.go`: + +| Column | Type | Description | +| --- | --- | --- | +| `id` | `VARCHAR(36)` | Report config UUID (primary key) | +| `name` | `VARCHAR(1024)` | Report name | +| `description` | `TEXT` | Report description | +| `domain_id` | `VARCHAR(36)` | Domain ID | +| `status` | `SMALLINT` | 0 = enabled, 1 = disabled, 2 = deleted | +| `created_at` | `TIMESTAMP` | Creation timestamp | +| `created_by` | `VARCHAR(254)` | Creator user ID | +| `updated_at` | `TIMESTAMP` | Last update timestamp | +| `updated_by` | `VARCHAR(254)` | Last updater user ID | +| `due` | `TIMESTAMPTZ` | Next scheduled execution time | +| `recurring` | `SMALLINT` | Recurring type | +| `recurring_period` | `SMALLINT` | Recurring period | +| `start_datetime` | `TIMESTAMP` | Schedule start time | +| `config` | `JSONB` | Metric config (from/to/title/format/aggregation) | +| `email` | `JSONB` | Email settings | +| `metrics` | `JSONB` | Requested metrics list | +| `report_template` | `TEXT` | Custom HTML template | + +## Deployment + +### Build and run locally + +```bash +make reports + +MG_REPORTS_LOG_LEVEL=debug \ +MG_REPORTS_HTTP_PORT=9017 \ +MG_REPORTS_DB_HOST=localhost \ +MG_REPORTS_DB_PORT=5432 \ +MG_REPORTS_DB_USER=magistrala \ +MG_REPORTS_DB_PASS=magistrala \ +MG_REPORTS_DB_NAME=reports \ +MG_PDF_CONVERTER_URL=http://localhost:4000/forms/chromium/convert/html \ +SMQ_AUTH_GRPC_URL=localhost:7001 \ +SMQ_AUTH_GRPC_TIMEOUT=300s \ +SMQ_DOMAINS_GRPC_URL=localhost:7003 \ +SMQ_DOMAINS_GRPC_TIMEOUT=300s \ +MG_TIMESCALE_READER_GRPC_URL=localhost:7011 \ +MG_TIMESCALE_READER_GRPC_TIMEOUT=300s \ +./build/reports +``` + +### Docker Compose + +The service is available as a Docker container. Refer to [docker/docker-compose.yaml](https://github.com/absmach/magistrala/blob/main/docker/docker-compose.yaml) for the `reports`, `reports-db`, and `pdf-generator` services and their environment variables. For a full local stack, ensure auth, domains, readers, and the PDF generator are running. + +```bash +docker compose -f docker/docker-compose.yaml up reports reports-db pdf-generator +``` + +### Health check + +```bash +curl -X GET http://localhost:9017/health \ + -H "accept: application/health+json" +``` + +## Testing + +```bash +go test ./reports/... +``` + +## Usage + +The Reports service supports the following operations: + +| Operation | Method & Path | Description | +| --- | --- | --- | +| `generateReport` | `POST /{domainID}/reports` | Generate a report (`action` query param) | +| `addReportConfig` | `POST /{domainID}/reports/configs` | Create a report configuration | +| `listReportsConfig` | `GET /{domainID}/reports/configs` | List report configurations | +| `viewReportConfig` | `GET /{domainID}/reports/configs/{reportID}` | View a report configuration | +| `updateReportConfig` | `PATCH /{domainID}/reports/configs/{reportID}` | Update a report configuration | +| `updateReportSchedule` | `PATCH /{domainID}/reports/configs/{reportID}/schedule` | Update schedule | +| `enableReportConfig` | `POST /{domainID}/reports/configs/{reportID}/enable` | Enable a report configuration | +| `disableReportConfig` | `POST /{domainID}/reports/configs/{reportID}/disable` | Disable a report configuration | +| `deleteReportConfig` | `DELETE /{domainID}/reports/configs/{reportID}` | Delete a report configuration | +| `updateReportTemplate` | `PUT /{domainID}/reports/configs/{reportID}/template` | Update custom template | +| `viewReportTemplate` | `GET /{domainID}/reports/configs/{reportID}/template` | View custom template | +| `deleteReportTemplate` | `DELETE /{domainID}/reports/configs/{reportID}/template` | Delete custom template | +| `health` | `GET /health` | Service health check | + +List filters: `offset`, `limit`, `status`, `name`, `order` (`name`, `created_at`, `updated_at`), and `dir` (`asc`, `desc`). + +Time ranges use relative expressions parsed by `pkg/reltime`, such as `now()` or `now()-24h` (units: `s`, `m`, `h`, `d`, `w`). Aggregation intervals use Go duration strings like `15m` or `1h`. File output formats are `pdf` and `csv`. + +### Example: Generate a report + +```bash +curl -X POST "http://localhost:9017//reports?action=view" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "name": "temperature-view", + "metrics": [ + { + "channel_id": "", + "client_ids": [""], + "name": "temperature", + "subtopic": "sensor" + } + ], + "config": { + "from": "now()-24h", + "to": "now()", + "title": "Temperature (last 24h)", + "timezone": "UTC", + "aggregation": { + "agg_type": "avg", + "interval": "1h" + } + } + }' +``` + +### Example: Generate and email a report + +```bash +curl -X POST "http://localhost:9017//reports?action=email" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "name": "temperature-email", + "metrics": [ + { + "channel_id": "", + "name": "temperature" + } + ], + "config": { + "from": "now()-1d", + "to": "now()", + "title": "Daily Temperature", + "file_format": "csv" + }, + "email": { + "to": ["ops@example.com"], + "subject": "Daily temperature report", + "content": "Report attached." + } + }' + +``` + +### Example: Create a scheduled report config + +```bash +curl -X POST "http://localhost:9017//reports/configs" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "name": "daily-temperature", + "description": "Daily temperature summary", + "metrics": [ + { + "channel_id": "", + "name": "temperature" + } + ], + "config": { + "from": "now()-1d", + "to": "now()", + "title": "Daily Temperature", + "file_format": "pdf", + "aggregation": { + "agg_type": "avg", + "interval": "1h" + } + }, + "email": { + "to": ["ops@example.com"], + "subject": "Daily temperature report", + "content": "Report attached." + }, + "schedule": { + "start_datetime": "2025-01-01T00:00:00Z", + "recurring": "daily", + "recurring_period": 1 + } + }' +``` + +### Example: Update a report template + +```bash +curl -X PUT "http://localhost:9017//reports/configs//template" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "report_template": "

{{$.Title}}

{{range .Reports}}{{range .Messages}}{{formatTime .Time}} {{formatValue .}}{{end}}{{end}}" + }' +``` + +### Example: Enable a report config + +```bash +curl -X POST "http://localhost:9017//reports/configs//enable" \ + -H "Authorization: Bearer " +``` + +For an in-depth explanation of our Reports Service, see the see the [official documentation][doc]. + +[doc]: https://docs.magistrala.absmach.eu/dev-guide/reports/ diff --git a/tools/e2e/README.md b/tools/e2e/README.md index c1412c8d4..b455a9b87 100644 --- a/tools/e2e/README.md +++ b/tools/e2e/README.md @@ -17,7 +17,7 @@ Tool for testing end-to-end flow of SuperMQ by doing a couple of operations name 1. Creating, viewing, updating and changing status of users, groups, clients and channels. 2. Connecting users and groups to each other and clients and channels to each other. 3. Sending messages from clients to channels on all 4 protocol adapters (HTTP, WS, CoAP and MQTT). -Complete documentation is available at https://docs.supermq.abstractmachines.fr +Complete documentation is available at https://docs.supermq.absmach.eu Usage: diff --git a/tools/e2e/cmd/main.go b/tools/e2e/cmd/main.go index 3a6d6715a..2a1e92f4e 100644 --- a/tools/e2e/cmd/main.go +++ b/tools/e2e/cmd/main.go @@ -24,7 +24,7 @@ func main() { "1. Creating, viewing, updating and changing status of users, groups, clients and channels.\n" + "2. Connecting users and groups to each other and clients and channels to each other.\n" + "3. Sending messages from clients to channels on all 4 protocol adapters (HTTP, WS, CoAP and MQTT).\n" + - "Complete documentation is available at https://docs.supermq.abstractmachines.fr", + "Complete documentation is available at https://docs.supermq.absmach.eu", Example: "Here is a simple example of using e2e tool.\n" + "Use the following commands from the root supermq directory:\n\n" + "go run tools/e2e/cmd/main.go\n" + diff --git a/tools/mqtt-bench/README.md b/tools/mqtt-bench/README.md index 86a6bc087..2acd5dd00 100644 --- a/tools/mqtt-bench/README.md +++ b/tools/mqtt-bench/README.md @@ -21,7 +21,7 @@ The tool supports multiple concurrent clients, publishers and subscribers config ``` ./mqtt-bench --help Tool for extensive load and benchmarking of MQTT brokers used within SuperMQ platform. -Complete documentation is available at https://docs.supermq.abstractmachines.fr +Complete documentation is available at https://docs.supermq.absmach.eu Usage: mqtt-bench [flags] diff --git a/tools/mqtt-bench/cmd/main.go b/tools/mqtt-bench/cmd/main.go index 9e58fcb1c..3a73f22d9 100644 --- a/tools/mqtt-bench/cmd/main.go +++ b/tools/mqtt-bench/cmd/main.go @@ -21,7 +21,7 @@ func main() { Use: "mqtt-bench", Short: "mqtt-bench is MQTT benchmark tool for SuperMQ", Long: `Tool for extensive load and benchmarking of MQTT brokers used within the SuperMQ platform. -Complete documentation is available at https://docs.supermq.abstractmachines.fr`, +Complete documentation is available at https://docs.supermq.absmach.eu`, Run: func(cmd *cobra.Command, args []string) { if confFile != "" { viper.SetConfigFile(confFile) diff --git a/tools/provision/README.md b/tools/provision/README.md index d3a95e5eb..6cf027720 100644 --- a/tools/provision/README.md +++ b/tools/provision/README.md @@ -17,7 +17,7 @@ make ```bash ./provision --help Tool for provisioning series of SuperMQ channels and clients and connecting them together. -Complete documentation is available at https://docs.supermq.abstractmachines.fr +Complete documentation is available at https://docs.supermq.absmach.eu Usage: provision [flags] diff --git a/tools/provision/cmd/main.go b/tools/provision/cmd/main.go index 632d8555a..8f0f65905 100644 --- a/tools/provision/cmd/main.go +++ b/tools/provision/cmd/main.go @@ -18,7 +18,7 @@ func main() { Use: "provision", Short: "provision is provisioning tool for SuperMQ", Long: `Tool for provisioning series of SuperMQ channels and clients and connecting them together. -Complete documentation is available at https://docs.supermq.abstractmachines.fr`, +Complete documentation is available at https://docs.supermq.absmach.eu`, Run: func(cmd *cobra.Command, _ []string) { if err := provision.Provision(cmd.Context(), pconf); err != nil { log.Fatal(err)