opensource/magistrala
1
0
Fork 0
mirror of https://github.com/absmach/magistrala.git synced 2026-06-23 04:10:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

SMQ-2629 - Remove Readers and Consumers (#2641)

Browse Source 
Signed-off-by: Felix Gateru <felix.gateru@gmail.com>
This commit is contained in:
Felix Gateru
2025-01-10 13:26:39 +03:00
committed by GitHub
parent df5d752c4b
commit a9169276e5
122 changed files with 6 additions and 13684 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/api-tests.yml
-9
View File
@@ -14,13 +14,11 @@ on:
- "certs/api/**"
- "channels/api/http/**"
- "clients/api/http/**"
- "consumers/notifiers/api/**"
- "domains/api/http/**"
- "groups/api/http/**"
- "http/api/**"
- "invitations/api/**"
- "journal/api/**"
- "readers/api/**"
- "users/api/**"
env:
@@ -38,8 +36,6 @@ env:
INVITATIONS_URL: http://localhost:9020
AUTH_URL: http://localhost:9001
CERTS_URL: http://localhost:9019
POSTGRES_READER_URL: http://localhost:9009
TIMESCALE_READER_URL: http://localhost:9011
JOURNAL_URL: http://localhost:9021
jobs:
@@ -104,11 +100,6 @@ jobs:
- "apidocs/openapi/invitations.yml"
- "invitations/api/**"
readers:
- ".github/workflows/api-tests.yml"
- "apidocs/openapi/readers.yml"
- "readers/api/**"
clients:
- ".github/workflows/api-tests.yml"
- "apidocs/openapi/clients.yml"
.github/workflows/check-generated-files.yml
-10
View File
@@ -62,10 +62,6 @@ jobs:
- "users/emailer.go"
- "users/hasher.go"
- "mqtt/events/streams.go"
- "readers/messages.go"
- "consumers/notifiers/notifier.go"
- "consumers/notifiers/service.go"
- "consumers/notifiers/subscriptions.go"
- "certs/certs.go"
- "certs/pki/vault.go"
- "certs/service.go"
@@ -145,9 +141,6 @@ jobs:
mv ./users/mocks/service.go ./users/mocks/service.go.tmp
mv ./journal/mocks/repository.go ./journal/mocks/repository.go.tmp
mv ./journal/mocks/service.go ./journal/mocks/service.go.tmp
mv ./consumers/notifiers/mocks/notifier.go ./consumers/notifiers/mocks/notifier.go.tmp
mv ./consumers/notifiers/mocks/repository.go ./consumers/notifiers/mocks/repository.go.tmp
mv ./consumers/notifiers/mocks/service.go ./consumers/notifiers/mocks/service.go.tmp
mv ./certs/mocks/pki.go ./certs/mocks/pki.go.tmp
mv ./certs/mocks/service.go ./certs/mocks/service.go.tmp
mv ./clients/private/mocks/service.go ./clients/private/mocks/service.go.tmp
@@ -206,9 +199,6 @@ jobs:
check_mock_changes ./users/mocks/service.go " ./users/mocks/service.go"
check_mock_changes ./journal/mocks/repository.go " ./journal/mocks/repository.go"
check_mock_changes ./journal/mocks/service.go " ./journal/mocks/service.go"
check_mock_changes ./consumers/notifiers/mocks/notifier.go " ./consumers/notifiers/mocks/notifier.go"
check_mock_changes ./consumers/notifiers/mocks/repository.go " ./consumers/notifiers/mocks/repository.go"
check_mock_changes ./consumers/notifiers/mocks/service.go " ./consumers/notifiers/mocks/service.go"
check_mock_changes ./certs/mocks/pki.go " ./certs/mocks/pki.go"
check_mock_changes ./certs/mocks/service.go " ./certs/mocks/service.go"
check_mock_changes ./clients/private/mocks/service.go " ./clients/private/mocks/service.go"
.github/workflows/tests.yml
-34
View File
@@ -131,19 +131,6 @@ jobs:
- "clients/**"
- "pkg/messaging/**"
consumers:
- "consumers/**"
- "cmd/postgres-writer/**"
- "cmd/timescale-writer/**"
- "cmd/smpp-notifier/**"
- "cmd/smtp-notifier/**"
- "auth.pb.go"
- "auth_grpc.pb.go"
- "auth/**"
- "pkg/ulid/**"
- "pkg/uuid/**"
- "pkg/messaging/**"
domains:
- "domain/**"
- "cmd/domain/**"
@@ -225,14 +212,12 @@ jobs:
- "pkg/groups/**"
- "auth/**"
- "certs/**"
- "consumers/**"
- "http/**"
- "internal/*"
- "internal/api/**"
- "internal/apiutil/**"
- "internal/groups/**"
- "invitations/**"
- "readers/**"
- "clients/**"
- "users/**"
@@ -245,15 +230,6 @@ jobs:
pkg-uuid:
- "pkg/uuid/**"
readers:
- "readers/**"
- "cmd/postgres-reader/**"
- "cmd/timescale-reader/**"
- "auth.pb.go"
- "auth_grpc.pb.go"
- "clients/**"
- "auth/**"
users:
- "users/**"
- "cmd/users/**"
@@ -306,11 +282,6 @@ jobs:
run: |
go test --race -v -count=1 -coverprofile=coverage/coap.out ./coap/...
- name: Run consumers tests
if: steps.changes.outputs.consumers == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/consumers.out ./consumers/...
- name: Run HTTP tests
if: steps.changes.outputs.http == 'true' || steps.changes.outputs.workflow == 'true'
run: |
@@ -376,11 +347,6 @@ jobs:
run: |
go test --race -v -count=1 -coverprofile=coverage/pkg-uuid.out ./pkg/uuid/...
- name: Run readers tests
if: steps.changes.outputs.readers == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/readers.out ./readers/...
- name: Run clients tests
if: steps.changes.outputs.clients == 'true' || steps.changes.outputs.workflow == 'true'
run: |
Makefile
+3 -5
View File
@@ -3,9 +3,8 @@
SMQ_DOCKER_IMAGE_NAME_PREFIX ?= supermq
BUILD_DIR ?= build
SERVICES = auth users clients groups channels domains http coap ws postgres-writer postgres-reader timescale-writer \
timescale-reader cli mqtt certs invitations journal
TEST_API_SERVICES = journal auth certs http invitations notifiers readers clients users channels groups domains
SERVICES = auth users clients groups channels domains http coap ws cli mqtt certs invitations journal
TEST_API_SERVICES = journal auth certs http invitations clients users channels groups domains
TEST_API = $(addprefix test_api_,$(TEST_API_SERVICES))
DOCKERS = $(addprefix docker_,$(SERVICES))
DOCKERS_DEV = $(addprefix docker_dev_,$(SERVICES))
@@ -73,7 +72,7 @@ define make_docker_dev
-f docker/Dockerfile.dev ./build
endef
ADDON_SERVICES = journal certs timescale-reader timescale-writer postgres-reader postgres-writer
ADDON_SERVICES = journal certs
EXTERNAL_SERVICES = vault prometheus
@@ -176,7 +175,6 @@ test_api_http: TEST_API_URL := http://localhost:8008
test_api_invitations: TEST_API_URL := http://localhost:9020
test_api_auth: TEST_API_URL := http://localhost:9001
test_api_certs: TEST_API_URL := http://localhost:9019
test_api_readers: TEST_API_URL := http://localhost:9009 # This can be the URL of any reader service.
test_api_journal: TEST_API_URL := http://localhost:9021
$(TEST_API):
apidocs/openapi/notifiers.yml
-292
View File
@@ -1,292 +0,0 @@
# Copyright (c) Abstract Machines
# SPDX-License-Identifier: Apache-2.0
openapi: 3.0.1
info:
title: SuperMQ Notifiers service
description: |
HTTP API for Notifiers service.
Some useful links:
- [The SuperMQ repository](https://github.com/absmach/supermq)
contact:
email: info@abstractmachines.fr
license:
name: Apache 2.0
url: https://github.com/absmach/supermq/blob/main/LICENSE
version: 0.15.1
servers:
- url: http://localhost:9014
- url: https://localhost:9014
- url: http://localhost:9015
- url: https://localhost:9015
tags:
- name: notifiers
description: Everything about your Notifiers
externalDocs:
description: Find out more about notifiers
url: https://docs.supermq.abstractmachines.fr/
paths:
/subscriptions:
post:
operationId: createSubscription
summary: Create subscription
description: Creates a new subscription give a topic and contact.
tags:
- notifiers
requestBody:
$ref: "#/components/requestBodies/Create"
responses:
"201":
$ref: "#/components/responses/Create"
"400":
description: Failed due to malformed JSON.
"401":
description: Missing or invalid access token provided.
"403":
description: Failed to perform authorization over the entity.
"409":
description: Failed due to using an existing topic and contact.
"415":
description: Missing or invalid content type.
"422":
description: Database can't process request.
"500":
$ref: "#/components/responses/ServiceError"
get:
operationId: listSubscriptions
summary: List subscriptions
description: List subscriptions given list parameters.
tags:
- notifiers
parameters:
- $ref: "#/components/parameters/Topic"
- $ref: "#/components/parameters/Contact"
- $ref: "#/components/parameters/Offset"
- $ref: "#/components/parameters/Limit"
responses:
"200":
$ref: "#/components/responses/Page"
"400":
description: Failed due to malformed query parameters.
"401":
description: Missing or invalid access token provided.
"403":
description: Failed to perform authorization over the entity.
"404":
description: A non-existent entity request.
"422":
description: Database can't process request.
"500":
$ref: "#/components/responses/ServiceError"
/subscriptions/{id}:
get:
operationId: viewSubscription
summary: Get subscription with the provided id
description: Retrieves a subscription with the provided id.
tags:
- notifiers
parameters:
- $ref: "#/components/parameters/Id"
responses:
"200":
$ref: "#/components/responses/View"
"400":
description: Failed due to malformed ID.
"401":
description: Missing or invalid access token provided.
"403":
description: Failed to perform authorization over the entity.
"404":
description: A non-existent entity request.
"422":
description: Database can't process request.
"500":
$ref: "#/components/responses/ServiceError"
delete:
operationId: removeSubscription
summary: Delete subscription with the provided id
description: Removes a subscription with the provided id.
tags:
- notifiers
parameters:
- $ref: "#/components/parameters/Id"
responses:
"204":
description: Subscription removed
"401":
description: Missing or invalid access token provided.
"403":
description: Failed to perform authorization over the entity.
"404":
description: A non-existent entity request.
"422":
description: Database can't process request.
"500":
$ref: "#/components/responses/ServiceError"
/health:
get:
summary: Retrieves service health check info.
tags:
- health
security: []
responses:
"200":
$ref: "#/components/responses/HealthRes"
"500":
$ref: "#/components/responses/ServiceError"
components:
schemas:
Subscription:
type: object
properties:
id:
type: string
format: ulid
example: 01EWDVKBQSG80B6PQRS9PAAY35
description: ULID id of the subscription.
owner_id:
type: string
format: uuid
example: 18167738-f7a8-4e96-a123-58c3cd14de3a
description: An id of the owner who created subscription.
topic:
type: string
example: topic.subtopic
description: Topic to which the user subscribes.
contact:
type: string
example: user@example.com
description: The contact of the user to which the notification will be sent.
CreateSubscription:
type: object
properties:
topic:
type: string
example: topic.subtopic
description: Topic to which the user subscribes.
contact:
type: string
example: user@example.com
description: The contact of the user to which the notification will be sent.
Page:
type: object
properties:
subscriptions:
type: array
minItems: 0
uniqueItems: true
items:
$ref: "#/components/schemas/Subscription"
total:
type: integer
description: Total number of items.
offset:
type: integer
description: Number of items to skip during retrieval.
limit:
type: integer
description: Maximum number of items to return in one page.
parameters:
Id:
name: id
description: Unique identifier.
in: path
schema:
type: string
format: ulid
required: true
Limit:
name: limit
description: Size of the subset to retrieve.
in: query
schema:
type: integer
default: 10
maximum: 100
minimum: 1
required: false
Offset:
name: offset
description: Number of items to skip during retrieval.
in: query
schema:
type: integer
default: 0
minimum: 0
required: false
Topic:
name: topic
description: Topic name.
in: query
schema:
type: string
required: false
Contact:
name: contact
description: Subscription contact.
in: query
schema:
type: string
required: false
requestBodies:
Create:
description: JSON-formatted document describing the new subscription to be created
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/CreateSubscription"
responses:
Create:
description: Created a new subscription.
headers:
Location:
content:
text/plain:
schema:
type: string
description: Created subscription relative URL
example: /subscriptions/{id}
View:
description: View subscription.
content:
application/json:
schema:
$ref: "#/components/schemas/Subscription"
links:
delete:
operationId: removeSubscription
parameters:
id: $response.body#/id
Page:
description: Data retrieved.
content:
application/json:
schema:
$ref: "#/components/schemas/Page"
ServiceError:
description: Unexpected server-side error occurred.
HealthRes:
description: Service Health Check.
content:
application/health+json:
schema:
$ref: "./schemas/health_info.yml"
securitySchemes:
bearerAuth:
type: http
scheme: bearer
bearerFormat: JWT
description: |
* Users access: "Authorization: Bearer <user_token>"
security:
- bearerAuth: []
cli/config.go
-8
View File
@@ -21,7 +21,6 @@ const (
defURL string = "http://localhost"
defUsersURL string = defURL + ":9002"
defCLientsURL string = defURL + ":9000"
defReaderURL string = defURL + ":9011"
defDomainsURL string = defURL + ":8189"
defCertsURL string = defURL + ":9019"
defInvitationsURL string = defURL + ":9020"
@@ -37,7 +36,6 @@ const (
type remotes struct {
ClientsURL string `toml:"clients_url"`
UsersURL string `toml:"users_url"`
ReaderURL string `toml:"reader_url"`
DomainsURL string `toml:"domains_url"`
HTTPAdapterURL string `toml:"http_adapter_url"`
CertsURL string `toml:"certs_url"`
@@ -107,7 +105,6 @@ func ParseConfig(sdkConf smqsdk.Config) (smqsdk.Config, error) {
Remotes: remotes{
ClientsURL: defCLientsURL,
UsersURL: defUsersURL,
ReaderURL: defReaderURL,
DomainsURL: defDomainsURL,
HTTPAdapterURL: defHTTPURL,
CertsURL: defCertsURL,
@@ -176,10 +173,6 @@ func ParseConfig(sdkConf smqsdk.Config) (smqsdk.Config, error) {
sdkConf.UsersURL = config.Remotes.UsersURL
}
if sdkConf.ReaderURL == "" && config.Remotes.ReaderURL != "" {
sdkConf.ReaderURL = config.Remotes.ReaderURL
}
if sdkConf.DomainsURL == "" && config.Remotes.DomainsURL != "" {
sdkConf.DomainsURL = config.Remotes.DomainsURL
}
@@ -253,7 +246,6 @@ func setConfigValue(key, value string) error {
configKeyToField := map[string]interface{}{
"clients_url": &config.Remotes.ClientsURL,
"users_url": &config.Remotes.UsersURL,
"reader_url": &config.Remotes.ReaderURL,
"http_adapter_url": &config.Remotes.HTTPAdapterURL,
"certs_url": &config.Remotes.CertsURL,
"tls_verification": &config.Remotes.TLSVerification,
cli/consumers.go
-100
View File
@@ -1,100 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package cli
import (
smqsdk "github.com/absmach/supermq/pkg/sdk"
"github.com/spf13/cobra"
)
var cmdSubscription = []cobra.Command{
{
Use: "create <topic> <contact> <user_auth_token>",
Short: "Create subscription",
Long: `Create new subscription`,
Run: func(cmd *cobra.Command, args []string) {
if len(args) != 3 {
logUsageCmd(*cmd, cmd.Use)
return
}
id, err := sdk.CreateSubscription(args[0], args[1], args[2])
if err != nil {
logErrorCmd(*cmd, err)
return
}
logCreatedCmd(*cmd, id)
},
},
{
Use: "get [all | <sub_id>] <user_auth_token>",
Short: "Get subscription",
Long: `Get subscription.
all - lists all subscriptions
<sub_id> - view subscription of <sub_id>`,
Run: func(cmd *cobra.Command, args []string) {
if len(args) != 2 {
logUsageCmd(*cmd, cmd.Use)
return
}
pageMetadata := smqsdk.PageMetadata{
Offset: Offset,
Limit: Limit,
Topic: Topic,
Contact: Contact,
}
if args[0] == "all" {
sub, err := sdk.ListSubscriptions(pageMetadata, args[1])
if err != nil {
logErrorCmd(*cmd, err)
return
}
logJSONCmd(*cmd, sub)
return
}
c, err := sdk.ViewSubscription(args[0], args[1])
if err != nil {
logErrorCmd(*cmd, err)
return
}
logJSONCmd(*cmd, c)
},
},
{
Use: "remove <sub_id> <user_auth_token>",
Short: "Remove subscription",
Long: `Removes removes a subscription with the provided id`,
Run: func(cmd *cobra.Command, args []string) {
if len(args) != 2 {
logUsageCmd(*cmd, cmd.Use)
return
}
if err := sdk.DeleteSubscription(args[0], args[1]); err != nil {
logErrorCmd(*cmd, err)
return
}
logOKCmd(*cmd)
},
},
}
// NewSubscriptionCmd returns subscription command.
func NewSubscriptionCmd() *cobra.Command {
cmd := cobra.Command{
Use: "subscription [create | get | remove ]",
Short: "Subscription management",
Long: `Subscription management: create, get, or delete subscription`,
}
for i := range cmdSubscription {
cmd.AddCommand(&cmdSubscription[i])
}
return &cmd
}
cli/consumers_test.go
-273
View File
@@ -1,273 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package cli_test
import (
"encoding/json"
"fmt"
"net/http"
"strings"
"testing"
"github.com/absmach/supermq/cli"
"github.com/absmach/supermq/internal/testsutil"
"github.com/absmach/supermq/pkg/errors"
svcerr "github.com/absmach/supermq/pkg/errors/service"
mgsdk "github.com/absmach/supermq/pkg/sdk"
sdkmocks "github.com/absmach/supermq/pkg/sdk/mocks"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
var subscription = mgsdk.Subscription{
ID: testsutil.GenerateUUID(&testing.T{}),
OwnerID: user.ID,
Topic: "topic",
Contact: "identity@example.com",
}
func TestCreateSubscriptionCmd(t *testing.T) {
sdkMock := new(sdkmocks.SDK)
cli.SetSDK(sdkMock)
subCmd := cli.NewSubscriptionCmd()
rootCmd := setFlags(subCmd)
cases := []struct {
desc string
args []string
logType outputLog
errLogMessage string
sdkErr errors.SDKError
response string
id string
}{
{
desc: "create subscription successfully",
args: []string{
subscription.Topic,
subscription.Contact,
validToken,
},
id: user.ID,
response: fmt.Sprintf("\ncreated: %s\n\n", user.ID),
logType: createLog,
},
{
desc: "create subscription with invalid args",
args: []string{
subscription.Topic,
subscription.Contact,
validToken,
extraArg,
},
logType: usageLog,
},
{
desc: "create subscription with invalid token",
args: []string{
subscription.Topic,
subscription.Contact,
invalidToken,
},
logType: errLog,
sdkErr: errors.NewSDKErrorWithStatus(svcerr.ErrAuthorization, http.StatusForbidden),
errLogMessage: fmt.Sprintf("\nerror: %s\n\n", errors.NewSDKErrorWithStatus(svcerr.ErrAuthorization, http.StatusForbidden)),
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
sdkCall := sdkMock.On("CreateSubscription", tc.args[0], tc.args[1], tc.args[2]).Return(tc.id, tc.sdkErr)
out := executeCommand(t, rootCmd, append([]string{createCmd}, tc.args...)...)
switch tc.logType {
case usageLog:
assert.False(t, strings.Contains(out, rootCmd.Use), fmt.Sprintf("%s invalid usage: %s", tc.desc, out))
case errLog:
assert.Equal(t, tc.errLogMessage, out, fmt.Sprintf("%s unexpected error response: expected %s got errLogMessage:%s", tc.desc, tc.errLogMessage, out))
case createLog:
assert.Equal(t, tc.response, out, fmt.Sprintf("%s unexpected error response: expected %s got errLogMessage:%s", tc.desc, tc.response, out))
}
sdkCall.Unset()
})
}
}
func TestGetSubscriptionsCmd(t *testing.T) {
sdkMock := new(sdkmocks.SDK)
cli.SetSDK(sdkMock)
subCmd := cli.NewSubscriptionCmd()
rootCmd := setFlags(subCmd)
var sub mgsdk.Subscription
var page mgsdk.SubscriptionPage
cases := []struct {
desc string
args []string
sdkErr errors.SDKError
page mgsdk.SubscriptionPage
subscription mgsdk.Subscription
logType outputLog
errLogMessage string
}{
{
desc: "get all subscriptions successfully",
args: []string{
all,
token,
},
page: mgsdk.SubscriptionPage{
Subscriptions: []mgsdk.Subscription{subscription},
},
logType: entityLog,
},
{
desc: "get subscription with id",
args: []string{
subscription.ID,
token,
},
logType: entityLog,
subscription: subscription,
},
{
desc: "get subscriptions with invalid args",
args: []string{
all,
token,
extraArg,
},
logType: usageLog,
},
{
desc: "get all subscriptions with invalid token",
args: []string{
all,
invalidToken,
},
logType: errLog,
sdkErr: errors.NewSDKErrorWithStatus(svcerr.ErrAuthorization, http.StatusForbidden),
errLogMessage: fmt.Sprintf("\nerror: %s\n\n", errors.NewSDKErrorWithStatus(svcerr.ErrAuthorization, http.StatusForbidden)),
},
{
desc: "get subscription without domain token",
args: []string{
subscription.ID,
tokenWithoutDomain,
},
logType: errLog,
sdkErr: errors.NewSDKErrorWithStatus(svcerr.ErrDomainAuthorization, http.StatusForbidden),
errLogMessage: fmt.Sprintf("\nerror: %s\n\n", errors.NewSDKErrorWithStatus(svcerr.ErrDomainAuthorization, http.StatusForbidden)),
},
{
desc: "get subscription with invalid id",
args: []string{
invalidID,
token,
},
sdkErr: errors.NewSDKErrorWithStatus(svcerr.ErrAuthorization, http.StatusForbidden),
errLogMessage: fmt.Sprintf("\nerror: %s\n\n", errors.NewSDKErrorWithStatus(svcerr.ErrAuthorization, http.StatusForbidden)),
logType: errLog,
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
sdkCall := sdkMock.On("ViewSubscription", tc.args[0], tc.args[1]).Return(tc.subscription, tc.sdkErr)
sdkCall1 := sdkMock.On("ListSubscriptions", mock.Anything, tc.args[1]).Return(tc.page, tc.sdkErr)
out := executeCommand(t, rootCmd, append([]string{getCmd}, tc.args...)...)
switch tc.logType {
case entityLog:
if tc.args[1] == all {
err := json.Unmarshal([]byte(out), &page)
assert.Nil(t, err)
assert.Equal(t, tc.page, page, fmt.Sprintf("%v unexpected response, expected: %v, got: %v", tc.desc, tc.page, page))
} else {
err := json.Unmarshal([]byte(out), &sub)
assert.Nil(t, err)
assert.Equal(t, tc.subscription, sub, fmt.Sprintf("%v unexpected response, expected: %v, got: %v", tc.desc, tc.subscription, sub))
}
case errLog:
assert.Equal(t, tc.errLogMessage, out, fmt.Sprintf("%s unexpected error response: expected %s got errLogMessage:%s", tc.desc, tc.errLogMessage, out))
case usageLog:
assert.False(t, strings.Contains(out, rootCmd.Use), fmt.Sprintf("%s invalid usage: %s", tc.desc, out))
}
sdkCall.Unset()
sdkCall1.Unset()
})
}
}
func TestRemoveSubscriptionCmd(t *testing.T) {
sdkMock := new(sdkmocks.SDK)
cli.SetSDK(sdkMock)
subCmd := cli.NewSubscriptionCmd()
rootCmd := setFlags(subCmd)
cases := []struct {
desc string
args []string
sdkErr errors.SDKError
logType outputLog
errLogMessage string
}{
{
desc: "remove subscription successfully",
args: []string{
subscription.ID,
token,
},
logType: okLog,
},
{
desc: "remove subscription with invalid args",
args: []string{
subscription.ID,
token,
extraArg,
},
logType: usageLog,
},
{
desc: "remove subscription with invalid subscription id",
args: []string{
invalidID,
token,
},
sdkErr: errors.NewSDKErrorWithStatus(svcerr.ErrAuthorization, http.StatusForbidden),
errLogMessage: fmt.Sprintf("\nerror: %s\n\n", errors.NewSDKErrorWithStatus(svcerr.ErrAuthorization, http.StatusForbidden)),
logType: errLog,
},
{
desc: "remove subscription with invalid token",
args: []string{
subscription.ID,
invalidToken,
},
sdkErr: errors.NewSDKErrorWithStatus(svcerr.ErrAuthorization, http.StatusForbidden),
errLogMessage: fmt.Sprintf("\nerror: %s\n\n", errors.NewSDKErrorWithStatus(svcerr.ErrAuthorization, http.StatusForbidden)),
logType: errLog,
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
sdkCall := sdkMock.On("DeleteSubscription", tc.args[0], tc.args[1]).Return(tc.sdkErr)
out := executeCommand(t, rootCmd, append([]string{rmCmd}, tc.args...)...)
switch tc.logType {
case okLog:
assert.True(t, strings.Contains(out, "ok"), fmt.Sprintf("%s unexpected response: expected success message, got: %v", tc.desc, out))
case errLog:
assert.Equal(t, tc.errLogMessage, out, fmt.Sprintf("%s unexpected error response: expected %s got errLogMessage:%s", tc.desc, tc.errLogMessage, out))
case usageLog:
assert.False(t, strings.Contains(out, rootCmd.Use), fmt.Sprintf("%s invalid usage: %s", tc.desc, out))
}
sdkCall.Unset()
})
}
}
cli/message.go
+3 -33
View File
@@ -3,10 +3,7 @@
package cli
import (
smqsdk "github.com/absmach/supermq/pkg/sdk"
"github.com/spf13/cobra"
)
import "github.com/spf13/cobra"
var cmdMessages = []cobra.Command{
{
@@ -27,41 +24,14 @@ var cmdMessages = []cobra.Command{
logOKCmd(*cmd)
},
},
{
Use: "read <channel_id.subtopic> <domain_id> <user_token>",
Short: "Read messages",
Long: "Reads all channel messages\n" +
"Usage:\n" +
"\tsupermq-cli messages read <channel_id.subtopic> <domain_id> <user_token> --offset <offset> --limit <limit> - lists all messages with provided offset and limit\n",
Run: func(cmd *cobra.Command, args []string) {
if len(args) != 3 {
logUsageCmd(*cmd, cmd.Use)
return
}
pageMetadata := smqsdk.MessagePageMetadata{
PageMetadata: smqsdk.PageMetadata{
Offset: Offset,
Limit: Limit,
},
}
m, err := sdk.ReadMessages(pageMetadata, args[0], args[1], args[2])
if err != nil {
logErrorCmd(*cmd, err)
return
}
logJSONCmd(*cmd, m)
},
},
}
// NewMessagesCmd returns messages command.
func NewMessagesCmd() *cobra.Command {
cmd := cobra.Command{
Use: "messages [send | read]",
Short: "Send or read messages",
Long: `Send or read messages using the http-adapter and the configured database reader`,
Short: "Send messages",
Long: `Send messages using the http-adapter`,
}
for i := range cmdMessages {
cli/message_test.go
-83
View File
@@ -4,7 +4,6 @@
package cli_test
import (
"encoding/json"
"fmt"
"net/http"
"strings"
@@ -13,11 +12,8 @@ import (
"github.com/absmach/supermq/cli"
"github.com/absmach/supermq/pkg/errors"
svcerr "github.com/absmach/supermq/pkg/errors/service"
mgsdk "github.com/absmach/supermq/pkg/sdk"
sdkmocks "github.com/absmach/supermq/pkg/sdk/mocks"
"github.com/absmach/supermq/pkg/transformers/senml"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
func TestSendMesageCmd(t *testing.T) {
@@ -84,82 +80,3 @@ func TestSendMesageCmd(t *testing.T) {
})
}
}
func TestReadMesageCmd(t *testing.T) {
sdkMock := new(sdkmocks.SDK)
cli.SetSDK(sdkMock)
messageCmd := cli.NewMessagesCmd()
rootCmd := setFlags(messageCmd)
var mp mgsdk.MessagesPage
cases := []struct {
desc string
args []string
logType outputLog
errLogMessage string
sdkErr errors.SDKError
page mgsdk.MessagesPage
}{
{
desc: "read message successfully",
args: []string{
channel.ID,
domainID,
validToken,
},
page: mgsdk.MessagesPage{
PageRes: mgsdk.PageRes{
Total: 1,
Offset: 0,
Limit: 10,
},
Messages: []senml.Message{
{
Channel: channel.ID,
},
},
},
logType: entityLog,
},
{
desc: "read message with invalid args",
args: []string{
channel.ID,
domainID,
validToken,
extraArg,
},
logType: usageLog,
},
{
desc: "read message with invalid token",
args: []string{
channel.ID,
domainID,
invalidToken,
},
sdkErr: errors.NewSDKErrorWithStatus(svcerr.ErrAuthorization, http.StatusUnauthorized),
errLogMessage: fmt.Sprintf("\nerror: %s\n\n", errors.NewSDKErrorWithStatus(svcerr.ErrAuthorization, http.StatusUnauthorized)),
logType: errLog,
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
sdkCall := sdkMock.On("ReadMessages", mock.Anything, tc.args[0], tc.args[1], tc.args[2]).Return(tc.page, tc.sdkErr)
out := executeCommand(t, rootCmd, append([]string{readCmd}, tc.args...)...)
switch tc.logType {
case entityLog:
err := json.Unmarshal([]byte(out), &mp)
assert.Nil(t, err)
assert.Equal(t, tc.page, mp, fmt.Sprintf("%s unexpected response: expected: %v, got: %v", tc.desc, tc.page, mp))
case errLog:
assert.Equal(t, tc.errLogMessage, out, fmt.Sprintf("%s unexpected error response: expected %s got errLogMessage:%s", tc.desc, tc.errLogMessage, out))
case usageLog:
assert.False(t, strings.Contains(out, rootCmd.Use), fmt.Sprintf("%s invalid usage: %s", tc.desc, out))
}
sdkCall.Unset()
})
}
}
cli/utils.go
-8
View File
@@ -77,14 +77,6 @@ func logOKCmd(cmd cobra.Command) {
fmt.Fprintf(cmd.OutOrStdout(), "\n%s\n\n", color.BlueString("ok"))
}
func logCreatedCmd(cmd cobra.Command, e string) {
if RawOutput {
fmt.Fprintln(cmd.OutOrStdout(), e)
} else {
fmt.Fprintf(cmd.OutOrStdout(), color.BlueString("\ncreated: %s\n\n"), e)
}
}
func logRevokedTimeCmd(cmd cobra.Command, t time.Time) {
if RawOutput {
fmt.Fprintln(cmd.OutOrStdout(), t)
cmd/cli/main.go
-10
View File
@@ -42,7 +42,6 @@ func main() {
channelsCmd := cli.NewChannelsCmd()
messagesCmd := cli.NewMessagesCmd()
certsCmd := cli.NewCertsCmd()
subscriptionsCmd := cli.NewSubscriptionCmd()
configCmd := cli.NewConfigCmd()
invitationsCmd := cli.NewInvitationsCmd()
journalCmd := cli.NewJournalCmd()
@@ -56,7 +55,6 @@ func main() {
rootCmd.AddCommand(channelsCmd)
rootCmd.AddCommand(messagesCmd)
rootCmd.AddCommand(certsCmd)
rootCmd.AddCommand(subscriptionsCmd)
rootCmd.AddCommand(configCmd)
rootCmd.AddCommand(invitationsCmd)
rootCmd.AddCommand(journalCmd)
@@ -102,14 +100,6 @@ func main() {
"HTTP adapter URL",
)
rootCmd.PersistentFlags().StringVarP(
&sdkConf.ReaderURL,
"reader-url",
"R",
sdkConf.ReaderURL,
"Reader URL",
)
rootCmd.PersistentFlags().StringVarP(
&sdkConf.InvitationsURL,
"invitations-url",
cmd/postgres-reader/main.go
-171
View File
@@ -1,171 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package main contains postgres-reader main function to start the postgres-reader service.
package main
import (
"context"
"fmt"
"log"
"log/slog"
"os"
chclient "github.com/absmach/callhome/pkg/client"
"github.com/absmach/supermq"
smqlog "github.com/absmach/supermq/logger"
"github.com/absmach/supermq/pkg/authn/authsvc"
"github.com/absmach/supermq/pkg/grpcclient"
pgclient "github.com/absmach/supermq/pkg/postgres"
"github.com/absmach/supermq/pkg/prometheus"
"github.com/absmach/supermq/pkg/server"
httpserver "github.com/absmach/supermq/pkg/server/http"
"github.com/absmach/supermq/pkg/uuid"
"github.com/absmach/supermq/readers"
httpapi "github.com/absmach/supermq/readers/api"
"github.com/absmach/supermq/readers/postgres"
"github.com/caarlos0/env/v11"
"github.com/jmoiron/sqlx"
"golang.org/x/sync/errgroup"
)
const (
svcName = "postgres-reader"
envPrefixDB = "SMQ_POSTGRES_"
envPrefixHTTP = "SMQ_POSTGRES_READER_HTTP_"
envPrefixAuth = "SMQ_AUTH_GRPC_"
envPrefixClients = "SMQ_CLIENTS_AUTH_GRPC_"
envPrefixChannels = "SMQ_CHANNELS_GRPC_"
defDB = "supermq"
defSvcHTTPPort = "9009"
)
type config struct {
LogLevel string `env:"SMQ_POSTGRES_READER_LOG_LEVEL" envDefault:"info"`
SendTelemetry bool `env:"SMQ_SEND_TELEMETRY" envDefault:"true"`
InstanceID string `env:"SMQ_POSTGRES_READER_INSTANCE_ID" envDefault:""`
}
func main() {
ctx, cancel := context.WithCancel(context.Background())
g, ctx := errgroup.WithContext(ctx)
cfg := config{}
if err := env.Parse(&cfg); err != nil {
log.Fatalf("failed to load %s configuration : %s", svcName, err)
}
logger, err := smqlog.New(os.Stdout, cfg.LogLevel)
if err != nil {
log.Fatalf("failed to init logger: %s", err.Error())
}
var exitCode int
defer smqlog.ExitWithError(&exitCode)
if cfg.InstanceID == "" {
if cfg.InstanceID, err = uuid.New().ID(); err != nil {
logger.Error(fmt.Sprintf("failed to generate instanceID: %s", err))
exitCode = 1
return
}
}
dbConfig := pgclient.Config{}
if err := env.ParseWithOptions(&dbConfig, env.Options{Prefix: envPrefixDB}); err != nil {
logger.Error(err.Error())
exitCode = 1
return
}
db, err := pgclient.Connect(dbConfig)
if err != nil {
logger.Error(fmt.Sprintf("failed to setup postgres database : %s", err))
exitCode = 1
return
}
defer db.Close()
clientsClientCfg := grpcclient.Config{}
if err := env.ParseWithOptions(&clientsClientCfg, env.Options{Prefix: envPrefixClients}); err != nil {
logger.Error(fmt.Sprintf("failed to load clients gRPC client configuration : %s", err))
exitCode = 1
return
}
clientsClient, clientsHandler, err := grpcclient.SetupClientsClient(ctx, clientsClientCfg)
if err != nil {
logger.Error(err.Error())
exitCode = 1
return
}
defer clientsHandler.Close()
logger.Info("Clients service gRPC client successfully connected to clients gRPC server " + clientsHandler.Secure())
channelsClientCfg := grpcclient.Config{}
if err := env.ParseWithOptions(&channelsClientCfg, env.Options{Prefix: envPrefixChannels}); err != nil {
logger.Error(fmt.Sprintf("failed to load channels gRPC client configuration : %s", err))
exitCode = 1
return
}
channelsClient, channelsHandler, err := grpcclient.SetupChannelsClient(ctx, channelsClientCfg)
if err != nil {
logger.Error(err.Error())
exitCode = 1
return
}
defer channelsHandler.Close()
logger.Info("Channels service gRPC client successfully connected to channels gRPC server " + channelsHandler.Secure())
authnCfg := grpcclient.Config{}
if err := env.ParseWithOptions(&authnCfg, env.Options{Prefix: envPrefixAuth}); err != nil {
logger.Error(fmt.Sprintf("failed to load auth gRPC client configuration : %s", err))
exitCode = 1
return
}
authn, authnHandler, err := authsvc.NewAuthentication(ctx, authnCfg)
if err != nil {
logger.Error(err.Error())
exitCode = 1
return
}
defer authnHandler.Close()
logger.Info("authn successfully connected to auth gRPC server " + authnHandler.Secure())
repo := newService(db, logger)
httpServerConfig := server.Config{Port: defSvcHTTPPort}
if err := env.ParseWithOptions(&httpServerConfig, env.Options{Prefix: envPrefixHTTP}); err != nil {
logger.Error(fmt.Sprintf("failed to load %s HTTP server configuration : %s", svcName, err))
exitCode = 1
return
}
hs := httpserver.NewServer(ctx, cancel, svcName, httpServerConfig, httpapi.MakeHandler(repo, authn, clientsClient, channelsClient, svcName, cfg.InstanceID), logger)
if cfg.SendTelemetry {
chc := chclient.New(svcName, supermq.Version, logger, cancel)
go chc.CallHome(ctx)
}
g.Go(func() error {
return hs.Start()
})
g.Go(func() error {
return server.StopSignalHandler(ctx, cancel, logger, svcName, hs)
})
if err := g.Wait(); err != nil {
logger.Error(fmt.Sprintf("Postgres reader service terminated: %s", err))
}
}
func newService(db *sqlx.DB, logger *slog.Logger) readers.MessageRepository {
svc := postgres.New(db)
svc = httpapi.LoggingMiddleware(svc, logger)
counter, latency := prometheus.MakeMetrics("postgres", "message_reader")
svc = httpapi.MetricsMiddleware(svc, counter, latency)
return svc
}
cmd/postgres-writer/main.go
-154
View File
@@ -1,154 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package main contains postgres-writer main function to start the postgres-writer service.
package main
import (
"context"
"fmt"
"log"
"log/slog"
"net/url"
"os"
chclient "github.com/absmach/callhome/pkg/client"
"github.com/absmach/supermq"
"github.com/absmach/supermq/consumers"
consumertracing "github.com/absmach/supermq/consumers/tracing"
httpapi "github.com/absmach/supermq/consumers/writers/api"
writerpg "github.com/absmach/supermq/consumers/writers/postgres"
smqlog "github.com/absmach/supermq/logger"
jaegerclient "github.com/absmach/supermq/pkg/jaeger"
"github.com/absmach/supermq/pkg/messaging/brokers"
brokerstracing "github.com/absmach/supermq/pkg/messaging/brokers/tracing"
pgclient "github.com/absmach/supermq/pkg/postgres"
"github.com/absmach/supermq/pkg/prometheus"
"github.com/absmach/supermq/pkg/server"
httpserver "github.com/absmach/supermq/pkg/server/http"
"github.com/absmach/supermq/pkg/uuid"
"github.com/caarlos0/env/v11"
"github.com/jmoiron/sqlx"
"golang.org/x/sync/errgroup"
)
const (
svcName = "postgres-writer"
envPrefixDB = "SMQ_POSTGRES_"
envPrefixHTTP = "SMQ_POSTGRES_WRITER_HTTP_"
defDB = "messages"
defSvcHTTPPort = "9010"
)
type config struct {
LogLevel string `env:"SMQ_POSTGRES_WRITER_LOG_LEVEL" envDefault:"info"`
ConfigPath string `env:"SMQ_POSTGRES_WRITER_CONFIG_PATH" envDefault:"/config.toml"`
BrokerURL string `env:"SMQ_MESSAGE_BROKER_URL" envDefault:"nats://localhost:4222"`
JaegerURL url.URL `env:"SMQ_JAEGER_URL" envDefault:"http://localhost:4318/v1/traces"`
SendTelemetry bool `env:"SMQ_SEND_TELEMETRY" envDefault:"true"`
InstanceID string `env:"SMQ_POSTGRES_WRITER_INSTANCE_ID" envDefault:""`
TraceRatio float64 `env:"SMQ_JAEGER_TRACE_RATIO" envDefault:"1.0"`
}
func main() {
ctx, cancel := context.WithCancel(context.Background())
g, ctx := errgroup.WithContext(ctx)
cfg := config{}
if err := env.Parse(&cfg); err != nil {
log.Fatalf("failed to load %s configuration : %s", svcName, err)
}
logger, err := smqlog.New(os.Stdout, cfg.LogLevel)
if err != nil {
log.Fatalf("failed to init logger: %s", err.Error())
}
var exitCode int
defer smqlog.ExitWithError(&exitCode)
if cfg.InstanceID == "" {
if cfg.InstanceID, err = uuid.New().ID(); err != nil {
logger.Error(fmt.Sprintf("failed to generate instanceID: %s", err))
exitCode = 1
return
}
}
httpServerConfig := server.Config{Port: defSvcHTTPPort}
if err := env.ParseWithOptions(&httpServerConfig, env.Options{Prefix: envPrefixHTTP}); err != nil {
logger.Error(fmt.Sprintf("failed to load %s HTTP server configuration : %s", svcName, err))
exitCode = 1
return
}
dbConfig := pgclient.Config{Name: defDB}
if err := env.ParseWithOptions(&dbConfig, env.Options{Prefix: envPrefixDB}); err != nil {
logger.Error(fmt.Sprintf("failed to load %s Postgres configuration : %s", svcName, err))
exitCode = 1
return
}
db, err := pgclient.Setup(dbConfig, *writerpg.Migration())
if err != nil {
logger.Error(err.Error())
}
defer db.Close()
tp, err := jaegerclient.NewProvider(ctx, svcName, cfg.JaegerURL, cfg.InstanceID, cfg.TraceRatio)
if err != nil {
logger.Error(fmt.Sprintf("Failed to init Jaeger: %s", err))
exitCode = 1
return
}
defer func() {
if err := tp.Shutdown(ctx); err != nil {
logger.Error(fmt.Sprintf("Error shutting down tracer provider: %v", err))
}
}()
tracer := tp.Tracer(svcName)
pubSub, err := brokers.NewPubSub(ctx, cfg.BrokerURL, logger)
if err != nil {
logger.Error(fmt.Sprintf("failed to connect to message broker: %s", err))
exitCode = 1
return
}
defer pubSub.Close()
pubSub = brokerstracing.NewPubSub(httpServerConfig, tracer, pubSub)
repo := newService(db, logger)
repo = consumertracing.NewBlocking(tracer, repo, httpServerConfig)
if err = consumers.Start(ctx, svcName, pubSub, repo, cfg.ConfigPath, logger); err != nil {
logger.Error(fmt.Sprintf("failed to create Postgres writer: %s", err))
exitCode = 1
return
}
hs := httpserver.NewServer(ctx, cancel, svcName, httpServerConfig, httpapi.MakeHandler(svcName, cfg.InstanceID), logger)
if cfg.SendTelemetry {
chc := chclient.New(svcName, supermq.Version, logger, cancel)
go chc.CallHome(ctx)
}
g.Go(func() error {
return hs.Start()
})
g.Go(func() error {
return server.StopSignalHandler(ctx, cancel, logger, svcName, hs)
})
if err := g.Wait(); err != nil {
logger.Error(fmt.Sprintf("Postgres writer service terminated: %s", err))
}
}
func newService(db *sqlx.DB, logger *slog.Logger) consumers.BlockingConsumer {
svc := writerpg.New(db)
svc = httpapi.LoggingMiddleware(svc, logger)
counter, latency := prometheus.MakeMetrics("postgres", "message_writer")
svc = httpapi.MetricsMiddleware(svc, counter, latency)
return svc
}
cmd/timescale-reader/main.go
-170
View File
@@ -1,170 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package main contains timescale-reader main function to start the timescale-reader service.
package main
import (
"context"
"fmt"
"log"
"log/slog"
"os"
chclient "github.com/absmach/callhome/pkg/client"
"github.com/absmach/supermq"
smqlog "github.com/absmach/supermq/logger"
"github.com/absmach/supermq/pkg/authn/authsvc"
"github.com/absmach/supermq/pkg/grpcclient"
pgclient "github.com/absmach/supermq/pkg/postgres"
"github.com/absmach/supermq/pkg/prometheus"
"github.com/absmach/supermq/pkg/server"
httpserver "github.com/absmach/supermq/pkg/server/http"
"github.com/absmach/supermq/pkg/uuid"
"github.com/absmach/supermq/readers"
httpapi "github.com/absmach/supermq/readers/api"
"github.com/absmach/supermq/readers/timescale"
"github.com/caarlos0/env/v11"
"github.com/jmoiron/sqlx"
"golang.org/x/sync/errgroup"
)
const (
svcName = "timescaledb-reader"
envPrefixDB = "SMQ_TIMESCALE_"
envPrefixHTTP = "SMQ_TIMESCALE_READER_HTTP_"
envPrefixAuth = "SMQ_AUTH_GRPC_"
envPrefixClients = "SMQ_CLIENTS_AUTH_GRPC_"
envPrefixChannels = "SMQ_CHANNELS_GRPC_"
defDB = "messages"
defSvcHTTPPort = "9011"
)
type config struct {
LogLevel string `env:"SMQ_TIMESCALE_READER_LOG_LEVEL" envDefault:"info"`
SendTelemetry bool `env:"SMQ_SEND_TELEMETRY" envDefault:"true"`
InstanceID string `env:"SMQ_TIMESCALE_READER_INSTANCE_ID" envDefault:""`
}
func main() {
ctx, cancel := context.WithCancel(context.Background())
g, ctx := errgroup.WithContext(ctx)
cfg := config{}
if err := env.Parse(&cfg); err != nil {
log.Fatalf("failed to load %s configuration : %s", svcName, err)
}
logger, err := smqlog.New(os.Stdout, cfg.LogLevel)
if err != nil {
log.Fatalf("failed to init logger: %s", err.Error())
}
var exitCode int
defer smqlog.ExitWithError(&exitCode)
if cfg.InstanceID == "" {
if cfg.InstanceID, err = uuid.New().ID(); err != nil {
logger.Error(fmt.Sprintf("failed to generate instanceID: %s", err))
exitCode = 1
return
}
}
dbConfig := pgclient.Config{Name: defDB}
if err := env.ParseWithOptions(&dbConfig, env.Options{Prefix: envPrefixDB}); err != nil {
logger.Error(err.Error())
exitCode = 1
return
}
db, err := pgclient.Connect(dbConfig)
if err != nil {
logger.Error(err.Error())
}
defer db.Close()
repo := newService(db, logger)
clientsClientCfg := grpcclient.Config{}
if err := env.ParseWithOptions(&clientsClientCfg, env.Options{Prefix: envPrefixClients}); err != nil {
logger.Error(fmt.Sprintf("failed to load %s auth configuration : %s", svcName, err))
exitCode = 1
return
}
clientsClient, clientsHandler, err := grpcclient.SetupClientsClient(ctx, clientsClientCfg)
if err != nil {
logger.Error(err.Error())
exitCode = 1
return
}
defer clientsHandler.Close()
logger.Info("ClientsService gRPC client successfully connected to clients gRPC server " + clientsHandler.Secure())
channelsClientCfg := grpcclient.Config{}
if err := env.ParseWithOptions(&channelsClientCfg, env.Options{Prefix: envPrefixChannels}); err != nil {
logger.Error(fmt.Sprintf("failed to load channels gRPC client configuration : %s", err))
exitCode = 1
return
}
channelsClient, channelsHandler, err := grpcclient.SetupChannelsClient(ctx, channelsClientCfg)
if err != nil {
logger.Error(err.Error())
exitCode = 1
return
}
defer channelsHandler.Close()
logger.Info("Channels service gRPC client successfully connected to channels gRPC server " + channelsHandler.Secure())
authnCfg := grpcclient.Config{}
if err := env.ParseWithOptions(&authnCfg, env.Options{Prefix: envPrefixAuth}); err != nil {
logger.Error(fmt.Sprintf("failed to load auth gRPC client configuration : %s", err))
exitCode = 1
return
}
authn, authnHandler, err := authsvc.NewAuthentication(ctx, authnCfg)
if err != nil {
logger.Error(err.Error())
exitCode = 1
return
}
defer authnHandler.Close()
logger.Info("authn successfully connected to auth gRPC server " + authnHandler.Secure())
httpServerConfig := server.Config{Port: defSvcHTTPPort}
if err := env.ParseWithOptions(&httpServerConfig, env.Options{Prefix: envPrefixHTTP}); err != nil {
logger.Error(fmt.Sprintf("failed to load %s HTTP server configuration : %s", svcName, err))
exitCode = 1
return
}
hs := httpserver.NewServer(ctx, cancel, svcName, httpServerConfig, httpapi.MakeHandler(repo, authn, clientsClient, channelsClient, svcName, cfg.InstanceID), logger)
if cfg.SendTelemetry {
chc := chclient.New(svcName, supermq.Version, logger, cancel)
go chc.CallHome(ctx)
}
g.Go(func() error {
return hs.Start()
})
g.Go(func() error {
return server.StopSignalHandler(ctx, cancel, logger, svcName, hs)
})
if err := g.Wait(); err != nil {
logger.Error(fmt.Sprintf("Timescale reader service terminated: %s", err))
}
}
func newService(db *sqlx.DB, logger *slog.Logger) readers.MessageRepository {
svc := timescale.New(db)
svc = httpapi.LoggingMiddleware(svc, logger)
counter, latency := prometheus.MakeMetrics("timescale", "message_reader")
svc = httpapi.MetricsMiddleware(svc, counter, latency)
return svc
}
cmd/timescale-writer/main.go
-156
View File
@@ -1,156 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package main contains timescale-writer main function to start the timescale-writer service.
package main
import (
"context"
"fmt"
"log"
"log/slog"
"net/url"
"os"
chclient "github.com/absmach/callhome/pkg/client"
"github.com/absmach/supermq"
"github.com/absmach/supermq/consumers"
consumertracing "github.com/absmach/supermq/consumers/tracing"
httpapi "github.com/absmach/supermq/consumers/writers/api"
"github.com/absmach/supermq/consumers/writers/timescale"
smqlog "github.com/absmach/supermq/logger"
jaegerclient "github.com/absmach/supermq/pkg/jaeger"
"github.com/absmach/supermq/pkg/messaging/brokers"
brokerstracing "github.com/absmach/supermq/pkg/messaging/brokers/tracing"
pgclient "github.com/absmach/supermq/pkg/postgres"
"github.com/absmach/supermq/pkg/prometheus"
"github.com/absmach/supermq/pkg/server"
httpserver "github.com/absmach/supermq/pkg/server/http"
"github.com/absmach/supermq/pkg/uuid"
"github.com/caarlos0/env/v11"
"github.com/jmoiron/sqlx"
"golang.org/x/sync/errgroup"
)
const (
svcName = "timescaledb-writer"
envPrefixDB = "SMQ_TIMESCALE_"
envPrefixHTTP = "SMQ_TIMESCALE_WRITER_HTTP_"
defDB = "messages"
defSvcHTTPPort = "9012"
)
type config struct {
LogLevel string `env:"SMQ_TIMESCALE_WRITER_LOG_LEVEL" envDefault:"info"`
ConfigPath string `env:"SMQ_TIMESCALE_WRITER_CONFIG_PATH" envDefault:"/config.toml"`
BrokerURL string `env:"SMQ_MESSAGE_BROKER_URL" envDefault:"nats://localhost:4222"`
JaegerURL url.URL `env:"SMQ_JAEGER_URL" envDefault:"http://localhost:4318/v1/traces"`
SendTelemetry bool `env:"SMQ_SEND_TELEMETRY" envDefault:"true"`
InstanceID string `env:"SMQ_TIMESCALE_WRITER_INSTANCE_ID" envDefault:""`
TraceRatio float64 `env:"SMQ_JAEGER_TRACE_RATIO" envDefault:"1.0"`
}
func main() {
ctx, cancel := context.WithCancel(context.Background())
g, ctx := errgroup.WithContext(ctx)
cfg := config{}
if err := env.Parse(&cfg); err != nil {
log.Fatalf("failed to load %s service configuration : %s", svcName, err)
}
logger, err := smqlog.New(os.Stdout, cfg.LogLevel)
if err != nil {
log.Fatalf("failed to init logger: %s", err.Error())
}
var exitCode int
defer smqlog.ExitWithError(&exitCode)
if cfg.InstanceID == "" {
if cfg.InstanceID, err = uuid.New().ID(); err != nil {
logger.Error(fmt.Sprintf("failed to generate instanceID: %s", err))
exitCode = 1
return
}
}
httpServerConfig := server.Config{Port: defSvcHTTPPort}
if err := env.ParseWithOptions(&httpServerConfig, env.Options{Prefix: envPrefixHTTP}); err != nil {
logger.Error(fmt.Sprintf("failed to load %s HTTP server configuration : %s", svcName, err))
exitCode = 1
return
}
dbConfig := pgclient.Config{Name: defDB}
if err := env.ParseWithOptions(&dbConfig, env.Options{Prefix: envPrefixDB}); err != nil {
logger.Error(fmt.Sprintf("failed to load %s Postgres configuration : %s", svcName, err))
exitCode = 1
return
}
db, err := pgclient.Setup(dbConfig, *timescale.Migration())
if err != nil {
logger.Error(err.Error())
exitCode = 1
return
}
defer db.Close()
tp, err := jaegerclient.NewProvider(ctx, svcName, cfg.JaegerURL, cfg.InstanceID, cfg.TraceRatio)
if err != nil {
logger.Error(fmt.Sprintf("Failed to init Jaeger: %s", err))
exitCode = 1
return
}
defer func() {
if err := tp.Shutdown(ctx); err != nil {
logger.Error(fmt.Sprintf("Error shutting down tracer provider: %v", err))
}
}()
tracer := tp.Tracer(svcName)
repo := newService(db, logger)
repo = consumertracing.NewBlocking(tracer, repo, httpServerConfig)
pubSub, err := brokers.NewPubSub(ctx, cfg.BrokerURL, logger)
if err != nil {
logger.Error(fmt.Sprintf("failed to connect to message broker: %s", err))
exitCode = 1
return
}
defer pubSub.Close()
pubSub = brokerstracing.NewPubSub(httpServerConfig, tracer, pubSub)
if err = consumers.Start(ctx, svcName, pubSub, repo, cfg.ConfigPath, logger); err != nil {
logger.Error(fmt.Sprintf("failed to create Timescale writer: %s", err))
exitCode = 1
return
}
hs := httpserver.NewServer(ctx, cancel, svcName, httpServerConfig, httpapi.MakeHandler(svcName, cfg.InstanceID), logger)
if cfg.SendTelemetry {
chc := chclient.New(svcName, supermq.Version, logger, cancel)
go chc.CallHome(ctx)
}
g.Go(func() error {
return hs.Start()
})
g.Go(func() error {
return server.StopSignalHandler(ctx, cancel, logger, svcName, hs)
})
if err := g.Wait(); err != nil {
logger.Error(fmt.Sprintf("Timescale writer service terminated: %s", err))
}
}
func newService(db *sqlx.DB, logger *slog.Logger) consumers.BlockingConsumer {
svc := timescale.New(db)
svc = httpapi.LoggingMiddleware(svc, logger)
counter, latency := prometheus.MakeMetrics("timescale", "message_writer")
svc = httpapi.MetricsMiddleware(svc, counter, latency)
return svc
}
config.toml
-1
View File
@@ -16,7 +16,6 @@ user_token = ""
host_url = "http://localhost"
http_adapter_url = "http://localhost:8008"
invitations_url = "http://localhost:9020"
reader_url = "http://localhost:9011"
clients_url = "http://localhost:9000"
tls_verification = false
users_url = "http://localhost:9002"
consumers/notifiers/README.md
-23
View File
@@ -1,23 +0,0 @@
# Notifiers service
Notifiers service provides a service for sending notifications using Notifiers.
Notifiers service can be configured to use different types of Notifiers to send
different types of notifications such as SMS messages, emails, or push notifications.
Service is extensible so that new implementations of Notifiers can be easily added.
Notifiers **are not standalone services** but rather dependencies used by Notifiers service
for sending notifications over specific protocols.
## Configuration
The service is configured using the environment variables.
The environment variables needed for service configuration depend on the underlying Notifier.
An example of the service configuration for SMTP Notifier can be found [in SMTP Notifier documentation](smtp/README.md).
Note that any unset variables will be replaced with their
default values.
## Usage
Subscriptions service will start consuming messages and sending notifications when a message is received.
[doc]: https://docs.supermq.abstractmachines.fr
consumers/notifiers/api/doc.go
-6
View File
@@ -1,6 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package api contains API-related concerns: endpoint definitions, middlewares
// and all resource representations.
package api
consumers/notifiers/api/endpoint.go
-103
View File
@@ -1,103 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api
import (
"context"
apiutil "github.com/absmach/supermq/api/http/util"
notifiers "github.com/absmach/supermq/consumers/notifiers"
"github.com/absmach/supermq/pkg/errors"
"github.com/go-kit/kit/endpoint"
)
func createSubscriptionEndpoint(svc notifiers.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
req := request.(createSubReq)
if err := req.validate(); err != nil {
return createSubRes{}, errors.Wrap(apiutil.ErrValidation, err)
}
sub := notifiers.Subscription{
Contact: req.Contact,
Topic: req.Topic,
}
id, err := svc.CreateSubscription(ctx, req.token, sub)
if err != nil {
return createSubRes{}, err
}
ucr := createSubRes{
ID: id,
}
return ucr, nil
}
}
func viewSubscriptionEndpint(svc notifiers.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
req := request.(subReq)
if err := req.validate(); err != nil {
return viewSubRes{}, errors.Wrap(apiutil.ErrValidation, err)
}
sub, err := svc.ViewSubscription(ctx, req.token, req.id)
if err != nil {
return viewSubRes{}, err
}
res := viewSubRes{
ID: sub.ID,
OwnerID: sub.OwnerID,
Contact: sub.Contact,
Topic: sub.Topic,
}
return res, nil
}
}
func listSubscriptionsEndpoint(svc notifiers.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
req := request.(listSubsReq)
if err := req.validate(); err != nil {
return listSubsRes{}, errors.Wrap(apiutil.ErrValidation, err)
}
pm := notifiers.PageMetadata{
Topic: req.topic,
Contact: req.contact,
Offset: req.offset,
Limit: int(req.limit),
}
page, err := svc.ListSubscriptions(ctx, req.token, pm)
if err != nil {
return listSubsRes{}, err
}
res := listSubsRes{
Offset: page.Offset,
Limit: page.Limit,
Total: page.Total,
}
for _, sub := range page.Subscriptions {
r := viewSubRes{
ID: sub.ID,
OwnerID: sub.OwnerID,
Contact: sub.Contact,
Topic: sub.Topic,
}
res.Subscriptions = append(res.Subscriptions, r)
}
return res, nil
}
}
func deleteSubscriptionEndpint(svc notifiers.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
req := request.(subReq)
if err := req.validate(); err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
if err := svc.RemoveSubscription(ctx, req.token, req.id); err != nil {
return nil, err
}
return removeSubRes{}, nil
}
}
consumers/notifiers/api/endpoint_test.go
-548
View File
@@ -1,548 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api_test
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"path"
"strings"
"testing"
apiutil "github.com/absmach/supermq/api/http/util"
"github.com/absmach/supermq/consumers/notifiers"
"github.com/absmach/supermq/consumers/notifiers/api"
"github.com/absmach/supermq/consumers/notifiers/mocks"
"github.com/absmach/supermq/internal/testsutil"
smqlog "github.com/absmach/supermq/logger"
svcerr "github.com/absmach/supermq/pkg/errors/service"
"github.com/absmach/supermq/pkg/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
const (
contentType = "application/json"
email = "user@example.com"
contact1 = "email1@example.com"
contact2 = "email2@example.com"
token = "token"
invalidToken = "invalid"
topic = "topic"
instanceID = "5de9b29a-feb9-11ed-be56-0242ac120002"
validID = "d4ebb847-5d0e-4e46-bdd9-b6aceaaa3a22"
)
var (
notFoundRes = toJSON(apiutil.ErrorRes{Msg: svcerr.ErrNotFound.Error()})
unauthRes = toJSON(apiutil.ErrorRes{Msg: svcerr.ErrAuthentication.Error()})
invalidRes = toJSON(apiutil.ErrorRes{Err: apiutil.ErrInvalidQueryParams.Error(), Msg: apiutil.ErrValidation.Error()})
missingTokRes = toJSON(apiutil.ErrorRes{Err: apiutil.ErrBearerToken.Error(), Msg: apiutil.ErrValidation.Error()})
)
type testRequest struct {
client *http.Client
method string
url string
contentType string
token string
body io.Reader
}
func (tr testRequest) make() (*http.Response, error) {
req, err := http.NewRequest(tr.method, tr.url, tr.body)
if err != nil {
return nil, err
}
if tr.token != "" {
req.Header.Set("Authorization", apiutil.BearerPrefix+tr.token)
}
if tr.contentType != "" {
req.Header.Set("Content-Type", tr.contentType)
}
return tr.client.Do(req)
}
func newServer() (*httptest.Server, *mocks.Service) {
logger := smqlog.NewMock()
svc := new(mocks.Service)
mux := api.MakeHandler(svc, logger, instanceID)
return httptest.NewServer(mux), svc
}
func toJSON(data interface{}) string {
jsonData, err := json.Marshal(data)
if err != nil {
return ""
}
return string(jsonData)
}
func TestCreate(t *testing.T) {
ss, svc := newServer()
defer ss.Close()
sub := notifiers.Subscription{
Topic: topic,
Contact: contact1,
}
data := toJSON(sub)
emptyTopic := toJSON(notifiers.Subscription{Contact: contact1})
emptyContact := toJSON(notifiers.Subscription{Topic: "topic123"})
cases := []struct {
desc string
req string
contentType string
auth string
status int
location string
err error
}{
{
desc: "add successfully",
req: data,
contentType: contentType,
auth: token,
status: http.StatusCreated,
location: fmt.Sprintf("/subscriptions/%s%012d", uuid.Prefix, 1),
err: nil,
},
{
desc: "add an existing subscription",
req: data,
contentType: contentType,
auth: token,
status: http.StatusConflict,
location: "",
err: svcerr.ErrConflict,
},
{
desc: "add with empty topic",
req: emptyTopic,
contentType: contentType,
auth: token,
status: http.StatusBadRequest,
location: "",
err: svcerr.ErrMalformedEntity,
},
{
desc: "add with empty contact",
req: emptyContact,
contentType: contentType,
auth: token,
status: http.StatusBadRequest,
location: "",
err: svcerr.ErrMalformedEntity,
},
{
desc: "add with invalid auth token",
req: data,
contentType: contentType,
auth: invalidToken,
status: http.StatusUnauthorized,
location: "",
err: svcerr.ErrAuthentication,
},
{
desc: "add with empty auth token",
req: data,
contentType: contentType,
auth: "",
status: http.StatusUnauthorized,
location: "",
err: svcerr.ErrAuthentication,
},
{
desc: "add with invalid request format",
req: "}",
contentType: contentType,
auth: token,
status: http.StatusBadRequest,
location: "",
err: svcerr.ErrMalformedEntity,
},
{
desc: "add without content type",
req: data,
contentType: "",
auth: token,
status: http.StatusUnsupportedMediaType,
location: "",
err: apiutil.ErrUnsupportedContentType,
},
}
for _, tc := range cases {
svcCall := svc.On("CreateSubscription", mock.Anything, tc.auth, sub).Return(path.Base(tc.location), tc.err)
req := testRequest{
client: ss.Client(),
method: http.MethodPost,
url: fmt.Sprintf("%s/subscriptions", ss.URL),
contentType: tc.contentType,
token: tc.auth,
body: strings.NewReader(tc.req),
}
res, err := req.make()
assert.Nil(t, err, fmt.Sprintf("%s: unexpected error %s", tc.desc, err))
location := res.Header.Get("Location")
assert.Equal(t, tc.status, res.StatusCode, fmt.Sprintf("%s: expected status code %d got %d", tc.desc, tc.status, res.StatusCode))
assert.Equal(t, tc.location, location, fmt.Sprintf("%s: expected location %s got %s", tc.desc, tc.location, location))
svcCall.Unset()
}
}
func TestView(t *testing.T) {
ss, svc := newServer()
defer ss.Close()
sub := notifiers.Subscription{
Topic: topic,
Contact: contact1,
ID: testsutil.GenerateUUID(t),
OwnerID: validID,
}
sr := subRes{
ID: sub.ID,
OwnerID: validID,
Contact: sub.Contact,
Topic: sub.Topic,
}
data := toJSON(sr)
cases := []struct {
desc string
id string
auth string
status int
res string
err error
Sub notifiers.Subscription
}{
{
desc: "view successfully",
id: sub.ID,
auth: token,
status: http.StatusOK,
res: data,
err: nil,
Sub: sub,
},
{
desc: "view not existing",
id: "not existing",
auth: token,
status: http.StatusNotFound,
res: notFoundRes,
err: svcerr.ErrNotFound,
},
{
desc: "view with invalid auth token",
id: sub.ID,
auth: invalidToken,
status: http.StatusUnauthorized,
res: unauthRes,
err: svcerr.ErrAuthentication,
},
{
desc: "view with empty auth token",
id: sub.ID,
auth: "",
status: http.StatusUnauthorized,
res: missingTokRes,
err: svcerr.ErrAuthentication,
},
}
for _, tc := range cases {
svcCall := svc.On("ViewSubscription", mock.Anything, tc.auth, tc.id).Return(tc.Sub, tc.err)
req := testRequest{
client: ss.Client(),
method: http.MethodGet,
url: fmt.Sprintf("%s/subscriptions/%s", ss.URL, tc.id),
token: tc.auth,
}
res, err := req.make()
assert.Nil(t, err, fmt.Sprintf("%s: unexpected request error %s", tc.desc, err))
body, err := io.ReadAll(res.Body)
assert.Nil(t, err, fmt.Sprintf("%s: unexpected read error %s", tc.desc, err))
data := strings.Trim(string(body), "\n")
assert.Equal(t, tc.status, res.StatusCode, fmt.Sprintf("%s: expected status code %d got %d", tc.desc, tc.status, res.StatusCode))
assert.Equal(t, tc.res, data, fmt.Sprintf("%s: expected body %s got %s", tc.desc, tc.res, data))
svcCall.Unset()
}
}
func TestList(t *testing.T) {
ss, svc := newServer()
defer ss.Close()
const numSubs = 100
var subs []subRes
var sub notifiers.Subscription
for i := 0; i < numSubs; i++ {
sub = notifiers.Subscription{
Topic: fmt.Sprintf("topic.subtopic.%d", i),
Contact: contact1,
ID: testsutil.GenerateUUID(t),
}
if i%2 == 0 {
sub.Contact = contact2
}
sr := subRes{
ID: sub.ID,
OwnerID: validID,
Contact: sub.Contact,
Topic: sub.Topic,
}
subs = append(subs, sr)
}
noLimit := toJSON(page{Offset: 5, Limit: 20, Total: numSubs, Subscriptions: subs[5:25]})
one := toJSON(page{Offset: 0, Limit: 20, Total: 1, Subscriptions: subs[10:11]})
var contact2Subs []subRes
for i := 20; i < 40; i += 2 {
contact2Subs = append(contact2Subs, subs[i])
}
contactList := toJSON(page{Offset: 10, Limit: 10, Total: 50, Subscriptions: contact2Subs})
cases := []struct {
desc string
query map[string]string
auth string
status int
res string
err error
page notifiers.Page
}{
{
desc: "list default limit",
query: map[string]string{
"offset": "5",
},
auth: token,
status: http.StatusOK,
res: noLimit,
err: nil,
page: notifiers.Page{
PageMetadata: notifiers.PageMetadata{
Offset: 5,
Limit: 20,
},
Total: numSubs,
Subscriptions: subscriptionsSlice(subs, 5, 25),
},
},
{
desc: "list not existing",
query: map[string]string{
"topic": "not-found-topic",
},
auth: token,
status: http.StatusNotFound,
res: notFoundRes,
err: svcerr.ErrNotFound,
},
{
desc: "list one with topic",
query: map[string]string{
"topic": "topic.subtopic.10",
},
auth: token,
status: http.StatusOK,
res: one,
err: nil,
page: notifiers.Page{
PageMetadata: notifiers.PageMetadata{
Offset: 0,
Limit: 20,
},
Total: 1,
Subscriptions: subscriptionsSlice(subs, 10, 11),
},
},
{
desc: "list with contact",
query: map[string]string{
"contact": contact2,
"offset": "10",
"limit": "10",
},
auth: token,
status: http.StatusOK,
res: contactList,
err: nil,
page: notifiers.Page{
PageMetadata: notifiers.PageMetadata{
Offset: 10,
Limit: 10,
},
Total: 50,
Subscriptions: subscriptionsSlice(contact2Subs, 0, 10),
},
},
{
desc: "list with invalid query",
query: map[string]string{
"offset": "two",
},
auth: token,
status: http.StatusBadRequest,
res: invalidRes,
err: svcerr.ErrMalformedEntity,
},
{
desc: "list with invalid auth token",
auth: invalidToken,
status: http.StatusUnauthorized,
res: unauthRes,
err: svcerr.ErrAuthentication,
},
{
desc: "list with empty auth token",
auth: "",
status: http.StatusUnauthorized,
res: missingTokRes,
err: svcerr.ErrAuthentication,
},
}
for _, tc := range cases {
svcCall := svc.On("ListSubscriptions", mock.Anything, tc.auth, mock.Anything).Return(tc.page, tc.err)
req := testRequest{
client: ss.Client(),
method: http.MethodGet,
url: fmt.Sprintf("%s/subscriptions%s", ss.URL, makeQuery(tc.query)),
token: tc.auth,
}
res, err := req.make()
assert.Nil(t, err, fmt.Sprintf("%s: unexpected error %s", tc.desc, err))
body, err := io.ReadAll(res.Body)
assert.Nil(t, err, fmt.Sprintf("%s: unexpected error %s", tc.desc, err))
data := strings.Trim(string(body), "\n")
assert.Equal(t, tc.status, res.StatusCode, fmt.Sprintf("%s: expected status code %d got %d", tc.desc, tc.status, res.StatusCode))
assert.Equal(t, tc.res, data, fmt.Sprintf("%s: got unexpected body\n", tc.desc))
svcCall.Unset()
}
}
func TestRemove(t *testing.T) {
ss, svc := newServer()
defer ss.Close()
id := testsutil.GenerateUUID(t)
cases := []struct {
desc string
id string
auth string
status int
res string
err error
}{
{
desc: "remove successfully",
id: id,
auth: token,
status: http.StatusNoContent,
err: nil,
},
{
desc: "remove not existing",
id: "not existing",
auth: token,
status: http.StatusNotFound,
err: svcerr.ErrNotFound,
},
{
desc: "remove empty id",
id: "",
auth: token,
status: http.StatusBadRequest,
err: svcerr.ErrMalformedEntity,
},
{
desc: "view with invalid auth token",
id: id,
auth: invalidToken,
status: http.StatusUnauthorized,
res: unauthRes,
err: svcerr.ErrAuthentication,
},
{
desc: "view with empty auth token",
id: id,
auth: "",
status: http.StatusUnauthorized,
res: missingTokRes,
err: svcerr.ErrAuthentication,
},
}
for _, tc := range cases {
svcCall := svc.On("RemoveSubscription", mock.Anything, tc.auth, tc.id).Return(tc.err)
req := testRequest{
client: ss.Client(),
method: http.MethodDelete,
url: fmt.Sprintf("%s/subscriptions/%s", ss.URL, tc.id),
token: tc.auth,
}
res, err := req.make()
assert.Nil(t, err, fmt.Sprintf("%s: unexpected error %s", tc.desc, err))
assert.Equal(t, tc.status, res.StatusCode, fmt.Sprintf("%s: expected status code %d got %d", tc.desc, tc.status, res.StatusCode))
svcCall.Unset()
}
}
func makeQuery(m map[string]string) string {
var ret string
for k, v := range m {
ret += fmt.Sprintf("&%s=%s", k, v)
}
if ret != "" {
return fmt.Sprintf("?%s", ret[1:])
}
return ""
}
type subRes struct {
ID string `json:"id"`
OwnerID string `json:"owner_id"`
Contact string `json:"contact"`
Topic string `json:"topic"`
}
type page struct {
Offset uint `json:"offset"`
Limit int `json:"limit"`
Total uint `json:"total,omitempty"`
Subscriptions []subRes `json:"subscriptions,omitempty"`
}
func subscriptionsSlice(subs []subRes, start, end int) []notifiers.Subscription {
var res []notifiers.Subscription
for i := start; i < end; i++ {
sub := subs[i]
res = append(res, notifiers.Subscription{
ID: sub.ID,
OwnerID: sub.OwnerID,
Contact: sub.Contact,
Topic: sub.Topic,
})
}
return res
}
consumers/notifiers/api/logging.go
-131
View File
@@ -1,131 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
//go:build !test
package api
import (
"context"
"log/slog"
"time"
"github.com/absmach/supermq/consumers/notifiers"
)
var _ notifiers.Service = (*loggingMiddleware)(nil)
type loggingMiddleware struct {
logger *slog.Logger
svc notifiers.Service
}
// LoggingMiddleware adds logging facilities to the core service.
func LoggingMiddleware(svc notifiers.Service, logger *slog.Logger) notifiers.Service {
return &loggingMiddleware{logger, svc}
}
// CreateSubscription logs the create_subscription request. It logs subscription ID and topic and the time it took to complete the request.
// If the request fails, it logs the error.
func (lm *loggingMiddleware) CreateSubscription(ctx context.Context, token string, sub notifiers.Subscription) (id string, err error) {
defer func(begin time.Time) {
args := []any{
slog.String("duration", time.Since(begin).String()),
slog.Group("subscription",
slog.String("topic", sub.Topic),
slog.String("id", id),
),
}
if err != nil {
args = append(args, slog.Any("error", err))
lm.logger.Warn("Create subscription failed", args...)
return
}
lm.logger.Info("Create subscription completed successfully", args...)
}(time.Now())
return lm.svc.CreateSubscription(ctx, token, sub)
}
// ViewSubscription logs the view_subscription request. It logs subscription topic and id and the time it took to complete the request.
// If the request fails, it logs the error.
func (lm *loggingMiddleware) ViewSubscription(ctx context.Context, token, topic string) (sub notifiers.Subscription, err error) {
defer func(begin time.Time) {
args := []any{
slog.String("duration", time.Since(begin).String()),
slog.Group("subscription",
slog.String("topic", topic),
slog.String("id", sub.ID),
),
}
if err != nil {
args = append(args, slog.Any("error", err))
lm.logger.Warn("View subscription failed", args...)
return
}
lm.logger.Info("View subscription completed successfully", args...)
}(time.Now())
return lm.svc.ViewSubscription(ctx, token, topic)
}
// ListSubscriptions logs the list_subscriptions request. It logs page metadata and subscription topic and the time it took to complete the request.
// If the request fails, it logs the error.
func (lm *loggingMiddleware) ListSubscriptions(ctx context.Context, token string, pm notifiers.PageMetadata) (res notifiers.Page, err error) {
defer func(begin time.Time) {
args := []any{
slog.String("duration", time.Since(begin).String()),
slog.Group("page",
slog.String("topic", pm.Topic),
slog.Int("limit", pm.Limit),
slog.Uint64("offset", uint64(pm.Offset)),
slog.Uint64("total", uint64(res.Total)),
),
}
if err != nil {
args = append(args, slog.Any("error", err))
lm.logger.Warn("List subscriptions failed", args...)
return
}
lm.logger.Info("List subscriptions completed successfully", args...)
}(time.Now())
return lm.svc.ListSubscriptions(ctx, token, pm)
}
// RemoveSubscription logs the remove_subscription request. It logs subscription ID and the time it took to complete the request.
// If the request fails, it logs the error.
func (lm *loggingMiddleware) RemoveSubscription(ctx context.Context, token, id string) (err error) {
defer func(begin time.Time) {
args := []any{
slog.String("duration", time.Since(begin).String()),
slog.String("subscription_id", id),
}
if err != nil {
args = append(args, slog.Any("error", err))
lm.logger.Warn("Remove subscription failed", args...)
return
}
lm.logger.Info("Remove subscription completed successfully", args...)
}(time.Now())
return lm.svc.RemoveSubscription(ctx, token, id)
}
// ConsumeBlocking logs the consume_blocking request. It logs the time it took to complete the request.
// If the request fails, it logs the error.
func (lm *loggingMiddleware) ConsumeBlocking(ctx context.Context, msg interface{}) (err error) {
defer func(begin time.Time) {
args := []any{
slog.String("duration", time.Since(begin).String()),
}
if err != nil {
args = append(args, slog.Any("error", err))
lm.logger.Warn("Blocking consumer failed to consume messages successfully", args...)
return
}
lm.logger.Info("Blocking consumer consumed messages successfully", args...)
}(time.Now())
return lm.svc.ConsumeBlocking(ctx, msg)
}
consumers/notifiers/api/metrics.go
-81
View File
@@ -1,81 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
//go:build !test
package api
import (
"context"
"time"
"github.com/absmach/supermq/consumers/notifiers"
"github.com/go-kit/kit/metrics"
)
var _ notifiers.Service = (*metricsMiddleware)(nil)
type metricsMiddleware struct {
counter metrics.Counter
latency metrics.Histogram
svc notifiers.Service
}
// MetricsMiddleware instruments core service by tracking request count and latency.
func MetricsMiddleware(svc notifiers.Service, counter metrics.Counter, latency metrics.Histogram) notifiers.Service {
return &metricsMiddleware{
counter: counter,
latency: latency,
svc: svc,
}
}
// CreateSubscription instruments CreateSubscription method with metrics.
func (ms *metricsMiddleware) CreateSubscription(ctx context.Context, token string, sub notifiers.Subscription) (string, error) {
defer func(begin time.Time) {
ms.counter.With("method", "create_subscription").Add(1)
ms.latency.With("method", "create_subscription").Observe(time.Since(begin).Seconds())
}(time.Now())
return ms.svc.CreateSubscription(ctx, token, sub)
}
// ViewSubscription instruments ViewSubscription method with metrics.
func (ms *metricsMiddleware) ViewSubscription(ctx context.Context, token, topic string) (notifiers.Subscription, error) {
defer func(begin time.Time) {
ms.counter.With("method", "view_subscription").Add(1)
ms.latency.With("method", "view_subscription").Observe(time.Since(begin).Seconds())
}(time.Now())
return ms.svc.ViewSubscription(ctx, token, topic)
}
// ListSubscriptions instruments ListSubscriptions method with metrics.
func (ms *metricsMiddleware) ListSubscriptions(ctx context.Context, token string, pm notifiers.PageMetadata) (notifiers.Page, error) {
defer func(begin time.Time) {
ms.counter.With("method", "list_subscriptions").Add(1)
ms.latency.With("method", "list_subscriptions").Observe(time.Since(begin).Seconds())
}(time.Now())
return ms.svc.ListSubscriptions(ctx, token, pm)
}
// RemoveSubscription instruments RemoveSubscription method with metrics.
func (ms *metricsMiddleware) RemoveSubscription(ctx context.Context, token, id string) error {
defer func(begin time.Time) {
ms.counter.With("method", "remove_subscription").Add(1)
ms.latency.With("method", "remove_subscription").Observe(time.Since(begin).Seconds())
}(time.Now())
return ms.svc.RemoveSubscription(ctx, token, id)
}
// ConsumeBlocking instruments ConsumeBlocking method with metrics.
func (ms *metricsMiddleware) ConsumeBlocking(ctx context.Context, msg interface{}) error {
defer func(begin time.Time) {
ms.counter.With("method", "consume").Add(1)
ms.latency.With("method", "consume").Observe(time.Since(begin).Seconds())
}(time.Now())
return ms.svc.ConsumeBlocking(ctx, msg)
}
consumers/notifiers/api/requests.go
-55
View File
@@ -1,55 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api
import apiutil "github.com/absmach/supermq/api/http/util"
type createSubReq struct {
token string
Topic string `json:"topic,omitempty"`
Contact string `json:"contact,omitempty"`
}
func (req createSubReq) validate() error {
if req.token == "" {
return apiutil.ErrBearerToken
}
if req.Topic == "" {
return apiutil.ErrInvalidTopic
}
if req.Contact == "" {
return apiutil.ErrInvalidContact
}
return nil
}
type subReq struct {
token string
id string
}
func (req subReq) validate() error {
if req.token == "" {
return apiutil.ErrBearerToken
}
if req.id == "" {
return apiutil.ErrMissingID
}
return nil
}
type listSubsReq struct {
token string
topic string
contact string
offset uint
limit uint
}
func (req listSubsReq) validate() error {
if req.token == "" {
return apiutil.ErrBearerToken
}
return nil
}
consumers/notifiers/api/responses.go
-88
View File
@@ -1,88 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api
import (
"fmt"
"net/http"
"github.com/absmach/supermq"
)
var (
_ supermq.Response = (*createSubRes)(nil)
_ supermq.Response = (*viewSubRes)(nil)
_ supermq.Response = (*listSubsRes)(nil)
_ supermq.Response = (*removeSubRes)(nil)
)
type createSubRes struct {
ID string
}
func (res createSubRes) Code() int {
return http.StatusCreated
}
func (res createSubRes) Headers() map[string]string {
return map[string]string{
"Location": fmt.Sprintf("/subscriptions/%s", res.ID),
}
}
func (res createSubRes) Empty() bool {
return true
}
type viewSubRes struct {
ID string `json:"id"`
OwnerID string `json:"owner_id"`
Contact string `json:"contact"`
Topic string `json:"topic"`
}
func (res viewSubRes) Code() int {
return http.StatusOK
}
func (res viewSubRes) Headers() map[string]string {
return map[string]string{}
}
func (res viewSubRes) Empty() bool {
return false
}
type listSubsRes struct {
Offset uint `json:"offset"`
Limit int `json:"limit"`
Total uint `json:"total,omitempty"`
Subscriptions []viewSubRes `json:"subscriptions,omitempty"`
}
func (res listSubsRes) Code() int {
return http.StatusOK
}
func (res listSubsRes) Headers() map[string]string {
return map[string]string{}
}
func (res listSubsRes) Empty() bool {
return false
}
type removeSubRes struct{}
func (res removeSubRes) Code() int {
return http.StatusNoContent
}
func (res removeSubRes) Headers() map[string]string {
return map[string]string{}
}
func (res removeSubRes) Empty() bool {
return true
}
consumers/notifiers/api/transport.go
-131
View File
@@ -1,131 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api
import (
"context"
"encoding/json"
"log/slog"
"net/http"
"strings"
"github.com/absmach/supermq"
api "github.com/absmach/supermq/api/http"
apiutil "github.com/absmach/supermq/api/http/util"
"github.com/absmach/supermq/consumers/notifiers"
"github.com/absmach/supermq/pkg/errors"
"github.com/go-chi/chi/v5"
kithttp "github.com/go-kit/kit/transport/http"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)
const (
contentType = "application/json"
offsetKey = "offset"
limitKey = "limit"
topicKey = "topic"
contactKey = "contact"
defOffset = 0
defLimit = 20
)
// MakeHandler returns a HTTP handler for API endpoints.
func MakeHandler(svc notifiers.Service, logger *slog.Logger, instanceID string) http.Handler {
opts := []kithttp.ServerOption{
kithttp.ServerErrorEncoder(apiutil.LoggingErrorEncoder(logger, api.EncodeError)),
}
mux := chi.NewRouter()
mux.Route("/subscriptions", func(r chi.Router) {
r.Post("/", otelhttp.NewHandler(kithttp.NewServer(
createSubscriptionEndpoint(svc),
decodeCreate,
api.EncodeResponse,
opts...,
), "create").ServeHTTP)
r.Get("/", otelhttp.NewHandler(kithttp.NewServer(
listSubscriptionsEndpoint(svc),
decodeList,
api.EncodeResponse,
opts...,
), "list").ServeHTTP)
r.Delete("/", otelhttp.NewHandler(kithttp.NewServer(
deleteSubscriptionEndpint(svc),
decodeSubscription,
api.EncodeResponse,
opts...,
), "delete").ServeHTTP)
r.Get("/{subID}", otelhttp.NewHandler(kithttp.NewServer(
viewSubscriptionEndpint(svc),
decodeSubscription,
api.EncodeResponse,
opts...,
), "view").ServeHTTP)
r.Delete("/{subID}", otelhttp.NewHandler(kithttp.NewServer(
deleteSubscriptionEndpint(svc),
decodeSubscription,
api.EncodeResponse,
opts...,
), "delete").ServeHTTP)
})
mux.Get("/health", supermq.Health("notifier", instanceID))
mux.Handle("/metrics", promhttp.Handler())
return mux
}
func decodeCreate(_ context.Context, r *http.Request) (interface{}, error) {
if !strings.Contains(r.Header.Get("Content-Type"), contentType) {
return nil, errors.Wrap(apiutil.ErrValidation, apiutil.ErrUnsupportedContentType)
}
req := createSubReq{token: apiutil.ExtractBearerToken(r)}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, errors.Wrap(err, errors.ErrMalformedEntity))
}
return req, nil
}
func decodeSubscription(_ context.Context, r *http.Request) (interface{}, error) {
req := subReq{
id: chi.URLParam(r, "subID"),
token: apiutil.ExtractBearerToken(r),
}
return req, nil
}
func decodeList(_ context.Context, r *http.Request) (interface{}, error) {
req := listSubsReq{token: apiutil.ExtractBearerToken(r)}
vals := r.URL.Query()[topicKey]
if len(vals) > 0 {
req.topic = vals[0]
}
vals = r.URL.Query()[contactKey]
if len(vals) > 0 {
req.contact = vals[0]
}
offset, err := apiutil.ReadNumQuery[uint64](r, offsetKey, defOffset)
if err != nil {
return listSubsReq{}, errors.Wrap(apiutil.ErrValidation, err)
}
req.offset = uint(offset)
limit, err := apiutil.ReadNumQuery[uint64](r, limitKey, defLimit)
if err != nil {
return listSubsReq{}, errors.Wrap(apiutil.ErrValidation, err)
}
req.limit = uint(limit)
return req, nil
}
consumers/notifiers/doc.go
-6
View File
@@ -1,6 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package notifiers contain the domain concept definitions needed to
// support SuperMQ notifications functionality.
package notifiers
consumers/notifiers/mocks/doc.go
-5
View File
@@ -1,5 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package mocks contains mocks for testing purposes.
package mocks
consumers/notifiers/mocks/notifier.go
-47
View File
@@ -1,47 +0,0 @@
// Code generated by mockery v2.43.2. DO NOT EDIT.
// Copyright (c) Abstract Machines
package mocks
import (
messaging "github.com/absmach/supermq/pkg/messaging"
mock "github.com/stretchr/testify/mock"
)
// Notifier is an autogenerated mock type for the Notifier type
type Notifier struct {
mock.Mock
}
// Notify provides a mock function with given fields: from, to, msg
func (_m *Notifier) Notify(from string, to []string, msg *messaging.Message) error {
ret := _m.Called(from, to, msg)
if len(ret) == 0 {
panic("no return value specified for Notify")
}
var r0 error
if rf, ok := ret.Get(0).(func(string, []string, *messaging.Message) error); ok {
r0 = rf(from, to, msg)
} else {
r0 = ret.Error(0)
}
return r0
}
// NewNotifier creates a new instance of Notifier. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewNotifier(t interface {
mock.TestingT
Cleanup(func())
}) *Notifier {
mock := &Notifier{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
consumers/notifiers/mocks/repository.go
-133
View File
@@ -1,133 +0,0 @@
// Code generated by mockery v2.43.2. DO NOT EDIT.
// Copyright (c) Abstract Machines
package mocks
import (
context "context"
notifiers "github.com/absmach/supermq/consumers/notifiers"
mock "github.com/stretchr/testify/mock"
)
// SubscriptionsRepository is an autogenerated mock type for the SubscriptionsRepository type
type SubscriptionsRepository struct {
mock.Mock
}
// Remove provides a mock function with given fields: ctx, id
func (_m *SubscriptionsRepository) Remove(ctx context.Context, id string) error {
ret := _m.Called(ctx, id)
if len(ret) == 0 {
panic("no return value specified for Remove")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, string) error); ok {
r0 = rf(ctx, id)
} else {
r0 = ret.Error(0)
}
return r0
}
// Retrieve provides a mock function with given fields: ctx, id
func (_m *SubscriptionsRepository) Retrieve(ctx context.Context, id string) (notifiers.Subscription, error) {
ret := _m.Called(ctx, id)
if len(ret) == 0 {
panic("no return value specified for Retrieve")
}
var r0 notifiers.Subscription
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) (notifiers.Subscription, error)); ok {
return rf(ctx, id)
}
if rf, ok := ret.Get(0).(func(context.Context, string) notifiers.Subscription); ok {
r0 = rf(ctx, id)
} else {
r0 = ret.Get(0).(notifiers.Subscription)
}
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, id)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// RetrieveAll provides a mock function with given fields: ctx, pm
func (_m *SubscriptionsRepository) RetrieveAll(ctx context.Context, pm notifiers.PageMetadata) (notifiers.Page, error) {
ret := _m.Called(ctx, pm)
if len(ret) == 0 {
panic("no return value specified for RetrieveAll")
}
var r0 notifiers.Page
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, notifiers.PageMetadata) (notifiers.Page, error)); ok {
return rf(ctx, pm)
}
if rf, ok := ret.Get(0).(func(context.Context, notifiers.PageMetadata) notifiers.Page); ok {
r0 = rf(ctx, pm)
} else {
r0 = ret.Get(0).(notifiers.Page)
}
if rf, ok := ret.Get(1).(func(context.Context, notifiers.PageMetadata) error); ok {
r1 = rf(ctx, pm)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Save provides a mock function with given fields: ctx, sub
func (_m *SubscriptionsRepository) Save(ctx context.Context, sub notifiers.Subscription) (string, error) {
ret := _m.Called(ctx, sub)
if len(ret) == 0 {
panic("no return value specified for Save")
}
var r0 string
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, notifiers.Subscription) (string, error)); ok {
return rf(ctx, sub)
}
if rf, ok := ret.Get(0).(func(context.Context, notifiers.Subscription) string); ok {
r0 = rf(ctx, sub)
} else {
r0 = ret.Get(0).(string)
}
if rf, ok := ret.Get(1).(func(context.Context, notifiers.Subscription) error); ok {
r1 = rf(ctx, sub)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// NewSubscriptionsRepository creates a new instance of SubscriptionsRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewSubscriptionsRepository(t interface {
mock.TestingT
Cleanup(func())
}) *SubscriptionsRepository {
mock := &SubscriptionsRepository{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
consumers/notifiers/mocks/service.go
-151
View File
@@ -1,151 +0,0 @@
// Code generated by mockery v2.43.2. DO NOT EDIT.
// Copyright (c) Abstract Machines
package mocks
import (
context "context"
notifiers "github.com/absmach/supermq/consumers/notifiers"
mock "github.com/stretchr/testify/mock"
)
// Service is an autogenerated mock type for the Service type
type Service struct {
mock.Mock
}
// ConsumeBlocking provides a mock function with given fields: ctx, messages
func (_m *Service) ConsumeBlocking(ctx context.Context, messages interface{}) error {
ret := _m.Called(ctx, messages)
if len(ret) == 0 {
panic("no return value specified for ConsumeBlocking")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, interface{}) error); ok {
r0 = rf(ctx, messages)
} else {
r0 = ret.Error(0)
}
return r0
}
// CreateSubscription provides a mock function with given fields: ctx, token, sub
func (_m *Service) CreateSubscription(ctx context.Context, token string, sub notifiers.Subscription) (string, error) {
ret := _m.Called(ctx, token, sub)
if len(ret) == 0 {
panic("no return value specified for CreateSubscription")
}
var r0 string
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, notifiers.Subscription) (string, error)); ok {
return rf(ctx, token, sub)
}
if rf, ok := ret.Get(0).(func(context.Context, string, notifiers.Subscription) string); ok {
r0 = rf(ctx, token, sub)
} else {
r0 = ret.Get(0).(string)
}
if rf, ok := ret.Get(1).(func(context.Context, string, notifiers.Subscription) error); ok {
r1 = rf(ctx, token, sub)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ListSubscriptions provides a mock function with given fields: ctx, token, pm
func (_m *Service) ListSubscriptions(ctx context.Context, token string, pm notifiers.PageMetadata) (notifiers.Page, error) {
ret := _m.Called(ctx, token, pm)
if len(ret) == 0 {
panic("no return value specified for ListSubscriptions")
}
var r0 notifiers.Page
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, notifiers.PageMetadata) (notifiers.Page, error)); ok {
return rf(ctx, token, pm)
}
if rf, ok := ret.Get(0).(func(context.Context, string, notifiers.PageMetadata) notifiers.Page); ok {
r0 = rf(ctx, token, pm)
} else {
r0 = ret.Get(0).(notifiers.Page)
}
if rf, ok := ret.Get(1).(func(context.Context, string, notifiers.PageMetadata) error); ok {
r1 = rf(ctx, token, pm)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// RemoveSubscription provides a mock function with given fields: ctx, token, id
func (_m *Service) RemoveSubscription(ctx context.Context, token string, id string) error {
ret := _m.Called(ctx, token, id)
if len(ret) == 0 {
panic("no return value specified for RemoveSubscription")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok {
r0 = rf(ctx, token, id)
} else {
r0 = ret.Error(0)
}
return r0
}
// ViewSubscription provides a mock function with given fields: ctx, token, id
func (_m *Service) ViewSubscription(ctx context.Context, token string, id string) (notifiers.Subscription, error) {
ret := _m.Called(ctx, token, id)
if len(ret) == 0 {
panic("no return value specified for ViewSubscription")
}
var r0 notifiers.Subscription
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, string) (notifiers.Subscription, error)); ok {
return rf(ctx, token, id)
}
if rf, ok := ret.Get(0).(func(context.Context, string, string) notifiers.Subscription); ok {
r0 = rf(ctx, token, id)
} else {
r0 = ret.Get(0).(notifiers.Subscription)
}
if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok {
r1 = rf(ctx, token, id)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// NewService creates a new instance of Service. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewService(t interface {
mock.TestingT
Cleanup(func())
}) *Service {
mock := &Service{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
consumers/notifiers/notifier.go
-22
View File
@@ -1,22 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package notifiers
import (
"errors"
"github.com/absmach/supermq/pkg/messaging"
)
// ErrNotify wraps sending notification errors.
var ErrNotify = errors.New("error sending notification")
// Notifier represents an API for sending notification.
//
//go:generate mockery --name Notifier --output=./mocks --filename notifier.go --quiet --note "Copyright (c) Abstract Machines"
type Notifier interface {
// Notify method is used to send notification for the
// received message to the provided list of receivers.
Notify(from string, to []string, msg *messaging.Message) error
}
consumers/notifiers/postgres/database.go
-74
View File
@@ -1,74 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package postgres
import (
"context"
"database/sql"
"fmt"
"github.com/jmoiron/sqlx"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
var _ Database = (*database)(nil)
type database struct {
db *sqlx.DB
tracer trace.Tracer
}
// Database provides a database interface.
type Database interface {
NamedExecContext(context.Context, string, interface{}) (sql.Result, error)
QueryRowxContext(context.Context, string, ...interface{}) *sqlx.Row
NamedQueryContext(context.Context, string, interface{}) (*sqlx.Rows, error)
GetContext(context.Context, interface{}, string, ...interface{}) error
}
// NewDatabase creates a SubscriptionsDatabase instance.
func NewDatabase(db *sqlx.DB, tracer trace.Tracer) Database {
return &database{
db: db,
tracer: tracer,
}
}
func (dm database) NamedExecContext(ctx context.Context, query string, args interface{}) (sql.Result, error) {
ctx, span := dm.addSpanTags(ctx, "NamedExecContext", query)
defer span.End()
return dm.db.NamedExecContext(ctx, query, args)
}
func (dm database) QueryRowxContext(ctx context.Context, query string, args ...interface{}) *sqlx.Row {
ctx, span := dm.addSpanTags(ctx, "QueryRowxContext", query)
defer span.End()
return dm.db.QueryRowxContext(ctx, query, args...)
}
func (dm database) NamedQueryContext(ctx context.Context, query string, args interface{}) (*sqlx.Rows, error) {
ctx, span := dm.addSpanTags(ctx, "NamedQueryContext", query)
defer span.End()
return dm.db.NamedQueryContext(ctx, query, args)
}
func (dm database) GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
ctx, span := dm.addSpanTags(ctx, "GetContext", query)
defer span.End()
return dm.db.GetContext(ctx, dest, query, args...)
}
func (dm database) addSpanTags(ctx context.Context, method, query string) (context.Context, trace.Span) {
ctx, span := dm.tracer.Start(ctx,
fmt.Sprintf("sql_%s", method),
trace.WithAttributes(
attribute.String("sql.statement", query),
attribute.String("span.kind", "client"),
attribute.String("peer.service", "postgres"),
attribute.String("db.type", "sql"),
),
)
return ctx, span
}
consumers/notifiers/postgres/doc.go
-6
View File
@@ -1,6 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package postgres contains repository implementations using PostgreSQL as
// the underlying database.
package postgres
consumers/notifiers/postgres/init.go
-28
View File
@@ -1,28 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package postgres
import migrate "github.com/rubenv/sql-migrate"
func Migration() *migrate.MemoryMigrationSource {
return &migrate.MemoryMigrationSource{
Migrations: []*migrate.Migration{
{
Id: "subscriptions_1",
Up: []string{
`CREATE TABLE IF NOT EXISTS subscriptions (
id VARCHAR(254) PRIMARY KEY,
owner_id VARCHAR(254) NOT NULL,
contact VARCHAR(254),
topic TEXT,
UNIQUE(topic, contact)
)`,
},
Down: []string{
"DROP TABLE IF EXISTS subscriptions",
},
},
},
}
}
consumers/notifiers/postgres/setup_test.go
-89
View File
@@ -1,89 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package postgres_test contains tests for PostgreSQL repository
// implementations.
package postgres_test
import (
"fmt"
"log"
"os"
"testing"
"github.com/absmach/supermq/consumers/notifiers/postgres"
pgclient "github.com/absmach/supermq/pkg/postgres"
"github.com/absmach/supermq/pkg/ulid"
_ "github.com/jackc/pgx/v5/stdlib" // required for SQL access
"github.com/jmoiron/sqlx"
"github.com/ory/dockertest/v3"
"github.com/ory/dockertest/v3/docker"
)
var (
idProvider = ulid.New()
db *sqlx.DB
)
func TestMain(m *testing.M) {
pool, err := dockertest.NewPool("")
if err != nil {
log.Fatalf("Could not connect to docker: %s", err)
}
container, err := pool.RunWithOptions(&dockertest.RunOptions{
Repository: "postgres",
Tag: "16.2-alpine",
Env: []string{
"POSTGRES_USER=test",
"POSTGRES_PASSWORD=test",
"POSTGRES_DB=test",
"listen_addresses = '*'",
},
}, func(config *docker.HostConfig) {
config.AutoRemove = true
config.RestartPolicy = docker.RestartPolicy{Name: "no"}
})
if err != nil {
log.Fatalf("Could not start container: %s", err)
}
port := container.GetPort("5432/tcp")
url := fmt.Sprintf("host=localhost port=%s user=test dbname=test password=test sslmode=disable", port)
if err := pool.Retry(func() error {
db, err = sqlx.Open("pgx", url)
if err != nil {
return err
}
return db.Ping()
}); err != nil {
log.Fatalf("Could not connect to docker: %s", err)
}
dbConfig := pgclient.Config{
Host: "localhost",
Port: port,
User: "test",
Pass: "test",
Name: "test",
SSLMode: "disable",
SSLCert: "",
SSLKey: "",
SSLRootCert: "",
}
if db, err = pgclient.Setup(dbConfig, *postgres.Migration()); err != nil {
log.Fatalf("Could not setup test DB connection: %s", err)
}
code := m.Run()
// Defers will not be run when using os.Exit
db.Close()
if err := pool.Purge(container); err != nil {
log.Fatalf("Could not purge container: %s", err)
}
os.Exit(code)
}
consumers/notifiers/postgres/subscriptions.go
-164
View File
@@ -1,164 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package postgres
import (
"context"
"database/sql"
"fmt"
"strings"
"github.com/absmach/supermq/consumers/notifiers"
"github.com/absmach/supermq/pkg/errors"
repoerr "github.com/absmach/supermq/pkg/errors/repository"
"github.com/jackc/pgerrcode"
"github.com/jackc/pgx/v5/pgconn"
)
var _ notifiers.SubscriptionsRepository = (*subscriptionsRepo)(nil)
type subscriptionsRepo struct {
db Database
}
// New instantiates a PostgreSQL implementation of Subscriptions repository.
func New(db Database) notifiers.SubscriptionsRepository {
return &subscriptionsRepo{
db: db,
}
}
func (repo subscriptionsRepo) Save(ctx context.Context, sub notifiers.Subscription) (string, error) {
q := `INSERT INTO subscriptions (id, owner_id, contact, topic) VALUES (:id, :owner_id, :contact, :topic) RETURNING id`
dbSub := dbSubscription{
ID: sub.ID,
OwnerID: sub.OwnerID,
Contact: sub.Contact,
Topic: sub.Topic,
}
row, err := repo.db.NamedQueryContext(ctx, q, dbSub)
if err != nil {
if pqErr, ok := err.(*pgconn.PgError); ok && pqErr.Code == pgerrcode.UniqueViolation {
return "", errors.Wrap(repoerr.ErrConflict, err)
}
return "", errors.Wrap(repoerr.ErrCreateEntity, err)
}
defer row.Close()
return sub.ID, nil
}
func (repo subscriptionsRepo) Retrieve(ctx context.Context, id string) (notifiers.Subscription, error) {
q := `SELECT id, owner_id, contact, topic FROM subscriptions WHERE id = $1`
sub := dbSubscription{}
if err := repo.db.QueryRowxContext(ctx, q, id).StructScan(&sub); err != nil {
if err == sql.ErrNoRows {
return notifiers.Subscription{}, errors.Wrap(repoerr.ErrNotFound, err)
}
return notifiers.Subscription{}, errors.Wrap(repoerr.ErrViewEntity, err)
}
return fromDBSub(sub), nil
}
func (repo subscriptionsRepo) RetrieveAll(ctx context.Context, pm notifiers.PageMetadata) (notifiers.Page, error) {
q := `SELECT id, owner_id, contact, topic FROM subscriptions`
args := make(map[string]interface{})
if pm.Topic != "" {
args["topic"] = pm.Topic
}
if pm.Contact != "" {
args["contact"] = pm.Contact
}
var condition string
if len(args) > 0 {
var cond []string
for k := range args {
cond = append(cond, fmt.Sprintf("%s = :%s", k, k))
}
condition = fmt.Sprintf(" WHERE %s", strings.Join(cond, " AND "))
q = fmt.Sprintf("%s%s", q, condition)
}
args["offset"] = pm.Offset
q = fmt.Sprintf("%s OFFSET :offset", q)
if pm.Limit > 0 {
q = fmt.Sprintf("%s LIMIT :limit", q)
args["limit"] = pm.Limit
}
rows, err := repo.db.NamedQueryContext(ctx, q, args)
if err != nil {
return notifiers.Page{}, errors.Wrap(repoerr.ErrViewEntity, err)
}
defer rows.Close()
var subs []notifiers.Subscription
for rows.Next() {
sub := dbSubscription{}
if err := rows.StructScan(&sub); err != nil {
return notifiers.Page{}, errors.Wrap(repoerr.ErrViewEntity, err)
}
subs = append(subs, fromDBSub(sub))
}
if len(subs) == 0 {
return notifiers.Page{}, repoerr.ErrNotFound
}
cq := fmt.Sprintf(`SELECT COUNT(*) FROM subscriptions %s`, condition)
total, err := total(ctx, repo.db, cq, args)
if err != nil {
return notifiers.Page{}, errors.Wrap(repoerr.ErrViewEntity, err)
}
ret := notifiers.Page{
PageMetadata: pm,
Total: total,
Subscriptions: subs,
}
return ret, nil
}
func (repo subscriptionsRepo) Remove(ctx context.Context, id string) error {
q := `DELETE from subscriptions WHERE id = $1`
if r := repo.db.QueryRowxContext(ctx, q, id); r.Err() != nil {
return errors.Wrap(repoerr.ErrRemoveEntity, r.Err())
}
return nil
}
func total(ctx context.Context, db Database, query string, params interface{}) (uint, error) {
rows, err := db.NamedQueryContext(ctx, query, params)
if err != nil {
return 0, err
}
defer rows.Close()
var total uint
if rows.Next() {
if err := rows.Scan(&total); err != nil {
return 0, err
}
}
return total, nil
}
type dbSubscription struct {
ID string `db:"id"`
OwnerID string `db:"owner_id"`
Contact string `db:"contact"`
Topic string `db:"topic"`
}
func fromDBSub(sub dbSubscription) notifiers.Subscription {
return notifiers.Subscription{
ID: sub.ID,
OwnerID: sub.OwnerID,
Contact: sub.Contact,
Topic: sub.Topic,
}
}
consumers/notifiers/postgres/subscriptions_test.go
-263
View File
@@ -1,263 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package postgres_test
import (
"context"
"fmt"
"testing"
"github.com/absmach/supermq/consumers/notifiers"
"github.com/absmach/supermq/consumers/notifiers/postgres"
"github.com/absmach/supermq/pkg/errors"
repoerr "github.com/absmach/supermq/pkg/errors/repository"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel"
)
const (
owner = "owner@example.com"
numSubs = 100
)
var tracer = otel.Tracer("tests")
func TestSave(t *testing.T) {
dbMiddleware := postgres.NewDatabase(db, tracer)
repo := postgres.New(dbMiddleware)
id1, err := idProvider.ID()
assert.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err))
id2, err := idProvider.ID()
assert.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err))
sub1 := notifiers.Subscription{
OwnerID: id1,
ID: id1,
Contact: owner,
Topic: "topic.subtopic",
}
sub2 := sub1
sub2.ID = id2
cases := []struct {
desc string
sub notifiers.Subscription
id string
err error
}{
{
desc: "save successfully",
sub: sub1,
id: id1,
err: nil,
},
{
desc: "save duplicate",
sub: sub2,
id: "",
err: repoerr.ErrConflict,
},
}
for _, tc := range cases {
id, err := repo.Save(context.Background(), tc.sub)
assert.Equal(t, tc.id, id, fmt.Sprintf("%s: expected id %s got %s\n", tc.desc, tc.id, id))
assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err))
}
}
func TestView(t *testing.T) {
dbMiddleware := postgres.NewDatabase(db, tracer)
repo := postgres.New(dbMiddleware)
id, err := idProvider.ID()
require.Nil(t, err, fmt.Sprintf("got an error creating id: %s", err))
sub := notifiers.Subscription{
OwnerID: id,
ID: id,
Contact: owner,
Topic: "view.subtopic",
}
ret, err := repo.Save(context.Background(), sub)
require.Nil(t, err, fmt.Sprintf("creating subscription must not fail: %s", err))
require.Equal(t, id, ret, fmt.Sprintf("provided id %s must be the same as the returned id %s", id, ret))
cases := []struct {
desc string
sub notifiers.Subscription
id string
err error
}{
{
desc: "retrieve successfully",
sub: sub,
id: id,
err: nil,
},
{
desc: "retrieve not existing",
sub: notifiers.Subscription{},
id: "non-existing",
err: repoerr.ErrNotFound,
},
}
for _, tc := range cases {
sub, err := repo.Retrieve(context.Background(), tc.id)
assert.Equal(t, tc.sub, sub, fmt.Sprintf("%s: expected sub %v got %v\n", tc.desc, tc.sub, sub))
assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err))
}
}
func TestRetrieveAll(t *testing.T) {
_, err := db.Exec("DELETE FROM subscriptions")
require.Nil(t, err, fmt.Sprintf("cleanup must not fail: %s", err))
dbMiddleware := postgres.NewDatabase(db, tracer)
repo := postgres.New(dbMiddleware)
var subs []notifiers.Subscription
for i := 0; i < numSubs; i++ {
id, err := idProvider.ID()
assert.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err))
sub := notifiers.Subscription{
OwnerID: "owner",
ID: id,
Contact: owner,
Topic: fmt.Sprintf("list.subtopic.%d", i),
}
ret, err := repo.Save(context.Background(), sub)
require.Nil(t, err, fmt.Sprintf("creating subscription must not fail: %s", err))
require.Equal(t, id, ret, fmt.Sprintf("provided id %s must be the same as the returned id %s", id, ret))
subs = append(subs, sub)
}
cases := []struct {
desc string
pageMeta notifiers.PageMetadata
page notifiers.Page
err error
}{
{
desc: "retrieve successfully",
pageMeta: notifiers.PageMetadata{
Offset: 10,
Limit: 2,
},
page: notifiers.Page{
Total: numSubs,
PageMetadata: notifiers.PageMetadata{
Offset: 10,
Limit: 2,
},
Subscriptions: subs[10:12],
},
err: nil,
},
{
desc: "retrieve with contact",
pageMeta: notifiers.PageMetadata{
Offset: 10,
Limit: 2,
Contact: owner,
},
page: notifiers.Page{
Total: numSubs,
PageMetadata: notifiers.PageMetadata{
Offset: 10,
Limit: 2,
Contact: owner,
},
Subscriptions: subs[10:12],
},
err: nil,
},
{
desc: "retrieve with topic",
pageMeta: notifiers.PageMetadata{
Offset: 0,
Limit: 2,
Topic: "list.subtopic.11",
},
page: notifiers.Page{
Total: 1,
PageMetadata: notifiers.PageMetadata{
Offset: 0,
Limit: 2,
Topic: "list.subtopic.11",
},
Subscriptions: subs[11:12],
},
err: nil,
},
{
desc: "retrieve with no limit",
pageMeta: notifiers.PageMetadata{
Offset: 0,
Limit: -1,
},
page: notifiers.Page{
Total: numSubs,
PageMetadata: notifiers.PageMetadata{
Limit: -1,
},
Subscriptions: subs,
},
err: nil,
},
}
for _, tc := range cases {
page, err := repo.RetrieveAll(context.Background(), tc.pageMeta)
assert.Equal(t, tc.page, page, fmt.Sprintf("%s: got unexpected page\n", tc.desc))
assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err))
}
}
func TestRemove(t *testing.T) {
dbMiddleware := postgres.NewDatabase(db, tracer)
repo := postgres.New(dbMiddleware)
id, err := idProvider.ID()
require.Nil(t, err, fmt.Sprintf("got an error creating id: %s", err))
sub := notifiers.Subscription{
OwnerID: id,
ID: id,
Contact: owner,
Topic: "remove.subtopic.%d",
}
ret, err := repo.Save(context.Background(), sub)
require.Nil(t, err, fmt.Sprintf("creating subscription must not fail: %s", err))
require.Equal(t, id, ret, fmt.Sprintf("provided id %s must be the same as the returned id %s", id, ret))
cases := []struct {
desc string
id string
err error
}{
{
desc: "remove successfully",
id: id,
err: nil,
},
{
desc: "remove not existing",
id: "empty",
err: nil,
},
}
for _, tc := range cases {
err := repo.Remove(context.Background(), tc.id)
assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err))
}
}
consumers/notifiers/service.go
-175
View File
@@ -1,175 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package notifiers
import (
"context"
"fmt"
"github.com/absmach/supermq"
"github.com/absmach/supermq/consumers"
smqauthn "github.com/absmach/supermq/pkg/authn"
"github.com/absmach/supermq/pkg/errors"
svcerr "github.com/absmach/supermq/pkg/errors/service"
"github.com/absmach/supermq/pkg/messaging"
)
// ErrMessage indicates an error converting a message to SuperMQ message.
var ErrMessage = errors.New("failed to convert to SuperMQ message")
var _ consumers.AsyncConsumer = (*notifierService)(nil)
// Service reprents a notification service.
//
//go:generate mockery --name Service --output=./mocks --filename service.go --quiet --note "Copyright (c) Abstract Machines"
type Service interface {
// CreateSubscription persists a subscription.
// Successful operation is indicated by non-nil error response.
CreateSubscription(ctx context.Context, token string, sub Subscription) (string, error)
// ViewSubscription retrieves the subscription for the given user and id.
ViewSubscription(ctx context.Context, token, id string) (Subscription, error)
// ListSubscriptions lists subscriptions having the provided user token and search params.
ListSubscriptions(ctx context.Context, token string, pm PageMetadata) (Page, error)
// RemoveSubscription removes the subscription having the provided identifier.
RemoveSubscription(ctx context.Context, token, id string) error
consumers.BlockingConsumer
}
var _ Service = (*notifierService)(nil)
type notifierService struct {
authn smqauthn.Authentication
subs SubscriptionsRepository
idp supermq.IDProvider
notifier Notifier
errCh chan error
from string
}
// New instantiates the subscriptions service implementation.
func New(authn smqauthn.Authentication, subs SubscriptionsRepository, idp supermq.IDProvider, notifier Notifier, from string) Service {
return &notifierService{
authn: authn,
subs: subs,
idp: idp,
notifier: notifier,
errCh: make(chan error, 1),
from: from,
}
}
func (ns *notifierService) CreateSubscription(ctx context.Context, token string, sub Subscription) (string, error) {
session, err := ns.authn.Authenticate(ctx, token)
if err != nil {
return "", err
}
sub.ID, err = ns.idp.ID()
if err != nil {
return "", err
}
sub.OwnerID = session.DomainUserID
id, err := ns.subs.Save(ctx, sub)
if err != nil {
return "", errors.Wrap(svcerr.ErrCreateEntity, err)
}
return id, nil
}
func (ns *notifierService) ViewSubscription(ctx context.Context, token, id string) (Subscription, error) {
if _, err := ns.authn.Authenticate(ctx, token); err != nil {
return Subscription{}, err
}
return ns.subs.Retrieve(ctx, id)
}
func (ns *notifierService) ListSubscriptions(ctx context.Context, token string, pm PageMetadata) (Page, error) {
if _, err := ns.authn.Authenticate(ctx, token); err != nil {
return Page{}, err
}
return ns.subs.RetrieveAll(ctx, pm)
}
func (ns *notifierService) RemoveSubscription(ctx context.Context, token, id string) error {
if _, err := ns.authn.Authenticate(ctx, token); err != nil {
return err
}
return ns.subs.Remove(ctx, id)
}
func (ns *notifierService) ConsumeBlocking(ctx context.Context, message interface{}) error {
msg, ok := message.(*messaging.Message)
if !ok {
return ErrMessage
}
topic := msg.GetChannel()
if msg.GetSubtopic() != "" {
topic = fmt.Sprintf("%s.%s", msg.GetChannel(), msg.GetSubtopic())
}
pm := PageMetadata{
Topic: topic,
Offset: 0,
Limit: -1,
}
page, err := ns.subs.RetrieveAll(ctx, pm)
if err != nil {
return err
}
var to []string
for _, sub := range page.Subscriptions {
to = append(to, sub.Contact)
}
if len(to) > 0 {
err := ns.notifier.Notify(ns.from, to, msg)
if err != nil {
return errors.Wrap(ErrNotify, err)
}
}
return nil
}
func (ns *notifierService) ConsumeAsync(ctx context.Context, message interface{}) {
msg, ok := message.(*messaging.Message)
if !ok {
ns.errCh <- ErrMessage
return
}
topic := msg.GetChannel()
if msg.GetSubtopic() != "" {
topic = fmt.Sprintf("%s.%s", msg.GetChannel(), msg.GetSubtopic())
}
pm := PageMetadata{
Topic: topic,
Offset: 0,
Limit: -1,
}
page, err := ns.subs.RetrieveAll(ctx, pm)
if err != nil {
ns.errCh <- err
return
}
var to []string
for _, sub := range page.Subscriptions {
to = append(to, sub.Contact)
}
if len(to) > 0 {
if err := ns.notifier.Notify(ns.from, to, msg); err != nil {
ns.errCh <- errors.Wrap(ErrNotify, err)
}
}
}
func (ns *notifierService) Errors() <-chan error {
return ns.errCh
}
consumers/notifiers/service_test.go
-359
View File
@@ -1,359 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package notifiers_test
import (
"context"
"fmt"
"testing"
"github.com/absmach/supermq/consumers/notifiers"
"github.com/absmach/supermq/consumers/notifiers/mocks"
"github.com/absmach/supermq/internal/testsutil"
smqauthn "github.com/absmach/supermq/pkg/authn"
authnmocks "github.com/absmach/supermq/pkg/authn/mocks"
"github.com/absmach/supermq/pkg/errors"
repoerr "github.com/absmach/supermq/pkg/errors/repository"
svcerr "github.com/absmach/supermq/pkg/errors/service"
"github.com/absmach/supermq/pkg/messaging"
"github.com/absmach/supermq/pkg/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
const (
total = 100
exampleUser1 = "token1"
exampleUser2 = "token2"
validID = "d4ebb847-5d0e-4e46-bdd9-b6aceaaa3a22"
)
func newService() (notifiers.Service, *authnmocks.Authentication, *mocks.SubscriptionsRepository) {
repo := new(mocks.SubscriptionsRepository)
auth := new(authnmocks.Authentication)
notifier := new(mocks.Notifier)
idp := uuid.NewMock()
from := "exampleFrom"
return notifiers.New(auth, repo, idp, notifier, from), auth, repo
}
func TestCreateSubscription(t *testing.T) {
svc, auth, repo := newService()
cases := []struct {
desc string
token string
sub notifiers.Subscription
id string
err error
authenticateErr error
userID string
}{
{
desc: "test success",
token: exampleUser1,
sub: notifiers.Subscription{Contact: exampleUser1, Topic: "valid.topic"},
id: uuid.Prefix + fmt.Sprintf("%012d", 1),
err: nil,
authenticateErr: nil,
userID: validID,
},
{
desc: "test already existing",
token: exampleUser1,
sub: notifiers.Subscription{Contact: exampleUser1, Topic: "valid.topic"},
id: "",
err: repoerr.ErrConflict,
authenticateErr: nil,
userID: validID,
},
{
desc: "test with empty token",
token: "",
sub: notifiers.Subscription{Contact: exampleUser1, Topic: "valid.topic"},
id: "",
err: svcerr.ErrAuthentication,
authenticateErr: svcerr.ErrAuthentication,
},
}
for _, tc := range cases {
repoCall := auth.On("Authenticate", context.Background(), tc.token).Return(smqauthn.Session{UserID: tc.userID}, tc.authenticateErr)
repoCall1 := repo.On("Save", context.Background(), mock.Anything).Return(tc.id, tc.err)
id, err := svc.CreateSubscription(context.Background(), tc.token, tc.sub)
assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err))
assert.Equal(t, tc.id, id, fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.id, id))
repoCall.Unset()
repoCall1.Unset()
}
}
func TestViewSubscription(t *testing.T) {
svc, auth, repo := newService()
sub := notifiers.Subscription{
Contact: exampleUser1,
Topic: "valid.topic",
ID: testsutil.GenerateUUID(t),
OwnerID: validID,
}
cases := []struct {
desc string
token string
id string
sub notifiers.Subscription
err error
authenticateErr error
userID string
}{
{
desc: "test success",
token: exampleUser1,
id: validID,
sub: sub,
err: nil,
authenticateErr: nil,
userID: validID,
},
{
desc: "test not existing",
token: exampleUser1,
id: "not_exist",
sub: notifiers.Subscription{},
err: svcerr.ErrNotFound,
authenticateErr: nil,
userID: validID,
},
{
desc: "test with empty token",
token: "",
id: validID,
sub: notifiers.Subscription{},
err: svcerr.ErrAuthentication,
authenticateErr: svcerr.ErrAuthentication,
},
}
for _, tc := range cases {
repoCall := auth.On("Authenticate", context.Background(), tc.token).Return(smqauthn.Session{UserID: tc.userID}, tc.authenticateErr)
repoCall1 := repo.On("Retrieve", context.Background(), tc.id).Return(tc.sub, tc.err)
sub, err := svc.ViewSubscription(context.Background(), tc.token, tc.id)
assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err))
assert.Equal(t, tc.sub, sub, fmt.Sprintf("%s: expected %v got %v\n", tc.desc, tc.sub, sub))
repoCall.Unset()
repoCall1.Unset()
}
}
func TestListSubscriptions(t *testing.T) {
svc, auth, repo := newService()
sub := notifiers.Subscription{Contact: exampleUser1, OwnerID: exampleUser1}
topic := "topic.subtopic"
var subs []notifiers.Subscription
for i := 0; i < total; i++ {
tmp := sub
if i%2 == 0 {
tmp.Contact = exampleUser2
tmp.OwnerID = exampleUser2
}
tmp.Topic = fmt.Sprintf("%s.%d", topic, i)
tmp.ID = testsutil.GenerateUUID(t)
tmp.OwnerID = validID
subs = append(subs, tmp)
}
var offsetSubs []notifiers.Subscription
for i := 20; i < 40; i += 2 {
offsetSubs = append(offsetSubs, subs[i])
}
cases := []struct {
desc string
token string
pageMeta notifiers.PageMetadata
page notifiers.Page
err error
authenticateErr error
userID string
}{
{
desc: "test success",
token: exampleUser1,
pageMeta: notifiers.PageMetadata{
Offset: 0,
Limit: 3,
},
err: nil,
page: notifiers.Page{
PageMetadata: notifiers.PageMetadata{
Offset: 0,
Limit: 3,
},
Subscriptions: subs[:3],
Total: total,
},
authenticateErr: nil,
userID: validID,
},
{
desc: "test not existing",
token: exampleUser1,
pageMeta: notifiers.PageMetadata{
Limit: 10,
Contact: "empty@example.com",
},
page: notifiers.Page{},
err: svcerr.ErrNotFound,
authenticateErr: nil,
userID: validID,
},
{
desc: "test with empty token",
token: "",
pageMeta: notifiers.PageMetadata{
Offset: 2,
Limit: 12,
Topic: "topic.subtopic.13",
},
page: notifiers.Page{},
err: svcerr.ErrAuthentication,
authenticateErr: svcerr.ErrAuthentication,
},
{
desc: "test with topic",
token: exampleUser1,
pageMeta: notifiers.PageMetadata{
Limit: 10,
Topic: fmt.Sprintf("%s.%d", topic, 4),
},
page: notifiers.Page{
PageMetadata: notifiers.PageMetadata{
Limit: 10,
Topic: fmt.Sprintf("%s.%d", topic, 4),
},
Subscriptions: subs[4:5],
Total: 1,
},
err: nil,
authenticateErr: nil,
userID: validID,
},
{
desc: "test with contact and offset",
token: exampleUser1,
pageMeta: notifiers.PageMetadata{
Offset: 10,
Limit: 10,
Contact: exampleUser2,
},
page: notifiers.Page{
PageMetadata: notifiers.PageMetadata{
Offset: 10,
Limit: 10,
Contact: exampleUser2,
},
Subscriptions: offsetSubs,
Total: uint(total / 2),
},
err: nil,
authenticateErr: nil,
userID: validID,
},
}
for _, tc := range cases {
repoCall := auth.On("Authenticate", context.Background(), tc.token).Return(smqauthn.Session{UserID: tc.userID}, tc.authenticateErr)
repoCall1 := repo.On("RetrieveAll", context.Background(), tc.pageMeta).Return(tc.page, tc.err)
page, err := svc.ListSubscriptions(context.Background(), tc.token, tc.pageMeta)
assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err))
assert.Equal(t, tc.page, page, fmt.Sprintf("%s: got unexpected page\n", tc.desc))
repoCall.Unset()
repoCall1.Unset()
}
}
func TestRemoveSubscription(t *testing.T) {
svc, auth, repo := newService()
sub := notifiers.Subscription{
ID: testsutil.GenerateUUID(t),
}
cases := []struct {
desc string
token string
id string
err error
authenticateErr error
userID string
}{
{
desc: "test success",
token: exampleUser1,
id: sub.ID,
err: nil,
authenticateErr: nil,
userID: validID,
},
{
desc: "test not existing",
token: exampleUser1,
id: "not_exist",
err: svcerr.ErrNotFound,
authenticateErr: nil,
userID: validID,
},
{
desc: "test with empty token",
token: "",
id: sub.ID,
err: svcerr.ErrAuthentication,
authenticateErr: svcerr.ErrAuthentication,
},
}
for _, tc := range cases {
repoCall := auth.On("Authenticate", context.Background(), tc.token).Return(smqauthn.Session{UserID: tc.userID}, tc.authenticateErr)
repoCall1 := repo.On("Remove", context.Background(), tc.id).Return(tc.err)
err := svc.RemoveSubscription(context.Background(), tc.token, tc.id)
assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err))
repoCall.Unset()
repoCall1.Unset()
}
}
func TestConsume(t *testing.T) {
svc, _, repo := newService()
msg := messaging.Message{
Channel: "topic",
Subtopic: "subtopic",
}
errMsg := messaging.Message{
Channel: "topic",
Subtopic: "subtopic-2",
}
cases := []struct {
desc string
msg *messaging.Message
err error
}{
{
desc: "test success",
msg: &msg,
err: nil,
},
{
desc: "test fail",
msg: &errMsg,
err: notifiers.ErrNotify,
},
}
for _, tc := range cases {
repoCall := repo.On("RetrieveAll", context.TODO(), mock.Anything).Return(notifiers.Page{}, tc.err)
err := svc.ConsumeBlocking(context.TODO(), tc.msg)
assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err))
repoCall.Unset()
}
}
consumers/notifiers/smtp/notifier.go
-40
View File
@@ -1,40 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package smtp
import (
"fmt"
"github.com/absmach/supermq/consumers/notifiers"
"github.com/absmach/supermq/internal/email"
"github.com/absmach/supermq/pkg/messaging"
)
const (
footer = "Sent by SuperMQ SMTP Notification"
contentTemplate = "A publisher with an id %s sent the message over %s with the following values \n %s"
)
var _ notifiers.Notifier = (*notifier)(nil)
type notifier struct {
agent *email.Agent
}
// New instantiates SMTP message notifier.
func New(agent *email.Agent) notifiers.Notifier {
return &notifier{agent: agent}
}
func (n *notifier) Notify(from string, to []string, msg *messaging.Message) error {
subject := fmt.Sprintf(`Notification for Channel %s`, msg.GetChannel())
if msg.GetSubtopic() != "" {
subject = fmt.Sprintf("%s and subtopic %s", subject, msg.GetSubtopic())
}
values := string(msg.GetPayload())
content := fmt.Sprintf(contentTemplate, msg.GetPublisher(), msg.GetProtocol(), values)
return n.agent.Send(to, from, subject, "", "", content, footer)
}
consumers/notifiers/subscriptions.go
-48
View File
@@ -1,48 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package notifiers
import "context"
// Subscription represents a user Subscription.
type Subscription struct {
ID string
OwnerID string
Contact string
Topic string
}
// Page represents page metadata with content.
type Page struct {
PageMetadata
Total uint
Subscriptions []Subscription
}
// PageMetadata contains page metadata that helps navigation.
type PageMetadata struct {
Offset uint
// Limit values less than 0 indicate no limit.
Limit int
Topic string
Contact string
}
// SubscriptionsRepository specifies a Subscription persistence API.
//
//go:generate mockery --name SubscriptionsRepository --output=./mocks --filename repository.go --quiet --note "Copyright (c) Abstract Machines"
type SubscriptionsRepository interface {
// Save persists a subscription. Successful operation is indicated by non-nil
// error response.
Save(ctx context.Context, sub Subscription) (string, error)
// Retrieve retrieves the subscription for the given id.
Retrieve(ctx context.Context, id string) (Subscription, error)
// RetrieveAll retrieves all the subscriptions for the given page metadata.
RetrieveAll(ctx context.Context, pm PageMetadata) (Page, error)
// Remove removes the subscription for the given ID.
Remove(ctx context.Context, id string) error
}
consumers/notifiers/tracing/doc.go
-12
View File
@@ -1,12 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package tracing provides tracing instrumentation for SuperMQ WebSocket adapter service.
//
// This package provides tracing middleware for SuperMQ WebSocket adapter service.
// It can be used to trace incoming requests and add tracing capabilities to
// SuperMQ WebSocket adapter service.
//
// For more details about tracing instrumentation for SuperMQ messaging refer
// to the documentation at https://docs.supermq.abstractmachines.fr/tracing/.
package tracing
consumers/notifiers/tracing/subscriptions.go
-73
View File
@@ -1,73 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package tracing contains middlewares that will add spans
// to existing traces.
package tracing
import (
"context"
"github.com/absmach/supermq/consumers/notifiers"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
const (
saveOp = "save_op"
retrieveOp = "retrieve_op"
retrieveAllOp = "retrieve_all_op"
removeOp = "remove_op"
)
var _ notifiers.SubscriptionsRepository = (*subRepositoryMiddleware)(nil)
type subRepositoryMiddleware struct {
tracer trace.Tracer
repo notifiers.SubscriptionsRepository
}
// New instantiates a new Subscriptions repository that
// tracks request and their latency, and adds spans to context.
func New(tracer trace.Tracer, repo notifiers.SubscriptionsRepository) notifiers.SubscriptionsRepository {
return subRepositoryMiddleware{
tracer: tracer,
repo: repo,
}
}
// Save traces the "Save" operation of the wrapped Subscriptions repository.
func (urm subRepositoryMiddleware) Save(ctx context.Context, sub notifiers.Subscription) (string, error) {
ctx, span := urm.tracer.Start(ctx, saveOp, trace.WithAttributes(
attribute.String("id", sub.ID),
attribute.String("contact", sub.Contact),
attribute.String("topic", sub.Topic),
))
defer span.End()
return urm.repo.Save(ctx, sub)
}
// Retrieve traces the "Retrieve" operation of the wrapped Subscriptions repository.
func (urm subRepositoryMiddleware) Retrieve(ctx context.Context, id string) (notifiers.Subscription, error) {
ctx, span := urm.tracer.Start(ctx, retrieveOp, trace.WithAttributes(attribute.String("id", id)))
defer span.End()
return urm.repo.Retrieve(ctx, id)
}
// RetrieveAll traces the "RetrieveAll" operation of the wrapped Subscriptions repository.
func (urm subRepositoryMiddleware) RetrieveAll(ctx context.Context, pm notifiers.PageMetadata) (notifiers.Page, error) {
ctx, span := urm.tracer.Start(ctx, retrieveAllOp)
defer span.End()
return urm.repo.RetrieveAll(ctx, pm)
}
// Remove traces the "Remove" operation of the wrapped Subscriptions repository.
func (urm subRepositoryMiddleware) Remove(ctx context.Context, id string) error {
ctx, span := urm.tracer.Start(ctx, removeOp, trace.WithAttributes(attribute.String("id", id)))
defer span.End()
return urm.repo.Remove(ctx, id)
}
consumers/tracing/consumers.go
-132
View File
@@ -1,132 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package tracing
import (
"context"
"fmt"
"github.com/absmach/supermq/consumers"
"github.com/absmach/supermq/pkg/server"
smqjson "github.com/absmach/supermq/pkg/transformers/json"
"github.com/absmach/supermq/pkg/transformers/senml"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
const (
consumeBlockingOP = "retrieve_blocking" // This is not specified in the open telemetry spec.
consumeAsyncOP = "retrieve_async" // This is not specified in the open telemetry spec.
)
var defaultAttributes = []attribute.KeyValue{
attribute.String("messaging.system", "nats"),
attribute.Bool("messaging.destination.anonymous", false),
attribute.String("messaging.destination.template", "channels/{channelID}/messages/*"),
attribute.Bool("messaging.destination.temporary", true),
attribute.String("network.protocol.name", "nats"),
attribute.String("network.protocol.version", "2.2.4"),
attribute.String("network.transport", "tcp"),
attribute.String("network.type", "ipv4"),
}
var (
_ consumers.AsyncConsumer = (*tracingMiddlewareAsync)(nil)
_ consumers.BlockingConsumer = (*tracingMiddlewareBlock)(nil)
)
type tracingMiddlewareAsync struct {
consumer consumers.AsyncConsumer
tracer trace.Tracer
host server.Config
}
type tracingMiddlewareBlock struct {
consumer consumers.BlockingConsumer
tracer trace.Tracer
host server.Config
}
// NewAsync creates a new traced consumers.AsyncConsumer service.
func NewAsync(tracer trace.Tracer, consumerAsync consumers.AsyncConsumer, host server.Config) consumers.AsyncConsumer {
return &tracingMiddlewareAsync{
consumer: consumerAsync,
tracer: tracer,
host: host,
}
}
// NewBlocking creates a new traced consumers.BlockingConsumer service.
func NewBlocking(tracer trace.Tracer, consumerBlock consumers.BlockingConsumer, host server.Config) consumers.BlockingConsumer {
return &tracingMiddlewareBlock{
consumer: consumerBlock,
tracer: tracer,
host: host,
}
}
// ConsumeBlocking traces consume operations for message/s consumed.
func (tm *tracingMiddlewareBlock) ConsumeBlocking(ctx context.Context, messages interface{}) error {
var span trace.Span
switch m := messages.(type) {
case smqjson.Messages:
if len(m.Data) > 0 {
firstMsg := m.Data[0]
ctx, span = createSpan(ctx, consumeBlockingOP, firstMsg.Publisher, firstMsg.Channel, firstMsg.Subtopic, len(m.Data), tm.host, trace.SpanKindConsumer, tm.tracer)
defer span.End()
}
case []senml.Message:
if len(m) > 0 {
firstMsg := m[0]
ctx, span = createSpan(ctx, consumeBlockingOP, firstMsg.Publisher, firstMsg.Channel, firstMsg.Subtopic, len(m), tm.host, trace.SpanKindConsumer, tm.tracer)
defer span.End()
}
}
return tm.consumer.ConsumeBlocking(ctx, messages)
}
// ConsumeAsync traces consume operations for message/s consumed.
func (tm *tracingMiddlewareAsync) ConsumeAsync(ctx context.Context, messages interface{}) {
var span trace.Span
switch m := messages.(type) {
case smqjson.Messages:
if len(m.Data) > 0 {
firstMsg := m.Data[0]
ctx, span = createSpan(ctx, consumeAsyncOP, firstMsg.Publisher, firstMsg.Channel, firstMsg.Subtopic, len(m.Data), tm.host, trace.SpanKindConsumer, tm.tracer)
defer span.End()
}
case []senml.Message:
if len(m) > 0 {
firstMsg := m[0]
ctx, span = createSpan(ctx, consumeAsyncOP, firstMsg.Publisher, firstMsg.Channel, firstMsg.Subtopic, len(m), tm.host, trace.SpanKindConsumer, tm.tracer)
defer span.End()
}
}
tm.consumer.ConsumeAsync(ctx, messages)
}
// Errors traces async consume errors.
func (tm *tracingMiddlewareAsync) Errors() <-chan error {
return tm.consumer.Errors()
}
func createSpan(ctx context.Context, operation, clientID, topic, subTopic string, noMessages int, cfg server.Config, spanKind trace.SpanKind, tracer trace.Tracer) (context.Context, trace.Span) {
subject := fmt.Sprintf("channels.%s.messages", topic)
if subTopic != "" {
subject = fmt.Sprintf("%s.%s", subject, subTopic)
}
spanName := fmt.Sprintf("%s %s", subject, operation)
kvOpts := []attribute.KeyValue{
attribute.String("messaging.operation", operation),
attribute.String("messaging.client_id", clientID),
attribute.String("messaging.destination.name", subject),
attribute.String("server.address", cfg.Host),
attribute.String("server.socket.port", cfg.Port),
attribute.Int("messaging.batch.message_count", noMessages),
}
kvOpts = append(kvOpts, defaultAttributes...)
return tracer.Start(ctx, spanName, trace.WithAttributes(kvOpts...), trace.WithSpanKind(spanKind))
}
consumers/writers/README.md
-16
View File
@@ -1,16 +0,0 @@
# Writers
Writers provide an implementation of various `message writers`.
Message writers are services that normalize (in `SenML` format)
SuperMQ messages and store them in specific data store.
Writers are optional services and are treated as plugins. In order to
run writer services, core services must be up and running. For more info
on the platform core services with its dependencies, please check out
the [Docker Compose][compose] file.
For an in-depth explanation of the usage of `writers`, as well as thorough
understanding of SuperMQ, please check out the [official documentation][doc].
[doc]: https://docs.supermq.abstractmachines.fr
[compose]: ../docker/docker-compose.yml
consumers/writers/api/doc.go
-6
View File
@@ -1,6 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package api contains API-related concerns: endpoint definitions, middlewares
// and all resource representations.
package api
consumers/writers/api/logging.go
-47
View File
@@ -1,47 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
//go:build !test
package api
import (
"context"
"log/slog"
"time"
"github.com/absmach/supermq/consumers"
)
var _ consumers.BlockingConsumer = (*loggingMiddleware)(nil)
type loggingMiddleware struct {
logger *slog.Logger
consumer consumers.BlockingConsumer
}
// LoggingMiddleware adds logging facilities to the adapter.
func LoggingMiddleware(consumer consumers.BlockingConsumer, logger *slog.Logger) consumers.BlockingConsumer {
return &loggingMiddleware{
logger: logger,
consumer: consumer,
}
}
// ConsumeBlocking logs the consume request. It logs the time it took to complete the request.
// If the request fails, it logs the error.
func (lm *loggingMiddleware) ConsumeBlocking(ctx context.Context, msgs interface{}) (err error) {
defer func(begin time.Time) {
args := []any{
slog.String("duration", time.Since(begin).String()),
}
if err != nil {
args = append(args, slog.Any("error", err))
lm.logger.Warn("Blocking consumer failed to consume messages successfully", args...)
return
}
lm.logger.Info("Blocking consumer consumed messages successfully", args...)
}(time.Now())
return lm.consumer.ConsumeBlocking(ctx, msgs)
}
consumers/writers/api/metrics.go
-41
View File
@@ -1,41 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
//go:build !test
package api
import (
"context"
"time"
"github.com/absmach/supermq/consumers"
"github.com/go-kit/kit/metrics"
)
var _ consumers.BlockingConsumer = (*metricsMiddleware)(nil)
type metricsMiddleware struct {
counter metrics.Counter
latency metrics.Histogram
consumer consumers.BlockingConsumer
}
// MetricsMiddleware returns new message repository
// with Save method wrapped to expose metrics.
func MetricsMiddleware(consumer consumers.BlockingConsumer, counter metrics.Counter, latency metrics.Histogram) consumers.BlockingConsumer {
return &metricsMiddleware{
counter: counter,
latency: latency,
consumer: consumer,
}
}
// ConsumeBlocking instruments ConsumeBlocking method with metrics.
func (mm *metricsMiddleware) ConsumeBlocking(ctx context.Context, msgs interface{}) error {
defer func(begin time.Time) {
mm.counter.With("method", "consume").Add(1)
mm.latency.With("method", "consume").Observe(time.Since(begin).Seconds())
}(time.Now())
return mm.consumer.ConsumeBlocking(ctx, msgs)
}
consumers/writers/api/transport.go
-21
View File
@@ -1,21 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api
import (
"net/http"
"github.com/absmach/supermq"
"github.com/go-chi/chi/v5"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
// MakeHandler returns a HTTP API handler with health check and metrics.
func MakeHandler(svcName, instanceID string) http.Handler {
r := chi.NewRouter()
r.Get("/health", supermq.Health(svcName, instanceID))
r.Handle("/metrics", promhttp.Handler())
return r
}
consumers/writers/doc.go
-6
View File
@@ -1,6 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package writers contain the domain concept definitions needed to
// support SuperMQ writer services functionality.
package writers
consumers/writers/postgres/README.md
-77
View File
@@ -1,77 +0,0 @@
# Postgres writer
Postgres writer provides message repository implementation for Postgres.
## Configuration
The service is configured using the environment variables presented in the
following table. Note that any unset variables will be replaced with their
default values.
| Variable | Description | Default |
| ------------------------------------ | --------------------------------------------------------------------------------- | ---------------------------- |
| SMQ_POSTGRES_WRITER_LOG_LEVEL | Service log level | info |
| SMQ_POSTGRES_WRITER_CONFIG_PATH | Config file path with Message broker subjects list, payload type and content-type | /config.toml |
| SMQ_POSTGRES_WRITER_HTTP_HOST | Service HTTP host | localhost |
| SMQ_POSTGRES_WRITER_HTTP_PORT | Service HTTP port | 9010 |
| SMQ_POSTGRES_WRITER_HTTP_SERVER_CERT | Service HTTP server certificate path | "" |
| SMQ_POSTGRES_WRITER_HTTP_SERVER_KEY | Service HTTP server key | "" |
| SMQ_POSTGRES_HOST | Postgres DB host | postgres |
| SMQ_POSTGRES_PORT | Postgres DB port | 5432 |
| SMQ_POSTGRES_USER | Postgres user | supermq |
| SMQ_POSTGRES_PASS | Postgres password | supermq |
| SMQ_POSTGRES_NAME | Postgres database name | messages |
| SMQ_POSTGRES_SSL_MODE | Postgres SSL mode | disabled |
| SMQ_POSTGRES_SSL_CERT | Postgres SSL certificate path | "" |
| SMQ_POSTGRES_SSL_KEY | Postgres SSL key | "" |
| SMQ_POSTGRES_SSL_ROOT_CERT | Postgres SSL root certificate path | "" |
| SMQ_MESSAGE_BROKER_URL | Message broker instance URL | nats://localhost:4222 |
| SMQ_JAEGER_URL | Jaeger server URL | http://jaeger:4318/v1/traces |
| SMQ_SEND_TELEMETRY | Send telemetry to supermq call home server | true |
| SMQ_POSTGRES_WRITER_INSTANCE_ID | Service instance ID | "" |
## Deployment
The service itself is distributed as Docker container. Check the [`postgres-writer`](https://github.com/absmach/supermq/blob/main/docker/addons/postgres-writer/docker-compose.yml#L34-L59) service section in docker-compose file to see how service is deployed.
To start the service, execute the following shell script:
```bash
# download the latest version of the service
git clone https://github.com/absmach/supermq
cd supermq
# compile the postgres writer
make postgres-writer
# copy binary to bin
make install
# Set the environment variables and run the service
SMQ_POSTGRES_WRITER_LOG_LEVEL=[Service log level] \
SMQ_POSTGRES_WRITER_CONFIG_PATH=[Config file path with Message broker subjects list, payload type and content-type] \
SMQ_POSTGRES_WRITER_HTTP_HOST=[Service HTTP host] \
SMQ_POSTGRES_WRITER_HTTP_PORT=[Service HTTP port] \
SMQ_POSTGRES_WRITER_HTTP_SERVER_CERT=[Service HTTP server cert] \
SMQ_POSTGRES_WRITER_HTTP_SERVER_KEY=[Service HTTP server key] \
SMQ_POSTGRES_HOST=[Postgres host] \
SMQ_POSTGRES_PORT=[Postgres port] \
SMQ_POSTGRES_USER=[Postgres user] \
SMQ_POSTGRES_PASS=[Postgres password] \
SMQ_POSTGRES_NAME=[Postgres database name] \
SMQ_POSTGRES_SSL_MODE=[Postgres SSL mode] \
SMQ_POSTGRES_SSL_CERT=[Postgres SSL cert] \
SMQ_POSTGRES_SSL_KEY=[Postgres SSL key] \
SMQ_POSTGRES_SSL_ROOT_CERT=[Postgres SSL Root cert] \
SMQ_MESSAGE_BROKER_URL=[Message broker instance URL] \
SMQ_JAEGER_URL=[Jaeger server URL] \
SMQ_SEND_TELEMETRY=[Send telemetry to supermq call home server] \
SMQ_POSTGRES_WRITER_INSTANCE_ID=[Service instance ID] \
$GOBIN/supermq-postgres-writer
```
## Usage
Starting service will start consuming normalized messages in SenML format.
consumers/writers/postgres/consumer.go
-213
View File
@@ -1,213 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package postgres
import (
"context"
"encoding/json"
"fmt"
"github.com/absmach/supermq/consumers"
"github.com/absmach/supermq/pkg/errors"
smqjson "github.com/absmach/supermq/pkg/transformers/json"
"github.com/absmach/supermq/pkg/transformers/senml"
"github.com/gofrs/uuid/v5"
"github.com/jackc/pgerrcode"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jmoiron/sqlx" // required for DB access
)
var (
errInvalidMessage = errors.New("invalid message representation")
errSaveMessage = errors.New("failed to save message to postgres database")
errTransRollback = errors.New("failed to rollback transaction")
errNoTable = errors.New("relation does not exist")
)
var _ consumers.BlockingConsumer = (*postgresRepo)(nil)
type postgresRepo struct {
db *sqlx.DB
}
// New returns new PostgreSQL writer.
func New(db *sqlx.DB) consumers.BlockingConsumer {
return &postgresRepo{db: db}
}
func (pr postgresRepo) ConsumeBlocking(ctx context.Context, message interface{}) (err error) {
switch m := message.(type) {
case smqjson.Messages:
return pr.saveJSON(ctx, m)
default:
return pr.saveSenml(ctx, m)
}
}
func (pr postgresRepo) saveSenml(ctx context.Context, messages interface{}) (err error) {
msgs, ok := messages.([]senml.Message)
if !ok {
return errSaveMessage
}
q := `INSERT INTO messages (id, channel, subtopic, publisher, protocol,
name, unit, value, string_value, bool_value, data_value, sum,
time, update_time)
VALUES (:id, :channel, :subtopic, :publisher, :protocol, :name, :unit,
:value, :string_value, :bool_value, :data_value, :sum,
:time, :update_time);`
tx, err := pr.db.BeginTxx(ctx, nil)
if err != nil {
return errors.Wrap(errSaveMessage, err)
}
defer func() {
if err != nil {
if txErr := tx.Rollback(); txErr != nil {
err = errors.Wrap(err, errors.Wrap(errTransRollback, txErr))
}
return
}
if err = tx.Commit(); err != nil {
err = errors.Wrap(errSaveMessage, err)
}
}()
for _, msg := range msgs {
id, err := uuid.NewV4()
if err != nil {
return err
}
m := senmlMessage{Message: msg, ID: id.String()}
if _, err := tx.NamedExec(q, m); err != nil {
pgErr, ok := err.(*pgconn.PgError)
if ok {
if pgErr.Code == pgerrcode.InvalidTextRepresentation {
return errors.Wrap(errSaveMessage, errInvalidMessage)
}
}
return errors.Wrap(errSaveMessage, err)
}
}
return err
}
func (pr postgresRepo) saveJSON(ctx context.Context, msgs smqjson.Messages) error {
if err := pr.insertJSON(ctx, msgs); err != nil {
if err == errNoTable {
if err := pr.createTable(msgs.Format); err != nil {
return err
}
return pr.insertJSON(ctx, msgs)
}
return err
}
return nil
}
func (pr postgresRepo) insertJSON(ctx context.Context, msgs smqjson.Messages) error {
tx, err := pr.db.BeginTxx(ctx, nil)
if err != nil {
return errors.Wrap(errSaveMessage, err)
}
defer func() {
if err != nil {
if txErr := tx.Rollback(); txErr != nil {
err = errors.Wrap(err, errors.Wrap(errTransRollback, txErr))
}
return
}
if err = tx.Commit(); err != nil {
err = errors.Wrap(errSaveMessage, err)
}
}()
q := `INSERT INTO %s (id, channel, created, subtopic, publisher, protocol, payload)
VALUES (:id, :channel, :created, :subtopic, :publisher, :protocol, :payload);`
q = fmt.Sprintf(q, msgs.Format)
for _, m := range msgs.Data {
var dbmsg jsonMessage
dbmsg, err = toJSONMessage(m)
if err != nil {
return errors.Wrap(errSaveMessage, err)
}
if _, err = tx.NamedExec(q, dbmsg); err != nil {
pgErr, ok := err.(*pgconn.PgError)
if ok {
switch pgErr.Code {
case pgerrcode.InvalidTextRepresentation:
return errors.Wrap(errSaveMessage, errInvalidMessage)
case pgerrcode.UndefinedTable:
return errNoTable
}
}
return err
}
}
return nil
}
func (pr postgresRepo) createTable(name string) error {
q := `CREATE TABLE IF NOT EXISTS %s (
id UUID,
created BIGINT,
channel VARCHAR(254),
subtopic VARCHAR(254),
publisher VARCHAR(254),
protocol TEXT,
payload JSONB,
PRIMARY KEY (id)
)`
q = fmt.Sprintf(q, name)
_, err := pr.db.Exec(q)
return err
}
type senmlMessage struct {
senml.Message
ID string `db:"id"`
}
type jsonMessage struct {
ID string `db:"id"`
Channel string `db:"channel"`
Created int64 `db:"created"`
Subtopic string `db:"subtopic"`
Publisher string `db:"publisher"`
Protocol string `db:"protocol"`
Payload []byte `db:"payload"`
}
func toJSONMessage(msg smqjson.Message) (jsonMessage, error) {
id, err := uuid.NewV4()
if err != nil {
return jsonMessage{}, err
}
data := []byte("{}")
if msg.Payload != nil {
b, err := json.Marshal(msg.Payload)
if err != nil {
return jsonMessage{}, errors.Wrap(errSaveMessage, err)
}
data = b
}
m := jsonMessage{
ID: id.String(),
Channel: msg.Channel,
Created: msg.Created,
Subtopic: msg.Subtopic,
Publisher: msg.Publisher,
Protocol: msg.Protocol,
Payload: data,
}
return m, nil
}
consumers/writers/postgres/consumer_test.go
-112
View File
@@ -1,112 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package postgres_test
import (
"context"
"fmt"
"testing"
"time"
"github.com/absmach/supermq/consumers/writers/postgres"
"github.com/absmach/supermq/pkg/transformers/json"
"github.com/absmach/supermq/pkg/transformers/senml"
"github.com/gofrs/uuid/v5"
"github.com/stretchr/testify/assert"
)
const (
msgsNum = 42
valueFields = 5
subtopic = "topic"
)
var (
v float64 = 5
stringV = "value"
boolV = true
dataV = "base64"
sum float64 = 42
)
func TestSaveSenml(t *testing.T) {
repo := postgres.New(db)
chid, err := uuid.NewV4()
assert.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err))
msg := senml.Message{}
msg.Channel = chid.String()
pubid, err := uuid.NewV4()
assert.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err))
msg.Publisher = pubid.String()
now := time.Now().Unix()
var msgs []senml.Message
for i := 0; i < msgsNum; i++ {
// Mix possible values as well as value sum.
count := i % valueFields
switch count {
case 0:
msg.Subtopic = subtopic
msg.Value = &v
case 1:
msg.BoolValue = &boolV
case 2:
msg.StringValue = &stringV
case 3:
msg.DataValue = &dataV
case 4:
msg.Sum = &sum
}
msg.Time = float64(now + int64(i))
msgs = append(msgs, msg)
}
err = repo.ConsumeBlocking(context.TODO(), msgs)
assert.Nil(t, err, fmt.Sprintf("expected no error got %s\n", err))
}
func TestSaveJSON(t *testing.T) {
repo := postgres.New(db)
chid, err := uuid.NewV4()
assert.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err))
pubid, err := uuid.NewV4()
assert.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err))
msg := json.Message{
Channel: chid.String(),
Publisher: pubid.String(),
Created: time.Now().Unix(),
Subtopic: "subtopic/format/some_json",
Protocol: "mqtt",
Payload: map[string]interface{}{
"field_1": 123,
"field_2": "value",
"field_3": false,
"field_4": 12.344,
"field_5": map[string]interface{}{
"field_1": "value",
"field_2": 42,
},
},
}
now := time.Now().Unix()
msgs := json.Messages{
Format: "some_json",
}
for i := 0; i < msgsNum; i++ {
msg.Created = now + int64(i)
msgs.Data = append(msgs.Data, msg)
}
err = repo.ConsumeBlocking(context.TODO(), msgs)
assert.Nil(t, err, fmt.Sprintf("expected no error got %s\n", err))
}
consumers/writers/postgres/doc.go
-6
View File
@@ -1,6 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package postgres contains repository implementations using Postgres as
// the underlying database.
package postgres
consumers/writers/postgres/init.go
-46
View File
@@ -1,46 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package postgres
import migrate "github.com/rubenv/sql-migrate"
// Migration of postgres-writer.
func Migration() *migrate.MemoryMigrationSource {
return &migrate.MemoryMigrationSource{
Migrations: []*migrate.Migration{
{
Id: "messages_1",
Up: []string{
`CREATE TABLE IF NOT EXISTS messages (
id UUID,
channel UUID,
subtopic VARCHAR(254),
publisher UUID,
protocol TEXT,
name TEXT,
unit TEXT,
value FLOAT,
string_value TEXT,
bool_value BOOL,
data_value BYTEA,
sum FLOAT,
time FLOAT,
update_time FLOAT,
PRIMARY KEY (id)
)`,
},
Down: []string{
"DROP TABLE messages",
},
},
{
Id: "messages_2",
Up: []string{
`ALTER TABLE messages DROP CONSTRAINT messages_pkey`,
`ALTER TABLE messages ADD PRIMARY KEY (time, publisher, subtopic, name)`,
},
},
},
}
}
consumers/writers/postgres/setup_test.go
-85
View File
@@ -1,85 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package postgres_test contains tests for PostgreSQL repository
// implementations.
package postgres_test
import (
"fmt"
"log"
"os"
"testing"
"github.com/absmach/supermq/consumers/writers/postgres"
pgclient "github.com/absmach/supermq/pkg/postgres"
"github.com/jmoiron/sqlx"
"github.com/ory/dockertest/v3"
"github.com/ory/dockertest/v3/docker"
)
var db *sqlx.DB
func TestMain(m *testing.M) {
pool, err := dockertest.NewPool("")
if err != nil {
log.Fatalf("Could not connect to docker: %s", err)
}
container, err := pool.RunWithOptions(&dockertest.RunOptions{
Repository: "postgres",
Tag: "16.2-alpine",
Env: []string{
"POSTGRES_USER=test",
"POSTGRES_PASSWORD=test",
"POSTGRES_DB=test",
"listen_addresses = '*'",
},
}, func(config *docker.HostConfig) {
config.AutoRemove = true
config.RestartPolicy = docker.RestartPolicy{Name: "no"}
})
if err != nil {
log.Fatalf("Could not start container: %s", err)
}
port := container.GetPort("5432/tcp")
if err := pool.Retry(func() error {
url := fmt.Sprintf("host=localhost port=%s user=test dbname=test password=test sslmode=disable", port)
db, err = sqlx.Open("pgx", url)
if err != nil {
return err
}
return db.Ping()
}); err != nil {
log.Fatalf("Could not connect to docker: %s", err)
}
dbConfig := pgclient.Config{
Host: "localhost",
Port: port,
User: "test",
Pass: "test",
Name: "test",
SSLMode: "disable",
SSLCert: "",
SSLKey: "",
SSLRootCert: "",
}
db, err = pgclient.Setup(dbConfig, *postgres.Migration())
if err != nil {
log.Fatalf("Could not setup test DB connection: %s", err)
}
code := m.Run()
// Defers will not be run when using os.Exit
db.Close()
if err := pool.Purge(container); err != nil {
log.Fatalf("Could not purge container: %s", err)
}
os.Exit(code)
}
consumers/writers/timescale/README.md
-76
View File
@@ -1,76 +0,0 @@
# Timescale writer
Timescale writer provides message repository implementation for Timescale.
## Configuration
The service is configured using the environment variables presented in the
following table. Note that any unset variables will be replaced with their
default values.
| Variable | Description | Default |
| ------------------------------------- | --------------------------------------------------------- | ---------------------------- |
| SMQ_TIMESCALE_WRITER_LOG_LEVEL | Service log level | info |
| SMQ_TIMESCALE_WRITER_CONFIG_PATH | Configuration file path with Message broker subjects list | /config.toml |
| SMQ_TIMESCALE_WRITER_HTTP_HOST | Service HTTP host | localhost |
| SMQ_TIMESCALE_WRITER_HTTP_PORT | Service HTTP port | 9012 |
| SMQ_TIMESCALE_WRITER_HTTP_SERVER_CERT | Service HTTP server certificate path | "" |
| SMQ_TIMESCALE_WRITER_HTTP_SERVER_KEY | Service HTTP server key | "" |
| SMQ_TIMESCALE_HOST | Timescale DB host | timescale |
| SMQ_TIMESCALE_PORT | Timescale DB port | 5432 |
| SMQ_TIMESCALE_USER | Timescale user | supermq |
| SMQ_TIMESCALE_PASS | Timescale password | supermq |
| SMQ_TIMESCALE_NAME | Timescale database name | messages |
| SMQ_TIMESCALE_SSL_MODE | Timescale SSL mode | disabled |
| SMQ_TIMESCALE_SSL_CERT | Timescale SSL certificate path | "" |
| SMQ_TIMESCALE_SSL_KEY | Timescale SSL key | "" |
| SMQ_TIMESCALE_SSL_ROOT_CERT | Timescale SSL root certificate path | "" |
| SMQ_MESSAGE_BROKER_URL | Message broker instance URL | nats://localhost:4222 |
| SMQ_JAEGER_URL | Jaeger server URL | http://jaeger:4318/v1/traces |
| SMQ_SEND_TELEMETRY | Send telemetry to supermq call home server | true |
| SMQ_TIMESCALE_WRITER_INSTANCE_ID | Timescale writer instance ID | "" |
## Deployment
The service itself is distributed as Docker container. Check the [`timescale-writer`](https://github.com/absmach/supermq/blob/main/docker/addons/timescale-writer/docker-compose.yml#L34-L59) service section in docker-compose file to see how service is deployed.
To start the service, execute the following shell script:
```bash
# download the latest version of the service
git clone https://github.com/absmach/supermq
cd supermq
# compile the timescale writer
make timescale-writer
# copy binary to bin
make install
# Set the environment variables and run the service
SMQ_TIMESCALE_WRITER_LOG_LEVEL=[Service log level] \
SMQ_TIMESCALE_WRITER_CONFIG_PATH=[Configuration file path with Message broker subjects list] \
SMQ_TIMESCALE_WRITER_HTTP_HOST=[Service HTTP host] \
SMQ_TIMESCALE_WRITER_HTTP_PORT=[Service HTTP port] \
SMQ_TIMESCALE_WRITER_HTTP_SERVER_CERT=[Service HTTP server cert] \
SMQ_TIMESCALE_WRITER_HTTP_SERVER_KEY=[Service HTTP server key] \
SMQ_TIMESCALE_HOST=[Timescale host] \
SMQ_TIMESCALE_PORT=[Timescale port] \
SMQ_TIMESCALE_USER=[Timescale user] \
SMQ_TIMESCALE_PASS=[Timescale password] \
SMQ_TIMESCALE_NAME=[Timescale database name] \
SMQ_TIMESCALE_SSL_MODE=[Timescale SSL mode] \
SMQ_TIMESCALE_SSL_CERT=[Timescale SSL cert] \
SMQ_TIMESCALE_SSL_KEY=[Timescale SSL key] \
SMQ_TIMESCALE_SSL_ROOT_CERT=[Timescale SSL Root cert] \
SMQ_MESSAGE_BROKER_URL=[Message broker instance URL] \
SMQ_JAEGER_URL=[Jaeger server URL] \
SMQ_SEND_TELEMETRY=[Send telemetry to supermq call home server] \
SMQ_TIMESCALE_WRITER_INSTANCE_ID=[Timescale writer instance ID] \
$GOBIN/supermq-timescale-writer
```
## Usage
Starting service will start consuming normalized messages in SenML format.
consumers/writers/timescale/consumer.go
-198
View File
@@ -1,198 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package timescale
import (
"context"
"encoding/json"
"fmt"
"github.com/absmach/supermq/consumers"
"github.com/absmach/supermq/pkg/errors"
smqjson "github.com/absmach/supermq/pkg/transformers/json"
"github.com/absmach/supermq/pkg/transformers/senml"
"github.com/jackc/pgerrcode"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jmoiron/sqlx" // required for DB access
)
var (
errInvalidMessage = errors.New("invalid message representation")
errSaveMessage = errors.New("failed to save message to timescale database")
errTransRollback = errors.New("failed to rollback transaction")
errNoTable = errors.New("relation does not exist")
)
var _ consumers.BlockingConsumer = (*timescaleRepo)(nil)
type timescaleRepo struct {
db *sqlx.DB
}
// New returns new TimescaleSQL writer.
func New(db *sqlx.DB) consumers.BlockingConsumer {
return &timescaleRepo{db: db}
}
func (tr *timescaleRepo) ConsumeBlocking(ctx context.Context, message interface{}) (err error) {
switch m := message.(type) {
case smqjson.Messages:
return tr.saveJSON(ctx, m)
default:
return tr.saveSenml(ctx, m)
}
}
func (tr timescaleRepo) saveSenml(ctx context.Context, messages interface{}) (err error) {
msgs, ok := messages.([]senml.Message)
if !ok {
return errSaveMessage
}
q := `INSERT INTO messages (channel, subtopic, publisher, protocol,
name, unit, value, string_value, bool_value, data_value, sum,
time, update_time)
VALUES (:channel, :subtopic, :publisher, :protocol, :name, :unit,
:value, :string_value, :bool_value, :data_value, :sum,
:time, :update_time);`
tx, err := tr.db.BeginTxx(ctx, nil)
if err != nil {
return errors.Wrap(errSaveMessage, err)
}
defer func() {
if err != nil {
if txErr := tx.Rollback(); txErr != nil {
err = errors.Wrap(err, errors.Wrap(errTransRollback, txErr))
}
return
}
if err = tx.Commit(); err != nil {
err = errors.Wrap(errSaveMessage, err)
}
}()
for _, msg := range msgs {
m := senmlMessage{Message: msg}
if _, err := tx.NamedExec(q, m); err != nil {
pgErr, ok := err.(*pgconn.PgError)
if ok {
if pgErr.Code == pgerrcode.InvalidTextRepresentation {
return errors.Wrap(errSaveMessage, errInvalidMessage)
}
}
return errors.Wrap(errSaveMessage, err)
}
}
return err
}
func (tr timescaleRepo) saveJSON(ctx context.Context, msgs smqjson.Messages) error {
if err := tr.insertJSON(ctx, msgs); err != nil {
if err == errNoTable {
if err := tr.createTable(msgs.Format); err != nil {
return err
}
return tr.insertJSON(ctx, msgs)
}
return err
}
return nil
}
func (tr timescaleRepo) insertJSON(ctx context.Context, msgs smqjson.Messages) error {
tx, err := tr.db.BeginTxx(ctx, nil)
if err != nil {
return errors.Wrap(errSaveMessage, err)
}
defer func() {
if err != nil {
if txErr := tx.Rollback(); txErr != nil {
err = errors.Wrap(err, errors.Wrap(errTransRollback, txErr))
}
return
}
if err = tx.Commit(); err != nil {
err = errors.Wrap(errSaveMessage, err)
}
}()
q := `INSERT INTO %s (channel, created, subtopic, publisher, protocol, payload)
VALUES (:channel, :created, :subtopic, :publisher, :protocol, :payload);`
q = fmt.Sprintf(q, msgs.Format)
for _, m := range msgs.Data {
var dbmsg jsonMessage
dbmsg, err = toJSONMessage(m)
if err != nil {
return errors.Wrap(errSaveMessage, err)
}
if _, err = tx.NamedExec(q, dbmsg); err != nil {
pgErr, ok := err.(*pgconn.PgError)
if ok {
switch pgErr.Code {
case pgerrcode.InvalidTextRepresentation:
return errors.Wrap(errSaveMessage, errInvalidMessage)
case pgerrcode.UndefinedTable:
return errNoTable
}
}
return err
}
}
return nil
}
func (tr timescaleRepo) createTable(name string) error {
q := `CREATE TABLE IF NOT EXISTS %s (
created BIGINT NOT NULL,
channel VARCHAR(254),
subtopic VARCHAR(254),
publisher VARCHAR(254),
protocol TEXT,
payload JSONB,
PRIMARY KEY (created, publisher, subtopic)
);`
q = fmt.Sprintf(q, name)
_, err := tr.db.Exec(q)
return err
}
type senmlMessage struct {
senml.Message
}
type jsonMessage struct {
Channel string `db:"channel"`
Created int64 `db:"created"`
Subtopic string `db:"subtopic"`
Publisher string `db:"publisher"`
Protocol string `db:"protocol"`
Payload []byte `db:"payload"`
}
func toJSONMessage(msg smqjson.Message) (jsonMessage, error) {
data := []byte("{}")
if msg.Payload != nil {
b, err := json.Marshal(msg.Payload)
if err != nil {
return jsonMessage{}, errors.Wrap(errSaveMessage, err)
}
data = b
}
m := jsonMessage{
Channel: msg.Channel,
Created: msg.Created,
Subtopic: msg.Subtopic,
Publisher: msg.Publisher,
Protocol: msg.Protocol,
Payload: data,
}
return m, nil
}
consumers/writers/timescale/consumer_test.go
-112
View File
@@ -1,112 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package timescale_test
import (
"context"
"fmt"
"testing"
"time"
"github.com/absmach/supermq/consumers/writers/timescale"
"github.com/absmach/supermq/pkg/transformers/json"
"github.com/absmach/supermq/pkg/transformers/senml"
"github.com/gofrs/uuid/v5"
"github.com/stretchr/testify/assert"
)
const (
msgsNum = 42
valueFields = 5
subtopic = "topic"
)
var (
v float64 = 5
stringV = "value"
boolV = true
dataV = "base64"
sum float64 = 42
)
func TestSaveSenml(t *testing.T) {
repo := timescale.New(db)
chid, err := uuid.NewV4()
assert.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err))
msg := senml.Message{}
msg.Channel = chid.String()
pubid, err := uuid.NewV4()
assert.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err))
msg.Publisher = pubid.String()
now := time.Now().Unix()
var msgs []senml.Message
for i := 0; i < msgsNum; i++ {
// Mix possible values as well as value sum.
count := i % valueFields
switch count {
case 0:
msg.Subtopic = subtopic
msg.Value = &v
case 1:
msg.BoolValue = &boolV
case 2:
msg.StringValue = &stringV
case 3:
msg.DataValue = &dataV
case 4:
msg.Sum = &sum
}
msg.Time = float64(now + int64(i))
msgs = append(msgs, msg)
}
err = repo.ConsumeBlocking(context.TODO(), msgs)
assert.Nil(t, err, fmt.Sprintf("expected no error got %s\n", err))
}
func TestSaveJSON(t *testing.T) {
repo := timescale.New(db)
chid, err := uuid.NewV4()
assert.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err))
pubid, err := uuid.NewV4()
assert.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err))
msg := json.Message{
Channel: chid.String(),
Publisher: pubid.String(),
Created: time.Now().Unix(),
Subtopic: "subtopic/format/some_json",
Protocol: "mqtt",
Payload: map[string]interface{}{
"field_1": 123,
"field_2": "value",
"field_3": false,
"field_4": 12.344,
"field_5": map[string]interface{}{
"field_1": "value",
"field_2": 42,
},
},
}
now := time.Now().Unix()
msgs := json.Messages{
Format: "some_json",
}
for i := 0; i < msgsNum; i++ {
msg.Created = now + int64(i)
msgs.Data = append(msgs.Data, msg)
}
err = repo.ConsumeBlocking(context.TODO(), msgs)
assert.Nil(t, err, fmt.Sprintf("expected no error got %s\n", err))
}
consumers/writers/timescale/doc.go
-6
View File
@@ -1,6 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package timescale contains repository implementations using Timescale as
// the underlying database.
package timescale
consumers/writers/timescale/init.go
-39
View File
@@ -1,39 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package timescale
import migrate "github.com/rubenv/sql-migrate"
// Migration of timescale-writer.
func Migration() *migrate.MemoryMigrationSource {
return &migrate.MemoryMigrationSource{
Migrations: []*migrate.Migration{
{
Id: "messages_1",
Up: []string{
`CREATE TABLE IF NOT EXISTS messages (
time BIGINT NOT NULL,
channel UUID,
subtopic VARCHAR(254),
publisher UUID,
protocol TEXT,
name VARCHAR(254),
unit TEXT,
value FLOAT,
string_value TEXT,
bool_value BOOL,
data_value BYTEA,
sum FLOAT,
update_time FLOAT,
PRIMARY KEY (time, publisher, subtopic, name)
);
SELECT create_hypertable('messages', 'time', create_default_indexes => FALSE, chunk_time_interval => 86400000, if_not_exists => TRUE);`,
},
Down: []string{
"DROP TABLE messages",
},
},
},
}
}
consumers/writers/timescale/setup_test.go
-85
View File
@@ -1,85 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package timescale_test contains tests for TimescaleSQL repository
// implementations.
package timescale_test
import (
"fmt"
"log"
"os"
"testing"
"github.com/absmach/supermq/consumers/writers/timescale"
pgclient "github.com/absmach/supermq/pkg/postgres"
"github.com/jmoiron/sqlx"
"github.com/ory/dockertest/v3"
"github.com/ory/dockertest/v3/docker"
)
var db *sqlx.DB
func TestMain(m *testing.M) {
pool, err := dockertest.NewPool("")
if err != nil {
log.Fatalf("Could not connect to docker: %s", err)
}
container, err := pool.RunWithOptions(&dockertest.RunOptions{
Repository: "timescale/timescaledb",
Tag: "2.13.1-pg16",
Env: []string{
"POSTGRES_USER=test",
"POSTGRES_PASSWORD=test",
"POSTGRES_DB=test",
"listen_addresses = '*'",
},
}, func(config *docker.HostConfig) {
config.AutoRemove = true
config.RestartPolicy = docker.RestartPolicy{Name: "no"}
})
if err != nil {
log.Fatalf("Could not start container: %s", err)
}
port := container.GetPort("5432/tcp")
if err := pool.Retry(func() error {
url := fmt.Sprintf("host=localhost port=%s user=test dbname=test password=test sslmode=disable", port)
db, err = sqlx.Open("pgx", url)
if err != nil {
return err
}
return db.Ping()
}); err != nil {
log.Fatalf("Could not connect to docker: %s", err)
}
dbConfig := pgclient.Config{
Host: "localhost",
Port: port,
User: "test",
Pass: "test",
Name: "test",
SSLMode: "disable",
SSLCert: "",
SSLKey: "",
SSLRootCert: "",
}
db, err = pgclient.Setup(dbConfig, *timescale.Migration())
if err != nil {
log.Fatalf("Could not setup test DB connection: %s", err)
}
code := m.Run()
// Defers will not be run when using os.Exit
db.Close()
if err := pool.Purge(container); err != nil {
log.Fatalf("Could not purge container: %s", err)
}
os.Exit(code)
}
docker/.env
-35
View File
@@ -170,7 +170,6 @@ SMQ_INVITATIONS_INSTANCE_ID=
SMQ_UI_LOG_LEVEL=debug
SMQ_UI_PORT=9095
SMQ_HTTP_ADAPTER_URL=http://http-adapter:8008
SMQ_READER_URL=http://timescale-reader:9011
SMQ_CLIENTS_URL=http://clients:9006
SMQ_USERS_URL=http://users:9002
SMQ_INVITATIONS_URL=http://invitations:9020
@@ -461,23 +460,6 @@ SMQ_POSTGRES_SSL_CERT=
SMQ_POSTGRES_SSL_KEY=
SMQ_POSTGRES_SSL_ROOT_CERT=
### Postgres Writer
SMQ_POSTGRES_WRITER_LOG_LEVEL=debug
SMQ_POSTGRES_WRITER_CONFIG_PATH=/config.toml
SMQ_POSTGRES_WRITER_HTTP_HOST=postgres-writer
SMQ_POSTGRES_WRITER_HTTP_PORT=9010
SMQ_POSTGRES_WRITER_HTTP_SERVER_CERT=
SMQ_POSTGRES_WRITER_HTTP_SERVER_KEY=
SMQ_POSTGRES_WRITER_INSTANCE_ID=
### Postgres Reader
SMQ_POSTGRES_READER_LOG_LEVEL=debug
SMQ_POSTGRES_READER_HTTP_HOST=postgres-reader
SMQ_POSTGRES_READER_HTTP_PORT=9009
SMQ_POSTGRES_READER_HTTP_SERVER_CERT=
SMQ_POSTGRES_READER_HTTP_SERVER_KEY=
SMQ_POSTGRES_READER_INSTANCE_ID=
### Timescale
SMQ_TIMESCALE_HOST=supermq-timescale
SMQ_TIMESCALE_PORT=5432
@@ -489,23 +471,6 @@ SMQ_TIMESCALE_SSL_CERT=
SMQ_TIMESCALE_SSL_KEY=
SMQ_TIMESCALE_SSL_ROOT_CERT=
### Timescale Writer
SMQ_TIMESCALE_WRITER_LOG_LEVEL=debug
SMQ_TIMESCALE_WRITER_CONFIG_PATH=/config.toml
SMQ_TIMESCALE_WRITER_HTTP_HOST=timescale-writer
SMQ_TIMESCALE_WRITER_HTTP_PORT=9012
SMQ_TIMESCALE_WRITER_HTTP_SERVER_CERT=
SMQ_TIMESCALE_WRITER_HTTP_SERVER_KEY=
SMQ_TIMESCALE_WRITER_INSTANCE_ID=
### Timescale Reader
SMQ_TIMESCALE_READER_LOG_LEVEL=debug
SMQ_TIMESCALE_READER_HTTP_HOST=timescale-reader
SMQ_TIMESCALE_READER_HTTP_PORT=9011
SMQ_TIMESCALE_READER_HTTP_SERVER_CERT=
SMQ_TIMESCALE_READER_HTTP_SERVER_KEY=
SMQ_TIMESCALE_READER_INSTANCE_ID=
### Journal
SMQ_JOURNAL_LOG_LEVEL=info
SMQ_JOURNAL_HTTP_HOST=journal
docker/addons/postgres-reader/docker-compose.yml
-117
View File
@@ -1,117 +0,0 @@
# Copyright (c) Abstract Machines
# SPDX-License-Identifier: Apache-2.0
# This docker-compose file contains optional Postgres-reader service for SuperMQ platform.
# Since this service is optional, this file is dependent of docker-compose.yml file
# from <project_root>/docker. In order to run this service, execute command:
# docker compose -f docker/docker-compose.yml -f docker/addons/postgres-reader/docker-compose.yml up
# from project root.
networks:
supermq-base-net:
services:
postgres-reader:
image: supermq/postgres-reader:${SMQ_RELEASE_TAG}
container_name: supermq-postgres-reader
restart: on-failure
environment:
SMQ_POSTGRES_READER_LOG_LEVEL: ${SMQ_POSTGRES_READER_LOG_LEVEL}
SMQ_POSTGRES_READER_HTTP_HOST: ${SMQ_POSTGRES_READER_HTTP_HOST}
SMQ_POSTGRES_READER_HTTP_PORT: ${SMQ_POSTGRES_READER_HTTP_PORT}
SMQ_POSTGRES_READER_HTTP_SERVER_CERT: ${SMQ_POSTGRES_READER_HTTP_SERVER_CERT}
SMQ_POSTGRES_READER_HTTP_SERVER_KEY: ${SMQ_POSTGRES_READER_HTTP_SERVER_KEY}
SMQ_POSTGRES_HOST: ${SMQ_POSTGRES_HOST}
SMQ_POSTGRES_PORT: ${SMQ_POSTGRES_PORT}
SMQ_POSTGRES_USER: ${SMQ_POSTGRES_USER}
SMQ_POSTGRES_PASS: ${SMQ_POSTGRES_PASS}
SMQ_POSTGRES_NAME: ${SMQ_POSTGRES_NAME}
SMQ_POSTGRES_SSL_MODE: ${SMQ_POSTGRES_SSL_MODE}
SMQ_POSTGRES_SSL_CERT: ${SMQ_POSTGRES_SSL_CERT}
SMQ_POSTGRES_SSL_KEY: ${SMQ_POSTGRES_SSL_KEY}
SMQ_POSTGRES_SSL_ROOT_CERT: ${SMQ_POSTGRES_SSL_ROOT_CERT}
SMQ_CLIENTS_AUTH_GRPC_URL: ${SMQ_CLIENTS_AUTH_GRPC_URL}
SMQ_CLIENTS_AUTH_GRPC_TIMEOUT: ${SMQ_CLIENTS_AUTH_GRPC_TIMEOUT}
SMQ_CLIENTS_AUTH_GRPC_CLIENT_CERT: ${SMQ_CLIENTS_AUTH_GRPC_CLIENT_CERT:+/clients-grpc-client.crt}
SMQ_CLIENTS_AUTH_GRPC_CLIENT_KEY: ${SMQ_CLIENTS_AUTH_GRPC_CLIENT_KEY:+/clients-grpc-client.key}
SMQ_CLIENTS_AUTH_GRPC_SERVER_CA_CERTS: ${SMQ_CLIENTS_AUTH_GRPC_SERVER_CA_CERTS:+/clients-grpc-server-ca.crt}
SMQ_CHANNELS_GRPC_URL: ${SMQ_CHANNELS_GRPC_URL}
SMQ_CHANNELS_GRPC_TIMEOUT: ${SMQ_CHANNELS_GRPC_TIMEOUT}
SMQ_CHANNELS_GRPC_CLIENT_CERT: ${SMQ_CHANNELS_GRPC_CLIENT_CERT:+/channels-grpc-client.crt}
SMQ_CHANNELS_GRPC_CLIENT_KEY: ${SMQ_CHANNELS_GRPC_CLIENT_KEY:+/channels-grpc-client.key}
SMQ_CHANNELS_GRPC_SERVER_CA_CERTS: ${SMQ_CHANNELS_GRPC_SERVER_CA_CERTS:+/channels-grpc-server-ca.crt}
SMQ_AUTH_GRPC_URL: ${SMQ_AUTH_GRPC_URL}
SMQ_AUTH_GRPC_TIMEOUT: ${SMQ_AUTH_GRPC_TIMEOUT}
SMQ_AUTH_GRPC_CLIENT_CERT: ${SMQ_AUTH_GRPC_CLIENT_CERT:+/auth-grpc-client.crt}
SMQ_AUTH_GRPC_CLIENT_KEY: ${SMQ_AUTH_GRPC_CLIENT_KEY:+/auth-grpc-client.key}
SMQ_AUTH_GRPC_SERVER_CA_CERTS: ${SMQ_AUTH_GRPC_SERVER_CA_CERTS:+/auth-grpc-server-ca.crt}
SMQ_SEND_TELEMETRY: ${SMQ_SEND_TELEMETRY}
SMQ_POSTGRES_READER_INSTANCE_ID: ${SMQ_POSTGRES_READER_INSTANCE_ID}
ports:
- ${SMQ_POSTGRES_READER_HTTP_PORT}:${SMQ_POSTGRES_READER_HTTP_PORT}
networks:
- supermq-base-net
volumes:
- type: bind
source: ${SMQ_ADDONS_CERTS_PATH_PREFIX}${SMQ_AUTH_GRPC_CLIENT_CERT:-./ssl/certs/dummy/client_cert}
target: /auth-grpc-client${SMQ_AUTH_GRPC_CLIENT_CERT:+.crt}
bind:
create_host_path: true
- type: bind
source: ${SMQ_ADDONS_CERTS_PATH_PREFIX}${SMQ_AUTH_GRPC_CLIENT_KEY:-./ssl/certs/dummy/client_key}
target: /auth-grpc-client${SMQ_AUTH_GRPC_CLIENT_KEY:+.key}
bind:
create_host_path: true
- type: bind
source: ${SMQ_ADDONS_CERTS_PATH_PREFIX}${SMQ_AUTH_GRPC_SERVER_CA_CERTS:-./ssl/certs/dummy/server_ca}
target: /auth-grpc-server-ca${SMQ_AUTH_GRPC_SERVER_CA_CERTS:+.crt}
bind:
create_host_path: true
# Clients gRPC mTLS client certificates
- type: bind
source: ${SMQ_ADDONS_CERTS_PATH_PREFIX}${SMQ_CLIENTS_AUTH_GRPC_CLIENT_CERT:-ssl/certs/dummy/client_cert}
target: /clients-grpc-client${SMQ_CLIENTS_AUTH_GRPC_CLIENT_CERT:+.crt}
bind:
create_host_path: true
- type: bind
source: ${SMQ_ADDONS_CERTS_PATH_PREFIX}${SMQ_CLIENTS_AUTH_GRPC_CLIENT_KEY:-ssl/certs/dummy/client_key}
target: /clients-grpc-client${SMQ_CLIENTS_AUTH_GRPC_CLIENT_KEY:+.key}
bind:
create_host_path: true
- type: bind
source: ${SMQ_ADDONS_CERTS_PATH_PREFIX}${SMQ_CLIENTS_AUTH_GRPC_SERVER_CA_CERTS:-ssl/certs/dummy/server_ca}
target: /clients-grpc-server-ca${SMQ_CLIENTS_AUTH_GRPC_SERVER_CA_CERTS:+.crt}
bind:
create_host_path: true
# Channels gRPC mTLS client certificates
- type: bind
source: ${SMQ_CHANNELS_AUTH_GRPC_CLIENT_CERT:-ssl/certs/dummy/client_cert}
target: /channels-grpc-client${SMQ_CHANNELS_AUTH_GRPC_CLIENT_CERT:+.crt}
bind:
create_host_path: true
- type: bind
source: ${SMQ_CHANNELS_AUTH_GRPC_CLIENT_KEY:-ssl/certs/dummy/client_key}
target: /channels-grpc-client${SMQ_CHANNELS_AUTH_GRPC_CLIENT_KEY:+.key}
bind:
create_host_path: true
- type: bind
source: ${SMQ_CHANNELS_AUTH_GRPC_SERVER_CA_CERTS:-ssl/certs/dummy/server_ca}
target: /channels-grpc-server-ca${SMQ_CHANNELS_AUTH_GRPC_SERVER_CA_CERTS:+.crt}
bind:
create_host_path: true
# Auth gRPC mTLS client certificates
- type: bind
source: ${SMQ_AUTH_GRPC_CLIENT_CERT:-ssl/certs/dummy/client_cert}
target: /auth-grpc-client${SMQ_AUTH_GRPC_CLIENT_CERT:+.crt}
bind:
create_host_path: true
- type: bind
source: ${SMQ_AUTH_GRPC_CLIENT_KEY:-ssl/certs/dummy/client_key}
target: /auth-grpc-client${SMQ_AUTH_GRPC_CLIENT_KEY:+.key}
bind:
create_host_path: true
- type: bind
source: ${SMQ_AUTH_GRPC_SERVER_CA_CERTS:-ssl/certs/dummy/server_ca}
target: /auth-grpc-server-ca${SMQ_AUTH_GRPC_SERVER_CA_CERTS:+.crt}
bind:
create_host_path: true
docker/addons/postgres-writer/config.toml
-19
View File
@@ -1,19 +0,0 @@
# Copyright (c) Abstract Machines
# SPDX-License-Identifier: Apache-2.0
# To listen all messsage broker subjects use default value "channels.>".
# To subscribe to specific subjects use values starting by "channels." and
# followed by a subtopic (e.g ["channels.<channel_id>.sub.topic.x", ...]).
[subscriber]
subjects = ["channels.>"]
[transformer]
# SenML or JSON
format = "senml"
# Used if format is SenML
content_type = "application/senml+json"
# Used as timestamp fields if format is JSON
time_fields = [{ field_name = "seconds_key", field_format = "unix", location = "UTC"},
{ field_name = "millis_key", field_format = "unix_ms", location = "UTC"},
{ field_name = "micros_key", field_format = "unix_us", location = "UTC"},
{ field_name = "nanos_key", field_format = "unix_ns", location = "UTC"}]
docker/addons/postgres-writer/docker-compose.yml
-63
View File
@@ -1,63 +0,0 @@
# Copyright (c) Abstract Machines
# SPDX-License-Identifier: Apache-2.0
# This docker-compose file contains optional Postgres and Postgres-writer services
# for SuperMQ platform. Since these are optional, this file is dependent of docker-compose file
# from <project_root>/docker. In order to run these services, execute command:
# docker compose -f docker/docker-compose.yml -f docker/addons/postgres-writer/docker-compose.yml up
# from project root. PostgreSQL default port (5432) is exposed, so you can use various tools for database
# inspection and data visualization.
networks:
supermq-base-net:
volumes:
supermq-postgres-writer-volume:
services:
postgres:
image: postgres:16.2-alpine
container_name: supermq-postgres
restart: on-failure
environment:
POSTGRES_USER: ${SMQ_POSTGRES_USER}
POSTGRES_PASSWORD: ${SMQ_POSTGRES_PASS}
POSTGRES_DB: ${SMQ_POSTGRES_NAME}
networks:
- supermq-base-net
volumes:
- supermq-postgres-writer-volume:/var/lib/postgresql/data
postgres-writer:
image: supermq/postgres-writer:${SMQ_RELEASE_TAG}
container_name: supermq-postgres-writer
depends_on:
- postgres
restart: on-failure
environment:
SMQ_POSTGRES_WRITER_LOG_LEVEL: ${SMQ_POSTGRES_WRITER_LOG_LEVEL}
SMQ_POSTGRES_WRITER_CONFIG_PATH: ${SMQ_POSTGRES_WRITER_CONFIG_PATH}
SMQ_POSTGRES_WRITER_HTTP_HOST: ${SMQ_POSTGRES_WRITER_HTTP_HOST}
SMQ_POSTGRES_WRITER_HTTP_PORT: ${SMQ_POSTGRES_WRITER_HTTP_PORT}
SMQ_POSTGRES_WRITER_HTTP_SERVER_CERT: ${SMQ_POSTGRES_WRITER_HTTP_SERVER_CERT}
SMQ_POSTGRES_WRITER_HTTP_SERVER_KEY: ${SMQ_POSTGRES_WRITER_HTTP_SERVER_KEY}
SMQ_POSTGRES_HOST: ${SMQ_POSTGRES_HOST}
SMQ_POSTGRES_PORT: ${SMQ_POSTGRES_PORT}
SMQ_POSTGRES_USER: ${SMQ_POSTGRES_USER}
SMQ_POSTGRES_PASS: ${SMQ_POSTGRES_PASS}
SMQ_POSTGRES_NAME: ${SMQ_POSTGRES_NAME}
SMQ_POSTGRES_SSL_MODE: ${SMQ_POSTGRES_SSL_MODE}
SMQ_POSTGRES_SSL_CERT: ${SMQ_POSTGRES_SSL_CERT}
SMQ_POSTGRES_SSL_KEY: ${SMQ_POSTGRES_SSL_KEY}
SMQ_POSTGRES_SSL_ROOT_CERT: ${SMQ_POSTGRES_SSL_ROOT_CERT}
SMQ_MESSAGE_BROKER_URL: ${SMQ_MESSAGE_BROKER_URL}
SMQ_JAEGER_URL: ${SMQ_JAEGER_URL}
SMQ_JAEGER_TRACE_RATIO: ${SMQ_JAEGER_TRACE_RATIO}
SMQ_SEND_TELEMETRY: ${SMQ_SEND_TELEMETRY}
SMQ_POSTGRES_WRITER_INSTANCE_ID: ${SMQ_POSTGRES_WRITER_INSTANCE_ID}
ports:
- ${SMQ_POSTGRES_WRITER_HTTP_PORT}:${SMQ_POSTGRES_WRITER_HTTP_PORT}
networks:
- supermq-base-net
volumes:
- ./config.toml:/config.toml
docker/addons/timescale-reader/docker-compose.yml
-117
View File
@@ -1,117 +0,0 @@
# Copyright (c) Abstract Machines
# SPDX-License-Identifier: Apache-2.0
# This docker-compose file contains optional Timescale-reader service for SuperMQ platform.
# Since this service is optional, this file is dependent of docker-compose.yml file
# from <project_root>/docker. In order to run this service, execute command:
# docker compose -f docker/docker-compose.yml -f docker/addons/timescale-reader/docker-compose.yml up
# from project root.
networks:
supermq-base-net:
services:
timescale-reader:
image: supermq/timescale-reader:${SMQ_RELEASE_TAG}
container_name: supermq-timescale-reader
restart: on-failure
environment:
SMQ_TIMESCALE_READER_LOG_LEVEL: ${SMQ_TIMESCALE_READER_LOG_LEVEL}
SMQ_TIMESCALE_READER_HTTP_HOST: ${SMQ_TIMESCALE_READER_HTTP_HOST}
SMQ_TIMESCALE_READER_HTTP_PORT: ${SMQ_TIMESCALE_READER_HTTP_PORT}
SMQ_TIMESCALE_READER_HTTP_SERVER_CERT: ${SMQ_TIMESCALE_READER_HTTP_SERVER_CERT}
SMQ_TIMESCALE_READER_HTTP_SERVER_KEY: ${SMQ_TIMESCALE_READER_HTTP_SERVER_KEY}
SMQ_TIMESCALE_HOST: ${SMQ_TIMESCALE_HOST}
SMQ_TIMESCALE_PORT: ${SMQ_TIMESCALE_PORT}
SMQ_TIMESCALE_USER: ${SMQ_TIMESCALE_USER}
SMQ_TIMESCALE_PASS: ${SMQ_TIMESCALE_PASS}
SMQ_TIMESCALE_NAME: ${SMQ_TIMESCALE_NAME}
SMQ_TIMESCALE_SSL_MODE: ${SMQ_TIMESCALE_SSL_MODE}
SMQ_TIMESCALE_SSL_CERT: ${SMQ_TIMESCALE_SSL_CERT}
SMQ_TIMESCALE_SSL_KEY: ${SMQ_TIMESCALE_SSL_KEY}
SMQ_TIMESCALE_SSL_ROOT_CERT: ${SMQ_TIMESCALE_SSL_ROOT_CERT}
SMQ_CLIENTS_AUTH_GRPC_URL: ${SMQ_CLIENTS_AUTH_GRPC_URL}
SMQ_CLIENTS_AUTH_GRPC_TIMEOUT: ${SMQ_CLIENTS_AUTH_GRPC_TIMEOUT}
SMQ_CLIENTS_AUTH_GRPC_CLIENT_CERT: ${SMQ_CLIENTS_AUTH_GRPC_CLIENT_CERT:+/clients-grpc-client.crt}
SMQ_CLIENTS_AUTH_GRPC_CLIENT_KEY: ${SMQ_CLIENTS_AUTH_GRPC_CLIENT_KEY:+/clients-grpc-client.key}
SMQ_CLIENTS_AUTH_GRPC_SERVER_CA_CERTS: ${SMQ_CLIENTS_AUTH_GRPC_SERVER_CA_CERTS:+/clients-grpc-server-ca.crt}
SMQ_CHANNELS_GRPC_URL: ${SMQ_CHANNELS_GRPC_URL}
SMQ_CHANNELS_GRPC_TIMEOUT: ${SMQ_CHANNELS_GRPC_TIMEOUT}
SMQ_CHANNELS_GRPC_CLIENT_CERT: ${SMQ_CHANNELS_GRPC_CLIENT_CERT:+/channels-grpc-client.crt}
SMQ_CHANNELS_GRPC_CLIENT_KEY: ${SMQ_CHANNELS_GRPC_CLIENT_KEY:+/channels-grpc-client.key}
SMQ_CHANNELS_GRPC_SERVER_CA_CERTS: ${SMQ_CHANNELS_GRPC_SERVER_CA_CERTS:+/channels-grpc-server-ca.crt}
SMQ_AUTH_GRPC_URL: ${SMQ_AUTH_GRPC_URL}
SMQ_AUTH_GRPC_TIMEOUT: ${SMQ_AUTH_GRPC_TIMEOUT}
SMQ_AUTH_GRPC_CLIENT_CERT: ${SMQ_AUTH_GRPC_CLIENT_CERT:+/auth-grpc-client.crt}
SMQ_AUTH_GRPC_CLIENT_KEY: ${SMQ_AUTH_GRPC_CLIENT_KEY:+/auth-grpc-client.key}
SMQ_AUTH_GRPC_SERVER_CA_CERTS: ${SMQ_AUTH_GRPC_SERVER_CA_CERTS:+/auth-grpc-server-ca.crt}
SMQ_SEND_TELEMETRY: ${SMQ_SEND_TELEMETRY}
SMQ_TIMESCALE_READER_INSTANCE_ID: ${SMQ_TIMESCALE_READER_INSTANCE_ID}
ports:
- ${SMQ_TIMESCALE_READER_HTTP_PORT}:${SMQ_TIMESCALE_READER_HTTP_PORT}
networks:
- supermq-base-net
volumes:
- type: bind
source: ${SMQ_ADDONS_CERTS_PATH_PREFIX}${SMQ_AUTH_GRPC_CLIENT_CERT:-./ssl/certs/dummy/client_cert}
target: /auth-grpc-client${SMQ_AUTH_GRPC_CLIENT_CERT:+.crt}
bind:
create_host_path: true
- type: bind
source: ${SMQ_ADDONS_CERTS_PATH_PREFIX}${SMQ_AUTH_GRPC_CLIENT_KEY:-./ssl/certs/dummy/client_key}
target: /auth-grpc-client${SMQ_AUTH_GRPC_CLIENT_KEY:+.key}
bind:
create_host_path: true
- type: bind
source: ${SMQ_ADDONS_CERTS_PATH_PREFIX}${SMQ_AUTH_GRPC_SERVER_CA_CERTS:-./ssl/certs/dummy/server_ca}
target: /auth-grpc-server-ca${SMQ_AUTH_GRPC_SERVER_CA_CERTS:+.crt}
bind:
create_host_path: true
# Clients gRPC mTLS client certificates
- type: bind
source: ${SMQ_ADDONS_CERTS_PATH_PREFIX}${SMQ_CLIENTS_AUTH_GRPC_CLIENT_CERT:-ssl/certs/dummy/client_cert}
target: /clients-grpc-client${SMQ_CLIENTS_AUTH_GRPC_CLIENT_CERT:+.crt}
bind:
create_host_path: true
- type: bind
source: ${SMQ_ADDONS_CERTS_PATH_PREFIX}${SMQ_CLIENTS_AUTH_GRPC_CLIENT_KEY:-ssl/certs/dummy/client_key}
target: /clients-grpc-client${SMQ_CLIENTS_AUTH_GRPC_CLIENT_KEY:+.key}
bind:
create_host_path: true
- type: bind
source: ${SMQ_ADDONS_CERTS_PATH_PREFIX}${SMQ_CLIENTS_AUTH_GRPC_SERVER_CA_CERTS:-ssl/certs/dummy/server_ca}
target: /clients-grpc-server-ca${SMQ_CLIENTS_AUTH_GRPC_SERVER_CA_CERTS:+.crt}
bind:
create_host_path: true
# Channels gRPC mTLS client certificates
- type: bind
source: ${SMQ_CHANNELS_AUTH_GRPC_CLIENT_CERT:-ssl/certs/dummy/client_cert}
target: /channels-grpc-client${SMQ_CHANNELS_AUTH_GRPC_CLIENT_CERT:+.crt}
bind:
create_host_path: true
- type: bind
source: ${SMQ_CHANNELS_AUTH_GRPC_CLIENT_KEY:-ssl/certs/dummy/client_key}
target: /channels-grpc-client${SMQ_CHANNELS_AUTH_GRPC_CLIENT_KEY:+.key}
bind:
create_host_path: true
- type: bind
source: ${SMQ_CHANNELS_AUTH_GRPC_SERVER_CA_CERTS:-ssl/certs/dummy/server_ca}
target: /channels-grpc-server-ca${SMQ_CHANNELS_AUTH_GRPC_SERVER_CA_CERTS:+.crt}
bind:
create_host_path: true
# Auth gRPC mTLS client certificates
- type: bind
source: ${SMQ_AUTH_GRPC_CLIENT_CERT:-ssl/certs/dummy/client_cert}
target: /auth-grpc-client${SMQ_AUTH_GRPC_CLIENT_CERT:+.crt}
bind:
create_host_path: true
- type: bind
source: ${SMQ_AUTH_GRPC_CLIENT_KEY:-ssl/certs/dummy/client_key}
target: /auth-grpc-client${SMQ_AUTH_GRPC_CLIENT_KEY:+.key}
bind:
create_host_path: true
- type: bind
source: ${SMQ_AUTH_GRPC_SERVER_CA_CERTS:-ssl/certs/dummy/server_ca}
target: /auth-grpc-server-ca${SMQ_AUTH_GRPC_SERVER_CA_CERTS:+.crt}
bind:
create_host_path: true
docker/addons/timescale-writer/config.toml
-8
View File
@@ -1,8 +0,0 @@
# Copyright (c) Abstract Machines
# SPDX-License-Identifier: Apache-2.0
# To listen all messsage broker subjects use default value "channels.>".
# To subscribe to specific subjects use values starting by "channels." and
# followed by a subtopic (e.g ["channels.<channel_id>.sub.topic.x", ...]).
[subjects]
filter = ["channels.>"]
docker/addons/timescale-writer/docker-compose.yml
-65
View File
@@ -1,65 +0,0 @@
# Copyright (c) Abstract Machines
# SPDX-License-Identifier: Apache-2.0
# This docker-compose file contains optional Timescale and Timescale-writer services
# for SuperMQ platform. Since these are optional, this file is dependent of docker-compose file
# from <project_root>/docker. In order to run these services, execute command:
# docker compose -f docker/docker-compose.yml -f docker/addons/timescale-writer/docker-compose.yml up
# from project root. PostgreSQL default port (5432) is exposed, so you can use various tools for database
# inspection and data visualization.
networks:
supermq-base-net:
volumes:
supermq-timescale-writer-volume:
services:
timescale:
image: timescale/timescaledb:2.13.1-pg16
container_name: supermq-timescale
restart: on-failure
environment:
POSTGRES_PASSWORD: ${SMQ_TIMESCALE_PASS}
POSTGRES_USER: ${SMQ_TIMESCALE_USER}
POSTGRES_DB: ${SMQ_TIMESCALE_NAME}
ports:
- 5433:5432
networks:
- supermq-base-net
volumes:
- supermq-timescale-writer-volume:/var/lib/timescalesql/data
timescale-writer:
image: supermq/timescale-writer:${SMQ_RELEASE_TAG}
container_name: supermq-timescale-writer
depends_on:
- timescale
restart: on-failure
environment:
SMQ_TIMESCALE_WRITER_LOG_LEVEL: ${SMQ_TIMESCALE_WRITER_LOG_LEVEL}
SMQ_TIMESCALE_WRITER_CONFIG_PATH: ${SMQ_TIMESCALE_WRITER_CONFIG_PATH}
SMQ_TIMESCALE_WRITER_HTTP_HOST: ${SMQ_TIMESCALE_WRITER_HTTP_HOST}
SMQ_TIMESCALE_WRITER_HTTP_PORT: ${SMQ_TIMESCALE_WRITER_HTTP_PORT}
SMQ_TIMESCALE_WRITER_HTTP_SERVER_CERT: ${SMQ_TIMESCALE_WRITER_HTTP_SERVER_CERT}
SMQ_TIMESCALE_WRITER_HTTP_SERVER_KEY: ${SMQ_TIMESCALE_WRITER_HTTP_SERVER_KEY}
SMQ_TIMESCALE_HOST: ${SMQ_TIMESCALE_HOST}
SMQ_TIMESCALE_PORT: ${SMQ_TIMESCALE_PORT}
SMQ_TIMESCALE_USER: ${SMQ_TIMESCALE_USER}
SMQ_TIMESCALE_PASS: ${SMQ_TIMESCALE_PASS}
SMQ_TIMESCALE_NAME: ${SMQ_TIMESCALE_NAME}
SMQ_TIMESCALE_SSL_MODE: ${SMQ_TIMESCALE_SSL_MODE}
SMQ_TIMESCALE_SSL_CERT: ${SMQ_TIMESCALE_SSL_CERT}
SMQ_TIMESCALE_SSL_KEY: ${SMQ_TIMESCALE_SSL_KEY}
SMQ_TIMESCALE_SSL_ROOT_CERT: ${SMQ_TIMESCALE_SSL_ROOT_CERT}
SMQ_MESSAGE_BROKER_URL: ${SMQ_MESSAGE_BROKER_URL}
SMQ_JAEGER_URL: ${SMQ_JAEGER_URL}
SMQ_JAEGER_TRACE_RATIO: ${SMQ_JAEGER_TRACE_RATIO}
SMQ_SEND_TELEMETRY: ${SMQ_SEND_TELEMETRY}
SMQ_TIMESCALE_WRITER_INSTANCE_ID: ${SMQ_TIMESCALE_WRITER_INSTANCE_ID}
ports:
- ${SMQ_TIMESCALE_WRITER_HTTP_PORT}:${SMQ_TIMESCALE_WRITER_HTTP_PORT}
networks:
- supermq-base-net
volumes:
- ./config.toml:/config.toml
docker/docker-compose.yml
-2
View File
@@ -1222,12 +1222,10 @@ services:
SMQ_UI_LOG_LEVEL: ${SMQ_UI_LOG_LEVEL}
SMQ_UI_PORT: ${SMQ_UI_PORT}
SMQ_HTTP_ADAPTER_URL: ${SMQ_HTTP_ADAPTER_URL}
SMQ_READER_URL: ${SMQ_READER_URL}
SMQ_CLIENTS_URL: ${SMQ_CLIENTS_URL}
SMQ_USERS_URL: ${SMQ_USERS_URL}
SMQ_INVITATIONS_URL: ${SMQ_INVITATIONS_URL}
SMQ_DOMAINS_URL: ${SMQ_DOMAINS_URL}
SMQ_BOOTSTRAP_URL: ${SMQ_BOOTSTRAP_URL}
SMQ_UI_HOST_URL: ${SMQ_UI_HOST_URL}
SMQ_UI_VERIFICATION_TLS: ${SMQ_UI_VERIFICATION_TLS}
SMQ_UI_CONTENT_TYPE: ${SMQ_UI_CONTENT_TYPE}
go.mod
-17
View File
@@ -19,13 +19,10 @@ require (
github.com/go-kit/kit v0.13.0
github.com/gofrs/uuid/v5 v5.3.0
github.com/google/uuid v1.6.0
github.com/gookit/color v1.5.4
github.com/gorilla/websocket v1.5.3
github.com/hashicorp/vault/api v1.15.0
github.com/hashicorp/vault/api/auth/approle v0.8.0
github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f
github.com/ivanpirog/coloredcobra v1.0.1
github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438
github.com/jackc/pgtype v1.14.4
github.com/jackc/pgx/v5 v5.7.2
github.com/jmoiron/sqlx v1.4.0
@@ -42,7 +39,6 @@ require (
github.com/redis/go-redis/v9 v9.7.0
github.com/rubenv/sql-migrate v1.7.1
github.com/spf13/cobra v1.8.1
github.com/spf13/viper v1.19.0
github.com/sqids/sqids-go v0.4.1
github.com/stretchr/testify v1.10.0
go.etcd.io/bbolt v1.3.11
@@ -56,7 +52,6 @@ require (
golang.org/x/crypto v0.32.0
golang.org/x/oauth2 v0.25.0
golang.org/x/sync v0.10.0
gonum.org/v1/gonum v0.15.1
google.golang.org/genproto/googleapis/rpc v0.0.0-20241219192143-6b3ec007d9bb
google.golang.org/grpc v1.69.2
google.golang.org/protobuf v1.36.2
@@ -88,7 +83,6 @@ require (
github.com/emirpasic/gods v1.18.1 // indirect
github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/go-errors/errors v1.5.1 // indirect
github.com/go-gorp/gorp/v3 v3.1.0 // indirect
@@ -126,7 +120,6 @@ require (
github.com/lestrrat-go/httprc v1.0.6 // indirect
github.com/lestrrat-go/iter v1.0.2 // indirect
github.com/lestrrat-go/option v1.0.1 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -138,7 +131,6 @@ require (
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/opencontainers/runc v1.1.14 // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/pion/dtls/v3 v3.0.2 // indirect
github.com/pion/logging v0.2.2 // indirect
github.com/pion/transport/v3 v3.0.7 // indirect
@@ -150,29 +142,21 @@ require (
github.com/prometheus/procfs v0.15.1 // indirect
github.com/rs/zerolog v1.33.0 // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
github.com/sagikazarmark/locafero v0.6.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/samber/lo v1.47.0 // indirect
github.com/segmentio/asm v1.2.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/smarty/assertions v1.15.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.7.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stoewer/go-strcase v1.3.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel/metric v1.33.0 // indirect
go.opentelemetry.io/proto/otlp v1.4.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
golang.org/x/net v0.33.0 // indirect
golang.org/x/sys v0.29.0 // indirect
@@ -180,7 +164,6 @@ require (
golang.org/x/time v0.8.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20241219192143-6b3ec007d9bb // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
go.sum
-43
View File
@@ -61,7 +61,6 @@ github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
@@ -94,15 +93,10 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM=
github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/go-chi/chi v1.5.5 h1:vOB/HbEMt9QqBqErz07QehcOKHaWFtuj87tTDVz2qXE=
@@ -158,8 +152,6 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
@@ -195,11 +187,8 @@ github.com/hashicorp/vault/api/auth/approle v0.8.0 h1:FuVtWZ0xD6+wz1x0l5s0b4852R
github.com/hashicorp/vault/api/auth/approle v0.8.0/go.mod h1:NV7O9r5JUtNdVnqVZeMHva81AIdpG0WoIQohNt1VCPM=
github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f h1:7LYC+Yfkj3CTRcShK0KOL/w6iTiKyqqBA9a41Wnggw8=
github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/ivanpirog/coloredcobra v1.0.1 h1:aURSdEmlR90/tSiWS0dMjdwOvCVUeYLfltLfbgNxrN4=
github.com/ivanpirog/coloredcobra v1.0.1/go.mod h1:iho4nEKcnwZFiniGSdcgdvRgZNjxm+h20acv8vqmN6Q=
github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
@@ -213,8 +202,6 @@ github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8
github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w=
github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM=
github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJyVOmBx6msDp/kfd1t9GRfny/mfJA0=
github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
@@ -298,17 +285,13 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -345,8 +328,6 @@ github.com/ory/dockertest/v3 v3.11.0/go.mod h1:VIPxS1gwT9NpPOrfD3rACs8Y9Z7yhzO4S
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
github.com/pion/dtls/v3 v3.0.2 h1:425DEeJ/jfuTTghhUDW0GtYZYIwwMtnKKJNMcWccTX0=
github.com/pion/dtls/v3 v3.0.2/go.mod h1:dfIXcFkKoujDQ+jtd8M6RgqKK3DuaUilm3YatAbGp5k=
github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
@@ -392,10 +373,6 @@ github.com/rubenv/sql-migrate v1.7.1/go.mod h1:Ob2Psprc0/3ggbM6wCzyYVFFuc6FyZrb2
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk=
github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc=
github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@@ -411,19 +388,10 @@ github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGB
github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec=
github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY=
github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w=
github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
github.com/sqids/sqids-go v0.4.1 h1:eQKYzmAZbLlRwHeHYPF35QhgxwZHLnlmVj9AkIj/rrw=
github.com/sqids/sqids-go v0.4.1/go.mod h1:EMwHuPQgSNFS0A49jESTfIQS+066XQTVhukrzEPScl8=
github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
@@ -446,8 +414,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -457,8 +423,6 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
@@ -501,8 +465,6 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
@@ -578,7 +540,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -633,8 +594,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0=
gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -663,8 +622,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
pkg/sdk/consumers.go
-89
View File
@@ -1,89 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package sdk
import (
"encoding/json"
"fmt"
"net/http"
"strings"
"github.com/absmach/supermq/pkg/errors"
)
const (
subscriptionEndpoint = "subscriptions"
)
type Subscription struct {
ID string `json:"id,omitempty"`
OwnerID string `json:"owner_id,omitempty"`
Topic string `json:"topic,omitempty"`
Contact string `json:"contact,omitempty"`
}
func (sdk mgSDK) CreateSubscription(topic, contact, token string) (string, errors.SDKError) {
sub := Subscription{
Topic: topic,
Contact: contact,
}
data, err := json.Marshal(sub)
if err != nil {
return "", errors.NewSDKError(err)
}
url := fmt.Sprintf("%s/%s", sdk.usersURL, subscriptionEndpoint)
headers, _, sdkerr := sdk.processRequest(http.MethodPost, url, token, data, nil, http.StatusCreated)
if sdkerr != nil {
return "", sdkerr
}
id := strings.TrimPrefix(headers.Get("Location"), fmt.Sprintf("/%s/", subscriptionEndpoint))
return id, nil
}
func (sdk mgSDK) ListSubscriptions(pm PageMetadata, token string) (SubscriptionPage, errors.SDKError) {
url, err := sdk.withQueryParams(sdk.usersURL, subscriptionEndpoint, pm)
if err != nil {
return SubscriptionPage{}, errors.NewSDKError(err)
}
_, body, sdkerr := sdk.processRequest(http.MethodGet, url, token, nil, nil, http.StatusOK)
if sdkerr != nil {
return SubscriptionPage{}, sdkerr
}
var sp SubscriptionPage
if err := json.Unmarshal(body, &sp); err != nil {
return SubscriptionPage{}, errors.NewSDKError(err)
}
return sp, nil
}
func (sdk mgSDK) ViewSubscription(id, token string) (Subscription, errors.SDKError) {
url := fmt.Sprintf("%s/%s/%s", sdk.usersURL, subscriptionEndpoint, id)
_, body, err := sdk.processRequest(http.MethodGet, url, token, nil, nil, http.StatusOK)
if err != nil {
return Subscription{}, err
}
var sub Subscription
if err := json.Unmarshal(body, &sub); err != nil {
return Subscription{}, errors.NewSDKError(err)
}
return sub, nil
}
func (sdk mgSDK) DeleteSubscription(id, token string) errors.SDKError {
url := fmt.Sprintf("%s/%s/%s", sdk.usersURL, subscriptionEndpoint, id)
_, _, err := sdk.processRequest(http.MethodDelete, url, token, nil, nil, http.StatusNoContent)
return err
}
pkg/sdk/consumers_test.go
-468
View File
@@ -1,468 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package sdk_test
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
apiutil "github.com/absmach/supermq/api/http/util"
"github.com/absmach/supermq/consumers/notifiers"
httpapi "github.com/absmach/supermq/consumers/notifiers/api"
notmocks "github.com/absmach/supermq/consumers/notifiers/mocks"
"github.com/absmach/supermq/internal/testsutil"
smqlog "github.com/absmach/supermq/logger"
"github.com/absmach/supermq/pkg/errors"
svcerr "github.com/absmach/supermq/pkg/errors/service"
sdk "github.com/absmach/supermq/pkg/sdk"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
var (
ownerID = testsutil.GenerateUUID(&testing.T{})
subID = testsutil.GenerateUUID(&testing.T{})
sdkSubReq = sdk.Subscription{
Topic: "topic",
Contact: "contact",
}
sdkSubRes = sdk.Subscription{
Topic: "topic",
Contact: "contact",
OwnerID: ownerID,
ID: subID,
}
notSubReq = notifiers.Subscription{
Contact: "contact",
Topic: "topic",
}
notSubRes = notifiers.Subscription{
Contact: "contact",
Topic: "topic",
OwnerID: ownerID,
ID: subID,
}
)
func setupSubscriptions() (*httptest.Server, *notmocks.Service) {
nsvc := new(notmocks.Service)
logger := smqlog.NewMock()
mux := httpapi.MakeHandler(nsvc, logger, instanceID)
return httptest.NewServer(mux), nsvc
}
func TestCreateSubscription(t *testing.T) {
ts, nsvc := setupSubscriptions()
defer ts.Close()
sdkConf := sdk.Config{
UsersURL: ts.URL,
MsgContentType: contentType,
TLSVerification: false,
}
mgsdk := sdk.NewSDK(sdkConf)
cases := []struct {
desc string
subscription sdk.Subscription
token string
empty bool
id string
svcReq notifiers.Subscription
svcErr error
svcRes string
err errors.SDKError
}{
{
desc: "create new subscription",
subscription: sdkSubReq,
token: validToken,
empty: false,
svcReq: notSubReq,
svcRes: subID,
svcErr: nil,
err: nil,
},
{
desc: "create new subscription with empty token",
subscription: sdkSubReq,
token: "",
empty: true,
svcReq: notifiers.Subscription{},
svcRes: "",
svcErr: nil,
err: errors.NewSDKErrorWithStatus(errors.Wrap(apiutil.ErrValidation, apiutil.ErrBearerToken), http.StatusUnauthorized),
},
{
desc: "create new subscription with invalid token",
subscription: sdkSubReq,
token: invalidToken,
empty: true,
svcReq: notSubReq,
svcRes: "",
svcErr: svcerr.ErrAuthentication,
err: errors.NewSDKErrorWithStatus(svcerr.ErrAuthentication, http.StatusUnauthorized),
},
{
desc: "create new subscription with empty topic",
subscription: sdk.Subscription{
Topic: "",
Contact: "contact",
},
token: validToken,
empty: true,
svcReq: notifiers.Subscription{},
svcErr: nil,
svcRes: "",
err: errors.NewSDKErrorWithStatus(errors.Wrap(apiutil.ErrValidation, apiutil.ErrInvalidTopic), http.StatusBadRequest),
},
{
desc: "create new subscription with empty contact",
subscription: sdk.Subscription{
Topic: "topic",
Contact: "",
},
token: validToken,
empty: true,
svcReq: notifiers.Subscription{},
svcErr: nil,
svcRes: "",
err: errors.NewSDKErrorWithStatus(errors.Wrap(apiutil.ErrValidation, apiutil.ErrInvalidContact), http.StatusBadRequest),
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
svcCall := nsvc.On("CreateSubscription", mock.Anything, tc.token, tc.svcReq).Return(tc.svcRes, tc.svcErr)
loc, err := mgsdk.CreateSubscription(tc.subscription.Topic, tc.subscription.Contact, tc.token)
assert.Equal(t, tc.err, err)
assert.Equal(t, tc.empty, loc == "")
if tc.err == nil {
ok := svcCall.Parent.AssertCalled(t, "CreateSubscription", mock.Anything, tc.token, tc.svcReq)
assert.True(t, ok)
}
svcCall.Unset()
})
}
}
func TestViewSubscription(t *testing.T) {
ts, nsvc := setupSubscriptions()
defer ts.Close()
sdkConf := sdk.Config{
UsersURL: ts.URL,
MsgContentType: contentType,
TLSVerification: false,
}
mgsdk := sdk.NewSDK(sdkConf)
cases := []struct {
desc string
subID string
token string
svcRes notifiers.Subscription
svcErr error
response sdk.Subscription
err errors.SDKError
}{
{
desc: "view existing subscription",
subID: subID,
token: validToken,
svcRes: notSubRes,
svcErr: nil,
response: sdkSubRes,
err: nil,
},
{
desc: "view non-existent subscription",
subID: wrongID,
token: validToken,
svcRes: notifiers.Subscription{},
svcErr: svcerr.ErrNotFound,
response: sdk.Subscription{},
err: errors.NewSDKErrorWithStatus(svcerr.ErrNotFound, http.StatusNotFound),
},
{
desc: "view subscription with invalid token",
subID: subID,
token: invalidToken,
svcRes: notifiers.Subscription{},
svcErr: svcerr.ErrAuthentication,
response: sdk.Subscription{},
err: errors.NewSDKErrorWithStatus(svcerr.ErrAuthentication, http.StatusUnauthorized),
},
{
desc: "view subscription with empty token",
subID: subID,
token: "",
svcRes: notifiers.Subscription{},
svcErr: nil,
response: sdk.Subscription{},
err: errors.NewSDKErrorWithStatus(errors.Wrap(apiutil.ErrValidation, apiutil.ErrBearerToken), http.StatusUnauthorized),
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
svcCall := nsvc.On("ViewSubscription", mock.Anything, tc.token, tc.subID).Return(tc.svcRes, tc.svcErr)
resp, err := mgsdk.ViewSubscription(tc.subID, tc.token)
assert.Equal(t, tc.err, err)
assert.Equal(t, tc.response, resp)
if tc.err == nil {
ok := svcCall.Parent.AssertCalled(t, "ViewSubscription", mock.Anything, tc.token, tc.subID)
assert.True(t, ok)
}
svcCall.Unset()
})
}
}
func TestListSubscription(t *testing.T) {
ts, nsvc := setupSubscriptions()
defer ts.Close()
sdkConf := sdk.Config{
UsersURL: ts.URL,
MsgContentType: contentType,
TLSVerification: false,
}
mgsdk := sdk.NewSDK(sdkConf)
nSubs := 10
noSubs := []notifiers.Subscription{}
sdSubs := []sdk.Subscription{}
for i := 0; i < nSubs; i++ {
nosub := notifiers.Subscription{
OwnerID: ownerID,
Topic: fmt.Sprintf("topic_%d", i),
Contact: fmt.Sprintf("contact_%d", i),
}
noSubs = append(noSubs, nosub)
sdsub := sdk.Subscription{
OwnerID: ownerID,
Topic: fmt.Sprintf("topic_%d", i),
Contact: fmt.Sprintf("contact_%d", i),
}
sdSubs = append(sdSubs, sdsub)
}
cases := []struct {
desc string
token string
pageMeta sdk.PageMetadata
svcReq notifiers.PageMetadata
svcRes notifiers.Page
svcErr error
response sdk.SubscriptionPage
err errors.SDKError
}{
{
desc: "list all subscription",
token: validToken,
pageMeta: sdk.PageMetadata{
Offset: 0,
Limit: 10,
},
svcReq: notifiers.PageMetadata{
Offset: 0,
Limit: 10,
},
svcRes: notifiers.Page{
Total: 10,
Subscriptions: noSubs,
},
svcErr: nil,
response: sdk.SubscriptionPage{
PageRes: sdk.PageRes{
Total: 10,
},
Subscriptions: sdSubs,
},
err: nil,
},
{
desc: "list subscription with specific topic",
token: validToken,
pageMeta: sdk.PageMetadata{
Offset: 0,
Limit: 10,
Topic: "topic_1",
},
svcReq: notifiers.PageMetadata{
Offset: 0,
Limit: 10,
Topic: "topic_1",
},
svcRes: notifiers.Page{
Total: uint(len(noSubs[1:2])),
Subscriptions: noSubs[1:2],
},
svcErr: nil,
response: sdk.SubscriptionPage{
PageRes: sdk.PageRes{
Total: uint64(len(sdSubs[1:2])),
},
Subscriptions: sdSubs[1:2],
},
err: nil,
},
{
desc: "list subscription with specific contact",
token: validToken,
pageMeta: sdk.PageMetadata{
Offset: 0,
Limit: 10,
Contact: "contact_1",
},
svcReq: notifiers.PageMetadata{
Offset: 0,
Limit: 10,
Contact: "contact_1",
},
svcRes: notifiers.Page{
Total: uint(len(noSubs[1:2])),
Subscriptions: noSubs[1:2],
},
svcErr: nil,
response: sdk.SubscriptionPage{
PageRes: sdk.PageRes{
Total: uint64(len(sdSubs[1:2])),
},
Subscriptions: sdSubs[1:2],
},
err: nil,
},
{
desc: "list subscription with invalid token",
token: invalidToken,
pageMeta: sdk.PageMetadata{
Offset: 0,
Limit: 10,
},
svcReq: notifiers.PageMetadata{
Offset: 0,
Limit: 10,
},
svcRes: notifiers.Page{},
svcErr: svcerr.ErrAuthentication,
response: sdk.SubscriptionPage{},
err: errors.NewSDKErrorWithStatus(svcerr.ErrAuthentication, http.StatusUnauthorized),
},
{
desc: "list subscription with empty token",
token: "",
pageMeta: sdk.PageMetadata{
Offset: 0,
Limit: 10,
},
svcReq: notifiers.PageMetadata{},
svcRes: notifiers.Page{},
svcErr: nil,
response: sdk.SubscriptionPage{},
err: errors.NewSDKErrorWithStatus(errors.Wrap(apiutil.ErrValidation, apiutil.ErrBearerToken), http.StatusUnauthorized),
},
{
desc: "list subscription with invalid page metadata",
token: validToken,
pageMeta: sdk.PageMetadata{
Offset: 0,
Limit: 10,
Metadata: sdk.Metadata{
"key": make(chan int),
},
},
svcReq: notifiers.PageMetadata{},
svcRes: notifiers.Page{},
svcErr: nil,
response: sdk.SubscriptionPage{},
err: errors.NewSDKError(errors.New("json: unsupported type: chan int")),
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
svcCall := nsvc.On("ListSubscriptions", mock.Anything, tc.token, tc.svcReq).Return(tc.svcRes, tc.svcErr)
resp, err := mgsdk.ListSubscriptions(tc.pageMeta, tc.token)
assert.Equal(t, tc.err, err)
assert.Equal(t, tc.response, resp)
if tc.err == nil {
ok := svcCall.Parent.AssertCalled(t, "ListSubscriptions", mock.Anything, tc.token, tc.svcReq)
assert.True(t, ok)
}
svcCall.Unset()
})
}
}
func TestDeleteSubscription(t *testing.T) {
ts, nsvc := setupSubscriptions()
defer ts.Close()
sdkConf := sdk.Config{
UsersURL: ts.URL,
MsgContentType: contentType,
TLSVerification: false,
}
mgsdk := sdk.NewSDK(sdkConf)
cases := []struct {
desc string
subID string
token string
svcErr error
err errors.SDKError
}{
{
desc: "delete existing subscription",
subID: subID,
token: validToken,
svcErr: nil,
err: nil,
},
{
desc: "delete non-existent subscription",
subID: wrongID,
token: validToken,
svcErr: svcerr.ErrRemoveEntity,
err: errors.NewSDKErrorWithStatus(svcerr.ErrRemoveEntity, http.StatusUnprocessableEntity),
},
{
desc: "delete subscription with invalid token",
subID: subID,
token: invalidToken,
svcErr: svcerr.ErrAuthentication,
err: errors.NewSDKErrorWithStatus(svcerr.ErrAuthentication, http.StatusUnauthorized),
},
{
desc: "delete subscription with empty token",
subID: subID,
token: "",
svcErr: nil,
err: errors.NewSDKErrorWithStatus(errors.Wrap(apiutil.ErrValidation, apiutil.ErrBearerToken), http.StatusUnauthorized),
},
{
desc: "delete subscription with empty subID",
subID: "",
token: validToken,
svcErr: nil,
err: errors.NewSDKErrorWithStatus(errors.Wrap(apiutil.ErrValidation, apiutil.ErrMissingID), http.StatusBadRequest),
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
svcCall := nsvc.On("RemoveSubscription", mock.Anything, tc.token, tc.subID).Return(tc.svcErr)
err := mgsdk.DeleteSubscription(tc.subID, tc.token)
assert.Equal(t, tc.err, err)
if tc.err == nil {
ok := svcCall.Parent.AssertCalled(t, "RemoveSubscription", mock.Anything, tc.token, tc.subID)
assert.True(t, ok)
}
svcCall.Unset()
})
}
}
pkg/sdk/health.go
-2
View File
@@ -38,8 +38,6 @@ func (sdk mgSDK) Health(service string) (HealthInfo, errors.SDKError) {
url = fmt.Sprintf("%s/health", sdk.usersURL)
case "certs":
url = fmt.Sprintf("%s/health", sdk.certsURL)
case "reader":
url = fmt.Sprintf("%s/health", sdk.readerURL)
case "http-adapter":
url = fmt.Sprintf("%s/health", sdk.httpAdapterURL)
}
pkg/sdk/health_test.go
-28
View File
@@ -5,17 +5,11 @@ package sdk_test
import (
"fmt"
"net/http/httptest"
"testing"
"github.com/absmach/supermq"
chmocks "github.com/absmach/supermq/channels/mocks"
climocks "github.com/absmach/supermq/clients/mocks"
authnmocks "github.com/absmach/supermq/pkg/authn/mocks"
"github.com/absmach/supermq/pkg/errors"
sdk "github.com/absmach/supermq/pkg/sdk"
readersapi "github.com/absmach/supermq/readers/api"
readersmocks "github.com/absmach/supermq/readers/mocks"
"github.com/stretchr/testify/assert"
)
@@ -29,9 +23,6 @@ func TestHealth(t *testing.T) {
certsTs, _, _ := setupCerts()
defer certsTs.Close()
readerTs := setupMinimalReader()
defer readerTs.Close()
httpAdapterTs, _ := setupMessages()
defer httpAdapterTs.Close()
@@ -39,7 +30,6 @@ func TestHealth(t *testing.T) {
ClientsURL: clientsTs.URL,
UsersURL: usersTs.URL,
CertsURL: certsTs.URL,
ReaderURL: readerTs.URL,
HTTPAdapterURL: httpAdapterTs.URL,
MsgContentType: contentType,
TLSVerification: false,
@@ -78,14 +68,6 @@ func TestHealth(t *testing.T) {
description: "certs service",
status: "pass",
},
{
desc: "get reader service health check",
service: "reader",
empty: false,
err: nil,
description: "test service",
status: "pass",
},
{
desc: "get http-adapter service health check",
service: "http-adapter",
@@ -107,13 +89,3 @@ func TestHealth(t *testing.T) {
})
}
}
func setupMinimalReader() *httptest.Server {
repo := new(readersmocks.MessageRepository)
channels := new(chmocks.ChannelsServiceClient)
authn := new(authnmocks.Authentication)
clients := new(climocks.ClientsServiceClient)
mux := readersapi.MakeHandler(repo, authn, clients, channels, "test", "")
return httptest.NewServer(mux)
}
pkg/sdk/message.go
-63
View File
@@ -4,11 +4,8 @@
package sdk
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
apiutil "github.com/absmach/supermq/api/http/util"
@@ -32,35 +29,6 @@ func (sdk mgSDK) SendMessage(chanName, msg, key string) errors.SDKError {
return err
}
func (sdk mgSDK) ReadMessages(pm MessagePageMetadata, chanName, domainID, token string) (MessagesPage, errors.SDKError) {
chanNameParts := strings.SplitN(chanName, ".", channelParts)
chanID := chanNameParts[0]
subtopicPart := ""
if len(chanNameParts) == channelParts {
subtopicPart = fmt.Sprintf("?subtopic=%s", chanNameParts[1])
}
msgURL, err := sdk.withMessageQueryParams(sdk.readerURL, fmt.Sprintf("channels/%s/messages%s", chanID, subtopicPart), pm)
if err != nil {
return MessagesPage{}, errors.NewSDKError(err)
}
header := make(map[string]string)
header["Content-Type"] = string(sdk.msgContentType)
_, body, sdkerr := sdk.processRequest(http.MethodGet, msgURL, token, nil, header, http.StatusOK)
if sdkerr != nil {
return MessagesPage{}, sdkerr
}
var mp MessagesPage
if err := json.Unmarshal(body, &mp); err != nil {
return MessagesPage{}, errors.NewSDKError(err)
}
return mp, nil
}
func (sdk *mgSDK) SetContentType(ct ContentType) errors.SDKError {
if ct != CTJSON && ct != CTJSONSenML && ct != CTBinary {
return errors.NewSDKError(apiutil.ErrUnsupportedContentType)
@@ -70,34 +38,3 @@ func (sdk *mgSDK) SetContentType(ct ContentType) errors.SDKError {
return nil
}
func (sdk mgSDK) withMessageQueryParams(baseURL, endpoint string, mpm MessagePageMetadata) (string, error) {
b, err := json.Marshal(mpm)
if err != nil {
return "", err
}
q := map[string]interface{}{}
if err := json.Unmarshal(b, &q); err != nil {
return "", err
}
ret := url.Values{}
for k, v := range q {
switch t := v.(type) {
case string:
ret.Add(k, t)
case float64:
ret.Add(k, strconv.FormatFloat(t, 'f', -1, 64))
case uint64:
ret.Add(k, strconv.FormatUint(t, 10))
case int64:
ret.Add(k, strconv.FormatInt(t, 10))
case json.Number:
ret.Add(k, t.String())
case bool:
ret.Add(k, strconv.FormatBool(t))
}
}
qs := ret.Encode()
return fmt.Sprintf("%s/%s?%s", baseURL, endpoint, qs), nil
}
pkg/sdk/message_test.go
-218
View File
@@ -19,16 +19,11 @@ import (
adapter "github.com/absmach/supermq/http"
"github.com/absmach/supermq/http/api"
smqlog "github.com/absmach/supermq/logger"
smqauthn "github.com/absmach/supermq/pkg/authn"
authnmocks "github.com/absmach/supermq/pkg/authn/mocks"
"github.com/absmach/supermq/pkg/errors"
svcerr "github.com/absmach/supermq/pkg/errors/service"
pubsub "github.com/absmach/supermq/pkg/messaging/mocks"
sdk "github.com/absmach/supermq/pkg/sdk"
"github.com/absmach/supermq/pkg/transformers/senml"
"github.com/absmach/supermq/readers"
readersapi "github.com/absmach/supermq/readers/api"
readersmocks "github.com/absmach/supermq/readers/mocks"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
@@ -60,16 +55,6 @@ func setupMessages() (*httptest.Server, *pubsub.PubSub) {
return httptest.NewServer(http.HandlerFunc(mp.ServeHTTP)), pub
}
func setupReaders() (*httptest.Server, *authnmocks.Authentication, *readersmocks.MessageRepository) {
repo := new(readersmocks.MessageRepository)
authn := new(authnmocks.Authentication)
clientsGRPCClient = new(climocks.ClientsServiceClient)
channelsGRPCClient = new(chmocks.ChannelsServiceClient)
mux := readersapi.MakeHandler(repo, authn, clientsGRPCClient, channelsGRPCClient, "test", "")
return httptest.NewServer(mux), authn, repo
}
func TestSendMessage(t *testing.T) {
ts, pub := setupMessages()
defer ts.Close()
@@ -207,206 +192,3 @@ func TestSetContentType(t *testing.T) {
assert.Equal(t, tc.err, err, fmt.Sprintf("%s: expected error %s, got %s", tc.desc, tc.err, err))
}
}
func TestReadMessages(t *testing.T) {
ts, authn, repo := setupReaders()
defer ts.Close()
channelID := "channelID"
msgValue := 1.6
boolVal := true
msg := senml.Message{
Name: "current",
Time: 1720000000,
Value: &msgValue,
Publisher: validID,
}
invalidMsg := "[{\"n\":\"current\",\"t\":-1,\"v\":1.6}]"
sdkConf := sdk.Config{
ReaderURL: ts.URL,
}
mgsdk := sdk.NewSDK(sdkConf)
cases := []struct {
desc string
token string
chanName string
domainID string
messagePageMeta sdk.MessagePageMetadata
authzErr error
authnErr error
repoRes readers.MessagesPage
repoErr error
response sdk.MessagesPage
err errors.SDKError
}{
{
desc: "read messages successfully",
token: validToken,
chanName: channelID,
domainID: validID,
messagePageMeta: sdk.MessagePageMetadata{
PageMetadata: sdk.PageMetadata{
Offset: 0,
Limit: 10,
Level: 0,
},
Publisher: validID,
BoolValue: &boolVal,
},
repoRes: readers.MessagesPage{
Total: 1,
Messages: []readers.Message{msg},
},
repoErr: nil,
response: sdk.MessagesPage{
PageRes: sdk.PageRes{
Total: 1,
},
Messages: []senml.Message{msg},
},
err: nil,
},
{
desc: "read messages successfully with subtopic",
token: validToken,
chanName: channelID + ".subtopic",
domainID: validID,
messagePageMeta: sdk.MessagePageMetadata{
PageMetadata: sdk.PageMetadata{
Offset: 0,
Limit: 10,
},
Publisher: validID,
},
repoRes: readers.MessagesPage{
Total: 1,
Messages: []readers.Message{msg},
},
repoErr: nil,
response: sdk.MessagesPage{
PageRes: sdk.PageRes{
Total: 1,
},
Messages: []senml.Message{msg},
},
err: nil,
},
{
desc: "read messages with invalid token",
token: invalidToken,
chanName: channelID,
domainID: validID,
messagePageMeta: sdk.MessagePageMetadata{
PageMetadata: sdk.PageMetadata{
Offset: 0,
Limit: 10,
},
Subtopic: "subtopic",
Publisher: validID,
},
authzErr: svcerr.ErrAuthorization,
repoRes: readers.MessagesPage{},
response: sdk.MessagesPage{},
err: errors.NewSDKErrorWithStatus(errors.Wrap(svcerr.ErrAuthorization, svcerr.ErrAuthorization), http.StatusUnauthorized),
},
{
desc: "read messages with empty token",
token: "",
chanName: channelID,
domainID: validID,
messagePageMeta: sdk.MessagePageMetadata{
PageMetadata: sdk.PageMetadata{
Offset: 0,
Limit: 10,
},
Subtopic: "subtopic",
Publisher: validID,
},
authnErr: svcerr.ErrAuthentication,
repoRes: readers.MessagesPage{},
response: sdk.MessagesPage{},
err: errors.NewSDKErrorWithStatus(errors.Wrap(apiutil.ErrValidation, apiutil.ErrBearerToken), http.StatusUnauthorized),
},
{
desc: "read messages with empty channel ID",
token: validToken,
chanName: "",
domainID: validID,
messagePageMeta: sdk.MessagePageMetadata{
PageMetadata: sdk.PageMetadata{
Offset: 0,
Limit: 10,
},
Subtopic: "subtopic",
Publisher: validID,
},
repoRes: readers.MessagesPage{},
repoErr: nil,
response: sdk.MessagesPage{},
err: errors.NewSDKErrorWithStatus(errors.Wrap(apiutil.ErrValidation, apiutil.ErrMissingID), http.StatusBadRequest),
},
{
desc: "read messages with invalid message page metadata",
token: validToken,
chanName: channelID,
domainID: validID,
messagePageMeta: sdk.MessagePageMetadata{
PageMetadata: sdk.PageMetadata{
Offset: 0,
Limit: 10,
Metadata: map[string]interface{}{
"key": make(chan int),
},
},
Subtopic: "subtopic",
Publisher: validID,
},
repoRes: readers.MessagesPage{},
repoErr: nil,
response: sdk.MessagesPage{},
err: errors.NewSDKError(errors.New("json: unsupported type: chan int")),
},
{
desc: "read messages with response that cannot be unmarshalled",
token: validToken,
chanName: channelID,
domainID: validID,
messagePageMeta: sdk.MessagePageMetadata{
PageMetadata: sdk.PageMetadata{
Offset: 0,
Limit: 10,
},
Subtopic: "subtopic",
Publisher: validID,
},
repoRes: readers.MessagesPage{
Total: 1,
Messages: []readers.Message{invalidMsg},
},
repoErr: nil,
response: sdk.MessagesPage{},
err: errors.NewSDKError(errors.New("json: cannot unmarshal string into Go struct field MessagesPage.messages of type senml.Message")),
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
authCall1 := authn.On("Authenticate", mock.Anything, tc.token).Return(smqauthn.Session{UserID: validID}, tc.authnErr)
authzCall := channelsGRPCClient.On("Authorize", mock.Anything, mock.Anything).Return(&grpcChannelsV1.AuthzRes{Authorized: true}, tc.authzErr)
repoCall := repo.On("ReadAll", channelID, mock.Anything).Return(tc.repoRes, tc.repoErr)
response, err := mgsdk.ReadMessages(tc.messagePageMeta, tc.chanName, tc.domainID, tc.token)
fmt.Println(err)
assert.Equal(t, tc.err, err)
assert.Equal(t, tc.response, response)
if tc.err == nil {
ok := repoCall.Parent.AssertCalled(t, "ReadAll", channelID, mock.Anything)
assert.True(t, ok)
}
authCall1.Unset()
authzCall.Unset()
repoCall.Unset()
})
}
}
pkg/sdk/mocks/sdk.go
-288
View File
@@ -1885,66 +1885,6 @@ func (_c *SDK_CreateGroupRole_Call) RunAndReturn(run func(string, string, sdk.Ro
return _c
}
// CreateSubscription provides a mock function with given fields: topic, contact, token
func (_m *SDK) CreateSubscription(topic string, contact string, token string) (string, errors.SDKError) {
ret := _m.Called(topic, contact, token)
if len(ret) == 0 {
panic("no return value specified for CreateSubscription")
}
var r0 string
var r1 errors.SDKError
if rf, ok := ret.Get(0).(func(string, string, string) (string, errors.SDKError)); ok {
return rf(topic, contact, token)
}
if rf, ok := ret.Get(0).(func(string, string, string) string); ok {
r0 = rf(topic, contact, token)
} else {
r0 = ret.Get(0).(string)
}
if rf, ok := ret.Get(1).(func(string, string, string) errors.SDKError); ok {
r1 = rf(topic, contact, token)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(errors.SDKError)
}
}
return r0, r1
}
// SDK_CreateSubscription_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateSubscription'
type SDK_CreateSubscription_Call struct {
*mock.Call
}
// CreateSubscription is a helper method to define mock.On call
// - topic string
// - contact string
// - token string
func (_e *SDK_Expecter) CreateSubscription(topic interface{}, contact interface{}, token interface{}) *SDK_CreateSubscription_Call {
return &SDK_CreateSubscription_Call{Call: _e.mock.On("CreateSubscription", topic, contact, token)}
}
func (_c *SDK_CreateSubscription_Call) Run(run func(topic string, contact string, token string)) *SDK_CreateSubscription_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(string), args[1].(string), args[2].(string))
})
return _c
}
func (_c *SDK_CreateSubscription_Call) Return(_a0 string, _a1 errors.SDKError) *SDK_CreateSubscription_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *SDK_CreateSubscription_Call) RunAndReturn(run func(string, string, string) (string, errors.SDKError)) *SDK_CreateSubscription_Call {
_c.Call.Return(run)
return _c
}
// CreateToken provides a mock function with given fields: lt
func (_m *SDK) CreateToken(lt sdk.Login) (sdk.Token, errors.SDKError) {
ret := _m.Called(lt)
@@ -2412,55 +2352,6 @@ func (_c *SDK_DeleteInvitation_Call) RunAndReturn(run func(string, string, strin
return _c
}
// DeleteSubscription provides a mock function with given fields: id, token
func (_m *SDK) DeleteSubscription(id string, token string) errors.SDKError {
ret := _m.Called(id, token)
if len(ret) == 0 {
panic("no return value specified for DeleteSubscription")
}
var r0 errors.SDKError
if rf, ok := ret.Get(0).(func(string, string) errors.SDKError); ok {
r0 = rf(id, token)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(errors.SDKError)
}
}
return r0
}
// SDK_DeleteSubscription_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteSubscription'
type SDK_DeleteSubscription_Call struct {
*mock.Call
}
// DeleteSubscription is a helper method to define mock.On call
// - id string
// - token string
func (_e *SDK_Expecter) DeleteSubscription(id interface{}, token interface{}) *SDK_DeleteSubscription_Call {
return &SDK_DeleteSubscription_Call{Call: _e.mock.On("DeleteSubscription", id, token)}
}
func (_c *SDK_DeleteSubscription_Call) Run(run func(id string, token string)) *SDK_DeleteSubscription_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(string), args[1].(string))
})
return _c
}
func (_c *SDK_DeleteSubscription_Call) Return(_a0 errors.SDKError) *SDK_DeleteSubscription_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *SDK_DeleteSubscription_Call) RunAndReturn(run func(string, string) errors.SDKError) *SDK_DeleteSubscription_Call {
_c.Call.Return(run)
return _c
}
// DeleteUser provides a mock function with given fields: id, token
func (_m *SDK) DeleteUser(id string, token string) errors.SDKError {
ret := _m.Called(id, token)
@@ -4502,65 +4393,6 @@ func (_c *SDK_ListDomainUsers_Call) RunAndReturn(run func(string, sdk.PageMetada
return _c
}
// ListSubscriptions provides a mock function with given fields: pm, token
func (_m *SDK) ListSubscriptions(pm sdk.PageMetadata, token string) (sdk.SubscriptionPage, errors.SDKError) {
ret := _m.Called(pm, token)
if len(ret) == 0 {
panic("no return value specified for ListSubscriptions")
}
var r0 sdk.SubscriptionPage
var r1 errors.SDKError
if rf, ok := ret.Get(0).(func(sdk.PageMetadata, string) (sdk.SubscriptionPage, errors.SDKError)); ok {
return rf(pm, token)
}
if rf, ok := ret.Get(0).(func(sdk.PageMetadata, string) sdk.SubscriptionPage); ok {
r0 = rf(pm, token)
} else {
r0 = ret.Get(0).(sdk.SubscriptionPage)
}
if rf, ok := ret.Get(1).(func(sdk.PageMetadata, string) errors.SDKError); ok {
r1 = rf(pm, token)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(errors.SDKError)
}
}
return r0, r1
}
// SDK_ListSubscriptions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListSubscriptions'
type SDK_ListSubscriptions_Call struct {
*mock.Call
}
// ListSubscriptions is a helper method to define mock.On call
// - pm sdk.PageMetadata
// - token string
func (_e *SDK_Expecter) ListSubscriptions(pm interface{}, token interface{}) *SDK_ListSubscriptions_Call {
return &SDK_ListSubscriptions_Call{Call: _e.mock.On("ListSubscriptions", pm, token)}
}
func (_c *SDK_ListSubscriptions_Call) Run(run func(pm sdk.PageMetadata, token string)) *SDK_ListSubscriptions_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(sdk.PageMetadata), args[1].(string))
})
return _c
}
func (_c *SDK_ListSubscriptions_Call) Return(_a0 sdk.SubscriptionPage, _a1 errors.SDKError) *SDK_ListSubscriptions_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *SDK_ListSubscriptions_Call) RunAndReturn(run func(sdk.PageMetadata, string) (sdk.SubscriptionPage, errors.SDKError)) *SDK_ListSubscriptions_Call {
_c.Call.Return(run)
return _c
}
// ListUserClients provides a mock function with given fields: userID, domainID, pm, token
func (_m *SDK) ListUserClients(userID string, domainID string, pm sdk.PageMetadata, token string) (sdk.ClientsPage, errors.SDKError) {
ret := _m.Called(userID, domainID, pm, token)
@@ -4683,67 +4515,6 @@ func (_c *SDK_Members_Call) RunAndReturn(run func(string, string, sdk.PageMetada
return _c
}
// ReadMessages provides a mock function with given fields: pm, chanID, domainID, token
func (_m *SDK) ReadMessages(pm sdk.MessagePageMetadata, chanID string, domainID string, token string) (sdk.MessagesPage, errors.SDKError) {
ret := _m.Called(pm, chanID, domainID, token)
if len(ret) == 0 {
panic("no return value specified for ReadMessages")
}
var r0 sdk.MessagesPage
var r1 errors.SDKError
if rf, ok := ret.Get(0).(func(sdk.MessagePageMetadata, string, string, string) (sdk.MessagesPage, errors.SDKError)); ok {
return rf(pm, chanID, domainID, token)
}
if rf, ok := ret.Get(0).(func(sdk.MessagePageMetadata, string, string, string) sdk.MessagesPage); ok {
r0 = rf(pm, chanID, domainID, token)
} else {
r0 = ret.Get(0).(sdk.MessagesPage)
}
if rf, ok := ret.Get(1).(func(sdk.MessagePageMetadata, string, string, string) errors.SDKError); ok {
r1 = rf(pm, chanID, domainID, token)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(errors.SDKError)
}
}
return r0, r1
}
// SDK_ReadMessages_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReadMessages'
type SDK_ReadMessages_Call struct {
*mock.Call
}
// ReadMessages is a helper method to define mock.On call
// - pm sdk.MessagePageMetadata
// - chanID string
// - domainID string
// - token string
func (_e *SDK_Expecter) ReadMessages(pm interface{}, chanID interface{}, domainID interface{}, token interface{}) *SDK_ReadMessages_Call {
return &SDK_ReadMessages_Call{Call: _e.mock.On("ReadMessages", pm, chanID, domainID, token)}
}
func (_c *SDK_ReadMessages_Call) Run(run func(pm sdk.MessagePageMetadata, chanID string, domainID string, token string)) *SDK_ReadMessages_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(sdk.MessagePageMetadata), args[1].(string), args[2].(string), args[3].(string))
})
return _c
}
func (_c *SDK_ReadMessages_Call) Return(_a0 sdk.MessagesPage, _a1 errors.SDKError) *SDK_ReadMessages_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *SDK_ReadMessages_Call) RunAndReturn(run func(sdk.MessagePageMetadata, string, string, string) (sdk.MessagesPage, errors.SDKError)) *SDK_ReadMessages_Call {
_c.Call.Return(run)
return _c
}
// RefreshToken provides a mock function with given fields: token
func (_m *SDK) RefreshToken(token string) (sdk.Token, errors.SDKError) {
ret := _m.Called(token)
@@ -7547,65 +7318,6 @@ func (_c *SDK_ViewCertByClient_Call) RunAndReturn(run func(string, string, strin
return _c
}
// ViewSubscription provides a mock function with given fields: id, token
func (_m *SDK) ViewSubscription(id string, token string) (sdk.Subscription, errors.SDKError) {
ret := _m.Called(id, token)
if len(ret) == 0 {
panic("no return value specified for ViewSubscription")
}
var r0 sdk.Subscription
var r1 errors.SDKError
if rf, ok := ret.Get(0).(func(string, string) (sdk.Subscription, errors.SDKError)); ok {
return rf(id, token)
}
if rf, ok := ret.Get(0).(func(string, string) sdk.Subscription); ok {
r0 = rf(id, token)
} else {
r0 = ret.Get(0).(sdk.Subscription)
}
if rf, ok := ret.Get(1).(func(string, string) errors.SDKError); ok {
r1 = rf(id, token)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(errors.SDKError)
}
}
return r0, r1
}
// SDK_ViewSubscription_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ViewSubscription'
type SDK_ViewSubscription_Call struct {
*mock.Call
}
// ViewSubscription is a helper method to define mock.On call
// - id string
// - token string
func (_e *SDK_Expecter) ViewSubscription(id interface{}, token interface{}) *SDK_ViewSubscription_Call {
return &SDK_ViewSubscription_Call{Call: _e.mock.On("ViewSubscription", id, token)}
}
func (_c *SDK_ViewSubscription_Call) Run(run func(id string, token string)) *SDK_ViewSubscription_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(string), args[1].(string))
})
return _c
}
func (_c *SDK_ViewSubscription_Call) Return(_a0 sdk.Subscription, _a1 errors.SDKError) *SDK_ViewSubscription_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *SDK_ViewSubscription_Call) RunAndReturn(run func(string, string) (sdk.Subscription, errors.SDKError)) *SDK_ViewSubscription_Call {
_c.Call.Return(run)
return _c
}
// NewSDK creates a new instance of SDK. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewSDK(t interface {
pkg/sdk/responses.go
-5
View File
@@ -72,11 +72,6 @@ type CertSerials struct {
PageRes
}
type SubscriptionPage struct {
Subscriptions []Subscription `json:"subscriptions"`
PageRes
}
type DomainsPage struct {
Domains []Domain `json:"domains"`
PageRes
pkg/sdk/sdk.go
-46
View File
@@ -1044,17 +1044,6 @@ type SDK interface {
// fmt.Println(err)
SendMessage(chanID, msg, key string) errors.SDKError
// ReadMessages read messages of specified channel.
//
// example:
// pm := sdk.MessagePageMetadata{
// Offset: 0,
// Limit: 10,
// }
// msgs, _ := sdk.ReadMessages(pm,"channelID", "domainID", "token")
// fmt.Println(msgs)
ReadMessages(pm MessagePageMetadata, chanID, domainID, token string) (MessagesPage, errors.SDKError)
// SetContentType sets message content type.
//
// example:
@@ -1097,38 +1086,6 @@ type SDK interface {
// fmt.Println(tm)
RevokeCert(clientID, domainID, token string) (time.Time, errors.SDKError)
// CreateSubscription creates a new subscription
//
// example:
// subscription, _ := sdk.CreateSubscription("topic", "contact", "token")
// fmt.Println(subscription)
CreateSubscription(topic, contact, token string) (string, errors.SDKError)
// ListSubscriptions list subscriptions given list parameters.
//
// example:
// pm := sdk.PageMetadata{
// Offset: 0,
// Limit: 10,
// }
// subscriptions, _ := sdk.ListSubscriptions(pm, "token")
// fmt.Println(subscriptions)
ListSubscriptions(pm PageMetadata, token string) (SubscriptionPage, errors.SDKError)
// ViewSubscription retrieves a subscription with the provided id.
//
// example:
// subscription, _ := sdk.ViewSubscription("id", "token")
// fmt.Println(subscription)
ViewSubscription(id, token string) (Subscription, errors.SDKError)
// DeleteSubscription removes a subscription with the provided id.
//
// example:
// err := sdk.DeleteSubscription("id", "token")
// fmt.Println(err)
DeleteSubscription(id, token string) errors.SDKError
// CreateDomain creates new domain and returns its details.
//
// example:
@@ -1370,7 +1327,6 @@ type SDK interface {
type mgSDK struct {
certsURL string
httpAdapterURL string
readerURL string
clientsURL string
usersURL string
groupsURL string
@@ -1389,7 +1345,6 @@ type mgSDK struct {
type Config struct {
CertsURL string
HTTPAdapterURL string
ReaderURL string
ClientsURL string
UsersURL string
GroupsURL string
@@ -1409,7 +1364,6 @@ func NewSDK(conf Config) SDK {
return &mgSDK{
certsURL: conf.CertsURL,
httpAdapterURL: conf.HTTPAdapterURL,
readerURL: conf.ReaderURL,
clientsURL: conf.ClientsURL,
usersURL: conf.UsersURL,
groupsURL: conf.GroupsURL,
readers/api/doc.go
-6
View File
@@ -1,6 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package api contains API-related concerns: endpoint definitions, middlewares
// and all resource representations.
package api
readers/api/endpoint.go
-41
View File
@@ -1,41 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api
import (
"context"
grpcChannelsV1 "github.com/absmach/supermq/api/grpc/channels/v1"
grpcClientsV1 "github.com/absmach/supermq/api/grpc/clients/v1"
apiutil "github.com/absmach/supermq/api/http/util"
smqauthn "github.com/absmach/supermq/pkg/authn"
"github.com/absmach/supermq/pkg/errors"
svcerr "github.com/absmach/supermq/pkg/errors/service"
"github.com/absmach/supermq/readers"
"github.com/go-kit/kit/endpoint"
)
func listMessagesEndpoint(svc readers.MessageRepository, authn smqauthn.Authentication, clients grpcClientsV1.ClientsServiceClient, channels grpcChannelsV1.ChannelsServiceClient) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
req := request.(listMessagesReq)
if err := req.validate(); err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
if err := authnAuthz(ctx, req, authn, clients, channels); err != nil {
return nil, errors.Wrap(svcerr.ErrAuthorization, err)
}
page, err := svc.ReadAll(req.chanID, req.pageMeta)
if err != nil {
return nil, err
}
return pageRes{
PageMetadata: page.PageMetadata,
Total: page.Total,
Messages: page.Messages,
}, nil
}
}
readers/api/endpoint_test.go
-1020
View File
File diff suppressed because it is too large Load Diff
readers/api/logging.go
-56
View File
@@ -1,56 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
//go:build !test
package api
import (
"log/slog"
"time"
"github.com/absmach/supermq/readers"
)
var _ readers.MessageRepository = (*loggingMiddleware)(nil)
type loggingMiddleware struct {
logger *slog.Logger
svc readers.MessageRepository
}
// LoggingMiddleware adds logging facilities to the core service.
func LoggingMiddleware(svc readers.MessageRepository, logger *slog.Logger) readers.MessageRepository {
return &loggingMiddleware{
logger: logger,
svc: svc,
}
}
func (lm *loggingMiddleware) ReadAll(chanID string, rpm readers.PageMetadata) (page readers.MessagesPage, err error) {
defer func(begin time.Time) {
args := []any{
slog.String("duration", time.Since(begin).String()),
slog.String("channel_id", chanID),
slog.Group("page",
slog.Uint64("offset", rpm.Offset),
slog.Uint64("limit", rpm.Limit),
slog.Uint64("total", page.Total),
),
}
if rpm.Subtopic != "" {
args = append(args, slog.String("subtopic", rpm.Subtopic))
}
if rpm.Publisher != "" {
args = append(args, slog.String("publisher", rpm.Publisher))
}
if err != nil {
args = append(args, slog.Any("error", err))
lm.logger.Warn("Read all failed", args...)
return
}
lm.logger.Info("Read all completed successfully", args...)
}(time.Now())
return lm.svc.ReadAll(chanID, rpm)
}
readers/api/metrics.go
-39
View File
@@ -1,39 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
//go:build !test
package api
import (
"time"
"github.com/absmach/supermq/readers"
"github.com/go-kit/kit/metrics"
)
var _ readers.MessageRepository = (*metricsMiddleware)(nil)
type metricsMiddleware struct {
counter metrics.Counter
latency metrics.Histogram
svc readers.MessageRepository
}
// MetricsMiddleware instruments core service by tracking request count and latency.
func MetricsMiddleware(svc readers.MessageRepository, counter metrics.Counter, latency metrics.Histogram) readers.MessageRepository {
return &metricsMiddleware{
counter: counter,
latency: latency,
svc: svc,
}
}
func (mm *metricsMiddleware) ReadAll(chanID string, rpm readers.PageMetadata) (readers.MessagesPage, error) {
defer func(begin time.Time) {
mm.counter.With("method", "read_all").Add(1)
mm.latency.With("method", "read_all").Observe(time.Since(begin).Seconds())
}(time.Now())
return mm.svc.ReadAll(chanID, rpm)
}
readers/api/requests.go
-67
View File
@@ -1,67 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api
import (
"slices"
"strings"
"time"
apiutil "github.com/absmach/supermq/api/http/util"
"github.com/absmach/supermq/readers"
)
const maxLimitSize = 1000
var validAggregations = []string{"MAX", "MIN", "AVG", "SUM", "COUNT"}
type listMessagesReq struct {
chanID string
token string
key string
pageMeta readers.PageMetadata
}
func (req listMessagesReq) validate() error {
if req.token == "" && req.key == "" {
return apiutil.ErrBearerToken
}
if req.chanID == "" {
return apiutil.ErrMissingID
}
if req.pageMeta.Limit < 1 || req.pageMeta.Limit > maxLimitSize {
return apiutil.ErrLimitSize
}
if req.pageMeta.Comparator != "" &&
req.pageMeta.Comparator != readers.EqualKey &&
req.pageMeta.Comparator != readers.LowerThanKey &&
req.pageMeta.Comparator != readers.LowerThanEqualKey &&
req.pageMeta.Comparator != readers.GreaterThanKey &&
req.pageMeta.Comparator != readers.GreaterThanEqualKey {
return apiutil.ErrInvalidComparator
}
if req.pageMeta.Aggregation != "" {
if req.pageMeta.From == 0 {
return apiutil.ErrMissingFrom
}
if req.pageMeta.To == 0 {
return apiutil.ErrMissingTo
}
if !slices.Contains(validAggregations, strings.ToUpper(req.pageMeta.Aggregation)) {
return apiutil.ErrInvalidAggregation
}
if _, err := time.ParseDuration(req.pageMeta.Interval); err != nil {
return apiutil.ErrInvalidInterval
}
}
return nil
}
readers/api/responses.go
-31
View File
@@ -1,31 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api
import (
"net/http"
"github.com/absmach/supermq"
"github.com/absmach/supermq/readers"
)
var _ supermq.Response = (*pageRes)(nil)
type pageRes struct {
readers.PageMetadata
Total uint64 `json:"total"`
Messages []readers.Message `json:"messages,omitempty"`
}
func (res pageRes) Headers() map[string]string {
return map[string]string{}
}
func (res pageRes) Code() int {
return http.StatusOK
}
func (res pageRes) Empty() bool {
return false
}
readers/api/transport.go
-289
View File
@@ -1,289 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api
import (
"context"
"encoding/json"
"net/http"
"github.com/absmach/supermq"
grpcChannelsV1 "github.com/absmach/supermq/api/grpc/channels/v1"
grpcClientsV1 "github.com/absmach/supermq/api/grpc/clients/v1"
apiutil "github.com/absmach/supermq/api/http/util"
smqauthn "github.com/absmach/supermq/pkg/authn"
"github.com/absmach/supermq/pkg/connections"
"github.com/absmach/supermq/pkg/errors"
svcerr "github.com/absmach/supermq/pkg/errors/service"
"github.com/absmach/supermq/pkg/policies"
"github.com/absmach/supermq/readers"
"github.com/go-chi/chi/v5"
kithttp "github.com/go-kit/kit/transport/http"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
const (
contentType = "application/json"
offsetKey = "offset"
limitKey = "limit"
formatKey = "format"
subtopicKey = "subtopic"
publisherKey = "publisher"
protocolKey = "protocol"
nameKey = "name"
valueKey = "v"
stringValueKey = "vs"
dataValueKey = "vd"
boolValueKey = "vb"
comparatorKey = "comparator"
fromKey = "from"
toKey = "to"
aggregationKey = "aggregation"
intervalKey = "interval"
defInterval = "1s"
defLimit = 10
defOffset = 0
defFormat = "messages"
)
// MakeHandler returns a HTTP handler for API endpoints.
func MakeHandler(svc readers.MessageRepository, authn smqauthn.Authentication, clients grpcClientsV1.ClientsServiceClient, channels grpcChannelsV1.ChannelsServiceClient, svcName, instanceID string) http.Handler {
opts := []kithttp.ServerOption{
kithttp.ServerErrorEncoder(encodeError),
}
mux := chi.NewRouter()
mux.Get("/channels/{chanID}/messages", kithttp.NewServer(
listMessagesEndpoint(svc, authn, clients, channels),
decodeList,
encodeResponse,
opts...,
).ServeHTTP)
mux.Get("/health", supermq.Health(svcName, instanceID))
mux.Handle("/metrics", promhttp.Handler())
return mux
}
func decodeList(_ context.Context, r *http.Request) (interface{}, error) {
offset, err := apiutil.ReadNumQuery[uint64](r, offsetKey, defOffset)
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
limit, err := apiutil.ReadNumQuery[uint64](r, limitKey, defLimit)
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
format, err := apiutil.ReadStringQuery(r, formatKey, defFormat)
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
subtopic, err := apiutil.ReadStringQuery(r, subtopicKey, "")
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
publisher, err := apiutil.ReadStringQuery(r, publisherKey, "")
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
protocol, err := apiutil.ReadStringQuery(r, protocolKey, "")
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
name, err := apiutil.ReadStringQuery(r, nameKey, "")
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
v, err := apiutil.ReadNumQuery[float64](r, valueKey, 0)
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
comparator, err := apiutil.ReadStringQuery(r, comparatorKey, "")
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
vs, err := apiutil.ReadStringQuery(r, stringValueKey, "")
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
vd, err := apiutil.ReadStringQuery(r, dataValueKey, "")
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
vb, err := apiutil.ReadBoolQuery(r, boolValueKey, false)
if err != nil && err != apiutil.ErrNotFoundParam {
return nil, err
}
from, err := apiutil.ReadNumQuery[float64](r, fromKey, 0)
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
to, err := apiutil.ReadNumQuery[float64](r, toKey, 0)
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
aggregation, err := apiutil.ReadStringQuery(r, aggregationKey, "")
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
var interval string
if aggregation != "" {
interval, err = apiutil.ReadStringQuery(r, intervalKey, defInterval)
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
}
req := listMessagesReq{
chanID: chi.URLParam(r, "chanID"),
token: apiutil.ExtractBearerToken(r),
key: apiutil.ExtractClientSecret(r),
pageMeta: readers.PageMetadata{
Offset: offset,
Limit: limit,
Format: format,
Subtopic: subtopic,
Publisher: publisher,
Protocol: protocol,
Name: name,
Value: v,
Comparator: comparator,
StringValue: vs,
DataValue: vd,
BoolValue: vb,
From: from,
To: to,
Aggregation: aggregation,
Interval: interval,
},
}
return req, nil
}
func encodeResponse(_ context.Context, w http.ResponseWriter, response interface{}) error {
w.Header().Set("Content-Type", contentType)
if ar, ok := response.(supermq.Response); ok {
for k, v := range ar.Headers() {
w.Header().Set(k, v)
}
w.WriteHeader(ar.Code())
if ar.Empty() {
return nil
}
}
return json.NewEncoder(w).Encode(response)
}
func encodeError(_ context.Context, err error, w http.ResponseWriter) {
var wrapper error
if errors.Contains(err, apiutil.ErrValidation) {
wrapper, err = errors.Unwrap(err)
}
switch {
case errors.Contains(err, nil):
case errors.Contains(err, apiutil.ErrInvalidQueryParams),
errors.Contains(err, svcerr.ErrMalformedEntity),
errors.Contains(err, apiutil.ErrMissingID),
errors.Contains(err, apiutil.ErrLimitSize),
errors.Contains(err, apiutil.ErrOffsetSize),
errors.Contains(err, apiutil.ErrInvalidComparator),
errors.Contains(err, apiutil.ErrInvalidAggregation),
errors.Contains(err, apiutil.ErrInvalidInterval),
errors.Contains(err, apiutil.ErrMissingFrom),
errors.Contains(err, apiutil.ErrMissingTo),
errors.Contains(err, apiutil.ErrMissingDomainID):
w.WriteHeader(http.StatusBadRequest)
case errors.Contains(err, svcerr.ErrAuthentication),
errors.Contains(err, svcerr.ErrAuthorization),
errors.Contains(err, apiutil.ErrBearerToken):
w.WriteHeader(http.StatusUnauthorized)
case errors.Contains(err, readers.ErrReadMessages):
w.WriteHeader(http.StatusInternalServerError)
default:
w.WriteHeader(http.StatusInternalServerError)
}
if wrapper != nil {
err = errors.Wrap(wrapper, err)
}
if errorVal, ok := err.(errors.Error); ok {
w.Header().Set("Content-Type", contentType)
if err := json.NewEncoder(w).Encode(errorVal); err != nil {
w.WriteHeader(http.StatusInternalServerError)
}
}
}
func authnAuthz(ctx context.Context, req listMessagesReq, authn smqauthn.Authentication, clients grpcClientsV1.ClientsServiceClient, channels grpcChannelsV1.ChannelsServiceClient) error {
clientID, clientType, err := authenticate(ctx, req, authn, clients)
if err != nil {
return nil
}
if err := authorize(ctx, clientID, clientType, req.chanID, channels); err != nil {
return err
}
return nil
}
func authenticate(ctx context.Context, req listMessagesReq, authn smqauthn.Authentication, clients grpcClientsV1.ClientsServiceClient) (clientID string, clientType string, err error) {
switch {
case req.token != "":
session, err := authn.Authenticate(ctx, req.token)
if err != nil {
return "", "", err
}
return session.DomainUserID, policies.UserType, nil
case req.key != "":
res, err := clients.Authenticate(ctx, &grpcClientsV1.AuthnReq{
ClientSecret: req.key,
})
if err != nil {
return "", "", err
}
if !res.GetAuthenticated() {
return "", "", svcerr.ErrAuthentication
}
return res.GetId(), policies.ClientType, nil
default:
return "", "", svcerr.ErrAuthentication
}
}
func authorize(ctx context.Context, clientID, clientType, chanID string, channels grpcChannelsV1.ChannelsServiceClient) (err error) {
res, err := channels.Authorize(ctx, &grpcChannelsV1.AuthzReq{
ClientId: clientID,
ClientType: clientType,
Type: uint32(connections.Subscribe),
ChannelId: chanID,
})
if err != nil {
return errors.Wrap(svcerr.ErrAuthorization, err)
}
if !res.GetAuthorized() {
return svcerr.ErrAuthorization
}
return nil
}
readers/postgres/README.md
-101
View File
@@ -1,101 +0,0 @@
# Postgres reader
Postgres reader provides message repository implementation for Postgres.
## Configuration
The service is configured using the environment variables presented in the
following table. Note that any unset variables will be replaced with their
default values.
| Variable | Description | Default |
| ------------------------------------ | -------------------------------------------- | ---------------------------- |
| SMQ_POSTGRES_READER_LOG_LEVEL | Service log level | info |
| SMQ_POSTGRES_READER_HTTP_HOST | Service HTTP host | localhost |
| SMQ_POSTGRES_READER_HTTP_PORT | Service HTTP port | 9009 |
| SMQ_POSTGRES_READER_HTTP_SERVER_CERT | Service HTTP server cert | "" |
| SMQ_POSTGRES_READER_HTTP_SERVER_KEY | Service HTTP server key | "" |
| SMQ_POSTGRES_HOST | Postgres DB host | localhost |
| SMQ_POSTGRES_PORT | Postgres DB port | 5432 |
| SMQ_POSTGRES_USER | Postgres user | supermq |
| SMQ_POSTGRES_PASS | Postgres password | supermq |
| SMQ_POSTGRES_NAME | Postgres database name | messages |
| SMQ_POSTGRES_SSL_MODE | Postgres SSL mode | disabled |
| SMQ_POSTGRES_SSL_CERT | Postgres SSL certificate path | "" |
| SMQ_POSTGRES_SSL_KEY | Postgres SSL key | "" |
| SMQ_POSTGRES_SSL_ROOT_CERT | Postgres SSL root certificate path | "" |
| SMQ_CLIENTS_AUTH_GRPC_URL | Clients service Auth gRPC URL | localhost:7000 |
| SMQ_CLIENTS_AUTH_GRPC_TIMEOUT | Clients service Auth gRPC timeout in seconds | 1s |
| SMQ_CLIENTS_AUTH_GRPC_CLIENT_TLS | Clients service Auth gRPC TLS mode flag | false |
| SMQ_CLIENTS_AUTH_GRPC_CA_CERTS | Clients service Auth gRPC CA certificates | "" |
| SMQ_AUTH_GRPC_URL | Auth service gRPC URL | localhost:7001 |
| SMQ_AUTH_GRPC_TIMEOUT | Auth service gRPC request timeout in seconds | 1s |
| SMQ_AUTH_GRPC_CLIENT_TLS | Auth service gRPC TLS mode flag | false |
| SMQ_AUTH_GRPC_CA_CERTS | Auth service gRPC CA certificates | "" |
| SMQ_JAEGER_URL | Jaeger server URL | http://jaeger:4318/v1/traces |
| SMQ_SEND_TELEMETRY | Send telemetry to supermq call home server | true |
| SMQ_POSTGRES_READER_INSTANCE_ID | Postgres reader instance ID | |
## Deployment
The service itself is distributed as Docker container. Check the [`postgres-reader`](https://github.com/absmach/supermq/blob/main/docker/addons/postgres-reader/docker-compose.yml#L17-L41) service section in
docker-compose file to see how service is deployed.
To start the service, execute the following shell script:
```bash
# download the latest version of the service
git clone https://github.com/absmach/supermq
cd supermq
# compile the postgres writer
make postgres-writer
# copy binary to bin
make install
# Set the environment variables and run the service
SMQ_POSTGRES_READER_LOG_LEVEL=[Service log level] \
SMQ_POSTGRES_READER_HTTP_HOST=[Service HTTP host] \
SMQ_POSTGRES_READER_HTTP_PORT=[Service HTTP port] \
SMQ_POSTGRES_READER_HTTP_SERVER_CERT=[Service HTTPS server certificate path] \
SMQ_POSTGRES_READER_HTTP_SERVER_KEY=[Service HTTPS server key path] \
SMQ_POSTGRES_HOST=[Postgres host] \
SMQ_POSTGRES_PORT=[Postgres port] \
SMQ_POSTGRES_USER=[Postgres user] \
SMQ_POSTGRES_PASS=[Postgres password] \
SMQ_POSTGRES_NAME=[Postgres database name] \
SMQ_POSTGRES_SSL_MODE=[Postgres SSL mode] \
SMQ_POSTGRES_SSL_CERT=[Postgres SSL cert] \
SMQ_POSTGRES_SSL_KEY=[Postgres SSL key] \
SMQ_POSTGRES_SSL_ROOT_CERT=[Postgres SSL Root cert] \
SMQ_CLIENTS_AUTH_GRPC_URL=[Clients service Auth GRPC URL] \
SMQ_CLIENTS_AUTH_GRPC_TIMEOUT=[Clients service Auth gRPC request timeout in seconds] \
SMQ_CLIENTS_AUTH_GRPC_CLIENT_TLS=[Clients service Auth gRPC TLS mode flag] \
SMQ_CLIENTS_AUTH_GRPC_CA_CERTS=[Clients service Auth gRPC CA certificates] \
SMQ_AUTH_GRPC_URL=[Auth service gRPC URL] \
SMQ_AUTH_GRPC_TIMEOUT=[Auth service gRPC request timeout in seconds] \
SMQ_AUTH_GRPC_CLIENT_TLS=[Auth service gRPC TLS mode flag] \
SMQ_AUTH_GRPC_CA_CERTS=[Auth service gRPC CA certificates] \
SMQ_JAEGER_URL=[Jaeger server URL] \
SMQ_SEND_TELEMETRY=[Send telemetry to supermq call home server] \
SMQ_POSTGRES_READER_INSTANCE_ID=[Postgres reader instance ID] \
$GOBIN/supermq-postgres-reader
```
## Usage
Starting service will start consuming normalized messages in SenML format.
Comparator Usage Guide:
| Comparator | Usage | Example |
| ---------- | --------------------------------------------------------------------------- | ---------------------------------- |
| eq | Return values that are equal to the query | eq["active"] -> "active" |
| ge | Return values that are substrings of the query | ge["tiv"] -> "active" and "tiv" |
| gt | Return values that are substrings of the query and not equal to the query | gt["tiv"] -> "active" |
| le | Return values that are superstrings of the query | le["active"] -> "tiv" |
| lt | Return values that are superstrings of the query and not equal to the query | lt["active"] -> "active" and "tiv" |
Official docs can be found [here](https://docs.supermq.abstractmachines.fr).
readers/postgres/doc.go
-6
View File
@@ -1,6 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package postgres contains repository implementations using Postgres as
// the underlying database.
package postgres
readers/postgres/init.go
-80
View File
@@ -1,80 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package postgres
import (
"fmt"
"github.com/jmoiron/sqlx"
migrate "github.com/rubenv/sql-migrate"
)
// Table for SenML messages.
const defTable = "messages"
// Config defines the options that are used when connecting to a PostgreSQL instance.
type Config struct {
Host string
Port string
User string
Pass string
Name string
SSLMode string
SSLCert string
SSLKey string
SSLRootCert string
}
// Connect creates a connection to the PostgreSQL instance and applies any
// unapplied database migrations. A non-nil error is returned to indicate
// failure.
func Connect(cfg Config) (*sqlx.DB, error) {
url := fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s", cfg.Host, cfg.Port, cfg.User, cfg.Name, cfg.Pass, cfg.SSLMode, cfg.SSLCert, cfg.SSLKey, cfg.SSLRootCert)
db, err := sqlx.Open("pgx", url)
if err != nil {
return nil, err
}
if err := migrateDB(db); err != nil {
return nil, err
}
return db, nil
}
func migrateDB(db *sqlx.DB) error {
migrations := &migrate.MemoryMigrationSource{
Migrations: []*migrate.Migration{
{
Id: "messages_1",
Up: []string{
`CREATE TABLE IF NOT EXISTS messages (
id UUID,
channel UUID,
subtopic VARCHAR(254),
publisher UUID,
protocol TEXT,
name TEXT,
unit TEXT,
value FLOAT,
string_value TEXT,
bool_value BOOL,
data_value TEXT,
sum FLOAT,
time FlOAT,
update_time FLOAT,
PRIMARY KEY (id)
)`,
},
Down: []string{
"DROP TABLE messages",
},
},
},
}
_, err := migrate.Exec(db.DB, "postgres", migrations, migrate.Up)
return err
}
readers/postgres/messages.go
-199
View File
@@ -1,199 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package postgres
import (
"encoding/json"
"fmt"
"github.com/absmach/supermq/pkg/errors"
"github.com/absmach/supermq/pkg/transformers/senml"
"github.com/absmach/supermq/readers"
"github.com/jackc/pgerrcode"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jmoiron/sqlx"
)
var _ readers.MessageRepository = (*postgresRepository)(nil)
type postgresRepository struct {
db *sqlx.DB
}
// New returns new PostgreSQL writer.
func New(db *sqlx.DB) readers.MessageRepository {
return &postgresRepository{
db: db,
}
}
func (tr postgresRepository) ReadAll(chanID string, rpm readers.PageMetadata) (readers.MessagesPage, error) {
order := "time"
format := defTable
if rpm.Format != "" && rpm.Format != defTable {
order = "created"
format = rpm.Format
}
cond := fmtCondition(chanID, rpm)
q := fmt.Sprintf(`SELECT * FROM %s
WHERE %s ORDER BY %s DESC
LIMIT :limit OFFSET :offset;`, format, cond, order)
params := map[string]interface{}{
"channel": chanID,
"limit": rpm.Limit,
"offset": rpm.Offset,
"subtopic": rpm.Subtopic,
"publisher": rpm.Publisher,
"name": rpm.Name,
"protocol": rpm.Protocol,
"value": rpm.Value,
"bool_value": rpm.BoolValue,
"string_value": rpm.StringValue,
"data_value": rpm.DataValue,
"from": rpm.From,
"to": rpm.To,
}
rows, err := tr.db.NamedQuery(q, params)
if err != nil {
if pgErr, ok := err.(*pgconn.PgError); ok {
if pgErr.Code == pgerrcode.UndefinedTable {
return readers.MessagesPage{}, nil
}
}
return readers.MessagesPage{}, errors.Wrap(readers.ErrReadMessages, err)
}
defer rows.Close()
page := readers.MessagesPage{
PageMetadata: rpm,
Messages: []readers.Message{},
}
switch format {
case defTable:
for rows.Next() {
msg := senmlMessage{Message: senml.Message{}}
if err := rows.StructScan(&msg); err != nil {
return readers.MessagesPage{}, errors.Wrap(readers.ErrReadMessages, err)
}
page.Messages = append(page.Messages, msg.Message)
}
default:
for rows.Next() {
msg := jsonMessage{}
if err := rows.StructScan(&msg); err != nil {
return readers.MessagesPage{}, errors.Wrap(readers.ErrReadMessages, err)
}
m, err := msg.toMap()
if err != nil {
return readers.MessagesPage{}, errors.Wrap(readers.ErrReadMessages, err)
}
page.Messages = append(page.Messages, m)
}
}
q = fmt.Sprintf(`SELECT COUNT(*) FROM %s WHERE %s;`, format, cond)
rows, err = tr.db.NamedQuery(q, params)
if err != nil {
return readers.MessagesPage{}, errors.Wrap(readers.ErrReadMessages, err)
}
defer rows.Close()
total := uint64(0)
if rows.Next() {
if err := rows.Scan(&total); err != nil {
return page, err
}
}
page.Total = total
return page, nil
}
func fmtCondition(chanID string, rpm readers.PageMetadata) string {
condition := `channel = :channel`
var query map[string]interface{}
meta, err := json.Marshal(rpm)
if err != nil {
return condition
}
if err := json.Unmarshal(meta, &query); err != nil {
return condition
}
for name := range query {
switch name {
case
"subtopic",
"publisher",
"name",
"protocol":
condition = fmt.Sprintf(`%s AND %s = :%s`, condition, name, name)
case "v":
comparator := readers.ParseValueComparator(query)
condition = fmt.Sprintf(`%s AND value %s :value`, condition, comparator)
case "vb":
condition = fmt.Sprintf(`%s AND bool_value = :bool_value`, condition)
case "vs":
comparator := readers.ParseValueComparator(query)
switch comparator {
case "=":
condition = fmt.Sprintf("%s AND string_value = :string_value ", condition)
case ">":
condition = fmt.Sprintf("%s AND string_value LIKE '%%' || :string_value || '%%' AND string_value <> :string_value", condition)
case ">=":
condition = fmt.Sprintf("%s AND string_value LIKE '%%' || :string_value || '%%'", condition)
case "<=":
condition = fmt.Sprintf("%s AND :string_value LIKE '%%' || string_value || '%%'", condition)
case "<":
condition = fmt.Sprintf("%s AND :string_value LIKE '%%' || string_value || '%%' AND string_value <> :string_value", condition)
}
case "vd":
comparator := readers.ParseValueComparator(query)
condition = fmt.Sprintf(`%s AND data_value %s :data_value`, condition, comparator)
case "from":
condition = fmt.Sprintf(`%s AND time >= :from`, condition)
case "to":
condition = fmt.Sprintf(`%s AND time < :to`, condition)
}
}
return condition
}
type senmlMessage struct {
ID string `db:"id"`
senml.Message
}
type jsonMessage struct {
ID string `db:"id"`
Channel string `db:"channel"`
Created int64 `db:"created"`
Subtopic string `db:"subtopic"`
Publisher string `db:"publisher"`
Protocol string `db:"protocol"`
Payload []byte `db:"payload"`
}
func (msg jsonMessage) toMap() (map[string]interface{}, error) {
ret := map[string]interface{}{
"id": msg.ID,
"channel": msg.Channel,
"created": msg.Created,
"subtopic": msg.Subtopic,
"publisher": msg.Publisher,
"protocol": msg.Protocol,
"payload": map[string]interface{}{},
}
pld := make(map[string]interface{})
if err := json.Unmarshal(msg.Payload, &pld); err != nil {
return nil, err
}
ret["payload"] = pld
return ret, nil
}
readers/postgres/messages_test.go
-687
View File
@@ -1,687 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package postgres_test
import (
"context"
"fmt"
"testing"
"time"
pwriter "github.com/absmach/supermq/consumers/writers/postgres"
"github.com/absmach/supermq/internal/testsutil"
"github.com/absmach/supermq/pkg/transformers/json"
"github.com/absmach/supermq/pkg/transformers/senml"
"github.com/absmach/supermq/readers"
preader "github.com/absmach/supermq/readers/postgres"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
subtopic = "subtopic"
msgsNum = 100
limit = 10
valueFields = 5
mqttProt = "mqtt"
httpProt = "http"
msgName = "temperature"
format1 = "format1"
format2 = "format2"
wrongID = "0"
)
var (
v float64 = 5
vs = "stringValue"
vb = true
vd = "dataValue"
sum float64 = 42
)
func TestReadSenml(t *testing.T) {
writer := pwriter.New(db)
chanID := testsutil.GenerateUUID(t)
pubID := testsutil.GenerateUUID(t)
pubID2 := testsutil.GenerateUUID(t)
wrongID := testsutil.GenerateUUID(t)
m := senml.Message{
Channel: chanID,
Publisher: pubID,
Protocol: mqttProt,
}
messages := []senml.Message{}
valueMsgs := []senml.Message{}
boolMsgs := []senml.Message{}
stringMsgs := []senml.Message{}
dataMsgs := []senml.Message{}
queryMsgs := []senml.Message{}
now := float64(time.Now().Unix())
for i := 0; i < msgsNum; i++ {
// Mix possible values as well as value sum.
msg := m
msg.Time = now - float64(i)
count := i % valueFields
switch count {
case 0:
msg.Value = &v
valueMsgs = append(valueMsgs, msg)
case 1:
msg.BoolValue = &vb
boolMsgs = append(boolMsgs, msg)
case 2:
msg.StringValue = &vs
stringMsgs = append(stringMsgs, msg)
case 3:
msg.DataValue = &vd
dataMsgs = append(dataMsgs, msg)
case 4:
msg.Sum = &sum
msg.Subtopic = subtopic
msg.Protocol = httpProt
msg.Publisher = pubID2
msg.Name = msgName
queryMsgs = append(queryMsgs, msg)
}
messages = append(messages, msg)
}
err := writer.ConsumeBlocking(context.TODO(), messages)
require.Nil(t, err, fmt.Sprintf("expected no error got %s\n", err))
reader := preader.New(db)
// Since messages are not saved in natural order,
// cases that return subset of messages are only
// checking data result set size, but not content.
cases := []struct {
desc string
chanID string
pageMeta readers.PageMetadata
page readers.MessagesPage
}{
{
desc: "read message page for existing channel",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: msgsNum,
},
page: readers.MessagesPage{
Total: msgsNum,
Messages: fromSenml(messages),
},
},
{
desc: "read message page for non-existent channel",
chanID: wrongID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: msgsNum,
},
page: readers.MessagesPage{
Messages: []readers.Message{},
},
},
{
desc: "read message last page",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: msgsNum - 20,
Limit: msgsNum,
},
page: readers.MessagesPage{
Total: msgsNum,
Messages: fromSenml(messages[msgsNum-20 : msgsNum]),
},
},
{
desc: "read message with non-existent subtopic",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: msgsNum,
Subtopic: "not-present",
},
page: readers.MessagesPage{
Messages: []readers.Message{},
},
},
{
desc: "read message with subtopic",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: uint64(len(queryMsgs)),
Subtopic: subtopic,
},
page: readers.MessagesPage{
Total: uint64(len(queryMsgs)),
Messages: fromSenml(queryMsgs),
},
},
{
desc: "read message with publisher",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: uint64(len(queryMsgs)),
Publisher: pubID2,
},
page: readers.MessagesPage{
Total: uint64(len(queryMsgs)),
Messages: fromSenml(queryMsgs),
},
},
{
desc: "read message with wrong format",
chanID: chanID,
pageMeta: readers.PageMetadata{
Format: "messagess",
Offset: 0,
Limit: uint64(len(queryMsgs)),
Publisher: pubID2,
},
page: readers.MessagesPage{
Total: 0,
Messages: []readers.Message{},
},
},
{
desc: "read message with protocol",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: uint64(len(queryMsgs)),
Protocol: httpProt,
},
page: readers.MessagesPage{
Total: uint64(len(queryMsgs)),
Messages: fromSenml(queryMsgs),
},
},
{
desc: "read message with name",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
Name: msgName,
},
page: readers.MessagesPage{
Total: uint64(len(queryMsgs)),
Messages: fromSenml(queryMsgs[0:limit]),
},
},
{
desc: "read message with value",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
Value: v,
},
page: readers.MessagesPage{
Total: uint64(len(valueMsgs)),
Messages: fromSenml(valueMsgs[0:limit]),
},
},
{
desc: "read message with value and equal comparator",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
Value: v,
Comparator: readers.EqualKey,
},
page: readers.MessagesPage{
Total: uint64(len(valueMsgs)),
Messages: fromSenml(valueMsgs[0:limit]),
},
},
{
desc: "read message with value and lower-than comparator",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
Value: v + 1,
Comparator: readers.LowerThanKey,
},
page: readers.MessagesPage{
Total: uint64(len(valueMsgs)),
Messages: fromSenml(valueMsgs[0:limit]),
},
},
{
desc: "read message with value and lower-than-or-equal comparator",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
Value: v + 1,
Comparator: readers.LowerThanEqualKey,
},
page: readers.MessagesPage{
Total: uint64(len(valueMsgs)),
Messages: fromSenml(valueMsgs[0:limit]),
},
},
{
desc: "read message with value and greater-than comparator",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
Value: v - 1,
Comparator: readers.GreaterThanKey,
},
page: readers.MessagesPage{
Total: uint64(len(valueMsgs)),
Messages: fromSenml(valueMsgs[0:limit]),
},
},
{
desc: "read message with value and greater-than-or-equal comparator",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
Value: v - 1,
Comparator: readers.GreaterThanEqualKey,
},
page: readers.MessagesPage{
Total: uint64(len(valueMsgs)),
Messages: fromSenml(valueMsgs[0:limit]),
},
},
{
desc: "read message with boolean value",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
BoolValue: vb,
},
page: readers.MessagesPage{
Total: uint64(len(boolMsgs)),
Messages: fromSenml(boolMsgs[0:limit]),
},
},
{
desc: "read message with string value",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
StringValue: vs,
},
page: readers.MessagesPage{
Total: uint64(len(stringMsgs)),
Messages: fromSenml(stringMsgs[0:limit]),
},
},
{
desc: "read message with string value and equal comparator",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
StringValue: vs,
Comparator: readers.EqualKey,
},
page: readers.MessagesPage{
Total: uint64(len(stringMsgs)),
Messages: fromSenml(stringMsgs[0:limit]),
},
},
{
desc: "read message with string value and lower-than comparator",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
StringValue: "a stringValues b",
Comparator: readers.LowerThanKey,
},
page: readers.MessagesPage{
Total: uint64(len(stringMsgs)),
Messages: fromSenml(stringMsgs[0:limit]),
},
},
{
desc: "read message with string value and lower-than-or-equal comparator",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
StringValue: vs,
Comparator: readers.LowerThanEqualKey,
},
page: readers.MessagesPage{
Total: uint64(len(stringMsgs)),
Messages: fromSenml(stringMsgs[0:limit]),
},
},
{
desc: "read message with string value and greater-than comparator",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
StringValue: "alu",
Comparator: readers.GreaterThanKey,
},
page: readers.MessagesPage{
Total: uint64(len(stringMsgs)),
Messages: fromSenml(stringMsgs[0:limit]),
},
},
{
desc: "read message with string value and greater-than-or-equal comparator",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
StringValue: vs,
Comparator: readers.GreaterThanEqualKey,
},
page: readers.MessagesPage{
Total: uint64(len(stringMsgs)),
Messages: fromSenml(stringMsgs[0:limit]),
},
},
{
desc: "read message with data value",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
DataValue: vd,
},
page: readers.MessagesPage{
Total: uint64(len(dataMsgs)),
Messages: fromSenml(dataMsgs[0:limit]),
},
},
{
desc: "read message with data value and lower-than comparator",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
DataValue: vd + string(rune(1)),
Comparator: readers.LowerThanKey,
},
page: readers.MessagesPage{
Total: uint64(len(dataMsgs)),
Messages: fromSenml(dataMsgs[0:limit]),
},
},
{
desc: "read message with data value and lower-than-or-equal comparator",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
DataValue: vd + string(rune(1)),
Comparator: readers.LowerThanEqualKey,
},
page: readers.MessagesPage{
Total: uint64(len(dataMsgs)),
Messages: fromSenml(dataMsgs[0:limit]),
},
},
{
desc: "read message with data value and greater-than comparator",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
DataValue: vd[:len(vd)-1],
Comparator: readers.GreaterThanKey,
},
page: readers.MessagesPage{
Total: uint64(len(dataMsgs)),
Messages: fromSenml(dataMsgs[0:limit]),
},
},
{
desc: "read message with data value and greater-than-or-equal comparator",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
DataValue: vd[:len(vd)-1],
Comparator: readers.GreaterThanEqualKey,
},
page: readers.MessagesPage{
Total: uint64(len(dataMsgs)),
Messages: fromSenml(dataMsgs[0:limit]),
},
},
{
desc: "read message with from",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: uint64(len(messages[0:21])),
From: messages[20].Time,
},
page: readers.MessagesPage{
Total: uint64(len(messages[0:21])),
Messages: fromSenml(messages[0:21]),
},
},
{
desc: "read message with to",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: uint64(len(messages[21:])),
To: messages[20].Time,
},
page: readers.MessagesPage{
Total: uint64(len(messages[21:])),
Messages: fromSenml(messages[21:]),
},
},
{
desc: "read message with from/to",
chanID: chanID,
pageMeta: readers.PageMetadata{
Offset: 0,
Limit: limit,
From: messages[5].Time,
To: messages[0].Time,
},
page: readers.MessagesPage{
Total: 5,
Messages: fromSenml(messages[1:6]),
},
},
}
for _, tc := range cases {
result, err := reader.ReadAll(tc.chanID, tc.pageMeta)
assert.Nil(t, err, fmt.Sprintf("%s: expected no error got %s", tc.desc, err))
assert.ElementsMatch(t, tc.page.Messages, result.Messages, fmt.Sprintf("%s: got incorrect list of senml Messages from ReadAll()", tc.desc))
assert.Equal(t, tc.page.Total, result.Total, fmt.Sprintf("%s: expected %v got %v", tc.desc, tc.page.Total, result.Total))
}
}
func TestReadJSON(t *testing.T) {
writer := pwriter.New(db)
id1 := testsutil.GenerateUUID(t)
m := json.Message{
Channel: id1,
Publisher: id1,
Created: time.Now().Unix(),
Subtopic: "subtopic/format/some_json",
Protocol: "coap",
Payload: map[string]interface{}{
"field_1": 123.0,
"field_2": "value",
"field_3": false,
"field_4": 12.344,
"field_5": map[string]interface{}{
"field_1": "value",
"field_2": 42.0,
},
},
}
messages1 := json.Messages{
Format: format1,
}
msgs1 := []map[string]interface{}{}
for i := 0; i < msgsNum; i++ {
msg := m
messages1.Data = append(messages1.Data, msg)
m := toMap(msg)
msgs1 = append(msgs1, m)
}
err := writer.ConsumeBlocking(context.TODO(), messages1)
require.Nil(t, err, fmt.Sprintf("expected no error got %s\n", err))
id2 := testsutil.GenerateUUID(t)
m = json.Message{
Channel: id2,
Publisher: id2,
Created: time.Now().Unix(),
Subtopic: "subtopic/other_format/some_other_json",
Protocol: "udp",
Payload: map[string]interface{}{
"field_1": "other_value",
"false_value": false,
"field_pi": 3.14159265,
},
}
messages2 := json.Messages{
Format: format2,
}
msgs2 := []map[string]interface{}{}
for i := 0; i < msgsNum; i++ {
msg := m
if i%2 == 0 {
msg.Protocol = httpProt
}
messages2.Data = append(messages2.Data, msg)
m := toMap(msg)
msgs2 = append(msgs2, m)
}
err = writer.ConsumeBlocking(context.TODO(), messages2)
require.Nil(t, err, fmt.Sprintf("expected no error got %s\n", err))
httpMsgs := []map[string]interface{}{}
for i := 0; i < msgsNum; i += 2 {
httpMsgs = append(httpMsgs, msgs2[i])
}
reader := preader.New(db)
cases := map[string]struct {
chanID string
pageMeta readers.PageMetadata
page readers.MessagesPage
}{
"read message page for existing channel": {
chanID: id1,
pageMeta: readers.PageMetadata{
Format: messages1.Format,
Offset: 0,
Limit: 10,
},
page: readers.MessagesPage{
Total: 100,
Messages: fromJSON(msgs1[:10]),
},
},
"read message page for non-existent channel": {
chanID: wrongID,
pageMeta: readers.PageMetadata{
Format: messages1.Format,
Offset: 0,
Limit: 10,
},
page: readers.MessagesPage{
Messages: []readers.Message{},
},
},
"read message last page": {
chanID: id2,
pageMeta: readers.PageMetadata{
Format: messages2.Format,
Offset: msgsNum - 20,
Limit: msgsNum,
},
page: readers.MessagesPage{
Total: msgsNum,
Messages: fromJSON(msgs2[msgsNum-20 : msgsNum]),
},
},
"read message with protocol": {
chanID: id2,
pageMeta: readers.PageMetadata{
Format: messages2.Format,
Offset: 0,
Limit: uint64(msgsNum / 2),
Protocol: httpProt,
},
page: readers.MessagesPage{
Total: uint64(msgsNum / 2),
Messages: fromJSON(httpMsgs),
},
},
}
for desc, tc := range cases {
result, err := reader.ReadAll(tc.chanID, tc.pageMeta)
for i := 0; i < len(result.Messages); i++ {
m := result.Messages[i]
// Remove id as it is not sent by the client.
delete(m.(map[string]interface{}), "id")
result.Messages[i] = m
}
assert.Nil(t, err, fmt.Sprintf("%s: expected no error got %s", desc, err))
assert.ElementsMatch(t, tc.page.Messages, result.Messages, fmt.Sprintf("%s: got incorrect list of json Messages from ReadAll()", desc))
assert.Equal(t, tc.page.Total, result.Total, fmt.Sprintf("%s: expected %v got %v", desc, tc.page.Total, result.Total))
}
}
func fromSenml(msg []senml.Message) []readers.Message {
var ret []readers.Message
for _, m := range msg {
ret = append(ret, m)
}
return ret
}
func fromJSON(msg []map[string]interface{}) []readers.Message {
var ret []readers.Message
for _, m := range msg {
ret = append(ret, m)
}
return ret
}
func toMap(msg json.Message) map[string]interface{} {
return map[string]interface{}{
"channel": msg.Channel,
"created": msg.Created,
"subtopic": msg.Subtopic,
"publisher": msg.Publisher,
"protocol": msg.Protocol,
"payload": map[string]interface{}(msg.Payload),
}
}
readers/postgres/setup_test.go
-83
View File
@@ -1,83 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package postgres_test contains tests for PostgreSQL repository
// implementations.
package postgres_test
import (
"fmt"
"log"
"os"
"testing"
"github.com/absmach/supermq/readers/postgres"
_ "github.com/jackc/pgx/v5/stdlib" // required for SQL access
"github.com/jmoiron/sqlx"
"github.com/ory/dockertest/v3"
"github.com/ory/dockertest/v3/docker"
)
var db *sqlx.DB
func TestMain(m *testing.M) {
pool, err := dockertest.NewPool("")
if err != nil {
log.Fatalf("Could not connect to docker: %s", err)
}
container, err := pool.RunWithOptions(&dockertest.RunOptions{
Repository: "postgres",
Tag: "16.2-alpine",
Env: []string{
"POSTGRES_USER=test",
"POSTGRES_PASSWORD=test",
"POSTGRES_DB=test",
"listen_addresses = '*'",
},
}, func(config *docker.HostConfig) {
config.AutoRemove = true
config.RestartPolicy = docker.RestartPolicy{Name: "no"}
})
if err != nil {
log.Fatalf("Could not start container: %s", err)
}
port := container.GetPort("5432/tcp")
url := fmt.Sprintf("host=localhost port=%s user=test dbname=test password=test sslmode=disable", port)
if err = pool.Retry(func() error {
db, err = sqlx.Open("pgx", url)
if err != nil {
return err
}
return db.Ping()
}); err != nil {
log.Fatalf("Could not connect to docker: %s", err)
}
dbConfig := postgres.Config{
Host: "localhost",
Port: port,
User: "test",
Pass: "test",
Name: "test",
SSLMode: "disable",
SSLCert: "",
SSLKey: "",
SSLRootCert: "",
}
if db, err = postgres.Connect(dbConfig); err != nil {
log.Fatalf("Could not setup test DB connection: %s", err)
}
code := m.Run()
// Defers will not be run when using os.Exit
db.Close()
if err = pool.Purge(container); err != nil {
log.Fatalf("Could not purge container: %s", err)
}
os.Exit(code)
}
readers/timescale/README.md
-99
View File
@@ -1,99 +0,0 @@
# Timescale reader
Timescale reader provides message repository implementation for Timescale.
## Configuration
The service is configured using the environment variables presented in the
following table. Note that any unset variables will be replaced with their
default values.
| Variable | Description | Default |
| ------------------------------------- | -------------------------------------------- | ---------------------------- |
| SMQ_TIMESCALE_READER_LOG_LEVEL | Service log level | info |
| SMQ_TIMESCALE_READER_HTTP_HOST | Service HTTP host | localhost |
| SMQ_TIMESCALE_READER_HTTP_PORT | Service HTTP port | 8180 |
| SMQ_TIMESCALE_READER_HTTP_SERVER_CERT | Service HTTP server certificate path | "" |
| SMQ_TIMESCALE_READER_HTTP_SERVER_KEY | Service HTTP server key path | "" |
| SMQ_TIMESCALE_HOST | Timescale DB host | localhost |
| SMQ_TIMESCALE_PORT | Timescale DB port | 5432 |
| SMQ_TIMESCALE_USER | Timescale user | supermq |
| SMQ_TIMESCALE_PASS | Timescale password | supermq |
| SMQ_TIMESCALE_NAME | Timescale database name | messages |
| SMQ_TIMESCALE_SSL_MODE | Timescale SSL mode | disabled |
| SMQ_TIMESCALE_SSL_CERT | Timescale SSL certificate path | "" |
| SMQ_TIMESCALE_SSL_KEY | Timescale SSL key | "" |
| SMQ_TIMESCALE_SSL_ROOT_CERT | Timescale SSL root certificate path | "" |
| SMQ_CLIENTS_AUTH_GRPC_URL | Clients service Auth gRPC URL | localhost:7000 |
| SMQ_CLIENTS_AUTH_GRPC_TIMEOUT | Clients service Auth gRPC timeout in seconds | 1s |
| SMQ_CLIENTS_AUTH_GRPC_CLIENT_TLS | Clients service Auth gRPC TLS enabled flag | false |
| SMQ_CLIENTS_AUTH_GRPC_CA_CERTS | Clients service Auth gRPC CA certificates | "" |
| SMQ_AUTH_GRPC_URL | Auth service gRPC URL | localhost:7001 |
| SMQ_AUTH_GRPC_TIMEOUT | Auth service gRPC timeout in seconds | 1s |
| SMQ_AUTH_GRPC_CLIENT_TLS | Auth service gRPC TLS enabled flag | false |
| SMQ_AUTH_GRPC_CA_CERT | Auth service gRPC CA certificate | "" |
| SMQ_JAEGER_URL | Jaeger server URL | http://jaeger:4318/v1/traces |
| SMQ_SEND_TELEMETRY | Send telemetry to supermq call home server | true |
| SMQ_TIMESCALE_READER_INSTANCE_ID | Timescale reader instance ID | "" |
## Deployment
The service itself is distributed as Docker container. Check the [`timescale-reader`](https://github.com/absmach/supermq/blob/main/docker/addons/timescale-reader/docker-compose.yml#L17-L41) service section in docker-compose file to see how service is deployed.
To start the service, execute the following shell script:
```bash
# download the latest version of the service
git clone https://github.com/absmach/supermq
cd supermq
# compile the timescale writer
make timescale-writer
# copy binary to bin
make install
# Set the environment variables and run the service
SMQ_TIMESCALE_READER_LOG_LEVEL=[Service log level] \
SMQ_TIMESCALE_READER_HTTP_HOST=[Service HTTP host] \
SMQ_TIMESCALE_READER_HTTP_PORT=[Service HTTP port] \
SMQ_TIMESCALE_READER_HTTP_SERVER_CERT=[Service HTTP server cert] \
SMQ_TIMESCALE_READER_HTTP_SERVER_KEY=[Service HTTP server key] \
SMQ_TIMESCALE_HOST=[Timescale host] \
SMQ_TIMESCALE_PORT=[Timescale port] \
SMQ_TIMESCALE_USER=[Timescale user] \
SMQ_TIMESCALE_PASS=[Timescale password] \
SMQ_TIMESCALE_NAME=[Timescale database name] \
SMQ_TIMESCALE_SSL_MODE=[Timescale SSL mode] \
SMQ_TIMESCALE_SSL_CERT=[Timescale SSL cert] \
SMQ_TIMESCALE_SSL_KEY=[Timescale SSL key] \
SMQ_TIMESCALE_SSL_ROOT_CERT=[Timescale SSL Root cert] \
SMQ_CLIENTS_AUTH_GRPC_URL=[Clients service Auth GRPC URL] \
SMQ_CLIENTS_AUTH_GRPC_TIMEOUT=[Clients service Auth gRPC request timeout in seconds] \
SMQ_CLIENTS_AUTH_GRPC_CLIENT_TLS=[Clients service Auth gRPC TLS enabled flag] \
SMQ_CLIENTS_AUTH_GRPC_CA_CERTS=[Clients service Auth gRPC CA certificates] \
SMQ_AUTH_GRPC_URL=[Auth service Auth gRPC URL] \
SMQ_AUTH_GRPC_TIMEOUT=[Auth service Auth gRPC request timeout in seconds] \
SMQ_AUTH_GRPC_CLIENT_TLS=[Auth service Auth gRPC TLS enabled flag] \
SMQ_AUTH_GRPC_CA_CERT=[Auth service Auth gRPC CA certificates] \
SMQ_JAEGER_URL=[Jaeger server URL] \
SMQ_SEND_TELEMETRY=[Send telemetry to supermq call home server] \
SMQ_TIMESCALE_READER_INSTANCE_ID=[Timescale reader instance ID] \
$GOBIN/supermq-timescale-reader
```
## Usage
Starting service will start consuming normalized messages in SenML format.
Comparator Usage Guide:
| Comparator | Usage | Example |
| ---------- | --------------------------------------------------------------------------- | ---------------------------------- |
| eq | Return values that are equal to the query | eq["active"] -> "active" |
| ge | Return values that are substrings of the query | ge["tiv"] -> "active" and "tiv" |
| gt | Return values that are substrings of the query and not equal to the query | gt["tiv"] -> "active" |
| le | Return values that are superstrings of the query | le["active"] -> "tiv" |
| lt | Return values that are superstrings of the query and not equal to the query | lt["active"] -> "active" and "tiv" |
Official docs can be found [here](https://docs.supermq.abstractmachines.fr).
readers/timescale/doc.go
-6
View File
@@ -1,6 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package timescale contains repository implementations using Timescale as
// the underlying database.
package timescale
readers/timescale/init.go
-80
View File
@@ -1,80 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package timescale
import (
"fmt"
"github.com/jmoiron/sqlx"
migrate "github.com/rubenv/sql-migrate"
)
// Table for SenML messages.
const defTable = "messages"
// Config defines the options that are used when connecting to a TimescaleSQL instance.
type Config struct {
Host string
Port string
User string
Pass string
Name string
SSLMode string
SSLCert string
SSLKey string
SSLRootCert string
}
// Connect creates a connection to the TimescaleSQL instance and applies any
// unapplied database migrations. A non-nil error is returned to indicate
// failure.
func Connect(cfg Config) (*sqlx.DB, error) {
url := fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=%s sslcert=%s sslkey=%s sslrootcert=%s", cfg.Host, cfg.Port, cfg.User, cfg.Name, cfg.Pass, cfg.SSLMode, cfg.SSLCert, cfg.SSLKey, cfg.SSLRootCert)
db, err := sqlx.Open("pgx", url)
if err != nil {
return nil, err
}
if err := migrateDB(db); err != nil {
return nil, err
}
return db, nil
}
func migrateDB(db *sqlx.DB) error {
migrations := &migrate.MemoryMigrationSource{
Migrations: []*migrate.Migration{
{
Id: "messages_1",
Up: []string{
`CREATE TABLE IF NOT EXISTS messages (
time BIGINT NOT NULL,
channel UUID,
subtopic VARCHAR(254),
publisher UUID,
protocol TEXT,
name VARCHAR(254),
unit TEXT,
value FLOAT,
string_value TEXT,
bool_value BOOL,
data_value BYTEA,
sum FLOAT,
update_time FLOAT,
PRIMARY KEY (time, publisher, subtopic, name)
);
SELECT create_hypertable('messages', 'time', create_default_indexes => FALSE, chunk_time_interval => 86400000, if_not_exists => TRUE);`,
},
Down: []string{
"DROP TABLE messages",
},
},
},
}
_, err := migrate.Exec(db.DB, "postgres", migrations, migrate.Up)
return err
}
readers/timescale/messages.go
-204
View File
@@ -1,204 +0,0 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package timescale
import (
"encoding/json"
"fmt"
"github.com/absmach/supermq/pkg/errors"
"github.com/absmach/supermq/pkg/transformers/senml"
"github.com/absmach/supermq/readers"
"github.com/jackc/pgerrcode"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jmoiron/sqlx" // required for DB access
)
var _ readers.MessageRepository = (*timescaleRepository)(nil)
type timescaleRepository struct {
db *sqlx.DB
}
// New returns new TimescaleSQL writer.
func New(db *sqlx.DB) readers.MessageRepository {
return &timescaleRepository{
db: db,
}
}
func (tr timescaleRepository) ReadAll(chanID string, rpm readers.PageMetadata) (readers.MessagesPage, error) {
order := "time"
format := defTable
if rpm.Format != "" && rpm.Format != defTable {
order = "created"
format = rpm.Format
}
q := fmt.Sprintf(`SELECT * FROM %s WHERE %s ORDER BY %s DESC LIMIT :limit OFFSET :offset;`, format, fmtCondition(rpm), order)
totalQuery := fmt.Sprintf(`SELECT COUNT(*) FROM %s WHERE %s;`, format, fmtCondition(rpm))
// If aggregation is provided, add time_bucket and aggregation to the query
const timeDivisor = 1000000000
if rpm.Aggregation != "" {
q = fmt.Sprintf(`SELECT EXTRACT(epoch FROM time_bucket('%s', to_timestamp(time/%d))) *%d AS time, %s(value) AS value, FIRST(publisher, time) AS publisher, FIRST(protocol, time) AS protocol, FIRST(subtopic, time) AS subtopic, FIRST(name,time) AS name, FIRST(unit, time) AS unit FROM %s WHERE %s GROUP BY 1 ORDER BY time DESC LIMIT :limit OFFSET :offset;`, rpm.Interval, timeDivisor, timeDivisor, rpm.Aggregation, format, fmtCondition(rpm))
totalQuery = fmt.Sprintf(`SELECT COUNT(*) FROM (SELECT EXTRACT(epoch FROM time_bucket('%s', to_timestamp(time/%d))) AS time, %s(value) AS value FROM %s WHERE %s GROUP BY 1) AS subquery;`, rpm.Interval, timeDivisor, rpm.Aggregation, format, fmtCondition(rpm))
}
params := map[string]interface{}{
"channel": chanID,
"limit": rpm.Limit,
"offset": rpm.Offset,
"subtopic": rpm.Subtopic,
"publisher": rpm.Publisher,
"name": rpm.Name,
"protocol": rpm.Protocol,
"value": rpm.Value,
"bool_value": rpm.BoolValue,
"string_value": rpm.StringValue,
"data_value": rpm.DataValue,
"from": rpm.From,
"to": rpm.To,
}
rows, err := tr.db.NamedQuery(q, params)
if err != nil {
if pgErr, ok := err.(*pgconn.PgError); ok {
if pgErr.Code == pgerrcode.UndefinedTable {
return readers.MessagesPage{}, nil
}
}
return readers.MessagesPage{}, errors.Wrap(readers.ErrReadMessages, err)
}
defer rows.Close()
page := readers.MessagesPage{
PageMetadata: rpm,
Messages: []readers.Message{},
}
switch format {
case defTable:
for rows.Next() {
msg := senmlMessage{Message: senml.Message{}}
if err := rows.StructScan(&msg); err != nil {
return readers.MessagesPage{}, errors.Wrap(readers.ErrReadMessages, err)
}
page.Messages = append(page.Messages, msg.Message)
}
default:
for rows.Next() {
msg := jsonMessage{}
if err := rows.StructScan(&msg); err != nil {
return readers.MessagesPage{}, errors.Wrap(readers.ErrReadMessages, err)
}
m, err := msg.toMap()
if err != nil {
return readers.MessagesPage{}, errors.Wrap(readers.ErrReadMessages, err)
}
page.Messages = append(page.Messages, m)
}
}
rows, err = tr.db.NamedQuery(totalQuery, params)
if err != nil {
return readers.MessagesPage{}, errors.Wrap(readers.ErrReadMessages, err)
}
defer rows.Close()
total := uint64(0)
if rows.Next() {
if err := rows.Scan(&total); err != nil {
return page, err
}
}
page.Total = total
return page, nil
}
func fmtCondition(rpm readers.PageMetadata) string {
condition := `channel = :channel`
var query map[string]interface{}
meta, err := json.Marshal(rpm)
if err != nil {
return condition
}
if err := json.Unmarshal(meta, &query); err != nil {
return condition
}
for name := range query {
switch name {
case
"subtopic",
"publisher",
"name",
"protocol":
condition = fmt.Sprintf(`%s AND %s = :%s`, condition, name, name)
case "v":
comparator := readers.ParseValueComparator(query)
condition = fmt.Sprintf(`%s AND value %s :value`, condition, comparator)
case "vb":
condition = fmt.Sprintf(`%s AND bool_value = :bool_value`, condition)
case "vs":
comparator := readers.ParseValueComparator(query)
switch comparator {
case "=":
condition = fmt.Sprintf("%s AND string_value = :string_value ", condition)
case ">":
condition = fmt.Sprintf("%s AND string_value LIKE '%%' || :string_value || '%%' AND string_value <> :string_value", condition)
case ">=":
condition = fmt.Sprintf("%s AND string_value LIKE '%%' || :string_value || '%%'", condition)
case "<=":
condition = fmt.Sprintf("%s AND :string_value LIKE '%%' || string_value || '%%'", condition)
case "<":
condition = fmt.Sprintf("%s AND :string_value LIKE '%%' || string_value || '%%' AND string_value <> :string_value", condition)
}
case "vd":
comparator := readers.ParseValueComparator(query)
condition = fmt.Sprintf(`%s AND data_value %s :data_value`, condition, comparator)
case "from":
condition = fmt.Sprintf(`%s AND time >= :from`, condition)
case "to":
condition = fmt.Sprintf(`%s AND time < :to`, condition)
}
}
return condition
}
type senmlMessage struct {
ID string `db:"id"`
senml.Message
}
type jsonMessage struct {
Channel string `db:"channel"`
Created int64 `db:"created"`
Subtopic string `db:"subtopic"`
Publisher string `db:"publisher"`
Protocol string `db:"protocol"`
Payload []byte `db:"payload"`
}
func (msg jsonMessage) toMap() (map[string]interface{}, error) {
ret := map[string]interface{}{
"channel": msg.Channel,
"created": msg.Created,
"subtopic": msg.Subtopic,
"publisher": msg.Publisher,
"protocol": msg.Protocol,
"payload": map[string]interface{}{},
}
pld := make(map[string]interface{})
if err := json.Unmarshal(msg.Payload, &pld); err != nil {
return nil, err
}
ret["payload"] = pld
return ret, nil
}

Some files were not shown because too many files have changed in this diff Show More