opensource/supermq
1
0
Fork 0
mirror of https://github.com/absmach/supermq.git synced 2026-06-23 06:40:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

MG-1965 - Process Event Logs (#2057)

Browse Source 
Signed-off-by: Rodney Osodo <28790446+rodneyosodo@users.noreply.github.com>
Signed-off-by: Rodney Osodo <socials@rodneyosodo.com>
This commit is contained in:
b1ackd0t
2024-06-27 17:38:20 +03:00
committed by GitHub
parent b0e37dacf4
commit 0794363a3c
66 changed files with 3968 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/api-tests.yml
+18
View File
@@ -16,6 +16,7 @@ on:
- "consumers/notifiers/api/**"
- "http/api/**"
- "invitations/api/**"
- "journal/api/**"
- "provision/api/**"
- "readers/api/**"
- "things/api/**"
@@ -44,6 +45,7 @@ env:
TIMESCALE_READER_URL: http://localhost:9011
SMPP_NOTIFIER_URL: http://localhost:9014
SMTP_NOTIFIER_URL: http://localhost:9015
JOURNAL_URL: http://localhost:9021
jobs:
api-test:
@@ -78,6 +80,11 @@ jobs:
id: changes
with:
filters: |
journal:
- ".github/workflows/api-tests.yml"
- "api/openapi/journal.yml"
- "journal/api/**"
auth:
- ".github/workflows/api-tests.yml"
- "api/openapi/auth.yml"
@@ -183,6 +190,17 @@ jobs:
report: false
args: '--header "Authorization: Bearer ${{ env.USER_TOKEN }}" --contrib-openapi-formats-uuid --hypothesis-suppress-health-check=filter_too_much --stateful=links'
- name: Run Journal API tests
if: steps.changes.outputs.journal == 'true'
uses: schemathesis/action@v1
with:
schema: api/openapi/journal.yml
base-url: ${{ env.JOURNAL_URL }}
checks: all
report: false
args: '--header "Authorization: Bearer ${{ env.USER_TOKEN }}" --contrib-openapi-formats-uuid --hypothesis-suppress-health-check=filter_too_much --stateful=links'
- name: Run Bootstrap API tests
if: steps.changes.outputs.bootstrap == 'true'
uses: schemathesis/action@v1
.github/workflows/check-generated-files.yml
+5
View File
@@ -75,6 +75,7 @@ jobs:
- "twins/twins.go"
- "twins/states.go"
- "twins/service.go"
- "journal/journal.go"
- name: Set up protoc
if: steps.changes.outputs.proto == 'true'
@@ -157,6 +158,8 @@ jobs:
mv ./twins/mocks/states.go ./twins/mocks/states.go.tmp
mv ./twins/mocks/repository.go ./twins/mocks/repository.go.tmp
mv ./twins/mocks/cache.go ./twins/mocks/cache.go.tmp
mv ./journal/mocks/repository.go ./journal/mocks/repository.go.tmp
mv ./journal/mocks/service.go ./journal/mocks/service.go.tmp
make mocks
@@ -208,3 +211,5 @@ jobs:
check_mock_changes ./twins/mocks/states.go "Twins States ./twins/mocks/states.go"
check_mock_changes ./twins/mocks/repository.go "Twins Repository ./twins/mocks/repository.go"
check_mock_changes ./twins/mocks/cache.go "Twins Cache ./twins/mocks/cache.go"
check_mock_changes ./journal/mocks/repository.go "Journal Repository ./journal/mocks/repository.go"
check_mock_changes ./journal/mocks/service.go "Journal Service ./journal/mocks/service.go"
.github/workflows/tests.yml
+13
View File
@@ -116,6 +116,14 @@ jobs:
- "pkg/uuid/**"
- "pkg/messaging/**"
journal:
- "journal/**"
- "cmd/journal/**"
- "auth.pb.go"
- "auth_grpc.pb.go"
- "auth/**"
- "pkg/events/**"
http:
- "http/**"
- "cmd/http/**"
@@ -268,6 +276,11 @@ jobs:
run: |
mkdir coverage
- name: Run Journal tests
if: steps.changes.outputs.journal == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/journal.out ./journal/...
- name: Run auth tests
if: steps.changes.outputs.auth == 'true' || steps.changes.outputs.workflow == 'true'
run: |
Makefile
+4 -3
View File
@@ -5,8 +5,8 @@ MG_DOCKER_IMAGE_NAME_PREFIX ?= magistrala
BUILD_DIR = build
SERVICES = auth users things http coap ws lora influxdb-writer influxdb-reader mongodb-writer \
mongodb-reader cassandra-writer cassandra-reader postgres-writer postgres-reader timescale-writer timescale-reader cli \
bootstrap opcua twins mqtt provision certs smtp-notifier smpp-notifier invitations
TEST_API_SERVICES = auth bootstrap certs http invitations notifiers provision readers things twins users
bootstrap opcua twins mqtt provision certs smtp-notifier smpp-notifier invitations journal
TEST_API_SERVICES = journal auth bootstrap certs http invitations notifiers provision readers things twins users
TEST_API = $(addprefix test_api_,$(TEST_API_SERVICES))
DOCKERS = $(addprefix docker_,$(SERVICES))
DOCKERS_DEV = $(addprefix docker_dev_,$(SERVICES))
@@ -73,7 +73,7 @@ endef
ADDON_SERVICES = bootstrap cassandra-reader cassandra-writer certs \
influxdb-reader influxdb-writer lora-adapter mongodb-reader mongodb-writer \
opcua-adapter postgres-reader postgres-writer provision smpp-notifier smtp-notifier \
timescale-reader timescale-writer twins
timescale-reader timescale-writer twins journal
EXTERNAL_SERVICES = vault prometheus
@@ -177,6 +177,7 @@ test_api_twins: TEST_API_URL := http://localhost:9018
test_api_provision: TEST_API_URL := http://localhost:9016
test_api_readers: TEST_API_URL := http://localhost:9009 # This can be the URL of any reader service.
test_api_notifiers: TEST_API_URL := http://localhost:9014 # This can be the URL of any notifier service.
test_api_journal: TEST_API_URL := http://localhost:9021
$(TEST_API):
$(call test_api_service,$(@),$(TEST_API_URL))
api/openapi/journal.yml
+285
View File
@@ -0,0 +1,285 @@
# Copyright (c) Abstract Machines
# SPDX-License-Identifier: Apache-2.0
openapi: 3.0.3
info:
title: Magistrala Journal Log Service
description: |
This is the Journal Log Server based on the OpenAPI 3.0 specification. It is the HTTP API for viewing journal log history. You can now help us improve the API whether it's by making changes to the definition itself or to the code.
Some useful links:
- [The Magistrala repository](https://github.com/absmach/magistrala)
contact:
email: info@mainflux.com
license:
name: Apache 2.0
url: https://github.com/absmach/magistrala/blob/master/LICENSE
version: 0.14.0
servers:
- url: http://localhost:9021
- url: https://localhost:9021
tags:
- name: journal-log
description: Everything about your Journal Log
externalDocs:
description: Find out more about Journal Log
url: http://docs.mainflux.io/
paths:
/journal/{entity_type}/{id}:
get:
tags:
- journal-log
summary: List journal log
description: |
Retrieves a list of journal. Due to performance concerns, data
is retrieved in subsets. The API must ensure that the entire
dataset is consumed either by making subsequent requests, or by
increasing the subset size of the initial request.
parameters:
- $ref: "#/components/parameters/entity_type"
- $ref: "#/components/parameters/id"
- $ref: "#/components/parameters/offset"
- $ref: "#/components/parameters/limit"
- $ref: "#/components/parameters/operation"
- $ref: "#/components/parameters/with_attributes"
- $ref: "#/components/parameters/with_metadata"
- $ref: "#/components/parameters/from"
- $ref: "#/components/parameters/to"
- $ref: "#/components/parameters/dir"
security:
- bearerAuth: []
responses:
"200":
$ref: "#/components/responses/JournalsPageRes"
"400":
description: Failed due to malformed query parameters.
"401":
description: Missing or invalid access token provided.
"403":
description: Failed to perform authorization over the entity.
"404":
description: A non-existent entity request.
"422":
description: Database can't process request.
"500":
$ref: "#/components/responses/ServiceError"
/health:
get:
summary: Retrieves service health check info.
tags:
- health
responses:
"200":
$ref: "#/components/responses/HealthRes"
"500":
$ref: "#/components/responses/ServiceError"
components:
schemas:
Journal:
type: object
properties:
operation:
type: string
example: user.create
description: Journal operation.
occurred_at:
type: string
format: date-time
example: "2024-01-11T12:05:07.449053Z"
description: Time when the journal occurred.
attributes:
type: object
description: Journal attributes.
example:
{
"created_at": "2024-06-12T11:34:32.991591Z",
"id": "29d425c8-542b-4614-8a4d-a5951945d720",
"identity": "Gawne-Havlicek@email.com",
"name": "Newgard-Frisina",
"status": "enabled",
"updated_at": "2024-06-12T11:34:33.116795Z",
"updated_by": "ad228f20-4741-47c5-bef7-d871b541c019",
}
metadata:
type: object
description: Journal payload.
example: { "Update": "Calvo-Felkins" }
xml:
name: journal
JournalPage:
type: object
properties:
journals:
type: array
minItems: 0
uniqueItems: true
items:
$ref: "#/components/schemas/Journal"
total:
type: integer
example: 1
description: Total number of items.
offset:
type: integer
description: Number of items to skip during retrieval.
limit:
type: integer
example: 10
description: Maximum number of items to return in one page.
required:
- journals
- total
- offset
Error:
type: object
properties:
error:
type: string
description: Error message
example: { "error": "malformed entity specification" }
parameters:
entity_type:
name: entity_type
description: Type of entity, e.g. user, group, thing, etc.
in: path
schema:
type: string
enum:
- user
- group
- thing
- channel
required: true
example: user
id:
name: id
description: Unique identifier for an entity, e.g. user, group, domain, etc. Used together with entity_type.
in: path
schema:
type: string
format: uuid
required: true
example: bb7edb32-2eac-4aad-aebe-ed96fe073879
offset:
name: offset
description: Number of items to skip during retrieval.
in: query
schema:
type: integer
default: 0
minimum: 0
required: false
example: "0"
limit:
name: limit
description: Size of the subset to retrieve.
in: query
schema:
type: integer
default: 10
maximum: 10
minimum: 1
required: false
example: "10"
operation:
name: operation
description: Journal operation.
in: query
schema:
type: string
required: false
example: user.create
with_attributes:
name: with_attributes
description: Include journal attributes.
in: query
schema:
type: boolean
required: false
example: true
with_metadata:
name: with_metadata
description: Include journal metadata.
in: query
schema:
type: boolean
required: false
example: true
from:
name: from
description: Start date in unix time.
in: query
schema:
type: string
format: int64
required: false
example: 1966777289
to:
name: to
description: End date in unix time.
in: query
schema:
type: string
format: int64
required: false
example: 1966777289
dir:
name: dir
description: Sort direction.
in: query
schema:
type: string
enum:
- asc
- desc
required: false
example: desc
responses:
JournalsPageRes:
description: Data retrieved.
content:
application/json:
schema:
$ref: "#/components/schemas/JournalPage"
HealthRes:
description: Service Health Check.
content:
application/health+json:
schema:
$ref: "./schemas/HealthInfo.yml"
ServiceError:
description: Unexpected server-side error occurred.
content:
application/json:
schema:
$ref: "#/components/schemas/Error"
securitySchemes:
bearerAuth:
type: http
scheme: bearer
bearerFormat: JWT
description: |
* User access: "Authorization: Bearer <user_access_token>"
security:
- bearerAuth: []
api/openapi/notifiers.yml
+3 -1
View File
@@ -43,6 +43,8 @@ paths:
$ref: "#/components/responses/Create"
"400":
description: Failed due to malformed JSON.
"401":
description: Missing or invalid access token provided.
"403":
description: Failed to perform authorization over the entity.
"409":
@@ -92,7 +94,7 @@ paths:
"200":
$ref: "#/components/responses/View"
"400":
description: Failed due to malformed query parameters.
description: Failed due to malformed ID.
"401":
description: Missing or invalid access token provided.
"403":
auth/events/events.go
+9 -45
View File
@@ -4,9 +4,6 @@
package events
import (
"encoding/json"
"fmt"
"strings"
"time"
"github.com/absmach/magistrala/auth"
@@ -59,16 +56,10 @@ func (cde createDomainEvent) Encode() (map[string]interface{}, error) {
val["permission"] = cde.Permission
}
if len(cde.Tags) > 0 {
tags := fmt.Sprintf("[%s]", strings.Join(cde.Tags, ","))
val["tags"] = tags
val["tags"] = cde.Tags
}
if cde.Metadata != nil {
metadata, err := json.Marshal(cde.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = cde.Metadata
}
return val, nil
@@ -91,16 +82,10 @@ func (rde retrieveDomainEvent) Encode() (map[string]interface{}, error) {
val["name"] = rde.Name
}
if len(rde.Tags) > 0 {
tags := fmt.Sprintf("[%s]", strings.Join(rde.Tags, ","))
val["tags"] = tags
val["tags"] = rde.Tags
}
if rde.Metadata != nil {
metadata, err := json.Marshal(rde.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = rde.Metadata
}
if !rde.UpdatedAt.IsZero() {
@@ -124,12 +109,7 @@ func (rpe retrieveDomainPermissionsEvent) Encode() (map[string]interface{}, erro
}
if rpe.permissions != nil {
permissions, err := json.Marshal(rpe.permissions)
if err != nil {
return map[string]interface{}{}, err
}
val["permissions"] = permissions
val["permissions"] = rpe.permissions
}
return val, nil
@@ -155,16 +135,10 @@ func (ude updateDomainEvent) Encode() (map[string]interface{}, error) {
val["name"] = ude.Name
}
if len(ude.Tags) > 0 {
tags := fmt.Sprintf("[%s]", strings.Join(ude.Tags, ","))
val["tags"] = tags
val["tags"] = ude.Tags
}
if ude.Metadata != nil {
metadata, err := json.Marshal(ude.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = ude.Metadata
}
return val, nil
@@ -210,12 +184,7 @@ func (lde listDomainsEvent) Encode() (map[string]interface{}, error) {
val["dir"] = lde.Dir
}
if lde.Metadata != nil {
metadata, err := json.Marshal(lde.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = lde.Metadata
}
if lde.Tag != "" {
val["tag"] = lde.Tag
@@ -298,12 +267,7 @@ func (lde listUserDomainsEvent) Encode() (map[string]interface{}, error) {
val["dir"] = lde.Dir
}
if lde.Metadata != nil {
metadata, err := json.Marshal(lde.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = lde.Metadata
}
if lde.Tag != "" {
val["tag"] = lde.Tag
bootstrap/events/consumer/streams.go
+18 -55
View File
@@ -5,7 +5,6 @@ package consumer
import (
"context"
"encoding/json"
"time"
"github.com/absmach/magistrala/bootstrap"
@@ -69,6 +68,9 @@ func (es *eventHandler) Handle(ctx context.Context, event events.Event) error {
if thingID == "" {
return svcerr.ErrMalformedEntity
}
}
for _, thingID := range dte.thingIDs {
if err = es.svc.DisconnectThingHandler(ctx, dte.channelID, thingID); err != nil {
return err
}
@@ -89,50 +91,47 @@ func (es *eventHandler) Handle(ctx context.Context, event events.Event) error {
func decodeRemoveThing(event map[string]interface{}) removeEvent {
return removeEvent{
id: read(event, "id", ""),
id: events.Read(event, "id", ""),
}
}
func decodeUpdateChannel(event map[string]interface{}) updateChannelEvent {
strmeta := read(event, "metadata", "{}")
var metadata map[string]interface{}
if err := json.Unmarshal([]byte(strmeta), &metadata); err != nil {
metadata = map[string]interface{}{}
}
metadata := events.Read(event, "metadata", map[string]interface{}{})
return updateChannelEvent{
id: read(event, "id", ""),
name: read(event, "name", ""),
id: events.Read(event, "id", ""),
name: events.Read(event, "name", ""),
metadata: metadata,
updatedAt: readTime(event, "updated_at", time.Now()),
updatedBy: read(event, "updated_by", ""),
updatedAt: events.Read(event, "updated_at", time.Now()),
updatedBy: events.Read(event, "updated_by", ""),
}
}
func decodeRemoveChannel(event map[string]interface{}) removeEvent {
return removeEvent{
id: read(event, "id", ""),
id: events.Read(event, "id", ""),
}
}
func decodeConnectThing(event map[string]interface{}) connectionEvent {
if read(event, "memberKind", "") != memberKind && read(event, "relation", "") != relation {
if events.Read(event, "memberKind", "") != memberKind && events.Read(event, "relation", "") != relation {
return connectionEvent{}
}
return connectionEvent{
channelID: read(event, "group_id", ""),
thingIDs: ReadStringSlice(event, "member_ids"),
channelID: events.Read(event, "group_id", ""),
thingIDs: events.ReadStringSlice(event, "member_ids"),
}
}
func decodeDisconnectThing(event map[string]interface{}) connectionEvent {
if read(event, "memberKind", "") != memberKind && read(event, "relation", "") != relation {
if events.Read(event, "memberKind", "") != memberKind && events.Read(event, "relation", "") != relation {
return connectionEvent{}
}
return connectionEvent{
channelID: read(event, "group_id", ""),
thingIDs: ReadStringSlice(event, "member_ids"),
channelID: events.Read(event, "group_id", ""),
thingIDs: events.ReadStringSlice(event, "member_ids"),
}
}
@@ -144,42 +143,6 @@ func (es *eventHandler) handleUpdateChannel(ctx context.Context, uce updateChann
UpdatedAt: uce.updatedAt,
UpdatedBy: uce.updatedBy,
}
return es.svc.UpdateChannelHandler(ctx, channel)
}
func read(event map[string]interface{}, key, def string) string {
val, ok := event[key].(string)
if !ok {
return def
}
return val
}
// ReadStringSlice reads string slice from event map.
// If value is not a string slice, returns empty slice.
func ReadStringSlice(event map[string]interface{}, key string) []string {
var res []string
vals, ok := event[key].([]interface{})
if !ok {
return res
}
for _, v := range vals {
if s, ok := v.(string); ok {
res = append(res, s)
}
}
return res
}
func readTime(event map[string]interface{}, key string, def time.Time) time.Time {
val, ok := event[key].(time.Time)
if !ok {
return def
}
return val
}
bootstrap/events/producer/events.go
+10 -40
View File
@@ -4,14 +4,12 @@
package producer
import (
"encoding/json"
"github.com/absmach/magistrala/bootstrap"
"github.com/absmach/magistrala/pkg/events"
)
const (
configPrefix = "config."
configPrefix = "bootstrap.config."
configCreate = configPrefix + "create"
configUpdate = configPrefix + "update"
configRemove = configPrefix + "remove"
@@ -19,18 +17,18 @@ const (
configList = configPrefix + "list"
configHandlerRemove = configPrefix + "remove_handler"
thingPrefix = "thing."
thingPrefix = "bootstrap.thing."
thingBootstrap = thingPrefix + "bootstrap"
thingStateChange = thingPrefix + "change_state"
thingUpdateConnections = thingPrefix + "update_connections"
thingConnect = thingPrefix + "connect"
thingDisconnect = thingPrefix + "disconnect"
channelPrefix = "group."
channelPrefix = "bootstrap.channel."
channelHandlerRemove = channelPrefix + "remove_handler"
channelUpdateHandler = channelPrefix + "update_handler"
certUpdate = "cert.update"
certUpdate = "bootstrap.cert.update"
)
var (
@@ -74,11 +72,7 @@ func (ce configEvent) Encode() (map[string]interface{}, error) {
for i, ch := range ce.Channels {
channels[i] = ch.ID
}
data, err := json.Marshal(channels)
if err != nil {
return map[string]interface{}{}, err
}
val["channels"] = string(data)
val["channels"] = channels
}
if ce.ClientCert != "" {
val["client_cert"] = ce.ClientCert
@@ -121,21 +115,11 @@ func (rce listConfigsEvent) Encode() (map[string]interface{}, error) {
"operation": configList,
}
if len(rce.fullMatch) > 0 {
data, err := json.Marshal(rce.fullMatch)
if err != nil {
return map[string]interface{}{}, err
}
val["full_match"] = data
val["full_match"] = rce.fullMatch
}
if len(rce.partialMatch) > 0 {
data, err := json.Marshal(rce.partialMatch)
if err != nil {
return map[string]interface{}{}, err
}
val["full_match"] = data
val["full_match"] = rce.partialMatch
}
return val, nil
}
@@ -173,11 +157,7 @@ func (be bootstrapEvent) Encode() (map[string]interface{}, error) {
for i, ch := range be.Channels {
channels[i] = ch.ID
}
data, err := json.Marshal(channels)
if err != nil {
return map[string]interface{}{}, err
}
val["channels"] = string(data)
val["channels"] = channels
}
if be.ClientCert != "" {
val["client_cert"] = be.ClientCert
@@ -213,14 +193,9 @@ type updateConnectionsEvent struct {
}
func (uce updateConnectionsEvent) Encode() (map[string]interface{}, error) {
data, err := json.Marshal(uce.mgChannels)
if err != nil {
return map[string]interface{}{}, err
}
return map[string]interface{}{
"thing_id": uce.mgThing,
"channels": string(data),
"channels": uce.mgChannels,
"operation": thingUpdateConnections,
}, nil
}
@@ -267,12 +242,7 @@ func (uche updateChannelHandlerEvent) Encode() (map[string]interface{}, error) {
val["name"] = uche.Name
}
if uche.Metadata != nil {
metadata, err := json.Marshal(uche.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = uche.Metadata
}
return val, nil
}
bootstrap/events/producer/streams_test.go
+10 -20
View File
@@ -5,7 +5,6 @@ package producer_test
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
@@ -123,7 +122,7 @@ func TestAdd(t *testing.T) {
"thing_id": "1",
"owner": email,
"name": config.Name,
"channels": strings.Join(channels, ", "),
"channels": channels,
"external_id": config.ExternalID,
"content": config.Content,
"timestamp": time.Now().Unix(),
@@ -238,8 +237,6 @@ func TestUpdate(t *testing.T) {
nonExisting.ThingID = "unknown"
channels := []string{modified.Channels[0].ID, modified.Channels[1].ID}
chs, err := json.Marshal(channels)
assert.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err))
cases := []struct {
desc string
@@ -258,7 +255,7 @@ func TestUpdate(t *testing.T) {
"content": modified.Content,
"timestamp": time.Now().UnixNano(),
"operation": configUpdate,
"channels": string(chs),
"channels": channels,
"external_id": modified.ExternalID,
"thing_id": modified.ThingID,
"owner": validID,
@@ -340,7 +337,7 @@ func TestUpdateConnections(t *testing.T) {
err: nil,
event: map[string]interface{}{
"thing_id": saved.ThingID,
"channels": "2",
"channels": []string{"2"},
"timestamp": time.Now().Unix(),
"operation": thingUpdateConnections,
},
@@ -1226,21 +1223,14 @@ func test(t *testing.T, expected, actual map[string]interface{}, description str
delete(actual, "occurred_at")
}
if expected["channels"] != nil || actual["channels"] != nil {
ech := expected["channels"]
ach := actual["channels"]
exchs := expected["channels"].([]interface{})
achs := actual["channels"].([]interface{})
che := []string{}
err = json.Unmarshal([]byte(ech.(string)), &che)
require.Nil(t, err, fmt.Sprintf("%s: expected to get a valid channels, got %s", description, err))
cha := []string{}
err = json.Unmarshal([]byte(ach.(string)), &cha)
require.Nil(t, err, fmt.Sprintf("%s: expected to get a valid channels, got %s", description, err))
if assert.ElementsMatchf(t, che, cha, "%s: got incorrect channels\n", description) {
delete(expected, "channels")
delete(actual, "channels")
if exchs != nil && achs != nil {
if assert.Len(t, exchs, len(achs), fmt.Sprintf("%s: got incorrect number of channels\n", description)) {
for _, exch := range exchs {
assert.Contains(t, achs, exch, fmt.Sprintf("%s: got incorrect channel\n", description))
}
}
}
cli/config.go
+7
View File
@@ -27,6 +27,7 @@ const (
defCertsURL string = defURL + ":9019"
defInvitationsURL string = defURL + ":9020"
defHTTPURL string = defURL + ":8008"
defJournalURL string = defURL + ":9021"
defTLSVerification bool = false
defOffset string = "0"
defLimit string = "10"
@@ -43,6 +44,7 @@ type remotes struct {
BootstrapURL string `toml:"bootstrap_url"`
CertsURL string `toml:"certs_url"`
InvitationsURL string `toml:"invitations_url"`
JournalURL string `toml:"journal_url"`
TLSVerification bool `toml:"tls_verification"`
}
@@ -112,6 +114,7 @@ func ParseConfig(sdkConf mgxsdk.Config) (mgxsdk.Config, error) {
BootstrapURL: defBootstrapURL,
CertsURL: defCertsURL,
InvitationsURL: defInvitationsURL,
JournalURL: defJournalURL,
TLSVerification: defTLSVerification,
},
Filter: filter{
@@ -198,6 +201,10 @@ func ParseConfig(sdkConf mgxsdk.Config) (mgxsdk.Config, error) {
sdkConf.InvitationsURL = config.Remotes.InvitationsURL
}
if sdkConf.JournalURL == "" && config.Remotes.JournalURL != "" {
sdkConf.JournalURL = config.Remotes.JournalURL
}
sdkConf.TLSVerification = config.Remotes.TLSVerification || sdkConf.TLSVerification
return sdkConf, nil
cli/journal.go
+50
View File
@@ -0,0 +1,50 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package cli
import (
mgxsdk "github.com/absmach/magistrala/pkg/sdk/go"
"github.com/spf13/cobra"
)
var cmdJournal = cobra.Command{
Use: "get <entity_type> <entity_id> <user_auth_token>",
Short: "Get journal",
Long: "Get journal\n" +
"Usage:\n" +
"\tmagistrala-cli journal get <entity_type> <entity_id> <user_auth_token> - lists journal logs\n" +
"\tmagistrala-cli journal get <entity_type> <entity_id> <user_auth_token> --offset <offset> --limit <limit> - lists journal logs with provided offset and limit\n",
Run: func(cmd *cobra.Command, args []string) {
if len(args) != 3 {
logUsage(cmd.Use)
return
}
pageMetadata := mgxsdk.PageMetadata{
Offset: Offset,
Limit: Limit,
}
journal, err := sdk.Journal(args[0], args[1], pageMetadata, args[2])
if err != nil {
logError(err)
return
}
logJSON(journal)
},
}
// NewJournalCmd returns journal log command.
func NewJournalCmd() *cobra.Command {
cmd := cobra.Command{
Use: "journal get",
Short: "journal log",
Long: `journal to read journal log`,
}
cmd.AddCommand(&cmdJournal)
return &cmd
}
cmd/cli/main.go
+11 -3
View File
@@ -12,9 +12,7 @@ import (
"github.com/spf13/cobra"
)
const (
defURL string = "http://localhost"
)
const defURL string = "http://localhost"
func main() {
msgContentType := string(sdk.CTJSONSenML)
@@ -52,6 +50,7 @@ func main() {
subscriptionsCmd := cli.NewSubscriptionCmd()
configCmd := cli.NewConfigCmd()
invitationsCmd := cli.NewInvitationsCmd()
journalCmd := cli.NewJournalCmd()
// Root Commands
rootCmd.AddCommand(healthCmd)
@@ -67,6 +66,7 @@ func main() {
rootCmd.AddCommand(subscriptionsCmd)
rootCmd.AddCommand(configCmd)
rootCmd.AddCommand(invitationsCmd)
rootCmd.AddCommand(journalCmd)
// Root Flags
rootCmd.PersistentFlags().StringVarP(
@@ -133,6 +133,14 @@ func main() {
"Inivitations URL",
)
rootCmd.PersistentFlags().StringVarP(
&sdkConf.JournalURL,
"journal-url",
"a",
sdkConf.JournalURL,
"Journal Log URL",
)
rootCmd.PersistentFlags().StringVarP(
&sdkConf.HostURL,
"host-url",
cmd/journal/main.go
+181
View File
@@ -0,0 +1,181 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package main contains journal main function to start the journal service.
package main
import (
"context"
"fmt"
"log"
"log/slog"
"net/url"
"os"
chclient "github.com/absmach/callhome/pkg/client"
"github.com/absmach/magistrala"
"github.com/absmach/magistrala/internal"
jaegerclient "github.com/absmach/magistrala/internal/clients/jaeger"
pgclient "github.com/absmach/magistrala/internal/clients/postgres"
"github.com/absmach/magistrala/internal/postgres"
"github.com/absmach/magistrala/internal/server"
httpserver "github.com/absmach/magistrala/internal/server/http"
"github.com/absmach/magistrala/journal"
"github.com/absmach/magistrala/journal/api"
"github.com/absmach/magistrala/journal/events"
"github.com/absmach/magistrala/journal/middleware"
journalpg "github.com/absmach/magistrala/journal/postgres"
mglog "github.com/absmach/magistrala/logger"
"github.com/absmach/magistrala/pkg/auth"
"github.com/absmach/magistrala/pkg/events/store"
"github.com/absmach/magistrala/pkg/uuid"
"github.com/caarlos0/env/v10"
"github.com/jmoiron/sqlx"
"go.opentelemetry.io/otel/trace"
"golang.org/x/sync/errgroup"
)
const (
svcName = "journal"
envPrefixDB = "MG_JOURNAL_"
envPrefixHTTP = "MG_JOURNAL_HTTP_"
envPrefixAuth = "MG_AUTH_GRPC_"
defDB = "journal"
defSvcHTTPPort = "9021"
)
type config struct {
LogLevel string `env:"MG_JOURNAL_LOG_LEVEL" envDefault:"info"`
ESURL string `env:"MG_ES_URL" envDefault:"nats://localhost:4222"`
JaegerURL url.URL `env:"MG_JAEGER_URL" envDefault:"http://jaeger:14268/api/traces"`
SendTelemetry bool `env:"MG_SEND_TELEMETRY" envDefault:"true"`
InstanceID string `env:"MG_JOURNAL_INSTANCE_ID" envDefault:""`
TraceRatio float64 `env:"MG_JAEGER_TRACE_RATIO" envDefault:"1.0"`
}
func main() {
ctx, cancel := context.WithCancel(context.Background())
g, ctx := errgroup.WithContext(ctx)
cfg := config{}
if err := env.Parse(&cfg); err != nil {
log.Fatalf("failed to load %s configuration : %s", svcName, err)
}
logger, err := mglog.New(os.Stdout, cfg.LogLevel)
if err != nil {
log.Fatalf("failed to init logger: %s", err)
}
var exitCode int
defer mglog.ExitWithError(&exitCode)
if cfg.InstanceID == "" {
if cfg.InstanceID, err = uuid.New().ID(); err != nil {
logger.Error(fmt.Sprintf("failed to generate instanceID: %s", err))
exitCode = 1
return
}
}
dbConfig := pgclient.Config{Name: defDB}
if err := env.ParseWithOptions(&dbConfig, env.Options{Prefix: envPrefixDB}); err != nil {
logger.Error(err.Error())
exitCode = 1
return
}
db, err := pgclient.Setup(dbConfig, *journalpg.Migration())
if err != nil {
logger.Error(err.Error())
exitCode = 1
return
}
defer db.Close()
authConfig := auth.Config{}
if err := env.ParseWithOptions(&authConfig, env.Options{Prefix: envPrefixAuth}); err != nil {
logger.Error(fmt.Sprintf("failed to load %s auth configuration : %s", svcName, err))
exitCode = 1
return
}
ac, acHandler, err := auth.Setup(ctx, authConfig)
if err != nil {
logger.Error(err.Error())
exitCode = 1
return
}
defer acHandler.Close()
logger.Info("Successfully connected to auth grpc server " + acHandler.Secure())
tp, err := jaegerclient.NewProvider(ctx, svcName, cfg.JaegerURL, cfg.InstanceID, cfg.TraceRatio)
if err != nil {
logger.Error(fmt.Sprintf("failed to init Jaeger: %s", err))
exitCode = 1
return
}
defer func() {
if err := tp.Shutdown(ctx); err != nil {
logger.Error(fmt.Sprintf("error shutting down tracer provider: %s", err))
}
}()
tracer := tp.Tracer(svcName)
svc := newService(db, dbConfig, ac, logger, tracer)
subscriber, err := store.NewSubscriber(ctx, cfg.ESURL, logger)
if err != nil {
logger.Error(fmt.Sprintf("failed to create subscriber: %s", err))
exitCode = 1
return
}
logger.Info("Subscribed to Event Store")
if err := events.Start(ctx, svcName, subscriber, svc); err != nil {
logger.Error("failed to start %s service: %s", svcName, err)
exitCode = 1
return
}
httpServerConfig := server.Config{Port: defSvcHTTPPort}
if err := env.ParseWithOptions(&httpServerConfig, env.Options{Prefix: envPrefixHTTP}); err != nil {
logger.Error(fmt.Sprintf("failed to load %s HTTP server configuration : %s", svcName, err.Error()))
exitCode = 1
return
}
hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svc, logger, svcName, cfg.InstanceID), logger)
if cfg.SendTelemetry {
chc := chclient.New(svcName, magistrala.Version, logger, cancel)
go chc.CallHome(ctx)
}
g.Go(func() error {
return hs.Start()
})
g.Go(func() error {
return server.StopSignalHandler(ctx, cancel, logger, svcName, hs)
})
if err := g.Wait(); err != nil {
logger.Error(fmt.Sprintf("%s service terminated: %s", svcName, err))
}
}
func newService(db *sqlx.DB, dbConfig pgclient.Config, authClient magistrala.AuthServiceClient, logger *slog.Logger, tracer trace.Tracer) journal.Service {
database := postgres.NewDatabase(db, dbConfig, tracer)
repo := journalpg.NewRepository(database)
idp := uuid.New()
svc := journal.NewService(idp, repo, authClient)
svc = middleware.LoggingMiddleware(svc, logger)
counter, latency := internal.MakeMetrics("journal", "journal_writer")
svc = middleware.MetricsMiddleware(svc, counter, latency)
svc = middleware.Tracing(svc, tracer)
return svc
}
config.toml
+1
View File
@@ -10,6 +10,7 @@ user_token = ""
topic = ""
[remotes]
journal_url = "http://localhost:9021"
bootstrap_url = "http://localhost:9013"
certs_url = "http://localhost:9019"
domains_url = "http://localhost:8189"
docker/.env
+17 -1
View File
@@ -122,7 +122,6 @@ MG_SPICEDB_HOST=magistrala-spicedb
MG_SPICEDB_PORT=50051
MG_SPICEDB_DATASTORE_ENGINE=postgres
### Invitations
MG_INVITATIONS_LOG_LEVEL=info
MG_INVITATIONS_HTTP_HOST=invitations
@@ -613,6 +612,23 @@ MG_SMPP_SRC_ADDR_NPI=0
MG_SMPP_DST_ADDR_NPI=1
MG_SMPP_NOTIFIER_INSTANCE_ID=
### Journal
MG_JOURNAL_LOG_LEVEL=info
MG_JOURNAL_HTTP_HOST=journal
MG_JOURNAL_HTTP_PORT=9021
MG_JOURNAL_HTTP_SERVER_CERT=
MG_JOURNAL_HTTP_SERVER_KEY=
MG_JOURNAL_HOST=magistrala-journal-db
MG_JOURNAL_PORT=5432
MG_JOURNAL_USER=magistrala
MG_JOURNAL_PASS=magistrala
MG_JOURNAL_NAME=journal
MG_JOURNAL_SSL_MODE=disable
MG_JOURNAL_SSL_CERT=
MG_JOURNAL_SSL_KEY=
MG_JOURNAL_SSL_ROOT_CERT=
MG_JOURNAL_INSTANCE_ID=
### GRAFANA and PROMETHEUS
MG_PROMETHEUS_PORT=9090
MG_GRAFANA_PORT=3000
docker/addons/journal/docker-compose.yml
+67
View File
@@ -0,0 +1,67 @@
# Copyright (c) Abstract Machines
# SPDX-License-Identifier: Apache-2.0
# This docker-compose file contains optional Postgres and journal services
# for Magistrala platform. Since these are optional, this file is dependent of docker-compose file
# from <project_root>/docker. In order to run these services, execute command:
# docker-compose -f docker/docker-compose.yml -f docker/addons/journal/docker-compose.yml up
# from project root. PostgreSQL default port (5432) is exposed, so you can use various tools for database
# inspection and data visualization.
networks:
magistrala-base-net:
volumes:
magistrala-journal-volume:
services:
journal-db:
image: postgres:16.2-alpine
container_name: magistrala-journal-db
restart: on-failure
command: postgres -c "max_connections=${MG_POSTGRES_MAX_CONNECTIONS}"
environment:
POSTGRES_USER: ${MG_JOURNAL_USER}
POSTGRES_PASSWORD: ${MG_JOURNAL_PASS}
POSTGRES_DB: ${MG_JOURNAL_NAME}
MG_POSTGRES_MAX_CONNECTIONS: ${MG_POSTGRES_MAX_CONNECTIONS}
networks:
- magistrala-base-net
volumes:
- magistrala-journal-volume:/var/lib/postgresql/data
journal:
image: magistrala/journal:${MG_RELEASE_TAG}
container_name: magistrala-journal
depends_on:
- journal-db
restart: on-failure
environment:
MG_JOURNAL_LOG_LEVEL: ${MG_JOURNAL_LOG_LEVEL}
MG_JOURNAL_HTTP_HOST: ${MG_JOURNAL_HTTP_HOST}
MG_JOURNAL_HTTP_PORT: ${MG_JOURNAL_HTTP_PORT}
MG_JOURNAL_HTTP_SERVER_CERT: ${MG_JOURNAL_HTTP_SERVER_CERT}
MG_JOURNAL_HTTP_SERVER_KEY: ${MG_JOURNAL_HTTP_SERVER_KEY}
MG_JOURNAL_HOST: ${MG_JOURNAL_HOST}
MG_JOURNAL_PORT: ${MG_JOURNAL_PORT}
MG_JOURNAL_USER: ${MG_JOURNAL_USER}
MG_JOURNAL_PASS: ${MG_JOURNAL_PASS}
MG_JOURNAL_NAME: ${MG_JOURNAL_NAME}
MG_JOURNAL_SSL_MODE: ${MG_JOURNAL_SSL_MODE}
MG_JOURNAL_SSL_CERT: ${MG_JOURNAL_SSL_CERT}
MG_JOURNAL_SSL_KEY: ${MG_JOURNAL_SSL_KEY}
MG_JOURNAL_SSL_ROOT_CERT: ${MG_JOURNAL_SSL_ROOT_CERT}
MG_AUTH_GRPC_URL: ${MG_AUTH_GRPC_URL}
MG_AUTH_GRPC_TIMEOUT: ${MG_AUTH_GRPC_TIMEOUT}
MG_AUTH_GRPC_CLIENT_CERT: ${MG_AUTH_GRPC_CLIENT_CERT:+/auth-grpc-client.crt}
MG_AUTH_GRPC_CLIENT_KEY: ${MG_AUTH_GRPC_CLIENT_KEY:+/auth-grpc-client.key}
MG_AUTH_GRPC_SERVER_CA_CERTS: ${MG_AUTH_GRPC_SERVER_CA_CERTS:+/auth-grpc-server-ca.crt}
MG_ES_URL: ${MG_ES_URL}
MG_JAEGER_URL: ${MG_JAEGER_URL}
MG_JAEGER_TRACE_RATIO: ${MG_JAEGER_TRACE_RATIO}
MG_SEND_TELEMETRY: ${MG_SEND_TELEMETRY}
MG_JOURNAL_INSTANCE_ID: ${MG_JOURNAL_INSTANCE_ID}
ports:
- ${MG_JOURNAL_HTTP_PORT}:${MG_JOURNAL_HTTP_PORT}
networks:
- magistrala-base-net
internal/api/common.go
+6 -1
View File
@@ -154,7 +154,12 @@ func EncodeError(_ context.Context, err error, w http.ResponseWriter) {
errors.Contains(err, apiutil.ErrInvalidTopic),
errors.Contains(err, bootstrap.ErrAddBootstrap),
errors.Contains(err, apiutil.ErrInvalidCertData),
errors.Contains(err, apiutil.ErrEmptyMessage):
errors.Contains(err, apiutil.ErrEmptyMessage),
errors.Contains(err, apiutil.ErrInvalidLevel),
errors.Contains(err, apiutil.ErrInvalidDirection),
errors.Contains(err, apiutil.ErrInvalidEntityType),
errors.Contains(err, apiutil.ErrMissingEntityType),
errors.Contains(err, apiutil.ErrInvalidTimeFormat):
err = unwrap(err)
w.WriteHeader(http.StatusBadRequest)
internal/apiutil/errors.go
+9
View File
@@ -167,4 +167,13 @@ var (
// ErrEmptyMessage indicates empty message.
ErrEmptyMessage = errors.New("empty message")
// ErrMissingEntityType indicates missing entity type.
ErrMissingEntityType = errors.New("missing entity type")
// ErrInvalidEntityType indicates invalid entity type.
ErrInvalidEntityType = errors.New("invalid entity type")
// ErrInvalidTimeFormat indicates invalid time format i.e not unix time.
ErrInvalidTimeFormat = errors.New("invalid time format use unix time")
)
internal/groups/events/events.go
+14 -42
View File
@@ -4,14 +4,13 @@
package events
import (
"encoding/json"
"time"
"github.com/absmach/magistrala/pkg/events"
groups "github.com/absmach/magistrala/pkg/groups"
)
const (
var (
groupPrefix = "group."
groupCreate = groupPrefix + "create"
groupUpdate = groupPrefix + "update"
@@ -46,15 +45,13 @@ type assignEvent struct {
}
func (cge assignEvent) Encode() (map[string]interface{}, error) {
val := map[string]interface{}{
return map[string]interface{}{
"operation": groupAssign,
"member_ids": cge.memberIDs,
"relation": cge.relation,
"memberKind": cge.memberKind,
"group_id": cge.groupID,
}
return val, nil
}, nil
}
type unassignEvent struct {
@@ -65,15 +62,13 @@ type unassignEvent struct {
}
func (cge unassignEvent) Encode() (map[string]interface{}, error) {
val := map[string]interface{}{
return map[string]interface{}{
"operation": groupUnassign,
"member_ids": cge.memberIDs,
"relation": cge.relation,
"memberKind": cge.memberKind,
"group_id": cge.groupID,
}
return val, nil
}, nil
}
type createGroupEvent struct {
@@ -101,12 +96,7 @@ func (cge createGroupEvent) Encode() (map[string]interface{}, error) {
val["description"] = cge.Description
}
if cge.Metadata != nil {
metadata, err := json.Marshal(cge.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = cge.Metadata
}
if cge.Status.String() != "" {
val["status"] = cge.Status.String()
@@ -142,12 +132,7 @@ func (uge updateGroupEvent) Encode() (map[string]interface{}, error) {
val["description"] = uge.Description
}
if uge.Metadata != nil {
metadata, err := json.Marshal(uge.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = uge.Metadata
}
if !uge.CreatedAt.IsZero() {
val["created_at"] = uge.CreatedAt
@@ -199,12 +184,7 @@ func (vge viewGroupEvent) Encode() (map[string]interface{}, error) {
val["description"] = vge.Description
}
if vge.Metadata != nil {
metadata, err := json.Marshal(vge.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = vge.Metadata
}
if !vge.CreatedAt.IsZero() {
val["created_at"] = vge.CreatedAt
@@ -227,11 +207,10 @@ type viewGroupPermsEvent struct {
}
func (vgpe viewGroupPermsEvent) Encode() (map[string]interface{}, error) {
val := map[string]interface{}{
return map[string]interface{}{
"operation": groupViewPerms,
"permissions": vgpe.permissions,
}
return val, nil
}, nil
}
type listGroupEvent struct {
@@ -256,12 +235,7 @@ func (lge listGroupEvent) Encode() (map[string]interface{}, error) {
val["tag"] = lge.Tag
}
if lge.Metadata != nil {
metadata, err := json.Marshal(lge.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = lge.Metadata
}
if lge.Status.String() != "" {
val["status"] = lge.Status.String()
@@ -277,14 +251,12 @@ type listGroupMembershipEvent struct {
}
func (lgme listGroupMembershipEvent) Encode() (map[string]interface{}, error) {
val := map[string]interface{}{
return map[string]interface{}{
"operation": groupListMemberships,
"group_id": lgme.groupID,
"id": lgme.groupID,
"permission": lgme.permission,
"member_kind": lgme.memberKind,
}
return val, nil
}, nil
}
type deleteGroupEvent struct {
journal/api/doc.go
+6
View File
@@ -0,0 +1,6 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package api contains API-related concerns: endpoint definitions, middlewares
// and all resource representations.
package api
journal/api/endpoint.go
+31
View File
@@ -0,0 +1,31 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api
import (
"context"
"github.com/absmach/magistrala/internal/apiutil"
"github.com/absmach/magistrala/journal"
"github.com/absmach/magistrala/pkg/errors"
"github.com/go-kit/kit/endpoint"
)
func retrieveJournalsEndpoint(svc journal.Service) endpoint.Endpoint {
return func(ctx context.Context, request interface{}) (interface{}, error) {
req := request.(retrieveJournalsReq)
if err := req.validate(); err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
page, err := svc.RetrieveAll(ctx, req.token, req.page)
if err != nil {
return nil, err
}
return pageRes{
JournalsPage: page,
}, nil
}
}
journal/api/endpoint_test.go
+282
View File
@@ -0,0 +1,282 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api_test
import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"time"
"github.com/absmach/magistrala/internal/apiutil"
"github.com/absmach/magistrala/journal"
"github.com/absmach/magistrala/journal/api"
"github.com/absmach/magistrala/journal/mocks"
mglog "github.com/absmach/magistrala/logger"
svcerr "github.com/absmach/magistrala/pkg/errors/service"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
var validToken = "valid"
type testRequest struct {
client *http.Client
method string
url string
token string
body io.Reader
}
func (tr testRequest) make() (*http.Response, error) {
req, err := http.NewRequest(tr.method, tr.url, tr.body)
if err != nil {
return nil, err
}
if tr.token != "" {
req.Header.Set("Authorization", apiutil.BearerPrefix+tr.token)
}
return tr.client.Do(req)
}
func newjournalServer() (*httptest.Server, *mocks.Service) {
svc := new(mocks.Service)
logger := mglog.NewMock()
mux := api.MakeHandler(svc, logger, "journal-log", "test")
return httptest.NewServer(mux), svc
}
func TestListJournalsEndpoint(t *testing.T) {
es, svc := newjournalServer()
cases := []struct {
desc string
token string
url string
contentType string
status int
svcErr error
}{
{
desc: "successful",
token: validToken,
url: "/user/123",
status: http.StatusOK,
svcErr: nil,
},
{
desc: "empty token",
token: "",
url: "/user/123",
status: http.StatusUnauthorized,
svcErr: nil,
},
{
desc: "with service error",
token: validToken,
url: "/user/123",
status: http.StatusForbidden,
svcErr: svcerr.ErrAuthorization,
},
{
desc: "with offset",
token: validToken,
url: "/user/123?offset=10",
status: http.StatusOK,
svcErr: nil,
},
{
desc: "with invalid offset",
token: validToken,
url: "/user/123?offset=ten",
status: http.StatusBadRequest,
svcErr: nil,
},
{
desc: "with limit",
token: validToken,
url: "/user/123?limit=10",
status: http.StatusOK,
svcErr: nil,
},
{
desc: "with invalid limit",
token: validToken,
url: "/user/123?limit=ten",
status: http.StatusBadRequest,
svcErr: nil,
},
{
desc: "with operation",
token: validToken,
url: "/user/123?operation=user.create",
status: http.StatusOK,
svcErr: nil,
},
{
desc: "with malformed operation",
token: validToken,
url: "/user/123?operation=user.create&operation=user.update",
status: http.StatusBadRequest,
svcErr: nil,
},
{
desc: "with from",
token: validToken,
url: fmt.Sprintf("/user/123?from=%d", time.Now().Unix()),
status: http.StatusOK,
svcErr: nil,
},
{
desc: "with invalid from",
token: validToken,
url: "/user/123?from=ten",
status: http.StatusBadRequest,
svcErr: nil,
},
{
desc: "with invalid from as UnixNano",
token: validToken,
url: fmt.Sprintf("/user/123?from=%d", time.Now().UnixNano()),
status: http.StatusBadRequest,
svcErr: nil,
},
{
desc: "with to",
token: validToken,
url: fmt.Sprintf("/user/123?to=%d", time.Now().Unix()),
status: http.StatusOK,
svcErr: nil,
},
{
desc: "with invalid to",
token: validToken,
url: "/user/123?to=ten",
status: http.StatusBadRequest,
svcErr: nil,
},
{
desc: "with invalid to as UnixNano",
token: validToken,
url: fmt.Sprintf("/user/123?to=%d", time.Now().UnixNano()),
status: http.StatusBadRequest,
svcErr: nil,
},
{
desc: "with attributes",
token: validToken,
url: fmt.Sprintf("/user/123?with_attributes=%s", strconv.FormatBool(true)),
status: http.StatusOK,
svcErr: nil,
},
{
desc: "with invalid attributes",
token: validToken,
url: "/user/123?with_attributes=ten",
status: http.StatusBadRequest,
svcErr: nil,
},
{
desc: "with metadata",
token: validToken,
url: fmt.Sprintf("/user/123?with_metadata=%s", strconv.FormatBool(true)),
status: http.StatusOK,
svcErr: nil,
},
{
desc: "with invalid metadata",
token: validToken,
url: "/user/123?with_metadata=ten",
status: http.StatusBadRequest,
svcErr: nil,
},
{
desc: "with asc direction",
token: validToken,
url: "/user/123?dir=asc",
status: http.StatusOK,
svcErr: nil,
},
{
desc: "with desc direction",
token: validToken,
url: "/user/123?dir=desc",
status: http.StatusOK,
svcErr: nil,
},
{
desc: "with invalid direction",
token: validToken,
url: "/user/123?dir=ten",
status: http.StatusBadRequest,
svcErr: nil,
},
{
desc: "with malformed direction",
token: validToken,
url: "/user/123?dir=invalid&dir=invalid2",
status: http.StatusBadRequest,
svcErr: nil,
},
{
desc: "with invalid entity type",
token: validToken,
url: "/invalid/123",
status: http.StatusBadRequest,
svcErr: nil,
},
{
desc: "with all query params",
token: validToken,
url: "/user/123?offset=10&limit=10&operation=user.create&from=0&to=10&with_attributes=true&with_metadata=true&dir=asc",
status: http.StatusOK,
svcErr: nil,
},
{
desc: "with empty url",
token: validToken,
url: "",
status: http.StatusNotFound,
svcErr: nil,
},
{
desc: "with empty entity type",
token: validToken,
url: "//123",
status: http.StatusBadRequest,
svcErr: nil,
},
{
desc: "with empty entity ID",
token: validToken,
url: "/user/",
status: http.StatusNotFound,
svcErr: nil,
},
}
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
svcCall := svc.On("RetrieveAll", mock.Anything, c.token, mock.Anything).Return(journal.JournalsPage{}, c.svcErr)
req := testRequest{
client: es.Client(),
method: http.MethodGet,
url: es.URL + "/journal" + c.url,
token: c.token,
}
resp, err := req.make()
assert.Nil(t, err, c.desc)
defer resp.Body.Close()
assert.Equal(t, c.status, resp.StatusCode, c.desc)
svcCall.Unset()
})
}
}
journal/api/requests.go
+32
View File
@@ -0,0 +1,32 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api
import (
"github.com/absmach/magistrala/internal/api"
"github.com/absmach/magistrala/internal/apiutil"
"github.com/absmach/magistrala/journal"
)
type retrieveJournalsReq struct {
token string
page journal.Page
}
func (req retrieveJournalsReq) validate() error {
if req.token == "" {
return apiutil.ErrBearerToken
}
if req.page.Limit > api.DefLimit {
return apiutil.ErrLimitSize
}
if req.page.Direction != "" && req.page.Direction != api.AscDir && req.page.Direction != api.DescDir {
return apiutil.ErrInvalidDirection
}
if req.page.EntityID == "" {
return apiutil.ErrMissingID
}
return nil
}
journal/api/requests_test.go
+126
View File
@@ -0,0 +1,126 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api
import (
"testing"
"github.com/absmach/magistrala/internal/api"
"github.com/absmach/magistrala/internal/apiutil"
"github.com/absmach/magistrala/journal"
"github.com/stretchr/testify/assert"
)
var (
token = "token"
limit uint64 = 10
)
func TestRetrieveJournalsReqValidate(t *testing.T) {
cases := []struct {
desc string
req retrieveJournalsReq
err error
}{
{
desc: "valid",
req: retrieveJournalsReq{
token: token,
page: journal.Page{
Limit: limit,
EntityID: "id",
EntityType: journal.UserEntity,
},
},
err: nil,
},
{
desc: "missing token",
req: retrieveJournalsReq{
page: journal.Page{
Limit: limit,
EntityID: "id",
EntityType: journal.UserEntity,
},
},
err: apiutil.ErrBearerToken,
},
{
desc: "invalid limit size",
req: retrieveJournalsReq{
token: token,
page: journal.Page{
Limit: api.DefLimit + 1,
EntityID: "id",
EntityType: journal.UserEntity,
},
},
err: apiutil.ErrLimitSize,
},
{
desc: "invalid sorting direction",
req: retrieveJournalsReq{
token: token,
page: journal.Page{
Limit: limit,
Direction: "invalid",
EntityID: "id",
EntityType: journal.UserEntity,
},
},
err: apiutil.ErrInvalidDirection,
},
{
desc: "valid id and entity type",
req: retrieveJournalsReq{
token: token,
page: journal.Page{
Limit: limit,
EntityID: "id",
EntityType: journal.UserEntity,
},
},
err: nil,
},
{
desc: "valid id and empty entity type",
req: retrieveJournalsReq{
token: token,
page: journal.Page{
Limit: limit,
EntityID: "id",
},
},
err: nil,
},
{
desc: "empty id and empty entity type",
req: retrieveJournalsReq{
token: token,
page: journal.Page{
Limit: limit,
},
},
err: apiutil.ErrMissingID,
},
{
desc: "empty id and valid entity type",
req: retrieveJournalsReq{
token: token,
page: journal.Page{
Limit: limit,
EntityType: journal.UserEntity,
},
},
err: apiutil.ErrMissingID,
},
}
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
err := c.req.validate()
assert.Equal(t, c.err, err)
})
}
}
journal/api/responses.go
+29
View File
@@ -0,0 +1,29 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api
import (
"net/http"
"github.com/absmach/magistrala"
"github.com/absmach/magistrala/journal"
)
var _ magistrala.Response = (*pageRes)(nil)
type pageRes struct {
journal.JournalsPage `json:",inline"`
}
func (res pageRes) Headers() map[string]string {
return map[string]string{}
}
func (res pageRes) Code() int {
return http.StatusOK
}
func (res pageRes) Empty() bool {
return false
}
journal/api/transport.go
+129
View File
@@ -0,0 +1,129 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package api
import (
"context"
"log/slog"
"math"
"net/http"
"strings"
"time"
"github.com/absmach/magistrala"
"github.com/absmach/magistrala/internal/api"
"github.com/absmach/magistrala/internal/apiutil"
"github.com/absmach/magistrala/journal"
"github.com/absmach/magistrala/pkg/errors"
"github.com/go-chi/chi/v5"
kithttp "github.com/go-kit/kit/transport/http"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)
const (
operationKey = "operation"
fromKey = "from"
toKey = "to"
attributesKey = "with_attributes"
metadataKey = "with_metadata"
entityIDKey = "id"
entityTypeKey = "entity_type"
)
// MakeHandler returns a HTTP API handler with health check and metrics.
func MakeHandler(svc journal.Service, logger *slog.Logger, svcName, instanceID string) http.Handler {
opts := []kithttp.ServerOption{
kithttp.ServerErrorEncoder(apiutil.LoggingErrorEncoder(logger, api.EncodeError)),
}
mux := chi.NewRouter()
mux.Get("/journal/{entityType}/{entityID}", otelhttp.NewHandler(kithttp.NewServer(
retrieveJournalsEndpoint(svc),
decodeRetrieveJournalReq,
api.EncodeResponse,
opts...,
), "list_journals").ServeHTTP)
mux.Get("/health", magistrala.Health(svcName, instanceID))
mux.Handle("/metrics", promhttp.Handler())
return mux
}
func decodeRetrieveJournalReq(_ context.Context, r *http.Request) (interface{}, error) {
offset, err := apiutil.ReadNumQuery[uint64](r, api.OffsetKey, api.DefOffset)
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
limit, err := apiutil.ReadNumQuery[uint64](r, api.LimitKey, api.DefLimit)
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
operation, err := apiutil.ReadStringQuery(r, operationKey, "")
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
from, err := apiutil.ReadNumQuery[int64](r, fromKey, 0)
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
if from > math.MaxInt32 {
return nil, errors.Wrap(apiutil.ErrValidation, apiutil.ErrInvalidTimeFormat)
}
var fromTime time.Time
if from != 0 {
fromTime = time.Unix(from, 0)
}
to, err := apiutil.ReadNumQuery[int64](r, toKey, 0)
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
if to > math.MaxInt32 {
return nil, errors.Wrap(apiutil.ErrValidation, apiutil.ErrInvalidTimeFormat)
}
var toTime time.Time
if to != 0 {
toTime = time.Unix(to, 0)
}
attributes, err := apiutil.ReadBoolQuery(r, attributesKey, false)
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
metadata, err := apiutil.ReadBoolQuery(r, metadataKey, false)
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
dir, err := apiutil.ReadStringQuery(r, api.DirKey, api.DescDir)
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
entityType, err := journal.ToEntityType(chi.URLParam(r, "entityType"))
if err != nil {
return nil, errors.Wrap(apiutil.ErrValidation, err)
}
if entityType == journal.ChannelEntity {
operation = strings.ReplaceAll(operation, "channel", "group")
}
req := retrieveJournalsReq{
token: apiutil.ExtractBearerToken(r),
page: journal.Page{
Offset: offset,
Limit: limit,
Operation: operation,
From: fromTime,
To: toTime,
WithAttributes: attributes,
WithMetadata: metadata,
EntityID: chi.URLParam(r, "entityID"),
EntityType: entityType,
Direction: dir,
},
}
return req, nil
}
journal/doc.go
+7
View File
@@ -0,0 +1,7 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package journal contains the journal service.
// This service is responsible for storing events from the event store to a
// journal log repository. It is also responsible for providing a REST API to query events.
package journal
journal/events/consumer.go
+85
View File
@@ -0,0 +1,85 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package events
import (
"context"
"errors"
"time"
"github.com/absmach/magistrala/journal"
"github.com/absmach/magistrala/pkg/events"
"github.com/absmach/magistrala/pkg/events/store"
)
var ErrMissingOccurredAt = errors.New("missing occurred_at")
// Start method starts consuming messages received from Event store.
func Start(ctx context.Context, consumer string, sub events.Subscriber, service journal.Service) error {
subCfg := events.SubscriberConfig{
Consumer: consumer,
Stream: store.StreamAllEvents,
Handler: Handle(service),
}
return sub.Subscribe(ctx, subCfg)
}
func Handle(service journal.Service) handleFunc {
return func(ctx context.Context, event events.Event) error {
data, err := event.Encode()
if err != nil {
return err
}
operation, ok := data["operation"].(string)
if !ok {
return errors.New("missing operation")
}
delete(data, "operation")
if operation == "" {
return errors.New("missing operation")
}
occurredAt, ok := data["occurred_at"].(float64)
if !ok {
return ErrMissingOccurredAt
}
delete(data, "occurred_at")
if occurredAt == 0 {
return ErrMissingOccurredAt
}
metadata, ok := data["metadata"].(map[string]interface{})
if !ok {
metadata = make(map[string]interface{})
}
delete(data, "metadata")
if len(data) == 0 {
return errors.New("missing attributes")
}
j := journal.Journal{
Operation: operation,
OccurredAt: time.Unix(0, int64(occurredAt)),
Attributes: data,
Metadata: metadata,
}
return service.Save(ctx, j)
}
}
type handleFunc func(ctx context.Context, event events.Event) error
func (h handleFunc) Handle(ctx context.Context, event events.Event) error {
return h(ctx, event)
}
func (h handleFunc) Cancel() error {
return nil
}
journal/events/consumer_test.go
+276
View File
@@ -0,0 +1,276 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package events_test
import (
"context"
"encoding/json"
"errors"
"math/rand"
"strings"
"testing"
"time"
"github.com/absmach/magistrala/internal/testsutil"
"github.com/absmach/magistrala/journal"
aevents "github.com/absmach/magistrala/journal/events"
"github.com/absmach/magistrala/journal/mocks"
repoerr "github.com/absmach/magistrala/pkg/errors/repository"
"github.com/absmach/magistrala/pkg/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
var (
operation = "users.create"
payload = map[string]interface{}{
"temperature": rand.Float64(),
"humidity": float64(rand.Intn(1000)),
"locations": []interface{}{
strings.Repeat("a", 100),
strings.Repeat("a", 100),
},
"status": "active",
}
idProvider = uuid.New()
)
type testEvent struct {
data map[string]interface{}
err error
}
func (e testEvent) Encode() (map[string]interface{}, error) {
return e.data, e.err
}
func NewTestEvent(data map[string]interface{}, err error) testEvent {
return testEvent{data: data, err: err}
}
func TestHandle(t *testing.T) {
repo := new(mocks.Repository)
svc := journal.NewService(idProvider, repo, nil)
cases := []struct {
desc string
event map[string]interface{}
encodeErr error
repoErr error
err error
}{
{
desc: "success",
event: map[string]interface{}{
"operation": operation,
"occurred_at": float64(time.Now().UnixNano()),
"id": testsutil.GenerateUUID(t),
"tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)},
"number": float64(rand.Intn(1000)),
"metadata": payload,
},
err: nil,
},
{
desc: "with encode error",
event: map[string]interface{}{
"operation": operation,
"occurred_at": float64(time.Now().UnixNano()),
"id": testsutil.GenerateUUID(t),
"tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)},
"number": float64(rand.Intn(1000)),
"metadata": payload,
},
encodeErr: errors.New("encode error"),
err: errors.New("encode error"),
},
{
desc: "with missing operation",
event: map[string]interface{}{
"occurred_at": float64(time.Now().UnixNano()),
"id": testsutil.GenerateUUID(t),
"tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)},
"number": float64(rand.Intn(1000)),
"metadata": payload,
},
err: errors.New("missing operation"),
},
{
desc: "with empty operation",
event: map[string]interface{}{
"operation": "",
"occurred_at": float64(time.Now().UnixNano()),
"id": testsutil.GenerateUUID(t),
"tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)},
"number": float64(rand.Intn(1000)),
"metadata": payload,
},
err: errors.New("missing operation"),
},
{
desc: "with invalid operation",
event: map[string]interface{}{
"operation": 1,
"occurred_at": float64(time.Now().UnixNano()),
"id": testsutil.GenerateUUID(t),
"tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)},
"number": float64(rand.Intn(1000)),
"metadata": payload,
},
err: errors.New("missing operation"),
},
{
desc: "with missing occurred_at",
event: map[string]interface{}{
"operation": operation,
"id": testsutil.GenerateUUID(t),
"tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)},
"number": float64(rand.Intn(1000)),
"metadata": payload,
},
err: aevents.ErrMissingOccurredAt,
},
{
desc: "with empty occurred_at",
event: map[string]interface{}{
"operation": operation,
"occurred_at": float64(0),
"id": testsutil.GenerateUUID(t),
"tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)},
"number": float64(rand.Intn(1000)),
"metadata": payload,
},
err: aevents.ErrMissingOccurredAt,
},
{
desc: "with invalid occurred_at",
event: map[string]interface{}{
"operation": operation,
"occurred_at": "invalid",
"id": testsutil.GenerateUUID(t),
"tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)},
"number": float64(rand.Intn(1000)),
"metadata": payload,
},
err: aevents.ErrMissingOccurredAt,
},
{
desc: "with missing metadata",
event: map[string]interface{}{
"operation": operation,
"occurred_at": float64(time.Now().UnixNano()),
"id": testsutil.GenerateUUID(t),
"tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)},
"number": float64(rand.Intn(1000)),
},
err: nil,
},
{
desc: "with empty metadata",
event: map[string]interface{}{
"operation": operation,
"occurred_at": float64(time.Now().UnixNano()),
"id": testsutil.GenerateUUID(t),
"tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)},
"number": float64(rand.Intn(1000)),
"metadata": map[string]interface{}{},
},
err: nil,
},
{
desc: "with invalid metadata",
event: map[string]interface{}{
"operation": operation,
"occurred_at": float64(time.Now().UnixNano()),
"id": testsutil.GenerateUUID(t),
"tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)},
"number": float64(rand.Intn(1000)),
"metadata": 1,
},
err: nil,
},
{
desc: "with missing attributes",
event: map[string]interface{}{
"operation": operation,
"occurred_at": float64(time.Now().UnixNano()),
"metadata": payload,
},
err: errors.New("missing attributes"),
},
{
desc: "with empty attributes",
event: map[string]interface{}{
"operation": operation,
"occurred_at": float64(time.Now().UnixNano()),
"id": "",
"tags": []interface{}{},
"number": float64(0),
"metadata": payload,
},
err: nil,
},
{
desc: "with invalid attributes",
event: map[string]interface{}{
"operation": operation,
"occurred_at": float64(time.Now().UnixNano()),
"nested": map[string]interface{}{
"key": float64(rand.Intn(1000)),
"nested": map[string]interface{}{
"key": float64(rand.Intn(1000)),
"nested": map[string]interface{}{
"key": float64(rand.Intn(1000)),
"nested": map[string]interface{}{
"key": float64(rand.Intn(1000)),
"nested": map[string]interface{}{
"key": float64(rand.Intn(1000)),
"nested": map[string]interface{}{
"key": float64(rand.Intn(1000)),
},
},
},
},
},
},
"metadata": payload,
},
err: nil,
},
{
desc: "success",
event: map[string]interface{}{
"operation": operation,
"occurred_at": float64(time.Now().UnixNano()),
"id": testsutil.GenerateUUID(t),
"tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)},
"number": float64(rand.Intn(1000)),
"metadata": payload,
},
repoErr: repoerr.ErrCreateEntity,
err: repoerr.ErrCreateEntity,
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
data, err := json.Marshal(tc.event)
assert.NoError(t, err)
event := map[string]interface{}{}
err = json.Unmarshal(data, &event)
assert.NoError(t, err)
repoCall := repo.On("Save", context.Background(), mock.Anything).Return(tc.repoErr)
err = aevents.Handle(svc)(context.Background(), NewTestEvent(event, tc.encodeErr))
switch {
case tc.err == nil:
assert.NoError(t, err)
default:
assert.ErrorContains(t, err, tc.err.Error())
}
repoCall.Unset()
})
}
}
journal/events/doc.go
+7
View File
@@ -0,0 +1,7 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package events provides the event consumer for the journal service.
// This package is responsible for consuming events from the event store and
// processing them.
package events
journal/journal.go
+158
View File
@@ -0,0 +1,158 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package journal
import (
"context"
"encoding/json"
"time"
"github.com/absmach/magistrala/auth"
"github.com/absmach/magistrala/internal/apiutil"
)
type EntityType uint8
const (
UserEntity EntityType = iota
GroupEntity
ThingEntity
ChannelEntity
)
// String representation of the possible entity type values.
const (
userEntityType = "user"
groupEntityType = "group"
thingEntityType = "thing"
channelEntityType = "channel"
)
// String converts entity type to string literal.
func (e EntityType) String() string {
switch e {
case UserEntity:
return userEntityType
case GroupEntity:
return groupEntityType
case ThingEntity:
return thingEntityType
case ChannelEntity:
return channelEntityType
default:
return ""
}
}
// AuthString returns the entity type as a string for authorization.
func (e EntityType) AuthString() string {
switch e {
case UserEntity:
return auth.UserType
case GroupEntity, ChannelEntity:
return auth.GroupType
case ThingEntity:
return auth.ThingType
default:
return ""
}
}
// ToEntityType converts string value to a valid entity type.
func ToEntityType(entityType string) (EntityType, error) {
switch entityType {
case userEntityType:
return UserEntity, nil
case groupEntityType:
return GroupEntity, nil
case thingEntityType:
return ThingEntity, nil
case channelEntityType:
return ChannelEntity, nil
default:
return EntityType(0), apiutil.ErrInvalidEntityType
}
}
// Query returns the SQL condition for the entity type.
func (e EntityType) Query() string {
switch e {
case UserEntity:
return "((operation LIKE 'user.%' AND attributes->>'id' = :entity_id) OR (attributes->>'user_id' = :entity_id))"
case GroupEntity, ChannelEntity:
return "((operation LIKE 'group.%' AND attributes->>'id' = :entity_id) OR (attributes->>'group_id' = :entity_id))"
case ThingEntity:
return "((operation LIKE 'thing.%' AND attributes->>'id' = :entity_id) OR (attributes->>'thing_id' = :entity_id))"
default:
return ""
}
}
// Journal represents an event journal that occurred in the system.
type Journal struct {
ID string `json:"id,omitempty" db:"id"`
Operation string `json:"operation,omitempty" db:"operation,omitempty"`
OccurredAt time.Time `json:"occurred_at,omitempty" db:"occurred_at,omitempty"`
Attributes map[string]interface{} `json:"attributes,omitempty" db:"attributes,omitempty"` // This is extra information about the journal for example thing_id, user_id, group_id etc.
Metadata map[string]interface{} `json:"metadata,omitempty" db:"metadata,omitempty"` // This is decoded metadata from the journal.
}
// JournalsPage represents a page of journals.
type JournalsPage struct {
Total uint64 `json:"total"`
Offset uint64 `json:"offset"`
Limit uint64 `json:"limit"`
Journals []Journal `json:"journals"`
}
// Page is used to filter journals.
type Page struct {
Offset uint64 `json:"offset" db:"offset"`
Limit uint64 `json:"limit" db:"limit"`
Operation string `json:"operation,omitempty" db:"operation,omitempty"`
From time.Time `json:"from,omitempty" db:"from,omitempty"`
To time.Time `json:"to,omitempty" db:"to,omitempty"`
WithAttributes bool `json:"with_attributes,omitempty"`
WithMetadata bool `json:"with_metadata,omitempty"`
EntityID string `json:"entity_id,omitempty" db:"entity_id,omitempty"`
EntityType EntityType `json:"entity_type,omitempty" db:"entity_type,omitempty"`
Direction string `json:"direction,omitempty"`
}
func (page JournalsPage) MarshalJSON() ([]byte, error) {
type Alias JournalsPage
a := struct {
Alias
}{
Alias: Alias(page),
}
if a.Journals == nil {
a.Journals = make([]Journal, 0)
}
return json.Marshal(a)
}
// Service provides access to the journal log service.
//
//go:generate mockery --name Service --output=./mocks --filename service.go --quiet --note "Copyright (c) Abstract Machines"
type Service interface {
// Save saves the journal to the database.
Save(ctx context.Context, journal Journal) error
// RetrieveAll retrieves all journals from the database with the given page.
RetrieveAll(ctx context.Context, token string, page Page) (JournalsPage, error)
}
// Repository provides access to the journal log database.
//
//go:generate mockery --name Repository --output=./mocks --filename repository.go --quiet --note "Copyright (c) Abstract Machines"
type Repository interface {
// Save persists the journal to a database.
Save(ctx context.Context, journal Journal) error
// RetrieveAll retrieves all journals from the database with the given page.
RetrieveAll(ctx context.Context, page Page) (JournalsPage, error)
}
journal/journal_test.go
+143
View File
@@ -0,0 +1,143 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package journal_test
import (
"fmt"
"testing"
"time"
"github.com/absmach/magistrala/internal/apiutil"
"github.com/absmach/magistrala/journal"
"github.com/stretchr/testify/assert"
)
func TestJournalsPage_MarshalJSON(t *testing.T) {
occurredAt := time.Now()
cases := []struct {
desc string
page journal.JournalsPage
res string
}{
{
desc: "empty page",
page: journal.JournalsPage{
Journals: []journal.Journal(nil),
},
res: `{"total":0,"offset":0,"limit":0,"journals":[]}`,
},
{
desc: "page with journals",
page: journal.JournalsPage{
Total: 1,
Offset: 0,
Limit: 0,
Journals: []journal.Journal{
{
Operation: "123",
OccurredAt: occurredAt,
Attributes: map[string]interface{}{"123": "123"},
Metadata: map[string]interface{}{"123": "123"},
},
},
},
res: fmt.Sprintf(`{"total":1,"offset":0,"limit":0,"journals":[{"operation":"123","occurred_at":"%s","attributes":{"123":"123"},"metadata":{"123":"123"}}]}`, occurredAt.Format(time.RFC3339Nano)),
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
data, err := tc.page.MarshalJSON()
assert.NoError(t, err, "Unexpected error: %v", err)
assert.Equal(t, tc.res, string(data))
})
}
}
func TestEntityType(t *testing.T) {
cases := []struct {
desc string
e journal.EntityType
str string
authString string
queryString string
}{
{
desc: "UserEntity",
e: journal.UserEntity,
str: "user",
authString: "user",
},
{
desc: "ThingEntity",
e: journal.ThingEntity,
str: "thing",
authString: "thing",
},
{
desc: "GroupEntity",
e: journal.GroupEntity,
str: "group",
authString: "group",
},
{
desc: "ChannelEntity",
e: journal.ChannelEntity,
str: "channel",
authString: "group",
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
assert.Equal(t, tc.str, tc.e.String())
assert.Equal(t, tc.authString, tc.e.AuthString())
assert.NotEmpty(t, tc.e.Query())
})
}
}
func TestToEntityType(t *testing.T) {
cases := []struct {
desc string
entityType string
expected journal.EntityType
expectedErr error
}{
{
desc: "UserEntity",
entityType: "user",
expected: journal.UserEntity,
},
{
desc: "ThingEntity",
entityType: "thing",
expected: journal.ThingEntity,
},
{
desc: "GroupEntity",
entityType: "group",
expected: journal.GroupEntity,
},
{
desc: "ChannelEntity",
entityType: "channel",
expected: journal.ChannelEntity,
},
{
desc: "Invalid entity type",
entityType: "invalid",
expectedErr: apiutil.ErrInvalidEntityType,
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
entityType, err := journal.ToEntityType(tc.entityType)
assert.Equal(t, tc.expected, entityType)
assert.Equal(t, tc.expectedErr, err)
})
}
}
journal/middleware/doc.go
+6
View File
@@ -0,0 +1,6 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package middleware provides middleware for the journal service.
// This is logging, metrics, and tracing middleware.
package middleware
journal/middleware/logging.go
+70
View File
@@ -0,0 +1,70 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package middleware
import (
"context"
"log/slog"
"time"
"github.com/absmach/magistrala/journal"
)
var _ journal.Service = (*loggingMiddleware)(nil)
type loggingMiddleware struct {
logger *slog.Logger
service journal.Service
}
// LoggingMiddleware adds logging facilities to the adapter.
func LoggingMiddleware(service journal.Service, logger *slog.Logger) journal.Service {
return &loggingMiddleware{
logger: logger,
service: service,
}
}
func (lm *loggingMiddleware) Save(ctx context.Context, j journal.Journal) (err error) {
defer func(begin time.Time) {
args := []any{
slog.String("duration", time.Since(begin).String()),
slog.Group("journal",
slog.String("occurred_at", j.OccurredAt.Format(time.RFC3339Nano)),
slog.String("operation", j.Operation),
),
}
if err != nil {
args = append(args, slog.Any("error", err))
lm.logger.Warn("Save journal failed", args...)
return
}
lm.logger.Info("Save journal completed successfully", args...)
}(time.Now())
return lm.service.Save(ctx, j)
}
func (lm *loggingMiddleware) RetrieveAll(ctx context.Context, token string, page journal.Page) (journalsPage journal.JournalsPage, err error) {
defer func(begin time.Time) {
args := []any{
slog.String("duration", time.Since(begin).String()),
slog.Group("page",
slog.String("operation", page.Operation),
slog.String("entity_type", page.EntityType.String()),
slog.Uint64("offset", page.Offset),
slog.Uint64("limit", page.Limit),
slog.Uint64("total", journalsPage.Total),
),
}
if err != nil {
args = append(args, slog.Any("error", err))
lm.logger.Warn("Retrieve all journals failed", args...)
return
}
lm.logger.Info("Retrieve all journals completed successfully", args...)
}(time.Now())
return lm.service.RetrieveAll(ctx, token, page)
}
journal/middleware/metrics.go
+48
View File
@@ -0,0 +1,48 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package middleware
import (
"context"
"time"
"github.com/absmach/magistrala/journal"
"github.com/go-kit/kit/metrics"
)
var _ journal.Service = (*metricsMiddleware)(nil)
type metricsMiddleware struct {
counter metrics.Counter
latency metrics.Histogram
service journal.Service
}
// MetricsMiddleware returns new message repository
// with Save method wrapped to expose metrics.
func MetricsMiddleware(service journal.Service, counter metrics.Counter, latency metrics.Histogram) journal.Service {
return &metricsMiddleware{
counter: counter,
latency: latency,
service: service,
}
}
func (mm *metricsMiddleware) Save(ctx context.Context, j journal.Journal) error {
defer func(begin time.Time) {
mm.counter.With("method", "save").Add(1)
mm.latency.With("method", "save").Observe(time.Since(begin).Seconds())
}(time.Now())
return mm.service.Save(ctx, j)
}
func (mm *metricsMiddleware) RetrieveAll(ctx context.Context, token string, page journal.Page) (journal.JournalsPage, error) {
defer func(begin time.Time) {
mm.counter.With("method", "retrieve_all").Add(1)
mm.latency.With("method", "retrieve_all").Observe(time.Since(begin).Seconds())
}(time.Now())
return mm.service.RetrieveAll(ctx, token, page)
}
journal/middleware/tracing.go
+46
View File
@@ -0,0 +1,46 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package middleware
import (
"context"
"github.com/absmach/magistrala/journal"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
var _ journal.Service = (*tracing)(nil)
type tracing struct {
tracer trace.Tracer
svc journal.Service
}
func Tracing(svc journal.Service, tracer trace.Tracer) journal.Service {
return &tracing{tracer, svc}
}
func (tm *tracing) Save(ctx context.Context, j journal.Journal) error {
ctx, span := tm.tracer.Start(ctx, "save", trace.WithAttributes(
attribute.String("occurred_at", j.OccurredAt.String()),
attribute.String("operation", j.Operation),
))
defer span.End()
return tm.svc.Save(ctx, j)
}
func (tm *tracing) RetrieveAll(ctx context.Context, token string, page journal.Page) (resp journal.JournalsPage, err error) {
ctx, span := tm.tracer.Start(ctx, "retrieve_all", trace.WithAttributes(
attribute.Int64("offset", int64(page.Offset)),
attribute.Int64("limit", int64(page.Limit)),
attribute.Int64("total", int64(resp.Total)),
attribute.String("entity_type", page.EntityType.String()),
attribute.String("operation", page.Operation),
))
defer span.End()
return tm.svc.RetrieveAll(ctx, token, page)
}
journal/mocks/doc.go
+5
View File
@@ -0,0 +1,5 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package mocks contains mocks for testing purposes.
package mocks
journal/mocks/repository.go
+77
View File
@@ -0,0 +1,77 @@
// Code generated by mockery v2.43.2. DO NOT EDIT.
// Copyright (c) Abstract Machines
package mocks
import (
context "context"
journal "github.com/absmach/magistrala/journal"
mock "github.com/stretchr/testify/mock"
)
// Repository is an autogenerated mock type for the Repository type
type Repository struct {
mock.Mock
}
// RetrieveAll provides a mock function with given fields: ctx, page
func (_m *Repository) RetrieveAll(ctx context.Context, page journal.Page) (journal.JournalsPage, error) {
ret := _m.Called(ctx, page)
if len(ret) == 0 {
panic("no return value specified for RetrieveAll")
}
var r0 journal.JournalsPage
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, journal.Page) (journal.JournalsPage, error)); ok {
return rf(ctx, page)
}
if rf, ok := ret.Get(0).(func(context.Context, journal.Page) journal.JournalsPage); ok {
r0 = rf(ctx, page)
} else {
r0 = ret.Get(0).(journal.JournalsPage)
}
if rf, ok := ret.Get(1).(func(context.Context, journal.Page) error); ok {
r1 = rf(ctx, page)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Save provides a mock function with given fields: ctx, _a1
func (_m *Repository) Save(ctx context.Context, _a1 journal.Journal) error {
ret := _m.Called(ctx, _a1)
if len(ret) == 0 {
panic("no return value specified for Save")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, journal.Journal) error); ok {
r0 = rf(ctx, _a1)
} else {
r0 = ret.Error(0)
}
return r0
}
// NewRepository creates a new instance of Repository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewRepository(t interface {
mock.TestingT
Cleanup(func())
}) *Repository {
mock := &Repository{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
journal/mocks/service.go
+77
View File
@@ -0,0 +1,77 @@
// Code generated by mockery v2.43.2. DO NOT EDIT.
// Copyright (c) Abstract Machines
package mocks
import (
context "context"
journal "github.com/absmach/magistrala/journal"
mock "github.com/stretchr/testify/mock"
)
// Service is an autogenerated mock type for the Service type
type Service struct {
mock.Mock
}
// RetrieveAll provides a mock function with given fields: ctx, token, page
func (_m *Service) RetrieveAll(ctx context.Context, token string, page journal.Page) (journal.JournalsPage, error) {
ret := _m.Called(ctx, token, page)
if len(ret) == 0 {
panic("no return value specified for RetrieveAll")
}
var r0 journal.JournalsPage
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, journal.Page) (journal.JournalsPage, error)); ok {
return rf(ctx, token, page)
}
if rf, ok := ret.Get(0).(func(context.Context, string, journal.Page) journal.JournalsPage); ok {
r0 = rf(ctx, token, page)
} else {
r0 = ret.Get(0).(journal.JournalsPage)
}
if rf, ok := ret.Get(1).(func(context.Context, string, journal.Page) error); ok {
r1 = rf(ctx, token, page)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Save provides a mock function with given fields: ctx, _a1
func (_m *Service) Save(ctx context.Context, _a1 journal.Journal) error {
ret := _m.Called(ctx, _a1)
if len(ret) == 0 {
panic("no return value specified for Save")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, journal.Journal) error); ok {
r0 = rf(ctx, _a1)
} else {
r0 = ret.Error(0)
}
return r0
}
// NewService creates a new instance of Service. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewService(t interface {
mock.TestingT
Cleanup(func())
}) *Service {
mock := &Service{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
journal/postgres/doc.go
+5
View File
@@ -0,0 +1,5 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package postgres provides a postgres implementation of the journal log repository.
package postgres
journal/postgres/init.go
+36
View File
@@ -0,0 +1,36 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package postgres
import (
_ "github.com/jackc/pgx/v5/stdlib" // required for SQL access
migrate "github.com/rubenv/sql-migrate"
)
func Migration() *migrate.MemoryMigrationSource {
return &migrate.MemoryMigrationSource{
Migrations: []*migrate.Migration{
{
Id: "journal_01",
Up: []string{
`CREATE TABLE IF NOT EXISTS journal (
id VARCHAR(36) PRIMARY KEY,
operation VARCHAR NOT NULL,
occurred_at TIMESTAMP NOT NULL,
attributes JSONB NOT NULL,
metadata JSONB,
UNIQUE(operation, occurred_at, attributes)
)`,
`CREATE INDEX idx_journal_default_user_filter ON journal(operation, (attributes->>'id'), (attributes->>'user_id'), occurred_at DESC);`,
`CREATE INDEX idx_journal_default_group_filter ON journal(operation, (attributes->>'id'), (attributes->>'group_id'), occurred_at DESC);`,
`CREATE INDEX idx_journal_default_thing_filter ON journal(operation, (attributes->>'id'), (attributes->>'thing_id'), occurred_at DESC);`,
`CREATE INDEX idx_journal_default_channel_filter ON journal(operation, (attributes->>'id'), (attributes->>'channel_id'), occurred_at DESC);`,
},
Down: []string{
`DROP TABLE IF EXISTS journal`,
},
},
},
}
}
journal/postgres/journal.go
+178
View File
@@ -0,0 +1,178 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package postgres
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/absmach/magistrala/internal/postgres"
"github.com/absmach/magistrala/journal"
"github.com/absmach/magistrala/pkg/errors"
repoerr "github.com/absmach/magistrala/pkg/errors/repository"
)
type repository struct {
db postgres.Database
}
func NewRepository(db postgres.Database) journal.Repository {
return &repository{db: db}
}
func (repo *repository) Save(ctx context.Context, j journal.Journal) (err error) {
q := `INSERT INTO journal (id, operation, occurred_at, attributes, metadata)
VALUES (:id, :operation, :occurred_at, :attributes, :metadata);`
dbJournal, err := toDBJournal(j)
if err != nil {
return errors.Wrap(repoerr.ErrCreateEntity, err)
}
if _, err = repo.db.NamedExecContext(ctx, q, dbJournal); err != nil {
return postgres.HandleError(repoerr.ErrCreateEntity, err)
}
return nil
}
func (repo *repository) RetrieveAll(ctx context.Context, page journal.Page) (journal.JournalsPage, error) {
query := pageQuery(page)
sq := "operation, occurred_at"
if page.WithAttributes {
sq += ", attributes"
}
if page.WithMetadata {
sq += ", metadata"
}
if page.Direction == "" {
page.Direction = "ASC"
}
q := fmt.Sprintf("SELECT %s FROM journal %s ORDER BY occurred_at %s LIMIT :limit OFFSET :offset;", sq, query, page.Direction)
rows, err := repo.db.NamedQueryContext(ctx, q, page)
if err != nil {
return journal.JournalsPage{}, postgres.HandleError(repoerr.ErrViewEntity, err)
}
defer rows.Close()
var items []journal.Journal
for rows.Next() {
var item dbJournal
if err = rows.StructScan(&item); err != nil {
return journal.JournalsPage{}, postgres.HandleError(repoerr.ErrViewEntity, err)
}
j, err := toJournal(item)
if err != nil {
return journal.JournalsPage{}, err
}
items = append(items, j)
}
tq := fmt.Sprintf(`SELECT COUNT(*) FROM journal %s;`, query)
total, err := postgres.Total(ctx, repo.db, tq, page)
if err != nil {
return journal.JournalsPage{}, postgres.HandleError(repoerr.ErrViewEntity, err)
}
journalsPage := journal.JournalsPage{
Total: total,
Offset: page.Offset,
Limit: page.Limit,
Journals: items,
}
return journalsPage, nil
}
func pageQuery(pm journal.Page) string {
var query []string
var emq string
if pm.Operation != "" {
query = append(query, "operation = :operation")
}
if !pm.From.IsZero() {
query = append(query, "occurred_at >= :from")
}
if !pm.To.IsZero() {
query = append(query, "occurred_at <= :to")
}
if pm.EntityID != "" {
query = append(query, pm.EntityType.Query())
}
if len(query) > 0 {
emq = fmt.Sprintf("WHERE %s", strings.Join(query, " AND "))
}
return emq
}
type dbJournal struct {
ID string `db:"id"`
Operation string `db:"operation"`
OccurredAt time.Time `db:"occurred_at"`
Attributes []byte `db:"attributes"`
Metadata []byte `db:"metadata"`
}
func toDBJournal(j journal.Journal) (dbJournal, error) {
if j.OccurredAt.IsZero() {
j.OccurredAt = time.Now()
}
attributes := []byte("{}")
if len(j.Attributes) > 0 {
b, err := json.Marshal(j.Attributes)
if err != nil {
return dbJournal{}, errors.Wrap(repoerr.ErrMalformedEntity, err)
}
attributes = b
}
metadata := []byte("{}")
if len(j.Metadata) > 0 {
b, err := json.Marshal(j.Metadata)
if err != nil {
return dbJournal{}, errors.Wrap(repoerr.ErrMalformedEntity, err)
}
metadata = b
}
return dbJournal{
ID: j.ID,
Operation: j.Operation,
OccurredAt: j.OccurredAt,
Attributes: attributes,
Metadata: metadata,
}, nil
}
func toJournal(dbj dbJournal) (journal.Journal, error) {
var attributes map[string]interface{}
if dbj.Attributes != nil {
if err := json.Unmarshal(dbj.Attributes, &attributes); err != nil {
return journal.Journal{}, errors.Wrap(repoerr.ErrMalformedEntity, err)
}
}
var metadata map[string]interface{}
if dbj.Metadata != nil {
if err := json.Unmarshal(dbj.Metadata, &metadata); err != nil {
return journal.Journal{}, errors.Wrap(repoerr.ErrMalformedEntity, err)
}
}
return journal.Journal{
Operation: dbj.Operation,
OccurredAt: dbj.OccurredAt,
Attributes: attributes,
Metadata: metadata,
}, nil
}
journal/postgres/journal_test.go
+724
View File
@@ -0,0 +1,724 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package postgres_test
import (
"context"
"fmt"
"math/rand"
"sort"
"strings"
"testing"
"time"
"github.com/absmach/magistrala/internal/testsutil"
"github.com/absmach/magistrala/journal"
"github.com/absmach/magistrala/journal/postgres"
"github.com/absmach/magistrala/pkg/errors"
repoerr "github.com/absmach/magistrala/pkg/errors/repository"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var (
operation = "user.create"
payload = map[string]interface{}{
"temperature": rand.Float64(),
"humidity": float64(rand.Intn(1000)),
"locations": []interface{}{
strings.Repeat("a", 100),
strings.Repeat("a", 100),
},
"status": "active",
"nested": map[string]interface{}{
"nested": map[string]interface{}{
"nested": map[string]interface{}{
"nested": map[string]interface{}{
"key": "value",
},
},
},
},
}
entityID = testsutil.GenerateUUID(&testing.T{})
thingOperation = "thing.create"
thingAttributesV1 = map[string]interface{}{
"id": entityID,
"status": "enabled",
"created_at": time.Now().Add(-time.Hour),
"name": "thing",
"tags": []interface{}{"tag1", "tag2"},
"domain": testsutil.GenerateUUID(&testing.T{}),
"metadata": payload,
"identity": testsutil.GenerateUUID(&testing.T{}),
}
thingAttributesV2 = map[string]interface{}{
"thing_id": entityID,
"metadata": payload,
}
userAttributesV1 = map[string]interface{}{
"id": entityID,
"status": "enabled",
"created_at": time.Now().Add(-time.Hour),
"name": "user",
"tags": []interface{}{"tag1", "tag2"},
"domain": testsutil.GenerateUUID(&testing.T{}),
"metadata": payload,
"identity": testsutil.GenerateUUID(&testing.T{}),
}
userAttributesV2 = map[string]interface{}{
"user_id": entityID,
"metadata": payload,
}
)
func TestJournalSave(t *testing.T) {
t.Cleanup(func() {
_, err := db.Exec("DELETE FROM journal")
require.Nil(t, err, fmt.Sprintf("clean journal unexpected error: %s", err))
})
repo := postgres.NewRepository(database)
occurredAt := time.Now()
id := testsutil.GenerateUUID(t)
cases := []struct {
desc string
journal journal.Journal
err error
}{
{
desc: "new journal successfully",
journal: journal.Journal{
ID: id,
Operation: operation,
OccurredAt: occurredAt,
Attributes: payload,
Metadata: payload,
},
err: nil,
},
{
desc: "with duplicate journal",
journal: journal.Journal{
ID: id,
Operation: operation,
OccurredAt: occurredAt,
Attributes: payload,
Metadata: payload,
},
err: repoerr.ErrConflict,
},
{
desc: "with massive journal metadata and attributes",
journal: journal.Journal{
ID: testsutil.GenerateUUID(t),
Operation: operation,
OccurredAt: time.Now(),
Attributes: map[string]interface{}{
"attributes": map[string]interface{}{
"attributes": map[string]interface{}{
"attributes": map[string]interface{}{
"attributes": map[string]interface{}{
"attributes": map[string]interface{}{
"data": payload,
},
"data": payload,
},
"data": payload,
},
"data": payload,
},
"data": payload,
},
"data": payload,
},
Metadata: map[string]interface{}{
"metadata": map[string]interface{}{
"metadata": map[string]interface{}{
"metadata": map[string]interface{}{
"metadata": map[string]interface{}{
"metadata": map[string]interface{}{
"data": payload,
},
"data": payload,
},
"data": payload,
},
"data": payload,
},
"data": payload,
},
"data": payload,
},
},
err: nil,
},
{
desc: "with nil journal operation",
journal: journal.Journal{
ID: testsutil.GenerateUUID(t),
OccurredAt: time.Now(),
Attributes: payload,
Metadata: payload,
},
err: repoerr.ErrCreateEntity,
},
{
desc: "with empty journal operation",
journal: journal.Journal{
ID: testsutil.GenerateUUID(t),
Operation: "",
OccurredAt: time.Now().Add(-time.Hour),
Attributes: payload,
Metadata: payload,
},
err: nil,
},
{
desc: "with nil journal occurred_at",
journal: journal.Journal{
ID: testsutil.GenerateUUID(t),
Operation: operation,
Attributes: payload,
Metadata: payload,
},
err: repoerr.ErrCreateEntity,
},
{
desc: "with empty journal occurred_at",
journal: journal.Journal{
ID: testsutil.GenerateUUID(t),
Operation: operation,
OccurredAt: time.Time{},
Attributes: payload,
Metadata: payload,
},
err: nil,
},
{
desc: "with nil journal attributes",
journal: journal.Journal{
ID: testsutil.GenerateUUID(t),
Operation: operation + ".with.nil.attributes",
OccurredAt: time.Now(),
Metadata: payload,
},
err: nil,
},
{
desc: "with invalid journal attributes",
journal: journal.Journal{
ID: testsutil.GenerateUUID(t),
Operation: operation,
OccurredAt: time.Now(),
Attributes: map[string]interface{}{"invalid": make(chan struct{})},
Metadata: payload,
},
err: repoerr.ErrCreateEntity,
},
{
desc: "with empty journal attributes",
journal: journal.Journal{
ID: testsutil.GenerateUUID(t),
Operation: operation + ".with.empty.attributes",
OccurredAt: time.Now(),
Attributes: map[string]interface{}{},
Metadata: payload,
},
err: nil,
},
{
desc: "with nil journal metadata",
journal: journal.Journal{
ID: testsutil.GenerateUUID(t),
Operation: operation + ".with.nil.metadata",
OccurredAt: time.Now(),
Attributes: payload,
},
err: nil,
},
{
desc: "with invalid journal metadata",
journal: journal.Journal{
ID: testsutil.GenerateUUID(t),
Operation: operation,
OccurredAt: time.Now(),
Metadata: map[string]interface{}{"invalid": make(chan struct{})},
Attributes: payload,
},
err: repoerr.ErrCreateEntity,
},
{
desc: "with empty journal metadata",
journal: journal.Journal{
ID: testsutil.GenerateUUID(t),
Operation: operation + ".with.empty.metadata",
OccurredAt: time.Now(),
Metadata: map[string]interface{}{},
Attributes: payload,
},
err: nil,
},
{
desc: "with empty journal",
journal: journal.Journal{},
err: repoerr.ErrCreateEntity,
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
switch err := repo.Save(context.Background(), tc.journal); {
case err == nil:
assert.Nil(t, err)
default:
assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err))
}
})
}
}
func TestJournalRetrieveAll(t *testing.T) {
t.Cleanup(func() {
_, err := db.Exec("DELETE FROM journal")
require.Nil(t, err, fmt.Sprintf("clean journal unexpected error: %s", err))
})
repo := postgres.NewRepository(database)
num := 200
var items []journal.Journal
for i := 0; i < num; i++ {
j := journal.Journal{
ID: testsutil.GenerateUUID(t),
Operation: fmt.Sprintf("%s-%d", operation, i),
OccurredAt: time.Now().UTC().Truncate(time.Millisecond),
Attributes: userAttributesV1,
Metadata: payload,
}
if i%2 == 0 {
j.Operation = fmt.Sprintf("%s-%d", thingOperation, i)
j.Attributes = thingAttributesV1
}
if i%3 == 0 {
j.Attributes = userAttributesV2
}
if i%5 == 0 {
j.Attributes = thingAttributesV2
}
err := repo.Save(context.Background(), j)
require.Nil(t, err, fmt.Sprintf("create journal unexpected error: %s", err))
j.ID = ""
items = append(items, j)
}
reversedItems := make([]journal.Journal, len(items))
copy(reversedItems, items)
sort.Slice(reversedItems, func(i, j int) bool {
return reversedItems[i].OccurredAt.After(reversedItems[j].OccurredAt)
})
cases := []struct {
desc string
page journal.Page
response journal.JournalsPage
err error
}{
{
desc: "successfully",
page: journal.Page{
Offset: 0,
Limit: 1,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 0,
Limit: 1,
Journals: items[:1],
},
err: nil,
},
{
desc: "with offset and empty limit",
page: journal.Page{
Offset: 10,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 10,
Limit: 0,
Journals: []journal.Journal(nil),
},
},
{
desc: "with limit and empty offset",
page: journal.Page{
Limit: 50,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 0,
Limit: 50,
Journals: items[:50],
},
},
{
desc: "with offset and limit",
page: journal.Page{
Offset: 10,
Limit: 50,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 10,
Limit: 50,
Journals: items[10:60],
},
},
{
desc: "with offset out of range",
page: journal.Page{
Offset: 1000,
Limit: 50,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 1000,
Limit: 50,
Journals: []journal.Journal(nil),
},
},
{
desc: "with offset and limit out of range",
page: journal.Page{
Offset: 170,
Limit: 50,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 170,
Limit: 50,
Journals: items[170:200],
},
},
{
desc: "with limit out of range",
page: journal.Page{
Offset: 0,
Limit: 1000,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 0,
Limit: 1000,
Journals: items,
},
},
{
desc: "with empty page",
page: journal.Page{},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 0,
Limit: 0,
Journals: []journal.Journal(nil),
},
},
{
desc: "with operation",
page: journal.Page{
Operation: items[0].Operation,
Offset: 0,
Limit: 10,
},
response: journal.JournalsPage{
Total: 1,
Offset: 0,
Limit: 10,
Journals: []journal.Journal{items[0]},
},
},
{
desc: "with invalid operation",
page: journal.Page{
Operation: strings.Repeat("a", 37),
Offset: 0,
Limit: 10,
},
response: journal.JournalsPage{
Total: 0,
Offset: 0,
Limit: 10,
Journals: []journal.Journal(nil),
},
},
{
desc: "with attributes",
page: journal.Page{
WithAttributes: true,
Offset: 0,
Limit: 10,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 0,
Limit: 10,
Journals: items[:10],
},
},
{
desc: "with metadata",
page: journal.Page{
WithMetadata: true,
Offset: 0,
Limit: 10,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 0,
Limit: 10,
Journals: items[:10],
},
},
{
desc: "with attributes and Metadata",
page: journal.Page{
WithAttributes: true,
WithMetadata: true,
Offset: 0,
Limit: 10,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 0,
Limit: 10,
Journals: items[:10],
},
},
{
desc: "with from",
page: journal.Page{
From: items[0].OccurredAt,
Offset: 0,
Limit: 10,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 0,
Limit: 10,
Journals: items[:10],
},
},
{
desc: "with invalid from",
page: journal.Page{
From: time.Now().UTC().Truncate(time.Millisecond).Add(time.Hour),
Offset: 0,
Limit: 10,
},
response: journal.JournalsPage{
Total: 0,
Offset: 0,
Limit: 10,
Journals: []journal.Journal(nil),
},
},
{
desc: "with to",
page: journal.Page{
To: items[num-1].OccurredAt,
Offset: 0,
Limit: 10,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 0,
Limit: 10,
Journals: items[:10],
},
},
{
desc: "with invalid to",
page: journal.Page{
To: time.Now().UTC().Truncate(time.Millisecond).Add(-time.Hour),
Offset: 0,
Limit: 10,
},
response: journal.JournalsPage{
Total: 0,
Offset: 0,
Limit: 10,
Journals: []journal.Journal(nil),
},
},
{
desc: "with from and to",
page: journal.Page{
From: items[0].OccurredAt,
To: items[num-1].OccurredAt,
Offset: 0,
Limit: 10,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 0,
Limit: 10,
Journals: items[:10],
},
},
{
desc: "with asc direction",
page: journal.Page{
Direction: "ASC",
Offset: 0,
Limit: 10,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 0,
Limit: 10,
Journals: items[:10],
},
},
{
desc: "with desc direction",
page: journal.Page{
Direction: "DESC",
Offset: 0,
Limit: 10,
},
response: journal.JournalsPage{
Total: uint64(num),
Offset: 0,
Limit: 10,
Journals: reversedItems[:10],
},
},
{
desc: "with user entity type",
page: journal.Page{
Offset: 0,
Limit: 10,
EntityID: entityID,
EntityType: journal.UserEntity,
},
response: journal.JournalsPage{
Total: uint64(len(extractEntities(items, journal.UserEntity, entityID))),
Offset: 0,
Limit: 10,
Journals: extractEntities(items, journal.UserEntity, entityID)[:10],
},
},
{
desc: "with user entity type, attributes and metadata",
page: journal.Page{
Offset: 0,
Limit: 10,
EntityID: entityID,
EntityType: journal.UserEntity,
WithAttributes: true,
WithMetadata: true,
},
response: journal.JournalsPage{
Total: uint64(len(extractEntities(items, journal.UserEntity, entityID))),
Offset: 0,
Limit: 10,
Journals: extractEntities(items, journal.UserEntity, entityID)[:10],
},
},
{
desc: "with thing entity type",
page: journal.Page{
Offset: 0,
Limit: 10,
EntityID: entityID,
EntityType: journal.ThingEntity,
},
response: journal.JournalsPage{
Total: uint64(len(extractEntities(items, journal.ThingEntity, entityID))),
Offset: 0,
Limit: 10,
Journals: extractEntities(items, journal.ThingEntity, entityID)[:10],
},
},
{
desc: "with invalid entity id",
page: journal.Page{
Offset: 0,
Limit: 10,
EntityID: testsutil.GenerateUUID(&testing.T{}),
EntityType: journal.ChannelEntity,
},
response: journal.JournalsPage{
Total: 0,
Offset: 0,
Limit: 10,
Journals: []journal.Journal(nil),
},
},
{
desc: "with all filters",
page: journal.Page{
Offset: 0,
Limit: 10,
Operation: items[0].Operation,
From: items[0].OccurredAt,
To: items[num-1].OccurredAt,
WithAttributes: true,
WithMetadata: true,
Direction: "asc",
},
response: journal.JournalsPage{
Total: 1,
Offset: 0,
Limit: 10,
Journals: []journal.Journal{items[0]},
},
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
page, err := repo.RetrieveAll(context.Background(), tc.page)
assert.Equal(t, tc.response.Total, page.Total)
assert.Equal(t, tc.response.Offset, page.Offset)
assert.Equal(t, tc.response.Limit, page.Limit)
for i := range tc.response.Journals {
tc.response.Journals[i].Attributes = map[string]interface{}{}
page.Journals[i].Attributes = map[string]interface{}{}
tc.response.Journals[i].Metadata = map[string]interface{}{}
page.Journals[i].Metadata = map[string]interface{}{}
}
assert.ElementsMatch(t, tc.response.Journals, page.Journals)
assert.Equal(t, tc.err, err)
})
}
}
func extractEntities(journals []journal.Journal, entityType journal.EntityType, entityID string) []journal.Journal {
var entities []journal.Journal
for _, j := range journals {
switch entityType {
case journal.UserEntity:
if strings.HasPrefix(j.Operation, "user.") && j.Attributes["id"] == entityID || j.Attributes["user_id"] == entityID {
entities = append(entities, j)
}
case journal.GroupEntity:
if strings.HasPrefix(j.Operation, "group.") && j.Attributes["id"] == entityID || j.Attributes["group_id"] == entityID {
entities = append(entities, j)
}
case journal.ThingEntity:
if strings.HasPrefix(j.Operation, "thing.") && j.Attributes["id"] == entityID || j.Attributes["thing_id"] == entityID {
entities = append(entities, j)
}
case journal.ChannelEntity:
if strings.HasPrefix(j.Operation, "channel.") && j.Attributes["id"] == entityID || j.Attributes["group_id"] == entityID {
entities = append(entities, j)
}
}
}
return entities
}
journal/postgres/setup_test.go
+94
View File
@@ -0,0 +1,94 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package postgres_test
import (
"database/sql"
"fmt"
"log"
"os"
"testing"
"time"
pgclient "github.com/absmach/magistrala/internal/clients/postgres"
"github.com/absmach/magistrala/internal/postgres"
apostgres "github.com/absmach/magistrala/journal/postgres"
"github.com/jmoiron/sqlx"
dockertest "github.com/ory/dockertest/v3"
"github.com/ory/dockertest/v3/docker"
"go.opentelemetry.io/otel"
)
var (
db *sqlx.DB
database postgres.Database
tracer = otel.Tracer("repo_tests")
)
func TestMain(m *testing.M) {
pool, err := dockertest.NewPool("")
if err != nil {
log.Fatalf("Could not connect to docker: %s", err)
}
container, err := pool.RunWithOptions(&dockertest.RunOptions{
Repository: "postgres",
Tag: "16.2-alpine",
Env: []string{
"POSTGRES_USER=test",
"POSTGRES_PASSWORD=test",
"POSTGRES_DB=test",
"listen_addresses = '*'",
},
}, func(config *docker.HostConfig) {
config.AutoRemove = true
config.RestartPolicy = docker.RestartPolicy{Name: "no"}
})
if err != nil {
log.Fatalf("Could not start container: %s", err)
}
port := container.GetPort("5432/tcp")
// exponential backoff-retry, because the application in the container might not be ready to accept connections yet
pool.MaxWait = 120 * time.Second
if err := pool.Retry(func() error {
url := fmt.Sprintf("host=localhost port=%s user=test dbname=test password=test sslmode=disable", port)
db, err := sql.Open("pgx", url)
if err != nil {
return err
}
return db.Ping()
}); err != nil {
log.Fatalf("Could not connect to docker: %s", err)
}
dbConfig := pgclient.Config{
Host: "localhost",
Port: port,
User: "test",
Pass: "test",
Name: "test",
SSLMode: "disable",
SSLCert: "",
SSLKey: "",
SSLRootCert: "",
}
if db, err = pgclient.Setup(dbConfig, *apostgres.Migration()); err != nil {
log.Fatalf("Could not setup test DB connection: %s", err)
}
database = postgres.NewDatabase(db, dbConfig, tracer)
code := m.Run()
// Defers will not be run when using os.Exit
db.Close()
if err := pool.Purge(container); err != nil {
log.Fatalf("Could not purge container: %s", err)
}
os.Exit(code)
}
journal/service.go
+85
View File
@@ -0,0 +1,85 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package journal
import (
"context"
"github.com/absmach/magistrala"
"github.com/absmach/magistrala/auth"
"github.com/absmach/magistrala/pkg/errors"
svcerr "github.com/absmach/magistrala/pkg/errors/service"
)
type service struct {
idProvider magistrala.IDProvider
auth magistrala.AuthServiceClient
repository Repository
}
func NewService(idp magistrala.IDProvider, repository Repository, authClient magistrala.AuthServiceClient) Service {
return &service{
idProvider: idp,
auth: authClient,
repository: repository,
}
}
func (svc *service) Save(ctx context.Context, journal Journal) error {
id, err := svc.idProvider.ID()
if err != nil {
return err
}
journal.ID = id
return svc.repository.Save(ctx, journal)
}
func (svc *service) RetrieveAll(ctx context.Context, token string, page Page) (JournalsPage, error) {
if err := svc.authorize(ctx, token, page.EntityID, page.EntityType.AuthString()); err != nil {
return JournalsPage{}, err
}
return svc.repository.RetrieveAll(ctx, page)
}
func (svc *service) authorize(ctx context.Context, token, entityID, entityType string) error {
user, err := svc.auth.Identify(ctx, &magistrala.IdentityReq{Token: token})
if err != nil {
return errors.Wrap(svcerr.ErrAuthentication, err)
}
permission := auth.ViewPermission
objectType := entityType
object := entityID
subject := user.GetId()
// If the entity is a user, we need to check if the user is an admin
if entityType == auth.UserType {
permission = auth.AdminPermission
objectType = auth.PlatformType
object = auth.MagistralaObject
subject = user.GetUserId()
}
req := &magistrala.AuthorizeReq{
Domain: user.GetDomainId(),
SubjectType: auth.UserType,
SubjectKind: auth.UsersKind,
Subject: subject,
Permission: permission,
ObjectType: objectType,
Object: object,
}
res, err := svc.auth.Authorize(ctx, req)
if err != nil {
return errors.Wrap(svcerr.ErrAuthorization, err)
}
if !res.GetAuthorized() {
return svcerr.ErrAuthorization
}
return nil
}
journal/service_test.go
+210
View File
@@ -0,0 +1,210 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package journal_test
import (
"context"
"fmt"
"math/rand"
"testing"
"time"
"github.com/absmach/magistrala"
"github.com/absmach/magistrala/auth"
authmocks "github.com/absmach/magistrala/auth/mocks"
"github.com/absmach/magistrala/internal/testsutil"
"github.com/absmach/magistrala/journal"
"github.com/absmach/magistrala/journal/mocks"
"github.com/absmach/magistrala/pkg/errors"
repoerr "github.com/absmach/magistrala/pkg/errors/repository"
svcerr "github.com/absmach/magistrala/pkg/errors/service"
"github.com/absmach/magistrala/pkg/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
var (
validJournal = journal.Journal{
Operation: "user.create",
OccurredAt: time.Now().Add(-time.Hour),
Attributes: map[string]interface{}{
"temperature": rand.Float64(),
"humidity": rand.Float64(),
},
Metadata: map[string]interface{}{
"sensor_id": rand.Intn(1000),
},
}
idProvider = uuid.New()
)
func TestSave(t *testing.T) {
repo := new(mocks.Repository)
authsvc := new(authmocks.AuthClient)
svc := journal.NewService(idProvider, repo, authsvc)
cases := []struct {
desc string
journal journal.Journal
repoErr error
err error
}{
{
desc: "successful with ID and EntityType",
journal: validJournal,
repoErr: nil,
err: nil,
},
{
desc: "with repo error",
repoErr: repoerr.ErrCreateEntity,
err: repoerr.ErrCreateEntity,
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
repoCall := repo.On("Save", context.Background(), mock.Anything).Return(tc.repoErr)
err := svc.Save(context.Background(), tc.journal)
assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err))
repoCall.Unset()
})
}
}
func TestReadAll(t *testing.T) {
repo := new(mocks.Repository)
authsvc := new(authmocks.AuthClient)
svc := journal.NewService(idProvider, repo, authsvc)
validToken := "token"
validPage := journal.Page{
Offset: 0,
Limit: 10,
EntityID: testsutil.GenerateUUID(t),
EntityType: journal.ThingEntity,
}
cases := []struct {
desc string
token string
page journal.Page
resp journal.JournalsPage
identifyRes *magistrala.IdentityRes
identifyErr error
authRes *magistrala.AuthorizeRes
authErr error
repoErr error
err error
}{
{
desc: "successful",
token: validToken,
page: validPage,
resp: journal.JournalsPage{
Total: 1,
Offset: 0,
Limit: 10,
Journals: []journal.Journal{validJournal},
},
identifyRes: &magistrala.IdentityRes{Id: testsutil.GenerateUUID(t), UserId: testsutil.GenerateUUID(t)},
authRes: &magistrala.AuthorizeRes{Authorized: true},
authErr: nil,
repoErr: nil,
err: nil,
},
{
desc: "successful for user",
token: validToken,
page: journal.Page{
Offset: 0,
Limit: 10,
EntityID: testsutil.GenerateUUID(t),
EntityType: journal.UserEntity,
},
resp: journal.JournalsPage{
Total: 1,
Offset: 0,
Limit: 10,
Journals: []journal.Journal{validJournal},
},
identifyRes: &magistrala.IdentityRes{Id: testsutil.GenerateUUID(t), UserId: testsutil.GenerateUUID(t)},
authRes: &magistrala.AuthorizeRes{Authorized: true},
authErr: nil,
repoErr: nil,
err: nil,
},
{
desc: "with identify error",
token: validToken,
page: validPage,
resp: journal.JournalsPage{},
identifyRes: &magistrala.IdentityRes{},
identifyErr: svcerr.ErrAuthentication,
err: svcerr.ErrAuthentication,
},
{
desc: "with repo error",
token: validToken,
page: validPage,
resp: journal.JournalsPage{},
identifyRes: &magistrala.IdentityRes{Id: testsutil.GenerateUUID(t), UserId: testsutil.GenerateUUID(t)},
authRes: &magistrala.AuthorizeRes{Authorized: true},
repoErr: repoerr.ErrViewEntity,
err: repoerr.ErrViewEntity,
},
{
desc: "with failed to authorize",
token: validToken,
page: validPage,
resp: journal.JournalsPage{},
identifyRes: &magistrala.IdentityRes{Id: testsutil.GenerateUUID(t), UserId: testsutil.GenerateUUID(t)},
authRes: &magistrala.AuthorizeRes{Authorized: false},
authErr: nil,
repoErr: nil,
err: svcerr.ErrAuthorization,
},
{
desc: "with error on authorize",
token: validToken,
page: validPage,
resp: journal.JournalsPage{},
identifyRes: &magistrala.IdentityRes{Id: testsutil.GenerateUUID(t), UserId: testsutil.GenerateUUID(t)},
authRes: &magistrala.AuthorizeRes{Authorized: true},
authErr: svcerr.ErrAuthorization,
repoErr: nil,
err: svcerr.ErrAuthorization,
},
}
for _, tc := range cases {
t.Run(tc.desc, func(t *testing.T) {
authReq := &magistrala.AuthorizeReq{
SubjectType: auth.UserType,
SubjectKind: auth.UsersKind,
Subject: tc.identifyRes.GetId(),
ObjectType: tc.page.EntityType.AuthString(),
Object: tc.page.EntityID,
Permission: auth.ViewPermission,
}
if tc.page.EntityType == journal.UserEntity {
authReq.Permission = auth.AdminPermission
authReq.ObjectType = auth.PlatformType
authReq.Object = auth.MagistralaObject
authReq.Subject = tc.identifyRes.GetUserId()
}
authCall := authsvc.On("Identify", context.Background(), &magistrala.IdentityReq{Token: tc.token}).Return(tc.identifyRes, tc.identifyErr)
authCall1 := authsvc.On("Authorize", context.Background(), authReq).Return(tc.authRes, tc.authErr)
repoCall := repo.On("RetrieveAll", context.Background(), tc.page).Return(tc.resp, tc.repoErr)
resp, err := svc.RetrieveAll(context.Background(), tc.token, tc.page)
if tc.err == nil {
assert.Equal(t, tc.resp, resp, tc.desc)
}
assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err))
repoCall.Unset()
authCall.Unset()
authCall1.Unset()
})
}
}
lora/events/events.go
+2 -2
View File
@@ -22,6 +22,6 @@ type removeChannelEvent struct {
}
type connectionThingEvent struct {
chanID string
thingID string
chanID string
thingIDs []string
}
lora/events/streams.go
+24 -45
View File
@@ -5,7 +5,6 @@ package events
import (
"context"
"encoding/json"
"errors"
"github.com/absmach/magistrala/lora"
@@ -58,35 +57,20 @@ func (es *eventHandler) Handle(ctx context.Context, event events.Event) error {
}
switch msg["operation"] {
case thingCreate:
case thingCreate, thingUpdate:
cte, derr := decodeCreateThing(msg)
if derr != nil {
err = derr
break
}
err = es.svc.CreateThing(ctx, cte.id, cte.loraDevEUI)
case thingUpdate:
ute, derr := decodeCreateThing(msg)
if derr != nil {
err = derr
break
}
err = es.svc.CreateThing(ctx, ute.id, ute.loraDevEUI)
case channelCreate:
case channelCreate, channelUpdate:
cce, derr := decodeCreateChannel(msg)
if derr != nil {
err = derr
break
}
err = es.svc.CreateChannel(ctx, cce.id, cce.loraAppID)
case channelUpdate:
uce, derr := decodeCreateChannel(msg)
if derr != nil {
err = derr
break
}
err = es.svc.CreateChannel(ctx, uce.id, uce.loraAppID)
case thingRemove:
rte := decodeRemoveThing(msg)
err = es.svc.RemoveThing(ctx, rte.id)
@@ -95,10 +79,22 @@ func (es *eventHandler) Handle(ctx context.Context, event events.Event) error {
err = es.svc.RemoveChannel(ctx, rce.id)
case thingConnect:
tce := decodeConnectionThing(msg)
err = es.svc.ConnectThing(ctx, tce.chanID, tce.thingID)
for _, thingID := range tce.thingIDs {
err = es.svc.ConnectThing(ctx, tce.chanID, thingID)
if err != nil {
return err
}
}
case thingDisconnect:
tde := decodeConnectionThing(msg)
err = es.svc.DisconnectThing(ctx, tde.chanID, tde.thingID)
for _, thingID := range tde.thingIDs {
err = es.svc.DisconnectThing(ctx, tde.chanID, thingID)
if err != nil {
return err
}
}
}
if err != nil && err != errMetadataType {
return err
@@ -108,14 +104,10 @@ func (es *eventHandler) Handle(ctx context.Context, event events.Event) error {
}
func decodeCreateThing(event map[string]interface{}) (createThingEvent, error) {
strmeta := read(event, "metadata", "{}")
var metadata map[string]interface{}
if err := json.Unmarshal([]byte(strmeta), &metadata); err != nil {
return createThingEvent{}, err
}
metadata := events.Read(event, "metadata", map[string]interface{}{})
cte := createThingEvent{
id: read(event, "id", ""),
id: events.Read(event, "id", ""),
}
m, ok := metadata[keyType]
@@ -139,19 +131,15 @@ func decodeCreateThing(event map[string]interface{}) (createThingEvent, error) {
func decodeRemoveThing(event map[string]interface{}) removeThingEvent {
return removeThingEvent{
id: read(event, "id", ""),
id: events.Read(event, "id", ""),
}
}
func decodeCreateChannel(event map[string]interface{}) (createChannelEvent, error) {
strmeta := read(event, "metadata", "{}")
var metadata map[string]interface{}
if err := json.Unmarshal([]byte(strmeta), &metadata); err != nil {
return createChannelEvent{}, err
}
metadata := events.Read(event, "metadata", map[string]interface{}{})
cce := createChannelEvent{
id: read(event, "id", ""),
id: events.Read(event, "id", ""),
}
m, ok := metadata[keyType]
@@ -175,22 +163,13 @@ func decodeCreateChannel(event map[string]interface{}) (createChannelEvent, erro
func decodeConnectionThing(event map[string]interface{}) connectionThingEvent {
return connectionThingEvent{
chanID: read(event, "chan_id", ""),
thingID: read(event, "thing_id", ""),
chanID: events.Read(event, "group_id", ""),
thingIDs: events.ReadStringSlice(event, "member_ids"),
}
}
func decodeRemoveChannel(event map[string]interface{}) removeChannelEvent {
return removeChannelEvent{
id: read(event, "id", ""),
id: events.Read(event, "id", ""),
}
}
func read(event map[string]interface{}, key, def string) string {
val, ok := event[key].(string)
if !ok {
return def
}
return val
}
mqtt/events/events.go
+4 -4
View File
@@ -9,14 +9,14 @@ var _ events.Event = (*mqttEvent)(nil)
type mqttEvent struct {
clientID string
eventType string
operation string
instance string
}
func (me mqttEvent) Encode() (map[string]interface{}, error) {
return map[string]interface{}{
"thing_id": me.clientID,
"event_type": me.eventType,
"instance": me.instance,
"thing_id": me.clientID,
"operation": me.operation,
"instance": me.instance,
}, nil
}
mqtt/events/streams.go
+2 -2
View File
@@ -42,7 +42,7 @@ func NewEventStore(ctx context.Context, url, instance string) (EventStore, error
func (es *eventStore) Connect(ctx context.Context, clientID string) error {
ev := mqttEvent{
clientID: clientID,
eventType: "connect",
operation: "connect",
instance: es.instance,
}
@@ -53,7 +53,7 @@ func (es *eventStore) Connect(ctx context.Context, clientID string) error {
func (es *eventStore) Disconnect(ctx context.Context, clientID string) error {
ev := mqttEvent{
clientID: clientID,
eventType: "disconnect",
operation: "disconnect",
instance: es.instance,
}
opcua/events/streams.go
+15 -60
View File
@@ -5,8 +5,6 @@ package events
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"github.com/absmach/magistrala/opcua"
@@ -14,16 +12,16 @@ import (
)
const (
keyType = "opcua"
keyNodeID = "node_id"
keyServerURI = "server_uri"
channelPrefix = "group."
thingPrefix = "thing."
keyType = "opcua"
keyNodeID = "node_id"
keyServerURI = "server_uri"
thingPrefix = "thing."
thingCreate = thingPrefix + "create"
thingUpdate = thingPrefix + "update"
thingRemove = thingPrefix + "remove"
channelPrefix = "channel."
channelCreate = channelPrefix + "create"
channelUpdate = channelPrefix + "update"
channelRemove = channelPrefix + "remove"
@@ -108,20 +106,10 @@ func (es *eventHandler) Handle(ctx context.Context, event events.Event) error {
}
func decodeCreateThing(event map[string]interface{}) (createThingEvent, error) {
strmeta := read(event, "metadata", "{}")
// Metadata is base64 encoded since it is marshalled as []byte.
meta, err := base64.StdEncoding.DecodeString(strmeta)
if err != nil {
return createThingEvent{}, err
}
var metadata map[string]interface{}
if err := json.Unmarshal(meta, &metadata); err != nil {
return createThingEvent{}, err
}
metadata := events.Read(event, "metadata", map[string]interface{}{})
cte := createThingEvent{
id: read(event, "id", ""),
id: events.Read(event, "id", ""),
}
metadataOpcua, ok := metadata[keyType]
@@ -145,23 +133,15 @@ func decodeCreateThing(event map[string]interface{}) (createThingEvent, error) {
func decodeRemoveThing(event map[string]interface{}) removeThingEvent {
return removeThingEvent{
id: read(event, "id", ""),
id: events.Read(event, "id", ""),
}
}
func decodeCreateChannel(event map[string]interface{}) (createChannelEvent, error) {
strmeta := read(event, "metadata", "{}")
meta, err := base64.StdEncoding.DecodeString(strmeta)
if err != nil {
return createChannelEvent{}, err
}
var metadata map[string]interface{}
if err := json.Unmarshal(meta, &metadata); err != nil {
return createChannelEvent{}, err
}
metadata := events.Read(event, "metadata", map[string]interface{}{})
cce := createChannelEvent{
id: read(event, "id", ""),
id: events.Read(event, "id", ""),
}
metadataOpcua, ok := metadata[keyType]
@@ -185,45 +165,20 @@ func decodeCreateChannel(event map[string]interface{}) (createChannelEvent, erro
func decodeRemoveChannel(event map[string]interface{}) removeChannelEvent {
return removeChannelEvent{
id: read(event, "id", ""),
id: events.Read(event, "id", ""),
}
}
func decodeConnectThing(event map[string]interface{}) connectThingEvent {
return connectThingEvent{
chanID: read(event, "group_id", ""),
thingIDs: readMemberIDs(event, "member_ids"),
chanID: events.Read(event, "group_id", ""),
thingIDs: events.ReadStringSlice(event, "member_ids"),
}
}
func decodeDisconnectThing(event map[string]interface{}) connectThingEvent {
return connectThingEvent{
chanID: read(event, "chan_id", ""),
thingIDs: readMemberIDs(event, "member_ids"),
chanID: events.Read(event, "group_id", ""),
thingIDs: events.ReadStringSlice(event, "member_ids"),
}
}
func read(event map[string]interface{}, key, def string) string {
val, ok := event[key].(string)
if !ok {
return def
}
return val
}
func readMemberIDs(event map[string]interface{}, key string) []string {
var memberIDs []string
val, ok := event[key].([]interface{})
if !ok {
return memberIDs
}
for _, v := range val {
if str, ok := v.(string); ok {
memberIDs = append(memberIDs, str)
}
}
return memberIDs
}
pkg/events/events.go
+30
View File
@@ -55,3 +55,33 @@ type Subscriber interface {
// Close gracefully closes event subscriber's connection.
Close() error
}
// Read reads value from event map.
// If value is not of type T, returns default value.
func Read[T any](event map[string]interface{}, key string, def T) T {
val, ok := event[key].(T)
if !ok {
return def
}
return val
}
// ReadStringSlice reads string slice from event map.
// If value is not a string slice, returns empty slice.
func ReadStringSlice(event map[string]interface{}, key string) []string {
var res []string
vals, ok := event[key].([]interface{})
if !ok {
return res
}
for _, v := range vals {
if s, ok := v.(string); ok {
res = append(res, s)
}
}
return res
}
pkg/events/redis/publisher.go
+7 -1
View File
@@ -5,6 +5,7 @@ package redis
import (
"context"
"encoding/json"
"sync"
"time"
@@ -45,11 +46,16 @@ func (es *pubEventStore) Publish(ctx context.Context, event events.Event) error
}
values["occurred_at"] = time.Now().UnixNano()
data, err := json.Marshal(values)
if err != nil {
return err
}
record := &redis.XAddArgs{
Stream: es.stream,
MaxLen: events.MaxEventStreamLen,
Approx: true,
Values: values,
Values: map[string]interface{}{"data": string(data)},
}
switch err := es.checkConnection(ctx); err {
pkg/events/redis/publisher_test.go
+14 -28
View File
@@ -5,11 +5,9 @@ package redis_test
import (
"context"
"encoding/json"
"errors"
"fmt"
"math/rand"
"strconv"
"testing"
"time"
@@ -33,23 +31,11 @@ type testEvent struct {
}
func (te testEvent) Encode() (map[string]interface{}, error) {
data := make(map[string]interface{})
for k, v := range te.Data {
switch v.(type) {
case string:
data[k] = v
case float64:
data[k] = v
default:
b, err := json.Marshal(v)
if err != nil {
return nil, err
}
data[k] = string(b)
}
if te.Data == nil {
return map[string]interface{}{}, nil
}
return data, nil
return te.Data, nil
}
func TestPublish(t *testing.T) {
@@ -87,12 +73,12 @@ func TestPublish(t *testing.T) {
desc: "publish event successfully",
err: nil,
event: map[string]interface{}{
"temperature": fmt.Sprintf("%f", rand.Float64()),
"humidity": fmt.Sprintf("%f", rand.Float64()),
"temperature": float64(rand.Float64()),
"humidity": float64(rand.Float64()),
"sensor_id": "abc123",
"location": "Earth",
"status": "normal",
"timestamp": fmt.Sprintf("%d", time.Now().UnixNano()),
"timestamp": float64(time.Now().UnixNano()),
"operation": "create",
"occurred_at": time.Now().UnixNano(),
},
@@ -106,22 +92,22 @@ func TestPublish(t *testing.T) {
desc: "publish event with invalid event location",
err: fmt.Errorf("json: unsupported type: chan int"),
event: map[string]interface{}{
"temperature": fmt.Sprintf("%f", rand.Float64()),
"humidity": fmt.Sprintf("%f", rand.Float64()),
"temperature": float64(rand.Float64()),
"humidity": float64(rand.Float64()),
"sensor_id": "abc123",
"location": make(chan int),
"status": "normal",
"timestamp": "invalid",
"operation": "create",
"occurred_at": time.Now().UnixNano(),
"occurred_at": float64(time.Now().UnixNano()),
},
},
{
desc: "publish event with nested sting value",
err: nil,
event: map[string]interface{}{
"temperature": fmt.Sprintf("%f", rand.Float64()),
"humidity": fmt.Sprintf("%f", rand.Float64()),
"temperature": float64(rand.Float64()),
"humidity": float64(rand.Float64()),
"sensor_id": "abc123",
"location": map[string]string{
"lat": fmt.Sprintf("%f", rand.Float64()),
@@ -130,7 +116,7 @@ func TestPublish(t *testing.T) {
"status": "normal",
"timestamp": "invalid",
"operation": "create",
"occurred_at": time.Now().UnixNano(),
"occurred_at": float64(time.Now().UnixNano()),
},
},
}
@@ -144,9 +130,9 @@ func TestPublish(t *testing.T) {
case nil:
receivedEvent := <-eventsChan
roa, err := strconv.ParseInt(receivedEvent["occurred_at"].(string), 10, 64)
roa := receivedEvent["occurred_at"].(float64)
assert.Nil(t, err)
if assert.WithinRange(t, time.Unix(0, roa), time.Now().Add(-time.Second), time.Now().Add(time.Second)) {
if assert.WithinRange(t, time.Unix(0, int64(roa)), time.Now().Add(-time.Second), time.Now().Add(time.Second)) {
delete(receivedEvent, "occurred_at")
delete(tc.event, "occurred_at")
}
pkg/events/redis/subscriber.go
+9 -1
View File
@@ -5,6 +5,7 @@ package redis
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
@@ -98,8 +99,15 @@ func (re redisEvent) Encode() (map[string]interface{}, error) {
func (es *subEventStore) handle(ctx context.Context, stream string, msgs []redis.XMessage, h events.EventHandler) {
for _, msg := range msgs {
var data map[string]interface{}
if err := json.Unmarshal([]byte(msg.Values["data"].(string)), &data); err != nil {
es.logger.Warn(fmt.Sprintf("failed to unmarshal redis event: %s", err))
return
}
event := redisEvent{
Data: msg.Values,
Data: data,
}
if err := h.Handle(ctx, event); err != nil {
pkg/events/store/store_nats.go
+3
View File
@@ -15,6 +15,9 @@ import (
"github.com/absmach/magistrala/pkg/events/nats"
)
// StreamAllEvents represents subject to subscribe for all the events.
const StreamAllEvents = "events.>"
func init() {
log.Println("The binary was build using nats as the events store")
}
pkg/events/store/store_rabbitmq.go
+3
View File
@@ -15,6 +15,9 @@ import (
"github.com/absmach/magistrala/pkg/events/rabbitmq"
)
// StreamAllEvents represents subject to subscribe for all the events.
const StreamAllEvents = "events.#"
func init() {
log.Println("The binary was build using rabbitmq as the events store")
}
pkg/events/store/store_redis.go
+3
View File
@@ -15,6 +15,9 @@ import (
"github.com/absmach/magistrala/pkg/events/redis"
)
// StreamAllEvents represents subject to subscribe for all the events.
const StreamAllEvents = ">"
func init() {
log.Println("The binary was build using redis as the events store")
}
pkg/sdk/go/groups_test.go
+1 -1
View File
@@ -441,7 +441,7 @@ func TestListGroups(t *testing.T) {
svcRes: groups.Page{},
svcErr: nil,
response: sdk.GroupsPage{},
err: errors.NewSDKErrorWithStatus(errors.Wrap(apiutil.ErrValidation, apiutil.ErrInvalidLevel), http.StatusInternalServerError),
err: errors.NewSDKErrorWithStatus(errors.Wrap(apiutil.ErrValidation, apiutil.ErrInvalidLevel), http.StatusBadRequest),
},
{
desc: "list groups with invalid page metadata",
pkg/sdk/go/journal.go
+56
View File
@@ -0,0 +1,56 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
package sdk
import (
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/absmach/magistrala/internal/apiutil"
"github.com/absmach/magistrala/pkg/errors"
)
const journalEndpoint = "journal"
type Journal struct {
ID string `json:"id,omitempty"`
Operation string `json:"operation,omitempty"`
OccurredAt time.Time `json:"occurred_at,omitempty"`
Payload Metadata `json:"payload,omitempty"`
}
type JournalsPage struct {
Total uint64 `json:"total"`
Offset uint64 `json:"offset"`
Limit uint64 `json:"limit"`
Journals []Journal `json:"journals"`
}
func (sdk mgSDK) Journal(entityType, entityID string, pm PageMetadata, token string) (journals JournalsPage, err error) {
if entityID == "" {
return JournalsPage{}, errors.NewSDKError(apiutil.ErrMissingID)
}
if entityType == "" {
return JournalsPage{}, errors.NewSDKError(apiutil.ErrMissingEntityType)
}
url, err := sdk.withQueryParams(sdk.journalURL, fmt.Sprintf("%s/%s/%s", journalEndpoint, entityType, entityID), pm)
if err != nil {
return JournalsPage{}, errors.NewSDKError(err)
}
_, body, sdkerr := sdk.processRequest(http.MethodGet, url, token, nil, nil, http.StatusOK)
if sdkerr != nil {
return JournalsPage{}, sdkerr
}
var journalsPage JournalsPage
if err := json.Unmarshal(body, &journalsPage); err != nil {
return JournalsPage{}, errors.NewSDKError(err)
}
return journalsPage, nil
}
pkg/sdk/go/sdk.go
+27
View File
@@ -116,6 +116,12 @@ type PageMetadata struct {
UserID string `json:"user_id,omitempty"`
DomainID string `json:"domain_id,omitempty"`
Relation string `json:"relation,omitempty"`
Operation string `json:"operation,omitempty"`
From int64 `json:"from,omitempty"`
To int64 `json:"to,omitempty"`
WithMetadata bool `json:"with_metadata,omitempty"`
WithAttributes bool `json:"with_attributes,omitempty"`
ID string `json:"id,omitempty"`
}
// Credentials represent client credentials: it contains
@@ -1148,6 +1154,13 @@ type SDK interface {
// err := sdk.DeleteInvitation("userID", "domainID", "token")
// fmt.Println(err)
DeleteInvitation(userID, domainID, token string) (err error)
// Journal returns a list of journal logs.
//
// For example:
// journals, _ := sdk.Journal("thing", "thingID", PageMetadata{Offset: 0, Limit: 10, Operation: "users.create"}, "token")
// fmt.Println(journals)
Journal(entityType, entityID string, pm PageMetadata, token string) (journal JournalsPage, err error)
}
type mgSDK struct {
@@ -1159,6 +1172,7 @@ type mgSDK struct {
usersURL string
domainsURL string
invitationsURL string
journalURL string
HostURL string
msgContentType ContentType
@@ -1176,6 +1190,7 @@ type Config struct {
UsersURL string
DomainsURL string
InvitationsURL string
JournalURL string
HostURL string
MsgContentType ContentType
@@ -1194,6 +1209,7 @@ func NewSDK(conf Config) SDK {
usersURL: conf.UsersURL,
domainsURL: conf.DomainsURL,
invitationsURL: conf.InvitationsURL,
journalURL: conf.JournalURL,
HostURL: conf.HostURL,
msgContentType: conf.MsgContentType,
@@ -1354,6 +1370,17 @@ func (pm PageMetadata) query() (string, error) {
if pm.Relation != "" {
q.Add("relation", pm.Relation)
}
if pm.Operation != "" {
q.Add("operation", pm.Operation)
}
if pm.From != 0 {
q.Add("from", strconv.FormatInt(pm.From, 10))
}
if pm.To != 0 {
q.Add("to", strconv.FormatInt(pm.To, 10))
}
q.Add("with_attributes", strconv.FormatBool(pm.WithAttributes))
q.Add("with_metadata", strconv.FormatBool(pm.WithMetadata))
return q.Encode(), nil
}
pkg/sdk/mocks/sdk.go
+28
View File
@@ -1412,6 +1412,34 @@ func (_m *SDK) IssueCert(thingID string, validity string, token string) (sdk.Cer
return r0, r1
}
// Journal provides a mock function with given fields: entityType, entityID, pm, token
func (_m *SDK) Journal(entityType string, entityID string, pm sdk.PageMetadata, token string) (sdk.JournalsPage, error) {
ret := _m.Called(entityType, entityID, pm, token)
if len(ret) == 0 {
panic("no return value specified for Journal")
}
var r0 sdk.JournalsPage
var r1 error
if rf, ok := ret.Get(0).(func(string, string, sdk.PageMetadata, string) (sdk.JournalsPage, error)); ok {
return rf(entityType, entityID, pm, token)
}
if rf, ok := ret.Get(0).(func(string, string, sdk.PageMetadata, string) sdk.JournalsPage); ok {
r0 = rf(entityType, entityID, pm, token)
} else {
r0 = ret.Get(0).(sdk.JournalsPage)
}
if rf, ok := ret.Get(1).(func(string, string, sdk.PageMetadata, string) error); ok {
r1 = rf(entityType, entityID, pm, token)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ListChannelUserGroups provides a mock function with given fields: channelID, pm, token
func (_m *SDK) ListChannelUserGroups(channelID string, pm sdk.PageMetadata, token string) (sdk.GroupsPage, errors.SDKError) {
ret := _m.Called(channelID, pm, token)
things/events/events.go
+12 -44
View File
@@ -4,9 +4,6 @@
package events
import (
"encoding/json"
"fmt"
"strings"
"time"
mgclients "github.com/absmach/magistrala/pkg/clients"
@@ -57,19 +54,13 @@ func (cce createClientEvent) Encode() (map[string]interface{}, error) {
val["name"] = cce.Name
}
if len(cce.Tags) > 0 {
tags := fmt.Sprintf("[%s]", strings.Join(cce.Tags, ","))
val["tags"] = tags
val["tags"] = cce.Tags
}
if cce.Domain != "" {
val["domain"] = cce.Domain
}
if cce.Metadata != nil {
metadata, err := json.Marshal(cce.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = cce.Metadata
}
if cce.Credentials.Identity != "" {
val["identity"] = cce.Credentials.Identity
@@ -100,8 +91,7 @@ func (uce updateClientEvent) Encode() (map[string]interface{}, error) {
val["name"] = uce.Name
}
if len(uce.Tags) > 0 {
tags := fmt.Sprintf("[%s]", strings.Join(uce.Tags, ","))
val["tags"] = tags
val["tags"] = uce.Tags
}
if uce.Domain != "" {
val["domain"] = uce.Domain
@@ -110,12 +100,7 @@ func (uce updateClientEvent) Encode() (map[string]interface{}, error) {
val["identity"] = uce.Credentials.Identity
}
if uce.Metadata != nil {
metadata, err := json.Marshal(uce.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = uce.Metadata
}
if !uce.CreatedAt.IsZero() {
val["created_at"] = uce.CreatedAt
@@ -158,8 +143,7 @@ func (vce viewClientEvent) Encode() (map[string]interface{}, error) {
val["name"] = vce.Name
}
if len(vce.Tags) > 0 {
tags := fmt.Sprintf("[%s]", strings.Join(vce.Tags, ","))
val["tags"] = tags
val["tags"] = vce.Tags
}
if vce.Domain != "" {
val["domain"] = vce.Domain
@@ -168,12 +152,7 @@ func (vce viewClientEvent) Encode() (map[string]interface{}, error) {
val["identity"] = vce.Credentials.Identity
}
if vce.Metadata != nil {
metadata, err := json.Marshal(vce.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = vce.Metadata
}
if !vce.CreatedAt.IsZero() {
val["created_at"] = vce.CreatedAt
@@ -227,12 +206,7 @@ func (lce listClientEvent) Encode() (map[string]interface{}, error) {
val["dir"] = lce.Dir
}
if lce.Metadata != nil {
metadata, err := json.Marshal(lce.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = lce.Metadata
}
if lce.Domain != "" {
val["domain"] = lce.Domain
@@ -247,8 +221,7 @@ func (lce listClientEvent) Encode() (map[string]interface{}, error) {
val["status"] = lce.Status.String()
}
if len(lce.IDs) > 0 {
ids := fmt.Sprintf("[%s]", strings.Join(lce.IDs, ","))
val["ids"] = ids
val["ids"] = lce.IDs
}
if lce.Identity != "" {
val["identity"] = lce.Identity
@@ -281,12 +254,7 @@ func (lcge listClientByGroupEvent) Encode() (map[string]interface{}, error) {
val["dir"] = lcge.Dir
}
if lcge.Metadata != nil {
metadata, err := json.Marshal(lcge.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = lcge.Metadata
}
if lcge.Domain != "" {
val["domain"] = lcge.Domain
@@ -314,7 +282,7 @@ type identifyClientEvent struct {
func (ice identifyClientEvent) Encode() (map[string]interface{}, error) {
return map[string]interface{}{
"operation": clientIdentify,
"thing_id": ice.thingID,
"id": ice.thingID,
}, nil
}
@@ -334,7 +302,7 @@ type authorizeClientEvent struct {
func (ice authorizeClientEvent) Encode() (map[string]interface{}, error) {
val := map[string]interface{}{
"operation": clientAuthorize,
"thing_id": ice.thingID,
"id": ice.thingID,
}
if ice.namespace != "" {
val["namespace"] = ice.namespace
@@ -379,7 +347,7 @@ func (sce shareClientEvent) Encode() (map[string]interface{}, error) {
"operation": clientPrefix + sce.action,
"id": sce.id,
"relation": sce.relation,
"user_ids": strings.Join(sce.userIDs, ","),
"user_ids": sce.userIDs,
}, nil
}
users/api/endpoint_test.go
+7
View File
@@ -616,6 +616,13 @@ func TestListClients(t *testing.T) {
status: http.StatusBadRequest,
err: apiutil.ErrInvalidQueryParams,
},
{
desc: "list users with invalid order direction",
token: validToken,
query: "dir=invalid",
status: http.StatusBadRequest,
err: apiutil.ErrValidation,
},
{
desc: "list users with duplicate order direction",
token: validToken,
users/events/events.go
+11 -48
View File
@@ -4,9 +4,6 @@
package events
import (
"encoding/json"
"fmt"
"strings"
"time"
mgclients "github.com/absmach/magistrala/pkg/clients"
@@ -64,19 +61,13 @@ func (cce createClientEvent) Encode() (map[string]interface{}, error) {
val["name"] = cce.Name
}
if len(cce.Tags) > 0 {
tags := fmt.Sprintf("[%s]", strings.Join(cce.Tags, ","))
val["tags"] = tags
val["tags"] = cce.Tags
}
if cce.Domain != "" {
val["domain"] = cce.Domain
}
if cce.Metadata != nil {
metadata, err := json.Marshal(cce.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = cce.Metadata
}
if cce.Credentials.Identity != "" {
val["identity"] = cce.Credentials.Identity
@@ -107,19 +98,13 @@ func (uce updateClientEvent) Encode() (map[string]interface{}, error) {
val["name"] = uce.Name
}
if len(uce.Tags) > 0 {
tags := fmt.Sprintf("[%s]", strings.Join(uce.Tags, ","))
val["tags"] = tags
val["tags"] = uce.Tags
}
if uce.Credentials.Identity != "" {
val["identity"] = uce.Credentials.Identity
}
if uce.Metadata != nil {
metadata, err := json.Marshal(uce.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = uce.Metadata
}
if !uce.CreatedAt.IsZero() {
val["created_at"] = uce.CreatedAt
@@ -162,8 +147,7 @@ func (vce viewClientEvent) Encode() (map[string]interface{}, error) {
val["name"] = vce.Name
}
if len(vce.Tags) > 0 {
tags := fmt.Sprintf("[%s]", strings.Join(vce.Tags, ","))
val["tags"] = tags
val["tags"] = vce.Tags
}
if vce.Domain != "" {
val["domain"] = vce.Domain
@@ -172,12 +156,7 @@ func (vce viewClientEvent) Encode() (map[string]interface{}, error) {
val["identity"] = vce.Credentials.Identity
}
if vce.Metadata != nil {
metadata, err := json.Marshal(vce.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = vce.Metadata
}
if !vce.CreatedAt.IsZero() {
val["created_at"] = vce.CreatedAt
@@ -209,8 +188,7 @@ func (vpe viewProfileEvent) Encode() (map[string]interface{}, error) {
val["name"] = vpe.Name
}
if len(vpe.Tags) > 0 {
tags := fmt.Sprintf("[%s]", strings.Join(vpe.Tags, ","))
val["tags"] = tags
val["tags"] = vpe.Tags
}
if vpe.Domain != "" {
val["domain"] = vpe.Domain
@@ -219,12 +197,7 @@ func (vpe viewProfileEvent) Encode() (map[string]interface{}, error) {
val["identity"] = vpe.Credentials.Identity
}
if vpe.Metadata != nil {
metadata, err := json.Marshal(vpe.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = vpe.Metadata
}
if !vpe.CreatedAt.IsZero() {
val["created_at"] = vpe.CreatedAt
@@ -264,12 +237,7 @@ func (lce listClientEvent) Encode() (map[string]interface{}, error) {
val["dir"] = lce.Dir
}
if lce.Metadata != nil {
metadata, err := json.Marshal(lce.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = lce.Metadata
}
if lce.Domain != "" {
val["domain"] = lce.Domain
@@ -316,12 +284,7 @@ func (lcge listClientByGroupEvent) Encode() (map[string]interface{}, error) {
val["dir"] = lcge.Dir
}
if lcge.Metadata != nil {
metadata, err := json.Marshal(lcge.Metadata)
if err != nil {
return map[string]interface{}{}, err
}
val["metadata"] = metadata
val["metadata"] = lcge.Metadata
}
if lcge.Domain != "" {
val["domain"] = lcge.Domain
@@ -349,7 +312,7 @@ type identifyClientEvent struct {
func (ice identifyClientEvent) Encode() (map[string]interface{}, error) {
return map[string]interface{}{
"operation": clientIdentify,
"user_id": ice.userID,
"id": ice.userID,
}, nil
}