NOISSUE - Add migration scripts for Rules, Alarms and Reports (#3482)
Continuous Delivery / lint-and-build (push) Has been cancelled
Deploy GitHub Pages / swagger-ui (push) Has been cancelled
CI Pipeline / Lint Proto (push) Has been cancelled
CI Pipeline / Detect Changes (push) Has been cancelled
Continuous Delivery / Build and Push Docker Images (push) Has been cancelled
CI Pipeline / lint-and-build (push) Has been cancelled
CI Pipeline / Test ${{ matrix.module }} (push) Has been cancelled
CI Pipeline / Upload Coverage (push) Has been cancelled

Signed-off-by: Arvindh <arvindh91@gmail.com>
Signed-off-by: nyagamunene <stevenyaga2014@gmail.com>
Co-authored-by: nyagamunene <stevenyaga2014@gmail.com>
This commit is contained in:
Arvindh
2026-04-30 12:16:50 +05:30
committed by GitHub
parent a0bc7c2108
commit d840aeb1b9
8 changed files with 2060 additions and 2 deletions
+48
View File
@@ -151,6 +151,54 @@ make run_latest
---
## Upgrade from v0.19.0 to v0.20.0
Before upgrading, back up the Domains, Rules Engine, Reports, Alarms, Auth, and SpiceDB databases.
v0.20.0 adds new domain admin actions for alarms and reports, and it requires existing rules and reports to have their built-in admin roles backfilled. The service database migrations run when the v0.20.0 services start, then the role backfill scripts must be run once.
For the default Docker Compose setup:
```bash
cd docker
docker compose up -d \
spicedb-db spicedb-migrate spicedb \
auth-db auth \
domains-db domains \
re-db re \
reports-db reports \
alarms-db alarms
```
Wait until the services are running. The `auth` service must start successfully because it loads the SpiceDB schema.
From the repository root, run the backfills:
```bash
go run ./scripts/re-backfill-roles/
go run ./scripts/reports-backfill-roles/
```
The scripts are idempotent. If they are interrupted, fix the issue and run them again.
Expected successful summaries:
```text
backfill finished processed=<number> skipped=<number> failed=0
```
After the backfills finish, verify that the services are still running:
```bash
cd docker
docker compose ps re reports alarms domains auth spicedb
```
For non-default deployments, make sure the database and SpiceDB connection settings used by the backfill scripts match your environment before running them.
---
## Usage
```bash
+1 -1
View File
@@ -2,5 +2,5 @@
// SPDX-License-Identifier: Apache-2.0
// Package hasher contains the domain concept definitions needed to
// support Supermq users password hasher sub-service functionality.
// support Magistrala users password hasher sub-service functionality.
package hasher
+5
View File
@@ -137,12 +137,16 @@ alarm:
- view: alarm_read_permission
- update: alarm_update_permission
- delete: alarm_delete_permission
- assign: alarm_assign_permission
- acknowledge: alarm_acknowledge_permission
- resolve: alarm_resolve_permission
- alarm_assign: alarm_assign_permission
- alarm_acknowledge: alarm_acknowledge_permission
- alarm_resolve: alarm_resolve_permission
rule:
operations:
- add: rule_create_permission
- create: rule_create_permission
- list: rule_read_permission
- view: read_permission
@@ -174,6 +178,7 @@ rule:
report:
operations:
- add: report_create_permission
- create: report_create_permission
- list: report_read_permission
- generate: report_read_permission
+63 -1
View File
@@ -174,7 +174,7 @@ func Migration() (*migrate.MemoryMigrationSource, error) {
{
Id: "domain_7",
Up: []string{
`UPDATE domains
`UPDATE domains
SET metadata = (COALESCE(metadata, '{}'::jsonb) || COALESCE(metadata->'ui', '{}'::jsonb)) - 'ui'
WHERE metadata ? 'ui' AND jsonb_typeof(metadata->'ui') = 'object'`,
},
@@ -182,6 +182,68 @@ func Migration() (*migrate.MemoryMigrationSource, error) {
`SELECT 1`,
},
},
{
Id: "domains_roles_4",
Up: []string{
`INSERT INTO domains_role_actions (role_id, action)
SELECT dr.id, a.action
FROM domains_roles dr
CROSS JOIN (VALUES
('rule_create'),
('rule_read'),
('rule_update'),
('rule_delete'),
('rule_manage_role'),
('rule_add_role_users'),
('rule_remove_role_users'),
('rule_view_role_users'),
('alarm_update'),
('alarm_read'),
('alarm_delete'),
('alarm_assign'),
('alarm_acknowledge'),
('alarm_resolve'),
('report_create'),
('report_read'),
('report_update'),
('report_delete'),
('report_manage_role'),
('report_add_role_users'),
('report_remove_role_users'),
('report_view_role_users')
) AS a(action)
WHERE dr.name = 'admin'
ON CONFLICT DO NOTHING;`,
},
Down: []string{
`DELETE FROM domains_role_actions
WHERE action IN (
'rule_create',
'rule_read',
'rule_update',
'rule_delete',
'rule_manage_role',
'rule_add_role_users',
'rule_remove_role_users',
'rule_view_role_users',
'alarm_update',
'alarm_read',
'alarm_delete',
'alarm_assign',
'alarm_acknowledge',
'alarm_resolve',
'report_create',
'report_read',
'report_update',
'report_delete',
'report_manage_role',
'report_add_role_users',
'report_remove_role_users',
'report_view_role_users'
)
AND role_id IN (SELECT id FROM domains_roles WHERE name = 'admin');`,
},
},
},
}
+536
View File
@@ -0,0 +1,536 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package main backfills missing built-in roles for rules.
package main
import (
"context"
"database/sql"
"fmt"
"io"
"log"
"os"
"strings"
mglog "github.com/absmach/magistrala/logger"
"github.com/absmach/magistrala/pkg/errors"
"github.com/absmach/magistrala/pkg/policies"
"github.com/absmach/magistrala/pkg/policies/spicedb"
pgclient "github.com/absmach/magistrala/pkg/postgres"
"github.com/absmach/magistrala/pkg/roles"
spicedbdecoder "github.com/absmach/magistrala/pkg/spicedb"
"github.com/absmach/magistrala/pkg/uuid"
"github.com/absmach/magistrala/re"
"github.com/absmach/magistrala/re/operations"
repg "github.com/absmach/magistrala/re/postgres"
v1 "github.com/authzed/authzed-go/proto/authzed/api/v1"
"github.com/authzed/authzed-go/v1"
"github.com/authzed/grpcutil"
"go.opentelemetry.io/otel/trace/noop"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
const cmdName = "re_backfill_roles"
var (
logLevel = "info"
dryRun = false
limit = 0
defaultMemberID = ""
spicedbHost = "localhost"
spicedbPort = "50051"
spicedbPreSharedKey = "12345678"
spicedbSchemaFile = "docker/spicedb/schema.zed"
dbConfig = pgclient.Config{
Host: "localhost",
Port: "6009",
User: "magistrala",
Pass: "magistrala",
Name: "rules_engine",
SSLMode: "disable",
}
)
type missingRule struct {
ID string `db:"id"`
Name string `db:"name"`
DomainID string `db:"domain_id"`
CreatedBy sql.NullString `db:"created_by"`
}
func main() {
ctx := context.Background()
if limit < 0 {
log.Fatalf("invalid limit %d: limit must be >= 0", limit)
}
logger, err := mglog.New(os.Stdout, logLevel)
if err != nil {
log.Fatalf("failed to init logger: %s", err)
}
var exitCode int
defer mglog.ExitWithError(&exitCode)
sqlDB, err := pgclient.Connect(dbConfig)
if err != nil {
logger.Error("failed to connect to postgres", "error", err)
exitCode = 1
return
}
defer sqlDB.Close()
database := pgclient.NewDatabase(sqlDB, dbConfig, noop.NewTracerProvider().Tracer(cmdName))
rulesRepo := repg.NewRepository(database)
rulesWithoutRoles, err := listRulesWithoutRoles(ctx, database, limit)
if err != nil {
logger.Error("failed to list rules without roles", "error", err)
exitCode = 1
return
}
logger.Info("loaded rules without roles", "count", len(rulesWithoutRoles), "dry_run", dryRun)
if len(rulesWithoutRoles) == 0 {
return
}
availableActions, builtInRoles, err := availableActionsAndBuiltInRoles(spicedbSchemaFile)
if err != nil {
logger.Error("failed to load built-in role actions", "error", err)
exitCode = 1
return
}
adminRoleActions, err := builtInRoleActionStrings(builtInRoles, re.BuiltInRoleAdmin)
if err != nil {
logger.Error("failed to resolve built-in admin role actions", "error", err)
exitCode = 1
return
}
authzedClient, err := newAuthzedClient(spicedbHost, spicedbPort, spicedbPreSharedKey)
if err != nil {
logger.Error("failed to connect to spicedb", "error", err)
exitCode = 1
return
}
if dryRun {
var processed, skipped int
for _, rule := range rulesWithoutRoles {
memberID := strings.TrimSpace(rule.CreatedBy.String)
if memberID == "" {
memberID = strings.TrimSpace(defaultMemberID)
}
if rule.DomainID == "" {
skipped++
logger.Warn("skipping rule without domain_id", "rule_id", rule.ID, "name", rule.Name)
continue
}
if memberID == "" {
skipped++
logger.Warn("skipping rule without created_by and no default member override", "rule_id", rule.ID, "name", rule.Name)
continue
}
isDomainMember, err := isDomainRoleMember(ctx, database, rule.DomainID, memberID)
if err != nil {
skipped++
logger.Warn(
"skipping rule after failed domain membership check",
"rule_id", rule.ID,
"name", rule.Name,
"domain_id", rule.DomainID,
"member_id", memberID,
"error", err,
)
continue
}
candidatePolicies := []policies.Policy{
{
SubjectType: policies.DomainType,
Subject: rule.DomainID,
Relation: policies.DomainRelation,
ObjectType: operations.EntityType,
Object: rule.ID,
},
}
policiesToAdd, existingPolicies, err := filterMissingPolicies(ctx, authzedClient.PermissionsServiceClient, candidatePolicies)
if err != nil {
skipped++
logger.Warn(
"skipping rule after failed spicedb policy lookup",
"rule_id", rule.ID,
"name", rule.Name,
"domain_id", rule.DomainID,
"error", err,
)
continue
}
for _, existing := range existingPolicies {
logger.Info(
"dry run: spicedb policy already exists, will not be re-added",
"rule_id", rule.ID,
"subject_type", existing.SubjectType,
"subject", existing.Subject,
"relation", existing.Relation,
"object_type", existing.ObjectType,
"object", existing.Object,
)
}
if !isDomainMember {
logger.Warn(
"created_by user is not a member of the domain; role will be provisioned without member",
"rule_id", rule.ID,
"name", rule.Name,
"domain_id", rule.DomainID,
"member_id", memberID,
"created_by_exists_in_domain", false,
"role_actions", adminRoleActions,
)
processed++
logger.Info(
"dry run: would provision missing built-in role without member",
"rule_id", rule.ID,
"name", rule.Name,
"domain_id", rule.DomainID,
"member_id", memberID,
"created_by_exists_in_domain", false,
"role_actions", adminRoleActions,
"role_name", re.BuiltInRoleAdmin.String(),
"new_optional_policies", len(policiesToAdd),
"existing_optional_policies", len(existingPolicies),
)
continue
}
processed++
logger.Info(
"dry run: would provision missing built-in role",
"rule_id", rule.ID,
"name", rule.Name,
"domain_id", rule.DomainID,
"member_id", memberID,
"created_by_exists_in_domain", true,
"role_actions", adminRoleActions,
"role_name", re.BuiltInRoleAdmin.String(),
"new_optional_policies", len(policiesToAdd),
"existing_optional_policies", len(existingPolicies),
)
}
logger.Info(
"backfill finished",
"processed", processed,
"skipped", skipped,
"failed", 0,
"dry_run", true,
)
return
}
policyService := spicedb.NewPolicyService(authzedClient, logger)
provisioner, err := roles.NewProvisionManageService(
operations.EntityType,
rulesRepo,
policyService,
uuid.New(),
availableActions,
builtInRoles,
)
if err != nil {
logger.Error("failed to create roles provisioner", "error", err)
exitCode = 1
return
}
var processed, skipped, failed int
for _, rule := range rulesWithoutRoles {
memberID := strings.TrimSpace(rule.CreatedBy.String)
if memberID == "" {
memberID = strings.TrimSpace(defaultMemberID)
}
if rule.DomainID == "" {
skipped++
logger.Warn("skipping rule without domain_id", "rule_id", rule.ID, "name", rule.Name)
continue
}
if memberID == "" {
skipped++
logger.Warn("skipping rule without created_by and no default member override", "rule_id", rule.ID, "name", rule.Name)
continue
}
assignMembers := []roles.Member{}
isDomainMember, err := isDomainRoleMember(ctx, database, rule.DomainID, memberID)
if err != nil {
failed++
logger.Error(
"failed to check domain membership before provisioning role",
"rule_id", rule.ID,
"name", rule.Name,
"domain_id", rule.DomainID,
"member_id", memberID,
"error", err,
)
continue
}
if isDomainMember {
assignMembers = []roles.Member{roles.Member(memberID)}
} else {
logger.Warn(
"created_by user is not a member of the domain; provisioning role without member",
"rule_id", rule.ID,
"name", rule.Name,
"domain_id", rule.DomainID,
"member_id", memberID,
"created_by_exists_in_domain", false,
"role_actions", adminRoleActions,
)
}
candidatePolicies := []policies.Policy{
{
SubjectType: policies.DomainType,
Subject: rule.DomainID,
Relation: policies.DomainRelation,
ObjectType: operations.EntityType,
Object: rule.ID,
},
}
optionalPolicies, existingPolicies, err := filterMissingPolicies(ctx, authzedClient.PermissionsServiceClient, candidatePolicies)
if err != nil {
failed++
logger.Error(
"failed to check existing spicedb policies",
"rule_id", rule.ID,
"name", rule.Name,
"domain_id", rule.DomainID,
"error", err,
)
continue
}
for _, existing := range existingPolicies {
logger.Info(
"spicedb policy already exists, skipping re-add",
"rule_id", rule.ID,
"subject_type", existing.SubjectType,
"subject", existing.Subject,
"relation", existing.Relation,
"object_type", existing.ObjectType,
"object", existing.Object,
)
}
newBuiltInRoleMembers := map[roles.BuiltInRoleName][]roles.Member{
re.BuiltInRoleAdmin: assignMembers,
}
if _, err := provisioner.AddNewEntitiesRoles(
ctx,
rule.DomainID,
memberID,
[]string{rule.ID},
optionalPolicies,
newBuiltInRoleMembers,
); err != nil {
failed++
logger.Error(
"failed to provision missing built-in role",
"rule_id", rule.ID,
"name", rule.Name,
"domain_id", rule.DomainID,
"member_id", memberID,
"error", err,
)
continue
}
processed++
logger.Info(
"provisioned missing built-in role",
"rule_id", rule.ID,
"name", rule.Name,
"domain_id", rule.DomainID,
"member_id", memberID,
"created_by_exists_in_domain", isDomainMember,
"member_added", len(assignMembers) > 0,
"role_actions", adminRoleActions,
"role_name", re.BuiltInRoleAdmin.String(),
"new_optional_policies", len(optionalPolicies),
"existing_optional_policies", len(existingPolicies),
)
}
logger.Info(
"backfill finished",
"processed", processed,
"skipped", skipped,
"failed", failed,
"dry_run", dryRun,
)
if failed > 0 {
exitCode = 1
}
}
func listRulesWithoutRoles(ctx context.Context, db pgclient.Database, limit int) ([]missingRule, error) {
params := map[string]any{}
query := `
SELECT r.id, r.name, r.domain_id, r.created_by
FROM rules r
WHERE NOT EXISTS (
SELECT 1
FROM rules_roles rr
WHERE rr.entity_id = r.id
)
`
query += " ORDER BY r.created_at ASC NULLS LAST, r.id ASC"
if limit > 0 {
query += " LIMIT :limit"
params["limit"] = limit
}
rows, err := db.NamedQueryContext(ctx, query, params)
if err != nil {
return nil, errors.Wrap(fmt.Errorf("failed to query rules without roles"), err)
}
defer rows.Close()
var rules []missingRule
for rows.Next() {
var rule missingRule
if err := rows.StructScan(&rule); err != nil {
return nil, errors.Wrap(fmt.Errorf("failed to scan rule without role"), err)
}
rules = append(rules, rule)
}
if err := rows.Err(); err != nil {
return nil, errors.Wrap(fmt.Errorf("failed to iterate rules without roles"), err)
}
return rules, nil
}
func isDomainRoleMember(ctx context.Context, db pgclient.Database, domainID, memberID string) (bool, error) {
const query = `
SELECT EXISTS (
SELECT 1
FROM domains_role_members drm
WHERE drm.entity_id = $1 AND drm.member_id = $2
)
`
var exists bool
if err := db.QueryRowxContext(ctx, query, domainID, memberID).Scan(&exists); err != nil {
return false, errors.Wrap(fmt.Errorf("failed to check domain role membership"), err)
}
return exists, nil
}
func newAuthzedClient(spicedbHost, spicedbPort, spicedbPreSharedKey string) (*authzed.ClientWithExperimental, error) {
return authzed.NewClientWithExperimentalAPIs(
fmt.Sprintf("%s:%s", spicedbHost, spicedbPort),
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpcutil.WithInsecureBearerToken(spicedbPreSharedKey),
)
}
// filterMissingPolicies splits the given policies into those that do not yet
// exist in SpiceDB (returned first) and those that already exist (returned
// second). Any error from SpiceDB short-circuits with an empty result.
func filterMissingPolicies(ctx context.Context, permClient v1.PermissionsServiceClient, ps []policies.Policy) ([]policies.Policy, []policies.Policy, error) {
missing := make([]policies.Policy, 0, len(ps))
existing := make([]policies.Policy, 0)
for _, p := range ps {
ok, err := policyExists(ctx, permClient, p)
if err != nil {
return nil, nil, err
}
if ok {
existing = append(existing, p)
continue
}
missing = append(missing, p)
}
return missing, existing, nil
}
// policyExists returns true when SpiceDB already contains a relationship
// matching the supplied policy on (object_type, object, relation, subject_type,
// subject). The lookup is fully consistent and capped at one row.
func policyExists(ctx context.Context, permClient v1.PermissionsServiceClient, p policies.Policy) (bool, error) {
req := &v1.ReadRelationshipsRequest{
Consistency: &v1.Consistency{
Requirement: &v1.Consistency_FullyConsistent{FullyConsistent: true},
},
RelationshipFilter: &v1.RelationshipFilter{
ResourceType: p.ObjectType,
OptionalResourceId: p.Object,
OptionalRelation: p.Relation,
OptionalSubjectFilter: &v1.SubjectFilter{
SubjectType: p.SubjectType,
OptionalSubjectId: p.Subject,
},
},
OptionalLimit: 1,
}
stream, err := permClient.ReadRelationships(ctx, req)
if err != nil {
return false, errors.Wrap(fmt.Errorf("failed to read spicedb relationships"), err)
}
for {
_, err := stream.Recv()
switch {
case err == nil:
return true, nil
case errors.Contains(err, io.EOF):
return false, nil
default:
return false, errors.Wrap(fmt.Errorf("failed to receive spicedb relationship"), err)
}
}
}
func availableActionsAndBuiltInRoles(spicedbSchemaFile string) ([]roles.Action, map[roles.BuiltInRoleName][]roles.Action, error) {
availableActions, err := spicedbdecoder.GetActionsFromSchema(spicedbSchemaFile, operations.EntityType)
if err != nil {
return []roles.Action{}, map[roles.BuiltInRoleName][]roles.Action{}, err
}
builtInRoles := map[roles.BuiltInRoleName][]roles.Action{
re.BuiltInRoleAdmin: availableActions,
}
return availableActions, builtInRoles, nil
}
func builtInRoleActionStrings(builtInRoles map[roles.BuiltInRoleName][]roles.Action, roleName roles.BuiltInRoleName) ([]string, error) {
actions, ok := builtInRoles[roleName]
if !ok {
return nil, fmt.Errorf("built-in role %q not found", roleName)
}
ret := make([]string, 0, len(actions))
for _, action := range actions {
ret = append(ret, action.String())
}
return ret, nil
}
+532
View File
@@ -0,0 +1,532 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package main backfills missing built-in roles for reports.
package main
import (
"context"
"database/sql"
"fmt"
"io"
"log"
"os"
"strings"
mglog "github.com/absmach/magistrala/logger"
"github.com/absmach/magistrala/pkg/errors"
"github.com/absmach/magistrala/pkg/policies"
"github.com/absmach/magistrala/pkg/policies/spicedb"
pgclient "github.com/absmach/magistrala/pkg/postgres"
"github.com/absmach/magistrala/pkg/roles"
spicedbdecoder "github.com/absmach/magistrala/pkg/spicedb"
"github.com/absmach/magistrala/pkg/uuid"
"github.com/absmach/magistrala/reports"
"github.com/absmach/magistrala/reports/operations"
repg "github.com/absmach/magistrala/reports/postgres"
v1 "github.com/authzed/authzed-go/proto/authzed/api/v1"
"github.com/authzed/authzed-go/v1"
"github.com/authzed/grpcutil"
"go.opentelemetry.io/otel/trace/noop"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
const (
cmdName = "reports_backfill_roles"
)
var (
logLevel = "info"
dryRun = false
limit = 0
defaultMemberID = ""
spicedbHost = "localhost"
spicedbPort = "50051"
spicedbPreSharedKey = "12345678"
spicedbSchemaFile = "docker/spicedb/schema.zed"
dbConfig = pgclient.Config{
Host: "localhost",
Port: "6020",
User: "magistrala",
Pass: "magistrala",
Name: "reports",
SSLMode: "disable",
}
)
type missingReport struct {
ID string `db:"id"`
Name string `db:"name"`
DomainID string `db:"domain_id"`
CreatedBy sql.NullString `db:"created_by"`
}
func main() {
ctx := context.Background()
if limit < 0 {
log.Fatalf("invalid limit %d: limit must be >= 0", limit)
}
logger, err := mglog.New(os.Stdout, logLevel)
if err != nil {
log.Fatalf("failed to init logger: %s", err)
}
var exitCode int
defer mglog.ExitWithError(&exitCode)
sqlDB, err := pgclient.Connect(dbConfig)
if err != nil {
logger.Error("failed to connect to postgres", "error", err)
exitCode = 1
return
}
defer sqlDB.Close()
database := pgclient.NewDatabase(sqlDB, dbConfig, noop.NewTracerProvider().Tracer(cmdName))
reportsRepo := repg.NewRepository(database)
reportsWithoutRoles, err := listReportsWithoutRoles(ctx, database, limit)
if err != nil {
logger.Error("failed to list reports without roles", "error", err)
exitCode = 1
return
}
logger.Info("loaded reports without roles", "count", len(reportsWithoutRoles), "dry_run", dryRun)
if len(reportsWithoutRoles) == 0 {
return
}
availableActions, builtInRoles, err := availableActionsAndBuiltInRoles(spicedbSchemaFile)
if err != nil {
logger.Error("failed to load built-in role actions", "error", err)
exitCode = 1
return
}
adminRoleActions, err := builtInRoleActionStrings(builtInRoles, reports.BuiltInRoleAdmin)
if err != nil {
logger.Error("failed to resolve built-in admin role actions", "error", err)
exitCode = 1
return
}
authzedClient, err := newAuthzedClient(spicedbHost, spicedbPort, spicedbPreSharedKey)
if err != nil {
logger.Error("failed to connect to spicedb", "error", err)
exitCode = 1
return
}
if dryRun {
var processed, skipped int
for _, report := range reportsWithoutRoles {
memberID := strings.TrimSpace(report.CreatedBy.String)
if memberID == "" {
memberID = strings.TrimSpace(defaultMemberID)
}
if report.DomainID == "" {
skipped++
logger.Warn("skipping report without domain_id", "report_id", report.ID, "name", report.Name)
continue
}
if memberID == "" {
skipped++
logger.Warn("skipping report without created_by and no default member override", "report_id", report.ID, "name", report.Name)
continue
}
isDomainMember, err := isDomainRoleMember(ctx, database, report.DomainID, memberID)
if err != nil {
skipped++
logger.Warn(
"skipping report after failed domain membership check",
"report_id", report.ID,
"name", report.Name,
"domain_id", report.DomainID,
"member_id", memberID,
"error", err,
)
continue
}
candidatePolicies := []policies.Policy{
{
SubjectType: policies.DomainType,
Subject: report.DomainID,
Relation: policies.DomainRelation,
ObjectType: operations.EntityType,
Object: report.ID,
},
}
policiesToAdd, existingPolicies, err := filterMissingPolicies(ctx, authzedClient.PermissionsServiceClient, candidatePolicies)
if err != nil {
skipped++
logger.Warn(
"skipping report after failed spicedb policy lookup",
"report_id", report.ID,
"name", report.Name,
"domain_id", report.DomainID,
"error", err,
)
continue
}
for _, existing := range existingPolicies {
logger.Info(
"dry run: spicedb policy already exists, will not be re-added",
"report_id", report.ID,
"subject_type", existing.SubjectType,
"subject", existing.Subject,
"relation", existing.Relation,
"object_type", existing.ObjectType,
"object", existing.Object,
)
}
if !isDomainMember {
logger.Warn(
"created_by user is not a member of the domain; role will be provisioned without member",
"report_id", report.ID,
"name", report.Name,
"domain_id", report.DomainID,
"member_id", memberID,
"created_by_exists_in_domain", false,
"role_actions", adminRoleActions,
)
processed++
logger.Info(
"dry run: would provision missing built-in role without member",
"report_id", report.ID,
"name", report.Name,
"domain_id", report.DomainID,
"member_id", memberID,
"created_by_exists_in_domain", false,
"role_actions", adminRoleActions,
"role_name", reports.BuiltInRoleAdmin.String(),
"new_optional_policies", len(policiesToAdd),
"existing_optional_policies", len(existingPolicies),
)
continue
}
processed++
logger.Info(
"dry run: would provision missing built-in role",
"report_id", report.ID,
"name", report.Name,
"domain_id", report.DomainID,
"member_id", memberID,
"created_by_exists_in_domain", true,
"role_actions", adminRoleActions,
"role_name", reports.BuiltInRoleAdmin.String(),
"new_optional_policies", len(policiesToAdd),
"existing_optional_policies", len(existingPolicies),
)
}
logger.Info(
"backfill finished",
"processed", processed,
"skipped", skipped,
"failed", 0,
"dry_run", true,
)
return
}
policyService := spicedb.NewPolicyService(authzedClient, logger)
provisioner, err := roles.NewProvisionManageService(
operations.EntityType,
reportsRepo,
policyService,
uuid.New(),
availableActions,
builtInRoles,
)
if err != nil {
logger.Error("failed to create roles provisioner", "error", err)
exitCode = 1
return
}
var processed, skipped, failed int
for _, report := range reportsWithoutRoles {
memberID := strings.TrimSpace(report.CreatedBy.String)
if memberID == "" {
memberID = strings.TrimSpace(defaultMemberID)
}
if report.DomainID == "" {
skipped++
logger.Warn("skipping report without domain_id", "report_id", report.ID, "name", report.Name)
continue
}
if memberID == "" {
skipped++
logger.Warn("skipping report without created_by and no default member override", "report_id", report.ID, "name", report.Name)
continue
}
assignMembers := []roles.Member{}
isDomainMember, err := isDomainRoleMember(ctx, database, report.DomainID, memberID)
if err != nil {
failed++
logger.Error(
"failed to check domain membership before provisioning role",
"report_id", report.ID,
"name", report.Name,
"domain_id", report.DomainID,
"member_id", memberID,
"error", err,
)
continue
}
if isDomainMember {
assignMembers = []roles.Member{roles.Member(memberID)}
} else {
logger.Warn(
"created_by user is not a member of the domain; provisioning role without member",
"report_id", report.ID,
"name", report.Name,
"domain_id", report.DomainID,
"member_id", memberID,
"created_by_exists_in_domain", false,
"role_actions", adminRoleActions,
)
}
candidatePolicies := []policies.Policy{
{
SubjectType: policies.DomainType,
Subject: report.DomainID,
Relation: policies.DomainRelation,
ObjectType: operations.EntityType,
Object: report.ID,
},
}
optionalPolicies, existingPolicies, err := filterMissingPolicies(ctx, authzedClient.PermissionsServiceClient, candidatePolicies)
if err != nil {
failed++
logger.Error(
"failed to check existing spicedb policies",
"report_id", report.ID,
"name", report.Name,
"domain_id", report.DomainID,
"error", err,
)
continue
}
for _, existing := range existingPolicies {
logger.Info(
"spicedb policy already exists, skipping re-add",
"report_id", report.ID,
"subject_type", existing.SubjectType,
"subject", existing.Subject,
"relation", existing.Relation,
"object_type", existing.ObjectType,
"object", existing.Object,
)
}
newBuiltInRoleMembers := map[roles.BuiltInRoleName][]roles.Member{
reports.BuiltInRoleAdmin: assignMembers,
}
if _, err := provisioner.AddNewEntitiesRoles(
ctx,
report.DomainID,
memberID,
[]string{report.ID},
optionalPolicies,
newBuiltInRoleMembers,
); err != nil {
failed++
logger.Error(
"failed to provision missing built-in role",
"report_id", report.ID,
"name", report.Name,
"domain_id", report.DomainID,
"member_id", memberID,
"error", err,
)
continue
}
processed++
logger.Info(
"provisioned missing built-in role",
"report_id", report.ID,
"name", report.Name,
"domain_id", report.DomainID,
"member_id", memberID,
"created_by_exists_in_domain", isDomainMember,
"member_added", len(assignMembers) > 0,
"role_actions", adminRoleActions,
"role_name", reports.BuiltInRoleAdmin.String(),
"new_optional_policies", len(optionalPolicies),
"existing_optional_policies", len(existingPolicies),
)
}
logger.Info(
"backfill finished",
"processed", processed,
"skipped", skipped,
"failed", failed,
"dry_run", dryRun,
)
if failed > 0 {
exitCode = 1
}
}
func listReportsWithoutRoles(ctx context.Context, db pgclient.Database, limit int) ([]missingReport, error) {
params := map[string]any{}
query := `
SELECT rc.id, rc.name, rc.domain_id, rc.created_by
FROM report_config rc
WHERE NOT EXISTS (
SELECT 1
FROM reports_roles rr
WHERE rr.entity_id = rc.id
)
`
query += " ORDER BY rc.created_at ASC NULLS LAST, rc.id ASC"
if limit > 0 {
query += " LIMIT :limit"
params["limit"] = limit
}
rows, err := db.NamedQueryContext(ctx, query, params)
if err != nil {
return nil, errors.Wrap(fmt.Errorf("failed to query reports without roles"), err)
}
defer rows.Close()
var reps []missingReport
for rows.Next() {
var rep missingReport
if err := rows.StructScan(&rep); err != nil {
return nil, errors.Wrap(fmt.Errorf("failed to scan report without role"), err)
}
reps = append(reps, rep)
}
if err := rows.Err(); err != nil {
return nil, errors.Wrap(fmt.Errorf("failed to iterate reports without roles"), err)
}
return reps, nil
}
func isDomainRoleMember(ctx context.Context, db pgclient.Database, domainID, memberID string) (bool, error) {
const query = `
SELECT EXISTS (
SELECT 1
FROM domains_role_members drm
WHERE drm.entity_id = $1 AND drm.member_id = $2
)
`
var exists bool
if err := db.QueryRowxContext(ctx, query, domainID, memberID).Scan(&exists); err != nil {
return false, errors.Wrap(fmt.Errorf("failed to check domain role membership"), err)
}
return exists, nil
}
func newAuthzedClient(spicedbHost, spicedbPort, spicedbPreSharedKey string) (*authzed.ClientWithExperimental, error) {
return authzed.NewClientWithExperimentalAPIs(
fmt.Sprintf("%s:%s", spicedbHost, spicedbPort),
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpcutil.WithInsecureBearerToken(spicedbPreSharedKey),
)
}
func filterMissingPolicies(ctx context.Context, permClient v1.PermissionsServiceClient, ps []policies.Policy) ([]policies.Policy, []policies.Policy, error) {
missing := make([]policies.Policy, 0, len(ps))
existing := make([]policies.Policy, 0)
for _, p := range ps {
ok, err := policyExists(ctx, permClient, p)
if err != nil {
return nil, nil, err
}
if ok {
existing = append(existing, p)
continue
}
missing = append(missing, p)
}
return missing, existing, nil
}
func policyExists(ctx context.Context, permClient v1.PermissionsServiceClient, p policies.Policy) (bool, error) {
req := &v1.ReadRelationshipsRequest{
Consistency: &v1.Consistency{
Requirement: &v1.Consistency_FullyConsistent{FullyConsistent: true},
},
RelationshipFilter: &v1.RelationshipFilter{
ResourceType: p.ObjectType,
OptionalResourceId: p.Object,
OptionalRelation: p.Relation,
OptionalSubjectFilter: &v1.SubjectFilter{
SubjectType: p.SubjectType,
OptionalSubjectId: p.Subject,
},
},
OptionalLimit: 1,
}
stream, err := permClient.ReadRelationships(ctx, req)
if err != nil {
return false, errors.Wrap(fmt.Errorf("failed to read spicedb relationships"), err)
}
for {
_, err := stream.Recv()
switch {
case err == nil:
return true, nil
case errors.Contains(err, io.EOF):
return false, nil
default:
return false, errors.Wrap(fmt.Errorf("failed to receive spicedb relationship"), err)
}
}
}
func availableActionsAndBuiltInRoles(spicedbSchemaFile string) ([]roles.Action, map[roles.BuiltInRoleName][]roles.Action, error) {
availableActions, err := spicedbdecoder.GetActionsFromSchema(spicedbSchemaFile, operations.EntityType)
if err != nil {
return []roles.Action{}, map[roles.BuiltInRoleName][]roles.Action{}, err
}
builtInRoles := map[roles.BuiltInRoleName][]roles.Action{
reports.BuiltInRoleAdmin: availableActions,
}
return availableActions, builtInRoles, nil
}
func builtInRoleActionStrings(builtInRoles map[roles.BuiltInRoleName][]roles.Action, roleName roles.BuiltInRoleName) ([]string, error) {
actions, ok := builtInRoles[roleName]
if !ok {
return nil, fmt.Errorf("built-in role %q not found", roleName)
}
ret := make([]string, 0, len(actions))
for _, action := range actions {
ret = append(ret, action.String())
}
return ret, nil
}
+424
View File
@@ -0,0 +1,424 @@
# Backfill Roles — Testing Guide
This document covers end-to-end testing of the migration scripts on the `migrations` branch:
| Script | Purpose |
|--------|---------|
| `scripts/re-backfill-roles/` | Backfills missing built-in admin roles for **rules** (RE service) |
| `scripts/reports-backfill-roles/` | Backfills missing built-in admin roles for **reports** |
| `scripts/seed-test-data/` | Seeds all required test data across databases and SpiceDB |
| `domains/postgres/init.go` | Migration adding `alarm_*` and `report_*` actions to the domain admin role |
---
## Prerequisites
### Infrastructure
Start the required containers:
```bash
cd docker
docker compose up -d \
spicedb-db spicedb-migrate spicedb \
auth-db auth \
domains-db domains \
re-db re \
reports-db reports \
alarms-db alarms
```
Wait until all services are healthy. Each service applies its own Postgres migrations on startup, creating the required schemas. The `auth` service is required because it writes the SpiceDB schema on startup — without it, the seed script and backfill scripts fail with `object definition not found`.
### Connection Details (docker-compose defaults)
| Service | Host | Port | User | Password | Database |
|---------|------|------|------|----------|----------|
| Domains DB | localhost | 6003 | magistrala | magistrala | domains |
| RE DB | localhost | 6009 | magistrala | magistrala | rules_engine |
| Reports DB | localhost | 6020 | magistrala | magistrala | reports |
| Alarms DB | localhost | 6019 | magistrala | magistrala | alarms |
| SpiceDB gRPC | localhost | 50051 | — | 12345678 (pre-shared key) | — |
### Fix Hard-Coded Configs (if needed)
The backfill scripts have hard-coded database configs. Before running, verify they match your environment:
**`scripts/re-backfill-roles/main.go` (lines 4855):**
```go
dbConfig = pgclient.Config{
Host: "localhost",
Port: "6009", // docker-compose: 6009 (NOT 15432)
User: "magistrala", // docker-compose: magistrala (NOT postgres)
Pass: "magistrala", // docker-compose: magistrala (NOT supermq)
Name: "rules_engine",
SSLMode: "disable",
}
```
**`scripts/reports-backfill-roles/main.go` (lines 4956):**
```go
dbConfig = pgclient.Config{
Host: "localhost",
Port: "6020", // docker-compose: 6020 (NOT 15432)
User: "magistrala", // docker-compose: magistrala (NOT postgres)
Pass: "magistrala", // docker-compose: magistrala (NOT supermq)
Name: "reports",
SSLMode: "disable",
}
```
**Both scripts — SpiceDB schema file (line 47):**
```go
spicedbSchemaFile = "docker/spicedb/schema.zed" // NOT combined-schema.zed
```
---
## Step 1 — Seed Test Data
```bash
go run ./scripts/seed-test-data/
```
This inserts deterministic test data across all four databases and SpiceDB. It is idempotent (uses `ON CONFLICT DO NOTHING`), so re-running is safe.
### What Gets Created
**Domain:**
| ID | Name |
|----|------|
| `d0000000-0000-0000-0000-000000000001` | seed-test-domain |
**Users:**
| ID | Domain Membership |
|----|-------------------|
| `u0000000-0000-0000-0000-000000000001` (user1) | Member of domain (in `domains_role_members`) |
| `u0000000-0000-0000-0000-000000000002` (user2) | NOT a domain member |
**Rules (RE DB) — 6 rules, 4 orphans:**
| Rule ID | Name | Scenario |
|---------|------|----------|
| `r0000000-...-000000000001` | rule-1-member-creator | Orphan. `created_by=user1` (domain member). Backfill should create role **with** member. |
| `r0000000-...-000000000002` | rule-2-nonmember-creator | Orphan. `created_by=user2` (NOT member). Backfill should create role **without** member. |
| `r0000000-...-000000000003` | rule-3-spicedb-exists | Orphan. `created_by=user1`. SpiceDB parent relation **pre-seeded**. Tests `policyExists` check. |
| `r0000000-...-000000000004` | rule-4-null-creator | Orphan. `created_by=NULL`. Should be **skipped**. |
| `r0000000-...-000000000005` | rule-5-no-domain | Orphan. `domain_id=""`. Should be **skipped**. |
| `r0000000-...-000000000006` | rule-6-has-role-already | Has `rules_roles` entry. Should **NOT appear** in orphan list. |
**Reports (Reports DB) — 5 reports, 3 orphans:**
| Report ID | Name | Scenario |
|-----------|------|----------|
| `rp000000-...-000000000001` | report-1-member-creator | Orphan. `created_by=user1`. Backfill should create role **with** member. |
| `rp000000-...-000000000002` | report-2-nonmember-creator | Orphan. `created_by=user2`. Backfill should create role **without** member. |
| `rp000000-...-000000000003` | report-3-spicedb-exists | Orphan. `created_by=user1`. SpiceDB parent **pre-seeded**. Tests `policyExists`. |
| `rp000000-...-000000000004` | report-4-null-creator | Orphan. `created_by=NULL`. Should be **skipped**. |
| `rp000000-...-000000000005` | report-5-has-role-already | Has `reports_roles` entry. Should **NOT appear**. |
**Alarms (Alarms DB) — 2 alarms:**
| Alarm ID | Linked Rule |
|----------|-------------|
| `a0000000-...-000000000001` | rule-1 |
| `a0000000-...-000000000002` | rule-2 |
**SpiceDB (pre-seeded parent relations):**
```
rule:r0000000-...-000000000003#domain@domain:d0000000-...-000000000001
report:rp000000-...-000000000003#domain@domain:d0000000-...-000000000001
```
---
## Step 2 — Verify Seed Data
### Check orphan rules in RE DB
```bash
psql -h localhost -p 6009 -U magistrala -d rules_engine -c "
SELECT r.id, r.name, r.domain_id, r.created_by
FROM rules r
WHERE NOT EXISTS (SELECT 1 FROM rules_roles rr WHERE rr.entity_id = r.id)
ORDER BY r.name;"
```
**Expected:** 5 rows (rule-1 through rule-5). **rule-6 should NOT appear** (it has a role).
### Check orphan reports in Reports DB
```bash
psql -h localhost -p 6020 -U magistrala -d reports -c "
SELECT rc.id, rc.name, rc.domain_id, rc.created_by
FROM report_config rc
WHERE NOT EXISTS (SELECT 1 FROM reports_roles rr WHERE rr.entity_id = rc.id)
ORDER BY rc.name;"
```
**Expected:** 4 rows (report-1 through report-4). **report-5 should NOT appear**.
### Check domain membership
```bash
psql -h localhost -p 6009 -U magistrala -d rules_engine -c "
SELECT * FROM domains_role_members
WHERE entity_id = 'd0000000-0000-0000-0000-000000000001';"
```
**Expected:** 1 row for `user1`. `user2` should NOT be present.
### Check SpiceDB pre-seeded relationships
```bash
zed relationship read rule \
--insecure --endpoint localhost:50051 --token 12345678
```
**Expected:** At least one relationship for `rule:r0000000-...-000000000003#domain@domain:d0000000-...-000000000001`.
---
## Step 3 — Test RE Backfill (Dry Run)
Set `dryRun = true` in `scripts/re-backfill-roles/main.go`, then:
```bash
go run ./scripts/re-backfill-roles/
```
### Expected Log Output
| Rule | Expected Log |
|------|-------------|
| rule-1 | `"dry run: would provision missing built-in role"` with `created_by_exists_in_domain=true` |
| rule-2 | `"created_by user is not a member of the domain"` + `"dry run: would provision missing built-in role without member"` |
| rule-3 | `"dry run: spicedb policy already exists, will not be re-added"` + `"dry run: would provision"` with `new_optional_policies=0, existing_optional_policies=1` |
| rule-4 | `"skipping rule without created_by and no default member override"` |
| rule-5 | `"skipping rule without domain_id"` |
| rule-6 | Does NOT appear at all |
### Verify No Side Effects
```bash
# Postgres: no new roles created
psql -h localhost -p 6009 -U magistrala -d rules_engine -c "
SELECT COUNT(*) FROM rules_roles
WHERE entity_id IN (
'r0000000-0000-0000-0000-000000000001',
'r0000000-0000-0000-0000-000000000002',
'r0000000-0000-0000-0000-000000000003'
);"
```
**Expected:** `0` (dry run should not write anything).
---
## Step 4 — Test RE Backfill (Real Run)
Set `dryRun = false` in `scripts/re-backfill-roles/main.go`, then:
```bash
go run ./scripts/re-backfill-roles/
```
### Expected Log Output
| Rule | Expected Log |
|------|-------------|
| rule-1 | `"provisioned missing built-in role"` with `member_added=true` |
| rule-2 | `"provisioned missing built-in role"` with `member_added=false` |
| rule-3 | `"spicedb policy already exists, skipping re-add"` + `"provisioned missing built-in role"` with `new_optional_policies=0` |
| rule-4 | `"skipping rule without created_by"` |
| rule-5 | `"skipping rule without domain_id"` |
Final summary should show: `processed=3, skipped=2, failed=0`.
### Verify in Postgres
```bash
# New roles exist for rules 1, 2, 3
psql -h localhost -p 6009 -U magistrala -d rules_engine -c "
SELECT rr.id, rr.entity_id, rr.name, rr.created_by
FROM rules_roles rr
ORDER BY rr.entity_id;"
# Role members: rule-1 should have user1; rule-2 and rule-3 check based on domain membership
psql -h localhost -p 6009 -U magistrala -d rules_engine -c "
SELECT rrm.role_id, rrm.member_id, rrm.entity_id
FROM rules_role_members rrm
ORDER BY rrm.entity_id;"
```
### Verify in SpiceDB
```bash
zed relationship read rule \
--insecure --endpoint localhost:50051 --token 12345678
```
**Expected:** Parent relations for rule-1 and rule-2 are newly created. Rule-3 already had one (no duplicate).
---
## Step 5 — Test Idempotency (Re-Run)
Run the same backfill again without any changes:
```bash
go run ./scripts/re-backfill-roles/
```
**Expected:** `"loaded rules without roles" count=2` and `"backfill finished" processed=0, skipped=2, failed=0`. The two remaining rows are rule-4 (`created_by=NULL`) and rule-5 (no `domain_id`), which always re-appear in the orphan query and are skipped each run. The idempotency signal is `processed=0` — no roles or SpiceDB writes are duplicated.
---
## Step 6 — Test Partial State (policyExists Path)
This specifically validates the SpiceDB pre-check. Simulate a scenario where Postgres lost the role but SpiceDB still has the parent relation:
```bash
# Delete just the Postgres role for rule-1
psql -h localhost -p 6009 -U magistrala -d rules_engine -c "
DELETE FROM rules_roles
WHERE entity_id = 'r0000000-0000-0000-0000-000000000001';"
# Re-run backfill
go run ./scripts/re-backfill-roles/
```
### Expected
- `"loaded rules without roles" count=1` (only rule-1 reappears)
- `"spicedb policy already exists, skipping re-add"` — the parent relation is detected and filtered out
- `"provisioned missing built-in role"` with `new_optional_policies=0, existing_optional_policies=1`
- The role row is re-created in Postgres **without** a duplicate SpiceDB write
### Verify
```bash
# Postgres: role restored
psql -h localhost -p 6009 -U magistrala -d rules_engine -c "
SELECT * FROM rules_roles
WHERE entity_id = 'r0000000-0000-0000-0000-000000000001';"
# SpiceDB: still exactly one parent relation (no duplicate)
zed relationship read rule:r0000000-0000-0000-0000-000000000001 \
--insecure --endpoint localhost:50051 --token 12345678
```
---
## Step 7 — Test Reports Backfill
Repeat Steps 36 for the reports backfill script:
```bash
# Dry run (set dryRun = true first)
go run ./scripts/reports-backfill-roles/
# Real run (set dryRun = false)
go run ./scripts/reports-backfill-roles/
```
### Expected behavior
| Report | Expected |
|--------|----------|
| report-1 | Role provisioned **with** member (user1 is domain member) |
| report-2 | Role provisioned **without** member (user2 not in domain) |
| report-3 | `"spicedb policy already exists"` + role provisioned with `new_optional_policies=0` |
| report-4 | Skipped (NULL `created_by`) |
| report-5 | Does not appear (already has role) |
### Verify
```bash
psql -h localhost -p 6020 -U magistrala -d reports -c "
SELECT rr.id, rr.entity_id, rr.name
FROM reports_roles rr
ORDER BY rr.entity_id;"
zed relationship read report \
--insecure --endpoint localhost:50051 --token 12345678
```
---
## Step 8 — Verify Domains Migration
The `domains/postgres/init.go` change adds `alarm_*` and `report_*` actions to the domain admin role. This is applied by the domains service on startup.
```bash
psql -h localhost -p 6003 -U magistrala -d domains -c "
SELECT action FROM domains_role_actions
WHERE role_id IN (SELECT id FROM domains_roles WHERE name = 'admin')
ORDER BY action;"
```
**Expected:** The result should include all of these new actions:
```
alarm_acknowledge
alarm_assign
alarm_delete
alarm_read
alarm_resolve
alarm_update
report_add_role_users
report_create
report_delete
report_manage_role
report_read
report_remove_role_users
report_update
report_view_role_users
```
---
## Step 9 — Cleanup (Optional)
To reset and re-test from scratch:
```bash
# Remove all seeded data from RE DB
psql -h localhost -p 6009 -U magistrala -d rules_engine -c "
DELETE FROM rules WHERE id LIKE 'r0000000-%';
DELETE FROM domains WHERE id = 'd0000000-0000-0000-0000-000000000001';"
# Remove all seeded data from Reports DB
psql -h localhost -p 6020 -U magistrala -d reports -c "
DELETE FROM report_config WHERE id LIKE 'rp000000-%';
DELETE FROM domains WHERE id = 'd0000000-0000-0000-0000-000000000001';"
# Remove all seeded data from Alarms DB
psql -h localhost -p 6019 -U magistrala -d alarms -c "
DELETE FROM alarms WHERE id LIKE 'a0000000-%';
DELETE FROM domains WHERE id = 'd0000000-0000-0000-0000-000000000001';"
# Remove SpiceDB relationships
zed relationship delete rule --insecure --endpoint localhost:50051 --token 12345678
zed relationship delete report --insecure --endpoint localhost:50051 --token 12345678
```
Then re-run `go run ./scripts/seed-test-data/` to start fresh.
---
## Troubleshooting
| Problem | Solution |
|---------|----------|
| `failed to connect to postgres` | Verify containers are running: `docker compose ps`. Check ports with `docker compose port re-db 5432`. |
| `failed to read spicedb relationships` | Ensure SpiceDB is running and schema is loaded. Check: `zed schema read --insecure --endpoint localhost:50051 --token 12345678` |
| `failed to load built-in role actions` | Verify `spicedbSchemaFile` points to `docker/spicedb/schema.zed` (not `combined-schema.zed`). |
| `no such table` errors during seed | Services haven't run yet to apply migrations. Start the full service (`re`, `reports`, `alarms`) at least once. |
| Script exits with `count=0` unexpectedly | All rules/reports already have roles. Check with the orphan queries from Step 2. |
+451
View File
@@ -0,0 +1,451 @@
// Copyright (c) Abstract Machines
// SPDX-License-Identifier: Apache-2.0
// Package main seeds test data across the domains, RE, reports, and alarms
// databases so that the backfill-roles scripts can be tested end-to-end.
//
// It creates one domain, two users (one domain member, one not), several
// rules/reports with and without pre-existing roles, a couple of alarms, and
// optionally a SpiceDB parent relation for one rule to exercise the
// "policy already exists" path.
//
// All IDs are deterministic so re-running is idempotent (INSERT … ON CONFLICT DO NOTHING).
package main
import (
"context"
"database/sql"
"fmt"
"log"
"time"
v1 "github.com/authzed/authzed-go/proto/authzed/api/v1"
"github.com/authzed/authzed-go/v1"
"github.com/authzed/grpcutil"
_ "github.com/jackc/pgx/v5/stdlib"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// ---------------------------------------------------------------------------
// Configuration — edit these to match your environment.
// Default values target the standard docker-compose setup.
// ---------------------------------------------------------------------------
var (
// Domains database.
domainsDB = dbConfig{host: "localhost", port: "6003", user: "magistrala", pass: "magistrala", name: "domains"}
// RE (rules engine) database.
reDB = dbConfig{host: "localhost", port: "6009", user: "magistrala", pass: "magistrala", name: "rules_engine"}
// Reports database.
reportsDB = dbConfig{host: "localhost", port: "6020", user: "magistrala", pass: "magistrala", name: "reports"}
// Alarms database.
alarmsDB = dbConfig{host: "localhost", port: "6019", user: "magistrala", pass: "magistrala", name: "alarms"}
// SpiceDB.
spicedbHost = "localhost"
spicedbPort = "50051"
spicedbPreSharedKey = "12345678"
// Whether to write a SpiceDB parent relation for rule-3 to test the
// "policy already exists" code path.
seedSpiceDB = true
)
// ---------------------------------------------------------------------------
// Deterministic test IDs
// ---------------------------------------------------------------------------
const (
domainID = "d0000000-0000-0000-0000-000000000001"
domainName = "seed-test-domain"
// user1 will be a domain member; user2 will NOT.
user1ID = "u0000000-0000-0000-0000-000000000001"
user2ID = "u0000000-0000-0000-0000-000000000002"
// Domain role (admin).
domainRoleID = "dr000000-0000-0000-0000-000000000001"
domainRoleName = "admin"
// Rules — orphans (no rules_roles entry).
rule1ID = "r0000000-0000-0000-0000-000000000001" // created_by=user1 (domain member) → backfill should assign member
rule2ID = "r0000000-0000-0000-0000-000000000002" // created_by=user2 (NOT member) → backfill should provision role without member
rule3ID = "r0000000-0000-0000-0000-000000000003" // created_by=user1, SpiceDB parent already exists → test policyExists
rule4ID = "r0000000-0000-0000-0000-000000000004" // created_by=NULL → should be skipped
rule5ID = "r0000000-0000-0000-0000-000000000005" // empty domain_id → should be skipped
// Rule with pre-existing role — should NOT appear in orphan list.
rule6ID = "r0000000-0000-0000-0000-000000000006"
rule6RoleID = "rr000000-0000-0000-0000-000000000006"
// Reports — orphans (no reports_roles entry).
report1ID = "rp000000-0000-0000-0000-000000000001" // created_by=user1 (domain member)
report2ID = "rp000000-0000-0000-0000-000000000002" // created_by=user2 (NOT member)
report3ID = "rp000000-0000-0000-0000-000000000003" // created_by=user1, SpiceDB parent already exists
report4ID = "rp000000-0000-0000-0000-000000000004" // created_by=NULL → skipped
// Report with pre-existing role.
report5ID = "rp000000-0000-0000-0000-000000000005"
report5RoleID = "rpr00000-0000-0000-0000-000000000005"
// Alarms (live in alarms DB).
alarm1ID = "a0000000-0000-0000-0000-000000000001"
alarm2ID = "a0000000-0000-0000-0000-000000000002"
)
type dbConfig struct {
host, port, user, pass, name string
}
func (c dbConfig) dsn() string {
return fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", c.host, c.port, c.user, c.pass, c.name)
}
func main() {
ctx := context.Background()
now := time.Now().UTC()
// -----------------------------------------------------------------------
// 1. Seed Domains DB
// -----------------------------------------------------------------------
log.Println("connecting to domains DB ...")
ddb := mustConnect(domainsDB)
defer ddb.Close()
seedDomainTables(ctx, ddb, now)
log.Println("domains DB seeded")
// -----------------------------------------------------------------------
// 2. Seed RE DB (includes domain tables + rules)
// -----------------------------------------------------------------------
log.Println("connecting to RE DB ...")
rdb := mustConnect(reDB)
defer rdb.Close()
seedDomainTables(ctx, rdb, now)
seedRules(ctx, rdb, now)
log.Println("RE DB seeded")
// -----------------------------------------------------------------------
// 3. Seed Reports DB (includes domain tables + report_config)
// -----------------------------------------------------------------------
log.Println("connecting to reports DB ...")
rpdb := mustConnect(reportsDB)
defer rpdb.Close()
seedDomainTables(ctx, rpdb, now)
seedReports(ctx, rpdb, now)
log.Println("reports DB seeded")
// -----------------------------------------------------------------------
// 4. Seed Alarms DB (includes domain + RE tables + alarms)
// -----------------------------------------------------------------------
log.Println("connecting to alarms DB ...")
adb := mustConnect(alarmsDB)
defer adb.Close()
seedDomainTables(ctx, adb, now)
seedRulesMinimal(ctx, adb, now) // alarms DB has rules tables via RE migration
seedAlarms(ctx, adb, now)
log.Println("alarms DB seeded")
// -----------------------------------------------------------------------
// 5. Optionally seed SpiceDB (parent relations for rule3 and report3)
// -----------------------------------------------------------------------
if seedSpiceDB {
log.Println("connecting to SpiceDB ...")
seedSpiceDBRelationships(ctx)
log.Println("SpiceDB seeded")
}
log.Println("all seed data inserted successfully")
printSummary()
}
// ---------------------------------------------------------------------------
// Domain tables (identical across all DBs that include domain migrations)
// ---------------------------------------------------------------------------
func seedDomainTables(ctx context.Context, db *sql.DB, now time.Time) {
mustExec(ctx, db, `
INSERT INTO domains (id, name, tags, metadata, route, created_at, updated_at, created_by, status)
VALUES ($1, $2, '{}', '{}', $3, $4, $4, $5, 0)
ON CONFLICT (id) DO NOTHING`,
domainID, domainName, "seed-test-domain", now, user1ID)
// Domain admin role
mustExec(ctx, db, `
INSERT INTO domains_roles (id, name, entity_id, created_at, updated_at, created_by)
VALUES ($1, $2, $3, $4, $4, $5)
ON CONFLICT (id) DO NOTHING`,
domainRoleID, domainRoleName, domainID, now, user1ID)
// Domain role actions (a representative subset)
actions := []string{
"domain_update", "domain_read", "domain_membership",
"domain_manage_role", "domain_add_role_users", "domain_remove_role_users", "domain_view_role_users",
"rule_create", "rule_read", "rule_update", "rule_delete",
"rule_manage_role", "rule_add_role_users", "rule_remove_role_users", "rule_view_role_users",
"report_create", "report_read", "report_update", "report_delete",
"report_manage_role", "report_add_role_users", "report_remove_role_users", "report_view_role_users",
"alarm_update", "alarm_read", "alarm_delete", "alarm_assign", "alarm_acknowledge", "alarm_resolve",
}
for _, action := range actions {
mustExec(ctx, db, `
INSERT INTO domains_role_actions (role_id, action)
VALUES ($1, $2)
ON CONFLICT DO NOTHING`,
domainRoleID, action)
}
// user1 is a domain member; user2 is NOT.
mustExec(ctx, db, `
INSERT INTO domains_role_members (role_id, member_id, entity_id)
VALUES ($1, $2, $3)
ON CONFLICT DO NOTHING`,
domainRoleID, user1ID, domainID)
}
// ---------------------------------------------------------------------------
// Rules (RE DB)
// ---------------------------------------------------------------------------
func seedRules(ctx context.Context, db *sql.DB, now time.Time) {
type rule struct {
id, name, domainID string
createdBy *string
}
u1 := strPtr(user1ID)
u2 := strPtr(user2ID)
rules := []rule{
{rule1ID, "rule-1-member-creator", domainID, u1},
{rule2ID, "rule-2-nonmember-creator", domainID, u2},
{rule3ID, "rule-3-spicedb-exists", domainID, u1},
{rule4ID, "rule-4-null-creator", domainID, nil},
{rule5ID, "rule-5-no-domain", "", u1},
{rule6ID, "rule-6-has-role-already", domainID, u1},
}
for _, r := range rules {
mustExec(ctx, db, `
INSERT INTO rules (id, name, domain_id, created_by, created_at, status, logic_type)
VALUES ($1, $2, $3, $4, $5, 0, 0)
ON CONFLICT (id) DO NOTHING`,
r.id, r.name, r.domainID, r.createdBy, now)
}
// rule6 already has a role → should NOT appear in orphan list.
mustExec(ctx, db, `
INSERT INTO rules_roles (id, name, entity_id, created_at, updated_at, created_by)
VALUES ($1, 'admin', $2, $3, $3, $4)
ON CONFLICT (id) DO NOTHING`,
rule6RoleID, rule6ID, now, user1ID)
mustExec(ctx, db, `
INSERT INTO rules_role_actions (role_id, action)
VALUES ($1, 'rule_read')
ON CONFLICT DO NOTHING`,
rule6RoleID)
mustExec(ctx, db, `
INSERT INTO rules_role_members (role_id, member_id, entity_id)
VALUES ($1, $2, $3)
ON CONFLICT DO NOTHING`,
rule6RoleID, user1ID, rule6ID)
}
// seedRulesMinimal inserts the same rules into the alarms DB (which has RE
// tables) so that foreign key constraints on rule_id can be satisfied.
func seedRulesMinimal(ctx context.Context, db *sql.DB, now time.Time) {
for _, r := range []struct{ id, name string }{
{rule1ID, "rule-1-member-creator"},
{rule2ID, "rule-2-nonmember-creator"},
} {
mustExec(ctx, db, `
INSERT INTO rules (id, name, domain_id, created_by, created_at, status, logic_type)
VALUES ($1, $2, $3, $4, $5, 0, 0)
ON CONFLICT (id) DO NOTHING`,
r.id, r.name, domainID, user1ID, now)
}
}
// ---------------------------------------------------------------------------
// Reports
// ---------------------------------------------------------------------------
func seedReports(ctx context.Context, db *sql.DB, now time.Time) {
type report struct {
id, name, domainID string
createdBy *string
}
u1 := strPtr(user1ID)
u2 := strPtr(user2ID)
reports := []report{
{report1ID, "report-1-member-creator", domainID, u1},
{report2ID, "report-2-nonmember-creator", domainID, u2},
{report3ID, "report-3-spicedb-exists", domainID, u1},
{report4ID, "report-4-null-creator", domainID, nil},
{report5ID, "report-5-has-role-already", domainID, u1},
}
for _, r := range reports {
mustExec(ctx, db, `
INSERT INTO report_config (id, name, domain_id, created_by, created_at, status)
VALUES ($1, $2, $3, $4, $5, 0)
ON CONFLICT (id) DO NOTHING`,
r.id, r.name, r.domainID, r.createdBy, now)
}
// report5 already has a role.
mustExec(ctx, db, `
INSERT INTO reports_roles (id, name, entity_id, created_at, updated_at, created_by)
VALUES ($1, 'admin', $2, $3, $3, $4)
ON CONFLICT (id) DO NOTHING`,
report5RoleID, report5ID, now, user1ID)
mustExec(ctx, db, `
INSERT INTO reports_role_actions (role_id, action)
VALUES ($1, 'report_read')
ON CONFLICT DO NOTHING`,
report5RoleID)
mustExec(ctx, db, `
INSERT INTO reports_role_members (role_id, member_id, entity_id)
VALUES ($1, $2, $3)
ON CONFLICT DO NOTHING`,
report5RoleID, user1ID, report5ID)
}
// ---------------------------------------------------------------------------
// Alarms
// ---------------------------------------------------------------------------
func seedAlarms(ctx context.Context, db *sql.DB, now time.Time) {
for _, a := range []struct{ id, ruleID string }{
{alarm1ID, rule1ID},
{alarm2ID, rule2ID},
} {
mustExec(ctx, db, `
INSERT INTO alarms (id, rule_id, domain_id, channel_id, subtopic, client_id,
measurement, value, unit, threshold, cause, status, severity, created_at)
VALUES ($1, $2, $3, 'ch000000-0000-0000-0000-000000000001', 'test/topic',
'cl000000-0000-0000-0000-000000000001', 'temperature', '42.5', 'C', '40.0',
'exceeded threshold', 0, 1, $4)
ON CONFLICT (id) DO NOTHING`,
a.id, a.ruleID, domainID, now)
}
}
// ---------------------------------------------------------------------------
// SpiceDB — write a parent relation for rule3 and report3 so the
// "policy already exists" code path is exercised.
// ---------------------------------------------------------------------------
func seedSpiceDBRelationships(ctx context.Context) {
addr := fmt.Sprintf("%s:%s", spicedbHost, spicedbPort)
client, err := authzed.NewClientWithExperimentalAPIs(
addr,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpcutil.WithInsecureBearerToken(spicedbPreSharedKey),
)
if err != nil {
log.Printf("WARNING: failed to connect to SpiceDB at %s: %v (skipping SpiceDB seed)", addr, err)
return
}
// Use TOUCH so re-running is idempotent.
updates := []*v1.RelationshipUpdate{
{
Operation: v1.RelationshipUpdate_OPERATION_TOUCH,
Relationship: &v1.Relationship{
Resource: &v1.ObjectReference{ObjectType: "rule", ObjectId: rule3ID},
Relation: "domain",
Subject: &v1.SubjectReference{Object: &v1.ObjectReference{ObjectType: "domain", ObjectId: domainID}},
},
},
{
Operation: v1.RelationshipUpdate_OPERATION_TOUCH,
Relationship: &v1.Relationship{
Resource: &v1.ObjectReference{ObjectType: "report", ObjectId: report3ID},
Relation: "domain",
Subject: &v1.SubjectReference{Object: &v1.ObjectReference{ObjectType: "domain", ObjectId: domainID}},
},
},
}
_, err = client.WriteRelationships(ctx, &v1.WriteRelationshipsRequest{Updates: updates})
if err != nil {
log.Printf("WARNING: failed to write SpiceDB relationships: %v", err)
return
}
log.Printf("wrote %d SpiceDB relationships (TOUCH)", len(updates))
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
func mustConnect(cfg dbConfig) *sql.DB {
db, err := sql.Open("pgx", cfg.dsn())
if err != nil {
log.Fatalf("failed to open %s: %v", cfg.name, err)
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
if err := db.PingContext(ctx); err != nil {
cancel()
log.Fatalf("failed to ping %s at %s:%s: %v", cfg.name, cfg.host, cfg.port, err)
}
cancel()
return db
}
func mustExec(ctx context.Context, db *sql.DB, query string, args ...any) {
if _, err := db.ExecContext(ctx, query, args...); err != nil {
log.Fatalf("exec failed: %v\nquery: %s\nargs: %v", err, query, args)
}
}
func strPtr(s string) *string { return &s }
func printSummary() {
fmt.Print(`
=== SEED DATA SUMMARY ===
Domain: ` + domainID + ` ("` + domainName + `")
Users:
user1 (domain member): ` + user1ID + `
user2 (NOT a member): ` + user2ID + `
Rules (RE DB):
` + rule1ID + ` rule-1-member-creator orphan, created_by=user1 → expect role WITH member
` + rule2ID + ` rule-2-nonmember-creator orphan, created_by=user2 → expect role WITHOUT member
` + rule3ID + ` rule-3-spicedb-exists orphan, created_by=user1, SpiceDB parent pre-seeded → test policyExists
` + rule4ID + ` rule-4-null-creator orphan, created_by=NULL → expect SKIPPED
` + rule5ID + ` rule-5-no-domain orphan, domain_id="" → expect SKIPPED
` + rule6ID + ` rule-6-has-role-already HAS role entry → should NOT appear in orphan list
Reports (Reports DB):
` + report1ID + ` report-1-member-creator orphan, created_by=user1 → expect role WITH member
` + report2ID + ` report-2-nonmember-creator orphan, created_by=user2 → expect role WITHOUT member
` + report3ID + ` report-3-spicedb-exists orphan, created_by=user1, SpiceDB parent pre-seeded → test policyExists
` + report4ID + ` report-4-null-creator orphan, created_by=NULL → expect SKIPPED
` + report5ID + ` report-5-has-role-already HAS role entry → should NOT appear in orphan list
Alarms (Alarms DB):
` + alarm1ID + ` alarm-1 (rule1)
` + alarm2ID + ` alarm-2 (rule2)
SpiceDB:
rule:` + rule3ID + `#domain@domain:` + domainID + ` (pre-seeded)
report:` + report3ID + `#domain@domain:` + domainID + ` (pre-seeded)
`)
}