diff --git a/README.md b/README.md index d73bc7ca6..7da4e6003 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,54 @@ make run_latest --- +## Upgrade from v0.19.0 to v0.20.0 + +Before upgrading, back up the Domains, Rules Engine, Reports, Alarms, Auth, and SpiceDB databases. + +v0.20.0 adds new domain admin actions for alarms and reports, and it requires existing rules and reports to have their built-in admin roles backfilled. The service database migrations run when the v0.20.0 services start, then the role backfill scripts must be run once. + +For the default Docker Compose setup: + +```bash +cd docker + +docker compose up -d \ + spicedb-db spicedb-migrate spicedb \ + auth-db auth \ + domains-db domains \ + re-db re \ + reports-db reports \ + alarms-db alarms +``` + +Wait until the services are running. The `auth` service must start successfully because it loads the SpiceDB schema. + +From the repository root, run the backfills: + +```bash +go run ./scripts/re-backfill-roles/ +go run ./scripts/reports-backfill-roles/ +``` + +The scripts are idempotent. If they are interrupted, fix the issue and run them again. + +Expected successful summaries: + +```text +backfill finished processed= skipped= failed=0 +``` + +After the backfills finish, verify that the services are still running: + +```bash +cd docker +docker compose ps re reports alarms domains auth spicedb +``` + +For non-default deployments, make sure the database and SpiceDB connection settings used by the backfill scripts match your environment before running them. + +--- + ## Usage ```bash diff --git a/auth/hasher/doc.go b/auth/hasher/doc.go index e762b1335..98be99226 100644 --- a/auth/hasher/doc.go +++ b/auth/hasher/doc.go @@ -2,5 +2,5 @@ // SPDX-License-Identifier: Apache-2.0 // Package hasher contains the domain concept definitions needed to -// support Supermq users password hasher sub-service functionality. +// support Magistrala users password hasher sub-service functionality. package hasher diff --git a/docker/permission.yaml b/docker/permission.yaml index 58d7a4375..958e072a4 100644 --- a/docker/permission.yaml +++ b/docker/permission.yaml @@ -137,12 +137,16 @@ alarm: - view: alarm_read_permission - update: alarm_update_permission - delete: alarm_delete_permission + - assign: alarm_assign_permission + - acknowledge: alarm_acknowledge_permission + - resolve: alarm_resolve_permission - alarm_assign: alarm_assign_permission - alarm_acknowledge: alarm_acknowledge_permission - alarm_resolve: alarm_resolve_permission rule: operations: + - add: rule_create_permission - create: rule_create_permission - list: rule_read_permission - view: read_permission @@ -174,6 +178,7 @@ rule: report: operations: + - add: report_create_permission - create: report_create_permission - list: report_read_permission - generate: report_read_permission diff --git a/domains/postgres/init.go b/domains/postgres/init.go index 4dfc333fd..0266ba389 100644 --- a/domains/postgres/init.go +++ b/domains/postgres/init.go @@ -174,7 +174,7 @@ func Migration() (*migrate.MemoryMigrationSource, error) { { Id: "domain_7", Up: []string{ - `UPDATE domains + `UPDATE domains SET metadata = (COALESCE(metadata, '{}'::jsonb) || COALESCE(metadata->'ui', '{}'::jsonb)) - 'ui' WHERE metadata ? 'ui' AND jsonb_typeof(metadata->'ui') = 'object'`, }, @@ -182,6 +182,68 @@ func Migration() (*migrate.MemoryMigrationSource, error) { `SELECT 1`, }, }, + { + Id: "domains_roles_4", + Up: []string{ + `INSERT INTO domains_role_actions (role_id, action) + SELECT dr.id, a.action + FROM domains_roles dr + CROSS JOIN (VALUES + ('rule_create'), + ('rule_read'), + ('rule_update'), + ('rule_delete'), + ('rule_manage_role'), + ('rule_add_role_users'), + ('rule_remove_role_users'), + ('rule_view_role_users'), + ('alarm_update'), + ('alarm_read'), + ('alarm_delete'), + ('alarm_assign'), + ('alarm_acknowledge'), + ('alarm_resolve'), + ('report_create'), + ('report_read'), + ('report_update'), + ('report_delete'), + ('report_manage_role'), + ('report_add_role_users'), + ('report_remove_role_users'), + ('report_view_role_users') + ) AS a(action) + WHERE dr.name = 'admin' + ON CONFLICT DO NOTHING;`, + }, + Down: []string{ + `DELETE FROM domains_role_actions + WHERE action IN ( + 'rule_create', + 'rule_read', + 'rule_update', + 'rule_delete', + 'rule_manage_role', + 'rule_add_role_users', + 'rule_remove_role_users', + 'rule_view_role_users', + 'alarm_update', + 'alarm_read', + 'alarm_delete', + 'alarm_assign', + 'alarm_acknowledge', + 'alarm_resolve', + 'report_create', + 'report_read', + 'report_update', + 'report_delete', + 'report_manage_role', + 'report_add_role_users', + 'report_remove_role_users', + 'report_view_role_users' + ) + AND role_id IN (SELECT id FROM domains_roles WHERE name = 'admin');`, + }, + }, }, } diff --git a/scripts/re-backfill-roles/main.go b/scripts/re-backfill-roles/main.go new file mode 100644 index 000000000..565f4f447 --- /dev/null +++ b/scripts/re-backfill-roles/main.go @@ -0,0 +1,536 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package main backfills missing built-in roles for rules. +package main + +import ( + "context" + "database/sql" + "fmt" + "io" + "log" + "os" + "strings" + + mglog "github.com/absmach/magistrala/logger" + "github.com/absmach/magistrala/pkg/errors" + "github.com/absmach/magistrala/pkg/policies" + "github.com/absmach/magistrala/pkg/policies/spicedb" + pgclient "github.com/absmach/magistrala/pkg/postgres" + "github.com/absmach/magistrala/pkg/roles" + spicedbdecoder "github.com/absmach/magistrala/pkg/spicedb" + "github.com/absmach/magistrala/pkg/uuid" + "github.com/absmach/magistrala/re" + "github.com/absmach/magistrala/re/operations" + repg "github.com/absmach/magistrala/re/postgres" + v1 "github.com/authzed/authzed-go/proto/authzed/api/v1" + "github.com/authzed/authzed-go/v1" + "github.com/authzed/grpcutil" + "go.opentelemetry.io/otel/trace/noop" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +const cmdName = "re_backfill_roles" + +var ( + logLevel = "info" + dryRun = false + limit = 0 + defaultMemberID = "" + spicedbHost = "localhost" + spicedbPort = "50051" + spicedbPreSharedKey = "12345678" + spicedbSchemaFile = "docker/spicedb/schema.zed" + dbConfig = pgclient.Config{ + Host: "localhost", + Port: "6009", + User: "magistrala", + Pass: "magistrala", + Name: "rules_engine", + SSLMode: "disable", + } +) + +type missingRule struct { + ID string `db:"id"` + Name string `db:"name"` + DomainID string `db:"domain_id"` + CreatedBy sql.NullString `db:"created_by"` +} + +func main() { + ctx := context.Background() + + if limit < 0 { + log.Fatalf("invalid limit %d: limit must be >= 0", limit) + } + + logger, err := mglog.New(os.Stdout, logLevel) + if err != nil { + log.Fatalf("failed to init logger: %s", err) + } + + var exitCode int + defer mglog.ExitWithError(&exitCode) + + sqlDB, err := pgclient.Connect(dbConfig) + if err != nil { + logger.Error("failed to connect to postgres", "error", err) + exitCode = 1 + return + } + defer sqlDB.Close() + + database := pgclient.NewDatabase(sqlDB, dbConfig, noop.NewTracerProvider().Tracer(cmdName)) + rulesRepo := repg.NewRepository(database) + + rulesWithoutRoles, err := listRulesWithoutRoles(ctx, database, limit) + if err != nil { + logger.Error("failed to list rules without roles", "error", err) + exitCode = 1 + return + } + + logger.Info("loaded rules without roles", "count", len(rulesWithoutRoles), "dry_run", dryRun) + if len(rulesWithoutRoles) == 0 { + return + } + + availableActions, builtInRoles, err := availableActionsAndBuiltInRoles(spicedbSchemaFile) + if err != nil { + logger.Error("failed to load built-in role actions", "error", err) + exitCode = 1 + return + } + + adminRoleActions, err := builtInRoleActionStrings(builtInRoles, re.BuiltInRoleAdmin) + if err != nil { + logger.Error("failed to resolve built-in admin role actions", "error", err) + exitCode = 1 + return + } + + authzedClient, err := newAuthzedClient(spicedbHost, spicedbPort, spicedbPreSharedKey) + if err != nil { + logger.Error("failed to connect to spicedb", "error", err) + exitCode = 1 + return + } + + if dryRun { + var processed, skipped int + + for _, rule := range rulesWithoutRoles { + memberID := strings.TrimSpace(rule.CreatedBy.String) + if memberID == "" { + memberID = strings.TrimSpace(defaultMemberID) + } + if rule.DomainID == "" { + skipped++ + logger.Warn("skipping rule without domain_id", "rule_id", rule.ID, "name", rule.Name) + continue + } + if memberID == "" { + skipped++ + logger.Warn("skipping rule without created_by and no default member override", "rule_id", rule.ID, "name", rule.Name) + continue + } + + isDomainMember, err := isDomainRoleMember(ctx, database, rule.DomainID, memberID) + if err != nil { + skipped++ + logger.Warn( + "skipping rule after failed domain membership check", + "rule_id", rule.ID, + "name", rule.Name, + "domain_id", rule.DomainID, + "member_id", memberID, + "error", err, + ) + continue + } + + candidatePolicies := []policies.Policy{ + { + SubjectType: policies.DomainType, + Subject: rule.DomainID, + Relation: policies.DomainRelation, + ObjectType: operations.EntityType, + Object: rule.ID, + }, + } + policiesToAdd, existingPolicies, err := filterMissingPolicies(ctx, authzedClient.PermissionsServiceClient, candidatePolicies) + if err != nil { + skipped++ + logger.Warn( + "skipping rule after failed spicedb policy lookup", + "rule_id", rule.ID, + "name", rule.Name, + "domain_id", rule.DomainID, + "error", err, + ) + continue + } + for _, existing := range existingPolicies { + logger.Info( + "dry run: spicedb policy already exists, will not be re-added", + "rule_id", rule.ID, + "subject_type", existing.SubjectType, + "subject", existing.Subject, + "relation", existing.Relation, + "object_type", existing.ObjectType, + "object", existing.Object, + ) + } + + if !isDomainMember { + logger.Warn( + "created_by user is not a member of the domain; role will be provisioned without member", + "rule_id", rule.ID, + "name", rule.Name, + "domain_id", rule.DomainID, + "member_id", memberID, + "created_by_exists_in_domain", false, + "role_actions", adminRoleActions, + ) + processed++ + logger.Info( + "dry run: would provision missing built-in role without member", + "rule_id", rule.ID, + "name", rule.Name, + "domain_id", rule.DomainID, + "member_id", memberID, + "created_by_exists_in_domain", false, + "role_actions", adminRoleActions, + "role_name", re.BuiltInRoleAdmin.String(), + "new_optional_policies", len(policiesToAdd), + "existing_optional_policies", len(existingPolicies), + ) + continue + } + + processed++ + logger.Info( + "dry run: would provision missing built-in role", + "rule_id", rule.ID, + "name", rule.Name, + "domain_id", rule.DomainID, + "member_id", memberID, + "created_by_exists_in_domain", true, + "role_actions", adminRoleActions, + "role_name", re.BuiltInRoleAdmin.String(), + "new_optional_policies", len(policiesToAdd), + "existing_optional_policies", len(existingPolicies), + ) + } + + logger.Info( + "backfill finished", + "processed", processed, + "skipped", skipped, + "failed", 0, + "dry_run", true, + ) + return + } + + policyService := spicedb.NewPolicyService(authzedClient, logger) + + provisioner, err := roles.NewProvisionManageService( + operations.EntityType, + rulesRepo, + policyService, + uuid.New(), + availableActions, + builtInRoles, + ) + if err != nil { + logger.Error("failed to create roles provisioner", "error", err) + exitCode = 1 + return + } + + var processed, skipped, failed int + + for _, rule := range rulesWithoutRoles { + memberID := strings.TrimSpace(rule.CreatedBy.String) + if memberID == "" { + memberID = strings.TrimSpace(defaultMemberID) + } + if rule.DomainID == "" { + skipped++ + logger.Warn("skipping rule without domain_id", "rule_id", rule.ID, "name", rule.Name) + continue + } + if memberID == "" { + skipped++ + logger.Warn("skipping rule without created_by and no default member override", "rule_id", rule.ID, "name", rule.Name) + continue + } + + assignMembers := []roles.Member{} + isDomainMember, err := isDomainRoleMember(ctx, database, rule.DomainID, memberID) + if err != nil { + failed++ + logger.Error( + "failed to check domain membership before provisioning role", + "rule_id", rule.ID, + "name", rule.Name, + "domain_id", rule.DomainID, + "member_id", memberID, + "error", err, + ) + continue + } + if isDomainMember { + assignMembers = []roles.Member{roles.Member(memberID)} + } else { + logger.Warn( + "created_by user is not a member of the domain; provisioning role without member", + "rule_id", rule.ID, + "name", rule.Name, + "domain_id", rule.DomainID, + "member_id", memberID, + "created_by_exists_in_domain", false, + "role_actions", adminRoleActions, + ) + } + + candidatePolicies := []policies.Policy{ + { + SubjectType: policies.DomainType, + Subject: rule.DomainID, + Relation: policies.DomainRelation, + ObjectType: operations.EntityType, + Object: rule.ID, + }, + } + optionalPolicies, existingPolicies, err := filterMissingPolicies(ctx, authzedClient.PermissionsServiceClient, candidatePolicies) + if err != nil { + failed++ + logger.Error( + "failed to check existing spicedb policies", + "rule_id", rule.ID, + "name", rule.Name, + "domain_id", rule.DomainID, + "error", err, + ) + continue + } + for _, existing := range existingPolicies { + logger.Info( + "spicedb policy already exists, skipping re-add", + "rule_id", rule.ID, + "subject_type", existing.SubjectType, + "subject", existing.Subject, + "relation", existing.Relation, + "object_type", existing.ObjectType, + "object", existing.Object, + ) + } + + newBuiltInRoleMembers := map[roles.BuiltInRoleName][]roles.Member{ + re.BuiltInRoleAdmin: assignMembers, + } + + if _, err := provisioner.AddNewEntitiesRoles( + ctx, + rule.DomainID, + memberID, + []string{rule.ID}, + optionalPolicies, + newBuiltInRoleMembers, + ); err != nil { + failed++ + logger.Error( + "failed to provision missing built-in role", + "rule_id", rule.ID, + "name", rule.Name, + "domain_id", rule.DomainID, + "member_id", memberID, + "error", err, + ) + continue + } + + processed++ + logger.Info( + "provisioned missing built-in role", + "rule_id", rule.ID, + "name", rule.Name, + "domain_id", rule.DomainID, + "member_id", memberID, + "created_by_exists_in_domain", isDomainMember, + "member_added", len(assignMembers) > 0, + "role_actions", adminRoleActions, + "role_name", re.BuiltInRoleAdmin.String(), + "new_optional_policies", len(optionalPolicies), + "existing_optional_policies", len(existingPolicies), + ) + } + + logger.Info( + "backfill finished", + "processed", processed, + "skipped", skipped, + "failed", failed, + "dry_run", dryRun, + ) + + if failed > 0 { + exitCode = 1 + } +} + +func listRulesWithoutRoles(ctx context.Context, db pgclient.Database, limit int) ([]missingRule, error) { + params := map[string]any{} + + query := ` + SELECT r.id, r.name, r.domain_id, r.created_by + FROM rules r + WHERE NOT EXISTS ( + SELECT 1 + FROM rules_roles rr + WHERE rr.entity_id = r.id + ) + ` + + query += " ORDER BY r.created_at ASC NULLS LAST, r.id ASC" + + if limit > 0 { + query += " LIMIT :limit" + params["limit"] = limit + } + + rows, err := db.NamedQueryContext(ctx, query, params) + if err != nil { + return nil, errors.Wrap(fmt.Errorf("failed to query rules without roles"), err) + } + defer rows.Close() + + var rules []missingRule + for rows.Next() { + var rule missingRule + if err := rows.StructScan(&rule); err != nil { + return nil, errors.Wrap(fmt.Errorf("failed to scan rule without role"), err) + } + rules = append(rules, rule) + } + if err := rows.Err(); err != nil { + return nil, errors.Wrap(fmt.Errorf("failed to iterate rules without roles"), err) + } + + return rules, nil +} + +func isDomainRoleMember(ctx context.Context, db pgclient.Database, domainID, memberID string) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM domains_role_members drm + WHERE drm.entity_id = $1 AND drm.member_id = $2 + ) + ` + + var exists bool + if err := db.QueryRowxContext(ctx, query, domainID, memberID).Scan(&exists); err != nil { + return false, errors.Wrap(fmt.Errorf("failed to check domain role membership"), err) + } + + return exists, nil +} + +func newAuthzedClient(spicedbHost, spicedbPort, spicedbPreSharedKey string) (*authzed.ClientWithExperimental, error) { + return authzed.NewClientWithExperimentalAPIs( + fmt.Sprintf("%s:%s", spicedbHost, spicedbPort), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpcutil.WithInsecureBearerToken(spicedbPreSharedKey), + ) +} + +// filterMissingPolicies splits the given policies into those that do not yet +// exist in SpiceDB (returned first) and those that already exist (returned +// second). Any error from SpiceDB short-circuits with an empty result. +func filterMissingPolicies(ctx context.Context, permClient v1.PermissionsServiceClient, ps []policies.Policy) ([]policies.Policy, []policies.Policy, error) { + missing := make([]policies.Policy, 0, len(ps)) + existing := make([]policies.Policy, 0) + for _, p := range ps { + ok, err := policyExists(ctx, permClient, p) + if err != nil { + return nil, nil, err + } + if ok { + existing = append(existing, p) + continue + } + missing = append(missing, p) + } + return missing, existing, nil +} + +// policyExists returns true when SpiceDB already contains a relationship +// matching the supplied policy on (object_type, object, relation, subject_type, +// subject). The lookup is fully consistent and capped at one row. +func policyExists(ctx context.Context, permClient v1.PermissionsServiceClient, p policies.Policy) (bool, error) { + req := &v1.ReadRelationshipsRequest{ + Consistency: &v1.Consistency{ + Requirement: &v1.Consistency_FullyConsistent{FullyConsistent: true}, + }, + RelationshipFilter: &v1.RelationshipFilter{ + ResourceType: p.ObjectType, + OptionalResourceId: p.Object, + OptionalRelation: p.Relation, + OptionalSubjectFilter: &v1.SubjectFilter{ + SubjectType: p.SubjectType, + OptionalSubjectId: p.Subject, + }, + }, + OptionalLimit: 1, + } + + stream, err := permClient.ReadRelationships(ctx, req) + if err != nil { + return false, errors.Wrap(fmt.Errorf("failed to read spicedb relationships"), err) + } + + for { + _, err := stream.Recv() + switch { + case err == nil: + return true, nil + case errors.Contains(err, io.EOF): + return false, nil + default: + return false, errors.Wrap(fmt.Errorf("failed to receive spicedb relationship"), err) + } + } +} + +func availableActionsAndBuiltInRoles(spicedbSchemaFile string) ([]roles.Action, map[roles.BuiltInRoleName][]roles.Action, error) { + availableActions, err := spicedbdecoder.GetActionsFromSchema(spicedbSchemaFile, operations.EntityType) + if err != nil { + return []roles.Action{}, map[roles.BuiltInRoleName][]roles.Action{}, err + } + + builtInRoles := map[roles.BuiltInRoleName][]roles.Action{ + re.BuiltInRoleAdmin: availableActions, + } + + return availableActions, builtInRoles, nil +} + +func builtInRoleActionStrings(builtInRoles map[roles.BuiltInRoleName][]roles.Action, roleName roles.BuiltInRoleName) ([]string, error) { + actions, ok := builtInRoles[roleName] + if !ok { + return nil, fmt.Errorf("built-in role %q not found", roleName) + } + + ret := make([]string, 0, len(actions)) + for _, action := range actions { + ret = append(ret, action.String()) + } + + return ret, nil +} diff --git a/scripts/reports-backfill-roles/main.go b/scripts/reports-backfill-roles/main.go new file mode 100644 index 000000000..d4e977495 --- /dev/null +++ b/scripts/reports-backfill-roles/main.go @@ -0,0 +1,532 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package main backfills missing built-in roles for reports. +package main + +import ( + "context" + "database/sql" + "fmt" + "io" + "log" + "os" + "strings" + + mglog "github.com/absmach/magistrala/logger" + "github.com/absmach/magistrala/pkg/errors" + "github.com/absmach/magistrala/pkg/policies" + "github.com/absmach/magistrala/pkg/policies/spicedb" + pgclient "github.com/absmach/magistrala/pkg/postgres" + "github.com/absmach/magistrala/pkg/roles" + spicedbdecoder "github.com/absmach/magistrala/pkg/spicedb" + "github.com/absmach/magistrala/pkg/uuid" + "github.com/absmach/magistrala/reports" + "github.com/absmach/magistrala/reports/operations" + repg "github.com/absmach/magistrala/reports/postgres" + v1 "github.com/authzed/authzed-go/proto/authzed/api/v1" + "github.com/authzed/authzed-go/v1" + "github.com/authzed/grpcutil" + "go.opentelemetry.io/otel/trace/noop" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +const ( + cmdName = "reports_backfill_roles" +) + +var ( + logLevel = "info" + dryRun = false + limit = 0 + defaultMemberID = "" + spicedbHost = "localhost" + spicedbPort = "50051" + spicedbPreSharedKey = "12345678" + spicedbSchemaFile = "docker/spicedb/schema.zed" + dbConfig = pgclient.Config{ + Host: "localhost", + Port: "6020", + User: "magistrala", + Pass: "magistrala", + Name: "reports", + SSLMode: "disable", + } +) + +type missingReport struct { + ID string `db:"id"` + Name string `db:"name"` + DomainID string `db:"domain_id"` + CreatedBy sql.NullString `db:"created_by"` +} + +func main() { + ctx := context.Background() + + if limit < 0 { + log.Fatalf("invalid limit %d: limit must be >= 0", limit) + } + + logger, err := mglog.New(os.Stdout, logLevel) + if err != nil { + log.Fatalf("failed to init logger: %s", err) + } + + var exitCode int + defer mglog.ExitWithError(&exitCode) + + sqlDB, err := pgclient.Connect(dbConfig) + if err != nil { + logger.Error("failed to connect to postgres", "error", err) + exitCode = 1 + return + } + defer sqlDB.Close() + + database := pgclient.NewDatabase(sqlDB, dbConfig, noop.NewTracerProvider().Tracer(cmdName)) + reportsRepo := repg.NewRepository(database) + + reportsWithoutRoles, err := listReportsWithoutRoles(ctx, database, limit) + if err != nil { + logger.Error("failed to list reports without roles", "error", err) + exitCode = 1 + return + } + + logger.Info("loaded reports without roles", "count", len(reportsWithoutRoles), "dry_run", dryRun) + if len(reportsWithoutRoles) == 0 { + return + } + + availableActions, builtInRoles, err := availableActionsAndBuiltInRoles(spicedbSchemaFile) + if err != nil { + logger.Error("failed to load built-in role actions", "error", err) + exitCode = 1 + return + } + + adminRoleActions, err := builtInRoleActionStrings(builtInRoles, reports.BuiltInRoleAdmin) + if err != nil { + logger.Error("failed to resolve built-in admin role actions", "error", err) + exitCode = 1 + return + } + + authzedClient, err := newAuthzedClient(spicedbHost, spicedbPort, spicedbPreSharedKey) + if err != nil { + logger.Error("failed to connect to spicedb", "error", err) + exitCode = 1 + return + } + + if dryRun { + var processed, skipped int + + for _, report := range reportsWithoutRoles { + memberID := strings.TrimSpace(report.CreatedBy.String) + if memberID == "" { + memberID = strings.TrimSpace(defaultMemberID) + } + if report.DomainID == "" { + skipped++ + logger.Warn("skipping report without domain_id", "report_id", report.ID, "name", report.Name) + continue + } + if memberID == "" { + skipped++ + logger.Warn("skipping report without created_by and no default member override", "report_id", report.ID, "name", report.Name) + continue + } + + isDomainMember, err := isDomainRoleMember(ctx, database, report.DomainID, memberID) + if err != nil { + skipped++ + logger.Warn( + "skipping report after failed domain membership check", + "report_id", report.ID, + "name", report.Name, + "domain_id", report.DomainID, + "member_id", memberID, + "error", err, + ) + continue + } + + candidatePolicies := []policies.Policy{ + { + SubjectType: policies.DomainType, + Subject: report.DomainID, + Relation: policies.DomainRelation, + ObjectType: operations.EntityType, + Object: report.ID, + }, + } + policiesToAdd, existingPolicies, err := filterMissingPolicies(ctx, authzedClient.PermissionsServiceClient, candidatePolicies) + if err != nil { + skipped++ + logger.Warn( + "skipping report after failed spicedb policy lookup", + "report_id", report.ID, + "name", report.Name, + "domain_id", report.DomainID, + "error", err, + ) + continue + } + for _, existing := range existingPolicies { + logger.Info( + "dry run: spicedb policy already exists, will not be re-added", + "report_id", report.ID, + "subject_type", existing.SubjectType, + "subject", existing.Subject, + "relation", existing.Relation, + "object_type", existing.ObjectType, + "object", existing.Object, + ) + } + + if !isDomainMember { + logger.Warn( + "created_by user is not a member of the domain; role will be provisioned without member", + "report_id", report.ID, + "name", report.Name, + "domain_id", report.DomainID, + "member_id", memberID, + "created_by_exists_in_domain", false, + "role_actions", adminRoleActions, + ) + processed++ + logger.Info( + "dry run: would provision missing built-in role without member", + "report_id", report.ID, + "name", report.Name, + "domain_id", report.DomainID, + "member_id", memberID, + "created_by_exists_in_domain", false, + "role_actions", adminRoleActions, + "role_name", reports.BuiltInRoleAdmin.String(), + "new_optional_policies", len(policiesToAdd), + "existing_optional_policies", len(existingPolicies), + ) + continue + } + + processed++ + logger.Info( + "dry run: would provision missing built-in role", + "report_id", report.ID, + "name", report.Name, + "domain_id", report.DomainID, + "member_id", memberID, + "created_by_exists_in_domain", true, + "role_actions", adminRoleActions, + "role_name", reports.BuiltInRoleAdmin.String(), + "new_optional_policies", len(policiesToAdd), + "existing_optional_policies", len(existingPolicies), + ) + } + + logger.Info( + "backfill finished", + "processed", processed, + "skipped", skipped, + "failed", 0, + "dry_run", true, + ) + return + } + + policyService := spicedb.NewPolicyService(authzedClient, logger) + + provisioner, err := roles.NewProvisionManageService( + operations.EntityType, + reportsRepo, + policyService, + uuid.New(), + availableActions, + builtInRoles, + ) + if err != nil { + logger.Error("failed to create roles provisioner", "error", err) + exitCode = 1 + return + } + + var processed, skipped, failed int + + for _, report := range reportsWithoutRoles { + memberID := strings.TrimSpace(report.CreatedBy.String) + if memberID == "" { + memberID = strings.TrimSpace(defaultMemberID) + } + if report.DomainID == "" { + skipped++ + logger.Warn("skipping report without domain_id", "report_id", report.ID, "name", report.Name) + continue + } + if memberID == "" { + skipped++ + logger.Warn("skipping report without created_by and no default member override", "report_id", report.ID, "name", report.Name) + continue + } + + assignMembers := []roles.Member{} + isDomainMember, err := isDomainRoleMember(ctx, database, report.DomainID, memberID) + if err != nil { + failed++ + logger.Error( + "failed to check domain membership before provisioning role", + "report_id", report.ID, + "name", report.Name, + "domain_id", report.DomainID, + "member_id", memberID, + "error", err, + ) + continue + } + if isDomainMember { + assignMembers = []roles.Member{roles.Member(memberID)} + } else { + logger.Warn( + "created_by user is not a member of the domain; provisioning role without member", + "report_id", report.ID, + "name", report.Name, + "domain_id", report.DomainID, + "member_id", memberID, + "created_by_exists_in_domain", false, + "role_actions", adminRoleActions, + ) + } + + candidatePolicies := []policies.Policy{ + { + SubjectType: policies.DomainType, + Subject: report.DomainID, + Relation: policies.DomainRelation, + ObjectType: operations.EntityType, + Object: report.ID, + }, + } + optionalPolicies, existingPolicies, err := filterMissingPolicies(ctx, authzedClient.PermissionsServiceClient, candidatePolicies) + if err != nil { + failed++ + logger.Error( + "failed to check existing spicedb policies", + "report_id", report.ID, + "name", report.Name, + "domain_id", report.DomainID, + "error", err, + ) + continue + } + for _, existing := range existingPolicies { + logger.Info( + "spicedb policy already exists, skipping re-add", + "report_id", report.ID, + "subject_type", existing.SubjectType, + "subject", existing.Subject, + "relation", existing.Relation, + "object_type", existing.ObjectType, + "object", existing.Object, + ) + } + + newBuiltInRoleMembers := map[roles.BuiltInRoleName][]roles.Member{ + reports.BuiltInRoleAdmin: assignMembers, + } + + if _, err := provisioner.AddNewEntitiesRoles( + ctx, + report.DomainID, + memberID, + []string{report.ID}, + optionalPolicies, + newBuiltInRoleMembers, + ); err != nil { + failed++ + logger.Error( + "failed to provision missing built-in role", + "report_id", report.ID, + "name", report.Name, + "domain_id", report.DomainID, + "member_id", memberID, + "error", err, + ) + continue + } + + processed++ + logger.Info( + "provisioned missing built-in role", + "report_id", report.ID, + "name", report.Name, + "domain_id", report.DomainID, + "member_id", memberID, + "created_by_exists_in_domain", isDomainMember, + "member_added", len(assignMembers) > 0, + "role_actions", adminRoleActions, + "role_name", reports.BuiltInRoleAdmin.String(), + "new_optional_policies", len(optionalPolicies), + "existing_optional_policies", len(existingPolicies), + ) + } + + logger.Info( + "backfill finished", + "processed", processed, + "skipped", skipped, + "failed", failed, + "dry_run", dryRun, + ) + + if failed > 0 { + exitCode = 1 + } +} + +func listReportsWithoutRoles(ctx context.Context, db pgclient.Database, limit int) ([]missingReport, error) { + params := map[string]any{} + + query := ` + SELECT rc.id, rc.name, rc.domain_id, rc.created_by + FROM report_config rc + WHERE NOT EXISTS ( + SELECT 1 + FROM reports_roles rr + WHERE rr.entity_id = rc.id + ) + ` + + query += " ORDER BY rc.created_at ASC NULLS LAST, rc.id ASC" + + if limit > 0 { + query += " LIMIT :limit" + params["limit"] = limit + } + + rows, err := db.NamedQueryContext(ctx, query, params) + if err != nil { + return nil, errors.Wrap(fmt.Errorf("failed to query reports without roles"), err) + } + defer rows.Close() + + var reps []missingReport + for rows.Next() { + var rep missingReport + if err := rows.StructScan(&rep); err != nil { + return nil, errors.Wrap(fmt.Errorf("failed to scan report without role"), err) + } + reps = append(reps, rep) + } + if err := rows.Err(); err != nil { + return nil, errors.Wrap(fmt.Errorf("failed to iterate reports without roles"), err) + } + + return reps, nil +} + +func isDomainRoleMember(ctx context.Context, db pgclient.Database, domainID, memberID string) (bool, error) { + const query = ` + SELECT EXISTS ( + SELECT 1 + FROM domains_role_members drm + WHERE drm.entity_id = $1 AND drm.member_id = $2 + ) + ` + + var exists bool + if err := db.QueryRowxContext(ctx, query, domainID, memberID).Scan(&exists); err != nil { + return false, errors.Wrap(fmt.Errorf("failed to check domain role membership"), err) + } + + return exists, nil +} + +func newAuthzedClient(spicedbHost, spicedbPort, spicedbPreSharedKey string) (*authzed.ClientWithExperimental, error) { + return authzed.NewClientWithExperimentalAPIs( + fmt.Sprintf("%s:%s", spicedbHost, spicedbPort), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpcutil.WithInsecureBearerToken(spicedbPreSharedKey), + ) +} + +func filterMissingPolicies(ctx context.Context, permClient v1.PermissionsServiceClient, ps []policies.Policy) ([]policies.Policy, []policies.Policy, error) { + missing := make([]policies.Policy, 0, len(ps)) + existing := make([]policies.Policy, 0) + for _, p := range ps { + ok, err := policyExists(ctx, permClient, p) + if err != nil { + return nil, nil, err + } + if ok { + existing = append(existing, p) + continue + } + missing = append(missing, p) + } + return missing, existing, nil +} + +func policyExists(ctx context.Context, permClient v1.PermissionsServiceClient, p policies.Policy) (bool, error) { + req := &v1.ReadRelationshipsRequest{ + Consistency: &v1.Consistency{ + Requirement: &v1.Consistency_FullyConsistent{FullyConsistent: true}, + }, + RelationshipFilter: &v1.RelationshipFilter{ + ResourceType: p.ObjectType, + OptionalResourceId: p.Object, + OptionalRelation: p.Relation, + OptionalSubjectFilter: &v1.SubjectFilter{ + SubjectType: p.SubjectType, + OptionalSubjectId: p.Subject, + }, + }, + OptionalLimit: 1, + } + + stream, err := permClient.ReadRelationships(ctx, req) + if err != nil { + return false, errors.Wrap(fmt.Errorf("failed to read spicedb relationships"), err) + } + + for { + _, err := stream.Recv() + switch { + case err == nil: + return true, nil + case errors.Contains(err, io.EOF): + return false, nil + default: + return false, errors.Wrap(fmt.Errorf("failed to receive spicedb relationship"), err) + } + } +} + +func availableActionsAndBuiltInRoles(spicedbSchemaFile string) ([]roles.Action, map[roles.BuiltInRoleName][]roles.Action, error) { + availableActions, err := spicedbdecoder.GetActionsFromSchema(spicedbSchemaFile, operations.EntityType) + if err != nil { + return []roles.Action{}, map[roles.BuiltInRoleName][]roles.Action{}, err + } + + builtInRoles := map[roles.BuiltInRoleName][]roles.Action{ + reports.BuiltInRoleAdmin: availableActions, + } + + return availableActions, builtInRoles, nil +} + +func builtInRoleActionStrings(builtInRoles map[roles.BuiltInRoleName][]roles.Action, roleName roles.BuiltInRoleName) ([]string, error) { + actions, ok := builtInRoles[roleName] + if !ok { + return nil, fmt.Errorf("built-in role %q not found", roleName) + } + + ret := make([]string, 0, len(actions)) + for _, action := range actions { + ret = append(ret, action.String()) + } + + return ret, nil +} diff --git a/scripts/seed-test-data/TESTING.md b/scripts/seed-test-data/TESTING.md new file mode 100644 index 000000000..bc610f090 --- /dev/null +++ b/scripts/seed-test-data/TESTING.md @@ -0,0 +1,424 @@ +# Backfill Roles — Testing Guide + +This document covers end-to-end testing of the migration scripts on the `migrations` branch: + +| Script | Purpose | +|--------|---------| +| `scripts/re-backfill-roles/` | Backfills missing built-in admin roles for **rules** (RE service) | +| `scripts/reports-backfill-roles/` | Backfills missing built-in admin roles for **reports** | +| `scripts/seed-test-data/` | Seeds all required test data across databases and SpiceDB | +| `domains/postgres/init.go` | Migration adding `alarm_*` and `report_*` actions to the domain admin role | + +--- + +## Prerequisites + +### Infrastructure + +Start the required containers: + +```bash +cd docker +docker compose up -d \ + spicedb-db spicedb-migrate spicedb \ + auth-db auth \ + domains-db domains \ + re-db re \ + reports-db reports \ + alarms-db alarms +``` + +Wait until all services are healthy. Each service applies its own Postgres migrations on startup, creating the required schemas. The `auth` service is required because it writes the SpiceDB schema on startup — without it, the seed script and backfill scripts fail with `object definition not found`. + +### Connection Details (docker-compose defaults) + +| Service | Host | Port | User | Password | Database | +|---------|------|------|------|----------|----------| +| Domains DB | localhost | 6003 | magistrala | magistrala | domains | +| RE DB | localhost | 6009 | magistrala | magistrala | rules_engine | +| Reports DB | localhost | 6020 | magistrala | magistrala | reports | +| Alarms DB | localhost | 6019 | magistrala | magistrala | alarms | +| SpiceDB gRPC | localhost | 50051 | — | 12345678 (pre-shared key) | — | + +### Fix Hard-Coded Configs (if needed) + +The backfill scripts have hard-coded database configs. Before running, verify they match your environment: + +**`scripts/re-backfill-roles/main.go` (lines 48–55):** + +```go +dbConfig = pgclient.Config{ + Host: "localhost", + Port: "6009", // docker-compose: 6009 (NOT 15432) + User: "magistrala", // docker-compose: magistrala (NOT postgres) + Pass: "magistrala", // docker-compose: magistrala (NOT supermq) + Name: "rules_engine", + SSLMode: "disable", +} +``` + +**`scripts/reports-backfill-roles/main.go` (lines 49–56):** + +```go +dbConfig = pgclient.Config{ + Host: "localhost", + Port: "6020", // docker-compose: 6020 (NOT 15432) + User: "magistrala", // docker-compose: magistrala (NOT postgres) + Pass: "magistrala", // docker-compose: magistrala (NOT supermq) + Name: "reports", + SSLMode: "disable", +} +``` + +**Both scripts — SpiceDB schema file (line 47):** + +```go +spicedbSchemaFile = "docker/spicedb/schema.zed" // NOT combined-schema.zed +``` + +--- + +## Step 1 — Seed Test Data + +```bash +go run ./scripts/seed-test-data/ +``` + +This inserts deterministic test data across all four databases and SpiceDB. It is idempotent (uses `ON CONFLICT DO NOTHING`), so re-running is safe. + +### What Gets Created + +**Domain:** + +| ID | Name | +|----|------| +| `d0000000-0000-0000-0000-000000000001` | seed-test-domain | + +**Users:** + +| ID | Domain Membership | +|----|-------------------| +| `u0000000-0000-0000-0000-000000000001` (user1) | Member of domain (in `domains_role_members`) | +| `u0000000-0000-0000-0000-000000000002` (user2) | NOT a domain member | + +**Rules (RE DB) — 6 rules, 4 orphans:** + +| Rule ID | Name | Scenario | +|---------|------|----------| +| `r0000000-...-000000000001` | rule-1-member-creator | Orphan. `created_by=user1` (domain member). Backfill should create role **with** member. | +| `r0000000-...-000000000002` | rule-2-nonmember-creator | Orphan. `created_by=user2` (NOT member). Backfill should create role **without** member. | +| `r0000000-...-000000000003` | rule-3-spicedb-exists | Orphan. `created_by=user1`. SpiceDB parent relation **pre-seeded**. Tests `policyExists` check. | +| `r0000000-...-000000000004` | rule-4-null-creator | Orphan. `created_by=NULL`. Should be **skipped**. | +| `r0000000-...-000000000005` | rule-5-no-domain | Orphan. `domain_id=""`. Should be **skipped**. | +| `r0000000-...-000000000006` | rule-6-has-role-already | Has `rules_roles` entry. Should **NOT appear** in orphan list. | + +**Reports (Reports DB) — 5 reports, 3 orphans:** + +| Report ID | Name | Scenario | +|-----------|------|----------| +| `rp000000-...-000000000001` | report-1-member-creator | Orphan. `created_by=user1`. Backfill should create role **with** member. | +| `rp000000-...-000000000002` | report-2-nonmember-creator | Orphan. `created_by=user2`. Backfill should create role **without** member. | +| `rp000000-...-000000000003` | report-3-spicedb-exists | Orphan. `created_by=user1`. SpiceDB parent **pre-seeded**. Tests `policyExists`. | +| `rp000000-...-000000000004` | report-4-null-creator | Orphan. `created_by=NULL`. Should be **skipped**. | +| `rp000000-...-000000000005` | report-5-has-role-already | Has `reports_roles` entry. Should **NOT appear**. | + +**Alarms (Alarms DB) — 2 alarms:** + +| Alarm ID | Linked Rule | +|----------|-------------| +| `a0000000-...-000000000001` | rule-1 | +| `a0000000-...-000000000002` | rule-2 | + +**SpiceDB (pre-seeded parent relations):** + +``` +rule:r0000000-...-000000000003#domain@domain:d0000000-...-000000000001 +report:rp000000-...-000000000003#domain@domain:d0000000-...-000000000001 +``` + +--- + +## Step 2 — Verify Seed Data + +### Check orphan rules in RE DB + +```bash +psql -h localhost -p 6009 -U magistrala -d rules_engine -c " + SELECT r.id, r.name, r.domain_id, r.created_by + FROM rules r + WHERE NOT EXISTS (SELECT 1 FROM rules_roles rr WHERE rr.entity_id = r.id) + ORDER BY r.name;" +``` + +**Expected:** 5 rows (rule-1 through rule-5). **rule-6 should NOT appear** (it has a role). + +### Check orphan reports in Reports DB + +```bash +psql -h localhost -p 6020 -U magistrala -d reports -c " + SELECT rc.id, rc.name, rc.domain_id, rc.created_by + FROM report_config rc + WHERE NOT EXISTS (SELECT 1 FROM reports_roles rr WHERE rr.entity_id = rc.id) + ORDER BY rc.name;" +``` + +**Expected:** 4 rows (report-1 through report-4). **report-5 should NOT appear**. + +### Check domain membership + +```bash +psql -h localhost -p 6009 -U magistrala -d rules_engine -c " + SELECT * FROM domains_role_members + WHERE entity_id = 'd0000000-0000-0000-0000-000000000001';" +``` + +**Expected:** 1 row for `user1`. `user2` should NOT be present. + +### Check SpiceDB pre-seeded relationships + +```bash +zed relationship read rule \ + --insecure --endpoint localhost:50051 --token 12345678 +``` + +**Expected:** At least one relationship for `rule:r0000000-...-000000000003#domain@domain:d0000000-...-000000000001`. + +--- + +## Step 3 — Test RE Backfill (Dry Run) + +Set `dryRun = true` in `scripts/re-backfill-roles/main.go`, then: + +```bash +go run ./scripts/re-backfill-roles/ +``` + +### Expected Log Output + +| Rule | Expected Log | +|------|-------------| +| rule-1 | `"dry run: would provision missing built-in role"` with `created_by_exists_in_domain=true` | +| rule-2 | `"created_by user is not a member of the domain"` + `"dry run: would provision missing built-in role without member"` | +| rule-3 | `"dry run: spicedb policy already exists, will not be re-added"` + `"dry run: would provision"` with `new_optional_policies=0, existing_optional_policies=1` | +| rule-4 | `"skipping rule without created_by and no default member override"` | +| rule-5 | `"skipping rule without domain_id"` | +| rule-6 | Does NOT appear at all | + +### Verify No Side Effects + +```bash +# Postgres: no new roles created +psql -h localhost -p 6009 -U magistrala -d rules_engine -c " + SELECT COUNT(*) FROM rules_roles + WHERE entity_id IN ( + 'r0000000-0000-0000-0000-000000000001', + 'r0000000-0000-0000-0000-000000000002', + 'r0000000-0000-0000-0000-000000000003' + );" +``` + +**Expected:** `0` (dry run should not write anything). + +--- + +## Step 4 — Test RE Backfill (Real Run) + +Set `dryRun = false` in `scripts/re-backfill-roles/main.go`, then: + +```bash +go run ./scripts/re-backfill-roles/ +``` + +### Expected Log Output + +| Rule | Expected Log | +|------|-------------| +| rule-1 | `"provisioned missing built-in role"` with `member_added=true` | +| rule-2 | `"provisioned missing built-in role"` with `member_added=false` | +| rule-3 | `"spicedb policy already exists, skipping re-add"` + `"provisioned missing built-in role"` with `new_optional_policies=0` | +| rule-4 | `"skipping rule without created_by"` | +| rule-5 | `"skipping rule without domain_id"` | + +Final summary should show: `processed=3, skipped=2, failed=0`. + +### Verify in Postgres + +```bash +# New roles exist for rules 1, 2, 3 +psql -h localhost -p 6009 -U magistrala -d rules_engine -c " + SELECT rr.id, rr.entity_id, rr.name, rr.created_by + FROM rules_roles rr + ORDER BY rr.entity_id;" + +# Role members: rule-1 should have user1; rule-2 and rule-3 check based on domain membership +psql -h localhost -p 6009 -U magistrala -d rules_engine -c " + SELECT rrm.role_id, rrm.member_id, rrm.entity_id + FROM rules_role_members rrm + ORDER BY rrm.entity_id;" +``` + +### Verify in SpiceDB + +```bash +zed relationship read rule \ + --insecure --endpoint localhost:50051 --token 12345678 +``` + +**Expected:** Parent relations for rule-1 and rule-2 are newly created. Rule-3 already had one (no duplicate). + +--- + +## Step 5 — Test Idempotency (Re-Run) + +Run the same backfill again without any changes: + +```bash +go run ./scripts/re-backfill-roles/ +``` + +**Expected:** `"loaded rules without roles" count=2` and `"backfill finished" processed=0, skipped=2, failed=0`. The two remaining rows are rule-4 (`created_by=NULL`) and rule-5 (no `domain_id`), which always re-appear in the orphan query and are skipped each run. The idempotency signal is `processed=0` — no roles or SpiceDB writes are duplicated. + +--- + +## Step 6 — Test Partial State (policyExists Path) + +This specifically validates the SpiceDB pre-check. Simulate a scenario where Postgres lost the role but SpiceDB still has the parent relation: + +```bash +# Delete just the Postgres role for rule-1 +psql -h localhost -p 6009 -U magistrala -d rules_engine -c " + DELETE FROM rules_roles + WHERE entity_id = 'r0000000-0000-0000-0000-000000000001';" + +# Re-run backfill +go run ./scripts/re-backfill-roles/ +``` + +### Expected + +- `"loaded rules without roles" count=1` (only rule-1 reappears) +- `"spicedb policy already exists, skipping re-add"` — the parent relation is detected and filtered out +- `"provisioned missing built-in role"` with `new_optional_policies=0, existing_optional_policies=1` +- The role row is re-created in Postgres **without** a duplicate SpiceDB write + +### Verify + +```bash +# Postgres: role restored +psql -h localhost -p 6009 -U magistrala -d rules_engine -c " + SELECT * FROM rules_roles + WHERE entity_id = 'r0000000-0000-0000-0000-000000000001';" + +# SpiceDB: still exactly one parent relation (no duplicate) +zed relationship read rule:r0000000-0000-0000-0000-000000000001 \ + --insecure --endpoint localhost:50051 --token 12345678 +``` + +--- + +## Step 7 — Test Reports Backfill + +Repeat Steps 3–6 for the reports backfill script: + +```bash +# Dry run (set dryRun = true first) +go run ./scripts/reports-backfill-roles/ + +# Real run (set dryRun = false) +go run ./scripts/reports-backfill-roles/ +``` + +### Expected behavior + +| Report | Expected | +|--------|----------| +| report-1 | Role provisioned **with** member (user1 is domain member) | +| report-2 | Role provisioned **without** member (user2 not in domain) | +| report-3 | `"spicedb policy already exists"` + role provisioned with `new_optional_policies=0` | +| report-4 | Skipped (NULL `created_by`) | +| report-5 | Does not appear (already has role) | + +### Verify + +```bash +psql -h localhost -p 6020 -U magistrala -d reports -c " + SELECT rr.id, rr.entity_id, rr.name + FROM reports_roles rr + ORDER BY rr.entity_id;" + +zed relationship read report \ + --insecure --endpoint localhost:50051 --token 12345678 +``` + +--- + +## Step 8 — Verify Domains Migration + +The `domains/postgres/init.go` change adds `alarm_*` and `report_*` actions to the domain admin role. This is applied by the domains service on startup. + +```bash +psql -h localhost -p 6003 -U magistrala -d domains -c " + SELECT action FROM domains_role_actions + WHERE role_id IN (SELECT id FROM domains_roles WHERE name = 'admin') + ORDER BY action;" +``` + +**Expected:** The result should include all of these new actions: + +``` +alarm_acknowledge +alarm_assign +alarm_delete +alarm_read +alarm_resolve +alarm_update +report_add_role_users +report_create +report_delete +report_manage_role +report_read +report_remove_role_users +report_update +report_view_role_users +``` + +--- + +## Step 9 — Cleanup (Optional) + +To reset and re-test from scratch: + +```bash +# Remove all seeded data from RE DB +psql -h localhost -p 6009 -U magistrala -d rules_engine -c " + DELETE FROM rules WHERE id LIKE 'r0000000-%'; + DELETE FROM domains WHERE id = 'd0000000-0000-0000-0000-000000000001';" + +# Remove all seeded data from Reports DB +psql -h localhost -p 6020 -U magistrala -d reports -c " + DELETE FROM report_config WHERE id LIKE 'rp000000-%'; + DELETE FROM domains WHERE id = 'd0000000-0000-0000-0000-000000000001';" + +# Remove all seeded data from Alarms DB +psql -h localhost -p 6019 -U magistrala -d alarms -c " + DELETE FROM alarms WHERE id LIKE 'a0000000-%'; + DELETE FROM domains WHERE id = 'd0000000-0000-0000-0000-000000000001';" + +# Remove SpiceDB relationships +zed relationship delete rule --insecure --endpoint localhost:50051 --token 12345678 +zed relationship delete report --insecure --endpoint localhost:50051 --token 12345678 +``` + +Then re-run `go run ./scripts/seed-test-data/` to start fresh. + +--- + +## Troubleshooting + +| Problem | Solution | +|---------|----------| +| `failed to connect to postgres` | Verify containers are running: `docker compose ps`. Check ports with `docker compose port re-db 5432`. | +| `failed to read spicedb relationships` | Ensure SpiceDB is running and schema is loaded. Check: `zed schema read --insecure --endpoint localhost:50051 --token 12345678` | +| `failed to load built-in role actions` | Verify `spicedbSchemaFile` points to `docker/spicedb/schema.zed` (not `combined-schema.zed`). | +| `no such table` errors during seed | Services haven't run yet to apply migrations. Start the full service (`re`, `reports`, `alarms`) at least once. | +| Script exits with `count=0` unexpectedly | All rules/reports already have roles. Check with the orphan queries from Step 2. | diff --git a/scripts/seed-test-data/main.go b/scripts/seed-test-data/main.go new file mode 100644 index 000000000..d8b535f39 --- /dev/null +++ b/scripts/seed-test-data/main.go @@ -0,0 +1,451 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package main seeds test data across the domains, RE, reports, and alarms +// databases so that the backfill-roles scripts can be tested end-to-end. +// +// It creates one domain, two users (one domain member, one not), several +// rules/reports with and without pre-existing roles, a couple of alarms, and +// optionally a SpiceDB parent relation for one rule to exercise the +// "policy already exists" path. +// +// All IDs are deterministic so re-running is idempotent (INSERT … ON CONFLICT DO NOTHING). +package main + +import ( + "context" + "database/sql" + "fmt" + "log" + "time" + + v1 "github.com/authzed/authzed-go/proto/authzed/api/v1" + "github.com/authzed/authzed-go/v1" + "github.com/authzed/grpcutil" + _ "github.com/jackc/pgx/v5/stdlib" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +// --------------------------------------------------------------------------- +// Configuration — edit these to match your environment. +// Default values target the standard docker-compose setup. +// --------------------------------------------------------------------------- + +var ( + // Domains database. + domainsDB = dbConfig{host: "localhost", port: "6003", user: "magistrala", pass: "magistrala", name: "domains"} + + // RE (rules engine) database. + reDB = dbConfig{host: "localhost", port: "6009", user: "magistrala", pass: "magistrala", name: "rules_engine"} + + // Reports database. + reportsDB = dbConfig{host: "localhost", port: "6020", user: "magistrala", pass: "magistrala", name: "reports"} + + // Alarms database. + alarmsDB = dbConfig{host: "localhost", port: "6019", user: "magistrala", pass: "magistrala", name: "alarms"} + + // SpiceDB. + spicedbHost = "localhost" + spicedbPort = "50051" + spicedbPreSharedKey = "12345678" + + // Whether to write a SpiceDB parent relation for rule-3 to test the + // "policy already exists" code path. + seedSpiceDB = true +) + +// --------------------------------------------------------------------------- +// Deterministic test IDs +// --------------------------------------------------------------------------- + +const ( + domainID = "d0000000-0000-0000-0000-000000000001" + domainName = "seed-test-domain" + + // user1 will be a domain member; user2 will NOT. + user1ID = "u0000000-0000-0000-0000-000000000001" + user2ID = "u0000000-0000-0000-0000-000000000002" + + // Domain role (admin). + domainRoleID = "dr000000-0000-0000-0000-000000000001" + domainRoleName = "admin" + + // Rules — orphans (no rules_roles entry). + rule1ID = "r0000000-0000-0000-0000-000000000001" // created_by=user1 (domain member) → backfill should assign member + rule2ID = "r0000000-0000-0000-0000-000000000002" // created_by=user2 (NOT member) → backfill should provision role without member + rule3ID = "r0000000-0000-0000-0000-000000000003" // created_by=user1, SpiceDB parent already exists → test policyExists + rule4ID = "r0000000-0000-0000-0000-000000000004" // created_by=NULL → should be skipped + rule5ID = "r0000000-0000-0000-0000-000000000005" // empty domain_id → should be skipped + + // Rule with pre-existing role — should NOT appear in orphan list. + rule6ID = "r0000000-0000-0000-0000-000000000006" + rule6RoleID = "rr000000-0000-0000-0000-000000000006" + + // Reports — orphans (no reports_roles entry). + report1ID = "rp000000-0000-0000-0000-000000000001" // created_by=user1 (domain member) + report2ID = "rp000000-0000-0000-0000-000000000002" // created_by=user2 (NOT member) + report3ID = "rp000000-0000-0000-0000-000000000003" // created_by=user1, SpiceDB parent already exists + report4ID = "rp000000-0000-0000-0000-000000000004" // created_by=NULL → skipped + + // Report with pre-existing role. + report5ID = "rp000000-0000-0000-0000-000000000005" + report5RoleID = "rpr00000-0000-0000-0000-000000000005" + + // Alarms (live in alarms DB). + alarm1ID = "a0000000-0000-0000-0000-000000000001" + alarm2ID = "a0000000-0000-0000-0000-000000000002" +) + +type dbConfig struct { + host, port, user, pass, name string +} + +func (c dbConfig) dsn() string { + return fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", c.host, c.port, c.user, c.pass, c.name) +} + +func main() { + ctx := context.Background() + now := time.Now().UTC() + + // ----------------------------------------------------------------------- + // 1. Seed Domains DB + // ----------------------------------------------------------------------- + log.Println("connecting to domains DB ...") + ddb := mustConnect(domainsDB) + defer ddb.Close() + + seedDomainTables(ctx, ddb, now) + log.Println("domains DB seeded") + + // ----------------------------------------------------------------------- + // 2. Seed RE DB (includes domain tables + rules) + // ----------------------------------------------------------------------- + log.Println("connecting to RE DB ...") + rdb := mustConnect(reDB) + defer rdb.Close() + + seedDomainTables(ctx, rdb, now) + seedRules(ctx, rdb, now) + log.Println("RE DB seeded") + + // ----------------------------------------------------------------------- + // 3. Seed Reports DB (includes domain tables + report_config) + // ----------------------------------------------------------------------- + log.Println("connecting to reports DB ...") + rpdb := mustConnect(reportsDB) + defer rpdb.Close() + + seedDomainTables(ctx, rpdb, now) + seedReports(ctx, rpdb, now) + log.Println("reports DB seeded") + + // ----------------------------------------------------------------------- + // 4. Seed Alarms DB (includes domain + RE tables + alarms) + // ----------------------------------------------------------------------- + log.Println("connecting to alarms DB ...") + adb := mustConnect(alarmsDB) + defer adb.Close() + + seedDomainTables(ctx, adb, now) + seedRulesMinimal(ctx, adb, now) // alarms DB has rules tables via RE migration + seedAlarms(ctx, adb, now) + log.Println("alarms DB seeded") + + // ----------------------------------------------------------------------- + // 5. Optionally seed SpiceDB (parent relations for rule3 and report3) + // ----------------------------------------------------------------------- + if seedSpiceDB { + log.Println("connecting to SpiceDB ...") + seedSpiceDBRelationships(ctx) + log.Println("SpiceDB seeded") + } + + log.Println("all seed data inserted successfully") + printSummary() +} + +// --------------------------------------------------------------------------- +// Domain tables (identical across all DBs that include domain migrations) +// --------------------------------------------------------------------------- + +func seedDomainTables(ctx context.Context, db *sql.DB, now time.Time) { + mustExec(ctx, db, ` + INSERT INTO domains (id, name, tags, metadata, route, created_at, updated_at, created_by, status) + VALUES ($1, $2, '{}', '{}', $3, $4, $4, $5, 0) + ON CONFLICT (id) DO NOTHING`, + domainID, domainName, "seed-test-domain", now, user1ID) + + // Domain admin role + mustExec(ctx, db, ` + INSERT INTO domains_roles (id, name, entity_id, created_at, updated_at, created_by) + VALUES ($1, $2, $3, $4, $4, $5) + ON CONFLICT (id) DO NOTHING`, + domainRoleID, domainRoleName, domainID, now, user1ID) + + // Domain role actions (a representative subset) + actions := []string{ + "domain_update", "domain_read", "domain_membership", + "domain_manage_role", "domain_add_role_users", "domain_remove_role_users", "domain_view_role_users", + "rule_create", "rule_read", "rule_update", "rule_delete", + "rule_manage_role", "rule_add_role_users", "rule_remove_role_users", "rule_view_role_users", + "report_create", "report_read", "report_update", "report_delete", + "report_manage_role", "report_add_role_users", "report_remove_role_users", "report_view_role_users", + "alarm_update", "alarm_read", "alarm_delete", "alarm_assign", "alarm_acknowledge", "alarm_resolve", + } + for _, action := range actions { + mustExec(ctx, db, ` + INSERT INTO domains_role_actions (role_id, action) + VALUES ($1, $2) + ON CONFLICT DO NOTHING`, + domainRoleID, action) + } + + // user1 is a domain member; user2 is NOT. + mustExec(ctx, db, ` + INSERT INTO domains_role_members (role_id, member_id, entity_id) + VALUES ($1, $2, $3) + ON CONFLICT DO NOTHING`, + domainRoleID, user1ID, domainID) +} + +// --------------------------------------------------------------------------- +// Rules (RE DB) +// --------------------------------------------------------------------------- + +func seedRules(ctx context.Context, db *sql.DB, now time.Time) { + type rule struct { + id, name, domainID string + createdBy *string + } + + u1 := strPtr(user1ID) + u2 := strPtr(user2ID) + + rules := []rule{ + {rule1ID, "rule-1-member-creator", domainID, u1}, + {rule2ID, "rule-2-nonmember-creator", domainID, u2}, + {rule3ID, "rule-3-spicedb-exists", domainID, u1}, + {rule4ID, "rule-4-null-creator", domainID, nil}, + {rule5ID, "rule-5-no-domain", "", u1}, + {rule6ID, "rule-6-has-role-already", domainID, u1}, + } + + for _, r := range rules { + mustExec(ctx, db, ` + INSERT INTO rules (id, name, domain_id, created_by, created_at, status, logic_type) + VALUES ($1, $2, $3, $4, $5, 0, 0) + ON CONFLICT (id) DO NOTHING`, + r.id, r.name, r.domainID, r.createdBy, now) + } + + // rule6 already has a role → should NOT appear in orphan list. + mustExec(ctx, db, ` + INSERT INTO rules_roles (id, name, entity_id, created_at, updated_at, created_by) + VALUES ($1, 'admin', $2, $3, $3, $4) + ON CONFLICT (id) DO NOTHING`, + rule6RoleID, rule6ID, now, user1ID) + + mustExec(ctx, db, ` + INSERT INTO rules_role_actions (role_id, action) + VALUES ($1, 'rule_read') + ON CONFLICT DO NOTHING`, + rule6RoleID) + + mustExec(ctx, db, ` + INSERT INTO rules_role_members (role_id, member_id, entity_id) + VALUES ($1, $2, $3) + ON CONFLICT DO NOTHING`, + rule6RoleID, user1ID, rule6ID) +} + +// seedRulesMinimal inserts the same rules into the alarms DB (which has RE +// tables) so that foreign key constraints on rule_id can be satisfied. +func seedRulesMinimal(ctx context.Context, db *sql.DB, now time.Time) { + for _, r := range []struct{ id, name string }{ + {rule1ID, "rule-1-member-creator"}, + {rule2ID, "rule-2-nonmember-creator"}, + } { + mustExec(ctx, db, ` + INSERT INTO rules (id, name, domain_id, created_by, created_at, status, logic_type) + VALUES ($1, $2, $3, $4, $5, 0, 0) + ON CONFLICT (id) DO NOTHING`, + r.id, r.name, domainID, user1ID, now) + } +} + +// --------------------------------------------------------------------------- +// Reports +// --------------------------------------------------------------------------- + +func seedReports(ctx context.Context, db *sql.DB, now time.Time) { + type report struct { + id, name, domainID string + createdBy *string + } + + u1 := strPtr(user1ID) + u2 := strPtr(user2ID) + + reports := []report{ + {report1ID, "report-1-member-creator", domainID, u1}, + {report2ID, "report-2-nonmember-creator", domainID, u2}, + {report3ID, "report-3-spicedb-exists", domainID, u1}, + {report4ID, "report-4-null-creator", domainID, nil}, + {report5ID, "report-5-has-role-already", domainID, u1}, + } + + for _, r := range reports { + mustExec(ctx, db, ` + INSERT INTO report_config (id, name, domain_id, created_by, created_at, status) + VALUES ($1, $2, $3, $4, $5, 0) + ON CONFLICT (id) DO NOTHING`, + r.id, r.name, r.domainID, r.createdBy, now) + } + + // report5 already has a role. + mustExec(ctx, db, ` + INSERT INTO reports_roles (id, name, entity_id, created_at, updated_at, created_by) + VALUES ($1, 'admin', $2, $3, $3, $4) + ON CONFLICT (id) DO NOTHING`, + report5RoleID, report5ID, now, user1ID) + + mustExec(ctx, db, ` + INSERT INTO reports_role_actions (role_id, action) + VALUES ($1, 'report_read') + ON CONFLICT DO NOTHING`, + report5RoleID) + + mustExec(ctx, db, ` + INSERT INTO reports_role_members (role_id, member_id, entity_id) + VALUES ($1, $2, $3) + ON CONFLICT DO NOTHING`, + report5RoleID, user1ID, report5ID) +} + +// --------------------------------------------------------------------------- +// Alarms +// --------------------------------------------------------------------------- + +func seedAlarms(ctx context.Context, db *sql.DB, now time.Time) { + for _, a := range []struct{ id, ruleID string }{ + {alarm1ID, rule1ID}, + {alarm2ID, rule2ID}, + } { + mustExec(ctx, db, ` + INSERT INTO alarms (id, rule_id, domain_id, channel_id, subtopic, client_id, + measurement, value, unit, threshold, cause, status, severity, created_at) + VALUES ($1, $2, $3, 'ch000000-0000-0000-0000-000000000001', 'test/topic', + 'cl000000-0000-0000-0000-000000000001', 'temperature', '42.5', 'C', '40.0', + 'exceeded threshold', 0, 1, $4) + ON CONFLICT (id) DO NOTHING`, + a.id, a.ruleID, domainID, now) + } +} + +// --------------------------------------------------------------------------- +// SpiceDB — write a parent relation for rule3 and report3 so the +// "policy already exists" code path is exercised. +// --------------------------------------------------------------------------- + +func seedSpiceDBRelationships(ctx context.Context) { + addr := fmt.Sprintf("%s:%s", spicedbHost, spicedbPort) + client, err := authzed.NewClientWithExperimentalAPIs( + addr, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpcutil.WithInsecureBearerToken(spicedbPreSharedKey), + ) + if err != nil { + log.Printf("WARNING: failed to connect to SpiceDB at %s: %v (skipping SpiceDB seed)", addr, err) + return + } + + // Use TOUCH so re-running is idempotent. + updates := []*v1.RelationshipUpdate{ + { + Operation: v1.RelationshipUpdate_OPERATION_TOUCH, + Relationship: &v1.Relationship{ + Resource: &v1.ObjectReference{ObjectType: "rule", ObjectId: rule3ID}, + Relation: "domain", + Subject: &v1.SubjectReference{Object: &v1.ObjectReference{ObjectType: "domain", ObjectId: domainID}}, + }, + }, + { + Operation: v1.RelationshipUpdate_OPERATION_TOUCH, + Relationship: &v1.Relationship{ + Resource: &v1.ObjectReference{ObjectType: "report", ObjectId: report3ID}, + Relation: "domain", + Subject: &v1.SubjectReference{Object: &v1.ObjectReference{ObjectType: "domain", ObjectId: domainID}}, + }, + }, + } + + _, err = client.WriteRelationships(ctx, &v1.WriteRelationshipsRequest{Updates: updates}) + if err != nil { + log.Printf("WARNING: failed to write SpiceDB relationships: %v", err) + return + } + log.Printf("wrote %d SpiceDB relationships (TOUCH)", len(updates)) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +func mustConnect(cfg dbConfig) *sql.DB { + db, err := sql.Open("pgx", cfg.dsn()) + if err != nil { + log.Fatalf("failed to open %s: %v", cfg.name, err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + if err := db.PingContext(ctx); err != nil { + cancel() + log.Fatalf("failed to ping %s at %s:%s: %v", cfg.name, cfg.host, cfg.port, err) + } + cancel() + return db +} + +func mustExec(ctx context.Context, db *sql.DB, query string, args ...any) { + if _, err := db.ExecContext(ctx, query, args...); err != nil { + log.Fatalf("exec failed: %v\nquery: %s\nargs: %v", err, query, args) + } +} + +func strPtr(s string) *string { return &s } + +func printSummary() { + fmt.Print(` +=== SEED DATA SUMMARY === + +Domain: ` + domainID + ` ("` + domainName + `") + +Users: + user1 (domain member): ` + user1ID + ` + user2 (NOT a member): ` + user2ID + ` + +Rules (RE DB): + ` + rule1ID + ` rule-1-member-creator orphan, created_by=user1 → expect role WITH member + ` + rule2ID + ` rule-2-nonmember-creator orphan, created_by=user2 → expect role WITHOUT member + ` + rule3ID + ` rule-3-spicedb-exists orphan, created_by=user1, SpiceDB parent pre-seeded → test policyExists + ` + rule4ID + ` rule-4-null-creator orphan, created_by=NULL → expect SKIPPED + ` + rule5ID + ` rule-5-no-domain orphan, domain_id="" → expect SKIPPED + ` + rule6ID + ` rule-6-has-role-already HAS role entry → should NOT appear in orphan list + +Reports (Reports DB): + ` + report1ID + ` report-1-member-creator orphan, created_by=user1 → expect role WITH member + ` + report2ID + ` report-2-nonmember-creator orphan, created_by=user2 → expect role WITHOUT member + ` + report3ID + ` report-3-spicedb-exists orphan, created_by=user1, SpiceDB parent pre-seeded → test policyExists + ` + report4ID + ` report-4-null-creator orphan, created_by=NULL → expect SKIPPED + ` + report5ID + ` report-5-has-role-already HAS role entry → should NOT appear in orphan list + +Alarms (Alarms DB): + ` + alarm1ID + ` alarm-1 (rule1) + ` + alarm2ID + ` alarm-2 (rule2) + +SpiceDB: + rule:` + rule3ID + `#domain@domain:` + domainID + ` (pre-seeded) + report:` + report3ID + `#domain@domain:` + domainID + ` (pre-seeded) +`) +}