Fix migrations

Signed-off-by: dusan <borovcanindusan1@gmail.com>

Makefile

@@ -330,6 +330,7 @@ provision-atom-tokens:
 # args="--verify" to reconcile afterwards.
 #   make migrate_atom                 # dry-run
 #   make migrate_atom args="--apply"  # perform the migration
+#   make migrate_atom args="--apply --fresh-atom"  # rebuild Atom schema first
 migrate_atom:
 	DOCKER_PROJECT="$(DOCKER_PROJECT)" tools/atom-migration/migrate.sh $(args)
 

docker/setup-tls.sh

@@ -6,6 +6,7 @@ set -eu
 
 ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
 ENV_FILE="$ROOT_DIR/docker/.env"
+TOKENS_ENV_FILE="$ROOT_DIR/docker/.env.tokens"
 COMPOSE_FILE="$ROOT_DIR/docker/docker-compose.yaml"
 
 HOST=${MG_PUBLIC_HOST:-}
@@ -124,7 +125,11 @@ comment_env_any() {
 }
 
 compose() {
-	docker compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" -p "$PROJECT" "$@"
+	if [ -f "$TOKENS_ENV_FILE" ]; then
+		docker compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" --env-file "$TOKENS_ENV_FILE" -p "$PROJECT" "$@"
+	else
+		docker compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" -p "$PROJECT" "$@"
+	fi
 }
 
 write_ui_proxy() {

tools/atom-migration/README.md

@@ -50,8 +50,36 @@ them directly. On `--apply`, the device-key CSV described below lands in
 Env overrides: `SRC_VOL_PREFIX` (old volume prefix, default
 `magistrala_magistrala-`), `SRC_DB_USER` / `SRC_DB_PASS` (old Postgres creds,
 default `magistrala`), `DOCKER_PROJECT` (run_latest project; default derived like
-the Makefile), `MIGRATE_PROJECT` (isolated project name, default `atommig`).
-Pass `--keep` to leave the stack up for debugging.
+the Makefile), `MIGRATE_PROJECT` (isolated project name, default `atommig`),
+`ATOM_IMAGE_TAG` (Atom image tag to seed with, default `latest`). Pass `--keep`
+to leave the stack up for debugging.
+
+### Atom schema freshness (`column "alias" does not exist`)
+
+The migrator writes the **current** Atom schema. The schema is created by the
+Atom image (`ghcr.io/absmach/atom:latest`, force-pulled each run) the first time
+its target volume is seeded. Two things follow:
+
+- A target volume that was **already seeded by an older Atom** (e.g. a previous
+  `make run_latest`) keeps that old schema — Atom does not re-run an
+  already-applied baseline, so newer columns like `tenants.alias` never appear.
+  The migrator then fails with `column "alias" of relation "tenants" does not
+  exist`.
+- The orchestrator guards against this: it waits for `tenants.alias` to exist
+  before running the migrator and aborts with guidance if it never does (instead
+  of failing mid-apply).
+
+Fix: rebuild the schema from scratch with
+
+```bash
+make migrate_atom args="--apply --fresh-atom"
+```
+
+`--fresh-atom` removes the existing Atom target volume so the current image lays
+down the current schema. **Destructive** for that volume only — it discards any
+data already in the Atom DB (the old per-service source volumes are never
+touched). Equivalent manual reset: `docker volume rm
+<DOCKER_PROJECT>_magistrala-atom-db-volume`.
 
 ### How volume names are resolved
 

tools/atom-migration/docker-compose.migrate.yaml

@@ -136,7 +136,8 @@ services:
   # Seeds / upgrades the Atom schema in the target volume on boot (migrations run
   # at startup). Certs disabled so it needs no mounted CA. No host ports.
   atom:
-    image: ghcr.io/absmach/atom:${MG_RELEASE_TAG:-latest}
+    image: ghcr.io/absmach/atom:${ATOM_IMAGE_TAG:-latest}
+    pull_policy: always
     restart: "no"
     depends_on:
       atom-db:

tools/atom-migration/migrate.sh

@@ -17,8 +17,13 @@
 #   tools/atom-migration/migrate.sh            # dry-run (reads + validates, writes nothing)
 #   tools/atom-migration/migrate.sh --apply    # perform the migration
 #   tools/atom-migration/migrate.sh --verify   # reconcile source vs Atom after apply
+#   tools/atom-migration/migrate.sh --apply --fresh-atom  # rebuild Atom schema from scratch
 #   tools/atom-migration/migrate.sh --apply --keep   # leave the stack running for debugging
 #
+# --fresh-atom discards any existing Atom target volume first, so the current Atom
+# image lays down the current schema. Use it when a previous run/`make run_latest`
+# left an older Atom schema in the volume (symptom: "column alias does not exist").
+#
 # Env overrides:
 #   DOCKER_PROJECT     run_latest Compose project (default: derived like the Makefile)
 #   SRC_VOL_PREFIX     old DB volume name prefix   (default: magistrala_magistrala-)
@@ -36,9 +41,11 @@ SRC_VOL_PREFIX="${SRC_VOL_PREFIX:-magistrala_magistrala-}"
 
 MIGRATOR_ARGS=()
 KEEP=false
+FRESH_ATOM=false
 for arg in "$@"; do
 	case "$arg" in
 		--keep) KEEP=true ;;
+		--fresh-atom) FRESH_ATOM=true ;;
 		--apply|--verify|--dry-run) MIGRATOR_ARGS+=("$arg") ;;
 		--unmapped-action=*|--report-dir=*) MIGRATOR_ARGS+=("$arg") ;;
 		-h|--help) grep '^#' "$0" | sed 's/^# \{0,1\}//' | head -40; exit 0 ;;
@@ -62,6 +69,14 @@ fi
 [[ -n "$DOCKER_PROJECT" ]] || die "could not determine DOCKER_PROJECT (set it explicitly)"
 ATOM_TARGET_VOLUME="${DOCKER_PROJECT}_magistrala-atom-db-volume"
 
+# Atom DB connection settings come from docker/.env (Compose interpolates them
+# from --env-file). Read the same values here so our readiness poll connects with
+# the right user/db -- defaulting to "atom" would loop forever if .env overrides
+# them.
+envget() { sed -nE "s/^[[:space:]]*$1=//p" "$ENV_FILE" | tail -1 | tr -d '"'"'"; }
+ATOM_DB_USER="$(envget ATOM_DB_USER)";     ATOM_DB_USER="${ATOM_DB_USER:-atom}"
+ATOM_DB_NAME="$(envget ATOM_DB_NAME)";     ATOM_DB_NAME="${ATOM_DB_NAME:-atom}"
+
 log "Isolated project : $MIGRATE_PROJECT"
 log "Source volumes   : ${SRC_VOL_PREFIX}<svc>-db-volume"
 log "Atom target vol  : $ATOM_TARGET_VOLUME"
@@ -76,6 +91,20 @@ done
 Run this on the machine whose stopped old Magistrala stack still has these
 volumes, or set SRC_VOL_PREFIX if the old Compose project used another name."
 
+# --fresh-atom: rebuild the target schema from scratch. A pre-existing Atom volume
+# (e.g. from an earlier `make run_latest`) keeps whatever schema it was seeded
+# with -- Atom's migrations won't re-run an already-applied baseline, so an old
+# schema (missing newer columns like tenants.alias) would survive and break the
+# load. Removing the volume forces the current Atom image to lay down the current
+# schema. Destructive: any data already in that Atom volume is discarded.
+if [[ "$FRESH_ATOM" == true ]]; then
+	if docker volume inspect "$ATOM_TARGET_VOLUME" >/dev/null 2>&1; then
+		log "Resetting Atom target volume $ATOM_TARGET_VOLUME (--fresh-atom)"
+		docker compose --env-file "$ENV_FILE" -p "$MIGRATE_PROJECT" -f "$COMPOSE_FILE" down >/dev/null 2>&1 || true
+		docker volume rm -f "$ATOM_TARGET_VOLUME" >/dev/null
+	fi
+fi
+
 # Atom target volume: created here if absent so the seed step can write to it;
 # `make run_latest` reuses the same name.
 docker volume inspect "$ATOM_TARGET_VOLUME" >/dev/null 2>&1 || {
@@ -111,9 +140,12 @@ log "Building migrator image"
 docker build -q -f "$SCRIPT_DIR/Dockerfile" -t magistrala/atom-migration:dev "$REPO_ROOT" >/dev/null
 
 # Variables consumed by the compose file. docker/.env is passed via --env-file so
-# ATOM_DB_* / MG_RELEASE_TAG interpolate; these exports take precedence.
+# ATOM_DB_* interpolate; these exports take precedence. The Atom image is tagged
+# independently of MG_RELEASE_TAG (the real compose pins :latest), so default to
+# latest here too; override with ATOM_IMAGE_TAG if you need a specific build.
 export SRC_VOL_PREFIX SRC_PG_IMAGE SRC_MOUNT SRC_PGDATA ATOM_TARGET_VOLUME
 export SRC_DB_USER="${SRC_DB_USER:-magistrala}" SRC_DB_PASS="${SRC_DB_PASS:-magistrala}"
+export ATOM_IMAGE_TAG="${ATOM_IMAGE_TAG:-latest}"
 export HOST_UID="$(id -u)" HOST_GID="$(id -g)"
 
 dc() { docker compose --env-file "$ENV_FILE" -p "$MIGRATE_PROJECT" -f "$COMPOSE_FILE" "$@"; }
@@ -135,14 +167,34 @@ dc up -d --wait domains-db users-db clients-db channels-db groups-db auth-db re-
 dc up -d atom
 
 log "Waiting for Atom to apply its schema into the target volume"
+# Require a column from a late migration (tenants.alias), not just table
+# existence: Atom adds tables early and columns later, so checking only for the
+# tables can race ahead of a not-yet-complete migration and hit the same
+# "column alias does not exist" error mid-apply. A timeout here with alias still
+# absent means the volume holds an old/incompatible Atom schema -> --fresh-atom.
+schema_ready_q="SELECT (to_regclass('public.entities') IS NOT NULL)
+	AND EXISTS (SELECT 1 FROM information_schema.columns
+		WHERE table_name='tenants' AND column_name='alias');"
+atom_cid="$(dc ps -q atom)"
 for i in $(seq 1 60); do
-	if dc exec -T atom-db psql -U "${ATOM_DB_USER:-atom}" -d "${ATOM_DB_NAME:-atom}" -tAc \
-		"SELECT to_regclass('public.tenants') IS NOT NULL AND to_regclass('public.entities') IS NOT NULL;" 2>/dev/null \
-		| grep -qx t; then
+	if dc exec -T atom-db psql -U "$ATOM_DB_USER" -d "$ATOM_DB_NAME" -tAc \
+		"$schema_ready_q" 2>/dev/null | grep -qx t; then
 		log "Atom schema ready"
 		break
 	fi
-	[[ $i -eq 60 ]] && die "Atom did not initialise its schema in time (check: dc logs atom)"
+	# If the Atom container has stopped it will never create the schema -- fail
+	# fast with its logs instead of waiting out the whole timeout.
+	if [[ -n "$atom_cid" ]] && [[ "$(docker inspect -f '{{.State.Running}}' "$atom_cid" 2>/dev/null)" != "true" ]]; then
+		echo "----- atom logs -----" >&2; dc logs --no-color --tail 50 atom >&2 || true
+		die "Atom container exited before applying its schema (see logs above).
+Common causes: missing/invalid ATOM_* settings in docker/.env (JWT_SECRET,
+ATOM_KEY_ENCRYPTION_KEY, ATOM_SERVICE_SECRET, ATOM_ADMIN_SECRET) or it could not
+reach atom-db."
+	fi
+	[[ $i -eq 60 ]] && { echo "----- atom logs -----" >&2; dc logs --no-color --tail 50 atom >&2 || true; die "Atom schema in $ATOM_TARGET_VOLUME never reached the expected version
+(tenants.alias missing) within the timeout. If Atom is still migrating, retry; if
+the volume holds an older Atom schema, re-run with --fresh-atom (discards that
+volume's Atom data) or 'docker volume rm $ATOM_TARGET_VOLUME'."; }
 	sleep 2
 done