fix(migrator): normalize Git sources URL correctly in the migrations BE-13152 (#3045)

api/datastore/migrator/migrate_2_43_0.go

@@ -69,7 +69,13 @@ type sourceDedupeKey struct {
 }
 
 func gitSourceKey(cfg *gittypes.RepoConfig) sourceDedupeKey {
-	key := sourceDedupeKey{url: cfg.URL}
+	url, err := gittypes.NormalizeURL(gittypes.SanitizeURL(cfg.URL))
+	if err != nil {
+		log.Warn().Err(err).Str("url", cfg.URL).Msg("failed to normalize git URL for deduplication, using raw URL")
+		url = cfg.URL
+	}
+
+	key := sourceDedupeKey{url: url}
 	if cfg.Authentication != nil {
 		key.username = cfg.Authentication.Username
 		key.password = cfg.Authentication.Password

api/datastore/migrator/migrate_2_43_0_test.go

@@ -193,6 +193,72 @@ func TestMigrateGitConfigToSources_2_43_0_DuplicateSourcesDeduped(t *testing.T)
 	}
 }
 
+func TestMigrateGitConfigToSources_2_43_0_DotGitSuffixDeduped(t *testing.T) {
+	t.Parallel()
+
+	conn := &boltdb.DbConnection{Path: t.TempDir()}
+	err := conn.Open()
+	require.NoError(t, err)
+	defer logs.CloseAndLogErr(conn)
+
+	stackSvc, err := stack.NewService(conn)
+	require.NoError(t, err)
+	sourceSvc, err := source.NewService(conn)
+	require.NoError(t, err)
+	workflowSvc, err := workflow.NewService(conn)
+	require.NoError(t, err)
+	rcSvc, err := resourcecontrol.NewService(conn)
+	require.NoError(t, err)
+
+	m := NewMigrator(&MigratorParameters{
+		StackService:           stackSvc,
+		SourceService:          sourceSvc,
+		WorkflowService:        workflowSvc,
+		ResourceControlService: rcSvc,
+	})
+
+	// One stack uses the .git suffix, the other does not. Both refer to the
+	// same repository and must share a single Source record after migration.
+	stack1 := &portainer.Stack{
+		ID:   1,
+		Name: "stack-a",
+		GitConfig: &gittypes.RepoConfig{
+			URL:           "https://github.com/example/shared-repo.git",
+			ReferenceName: "refs/heads/main",
+		},
+	}
+	stack2 := &portainer.Stack{
+		ID:   2,
+		Name: "stack-b",
+		GitConfig: &gittypes.RepoConfig{
+			URL:           "https://github.com/example/shared-repo",
+			ReferenceName: "refs/heads/develop",
+		},
+	}
+	err = conn.CreateObjectWithId(stack.BucketName, int(stack1.ID), stack1)
+	require.NoError(t, err)
+	err = conn.CreateObjectWithId(stack.BucketName, int(stack2.ID), stack2)
+	require.NoError(t, err)
+
+	err = m.migrateGitConfigToSources_2_43_0()
+	require.NoError(t, err)
+
+	sources, err := sourceSvc.ReadAll(adminUserContext)
+	require.NoError(t, err)
+	require.Len(t, sources, 1, "stacks whose URLs differ only in .git suffix must share one Source")
+
+	workflows, err := workflowSvc.ReadAll()
+	require.NoError(t, err)
+	require.Len(t, workflows, 2, "each stack must get its own Workflow")
+
+	sharedSourceID := sources[0].ID
+	for _, wf := range workflows {
+		require.Len(t, wf.Artifacts, 1)
+		require.Len(t, wf.Artifacts[0].Files, 1)
+		require.Equal(t, sharedSourceID, wf.Artifacts[0].Files[0].SourceID)
+	}
+}
+
 func TestMigrateGitConfigToSources_2_43_0_Idempotent(t *testing.T) {
 	t.Parallel()
 
@@ -439,6 +505,70 @@ func TestMigrateCustomTemplateGitConfigToSources_2_43_0_DuplicateSourcesDeduped(
 	require.Equal(t, sharedSrcID, migrated2.Artifact.Files[0].SourceID)
 }
 
+func TestMigrateCustomTemplateGitConfigToSources_2_43_0_DotGitSuffixDeduped(t *testing.T) {
+	t.Parallel()
+
+	conn := &boltdb.DbConnection{Path: t.TempDir()}
+	err := conn.Open()
+	require.NoError(t, err)
+	defer logs.CloseAndLogErr(conn)
+
+	stackSvc, err := stack.NewService(conn)
+	require.NoError(t, err)
+	sourceSvc, err := source.NewService(conn)
+	require.NoError(t, err)
+	customTemplateSvc, err := customtemplate.NewService(conn)
+	require.NoError(t, err)
+
+	m := NewMigrator(&MigratorParameters{
+		StackService:          stackSvc,
+		SourceService:         sourceSvc,
+		CustomTemplateService: customTemplateSvc,
+	})
+
+	// One template uses the .git suffix, the other does not. Both refer to the
+	// same repository and must share a single Source record after migration.
+	tmpl1 := &portainer.CustomTemplate{
+		ID:    1,
+		Title: "template-a",
+		GitConfig: &gittypes.RepoConfig{
+			URL:           "https://github.com/example/shared-repo.git",
+			ReferenceName: "refs/heads/main",
+		},
+	}
+	tmpl2 := &portainer.CustomTemplate{
+		ID:    2,
+		Title: "template-b",
+		GitConfig: &gittypes.RepoConfig{
+			URL:           "https://github.com/example/shared-repo",
+			ReferenceName: "refs/heads/develop",
+		},
+	}
+	err = conn.CreateObjectWithId(customtemplate.BucketName, int(tmpl1.ID), tmpl1)
+	require.NoError(t, err)
+	err = conn.CreateObjectWithId(customtemplate.BucketName, int(tmpl2.ID), tmpl2)
+	require.NoError(t, err)
+
+	err = m.migrateCustomTemplateGitConfigToSources_2_43_0()
+	require.NoError(t, err)
+
+	sources, err := sourceSvc.ReadAll(adminUserContext)
+	require.NoError(t, err)
+	require.Len(t, sources, 1, "templates whose URLs differ only in .git suffix must share one Source")
+
+	sharedSrcID := sources[0].ID
+
+	migrated1, err := customTemplateSvc.Read(tmpl1.ID)
+	require.NoError(t, err)
+	require.NotNil(t, migrated1.Artifact)
+	require.Equal(t, sharedSrcID, migrated1.Artifact.Files[0].SourceID)
+
+	migrated2, err := customTemplateSvc.Read(tmpl2.ID)
+	require.NoError(t, err)
+	require.NotNil(t, migrated2.Artifact)
+	require.Equal(t, sharedSrcID, migrated2.Artifact.Files[0].SourceID)
+}
+
 // TestMigrateGitConfigToSources_2_43_0_StandardUserStackPreservesAccess is the regression
 // test for the bug where stacks owned by standard users became inaccessible after upgrading
 // to 2.43. ResourceControls are stored in a separate bucket and are never embedded in the