refactor(gitops/sources): split source get/list handler [BE-13207] (#3203)

Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Chaim Lev-Ari
2026-07-21 17:11:20 +03:00
committed by GitHub
parent d09b50b95a
commit ae8e09f777
37 changed files with 984 additions and 898 deletions
+70 -51
View File
@@ -3208,8 +3208,8 @@ paths:
- gitops
get:
description: >-
Returns a single GitOps source with its connection settings and linked
workflows.
Returns a single GitOps source with its connection settings and access
rules.
**Access policy**: authenticated
operationId: GitOpsSourceGet
@@ -3367,6 +3367,44 @@ paths:
summary: Test the connection of a stored source
tags:
- gitops
"/gitops/sources/{id}/workflows":
get:
description: >-
Returns the workflows (stacks or edge stacks) currently deployed from
this source.
**Access policy**: authenticated
operationId: GitOpsSourceWorkflowsList
parameters:
- description: Source identifier
in: path
name: id
required: true
schema:
type: integer
responses:
"200":
description: OK
content:
application/json:
schema:
items:
$ref: "#/components/schemas/sources.Workflow"
type: array
"400":
description: Invalid request
"403":
description: Access denied
"404":
description: Source not found
"500":
description: Server error
security:
- ApiKeyAuth: []
- jwt: []
summary: List the workflows using a GitOps source
tags:
- gitops
/gitops/sources/git:
post:
description: |-
@@ -16647,13 +16685,6 @@ components:
example: 5m
type: string
type: object
sources.AutoUpdateInfo:
properties:
fetchInterval:
type: string
mechanism:
type: string
type: object
sources.ConnectionTestResult:
properties:
error:
@@ -16777,12 +16808,8 @@ components:
properties:
access:
$ref: "#/components/schemas/sources.SourceAccess"
autoUpdate:
$ref: "#/components/schemas/sources.AutoUpdateInfo"
connection:
$ref: "#/components/schemas/sources.connectionInfo"
environments:
type: integer
error:
type: string
id:
@@ -16800,12 +16827,6 @@ components:
$ref: "#/components/schemas/sources.SourceType"
url:
type: string
usedBy:
type: integer
workflows:
items:
$ref: "#/components/schemas/workflows.SourceWorkflow"
type: array
required:
- connection
- id
@@ -16834,6 +16855,36 @@ components:
- SourceStatusUnknown
- SourceStatusHealthy
- SourceStatusError
sources.Workflow:
properties:
creationDate:
type: integer
gitConfig:
$ref: "#/components/schemas/gittypes.RepoConfig"
id:
type: integer
lastSyncDate:
type: integer
name:
type: string
platform:
$ref: "#/components/schemas/workflows.DeploymentPlatform"
sourceId:
type: integer
status:
$ref: "#/components/schemas/workflows.WorkflowStatusObject"
target:
$ref: "#/components/schemas/workflows.Target"
type:
$ref: "#/components/schemas/workflows.Type"
required:
- id
- name
- platform
- status
- target
- type
type: object
sources.connectionInfo:
properties:
authentication:
@@ -21477,38 +21528,6 @@ components:
- DeploymentPlatformDockerStandalone
- DeploymentPlatformDockerSwarm
- DeploymentPlatformKubernetes
workflows.SourceWorkflow:
properties:
autoUpdate:
$ref: "#/components/schemas/portainer.AutoUpdateSettings"
creationDate:
type: integer
gitConfig:
$ref: "#/components/schemas/gittypes.RepoConfig"
id:
type: integer
lastSyncDate:
type: integer
name:
type: string
platform:
$ref: "#/components/schemas/workflows.DeploymentPlatform"
sourceId:
type: integer
status:
$ref: "#/components/schemas/workflows.WorkflowStatusObject"
target:
$ref: "#/components/schemas/workflows.Target"
type:
$ref: "#/components/schemas/workflows.Type"
required:
- id
- name
- platform
- status
- target
- type
type: object
workflows.Status:
enum:
- healthy
+66 -50
View File
@@ -5651,13 +5651,6 @@ definitions:
example: 5m
type: string
type: object
sources.AutoUpdateInfo:
properties:
fetchInterval:
type: string
mechanism:
type: string
type: object
sources.ConnectionTestResult:
properties:
error:
@@ -5781,12 +5774,8 @@ definitions:
properties:
access:
$ref: '#/definitions/sources.SourceAccess'
autoUpdate:
$ref: '#/definitions/sources.AutoUpdateInfo'
connection:
$ref: '#/definitions/sources.connectionInfo'
environments:
type: integer
error:
type: string
id:
@@ -5804,12 +5793,6 @@ definitions:
$ref: '#/definitions/sources.SourceType'
url:
type: string
usedBy:
type: integer
workflows:
items:
$ref: '#/definitions/workflows.SourceWorkflow'
type: array
required:
- connection
- id
@@ -5838,6 +5821,36 @@ definitions:
- SourceStatusUnknown
- SourceStatusHealthy
- SourceStatusError
sources.Workflow:
properties:
creationDate:
type: integer
gitConfig:
$ref: '#/definitions/gittypes.RepoConfig'
id:
type: integer
lastSyncDate:
type: integer
name:
type: string
platform:
$ref: '#/definitions/workflows.DeploymentPlatform'
sourceId:
type: integer
status:
$ref: '#/definitions/workflows.WorkflowStatusObject'
target:
$ref: '#/definitions/workflows.Target'
type:
$ref: '#/definitions/workflows.Type'
required:
- id
- name
- platform
- status
- target
- type
type: object
sources.connectionInfo:
properties:
authentication:
@@ -9658,38 +9671,6 @@ definitions:
- DeploymentPlatformDockerStandalone
- DeploymentPlatformDockerSwarm
- DeploymentPlatformKubernetes
workflows.SourceWorkflow:
properties:
autoUpdate:
$ref: '#/definitions/portainer.AutoUpdateSettings'
creationDate:
type: integer
gitConfig:
$ref: '#/definitions/gittypes.RepoConfig'
id:
type: integer
lastSyncDate:
type: integer
name:
type: string
platform:
$ref: '#/definitions/workflows.DeploymentPlatform'
sourceId:
type: integer
status:
$ref: '#/definitions/workflows.WorkflowStatusObject'
target:
$ref: '#/definitions/workflows.Target'
type:
$ref: '#/definitions/workflows.Type'
required:
- id
- name
- platform
- status
- target
- type
type: object
workflows.Status:
enum:
- healthy
@@ -12850,7 +12831,7 @@ paths:
- gitops
get:
description: |-
Returns a single GitOps source with its connection settings and linked workflows.
Returns a single GitOps source with its connection settings and access rules.
**Access policy**: authenticated
operationId: GitOpsSourceGet
parameters:
@@ -13004,6 +12985,41 @@ paths:
summary: Test the connection of a stored source
tags:
- gitops
/gitops/sources/{id}/workflows:
get:
description: |-
Returns the workflows (stacks or edge stacks) currently deployed from this source.
**Access policy**: authenticated
operationId: GitOpsSourceWorkflowsList
parameters:
- description: Source identifier
in: path
name: id
required: true
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
items:
$ref: '#/definitions/sources.Workflow'
type: array
"400":
description: Invalid request
"403":
description: Access denied
"404":
description: Source not found
"500":
description: Server error
security:
- ApiKeyAuth: []
- jwt: []
summary: List the workflows using a GitOps source
tags:
- gitops
/gitops/sources/git:
post:
consumes:
+12 -26
View File
@@ -1,8 +1,6 @@
package workflows
import (
"slices"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
@@ -116,11 +114,8 @@ func loadAccessibleStackMap(
continue
}
if stack.Type == portainer.KubernetesStack {
access := accessMap[stack.EndpointID]
if !access.IsKubeAdmin && !slices.Contains(access.NonAdminNamespaces, stack.Namespace) {
continue
}
if !isK8SNamespaceAccessible(stack, accessMap) {
continue
}
result[stack.ID] = stack
@@ -135,33 +130,27 @@ type SourceStats struct {
EndpointIDs set.Set[portainer.EndpointID]
}
// FetchSourceStats returns all sources and per-source stats for sources accessible to the given user.
// It applies the same access control as FetchWorkflows but skips git phase checks.
// FetchSourceStats returns per-source stats for sources accessible to the given user.
// It applies the same access control as FetchWorkflows
func FetchSourceStats(
tx dataservices.DataStoreTx,
k8sFactory *cli.ClientFactory,
sc *security.RestrictedRequestContext,
) ([]portainer.Source, map[portainer.SourceID]SourceStats, error) {
userContext := source.NewUserContext(sc.User, sc.UserMemberships)
sources, err := tx.Source().ReadAll(userContext)
if err != nil {
return nil, nil, err
}
) (map[portainer.SourceID]SourceStats, error) {
allStacks, err := tx.Stack().ReadAll(func(s portainer.Stack) bool { return s.WorkflowID != 0 })
if err != nil {
return nil, nil, err
return nil, err
}
endpointMap, err := BuildEndpointMap(tx, allStacks)
if err != nil {
return nil, nil, err
return nil, err
}
allStacks, err = FilterDockerStacksByAccess(tx, allStacks, sc)
if err != nil {
return nil, nil, err
return nil, err
}
workflowIDSet := make(set.Set[portainer.WorkflowID], len(allStacks))
@@ -176,7 +165,7 @@ func FetchSourceStats(
wfMap, err := LoadWorkflowMap(tx, workflowIDSet)
if err != nil {
return nil, nil, err
return nil, err
}
wfSources := make(map[portainer.WorkflowID][]portainer.SourceID, len(wfMap))
@@ -197,13 +186,10 @@ func FetchSourceStats(
accessMap, err := buildEndpointAccessMap(k8sFactory, sc, endpointMap)
if err != nil {
return nil, nil, err
return nil, err
}
stacks, err := filterK8SStacks(preFiltered, endpointMap, k8sFactory, accessMap)
if err != nil {
return nil, nil, err
}
stacks := filterK8SStacks(preFiltered, accessMap)
stats := make(map[portainer.SourceID]SourceStats)
@@ -215,7 +201,7 @@ func FetchSourceStats(
addSourceStats(stats, stackSourceIDs[stack.ID], epIDs)
}
return sources, stats, nil
return stats, nil
}
func addSourceStats(result map[portainer.SourceID]SourceStats, srcIDs []portainer.SourceID, epIDs []portainer.EndpointID) {
+2 -24
View File
@@ -290,28 +290,6 @@ func TestFetchWorkflows_HidesEmptyWorkflowWhenEndpointFilterActive(t *testing.T)
require.Empty(t, items)
}
func TestFetchSourceStats_ReturnsAllSources(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
require.NoError(t, tx.Source().Create(adminUserContext, &portainer.Source{Name: "source-1", Type: portainer.SourceTypeGit, Git: &gittypes.GitSource{URL: "http://github.com/org/repo1"}}))
require.NoError(t, tx.Source().Create(adminUserContext, &portainer.Source{Name: "source-2", Type: portainer.SourceTypeGit, Git: &gittypes.GitSource{URL: "http://github.com/org/repo2"}}))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
var sources []portainer.Source
require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
sources, _, err = FetchSourceStats(tx, nil, adminContext())
return err
}))
require.Len(t, sources, 2)
}
func TestFetchSourceStats_TracksWorkflowCountAndEndpoints(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
@@ -340,7 +318,7 @@ func TestFetchSourceStats_TracksWorkflowCountAndEndpoints(t *testing.T) {
var stats map[portainer.SourceID]SourceStats
require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
_, stats, err = FetchSourceStats(tx, nil, adminContext())
stats, err = FetchSourceStats(tx, nil, adminContext())
return err
}))
@@ -367,7 +345,7 @@ func TestFetchSourceStats_UnusedSourceHasZeroStats(t *testing.T) {
var stats map[portainer.SourceID]SourceStats
require.NoError(t, store.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
_, stats, err = FetchSourceStats(tx, nil, adminContext())
stats, err = FetchSourceStats(tx, nil, adminContext())
return err
}))
+17 -54
View File
@@ -3,11 +3,9 @@ package workflows
import (
"fmt"
"slices"
"strconv"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/models/kubernetes"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/internal/endpointutils"
@@ -142,57 +140,22 @@ func buildEndpointAccessMap(k8sFactory *cli.ClientFactory, sc *security.Restrict
return result, nil
}
// lookup only if env is kube and either not edge or (edge + not async)
func ShouldPerformEnvLookup(endpoint *portainer.Endpoint) bool {
return endpointutils.IsKubernetesEndpoint(endpoint) &&
(!endpointutils.IsEdgeEndpoint(endpoint) ||
(endpointutils.IsEdgeEndpoint(endpoint) && !endpoint.Edge.AsyncMode))
}
func filterK8SStacks(items []portainer.Stack, endpointMap map[portainer.EndpointID]portainer.Endpoint, k8sFactory *cli.ClientFactory, accessMap map[portainer.EndpointID]endpointAccess) ([]portainer.Stack, error) {
k8sStacks, result := slicesx.Partition(items, func(s portainer.Stack) bool {
return s.Type == portainer.KubernetesStack
})
groupedByEnvId := slicesx.GroupBy(k8sStacks, func(s portainer.Stack) portainer.EndpointID {
return s.EndpointID
})
for envID, stacks := range groupedByEnvId {
ep, ok := endpointMap[envID]
if !ok || !ShouldPerformEnvLookup(&ep) {
continue
}
kcl, err := k8sFactory.GetPrivilegedKubeClient(&ep)
if err != nil {
log.Warn().Err(err).Str("context", "filterK8SStacks").Int("endpoint_id", int(envID)).Msg("Failed to get kube client for endpoint, skipping")
continue
}
access := accessMap[envID]
kcl.SetIsKubeAdmin(access.IsKubeAdmin)
kcl.SetClientNonAdminNamespaces(access.NonAdminNamespaces)
apps, err := kcl.GetApplications("", "")
if err != nil {
log.Warn().Err(err).Str("context", "filterK8SStacks").Int("endpoint_id", int(envID)).Msg("Failed to get kube applications for endpoint, skipping")
continue
}
for _, s := range stacks {
idx := slices.IndexFunc(apps, func(app kubernetes.K8sApplication) bool {
return app.StackKind != "edge" && app.StackID == strconv.Itoa(int(s.ID))
})
if idx == -1 {
continue
}
app := apps[idx]
s.Name = app.Name
s.Namespace = app.ResourcePool
result = append(result, s)
}
// isK8SNamespaceAccessible reports whether a Kubernetes stack's stored namespace is visible to the
// user given the resolved endpoint access. Non-Kubernetes stacks always pass.
func isK8SNamespaceAccessible(stack portainer.Stack, accessMap map[portainer.EndpointID]endpointAccess) bool {
if stack.Type != portainer.KubernetesStack {
return true
}
return result, nil
access := accessMap[stack.EndpointID]
return access.IsKubeAdmin || slices.Contains(access.NonAdminNamespaces, stack.Namespace)
}
// filterK8SStacks drops Kubernetes stacks whose namespace is not accessible to the user. It relies
// on the stored stack namespace rather than querying the cluster, matching the workflow list's
// access filtering. Docker stacks pass through unchanged.
func filterK8SStacks(items []portainer.Stack, accessMap map[portainer.EndpointID]endpointAccess) []portainer.Stack {
return slicesx.Filter(items, func(s portainer.Stack) bool {
return isK8SNamespaceAccessible(s, accessMap)
})
}
+15 -106
View File
@@ -10,8 +10,6 @@ import (
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/kubernetes/cli"
appsv1 "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kfake "k8s.io/client-go/kubernetes/fake"
"github.com/stretchr/testify/assert"
@@ -89,113 +87,50 @@ func TestBuildEndpointAccessMap_AdminIsKubeAdmin(t *testing.T) {
require.Empty(t, result[1].NonAdminNamespaces)
}
func TestFilterK8SStacks_IncludesMatchingStack(t *testing.T) {
func TestFilterK8SStacks_AdminIncludesAllK8SStacks(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "default",
Labels: map[string]string{
"io.portainer.kubernetes.application.stackid": "1",
},
},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "my-app"}},
},
}
_, err := fakeKubeClient.AppsV1().Deployments("default").Create(t.Context(), deployment, metav1.CreateOptions{})
require.NoError(t, err)
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
endpointMap := map[portainer.EndpointID]portainer.Endpoint{
1: {ID: 1, Type: portainer.KubernetesLocalEnvironment},
}
stacks := []portainer.Stack{
{ID: 1, Name: "stack-name", EndpointID: 1, Type: portainer.KubernetesStack},
{ID: 1, Name: "stack-name", EndpointID: 1, Namespace: "default", Type: portainer.KubernetesStack},
}
accessMap := map[portainer.EndpointID]endpointAccess{
1: {IsKubeAdmin: true},
}
result, err := filterK8SStacks(stacks, endpointMap, factory, accessMap)
require.NoError(t, err)
result := filterK8SStacks(stacks, accessMap)
require.Len(t, result, 1)
assert.Equal(t, "my-app", result[0].Name)
assert.Equal(t, "stack-name", result[0].Name)
assert.Equal(t, "default", result[0].Namespace)
}
func TestFilterK8SStacks_ExcludesStackWhenNoMatchingDeployment(t *testing.T) {
func TestFilterK8SStacks_DockerStacksPassThrough(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
endpointMap := map[portainer.EndpointID]portainer.Endpoint{
1: {ID: 1, Type: portainer.KubernetesLocalEnvironment},
}
stacks := []portainer.Stack{
{ID: 1, Name: "stack-name", EndpointID: 1, Type: portainer.KubernetesStack},
{ID: 1, Name: "docker-stack", EndpointID: 1, Type: portainer.DockerComposeStack},
}
accessMap := map[portainer.EndpointID]endpointAccess{
1: {IsKubeAdmin: true},
}
result, err := filterK8SStacks(stacks, endpointMap, factory, accessMap)
require.NoError(t, err)
require.Empty(t, result)
// No access is resolved for the endpoint; Docker stacks must still pass through.
result := filterK8SStacks(stacks, map[portainer.EndpointID]endpointAccess{})
require.Len(t, result, 1)
assert.Equal(t, "docker-stack", result[0].Name)
}
func TestFilterK8SStacks_NonAdminWithNamespaceAccess(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "ns1",
Labels: map[string]string{
"io.portainer.kubernetes.application.stackid": "1",
},
},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "my-app"}},
},
}
_, err := fakeKubeClient.AppsV1().Deployments("ns1").Create(t.Context(), deployment, metav1.CreateOptions{})
require.NoError(t, err)
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
endpointMap := map[portainer.EndpointID]portainer.Endpoint{
1: {ID: 1, Type: portainer.KubernetesLocalEnvironment},
}
stacks := []portainer.Stack{
{ID: 1, Name: "stack-name", EndpointID: 1, Type: portainer.KubernetesStack},
{ID: 1, Name: "stack-name", EndpointID: 1, Namespace: "ns1", Type: portainer.KubernetesStack},
}
accessMap := map[portainer.EndpointID]endpointAccess{
1: {IsKubeAdmin: false, NonAdminNamespaces: []string{"ns1"}},
}
result, err := filterK8SStacks(stacks, endpointMap, factory, accessMap)
require.NoError(t, err)
result := filterK8SStacks(stacks, accessMap)
require.Len(t, result, 1)
assert.Equal(t, "my-app", result[0].Name)
assert.Equal(t, "stack-name", result[0].Name)
}
func TestResolveKubeAccess_NonAdminWithTeamMemberships(t *testing.T) {
@@ -250,40 +185,14 @@ func TestResolveKubeAccess_NonAdmin(t *testing.T) {
func TestFilterK8SStacks_NonAdminWithoutNamespaceAccess(t *testing.T) {
t.Parallel()
fakeKubeClient := kfake.NewSimpleClientset()
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "ns1",
Labels: map[string]string{
"io.portainer.kubernetes.application.stackid": "1",
},
},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "my-app"}},
},
}
_, err := fakeKubeClient.AppsV1().Deployments("ns1").Create(t.Context(), deployment, metav1.CreateOptions{})
require.NoError(t, err)
kcl := cli.NewTestKubeClient(fakeKubeClient)
factory := cli.NewTestClientFactory(1, kcl)
endpointMap := map[portainer.EndpointID]portainer.Endpoint{
1: {ID: 1, Type: portainer.KubernetesLocalEnvironment},
}
stacks := []portainer.Stack{
{ID: 1, Name: "stack-name", EndpointID: 1, Type: portainer.KubernetesStack},
{ID: 1, Name: "stack-name", EndpointID: 1, Namespace: "ns1", Type: portainer.KubernetesStack},
}
accessMap := map[portainer.EndpointID]endpointAccess{
1: {IsKubeAdmin: false, NonAdminNamespaces: []string{}},
}
result, err := filterK8SStacks(stacks, endpointMap, factory, accessMap)
require.NoError(t, err)
result := filterK8SStacks(stacks, accessMap)
require.Empty(t, result)
}
+32 -70
View File
@@ -6,7 +6,6 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/gitops/sources"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/set"
@@ -30,22 +29,29 @@ func BuildGroupEndpoints(tx dataservices.DataStoreTx, groups []portainer.EdgeGro
return m, nil
}
// MapStackToSourceWorkflow converts a stack to a SourceWorkflow. gitConfig is passed separately
// because EE embeds a different GitConfig type that shadows the CE field.
// source and artifact are the pre-computed git phase statuses from the caller.
func MapStackToSourceWorkflow(s portainer.Stack, sourceID portainer.SourceID, gitConfig *gittypes.RepoConfig, source, artifact WorkflowPhaseStatus) SourceWorkflow {
return SourceWorkflow{
ID: s.WorkflowID,
Name: s.Name,
// WorkflowMappingFields holds the fields shared between ArtifactDetail and the sources handler package's Workflow type.
type WorkflowMappingFields struct {
Type Type `json:"type" validate:"required"`
Name string `json:"name" validate:"required"`
Platform DeploymentPlatform `json:"platform"`
Status WorkflowStatusObject `json:"status"`
AutoUpdate *portainer.AutoUpdateSettings `json:"autoUpdate,omitempty"`
Target Target `json:"target"`
CreationDate int64 `json:"creationDate"`
LastSyncDate int64 `json:"lastSyncDate"`
}
// DeriveStackWorkflowFields computes the shared WorkflowMappingFields for a Stack-backed deployment.
func DeriveStackWorkflowFields(s portainer.Stack, source, artifact WorkflowPhaseStatus) WorkflowMappingFields {
return WorkflowMappingFields{
Type: TypeStack,
Name: s.Name,
Platform: platformFromStackType(s.Type),
Status: WorkflowStatusObject{
Source: source,
Artifact: artifact,
Target: deriveStackTargetState(s),
},
SourceID: sourceID,
GitConfig: gitConfig,
AutoUpdate: s.AutoUpdate,
Target: Target{
EndpointID: s.EndpointID,
@@ -56,26 +62,21 @@ func MapStackToSourceWorkflow(s portainer.Stack, sourceID portainer.SourceID, gi
}
}
// MapEdgeStackToSourceWorkflow converts an edge stack to a SourceWorkflow. gitConfig is passed separately
// because EE embeds a different GitConfig type that shadows the CE field.
// source and artifact are the pre-computed git phase statuses from the caller.
func MapEdgeStackToSourceWorkflow(wfID portainer.WorkflowID, es portainer.EdgeStack, sourceID portainer.SourceID, gitConfig *gittypes.RepoConfig, statuses []portainer.EdgeStackStatusForEnv, groupEndpoints map[portainer.EdgeGroupID][]portainer.EndpointID, source, artifact WorkflowPhaseStatus) SourceWorkflow {
// DeriveEdgeStackWorkflowFields computes the shared WorkflowMappingFields for an EdgeStack-backed deployment.
func DeriveEdgeStackWorkflowFields(es portainer.EdgeStack, statuses []portainer.EdgeStackStatusForEnv, groupEndpoints map[portainer.EdgeGroupID][]portainer.EndpointID, source, artifact WorkflowPhaseStatus) WorkflowMappingFields {
platform := DeploymentPlatformDockerStandalone
if es.DeploymentType == portainer.EdgeStackDeploymentKubernetes {
platform = DeploymentPlatformKubernetes
}
return SourceWorkflow{
ID: wfID,
Name: es.Name,
return WorkflowMappingFields{
Type: TypeEdgeStack,
Name: es.Name,
Platform: platform,
Status: WorkflowStatusObject{
Source: source,
Artifact: artifact,
Target: deriveEdgeStackTargetState(statuses),
},
SourceID: sourceID,
GitConfig: gitConfig,
Target: Target{
EdgeGroupIDs: es.EdgeGroups,
GroupStatus: edgeStackTargetStatuses(es.EdgeGroups, statuses, groupEndpoints),
@@ -86,55 +87,21 @@ func MapEdgeStackToSourceWorkflow(wfID portainer.WorkflowID, es portainer.EdgeSt
}
}
// MapStackToArtifactDetail converts a stack to an ArtifactDetail. source and artifact are the
// pre-computed git phase statuses from the caller; files are the artifact's resolved file refs.
// MapStackToArtifactDetail converts a stack to an ArtifactDetail.
func MapStackToArtifactDetail(stack portainer.Stack, files []portainer.ArtifactFile, source, artifact WorkflowPhaseStatus) ArtifactDetail {
return ArtifactDetail{
ID: int(stack.ID),
Type: TypeStack,
Name: stack.Name,
Platform: platformFromStackType(stack.Type),
Status: WorkflowStatusObject{
Source: source,
Artifact: artifact,
Target: deriveStackTargetState(stack),
},
AutoUpdate: stack.AutoUpdate,
Target: Target{
EndpointID: stack.EndpointID,
Namespace: stack.Namespace,
},
Files: mapFilesToFileDetails(files),
CreationDate: stack.CreationDate,
LastSyncDate: StackLastSyncDate(stack),
ID: int(stack.ID),
WorkflowMappingFields: DeriveStackWorkflowFields(stack, source, artifact),
Files: mapFilesToFileDetails(files),
}
}
// MapEdgeStackToArtifactDetail converts an edge stack to an ArtifactDetail. source and artifact are
// the pre-computed git phase statuses from the caller; files are the artifact's resolved file refs.
// MapEdgeStackToArtifactDetail converts an edge stack to an ArtifactDetail.
func MapEdgeStackToArtifactDetail(es portainer.EdgeStack, files []portainer.ArtifactFile, statuses []portainer.EdgeStackStatusForEnv, groupEndpoints map[portainer.EdgeGroupID][]portainer.EndpointID, source, artifact WorkflowPhaseStatus) ArtifactDetail {
platform := DeploymentPlatformDockerStandalone
if es.DeploymentType == portainer.EdgeStackDeploymentKubernetes {
platform = DeploymentPlatformKubernetes
}
return ArtifactDetail{
ID: int(es.ID),
Type: TypeEdgeStack,
Name: es.Name,
Platform: platform,
Status: WorkflowStatusObject{
Source: source,
Artifact: artifact,
Target: deriveEdgeStackTargetState(statuses),
},
Target: Target{
EdgeGroupIDs: es.EdgeGroups,
GroupStatus: edgeStackTargetStatuses(es.EdgeGroups, statuses, groupEndpoints),
ResolvedEndpointIDs: resolveEdgeGroupEndpoints(es.EdgeGroups, groupEndpoints),
},
Files: mapFilesToFileDetails(files),
CreationDate: es.CreationDate,
LastSyncDate: edgeStackLastSyncDate(statuses),
ID: int(es.ID),
WorkflowMappingFields: DeriveEdgeStackWorkflowFields(es, statuses, groupEndpoints, source, artifact),
Files: mapFilesToFileDetails(files),
}
}
@@ -227,8 +194,7 @@ func edgeStackTargetStatuses(
return result
}
// BuildWorkflow assembles a Workflow from a domain workflow and its resolved, access-filtered
// artifacts, aggregating the workflow-level status and dates across those artifacts.
// BuildWorkflow assembles a Workflow from a domain workflow and its resolved, access-filtered artifacts.
func BuildWorkflow(wf portainer.Workflow, artifacts []ArtifactDetail) Workflow {
creation, lastSync := SummaryDates(artifacts)
return Workflow{
@@ -241,8 +207,7 @@ func BuildWorkflow(wf portainer.Workflow, artifacts []ArtifactDetail) Workflow {
}
}
// workflowName returns the workflow's stored name, falling back to a placeholder when it has
// none (e.g. a Kubernetes stack deployed without a stack name).
// workflowName returns the workflow's stored name, falling back to a placeholder when it has none.
func workflowName(wf portainer.Workflow) string {
if wf.Name != "" {
return wf.Name
@@ -251,9 +216,7 @@ func workflowName(wf portainer.Workflow) string {
return "Unnamed workflow"
}
// ShouldHideWorkflow reports whether a workflow must be hidden from the list: it had configured
// artifacts but every one was filtered out (existence-hiding, mirroring the detail endpoint), or it
// has no artifacts while an endpoint filter is active (an empty workflow cannot match an endpoint).
// ShouldHideWorkflow reports whether a workflow must be hidden from the list (all artifacts filtered out, or none exist while an endpoint filter is active).
func ShouldHideWorkflow(wf portainer.Workflow, artifacts []ArtifactDetail, endpointIDSet set.Set[portainer.EndpointID]) bool {
if len(artifacts) > 0 {
return false
@@ -261,8 +224,7 @@ func ShouldHideWorkflow(wf portainer.Workflow, artifacts []ArtifactDetail, endpo
return len(wf.Artifacts) > 0 || len(endpointIDSet) > 0
}
// SummaryDates derives workflow-level dates from its artifacts: the earliest artifact creation
// date and the most recent artifact sync date. Zero values are ignored.
// SummaryDates derives the earliest creation date and most recent sync date across artifacts, ignoring zero values.
func SummaryDates(artifacts []ArtifactDetail) (creation, lastSync int64) {
for _, a := range artifacts {
if a.CreationDate != 0 && (creation == 0 || a.CreationDate < creation) {
-64
View File
@@ -4,7 +4,6 @@ import (
"testing"
portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -151,69 +150,6 @@ func TestEdgeStackTargetStatuses(t *testing.T) {
})
}
func TestMapEdgeStackToSourceWorkflow_DockerPlatform(t *testing.T) {
t.Parallel()
es := portainer.EdgeStack{
ID: 1,
Name: "docker-edge",
DeploymentType: portainer.EdgeStackDeploymentCompose,
EdgeGroups: []portainer.EdgeGroupID{1},
CreationDate: 1587399600,
}
cfg := &gittypes.RepoConfig{URL: "https://github.com/x/repo"}
w := MapEdgeStackToSourceWorkflow(2, es, 7, cfg, nil, map[portainer.EdgeGroupID][]portainer.EndpointID{1: {10}}, WorkflowPhaseStatus{Status: StatusHealthy}, WorkflowPhaseStatus{Status: StatusHealthy})
require.Equal(t, portainer.WorkflowID(2), w.ID)
require.Equal(t, es.Name, w.Name)
require.Equal(t, TypeEdgeStack, w.Type)
require.Equal(t, DeploymentPlatformDockerStandalone, w.Platform)
require.Equal(t, es.CreationDate, w.CreationDate)
require.Equal(t, cfg, w.GitConfig)
require.Equal(t, portainer.SourceID(7), w.SourceID)
require.Equal(t, []portainer.EdgeGroupID{1}, w.Target.EdgeGroupIDs)
}
func TestMapEdgeStackToSourceWorkflow_KubernetesPlatform(t *testing.T) {
t.Parallel()
es := portainer.EdgeStack{
ID: 2,
Name: "kube-edge",
DeploymentType: portainer.EdgeStackDeploymentKubernetes,
EdgeGroups: []portainer.EdgeGroupID{1},
}
w := MapEdgeStackToSourceWorkflow(1, es, 0, nil, nil, map[portainer.EdgeGroupID][]portainer.EndpointID{}, WorkflowPhaseStatus{Status: StatusUnknown}, WorkflowPhaseStatus{Status: StatusUnknown})
require.Equal(t, DeploymentPlatformKubernetes, w.Platform)
}
func TestMapEdgeStackToSourceWorkflow_GroupStatusesAndResolvedEndpoints(t *testing.T) {
t.Parallel()
statuses := []portainer.EdgeStackStatusForEnv{
{EndpointID: 10, Status: []portainer.EdgeStackDeploymentStatus{{Type: portainer.EdgeStackStatusRunning}}},
{EndpointID: 20, Status: []portainer.EdgeStackDeploymentStatus{{Type: portainer.EdgeStackStatusError, Error: "boom"}}},
}
groupEndpoints := map[portainer.EdgeGroupID][]portainer.EndpointID{
1: {10},
2: {20},
}
es := portainer.EdgeStack{
ID: 3,
Name: "multi-group",
EdgeGroups: []portainer.EdgeGroupID{1, 2},
}
w := MapEdgeStackToSourceWorkflow(5, es, 0, nil, statuses, groupEndpoints, WorkflowPhaseStatus{Status: StatusUnknown}, WorkflowPhaseStatus{Status: StatusUnknown})
require.Equal(t, StatusHealthy, w.Target.GroupStatus[1])
require.Equal(t, StatusError, w.Target.GroupStatus[2])
require.Len(t, w.Target.ResolvedEndpointIDs, 2)
}
func TestBuildWorkflow_NameFallback(t *testing.T) {
t.Parallel()
+6 -4
View File
@@ -12,10 +12,12 @@ func TestAggregateWorkflowStatus(t *testing.T) {
artifact := func(source, artifact, target Status) ArtifactDetail {
return ArtifactDetail{
Status: WorkflowStatusObject{
Source: WorkflowPhaseStatus{Status: source},
Artifact: WorkflowPhaseStatus{Status: artifact},
Target: WorkflowPhaseStatus{Status: target},
WorkflowMappingFields: WorkflowMappingFields{
Status: WorkflowStatusObject{
Source: WorkflowPhaseStatus{Status: source},
Artifact: WorkflowPhaseStatus{Status: artifact},
Target: WorkflowPhaseStatus{Status: target},
},
},
}
}
+3 -27
View File
@@ -4,7 +4,6 @@ import (
"fmt"
portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/gitops/sources"
)
@@ -66,7 +65,6 @@ type Target struct {
}
// WorkflowPhaseStatus represents the status of one phase (source, artifact, or target) of a workflow.
// All three phases share the Status type; source and artifact only ever emit healthy, error, or unknown.
type WorkflowPhaseStatus struct {
Status Status `json:"status"`
Error string `json:"error,omitempty"`
@@ -79,21 +77,6 @@ type WorkflowStatusObject struct {
Target WorkflowPhaseStatus `json:"target"`
}
// SourceWorkflow is the per-stack/edge-stack workflow shape embedded in a source's detail response.
type SourceWorkflow struct {
ID portainer.WorkflowID `json:"id" validate:"required"`
Name string `json:"name" validate:"required"`
Type Type `json:"type" validate:"required"`
Platform DeploymentPlatform `json:"platform" validate:"required"`
Status WorkflowStatusObject `json:"status" validate:"required"`
SourceID portainer.SourceID `json:"sourceId,omitempty"`
GitConfig *gittypes.RepoConfig `json:"gitConfig,omitempty"`
AutoUpdate *portainer.AutoUpdateSettings `json:"autoUpdate,omitempty"`
Target Target `json:"target" validate:"required"`
CreationDate int64 `json:"creationDate"`
LastSyncDate int64 `json:"lastSyncDate"`
}
// Workflow is the API representation of a workflow, used by both the list and detail endpoints.
type Workflow struct {
ID portainer.WorkflowID `json:"id" validate:"required"`
@@ -114,16 +97,9 @@ type StatusSummary struct {
// ArtifactDetail describes one Artifact's backing Stack or EdgeStack.
type ArtifactDetail struct {
ID int `json:"id" validate:"required"`
Type Type `json:"type" validate:"required"`
Name string `json:"name" validate:"required"`
Platform DeploymentPlatform `json:"platform"`
AutoUpdate *portainer.AutoUpdateSettings `json:"autoUpdate,omitempty"`
Target Target `json:"target"`
Status WorkflowStatusObject `json:"status"`
Files []ArtifactFileDetail `json:"files"`
CreationDate int64 `json:"creationDate"`
LastSyncDate int64 `json:"lastSyncDate"`
ID int `json:"id" validate:"required"`
WorkflowMappingFields
Files []ArtifactFileDetail `json:"files"`
}
// ArtifactFileDetail describe the representation of portainer.ArtifactFile used in API responses.
+3 -3
View File
@@ -11,7 +11,7 @@ import (
)
// FetchSourceWorkflows returns the workflows and stats for a single source.
func FetchSourceWorkflows(tx dataservices.DataStoreTx, src *portainer.Source) ([]ce.SourceWorkflow, ce.SourceStats, error) {
func FetchSourceWorkflows(tx dataservices.DataStoreTx, src *portainer.Source) ([]Workflow, ce.SourceStats, error) {
wfs, err := tx.Workflow().ReadAll(func(wf portainer.Workflow) bool {
return slices.ContainsFunc(wf.Artifacts, func(artifact portainer.Artifact) bool {
return slices.ContainsFunc(artifact.Files, func(f portainer.ArtifactFile) bool {
@@ -56,7 +56,7 @@ func FetchSourceWorkflows(tx dataservices.DataStoreTx, src *portainer.Source) ([
}
unknown := ce.WorkflowPhaseStatus{Status: ce.StatusUnknown}
items := make([]ce.SourceWorkflow, 0, len(stacks))
items := make([]Workflow, 0, len(stacks))
stats := ce.SourceStats{EndpointIDs: set.Set[portainer.EndpointID]{}}
for _, stack := range stacks {
@@ -66,7 +66,7 @@ func FetchSourceWorkflows(tx dataservices.DataStoreTx, src *portainer.Source) ([
cfg.ConfigFilePath = file.Path
cfg.ConfigHash = file.Hash
}
items = append(items, ce.MapStackToSourceWorkflow(stack, src.ID, cfg, unknown, unknown))
items = append(items, MapStackToWorkflow(stack, src.ID, cfg, unknown, unknown))
stats.WorkflowCount++
if stack.EndpointID != 0 {
stats.EndpointIDs.Add(stack.EndpointID)
+17 -64
View File
@@ -1,14 +1,12 @@
package sources
import (
"errors"
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
sourceDS "github.com/portainer/portainer/api/dataservices/source"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/gitops/workflows"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
@@ -24,29 +22,22 @@ type connectionInfo struct {
Authentication *gitAuthInfo `json:"authentication,omitempty"`
}
type AutoUpdateInfo struct {
Mechanism string `json:"mechanism,omitempty"`
FetchInterval string `json:"fetchInterval,omitempty"`
}
type SourceAccess struct {
Public bool `json:"public,omitempty"`
Users []portainer.UserID `json:"users,omitempty"`
Teams []portainer.TeamID `json:"teams,omitempty"`
}
// SourceDetail extends Source with connection settings and linked workflows.
// SourceDetail is the get-by-id response for a GitOps source
type SourceDetail struct {
Source
Connection connectionInfo `json:"connection" validate:"required"`
AutoUpdate *AutoUpdateInfo `json:"autoUpdate,omitempty"`
Workflows []workflows.SourceWorkflow `json:"workflows"`
Access SourceAccess `json:"access"`
SourceBase
Connection connectionInfo `json:"connection" validate:"required"`
Access SourceAccess `json:"access"`
}
// @id GitOpsSourceGet
// @summary Get a GitOps source by ID
// @description Returns a single GitOps source with its connection settings and linked workflows.
// @description Returns a single GitOps source with its connection settings and access rules.
// @description **Access policy**: authenticated
// @tags gitops
// @security ApiKeyAuth
@@ -73,46 +64,28 @@ func (h *Handler) getSource(w http.ResponseWriter, r *http.Request) *httperror.H
sourceID := portainer.SourceID(srcID)
var source *portainer.Source
var sourceWfs []workflows.SourceWorkflow
var stats workflows.SourceStats
err = h.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
userContext := sourceDS.NewUserContext(securityContext.User, securityContext.UserMemberships)
source, err = tx.Source().Read(userContext, sourceID)
if err != nil {
return err
var handlerErr *httperror.HandlerError
source, handlerErr = ReadSource(tx, userContext, sourceID)
if handlerErr != nil {
return handlerErr
}
sourceWfs, stats, err = FetchSourceWorkflows(tx, source)
return err
return nil
})
if h.dataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Source not found", err)
} else if errors.Is(err, sourceDS.ErrNotEnoughPermission) {
return httperror.Forbidden("Not enough permissions to retrieve source", err)
} else if err != nil {
return httperror.InternalServerError("Unable to retrieve source", err)
}
access := BuildSourceAccess(source)
detail := BuildSourceDetail(h.buildSource(source, stats), source.Git, sourceWfs, access)
return response.JSON(w, detail)
return response.TxFuncResponse(err, func() *httperror.HandlerError {
access := BuildSourceAccess(source)
detail := BuildSourceDetail(h.buildSourceBase(source), source.Git, access)
return response.JSON(w, detail)
})
}
func BuildSourceDetail(baseSource Source, cfg *gittypes.GitSource, sourceWfs []workflows.SourceWorkflow, access SourceAccess) SourceDetail {
var autoUpdate *AutoUpdateInfo
if len(sourceWfs) > 0 {
autoUpdate = BuildAutoUpdateInfo(sourceWfs[0].AutoUpdate)
}
func BuildSourceDetail(baseSource SourceBase, cfg *gittypes.GitSource, access SourceAccess) SourceDetail {
return SourceDetail{
Source: baseSource,
SourceBase: baseSource,
Connection: buildConnectionInfo(cfg),
AutoUpdate: autoUpdate,
Workflows: redactWorkflowCredentials(sourceWfs),
Access: access,
}
}
@@ -157,23 +130,3 @@ func buildGitAuthInfo(auth *gittypes.GitAuthentication) *gitAuthInfo {
Username: auth.Username,
}
}
func BuildAutoUpdateInfo(autoUpdate *portainer.AutoUpdateSettings) *AutoUpdateInfo {
if autoUpdate == nil {
return nil
}
switch {
case autoUpdate.Interval != "":
return &AutoUpdateInfo{
Mechanism: "Interval",
FetchInterval: autoUpdate.Interval,
}
case autoUpdate.Webhook != "":
return &AutoUpdateInfo{
Mechanism: "Webhook",
}
default:
return nil
}
}
@@ -53,65 +53,5 @@ func TestGetSource_ReturnsDetail(t *testing.T) {
detail := decodeSourceDetail(t, rr)
assert.Equal(t, srcID, detail.ID)
assert.Equal(t, "repo", detail.Name)
assert.Equal(t, 1, detail.UsedBy)
assert.True(t, detail.Connection.TLSSkipVerify)
require.Len(t, detail.Workflows, 1)
assert.Equal(t, "my-stack", detail.Workflows[0].Name)
}
func TestGetSource_RedactsCredentials(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
cfg := &gittypes.GitSource{
URL: "https://github.com/org/secure",
Authentication: &gittypes.GitAuthentication{Username: "user", Password: "s3cr3t"},
}
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
stack := &portainer.Stack{ID: 1, Name: "secure-stack"}
srcID = createGitWorkflow(t, tx, stack, cfg)
require.NoError(t, tx.Stack().Create(stack))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildGetReq(t, 1, strconv.Itoa(int(srcID))))
detail := decodeSourceDetail(t, rr)
require.Len(t, detail.Workflows, 1)
require.NotNil(t, detail.Workflows[0].GitConfig)
require.NotNil(t, detail.Workflows[0].GitConfig.Authentication)
assert.Equal(t, "user", detail.Workflows[0].GitConfig.Authentication.Username)
assert.Empty(t, detail.Workflows[0].GitConfig.Authentication.Password)
}
func TestGetSource_AutoUpdate(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
cfg := gitCfg("https://github.com/org/polled")
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
stack := &portainer.Stack{
ID: 1,
Name: "polled-stack",
AutoUpdate: &portainer.AutoUpdateSettings{Interval: "5m"},
}
srcID = createGitWorkflow(t, tx, stack, cfg)
require.NoError(t, tx.Stack().Create(stack))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildGetReq(t, 1, strconv.Itoa(int(srcID))))
detail := decodeSourceDetail(t, rr)
require.NotNil(t, detail.AutoUpdate)
assert.Equal(t, "Interval", detail.AutoUpdate.Mechanism)
assert.Equal(t, "5m", detail.AutoUpdate.FetchInterval)
}
@@ -45,6 +45,7 @@ func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStor
authenticatedRouter.Handle("", httperror.LoggerHandler(h.list)).Methods(http.MethodGet)
authenticatedRouter.Handle("/summary", httperror.LoggerHandler(h.summary)).Methods(http.MethodGet)
authenticatedRouter.Handle("/{id}", httperror.LoggerHandler(h.getSource)).Methods(http.MethodGet)
authenticatedRouter.Handle("/{id}/workflows", httperror.LoggerHandler(h.listSourceWorkflows)).Methods(http.MethodGet)
authenticatedRouter.Handle("/git", httperror.LoggerHandler(h.gitSourceCreate)).Methods(http.MethodPost)
authenticatedRouter.Handle("/test", httperror.LoggerHandler(h.gitSourceTest)).Methods(http.MethodPost)
authenticatedRouter.Handle("/{id}", httperror.LoggerHandler(h.gitSourceUpdate)).Methods(http.MethodPut)
@@ -106,6 +106,20 @@ func decodeSourceDetail(t *testing.T, rr *httptest.ResponseRecorder) SourceDetai
return item
}
func buildGetWorkflowsReq(t *testing.T, userID portainer.UserID, id string) *http.Request {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/gitops/sources/"+id+"/workflows", nil)
return withSecurityContext(req, userID)
}
func decodeSourceWorkflows(t *testing.T, rr *httptest.ResponseRecorder) []Workflow {
t.Helper()
require.Equal(t, http.StatusOK, rr.Code, "unexpected status: %s", rr.Body.String())
var items []Workflow
require.NoError(t, json.NewDecoder(rr.Body).Decode(&items))
return items
}
func gitCfg(url string) *gittypes.GitSource {
return &gittypes.GitSource{URL: url}
}
+18 -2
View File
@@ -9,6 +9,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/gitops/workflows"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/http/utils/filters"
@@ -20,6 +21,13 @@ import (
gocache "github.com/patrickmn/go-cache"
)
// Source is the list item response for a GitOps source
type Source struct {
SourceBase
UsedBy int `json:"usedBy"`
Environments int `json:"environments"`
}
// @id GitOpsSourcesList
// @summary List all GitOps sources
// @description Returns a deduplicated list of git repositories used across all GitOps workflows.
@@ -114,8 +122,16 @@ func (h *Handler) fetchSources(ctx context.Context, sc *security.RestrictedReque
var stats map[portainer.SourceID]workflows.SourceStats
if err := h.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
var err error
allSrcs, stats, err = workflows.FetchSourceStats(tx, h.k8sFactory, sc)
userContext := source.NewUserContext(sc.User, sc.UserMemberships)
sources, err := tx.Source().ReadAll(userContext)
if err != nil {
return err
}
allSrcs = sources
stats, err = workflows.FetchSourceStats(tx, h.k8sFactory, sc)
return err
}); err != nil {
return nil, err
@@ -0,0 +1,60 @@
package sources
import (
"net/http"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
sourceDS "github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
)
// @id GitOpsSourceWorkflowsList
// @summary List the workflows using a GitOps source
// @description Returns the workflows (stacks or edge stacks) currently deployed from this source.
// @description **Access policy**: authenticated
// @tags gitops
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "Source identifier"
// @success 200 {array} Workflow
// @failure 400 "Invalid request"
// @failure 403 "Access denied"
// @failure 404 "Source not found"
// @failure 500 "Server error"
// @router /gitops/sources/{id}/workflows [get]
func (h *Handler) listSourceWorkflows(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
srcID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid source identifier route variable", err)
}
securityContext, err := security.RetrieveRestrictedRequestContext(r)
if err != nil {
return httperror.InternalServerError("Unable to retrieve info from request context", err)
}
sourceID := portainer.SourceID(srcID)
var sourceWfs []Workflow
err = h.dataStore.ViewTx(func(tx dataservices.DataStoreTx) error {
userContext := sourceDS.NewUserContext(securityContext.User, securityContext.UserMemberships)
source, handlerErr := ReadSource(tx, userContext, sourceID)
if handlerErr != nil {
return handlerErr
}
var err error
sourceWfs, _, err = FetchSourceWorkflows(tx, source)
return err
})
return response.TxFuncResponse(err, func() *httperror.HandlerError {
return response.JSON(w, RedactWorkflowCredentials(sourceWfs))
})
}
@@ -0,0 +1,118 @@
package sources
import (
"net/http"
"net/http/httptest"
"strconv"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/datastore"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestListSourceWorkflows_NotFound(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildGetWorkflowsReq(t, 1, "999"))
assert.Equal(t, http.StatusNotFound, rr.Code)
}
func TestListSourceWorkflows_Forbidden(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
src := &portainer.Source{
Name: "private-repo",
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{URL: "https://github.com/org/private-repo"},
AdministratorsOnly: true,
}
require.NoError(t, tx.Source().Create(adminUserContext, src))
srcID = src.ID
require.NoError(t, tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole}))
return tx.User().Create(&portainer.User{ID: 2, Role: portainer.StandardUserRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/gitops/sources/"+strconv.Itoa(int(srcID))+"/workflows", nil)
req = req.WithContext(security.StoreTokenData(req, &portainer.TokenData{ID: 2}))
req = req.WithContext(security.StoreRestrictedRequestContext(req, &security.RestrictedRequestContext{
UserID: 2,
User: &portainer.User{ID: 2, Role: portainer.StandardUserRole},
}))
h.ServeHTTP(rr, req)
assert.Equal(t, http.StatusForbidden, rr.Code)
}
func TestListSourceWorkflows_ReturnsWorkflows(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
cfg := &gittypes.GitSource{
URL: "https://github.com/org/repo",
TLSSkipVerify: true,
}
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
stack := &portainer.Stack{ID: 1, Name: "my-stack"}
srcID = createGitWorkflow(t, tx, stack, cfg)
require.NoError(t, tx.Stack().Create(stack))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildGetWorkflowsReq(t, 1, strconv.Itoa(int(srcID))))
items := decodeSourceWorkflows(t, rr)
require.Len(t, items, 1)
assert.Equal(t, "my-stack", items[0].Name)
assert.Equal(t, srcID, items[0].SourceID)
}
func TestListSourceWorkflows_RedactsCredentials(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
cfg := &gittypes.GitSource{
URL: "https://github.com/org/secure",
Authentication: &gittypes.GitAuthentication{Username: "user", Password: "s3cr3t"},
}
var srcID portainer.SourceID
require.NoError(t, store.UpdateTx(func(tx dataservices.DataStoreTx) error {
stack := &portainer.Stack{ID: 1, Name: "secure-stack"}
srcID = createGitWorkflow(t, tx, stack, cfg)
require.NoError(t, tx.Stack().Create(stack))
return tx.User().Create(&portainer.User{ID: 1, Role: portainer.AdministratorRole})
}))
h := newTestHandler(t, store)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, buildGetWorkflowsReq(t, 1, strconv.Itoa(int(srcID))))
items := decodeSourceWorkflows(t, rr)
require.Len(t, items, 1)
require.NotNil(t, items[0].GitConfig)
require.NotNil(t, items[0].GitConfig.Authentication)
assert.Equal(t, "user", items[0].GitConfig.Authentication.Username)
assert.Empty(t, items[0].GitConfig.Authentication.Password)
}
@@ -0,0 +1,55 @@
package sources
import (
portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/gitops/workflows"
)
// Workflow is the per-stack/edge-stack workflow shape returned by a source's workflows endpoint.
type Workflow struct {
ID portainer.WorkflowID `json:"id" validate:"required"`
Name string `json:"name" validate:"required"`
Type workflows.Type `json:"type" validate:"required"`
Platform workflows.DeploymentPlatform `json:"platform" validate:"required"`
Status workflows.WorkflowStatusObject `json:"status" validate:"required"`
SourceID portainer.SourceID `json:"sourceId,omitempty"`
GitConfig *gittypes.RepoConfig `json:"gitConfig,omitempty"`
Target workflows.Target `json:"target" validate:"required"`
CreationDate int64 `json:"creationDate"`
LastSyncDate int64 `json:"lastSyncDate"`
}
// MapStackToWorkflow converts a stack to a Workflow
func MapStackToWorkflow(s portainer.Stack, sourceID portainer.SourceID, gitConfig *gittypes.RepoConfig, source, artifact workflows.WorkflowPhaseStatus) Workflow {
f := workflows.DeriveStackWorkflowFields(s, source, artifact)
return Workflow{
ID: s.WorkflowID,
Name: f.Name,
Type: f.Type,
Platform: f.Platform,
Status: f.Status,
SourceID: sourceID,
GitConfig: gitConfig,
Target: f.Target,
CreationDate: f.CreationDate,
LastSyncDate: f.LastSyncDate,
}
}
// MapEdgeStackToWorkflow converts an edge stack to a Workflow
func MapEdgeStackToWorkflow(wfID portainer.WorkflowID, es portainer.EdgeStack, sourceID portainer.SourceID, gitConfig *gittypes.RepoConfig, statuses []portainer.EdgeStackStatusForEnv, groupEndpoints map[portainer.EdgeGroupID][]portainer.EndpointID, source, artifact workflows.WorkflowPhaseStatus) Workflow {
f := workflows.DeriveEdgeStackWorkflowFields(es, statuses, groupEndpoints, source, artifact)
return Workflow{
ID: wfID,
Name: f.Name,
Type: f.Type,
Platform: f.Platform,
Status: f.Status,
SourceID: sourceID,
GitConfig: gitConfig,
Target: f.Target,
CreationDate: f.CreationDate,
LastSyncDate: f.LastSyncDate,
}
}
@@ -0,0 +1,74 @@
package sources
import (
"testing"
portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/gitops/workflows"
"github.com/stretchr/testify/require"
)
func TestMapEdgeStackToWorkflow_DockerPlatform(t *testing.T) {
t.Parallel()
es := portainer.EdgeStack{
ID: 1,
Name: "docker-edge",
DeploymentType: portainer.EdgeStackDeploymentCompose,
EdgeGroups: []portainer.EdgeGroupID{1},
CreationDate: 1587399600,
}
cfg := &gittypes.RepoConfig{URL: "https://github.com/x/repo"}
w := MapEdgeStackToWorkflow(2, es, 7, cfg, nil, map[portainer.EdgeGroupID][]portainer.EndpointID{1: {10}}, workflows.WorkflowPhaseStatus{Status: workflows.StatusHealthy}, workflows.WorkflowPhaseStatus{Status: workflows.StatusHealthy})
require.Equal(t, portainer.WorkflowID(2), w.ID)
require.Equal(t, es.Name, w.Name)
require.Equal(t, workflows.TypeEdgeStack, w.Type)
require.Equal(t, workflows.DeploymentPlatformDockerStandalone, w.Platform)
require.Equal(t, es.CreationDate, w.CreationDate)
require.Equal(t, cfg, w.GitConfig)
require.Equal(t, portainer.SourceID(7), w.SourceID)
require.Equal(t, []portainer.EdgeGroupID{1}, w.Target.EdgeGroupIDs)
}
func TestMapEdgeStackToWorkflow_KubernetesPlatform(t *testing.T) {
t.Parallel()
es := portainer.EdgeStack{
ID: 2,
Name: "kube-edge",
DeploymentType: portainer.EdgeStackDeploymentKubernetes,
EdgeGroups: []portainer.EdgeGroupID{1},
}
w := MapEdgeStackToWorkflow(1, es, 0, nil, nil, map[portainer.EdgeGroupID][]portainer.EndpointID{}, workflows.WorkflowPhaseStatus{Status: workflows.StatusUnknown}, workflows.WorkflowPhaseStatus{Status: workflows.StatusUnknown})
require.Equal(t, workflows.DeploymentPlatformKubernetes, w.Platform)
}
func TestMapEdgeStackToWorkflow_GroupStatusesAndResolvedEndpoints(t *testing.T) {
t.Parallel()
statuses := []portainer.EdgeStackStatusForEnv{
{EndpointID: 10, Status: []portainer.EdgeStackDeploymentStatus{{Type: portainer.EdgeStackStatusRunning}}},
{EndpointID: 20, Status: []portainer.EdgeStackDeploymentStatus{{Type: portainer.EdgeStackStatusError, Error: "boom"}}},
}
groupEndpoints := map[portainer.EdgeGroupID][]portainer.EndpointID{
1: {10},
2: {20},
}
es := portainer.EdgeStack{
ID: 3,
Name: "multi-group",
EdgeGroups: []portainer.EdgeGroupID{1, 2},
}
w := MapEdgeStackToWorkflow(5, es, 0, nil, statuses, groupEndpoints, workflows.WorkflowPhaseStatus{Status: workflows.StatusUnknown}, workflows.WorkflowPhaseStatus{Status: workflows.StatusUnknown})
require.Equal(t, workflows.StatusHealthy, w.Target.GroupStatus[1])
require.Equal(t, workflows.StatusError, w.Target.GroupStatus[2])
require.Len(t, w.Target.ResolvedEndpointIDs, 2)
}
+10 -12
View File
@@ -7,18 +7,16 @@ import (
"github.com/portainer/portainer/api/gitops/workflows"
)
// Source represents a unique git repository used as a GitOps source across one or more workflows.
type Source struct {
ID portainer.SourceID `json:"id" validate:"required"`
Name string `json:"name" validate:"required"`
Type SourceType `json:"type" validate:"required"`
URL string `json:"url" validate:"required"`
Status workflows.Status `json:"status" validate:"required"`
Error string `json:"error,omitempty"`
UsedBy int `json:"usedBy"`
Environments int `json:"environments"`
LastSync int64 `json:"lastSync"`
Interval string `json:"interval,omitempty" example:"5m"`
// SourceBase holds the source fields available without a workflow/stack stats scan.
type SourceBase struct {
ID portainer.SourceID `json:"id" validate:"required"`
Name string `json:"name" validate:"required"`
Type SourceType `json:"type" validate:"required"`
URL string `json:"url" validate:"required"`
Status workflows.Status `json:"status" validate:"required"`
Error string `json:"error,omitempty"`
LastSync int64 `json:"lastSync"`
Interval string `json:"interval,omitempty" example:"5m"`
}
type SourceType string
+43 -14
View File
@@ -5,8 +5,11 @@ import (
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
sourceDS "github.com/portainer/portainer/api/dataservices/source"
gittypes "github.com/portainer/portainer/api/git/types"
ce "github.com/portainer/portainer/api/gitops/workflows"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
)
const minPollingInterval = time.Minute
@@ -28,7 +31,7 @@ func validateInterval(interval string) error {
return nil
}
func (h *Handler) buildSource(src *portainer.Source, stats ce.SourceStats) Source {
func (h *Handler) buildSourceBase(src *portainer.Source) SourceBase {
phase := ce.SourceStatusToPhase(src.Status, src.StatusError)
url := ""
@@ -36,22 +39,48 @@ func (h *Handler) buildSource(src *portainer.Source, stats ce.SourceStats) Sourc
url = gittypes.SanitizeURL(src.Git.URL)
}
return Source{
ID: src.ID,
Name: src.Name,
Type: sourceTypeString(src.Type),
URL: url,
Status: phase.Status,
Error: phase.Error,
UsedBy: stats.WorkflowCount,
Environments: len(stats.EndpointIDs),
LastSync: src.LastSync,
Interval: src.Interval,
return SourceBase{
ID: src.ID,
Name: src.Name,
Type: sourceTypeString(src.Type),
URL: url,
Status: phase.Status,
Error: phase.Error,
LastSync: src.LastSync,
Interval: src.Interval,
}
}
func redactWorkflowCredentials(wfs []ce.SourceWorkflow) []ce.SourceWorkflow {
redacted := make([]ce.SourceWorkflow, len(wfs))
func (h *Handler) buildSource(src *portainer.Source, stats ce.SourceStats) Source {
return Source{
SourceBase: h.buildSourceBase(src),
UsedBy: stats.WorkflowCount,
Environments: len(stats.EndpointIDs),
}
}
// sourceStore is the minimal intersection of CE and EE DataStoreTx
type sourceStore interface {
Source() dataservices.SourceService
}
// ReadSource reads a source by ID
func ReadSource(tx sourceStore, userContext sourceDS.UserContext, sourceID portainer.SourceID) (*portainer.Source, *httperror.HandlerError) {
source, err := tx.Source().Read(userContext, sourceID)
if dataservices.IsErrObjectNotFound(err) {
return nil, httperror.NotFound("Source not found", err)
} else if errors.Is(err, sourceDS.ErrNotEnoughPermission) {
return nil, httperror.Forbidden("Not enough permissions to retrieve source", err)
} else if err != nil {
return nil, httperror.InternalServerError("Unable to retrieve source", err)
}
return source, nil
}
// RedactWorkflowCredentials returns a copy of wfs with each git credential's password cleared.
func RedactWorkflowCredentials(wfs []Workflow) []Workflow {
redacted := make([]Workflow, len(wfs))
for i, wf := range wfs {
redacted[i] = wf
if wf.GitConfig != nil && wf.GitConfig.Authentication != nil {
+7 -26
View File
@@ -3,9 +3,7 @@ package sources
import (
"testing"
portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
ce "github.com/portainer/portainer/api/gitops/workflows"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -16,10 +14,10 @@ func TestRedactWorkflowCredentials(t *testing.T) {
t.Run("clears password and preserves username", func(t *testing.T) {
t.Parallel()
wfs := []ce.SourceWorkflow{{GitConfig: &gittypes.RepoConfig{
wfs := []Workflow{{GitConfig: &gittypes.RepoConfig{
Authentication: &gittypes.GitAuthentication{Username: "user", Password: "s3cr3t"},
}}}
got := redactWorkflowCredentials(wfs)
got := RedactWorkflowCredentials(wfs)
require.NotNil(t, got[0].GitConfig.Authentication)
assert.Equal(t, "user", got[0].GitConfig.Authentication.Username)
assert.Empty(t, got[0].GitConfig.Authentication.Password)
@@ -27,42 +25,25 @@ func TestRedactWorkflowCredentials(t *testing.T) {
t.Run("does not mutate the original slice", func(t *testing.T) {
t.Parallel()
wfs := []ce.SourceWorkflow{{GitConfig: &gittypes.RepoConfig{
wfs := []Workflow{{GitConfig: &gittypes.RepoConfig{
Authentication: &gittypes.GitAuthentication{Password: "s3cr3t"},
}}}
_ = redactWorkflowCredentials(wfs)
_ = RedactWorkflowCredentials(wfs)
assert.Equal(t, "s3cr3t", wfs[0].GitConfig.Authentication.Password)
})
t.Run("nil GitConfig is safe", func(t *testing.T) {
t.Parallel()
assert.NotPanics(t, func() { redactWorkflowCredentials([]ce.SourceWorkflow{{}}) })
assert.NotPanics(t, func() { RedactWorkflowCredentials([]Workflow{{}}) })
})
t.Run("nil Authentication is safe", func(t *testing.T) {
t.Parallel()
wfs := []ce.SourceWorkflow{{GitConfig: &gittypes.RepoConfig{}}}
assert.NotPanics(t, func() { redactWorkflowCredentials(wfs) })
wfs := []Workflow{{GitConfig: &gittypes.RepoConfig{}}}
assert.NotPanics(t, func() { RedactWorkflowCredentials(wfs) })
})
}
func TestBuildAutoUpdateInfo(t *testing.T) {
t.Parallel()
assert.Nil(t, BuildAutoUpdateInfo(nil))
assert.Nil(t, BuildAutoUpdateInfo(&portainer.AutoUpdateSettings{}))
got := BuildAutoUpdateInfo(&portainer.AutoUpdateSettings{Interval: "5m"})
require.NotNil(t, got)
assert.Equal(t, "Interval", got.Mechanism)
assert.Equal(t, "5m", got.FetchInterval)
got = BuildAutoUpdateInfo(&portainer.AutoUpdateSettings{Webhook: "abc123"})
require.NotNil(t, got)
assert.Equal(t, "Webhook", got.Mechanism)
assert.Empty(t, got.FetchInterval)
}
func TestBuildConnectionInfo(t *testing.T) {
t.Parallel()
@@ -465,6 +465,9 @@ import type {
GitOpsSourcesUpdateGitData,
GitOpsSourcesUpdateGitErrors,
GitOpsSourcesUpdateGitResponses,
GitOpsSourceWorkflowsListData,
GitOpsSourceWorkflowsListErrors,
GitOpsSourceWorkflowsListResponses,
GitOpsWorkflowGetData,
GitOpsWorkflowGetErrors,
GitOpsWorkflowGetResponses,
@@ -1126,6 +1129,8 @@ import {
zGitOpsSourcesUpdateGitBody,
zGitOpsSourcesUpdateGitPath,
zGitOpsSourcesUpdateGitResponse,
zGitOpsSourceWorkflowsListPath,
zGitOpsSourceWorkflowsListResponse,
zGitOpsWorkflowGetPath,
zGitOpsWorkflowGetResponse,
zGitOpsWorkflowsListQuery,
@@ -4145,7 +4150,7 @@ export const gitOpsSourcesDelete = <ThrowOnError extends boolean = true>(
/**
* Get a GitOps source by ID
*
* Returns a single GitOps source with its connection settings and linked workflows.
* Returns a single GitOps source with its connection settings and access rules.
* **Access policy**: authenticated
*/
export const gitOpsSourceGet = <ThrowOnError extends boolean = true>(
@@ -4302,6 +4307,43 @@ export const gitOpsSourcesTestById = <ThrowOnError extends boolean = true>(
},
});
/**
* List the workflows using a GitOps source
*
* Returns the workflows (stacks or edge stacks) currently deployed from this source.
* **Access policy**: authenticated
*/
export const gitOpsSourceWorkflowsList = <ThrowOnError extends boolean = true>(
options: Options<GitOpsSourceWorkflowsListData, ThrowOnError>
): RequestResult<
GitOpsSourceWorkflowsListResponses,
GitOpsSourceWorkflowsListErrors,
ThrowOnError
> =>
(options.client ?? client).get<
GitOpsSourceWorkflowsListResponses,
GitOpsSourceWorkflowsListErrors,
ThrowOnError
>({
requestValidator: async (data) =>
await z
.object({
body: z.never().optional(),
path: zGitOpsSourceWorkflowsListPath,
query: z.never().optional(),
})
.parseAsync(data),
responseType: 'json',
responseValidator: async (data) =>
await zGitOpsSourceWorkflowsListResponse.parseAsync(data),
security: [
{ name: 'X-API-KEY', type: 'apiKey' },
{ name: 'Authorization', type: 'apiKey' },
],
url: '/gitops/sources/{id}/workflows',
...options,
});
/**
* Create a Git source
*
@@ -4353,11 +4353,6 @@ export type SettingsSettingsUpdatePayload = {
UserSessionTimeout?: string;
};
export type SourcesAutoUpdateInfo = {
fetchInterval?: string;
mechanism?: string;
};
export type SourcesConnectionTestResult = {
error?: string;
success?: boolean;
@@ -4420,9 +4415,7 @@ export type SourcesSourceAccessUpdatePayload = {
export type SourcesSourceDetail = {
access?: SourcesSourceAccess;
autoUpdate?: SourcesAutoUpdateInfo;
connection: SourcesConnectionInfo;
environments?: number;
error?: string;
id: number;
interval?: string;
@@ -4431,8 +4424,6 @@ export type SourcesSourceDetail = {
status: WorkflowsStatus;
type: SourcesSourceType;
url: string;
usedBy?: number;
workflows?: Array<WorkflowsSourceWorkflow>;
};
export const SourcesSourceType = {
@@ -4470,6 +4461,19 @@ export const SourcesStatus = {
export type SourcesStatus = (typeof SourcesStatus)[keyof typeof SourcesStatus];
export type SourcesWorkflow = {
creationDate?: number;
gitConfig?: GittypesRepoConfig;
id: number;
lastSyncDate?: number;
name: string;
platform: WorkflowsDeploymentPlatform;
sourceId?: number;
status: WorkflowsWorkflowStatusObject;
target: WorkflowsTarget;
type: WorkflowsType;
};
export type SourcesConnectionInfo = {
authentication?: SourcesGitAuthInfo;
tlsSkipVerify?: boolean;
@@ -8311,20 +8315,6 @@ export const WorkflowsDeploymentPlatform = {
export type WorkflowsDeploymentPlatform =
(typeof WorkflowsDeploymentPlatform)[keyof typeof WorkflowsDeploymentPlatform];
export type WorkflowsSourceWorkflow = {
autoUpdate?: PortainerAutoUpdateSettings;
creationDate?: number;
gitConfig?: GittypesRepoConfig;
id: number;
lastSyncDate?: number;
name: string;
platform: WorkflowsDeploymentPlatform;
sourceId?: number;
status: WorkflowsWorkflowStatusObject;
target: WorkflowsTarget;
type: WorkflowsType;
};
export const WorkflowsStatus = {
/**
* StatusHealthy
@@ -11753,6 +11743,47 @@ export type GitOpsSourcesTestByIdResponses = {
export type GitOpsSourcesTestByIdResponse =
GitOpsSourcesTestByIdResponses[keyof GitOpsSourcesTestByIdResponses];
export type GitOpsSourceWorkflowsListData = {
body?: never;
path: {
/**
* Source identifier
*/
id: number;
};
query?: never;
url: '/gitops/sources/{id}/workflows';
};
export type GitOpsSourceWorkflowsListErrors = {
/**
* Invalid request
*/
400: unknown;
/**
* Access denied
*/
403: unknown;
/**
* Source not found
*/
404: unknown;
/**
* Server error
*/
500: unknown;
};
export type GitOpsSourceWorkflowsListResponses = {
/**
* OK
*/
200: Array<SourcesWorkflow>;
};
export type GitOpsSourceWorkflowsListResponse =
GitOpsSourceWorkflowsListResponses[keyof GitOpsSourceWorkflowsListResponses];
export type GitOpsSourcesCreateGitData = {
/**
* Git source details
@@ -2026,11 +2026,6 @@ export const zSettingsSettingsUpdatePayload = z.object({
UserSessionTimeout: z.string().optional(),
});
export const zSourcesAutoUpdateInfo = z.object({
fetchInterval: z.string().optional(),
mechanism: z.string().optional(),
});
export const zSourcesConnectionTestResult = z.object({
error: z.string().optional(),
success: z.boolean().optional(),
@@ -3346,6 +3341,19 @@ export const zSourcesSource = z.object({
usedBy: z.int().optional(),
});
export const zSourcesSourceDetail = z.object({
access: zSourcesSourceAccess.optional(),
connection: zSourcesConnectionInfo,
error: z.string().optional(),
id: z.int(),
interval: z.string().optional(),
lastSync: z.int().optional(),
name: z.string(),
status: zWorkflowsStatus,
type: zSourcesSourceType,
url: z.string(),
});
export const zWorkflowsStatusSummary = z.object({
error: z.int().optional(),
healthy: z.int().optional(),
@@ -3375,21 +3383,7 @@ export const zWorkflowsWorkflowStatusObject = z.object({
target: zWorkflowsWorkflowPhaseStatus.optional(),
});
export const zWorkflowsArtifactDetail = z.object({
autoUpdate: zPortainerAutoUpdateSettings.optional(),
creationDate: z.int().optional(),
files: z.array(zWorkflowsArtifactFileDetail).optional(),
id: z.int(),
lastSyncDate: z.int().optional(),
name: z.string(),
platform: zWorkflowsDeploymentPlatform.optional(),
status: zWorkflowsWorkflowStatusObject.optional(),
target: zWorkflowsTarget.optional(),
type: zWorkflowsType,
});
export const zWorkflowsSourceWorkflow = z.object({
autoUpdate: zPortainerAutoUpdateSettings.optional(),
export const zSourcesWorkflow = z.object({
creationDate: z.int().optional(),
gitConfig: zGittypesRepoConfig.optional(),
id: z.int(),
@@ -3402,21 +3396,17 @@ export const zWorkflowsSourceWorkflow = z.object({
type: zWorkflowsType,
});
export const zSourcesSourceDetail = z.object({
access: zSourcesSourceAccess.optional(),
autoUpdate: zSourcesAutoUpdateInfo.optional(),
connection: zSourcesConnectionInfo,
environments: z.int().optional(),
error: z.string().optional(),
export const zWorkflowsArtifactDetail = z.object({
autoUpdate: zPortainerAutoUpdateSettings.optional(),
creationDate: z.int().optional(),
files: z.array(zWorkflowsArtifactFileDetail).optional(),
id: z.int(),
interval: z.string().optional(),
lastSync: z.int().optional(),
lastSyncDate: z.int().optional(),
name: z.string(),
status: zWorkflowsStatus,
type: zSourcesSourceType,
url: z.string(),
usedBy: z.int().optional(),
workflows: z.array(zWorkflowsSourceWorkflow).optional(),
platform: zWorkflowsDeploymentPlatform.optional(),
status: zWorkflowsWorkflowStatusObject.optional(),
target: zWorkflowsTarget.optional(),
type: zWorkflowsType,
});
export const zWorkflowsWorkflow = z.object({
@@ -4436,6 +4426,15 @@ export const zGitOpsSourcesTestByIdPath = z.object({
*/
export const zGitOpsSourcesTestByIdResponse = zSourcesConnectionTestResult;
export const zGitOpsSourceWorkflowsListPath = z.object({
id: z.int(),
});
/**
* OK
*/
export const zGitOpsSourceWorkflowsListResponse = z.array(zSourcesWorkflow);
/**
* Git source details
*/
@@ -16,7 +16,7 @@ import { SourceDetail, useSource } from '../queries/useSource';
import { SettingsTab } from './SettingsTab/SettingsTab';
import { WorkflowsTab } from './WorkflowsTab';
import { SourceResourceHeader } from './SourceResourceHeader';
import { CountDot } from './CountDot';
import { WorkflowsCountDot } from './WorkflowsCountDot';
import { AccessTab } from './AccessTab';
const breadcrumbs = [
@@ -43,7 +43,6 @@ export function ItemView() {
if (!source || sourceQuery.isError) {
const error = sourceQuery.error;
return (
<>
<PageHeader breadcrumbs={breadcrumbs} />
@@ -82,12 +81,11 @@ function PageContent({ source }: { source: SourceDetail }) {
{
name: (
<>
Workflows{' '}
<CountDot value={source.workflows?.length ?? 0} type="workflow" />
Workflows <WorkflowsCountDot sourceId={source.id} />
</>
),
icon: GitCommit,
widget: <WorkflowsTab workflows={source.workflows ?? []} />,
widget: <WorkflowsTab sourceId={source.id} />,
selectedTabParam: 'workflows',
},
{
@@ -1,38 +0,0 @@
import { RefreshCwIcon } from 'lucide-react';
import { Card } from '@@/primitives/Card';
import { AutoUpdateInfo } from '../../queries/useSource';
import { DetailField } from './DetailField';
interface Props {
autoUpdate?: AutoUpdateInfo;
}
export function AutoUpdateWidget({ autoUpdate }: Props) {
const mechanism = autoUpdate?.mechanism ?? '-';
const fetchInterval = autoUpdate?.fetchInterval ?? '-';
return (
<Card.Container>
<Card.Header
icon={RefreshCwIcon}
title="Change Detection"
subtitle="How Portainer detects new commits"
/>
<Card.Body>
<div className="grid grid-cols-2 gap-4">
<DetailField label="Mechanism">
<span className="text-gray-6 th-dark:text-gray-5">{mechanism}</span>
</DetailField>
<DetailField label="Fetch Interval">
<span className="text-gray-6 th-dark:text-gray-5">
{fetchInterval}
</span>
</DetailField>
</div>
</Card.Body>
</Card.Container>
);
}
@@ -2,7 +2,6 @@ import { SourceDetail } from '../../queries/useSource';
import { ConnectionDetailsWidget } from './ConnectionDetailsWidget';
import { AuthWidget } from './AuthWidget';
import { AutoUpdateWidget } from './AutoUpdateWidget';
import { PollingWidget } from './PollingWidget';
import { SyncStatusWidget } from './SyncStatusWidget';
import { SettingsForm } from './EditForm/SettingsForm';
@@ -25,7 +24,6 @@ export function SettingsTab({ source, isEditing, onEditingChange }: Props) {
<ConnectionDetailsWidget source={source} />
<AuthWidget auth={source?.connection.authentication} />
<PollingWidget interval={source.interval} />
<AutoUpdateWidget autoUpdate={source.autoUpdate} />
<SyncStatusWidget source={source} />
</>
);
@@ -1,4 +1,4 @@
import { GitBranch, GitCommitIcon, ClockIcon } from 'lucide-react';
import { GitBranch, GitCommitIcon, ClockIcon, Loader2 } from 'lucide-react';
import moment from 'moment';
import { useRouter } from '@uirouter/react';
@@ -15,6 +15,7 @@ import { StatusBadge } from '../../components/StatusBadge';
import { SOURCE_TYPES } from '../types';
import { SourceDetail } from '../queries/useSource';
import { useDeleteSourceMutation } from '../queries/useDeleteSourceMutation';
import { useSourceWorkflows } from '../queries/useSourceWorkflows';
import { TestConnectionButton } from './TestConnectionButton';
@@ -28,6 +29,9 @@ export function SourceResourceHeader({ source }: Props) {
const lastSyncLabel = source.lastSync
? moment.unix(source.lastSync).fromNow()
: '-';
const workflowsQuery = useSourceWorkflows(source.id);
const workflowsCount = workflowsQuery.data?.length;
const hasWorkflows = workflowsCount === undefined || workflowsCount > 0;
return (
<ResourceDetailHeader
@@ -49,7 +53,11 @@ export function SourceResourceHeader({ source }: Props) {
Workflows
</ResourceStatBlock.Label>
<ResourceStatBlock.Value align="center" size="base">
{source.usedBy ?? '-'}
<WorkflowsStatValue
count={workflowsCount}
isLoading={workflowsQuery.isLoading}
isError={workflowsQuery.isError}
/>
</ResourceStatBlock.Value>
</ResourceStatBlock>
<ResourceStatBlock>
@@ -72,7 +80,7 @@ export function SourceResourceHeader({ source }: Props) {
<div className="ml-auto">
<SourceDeleteButton
sourceId={source.id}
hasWorkflows={source.usedBy > 0}
hasWorkflows={hasWorkflows}
/>
</div>
</ActionBarShell>
@@ -81,6 +89,24 @@ export function SourceResourceHeader({ source }: Props) {
);
}
function WorkflowsStatValue({
count,
isLoading,
isError,
}: {
count: number | undefined;
isLoading: boolean;
isError: boolean;
}) {
if (isLoading) {
return <Icon icon={Loader2} className="animate-spin-slow" />;
}
if (isError) {
return <>-</>;
}
return <>{count}</>;
}
function SourceDeleteButton({
sourceId,
hasWorkflows,
@@ -0,0 +1,10 @@
import { useSourceWorkflows } from '../queries/useSourceWorkflows';
import { Source } from '../types';
import { CountDot } from './CountDot';
export function WorkflowsCountDot({ sourceId }: { sourceId: Source['id'] }) {
const workflowsQuery = useSourceWorkflows(sourceId);
return <CountDot value={workflowsQuery.data?.length} type="workflow" />;
}
@@ -11,37 +11,85 @@ import { WorkflowTarget, WorkflowType } from '../../workflows/types';
import { StatusBadge } from '../../components/StatusBadge';
import { getWorkflowLink } from '../../workflows/utils';
import { effectiveWorkflowStatus } from '../../workflows/status';
import { SourceWorkflow } from '../queries/useSource';
import {
SourceWorkflow,
useSourceWorkflows,
} from '../queries/useSourceWorkflows';
import { Source } from '../types';
interface Props {
workflows: SourceWorkflow[];
sourceId: Source['id'];
}
export function WorkflowsTab({ workflows }: Props) {
export function WorkflowsTab({ sourceId }: Props) {
const workflowsQuery = useSourceWorkflows(sourceId);
const workflows = workflowsQuery.data;
return (
<Card.Container>
<Card.Header
icon={GitCommitIcon}
title="Workflows"
subtitle={`${addPlural(
workflows.length,
'workflow'
)} using this source`}
subtitle={
workflows
? `${addPlural(workflows.length, 'workflow')} using this source`
: undefined
}
/>
{workflows.length === 0 ? (
<Card.Body>
<p className="text-muted text-sm">
No workflows are using this source.
</p>
</Card.Body>
) : (
<WorkflowsList workflows={workflows} />
)}
<WorkflowsBody
workflows={workflows}
isLoading={workflowsQuery.isLoading}
/>
</Card.Container>
);
}
function WorkflowsBody({
workflows,
isLoading,
}: {
workflows: Array<SourceWorkflow> | undefined;
isLoading: boolean;
}) {
if (isLoading) {
return <WorkflowsSkeleton />;
}
if (!workflows) {
return (
<Card.Body>
<p className="text-muted text-sm">Unable to load workflows.</p>
</Card.Body>
);
}
if (workflows.length === 0) {
return (
<Card.Body>
<p className="text-muted text-sm">
No workflows are using this source.
</p>
</Card.Body>
);
}
return <WorkflowsList workflows={workflows} />;
}
function WorkflowsSkeleton() {
return (
<div className="space-y-2 p-4">
{Array.from({ length: 3 }).map((_, index) => (
<div
key={index}
className="h-16 animate-pulse rounded-lg bg-gray-3 th-dark:bg-gray-8"
/>
))}
</div>
);
}
function WorkflowsList({ workflows }: { workflows: Array<SourceWorkflow> }) {
return (
<div className="space-y-2">
@@ -73,9 +121,11 @@ function WorkflowCard({ item }: { item: SourceWorkflow }) {
<StatusBadge status={effectiveWorkflowStatus(item).status} />
</div>
<div className="flex items-center gap-3">
<code className="bg-transparent p-0">
{item.gitConfig?.ConfigFilePath}
</code>
{item.gitConfig?.ConfigFilePath && (
<code className="bg-transparent p-0">
{item.gitConfig.ConfigFilePath}
</code>
)}
<span>
Last sync:{' '}
{item.lastSyncDate
@@ -3,4 +3,6 @@ export const sourceQueryKeys = {
list: (params: object) => [...sourceQueryKeys.all, 'list', params] as const,
summary: () => [...sourceQueryKeys.all, 'summary'] as const,
detail: (id: number) => [...sourceQueryKeys.all, 'detail', id] as const,
workflows: (id: number) =>
[...sourceQueryKeys.detail(id), 'workflows'] as const,
};
@@ -1,49 +1,15 @@
import { useQuery } from '@tanstack/react-query';
import {
type SourcesAutoUpdateInfo,
type SourcesSourceDetail,
WorkflowsSourceWorkflow,
WorkflowsWorkflowStatusObject,
WorkflowsStatus,
WorkflowsWorkflowPhaseStatus,
GittypesRepoConfig,
GittypesGitAuthentication,
} from '@api/types.gen';
import { type SourcesSourceDetail } from '@api/types.gen';
import { gitOpsSourceGet } from '@api/sdk.gen';
import { withError } from '@/react-tools/react-query';
import {
type RepoConfigResponse,
type GitAuthenticationResponse,
} from '@/react/portainer/gitops/types';
import { AuthTypeOption } from '@/react/portainer/account/git-credentials/types';
import { Source } from '../types';
import {
WorkflowPhaseStatus,
WorkflowStatus,
WorkflowStatusObject,
WorkflowTarget,
} from '../../workflows/types';
import { sourceQueryKeys } from './query-keys';
export type AutoUpdateInfo = SourcesAutoUpdateInfo;
export type SourceWorkflow = WorkflowsSourceWorkflow & {
status: WorkflowStatusObject;
sourceId?: number;
gitConfig?: RepoConfigResponse;
target: WorkflowTarget;
creationDate: number;
lastSyncDate: number;
};
export type SourceDetail = Omit<SourcesSourceDetail, 'workflows' | 'usedBy'> & {
workflows: Array<SourceWorkflow>;
usedBy: number;
};
export type SourceDetail = SourcesSourceDetail;
export function sourceOptions(id: Source['id']) {
return {
@@ -63,84 +29,5 @@ export function useSource(id: Source['id'] | undefined) {
export async function getSource(id: Source['id']): Promise<SourceDetail> {
const { data } = await gitOpsSourceGet({ path: { id } });
return toSourceDetails(data);
function toSourceDetails(source: SourcesSourceDetail): SourceDetail {
return {
...source,
workflows: source.workflows?.map(toWorkflow) ?? [],
usedBy: source.usedBy ?? 0,
};
function toWorkflow(workflow: WorkflowsSourceWorkflow): SourceWorkflow {
return {
...workflow,
creationDate: workflow.creationDate ?? 0,
lastSyncDate: workflow.lastSyncDate ?? 0,
status: toWorkflowStatusObject(workflow.status),
gitConfig: toWorkflowGitConfig(workflow.gitConfig),
};
}
function toWorkflowStatusObject(
statusObj: WorkflowsWorkflowStatusObject
): WorkflowStatusObject {
return {
...statusObj,
source: toPhaseStatus(statusObj.source),
artifact: toPhaseStatus(statusObj.artifact),
target: toPhaseStatus(statusObj.target),
};
}
}
function toPhaseStatus(
phaseStatus: WorkflowsWorkflowPhaseStatus | undefined
): WorkflowPhaseStatus {
return {
...phaseStatus,
status: toWorkflowStatus(phaseStatus?.status),
};
}
function toWorkflowStatus(
status: WorkflowsStatus | undefined
): WorkflowStatus {
if (!status) {
return 'unknown';
}
return status;
}
function toWorkflowGitConfig(
gitConfig: GittypesRepoConfig | undefined
): RepoConfigResponse | undefined {
if (!gitConfig) {
return undefined;
}
return {
URL: gitConfig.URL ?? '',
ReferenceName: gitConfig.ReferenceName ?? '',
ConfigFilePath: gitConfig.ConfigFilePath ?? '',
ConfigHash: gitConfig.ConfigHash ?? '',
TLSSkipVerify: gitConfig.TLSSkipVerify ?? false,
Authentication: toGitAuthentication(gitConfig.Authentication),
};
}
function toGitAuthentication(
auth: GittypesGitAuthentication | undefined
): GitAuthenticationResponse | undefined {
if (!auth) {
return undefined;
}
return {
Username: auth.Username,
Password: auth.Password,
AuthorizationType: auth.AuthorizationType as AuthTypeOption | undefined,
};
}
return data;
}
@@ -0,0 +1,94 @@
import { useQuery } from '@tanstack/react-query';
import {
type SourcesWorkflow,
GittypesRepoConfig,
GittypesGitAuthentication,
} from '@api/types.gen';
import { gitOpsSourceWorkflowsList } from '@api/sdk.gen';
import { withError } from '@/react-tools/react-query';
import {
type RepoConfigResponse,
type GitAuthenticationResponse,
} from '@/react/portainer/gitops/types';
import { AuthTypeOption } from '@/react/portainer/account/git-credentials/types';
import { Source } from '../types';
import { WorkflowStatusObject, WorkflowTarget } from '../../workflows/types';
import { toStatusObject } from '../../workflows/queries/mappers';
import { sourceQueryKeys } from './query-keys';
export type SourceWorkflow = SourcesWorkflow & {
status: WorkflowStatusObject;
sourceId?: number;
gitConfig?: RepoConfigResponse;
target: WorkflowTarget;
creationDate: number;
lastSyncDate: number;
};
export function sourceWorkflowsOptions(id: Source['id']) {
return {
queryKey: sourceQueryKeys.workflows(id!),
queryFn: () => getSourceWorkflows(id!),
...withError('Failed loading source workflows'),
};
}
export function useSourceWorkflows(id: Source['id'] | undefined) {
return useQuery({
...sourceWorkflowsOptions(id!),
enabled: !!id,
});
}
export async function getSourceWorkflows(
id: Source['id']
): Promise<Array<SourceWorkflow>> {
const { data } = await gitOpsSourceWorkflowsList({ path: { id } });
return data.map(toWorkflow);
function toWorkflow(workflow: SourcesWorkflow): SourceWorkflow {
return {
...workflow,
creationDate: workflow.creationDate ?? 0,
lastSyncDate: workflow.lastSyncDate ?? 0,
status: toStatusObject(workflow.status),
gitConfig: toWorkflowGitConfig(workflow.gitConfig),
};
}
function toWorkflowGitConfig(
gitConfig: GittypesRepoConfig | undefined
): RepoConfigResponse | undefined {
if (!gitConfig) {
return undefined;
}
return {
URL: gitConfig.URL ?? '',
ReferenceName: gitConfig.ReferenceName ?? '',
ConfigFilePath: gitConfig.ConfigFilePath ?? '',
ConfigHash: gitConfig.ConfigHash ?? '',
TLSSkipVerify: gitConfig.TLSSkipVerify ?? false,
Authentication: toGitAuthentication(gitConfig.Authentication),
};
}
function toGitAuthentication(
auth: GittypesGitAuthentication | undefined
): GitAuthenticationResponse | undefined {
if (!auth) {
return undefined;
}
return {
Username: auth.Username,
Password: auth.Password,
AuthorizationType: auth.AuthorizationType as AuthTypeOption | undefined,
};
}
}
@@ -29,7 +29,7 @@ function toArtifact(artifact: WorkflowsArtifactDetail): WorkflowArtifact {
};
}
function toStatusObject(
export function toStatusObject(
statusObj: WorkflowsWorkflowStatusObject | undefined
): WorkflowStatusObject {
return {
+1
View File
@@ -27,4 +27,5 @@ export const gitopsHandlers = [
connection: {},
});
}),
http.get('/api/gitops/sources/:id/workflows', () => HttpResponse.json([])),
];