feat(gitops): rework the flow so the sources are the trigger BE-13081 (#3180)

This commit is contained in:
andres-portainer
2026-07-14 12:52:42 -03:00
committed by GitHub
parent b8cb6dbe05
commit 0e9d7e3b5e
74 changed files with 1394 additions and 384 deletions
+10 -4
View File
@@ -27,6 +27,7 @@ import (
"github.com/portainer/portainer/api/exec"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/git"
"github.com/portainer/portainer/api/gitops/scheduling"
"github.com/portainer/portainer/api/http"
"github.com/portainer/portainer/api/http/proxy"
kubeproxy "github.com/portainer/portainer/api/http/proxy/factory/kubernetes"
@@ -571,10 +572,15 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
log.Fatal().Err(err).Msg("failed starting tunnel server")
}
scheduler := scheduler.NewScheduler(shutdownCtx)
sched := scheduler.NewScheduler(shutdownCtx)
stackDeployer := deployments.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer, dockerClientFactory, dataStore)
if err := deployments.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService); err != nil {
log.Fatal().Err(err).Msg("failed to start stack scheduler")
sourceScheduler := scheduling.NewSourceScheduler(sched, dataStore, scheduling.Deployers{
Stack: func(ctx context.Context, stackID portainer.StackID) error {
return deployments.RedeployWhenChanged(ctx, stackID, stackDeployer, dataStore, gitService)
},
})
if err := sourceScheduler.ReconcileAll(); err != nil {
log.Fatal().Err(err).Msg("failed to start source scheduler")
}
sslDBSettings, err := dataStore.SSLSettings().Settings()
@@ -649,7 +655,7 @@ func buildServer(flags *portainer.CLIFlags, shutdownCtx context.Context, shutdow
SSLService: sslService,
DockerClientFactory: dockerClientFactory,
KubernetesClientFactory: kubernetesClientFactory,
Scheduler: scheduler,
SourceScheduler: sourceScheduler,
ShutdownTrigger: shutdownTrigger,
StackDeployer: stackDeployer,
UpgradeService: upgradeService,
-1
View File
@@ -219,7 +219,6 @@ type (
StacksByName(name string) ([]portainer.Stack, error)
GetNextIdentifier() int
StackByWebhookID(ID string) (*portainer.Stack, error)
RefreshableStacks() ([]portainer.Stack, error)
}
// TagService represents a service for managing tag data
-13
View File
@@ -121,16 +121,3 @@ func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
return nil, err
}
// RefreshableStacks returns stacks that are configured for a periodic update
func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
stacks := make([]portainer.Stack, 0)
return stacks, service.Connection.GetAll(
BucketName,
&portainer.Stack{},
dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
return e.WorkflowID != 0 && e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
}),
)
}
@@ -83,25 +83,3 @@ func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack {
return stack
}
func Test_RefreshableStacks(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("skipping test in short mode. Normally takes ~1s to run.")
}
_, store := datastore.MustNewTestStore(t, false, true)
staticStack := portainer.Stack{ID: 1}
stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.AutoUpdateSettings{Webhook: "webhook"}}
intervalNoWorkflow := portainer.Stack{ID: 3, AutoUpdate: &portainer.AutoUpdateSettings{Interval: "1m"}}
refreshableStack := portainer.Stack{ID: 4, WorkflowID: 1, AutoUpdate: &portainer.AutoUpdateSettings{Interval: "1m"}}
for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &intervalNoWorkflow, &refreshableStack} {
err := store.Stack().Create(stack)
require.NoError(t, err)
}
stacks, err := store.Stack().RefreshableStacks()
require.NoError(t, err)
require.ElementsMatch(t, []portainer.Stack{refreshableStack}, stacks)
}
-13
View File
@@ -97,16 +97,3 @@ func (service ServiceTx) StackByWebhookID(id string) (*portainer.Stack, error) {
return nil, err
}
// RefreshableStacks returns stacks that are configured for a periodic update
func (service ServiceTx) RefreshableStacks() ([]portainer.Stack, error) {
stacks := make([]portainer.Stack, 0)
return stacks, service.Tx.GetAll(
BucketName,
&portainer.Stack{},
dataservices.FilterFn(&stacks, func(e portainer.Stack) bool {
return e.WorkflowID != 0 && e.AutoUpdate != nil && e.AutoUpdate.Interval != ""
}),
)
}
+121
View File
@@ -0,0 +1,121 @@
package migrator
import (
"fmt"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/rs/zerolog/log"
)
// backfillSourceInterval_2_44_0 copies AutoUpdate.Interval from each stack onto its linked
// Source.Interval. When a Source is shared by multiple stacks, the shortest interval wins.
func (m *Migrator) backfillSourceInterval_2_44_0() error {
log.Info().Msg("backfilling Source.Interval from deprecated Stack.AutoUpdate.Interval")
workflows, err := m.workflowService.ReadAll()
if err != nil {
return err
}
stackIDsBySource := make(map[portainer.SourceID][]portainer.StackID)
referencedStackIDs := make(map[portainer.StackID]struct{})
for _, wf := range workflows {
for _, artifact := range wf.Artifacts {
if artifact.StackID == 0 {
continue
}
for _, file := range artifact.Files {
stackIDsBySource[file.SourceID] = append(stackIDsBySource[file.SourceID], artifact.StackID)
referencedStackIDs[artifact.StackID] = struct{}{}
}
}
}
intervalByStack := make(map[portainer.StackID]string, len(referencedStackIDs))
for stackID := range referencedStackIDs {
s, err := m.stackService.Read(stackID)
if err != nil {
return fmt.Errorf("failed to read stack %d: %w", stackID, err)
}
if s.AutoUpdate != nil && s.AutoUpdate.Interval != "" {
intervalByStack[stackID] = s.AutoUpdate.Interval
}
}
adminUserContext := source.InsecureNewAdminContext()
for srcID, stackIDs := range stackIDsBySource {
if err := m.stackService.Connection.UpdateTx(func(tx portainer.Transaction) error {
return m.backfillSourceIntervalForGroup_2_44_0(tx, adminUserContext, srcID, stackIDs, intervalByStack)
}); err != nil {
return fmt.Errorf("failed to backfill interval for source %d: %w", srcID, err)
}
}
for stackID := range intervalByStack {
if err := m.stackService.Connection.UpdateTx(func(tx portainer.Transaction) error {
return m.clearAutoUpdateInterval_2_44_0(tx, stackID)
}); err != nil {
return fmt.Errorf("failed to clear auto update interval for stack %d: %w", stackID, err)
}
}
return nil
}
func (m *Migrator) backfillSourceIntervalForGroup_2_44_0(tx portainer.Transaction, adminUserContext source.UserContext, srcID portainer.SourceID, stackIDs []portainer.StackID, intervalByStack map[portainer.StackID]string) error {
var (
minInterval time.Duration
minIntervalStr string
)
for _, stackID := range stackIDs {
intervalStr, ok := intervalByStack[stackID]
if !ok {
continue
}
interval, err := time.ParseDuration(intervalStr)
if err != nil {
return fmt.Errorf("failed to parse auto update interval %q for stack %d: %w", intervalStr, stackID, err)
}
if minIntervalStr == "" || interval < minInterval {
minInterval = interval
minIntervalStr = intervalStr
}
}
if minIntervalStr == "" {
return nil
}
src, err := m.sourceService.Tx(tx).Read(adminUserContext, srcID)
if err != nil {
return fmt.Errorf("failed to read source %d: %w", srcID, err)
}
src.Interval = minIntervalStr
return m.sourceService.Tx(tx).Update(adminUserContext, srcID, src)
}
func (m *Migrator) clearAutoUpdateInterval_2_44_0(tx portainer.Transaction, stackID portainer.StackID) error {
s, err := m.stackService.Tx(tx).Read(stackID)
if err != nil {
return fmt.Errorf("failed to read stack %d: %w", stackID, err)
}
if s.AutoUpdate == nil {
return nil
}
s.AutoUpdate.Interval = ""
return m.stackService.Tx(tx).Update(s.ID, s)
}
@@ -0,0 +1,228 @@
package migrator
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/database/boltdb"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/dataservices/stack"
"github.com/portainer/portainer/api/dataservices/workflow"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/logs"
"github.com/stretchr/testify/require"
)
func TestBackfillSourceInterval_2_44_0_MinimumIntervalWins(t *testing.T) {
t.Parallel()
conn := &boltdb.DbConnection{Path: t.TempDir()}
err := conn.Open()
require.NoError(t, err)
defer logs.CloseAndLogErr(conn)
stackSvc, err := stack.NewService(conn)
require.NoError(t, err)
sourceSvc, err := source.NewService(conn)
require.NoError(t, err)
workflowSvc, err := workflow.NewService(conn)
require.NoError(t, err)
m := NewMigrator(&MigratorParameters{
StackService: stackSvc,
SourceService: sourceSvc,
WorkflowService: workflowSvc,
})
src := &portainer.Source{
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{URL: "https://github.com/example/shared-repo"},
}
err = sourceSvc.Create(adminUserContext, src)
require.NoError(t, err)
stackA := &portainer.Stack{ID: 1, Name: "stack-a", AutoUpdate: &portainer.AutoUpdateSettings{Interval: "10m"}}
err = stackSvc.Create(stackA)
require.NoError(t, err)
stackB := &portainer.Stack{ID: 2, Name: "stack-b", AutoUpdate: &portainer.AutoUpdateSettings{Interval: "5m"}}
err = stackSvc.Create(stackB)
require.NoError(t, err)
wfA := &portainer.Workflow{
Name: "stack-a",
Artifacts: []portainer.Artifact{{
StackID: stackA.ID,
Files: []portainer.ArtifactFile{{SourceID: src.ID}},
}},
}
err = workflowSvc.Create(wfA)
require.NoError(t, err)
wfB := &portainer.Workflow{
Name: "stack-b",
Artifacts: []portainer.Artifact{{
StackID: stackB.ID,
Files: []portainer.ArtifactFile{{SourceID: src.ID}},
}},
}
err = workflowSvc.Create(wfB)
require.NoError(t, err)
stackA.WorkflowID = wfA.ID
err = stackSvc.Update(stackA.ID, stackA)
require.NoError(t, err)
stackB.WorkflowID = wfB.ID
err = stackSvc.Update(stackB.ID, stackB)
require.NoError(t, err)
err = m.backfillSourceInterval_2_44_0()
require.NoError(t, err)
updatedSrc, err := sourceSvc.Read(adminUserContext, src.ID)
require.NoError(t, err)
require.Equal(t, "5m", updatedSrc.Interval)
updatedA, err := stackSvc.Read(stackA.ID)
require.NoError(t, err)
require.NotNil(t, updatedA.AutoUpdate)
require.Empty(t, updatedA.AutoUpdate.Interval)
updatedB, err := stackSvc.Read(stackB.ID)
require.NoError(t, err)
require.NotNil(t, updatedB.AutoUpdate)
require.Empty(t, updatedB.AutoUpdate.Interval)
}
func TestBackfillSourceInterval_2_44_0_WebhookOnlyStackLeftAlone(t *testing.T) {
t.Parallel()
conn := &boltdb.DbConnection{Path: t.TempDir()}
err := conn.Open()
require.NoError(t, err)
defer logs.CloseAndLogErr(conn)
stackSvc, err := stack.NewService(conn)
require.NoError(t, err)
sourceSvc, err := source.NewService(conn)
require.NoError(t, err)
workflowSvc, err := workflow.NewService(conn)
require.NoError(t, err)
m := NewMigrator(&MigratorParameters{
StackService: stackSvc,
SourceService: sourceSvc,
WorkflowService: workflowSvc,
})
src := &portainer.Source{
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{URL: "https://github.com/example/webhook-repo"},
}
err = sourceSvc.Create(adminUserContext, src)
require.NoError(t, err)
webhookStack := &portainer.Stack{
ID: 1,
Name: "webhook-stack",
AutoUpdate: &portainer.AutoUpdateSettings{
Webhook: "05de31a2-79fa-4644-9c12-faa67e5c49f0",
},
}
err = stackSvc.Create(webhookStack)
require.NoError(t, err)
wf := &portainer.Workflow{
Name: "webhook-stack",
Artifacts: []portainer.Artifact{{
StackID: webhookStack.ID,
Files: []portainer.ArtifactFile{{SourceID: src.ID}},
}},
}
err = workflowSvc.Create(wf)
require.NoError(t, err)
webhookStack.WorkflowID = wf.ID
err = stackSvc.Update(webhookStack.ID, webhookStack)
require.NoError(t, err)
err = m.backfillSourceInterval_2_44_0()
require.NoError(t, err)
updatedSrc, err := sourceSvc.Read(adminUserContext, src.ID)
require.NoError(t, err)
require.Empty(t, updatedSrc.Interval)
updatedStack, err := stackSvc.Read(webhookStack.ID)
require.NoError(t, err)
require.Equal(t, "05de31a2-79fa-4644-9c12-faa67e5c49f0", updatedStack.AutoUpdate.Webhook)
}
func TestBackfillSourceInterval_2_44_0_StackReferencingTwoSourcesBackfillsBoth(t *testing.T) {
t.Parallel()
conn := &boltdb.DbConnection{Path: t.TempDir()}
err := conn.Open()
require.NoError(t, err)
defer logs.CloseAndLogErr(conn)
stackSvc, err := stack.NewService(conn)
require.NoError(t, err)
sourceSvc, err := source.NewService(conn)
require.NoError(t, err)
workflowSvc, err := workflow.NewService(conn)
require.NoError(t, err)
m := NewMigrator(&MigratorParameters{
StackService: stackSvc,
SourceService: sourceSvc,
WorkflowService: workflowSvc,
})
srcA := &portainer.Source{Type: portainer.SourceTypeGit, Git: &gittypes.GitSource{URL: "https://github.com/example/repo-a"}}
err = sourceSvc.Create(adminUserContext, srcA)
require.NoError(t, err)
srcB := &portainer.Source{Type: portainer.SourceTypeGit, Git: &gittypes.GitSource{URL: "https://github.com/example/repo-b"}}
err = sourceSvc.Create(adminUserContext, srcB)
require.NoError(t, err)
multiSourceStack := &portainer.Stack{ID: 1, Name: "multi-source-stack", AutoUpdate: &portainer.AutoUpdateSettings{Interval: "5m"}}
err = stackSvc.Create(multiSourceStack)
require.NoError(t, err)
wf := &portainer.Workflow{
Name: "multi-source-stack",
Artifacts: []portainer.Artifact{{
StackID: multiSourceStack.ID,
Files: []portainer.ArtifactFile{
{SourceID: srcA.ID},
{SourceID: srcB.ID},
},
}},
}
err = workflowSvc.Create(wf)
require.NoError(t, err)
multiSourceStack.WorkflowID = wf.ID
err = stackSvc.Update(multiSourceStack.ID, multiSourceStack)
require.NoError(t, err)
err = m.backfillSourceInterval_2_44_0()
require.NoError(t, err)
updatedA, err := sourceSvc.Read(adminUserContext, srcA.ID)
require.NoError(t, err)
require.Equal(t, "5m", updatedA.Interval)
updatedB, err := sourceSvc.Read(adminUserContext, srcB.ID)
require.NoError(t, err)
require.Equal(t, "5m", updatedB.Interval)
updatedStack, err := stackSvc.Read(multiSourceStack.ID)
require.NoError(t, err)
require.NotNil(t, updatedStack.AutoUpdate)
require.Empty(t, updatedStack.AutoUpdate.Interval)
}
+2
View File
@@ -277,6 +277,8 @@ func (m *Migrator) initMigrations() {
m.migrateCustomTemplateGitConfigToSources_2_43_0,
)
m.addMigrations("2.44.0", m.backfillSourceInterval_2_44_0)
// WARNING: do not change migrations that have already been released!
// Add new migrations above...
+19 -1
View File
@@ -13547,7 +13547,12 @@ components:
example: false
type: boolean
Interval:
description: Auto update interval
description: >-
Auto update interval
Deprecated: polling interval now lives on the associated Source (Source.Interval).
Kept for DB backwards-compatibility only; new code must not read or write this field.
example: 1m30s
type: string
JobID:
@@ -15094,6 +15099,9 @@ components:
id:
example: 1
type: integer
interval:
example: 5m
type: string
lastSync:
example: 1587399600
type: integer
@@ -16585,6 +16593,8 @@ components:
type: boolean
authentication:
$ref: "#/components/schemas/sources.GitAuthenticationPayload"
interval:
type: string
name:
type: string
public:
@@ -16609,6 +16619,8 @@ components:
properties:
authentication:
$ref: "#/components/schemas/sources.GitAuthenticationUpdatePayload"
interval:
type: string
name:
type: string
tlsSkipVerify:
@@ -16624,6 +16636,9 @@ components:
type: string
id:
type: integer
interval:
example: 5m
type: string
lastSync:
type: integer
name:
@@ -16683,6 +16698,9 @@ components:
type: string
id:
type: integer
interval:
example: 5m
type: string
lastSync:
type: integer
name:
+17 -1
View File
@@ -2650,7 +2650,10 @@ definitions:
example: false
type: boolean
Interval:
description: Auto update interval
description: |-
Auto update interval
Deprecated: polling interval now lives on the associated Source (Source.Interval).
Kept for DB backwards-compatibility only; new code must not read or write this field.
example: 1m30s
type: string
JobID:
@@ -4179,6 +4182,9 @@ definitions:
id:
example: 1
type: integer
interval:
example: 5m
type: string
lastSync:
example: 1587399600
type: integer
@@ -5639,6 +5645,8 @@ definitions:
type: boolean
authentication:
$ref: '#/definitions/sources.GitAuthenticationPayload'
interval:
type: string
name:
type: string
public:
@@ -5663,6 +5671,8 @@ definitions:
properties:
authentication:
$ref: '#/definitions/sources.GitAuthenticationUpdatePayload'
interval:
type: string
name:
type: string
tlsSkipVerify:
@@ -5678,6 +5688,9 @@ definitions:
type: string
id:
type: integer
interval:
example: 5m
type: string
lastSync:
type: integer
name:
@@ -5737,6 +5750,9 @@ definitions:
type: string
id:
type: integer
interval:
example: 5m
type: string
lastSync:
type: integer
name:
+219
View File
@@ -0,0 +1,219 @@
package scheduling
import (
"context"
"fmt"
"sync"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/scheduler"
"github.com/rs/zerolog/log"
)
type Deployers struct {
Stack func(ctx context.Context, stackID portainer.StackID) error
EdgeStack func(ctx context.Context, edgeStackID portainer.EdgeStackID) error
}
type dataStore interface {
Source() dataservices.SourceService
Workflow() dataservices.WorkflowService
}
type SourceScheduler struct {
scheduler *scheduler.Scheduler
dataStore dataStore
deployers Deployers
mu sync.Mutex
jobs map[portainer.SourceID]jobEntry
}
type jobEntry struct {
jobID string
interval string
}
func NewSourceScheduler(s *scheduler.Scheduler, ds dataStore, deployers Deployers) *SourceScheduler {
return &SourceScheduler{
scheduler: s,
dataStore: ds,
deployers: deployers,
jobs: make(map[portainer.SourceID]jobEntry),
}
}
// ReconcileAll starts, updates, or stops the polling job of every source to match its current state.
func (s *SourceScheduler) ReconcileAll() error {
sysCtx := source.InsecureNewAdminContext()
sources, err := s.dataStore.Source().ReadAll(sysCtx)
if err != nil {
return fmt.Errorf("failed to read sources: %w", err)
}
for i := range sources {
if err := s.reconcileSource(&sources[i]); err != nil {
log.Warn().Err(err).Int("source_id", int(sources[i].ID)).Msg("failed to reconcile source polling job")
}
}
return nil
}
// Reconcile recomputes the desired polling state for a single source: it starts a job when the
// source becomes pollable, restarts it when the interval changes, and stops it when the source is
// gone, has no interval, or is no longer referenced by any workflow.
//
// It is a no-op when called on a nil scheduler or with a zero sourceID, so callers do not need to
// guard every call site.
func (s *SourceScheduler) Reconcile(sourceID portainer.SourceID) error {
if s == nil || sourceID == 0 {
return nil
}
sysCtx := source.InsecureNewAdminContext()
src, err := s.dataStore.Source().Read(sysCtx, sourceID)
if err != nil {
if dataservices.IsErrObjectNotFound(err) {
s.stop(sourceID)
return nil
}
return fmt.Errorf("failed to read source %d: %w", sourceID, err)
}
return s.reconcileSource(src)
}
func (s *SourceScheduler) reconcileSource(src *portainer.Source) error {
referenced, err := s.sourceReferenced(src.ID)
if err != nil {
return err
}
s.mu.Lock()
defer s.mu.Unlock()
if src.Interval == "" || !referenced {
s.stopLocked(src.ID)
return nil
}
d, err := time.ParseDuration(src.Interval)
if err != nil {
return fmt.Errorf("invalid interval %q for source %d: %w", src.Interval, src.ID, err)
}
if entry, ok := s.jobs[src.ID]; ok {
if entry.interval == src.Interval {
return nil
}
s.stopLocked(src.ID)
}
sourceID := src.ID
jobID := s.scheduler.StartJobEvery(d, func() error {
return s.tick(context.Background(), sourceID)
})
s.jobs[src.ID] = jobEntry{jobID: jobID, interval: src.Interval}
return nil
}
func (s *SourceScheduler) stop(sourceID portainer.SourceID) {
s.mu.Lock()
defer s.mu.Unlock()
s.stopLocked(sourceID)
}
func (s *SourceScheduler) stopLocked(sourceID portainer.SourceID) {
entry, ok := s.jobs[sourceID]
if !ok {
return
}
if err := s.scheduler.StopJob(entry.jobID); err != nil {
log.Warn().Err(err).Int("source_id", int(sourceID)).Msg("failed to stop source polling job")
}
delete(s.jobs, sourceID)
}
// tick runs one poll of a source: it redeploys every artifact that references the source.
// Individual deploy failures are logged and do not abort the remaining work; each deployer
// persists the resulting source and artifact status itself.
func (s *SourceScheduler) tick(ctx context.Context, sourceID portainer.SourceID) error {
workflows, err := s.dataStore.Workflow().ReadAll(func(wf portainer.Workflow) bool {
return workflowReferencesSource(wf, sourceID)
})
if err != nil {
return fmt.Errorf("failed to read workflows for source %d: %w", sourceID, err)
}
for _, wf := range workflows {
for _, a := range wf.Artifacts {
if !artifactReferencesSource(a, sourceID) {
continue
}
s.deployArtifact(ctx, a)
}
}
return nil
}
func (s *SourceScheduler) deployArtifact(ctx context.Context, a portainer.Artifact) {
if a.StackID != 0 && s.deployers.Stack != nil {
if err := s.deployers.Stack(ctx, a.StackID); err != nil {
log.Warn().Err(err).Int("stack_id", int(a.StackID)).Msg("source poll: stack redeploy failed")
}
}
if a.EdgeStackID != 0 && s.deployers.EdgeStack != nil {
if err := s.deployers.EdgeStack(ctx, a.EdgeStackID); err != nil {
log.Warn().Err(err).Int("edge_stack_id", int(a.EdgeStackID)).Msg("source poll: edge stack redeploy failed")
}
}
}
func (s *SourceScheduler) sourceReferenced(sourceID portainer.SourceID) (bool, error) {
workflows, err := s.dataStore.Workflow().ReadAll(func(wf portainer.Workflow) bool {
return workflowReferencesSource(wf, sourceID)
})
if err != nil {
return false, fmt.Errorf("failed to read workflows for source %d: %w", sourceID, err)
}
return len(workflows) > 0, nil
}
func workflowReferencesSource(wf portainer.Workflow, sourceID portainer.SourceID) bool {
for _, a := range wf.Artifacts {
if artifactReferencesSource(a, sourceID) {
return true
}
}
return false
}
func artifactReferencesSource(a portainer.Artifact, sourceID portainer.SourceID) bool {
for _, f := range a.Files {
if f.SourceID == sourceID {
return true
}
}
return false
}
+193
View File
@@ -0,0 +1,193 @@
package scheduling
import (
"context"
"sync"
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/datastore"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/scheduler"
"github.com/stretchr/testify/require"
)
func TestSourceScheduler_TickDeploysAllReferencingArtifacts(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
sysCtx := source.InsecureNewAdminContext()
src := &portainer.Source{
Name: "shared",
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{URL: "https://example.com/repo.git"},
}
err := store.Source().Create(sysCtx, src)
require.NoError(t, err)
otherSrc := &portainer.Source{
Name: "other",
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{URL: "https://example.com/other.git"},
}
err = store.Source().Create(sysCtx, otherSrc)
require.NoError(t, err)
// Two stacks and one edge stack reference the shared source across separate workflows.
err = store.Workflow().Create(&portainer.Workflow{
Name: "wf-stack-1",
Artifacts: []portainer.Artifact{
{StackID: 11, Files: []portainer.ArtifactFile{{SourceID: src.ID}}},
},
})
require.NoError(t, err)
err = store.Workflow().Create(&portainer.Workflow{
Name: "wf-stack-2",
Artifacts: []portainer.Artifact{
{StackID: 22, Files: []portainer.ArtifactFile{{SourceID: src.ID}}},
},
})
require.NoError(t, err)
err = store.Workflow().Create(&portainer.Workflow{
Name: "wf-edge",
Artifacts: []portainer.Artifact{
{EdgeStackID: 33, Files: []portainer.ArtifactFile{{SourceID: src.ID}}},
},
})
require.NoError(t, err)
// A workflow that only references the other source must not be touched.
err = store.Workflow().Create(&portainer.Workflow{
Name: "wf-unrelated",
Artifacts: []portainer.Artifact{
{StackID: 99, Files: []portainer.ArtifactFile{{SourceID: otherSrc.ID}}},
},
})
require.NoError(t, err)
var mu sync.Mutex
var stacks []portainer.StackID
var edgeStacks []portainer.EdgeStackID
sched := scheduler.NewScheduler(t.Context())
poller := NewSourceScheduler(sched, store, Deployers{
Stack: func(_ context.Context, id portainer.StackID) error {
mu.Lock()
defer mu.Unlock()
stacks = append(stacks, id)
return nil
},
EdgeStack: func(_ context.Context, id portainer.EdgeStackID) error {
mu.Lock()
defer mu.Unlock()
edgeStacks = append(edgeStacks, id)
return nil
},
})
err = poller.tick(t.Context(), src.ID)
require.NoError(t, err)
require.ElementsMatch(t, []portainer.StackID{11, 22}, stacks)
require.ElementsMatch(t, []portainer.EdgeStackID{33}, edgeStacks)
}
func TestSourceScheduler_ReconcileStartsRestartsAndStops(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
sysCtx := source.InsecureNewAdminContext()
// A long interval keeps the cron job from ever firing during the test; we only assert job state.
src := &portainer.Source{
Name: "polled",
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{URL: "https://example.com/repo.git"},
Interval: "1h",
}
err := store.Source().Create(sysCtx, src)
require.NoError(t, err)
err = store.Workflow().Create(&portainer.Workflow{
Name: "wf",
Artifacts: []portainer.Artifact{
{StackID: 11, Files: []portainer.ArtifactFile{{SourceID: src.ID}}},
},
})
require.NoError(t, err)
sched := scheduler.NewScheduler(t.Context())
poller := NewSourceScheduler(sched, store, Deployers{
Stack: func(_ context.Context, _ portainer.StackID) error { return nil },
})
// A referenced source with an interval gets a job.
err = poller.ReconcileAll()
require.NoError(t, err)
entry, ok := poller.jobs[src.ID]
require.True(t, ok)
require.Equal(t, "1h", entry.interval)
firstJobID := entry.jobID
// Changing the interval restarts the job with a fresh id.
src.Interval = "2h"
err = store.Source().Update(sysCtx, src.ID, src)
require.NoError(t, err)
err = poller.Reconcile(src.ID)
require.NoError(t, err)
entry, ok = poller.jobs[src.ID]
require.True(t, ok)
require.Equal(t, "2h", entry.interval)
require.NotEqual(t, firstJobID, entry.jobID)
// Clearing the interval stops the job.
src.Interval = ""
err = store.Source().Update(sysCtx, src.ID, src)
require.NoError(t, err)
err = poller.Reconcile(src.ID)
require.NoError(t, err)
_, ok = poller.jobs[src.ID]
require.False(t, ok)
}
func TestSourceScheduler_ReconcileSkipsUnreferencedSource(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, true)
sysCtx := source.InsecureNewAdminContext()
src := &portainer.Source{
Name: "orphan",
Type: portainer.SourceTypeGit,
Git: &gittypes.GitSource{URL: "https://example.com/repo.git"},
Interval: "1h",
}
err := store.Source().Create(sysCtx, src)
require.NoError(t, err)
sched := scheduler.NewScheduler(t.Context())
poller := NewSourceScheduler(sched, store, Deployers{
Stack: func(_ context.Context, _ portainer.StackID) error { return nil },
})
err = poller.Reconcile(src.ID)
require.NoError(t, err)
_, ok := poller.jobs[src.ID]
require.False(t, ok)
}
+1 -1
View File
@@ -11,7 +11,7 @@ import (
"github.com/stretchr/testify/require"
)
func TestValidateSourceForStack_ValidGitSource_ReturnsNil(t *testing.T) {
func TestValidateSourceForStack_ValidGitSource_ReturnsNoError(t *testing.T) {
t.Parallel()
_, store := datastore.MustNewTestStore(t, false, false)
+30
View File
@@ -0,0 +1,30 @@
package workflows
import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
)
// workflowDeleteStore is the minimal intersection of CE and EE DataStoreTx needed to safely delete a workflow.
type workflowDeleteStore interface {
Workflow() dataservices.WorkflowService
}
// DeleteIfSingleArtifact deletes the workflow identified by workflowID, but only when it has a
// single artifact.
func DeleteIfSingleArtifact(tx workflowDeleteStore, workflowID portainer.WorkflowID) error {
if workflowID == 0 {
return nil
}
wf, err := tx.Workflow().Read(workflowID)
if err != nil {
return err
}
if len(wf.Artifacts) > 1 {
return nil
}
return tx.Workflow().Delete(workflowID)
}
+3 -2
View File
@@ -5,6 +5,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/gitops/scheduling"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/kubernetes/cli"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
@@ -23,7 +24,7 @@ type Handler struct {
fileService portainer.FileService
}
func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, gitService portainer.GitService, fileService portainer.FileService, k8sFactory *cli.ClientFactory) *Handler {
func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, gitService portainer.GitService, fileService portainer.FileService, k8sFactory *cli.ClientFactory, sourceScheduler *scheduling.SourceScheduler) *Handler {
h := &Handler{
Router: mux.NewRouter(),
dataStore: dataStore,
@@ -39,7 +40,7 @@ func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStor
workflowsHandler := workflows.NewHandler(dataStore, gitService, k8sFactory)
authenticatedRouter.PathPrefix("/gitops/workflows").Handler(workflowsHandler)
sourcesHandler := sources.NewHandler(bouncer, dataStore, gitService, k8sFactory)
sourcesHandler := sources.NewHandler(bouncer, dataStore, gitService, k8sFactory, sourceScheduler)
authenticatedRouter.PathPrefix("/gitops/sources").Handler(sourcesHandler)
return h
@@ -14,6 +14,8 @@ import (
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/portainer/portainer/pkg/validate"
"github.com/rs/zerolog/log"
)
// GitAuthenticationPayload holds authentication parameters for a git source
@@ -36,6 +38,7 @@ type GitSourceCreatePayload struct {
URL string `json:"url" validate:"required"`
TLSSkipVerify bool `json:"tlsSkipVerify"`
Authentication *GitAuthenticationPayload `json:"authentication"`
Interval string `json:"interval"`
}
// Validate implements the portainer.Validatable interface
@@ -44,7 +47,7 @@ func (payload *GitSourceCreatePayload) Validate(_ *http.Request) error {
return errors.New("invalid repository URL. Must correspond to a valid URL format")
}
return nil
return validateInterval(payload.Interval)
}
// @id GitOpsSourcesCreateGit
@@ -96,6 +99,10 @@ func (h *Handler) gitSourceCreate(w http.ResponseWriter, r *http.Request) *httpe
h.invalidateCache()
if err := h.sourceScheduler.Reconcile(src.ID); err != nil {
log.Warn().Err(err).Int("source_id", int(src.ID)).Msg("source scheduler reconcile failed after source creation")
}
src.Git = gittypes.SanitizeGitSource(src.Git)
return response.JSONWithStatus(w, src, http.StatusCreated)
@@ -127,6 +134,7 @@ func BuildBaseGitSource(payload GitSourceCreatePayload) *portainer.Source {
TeamAccesses: payload.TeamAccesses,
Public: payload.Public,
AdministratorsOnly: payload.AdministratorsOnly,
Interval: payload.Interval,
}
}
@@ -13,6 +13,8 @@ import (
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/rs/zerolog/log"
)
var ErrSourceInUse = errors.New("source is used by one or more workflows or custom templates")
@@ -93,5 +95,9 @@ func (h *Handler) sourceDelete(w http.ResponseWriter, r *http.Request) *httperro
h.invalidateCache()
if err := h.sourceScheduler.Reconcile(portainer.SourceID(sourceID)); err != nil {
log.Warn().Err(err).Int("source_id", sourceID).Msg("source scheduler reconcile failed after source deletion")
}
return response.Empty(w)
}
+13 -10
View File
@@ -7,6 +7,7 @@ import (
gocache "github.com/patrickmn/go-cache"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/gitops/scheduling"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/kubernetes/cli"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
@@ -22,19 +23,21 @@ const (
// Handler is the HTTP handler for the GitOps sources API.
type Handler struct {
*mux.Router
dataStore dataservices.DataStore
gitService portainer.GitService
cache *gocache.Cache
k8sFactory *cli.ClientFactory
dataStore dataservices.DataStore
gitService portainer.GitService
cache *gocache.Cache
k8sFactory *cli.ClientFactory
sourceScheduler *scheduling.SourceScheduler
}
func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, gitService portainer.GitService, k8sFactory *cli.ClientFactory) *Handler {
func NewHandler(bouncer security.BouncerService, dataStore dataservices.DataStore, gitService portainer.GitService, k8sFactory *cli.ClientFactory, sourceScheduler *scheduling.SourceScheduler) *Handler {
h := &Handler{
Router: mux.NewRouter(),
dataStore: dataStore,
gitService: gitService,
cache: gocache.New(cacheTTL, cacheCleanupInterval),
k8sFactory: k8sFactory,
Router: mux.NewRouter(),
dataStore: dataStore,
gitService: gitService,
cache: gocache.New(cacheTTL, cacheCleanupInterval),
k8sFactory: k8sFactory,
sourceScheduler: sourceScheduler,
}
authenticatedRouter := h.PathPrefix("/gitops/sources").Subrouter()
@@ -48,7 +48,7 @@ func createGitWorkflow(t *testing.T, tx dataservices.DataStoreTx, stack *portain
func newTestHandler(t *testing.T, store dataservices.DataStore) *Handler {
t.Helper()
return NewHandler(testhelpers.NewTestRequestBouncer(), store, nil, nil)
return NewHandler(testhelpers.NewTestRequestBouncer(), store, nil, nil, nil)
}
// newTestHandlerNoCacheExpiry returns a handler whose source cache never expires,
+1
View File
@@ -18,6 +18,7 @@ type Source struct {
UsedBy int `json:"usedBy"`
Environments int `json:"environments"`
LastSync int64 `json:"lastSync"`
Interval string `json:"interval,omitempty" example:"5m"`
}
type SourceType string
@@ -14,6 +14,8 @@ import (
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/portainer/portainer/pkg/validate"
"github.com/rs/zerolog/log"
)
var (
@@ -26,6 +28,7 @@ type GitSourceUpdatePayload struct {
URL *string `json:"url"`
TLSSkipVerify *bool `json:"tlsSkipVerify"`
Authentication *GitAuthenticationUpdatePayload `json:"authentication"`
Interval *string `json:"interval"`
}
type GitAuthenticationUpdatePayload struct {
@@ -39,6 +42,10 @@ func (payload *GitSourceUpdatePayload) Validate(_ *http.Request) error {
return errors.New("invalid repository URL. Must correspond to a valid URL format")
}
if payload.Interval != nil {
return validateInterval(*payload.Interval)
}
return nil
}
@@ -115,6 +122,10 @@ func (h *Handler) gitSourceUpdate(w http.ResponseWriter, r *http.Request) *httpe
h.invalidateCache()
if err := h.sourceScheduler.Reconcile(src.ID); err != nil {
log.Warn().Err(err).Int("source_id", int(src.ID)).Msg("source scheduler reconcile failed after source update")
}
src.Git = gittypes.SanitizeGitSource(src.Git)
return response.JSON(w, src)
@@ -163,6 +174,10 @@ func ApplyBaseGitSourceChanges(src *portainer.Source, payload GitSourceUpdatePay
src.Git.TLSSkipVerify = *payload.TLSSkipVerify
}
if payload.Interval != nil {
src.Interval = *payload.Interval
}
return nil
}
+23
View File
@@ -1,11 +1,33 @@
package sources
import (
"errors"
"time"
portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
ce "github.com/portainer/portainer/api/gitops/workflows"
)
const minPollingInterval = time.Minute
func validateInterval(interval string) error {
if interval == "" {
return nil
}
d, err := time.ParseDuration(interval)
if err != nil {
return errors.New("invalid interval format")
}
if d < minPollingInterval {
return errors.New("interval must be at least 1 minute")
}
return nil
}
func (h *Handler) buildSource(src *portainer.Source, stats ce.SourceStats) Source {
phase := ce.SourceStatusToPhase(src.Status, src.StatusError)
@@ -24,6 +46,7 @@ func (h *Handler) buildSource(src *portainer.Source, stats ce.SourceStats) Sourc
UsedBy: stats.WorkflowCount,
Environments: len(stats.EndpointIDs),
LastSync: src.LastSync,
Interval: src.Interval,
}
}
+18 -11
View File
@@ -10,8 +10,8 @@ import (
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/git/update"
"github.com/portainer/portainer/api/gitops/sources"
"github.com/portainer/portainer/api/gitops/workflows"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/stacks/deployments"
"github.com/portainer/portainer/api/stacks/stackbuilders"
"github.com/portainer/portainer/api/stacks/stackutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
@@ -59,19 +59,23 @@ func (handler *Handler) checkAndCleanStackDupFromSwarm(_ http.ResponseWriter, _
return err
}
// stop scheduler updates of the stack before removal
if stack.AutoUpdate != nil {
deployments.StopAutoupdate(stack.ID, stack.AutoUpdate.JobID, handler.Scheduler)
var reconcileSourceID portainer.SourceID
if stack.WorkflowID != 0 {
if _, sid, err := loadGitConfigForStack(handler.DataStore, source.InsecureNewAdminContext(), stack.WorkflowID, stack.ID); err == nil {
reconcileSourceID = sid
}
if err := workflows.DeleteIfSingleArtifact(handler.DataStore, stack.WorkflowID); err != nil {
return err
}
}
err = handler.DataStore.Stack().Delete(stack.ID)
if err != nil {
if err := handler.DataStore.Stack().Delete(stack.ID); err != nil {
return err
}
if resourceControl != nil {
err = handler.DataStore.ResourceControl().Delete(resourceControl.ID)
if err != nil {
if err := handler.DataStore.ResourceControl().Delete(resourceControl.ID); err != nil {
log.Error().
Str("stack", fmt.Sprintf("%+v", stack)).
Msg("unable to remove the associated resource control from the database for stack")
@@ -79,14 +83,17 @@ func (handler *Handler) checkAndCleanStackDupFromSwarm(_ http.ResponseWriter, _
}
if exists, _ := handler.FileService.FileExists(stack.ProjectPath); exists {
err = handler.FileService.RemoveDirectory(stack.ProjectPath)
if err != nil {
if err := handler.FileService.RemoveDirectory(stack.ProjectPath); err != nil {
log.Warn().
Str("stack", fmt.Sprintf("%+v", stack)).
Msg("unable to remove stack files from disk for stack")
}
}
if err := handler.SourceScheduler.Reconcile(reconcileSourceID); err != nil {
log.Warn().Err(err).Msg("source scheduler reconcile failed after stack duplicate cleanup")
}
return nil
}
@@ -311,7 +318,7 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite
handler.DataStore,
handler.FileService,
handler.GitService,
handler.Scheduler,
handler.SourceScheduler,
handler.StackDeployer)
stack, httpErr := stackbuilders.Build(r.Context(), handler.DataStore, composeStackBuilder, &stackPayload, endpoint, userID)
@@ -265,7 +265,7 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr
k8sStackBuilder := stackbuilders.CreateKubernetesStackGitBuilder(handler.DataStore,
handler.FileService,
handler.GitService,
handler.Scheduler,
handler.SourceScheduler,
handler.StackDeployer,
handler.KubernetesDeployer,
user)
@@ -250,7 +250,7 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter,
handler.DataStore,
handler.FileService,
handler.GitService,
handler.Scheduler,
handler.SourceScheduler,
handler.StackDeployer)
stack, httpErr := stackbuilders.Build(r.Context(), handler.DataStore, swarmStackBuilder, &stackPayload, endpoint, userID)
+2 -2
View File
@@ -11,12 +11,12 @@ import (
"github.com/portainer/portainer/api/dataservices"
dockerclient "github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/docker/consts"
"github.com/portainer/portainer/api/gitops/scheduling"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/internal/endpointutils"
"github.com/portainer/portainer/api/kubernetes/cli"
"github.com/portainer/portainer/api/logs"
"github.com/portainer/portainer/api/scheduler"
"github.com/portainer/portainer/api/stacks/deployments"
"github.com/portainer/portainer/api/stacks/stackutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
@@ -41,7 +41,7 @@ type Handler struct {
ComposeStackManager portainer.ComposeStackManager
KubernetesDeployer portainer.KubernetesDeployer
KubernetesClientFactory *cli.ClientFactory
Scheduler *scheduler.Scheduler
SourceScheduler *scheduling.SourceScheduler
StackDeployer deployments.StackDeployer
}
+18
View File
@@ -64,6 +64,7 @@ func newStackResponse(tx dataservices.DataStoreTx, userContext source.UserContex
}
stack.GitConfig = gittypes.SanitizeRepoConfig(gitConfig)
fillAutoUpdateInterval(tx, userContext, stack)
return &stackResponse{Stack: *stack, GitSourceId: gitSourceID}, nil
}
@@ -80,6 +81,23 @@ func fillStackGitConfig(tx dataservices.DataStoreTx, userContext source.UserCont
}
stack.GitConfig = gittypes.SanitizeRepoConfig(gitConfig)
fillAutoUpdateInterval(tx, userContext, stack)
return nil
}
// fillAutoUpdateInterval restores the deprecated AutoUpdate.Interval field on API responses
// from the linked Source, so old API clients keep seeing polling intervals set through the GitOps
// Sources UI.
func fillAutoUpdateInterval(tx dataservices.DataStoreTx, userContext source.UserContext, stack *portainer.Stack) {
src, _, err := workflows.GitSourceAndArtifactForStack(tx, userContext, stack.WorkflowID, stack.ID)
if err != nil || src == nil || src.Interval == "" {
return
}
if stack.AutoUpdate == nil {
stack.AutoUpdate = &portainer.AutoUpdateSettings{}
}
stack.AutoUpdate.Interval = src.Interval
}
+24 -13
View File
@@ -9,10 +9,11 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/gitops/workflows"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/stacks/deployments"
"github.com/portainer/portainer/api/stacks/stackutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
@@ -113,26 +114,31 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt
return httperror.Forbidden(errMsg, errors.New(errMsg))
}
// stop scheduler updates of the stack before removal
if stack.AutoUpdate != nil {
deployments.StopAutoupdate(stack.ID, stack.AutoUpdate.JobID, handler.Scheduler)
}
if err := handler.deleteStack(r.Context(), securityContext.UserID, stack, endpoint); err != nil {
return httperror.InternalServerError(err.Error(), err)
}
var reconcileSourceID portainer.SourceID
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
if stack.WorkflowID != 0 {
if err := tx.Workflow().Delete(stack.WorkflowID); err != nil {
if _, sid, err := loadGitConfigForStack(tx, source.InsecureNewAdminContext(), stack.WorkflowID, stack.ID); err == nil {
reconcileSourceID = sid
}
if err := workflows.DeleteIfSingleArtifact(tx, stack.WorkflowID); err != nil {
return err
}
}
return tx.Stack().Delete(portainer.StackID(id))
}); err != nil {
return httperror.InternalServerError("Unable to remove the stack from the database", err)
}
if err := handler.SourceScheduler.Reconcile(reconcileSourceID); err != nil {
log.Warn().Err(err).Msg("source scheduler reconcile failed after stack deletion")
}
if resourceControl != nil {
if err := handler.DataStore.ResourceControl().Delete(resourceControl.ID); err != nil {
return httperror.InternalServerError("Unable to remove the associated resource control from the database", err)
@@ -328,11 +334,6 @@ func (handler *Handler) stackDeleteKubernetesByName(w http.ResponseWriter, r *ht
for _, stack := range stacksToDelete {
log.Debug().Msgf("Trying to delete Kubernetes stack id `%d`", stack.ID)
// stop scheduler updates of the stack before removal
if stack.AutoUpdate != nil {
deployments.StopAutoupdate(stack.ID, stack.AutoUpdate.JobID, handler.Scheduler)
}
err = handler.deleteStack(context.TODO(), securityContext.UserID, &stack, endpoint)
if err != nil {
log.Err(err).Msgf("Unable to delete Kubernetes stack `%d`", stack.ID)
@@ -341,12 +342,18 @@ func (handler *Handler) stackDeleteKubernetesByName(w http.ResponseWriter, r *ht
continue
}
var reconcileSourceID portainer.SourceID
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
if stack.WorkflowID != 0 {
if err := tx.Workflow().Delete(stack.WorkflowID); err != nil {
if _, sid, err := loadGitConfigForStack(tx, source.InsecureNewAdminContext(), stack.WorkflowID, stack.ID); err == nil {
reconcileSourceID = sid
}
if err := workflows.DeleteIfSingleArtifact(tx, stack.WorkflowID); err != nil {
return err
}
}
return tx.Stack().Delete(stack.ID)
}); err != nil {
errs = errors.Join(errs, err)
@@ -355,6 +362,10 @@ func (handler *Handler) stackDeleteKubernetesByName(w http.ResponseWriter, r *ht
continue
}
if err := handler.SourceScheduler.Reconcile(reconcileSourceID); err != nil {
log.Warn().Err(err).Msg("source scheduler reconcile failed after Kubernetes stack deletion")
}
if err := handler.FileService.RemoveDirectory(stack.ProjectPath); err != nil {
errs = errors.Join(errs, err)
log.Warn().Err(err).Msg("Unable to remove stack files from disk")
-12
View File
@@ -12,7 +12,6 @@ import (
"github.com/portainer/portainer/api/dataservices/source"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/stacks/deployments"
"github.com/portainer/portainer/api/stacks/stackutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
@@ -126,17 +125,6 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http
return httperror.Forbidden("Access denied to resource", httperrors.ErrResourceAccessDenied)
}
if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
deployments.StopAutoupdate(stack.ID, stack.AutoUpdate.JobID, handler.Scheduler)
jobID, e := deployments.StartAutoupdate(context.TODO(), stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService)
if e != nil {
return e
}
stack.AutoUpdate.JobID = jobID
}
if err := handler.startStack(context.TODO(), securityContext.UserID, stack, endpoint, securityContext); err != nil {
stack.Status = portainer.StackStatusError
stack.DeploymentStatus = append(stack.DeploymentStatus, portainer.StackDeploymentStatus{
-7
View File
@@ -10,7 +10,6 @@ import (
"github.com/portainer/portainer/api/dataservices/source"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/stacks/deployments"
"github.com/portainer/portainer/api/stacks/stackutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
@@ -103,12 +102,6 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe
return httperror.Conflict("Stack deployment is in progress", errors.New("stack deployment is in progress"))
}
// stop scheduler updates of the stack before stopping
if stack.AutoUpdate != nil && stack.AutoUpdate.JobID != "" {
deployments.StopAutoupdate(stack.ID, stack.AutoUpdate.JobID, handler.Scheduler)
stack.AutoUpdate.JobID = ""
}
stopErr := handler.stopStack(r.Context(), securityContext.UserID, stack, endpoint)
if stopErr != nil {
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
+22 -6
View File
@@ -9,6 +9,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/gitops/workflows"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/stacks/deployments"
@@ -103,14 +104,33 @@ func (handler *Handler) stackUpdate(w http.ResponseWriter, r *http.Request) *htt
}
var stack *portainer.Stack
var reconcileSourceID portainer.SourceID
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
preStack, err := tx.Stack().Read(portainer.StackID(stackID))
if err == nil && preStack.WorkflowID != 0 {
securityContext, scErr := security.RetrieveRestrictedRequestContext(r)
if scErr == nil {
uc := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
if _, sid, err := loadGitConfigForStack(tx, uc, preStack.WorkflowID, preStack.ID); err == nil {
reconcileSourceID = sid
}
}
}
var httpErr *httperror.HandlerError
stack, httpErr = handler.updateStackInTx(tx, r, portainer.StackID(stackID), portainer.EndpointID(endpointID))
if httpErr != nil {
return httpErr
}
return nil
})
if err == nil {
if err := handler.SourceScheduler.Reconcile(reconcileSourceID); err != nil {
log.Warn().Err(err).Msg("source scheduler reconcile failed after stack update")
}
}
return response.TxResponse(w, stack, err)
}
@@ -215,9 +235,7 @@ func (handler *Handler) updateAndDeployStack(tx dataservices.DataStoreTx, r *htt
}
func (handler *Handler) updateComposeStack(tx dataservices.DataStoreTx, r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint, gate *deployGate) *httperror.HandlerError {
// Must not be git based stack. stop the auto update job if there is any
if stack.AutoUpdate != nil {
deployments.StopAutoupdate(stack.ID, stack.AutoUpdate.JobID, handler.Scheduler)
stack.AutoUpdate = nil
}
if stack.WorkflowID != 0 {
@@ -235,7 +253,7 @@ func (handler *Handler) updateComposeStack(tx dataservices.DataStoreTx, r *http.
if stack.WorkflowID != 0 {
oldWorkflowID := stack.WorkflowID
stack.WorkflowID = 0
if err := tx.Workflow().Delete(oldWorkflowID); err != nil {
if err := workflows.DeleteIfSingleArtifact(tx, oldWorkflowID); err != nil {
return httperror.InternalServerError("Unable to remove git workflow records from database", err)
}
}
@@ -298,9 +316,7 @@ func (handler *Handler) updateComposeStack(tx dataservices.DataStoreTx, r *http.
}
func (handler *Handler) updateSwarmStack(tx dataservices.DataStoreTx, r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint, gate *deployGate) *httperror.HandlerError {
// Must not be git based stack. stop the auto update job if there is any
if stack.AutoUpdate != nil {
deployments.StopAutoupdate(stack.ID, stack.AutoUpdate.JobID, handler.Scheduler)
stack.AutoUpdate = nil
}
if stack.WorkflowID != 0 {
@@ -317,7 +333,7 @@ func (handler *Handler) updateSwarmStack(tx dataservices.DataStoreTx, r *http.Re
if stack.WorkflowID != 0 {
oldWorkflowID := stack.WorkflowID
stack.WorkflowID = 0
if err := tx.Workflow().Delete(oldWorkflowID); err != nil {
if err := workflows.DeleteIfSingleArtifact(tx, oldWorkflowID); err != nil {
return httperror.InternalServerError("Unable to remove git workflow records from database", err)
}
}
+23 -15
View File
@@ -14,13 +14,13 @@ import (
"github.com/portainer/portainer/api/gitops/sources"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/stacks/deployments"
"github.com/portainer/portainer/api/stacks/stackutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
type stackGitUpdatePayload struct {
@@ -167,11 +167,6 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *
return httperror.Forbidden(errMsg, errors.New(errMsg))
}
//stop the autoupdate job if there is any
if stack.AutoUpdate != nil {
deployments.StopAutoupdate(stack.ID, stack.AutoUpdate.JobID, handler.Scheduler)
}
// Record the current git config as the deployment baseline if it was never set (legacy stacks).
if stack.CurrentDeploymentInfo == nil {
stack.CurrentDeploymentInfo = &portainer.StackDeploymentInfo{
@@ -250,12 +245,9 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *
}
}
if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" {
if jobID, err := deployments.StartAutoupdate(context.TODO(), stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService); err != nil {
return err
} else {
stack.AutoUpdate.JobID = jobID
}
effectiveSourceID := sourceID
if payload.SourceID != 0 {
effectiveSourceID = payload.SourceID
}
var resp *stackResponse
@@ -263,16 +255,32 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *
if err := tx.Stack().Update(stack.ID, stack); err != nil {
return err
}
userContext := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
if err := saveStackGitConfig(tx, userContext, stack.WorkflowID, stack.ID, sourceID, payload.SourceID, gitConfig); err != nil {
uc := source.NewUserContext(securityContext.User, securityContext.UserMemberships)
if err := saveStackGitConfig(tx, uc, stack.WorkflowID, stack.ID, sourceID, payload.SourceID, gitConfig); err != nil {
return err
}
var err error
resp, err = newStackResponse(tx, userContext, stack)
resp, err = newStackResponse(tx, uc, stack)
return err
}); err != nil {
return httperror.InternalServerError("Unable to persist the stack changes inside the database", err)
}
if err := handler.SourceScheduler.Reconcile(effectiveSourceID); err != nil {
log.Warn().Err(err).Msg("source scheduler reconcile failed after stack git update")
}
// The stack may have just been repointed away from its previous source; reconcile it
// too so an orphaned source's polling job is stopped instead of running forever.
if sourceID != 0 && sourceID != effectiveSourceID {
if err := handler.SourceScheduler.Reconcile(sourceID); err != nil {
log.Warn().Err(err).Msg("source scheduler reconcile failed after stack git update")
}
}
return response.JSON(w, resp)
}
@@ -71,11 +71,6 @@ func (handler *Handler) updateKubernetesStack(tx dataservices.DataStoreTx, r *ht
return httperror.InternalServerError("Stack has no git config in source", errors.New("source has no git config"))
}
// Stop the autoupdate job if there is any
if stack.AutoUpdate != nil {
deployments.StopAutoupdate(stack.ID, stack.AutoUpdate.JobID, handler.Scheduler)
}
var payload kubernetesGitStackUpdatePayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
@@ -111,14 +106,6 @@ func (handler *Handler) updateKubernetesStack(tx dataservices.DataStoreTx, r *ht
gitConfig.Authentication = nil
}
if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" {
jobID, e := deployments.StartAutoupdate(context.TODO(), stack.ID, stack.AutoUpdate.Interval, handler.Scheduler, handler.StackDeployer, handler.DataStore, handler.GitService)
if e != nil {
return e
}
stack.AutoUpdate.JobID = jobID
}
if err := saveStackGitConfig(tx, userContext, stack.WorkflowID, stack.ID, sourceID, 0, gitConfig); err != nil {
return httperror.InternalServerError("Unable to update source git config", err)
}
+4 -4
View File
@@ -15,6 +15,7 @@ import (
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/docker"
dockerclient "github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/gitops/scheduling"
"github.com/portainer/portainer/api/http/csrf"
"github.com/portainer/portainer/api/http/handler"
"github.com/portainer/portainer/api/http/handler/auth"
@@ -65,7 +66,6 @@ import (
motdservice "github.com/portainer/portainer/api/motd"
"github.com/portainer/portainer/api/pendingactions"
"github.com/portainer/portainer/api/platform"
"github.com/portainer/portainer/api/scheduler"
"github.com/portainer/portainer/api/stacks/deployments"
libhelmtypes "github.com/portainer/portainer/pkg/libhelm/types"
@@ -104,7 +104,7 @@ type Server struct {
KubernetesClientFactory *cli.ClientFactory
KubernetesDeployer portainer.KubernetesDeployer
HelmPackageManager libhelmtypes.HelmPackageManager
Scheduler *scheduler.Scheduler
SourceScheduler *scheduling.SourceScheduler
ShutdownTrigger context.CancelFunc
StackDeployer deployments.StackDeployer
UpgradeService upgrade.Service
@@ -208,7 +208,7 @@ func (server *Server) Start(ctx context.Context) error {
var endpointHelmHandler = helm.NewHandler(requestBouncer, server.DataStore, server.JWTService, server.KubernetesDeployer, server.HelmPackageManager, server.KubeClusterAccessService)
var gitOperationHandler = gitops.NewHandler(requestBouncer, server.DataStore, server.GitService, server.FileService, server.KubernetesClientFactory)
var gitOperationHandler = gitops.NewHandler(requestBouncer, server.DataStore, server.GitService, server.FileService, server.KubernetesClientFactory, server.SourceScheduler)
var helmTemplatesHandler = helm.NewTemplateHandler(requestBouncer, server.HelmPackageManager)
@@ -249,7 +249,7 @@ func (server *Server) Start(ctx context.Context) error {
stackHandler.KubernetesClientFactory = server.KubernetesClientFactory
stackHandler.KubernetesDeployer = server.KubernetesDeployer
stackHandler.GitService = server.GitService
stackHandler.Scheduler = server.Scheduler
stackHandler.SourceScheduler = server.SourceScheduler
stackHandler.SwarmStackManager = server.SwarmStackManager
stackHandler.ComposeStackManager = server.ComposeStackManager
stackHandler.StackDeployer = server.StackDeployer
-12
View File
@@ -431,18 +431,6 @@ func (s *stubStacksService) StacksByEndpointID(endpointID portainer.EndpointID)
return result, nil
}
func (s *stubStacksService) RefreshableStacks() ([]portainer.Stack, error) {
result := make([]portainer.Stack, 0)
for _, stack := range s.stacks {
if stack.AutoUpdate != nil {
result = append(result, stack)
}
}
return result, nil
}
func (s *stubStacksService) StackByName(name string) (*portainer.Stack, error) {
for _, stack := range s.stacks {
if stack.Name == name {
+3
View File
@@ -50,6 +50,8 @@ type (
// AutoUpdateSettings represents the git auto sync config for stack deployment
AutoUpdateSettings struct {
// Auto update interval
// Deprecated: polling interval now lives on the associated Source (Source.Interval).
// Kept for DB backwards-compatibility only; new code must not read or write this field.
Interval string `example:"1m30s"`
// A UUID generated from client
Webhook string `example:"05de31a2-79fa-4644-9c12-faa67e5c49f0"`
@@ -1356,6 +1358,7 @@ type (
OwnerID UserID `json:"ownerID,omitempty"`
Status SourceStatus `json:"status,omitempty"`
StatusError string `json:"statusError,omitempty"`
Interval string `json:"interval,omitempty" example:"5m"`
}
SourceStatus int
-36
View File
@@ -1,36 +0,0 @@
package deployments
import (
"context"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/scheduler"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/rs/zerolog/log"
)
func StartAutoupdate(ctx context.Context, stackID portainer.StackID, interval string, scheduler *scheduler.Scheduler, stackDeployer StackDeployer, datastore dataservices.DataStore, gitService portainer.GitService) (jobID string, e *httperror.HandlerError) {
d, err := time.ParseDuration(interval)
if err != nil {
return "", httperror.BadRequest("Unable to parse stack's auto update interval", err)
}
jobID = scheduler.StartJobEvery(d, func() error {
return RedeployWhenChanged(ctx, stackID, stackDeployer, datastore, gitService)
})
return jobID, nil
}
func StopAutoupdate(stackID portainer.StackID, jobID string, scheduler *scheduler.Scheduler) {
if jobID == "" {
return
}
if err := scheduler.StopJob(jobID); err != nil {
log.Warn().Int("stack_id", int(stackID)).Msg("could not stop the job for the stack")
}
}
+4
View File
@@ -44,6 +44,10 @@ func RedeployWhenChanged(ctx context.Context, stackID portainer.StackID, deploye
return errors.WithMessagef(err, "failed to get the stack %v", stackID)
}
if stack.Status == portainer.StackStatusInactive {
return nil
}
// Webhook
if stack.AutoUpdate != nil && stack.AutoUpdate.Webhook != "" {
return redeployWhenChanged(ctx, stack, deployer, datastore, gitService, true)
+10 -12
View File
@@ -210,19 +210,19 @@ func Test_redeployWhenChanged_DoesNothingWhenNoGitChanges(t *testing.T) {
err = store.Source().Create(adminUserContext, src)
require.NoError(t, err, "failed to create source")
wf := &portainer.Workflow{Artifacts: []portainer.Artifact{{StackID: 1, Files: []portainer.ArtifactFile{{SourceID: src.ID}}}}}
wf := &portainer.Workflow{Artifacts: []portainer.Artifact{{StackID: 2, Files: []portainer.ArtifactFile{{SourceID: src.ID}}}}}
err = store.Workflow().Create(wf)
require.NoError(t, err, "failed to create workflow")
err = store.Stack().Create(&portainer.Stack{
ID: 1,
ID: 2,
CreatedBy: "admin",
ProjectPath: tmpDir,
WorkflowID: wf.ID,
})
require.NoError(t, err, "failed to create a test stack")
err = RedeployWhenChanged(t.Context(), 1, nil, store, testhelpers.NewGitService(nil, "oldHash"))
err = RedeployWhenChanged(t.Context(), 2, nil, store, testhelpers.NewGitService(nil, "oldHash"))
require.NoError(t, err)
updatedSrc, err := store.Source().Read(adminUserContext, src.ID)
@@ -262,20 +262,20 @@ func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) {
require.NoError(t, err, "failed to create source")
wf := &portainer.Workflow{Artifacts: []portainer.Artifact{{
StackID: 1,
StackID: 3,
Files: []portainer.ArtifactFile{{SourceID: src.ID}},
}}}
err = store.Workflow().Create(wf)
require.NoError(t, err, "failed to create workflow")
err = store.Stack().Create(&portainer.Stack{
ID: 1,
ID: 3,
CreatedBy: "admin",
WorkflowID: wf.ID,
})
require.NoError(t, err, "failed to create a test stack")
err = RedeployWhenChanged(t.Context(), 1, nil, store, testhelpers.NewGitService(cloneErr, "newHash"))
err = RedeployWhenChanged(t.Context(), 3, nil, store, testhelpers.NewGitService(cloneErr, "newHash"))
require.Error(t, err)
require.ErrorIs(t, err, cloneErr, "should failed to clone but didn't, check test setup")
@@ -286,7 +286,7 @@ func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) {
require.Zero(t, updatedSrc.LastSync)
}
func setupRedeployStore(t *testing.T, stackType portainer.StackType) (dataservices.DataStore, portainer.StackID) {
func setupRedeployStore(t *testing.T, stackType portainer.StackType, stackID portainer.StackID) (dataservices.DataStore, portainer.StackID) {
t.Helper()
_, store := datastore.MustNewTestStore(t, false, true)
@@ -312,8 +312,6 @@ func setupRedeployStore(t *testing.T, stackType portainer.StackType) (dataservic
err = store.Workflow().Create(wf)
require.NoError(t, err, "failed to create workflow")
const stackID portainer.StackID = 1
err = store.Stack().Create(&portainer.Stack{
ID: stackID,
EndpointID: 1,
@@ -330,7 +328,7 @@ func setupRedeployStore(t *testing.T, stackType portainer.StackType) (dataservic
func Test_redeployWhenChanged_DockerComposeStack(t *testing.T) {
t.Parallel()
store, stackID := setupRedeployStore(t, portainer.DockerComposeStack)
store, stackID := setupRedeployStore(t, portainer.DockerComposeStack, 4)
err := RedeployWhenChanged(t.Context(), stackID, noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash"))
require.NoError(t, err)
@@ -339,7 +337,7 @@ func Test_redeployWhenChanged_DockerComposeStack(t *testing.T) {
func Test_redeployWhenChanged_DockerSwarmStack(t *testing.T) {
t.Parallel()
store, stackID := setupRedeployStore(t, portainer.DockerSwarmStack)
store, stackID := setupRedeployStore(t, portainer.DockerSwarmStack, 5)
err := RedeployWhenChanged(t.Context(), stackID, noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash"))
require.NoError(t, err)
@@ -348,7 +346,7 @@ func Test_redeployWhenChanged_DockerSwarmStack(t *testing.T) {
func Test_redeployWhenChanged_KubernetesStack(t *testing.T) {
t.Parallel()
store, stackID := setupRedeployStore(t, portainer.KubernetesStack)
store, stackID := setupRedeployStore(t, portainer.KubernetesStack, 6)
err := RedeployWhenChanged(t.Context(), stackID, noopDeployer{}, store, testhelpers.NewGitService(nil, "newHash"))
require.NoError(t, err)
-35
View File
@@ -1,35 +0,0 @@
package deployments
import (
"context"
"time"
"github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/scheduler"
)
func StartStackSchedules(scheduler *scheduler.Scheduler, stackdeployer StackDeployer, datastore dataservices.DataStore, gitService portainer.GitService) error {
stacks, err := datastore.Stack().RefreshableStacks()
if err != nil {
return errors.Wrap(err, "failed to fetch refreshable stacks")
}
for _, stack := range stacks {
d, err := time.ParseDuration(stack.AutoUpdate.Interval)
if err != nil {
return errors.Wrap(err, "Unable to parse auto update interval")
}
stackID := stack.ID // to be captured by the scheduled function
jobID := scheduler.StartJobEvery(d, func() error {
return RedeployWhenChanged(context.Background(), stackID, stackdeployer, datastore, gitService)
})
stack.AutoUpdate.JobID = jobID
if err := datastore.Stack().Update(stack.ID, &stack); err != nil {
return errors.Wrap(err, "failed to update stack job id")
}
}
return nil
}
@@ -5,8 +5,8 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/gitops/scheduling"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/scheduler"
"github.com/portainer/portainer/api/stacks/deployments"
)
@@ -20,14 +20,14 @@ func CreateComposeStackGitBuilder(securityContext *security.RestrictedRequestCon
dataStore dataservices.DataStore,
fileService portainer.FileService,
gitService portainer.GitService,
scheduler *scheduler.Scheduler,
sourceScheduler *scheduling.SourceScheduler,
stackDeployer deployments.StackDeployer) *ComposeStackGitBuilder {
return &ComposeStackGitBuilder{
GitMethodStackBuilder: GitMethodStackBuilder{
StackBuilder: CreateStackBuilder(dataStore, fileService, stackDeployer),
gitService: gitService,
scheduler: scheduler,
StackBuilder: CreateStackBuilder(dataStore, fileService, stackDeployer),
gitService: gitService,
sourceScheduler: sourceScheduler,
},
SecurityContext: securityContext,
}
+5 -5
View File
@@ -5,7 +5,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/scheduler"
"github.com/portainer/portainer/api/gitops/scheduling"
"github.com/portainer/portainer/api/stacks/deployments"
)
@@ -19,16 +19,16 @@ type KubernetesStackGitBuilder struct {
func CreateKubernetesStackGitBuilder(dataStore dataservices.DataStore,
fileService portainer.FileService,
gitService portainer.GitService,
scheduler *scheduler.Scheduler,
sourceScheduler *scheduling.SourceScheduler,
stackDeployer deployments.StackDeployer,
kubernetesDeployer portainer.KubernetesDeployer,
user *portainer.User) *KubernetesStackGitBuilder {
return &KubernetesStackGitBuilder{
GitMethodStackBuilder: GitMethodStackBuilder{
StackBuilder: CreateStackBuilder(dataStore, fileService, stackDeployer),
gitService: gitService,
scheduler: scheduler,
StackBuilder: CreateStackBuilder(dataStore, fileService, stackDeployer),
gitService: gitService,
sourceScheduler: sourceScheduler,
},
kubernetesDeployer: kubernetesDeployer,
user: user,
+17 -36
View File
@@ -10,9 +10,8 @@ import (
"github.com/portainer/portainer/api/dataservices/source"
"github.com/portainer/portainer/api/filesystem"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/gitops/scheduling"
"github.com/portainer/portainer/api/gitops/workflows"
"github.com/portainer/portainer/api/scheduler"
"github.com/portainer/portainer/api/stacks/deployments"
"github.com/portainer/portainer/api/stacks/stackutils"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/ssrf"
@@ -20,8 +19,9 @@ import (
type GitMethodStackBuilder struct {
StackBuilder
gitService portainer.GitService
scheduler *scheduler.Scheduler
gitService portainer.GitService
sourceScheduler *scheduling.SourceScheduler
resolvedSourceID portainer.SourceID
}
func (b *GitMethodStackBuilder) prepare(ctx context.Context, payload *StackPayload, userID portainer.UserID) error {
@@ -127,8 +127,16 @@ func (b *GitMethodStackBuilder) prepare(ctx context.Context, payload *StackPaylo
PathStatus: portainer.SourceStatusHealthy,
}
var resolvedSrc *portainer.Source
if sourceID != 0 {
file.SourceID = sourceID
s, err := tx.Source().Read(userContext, sourceID)
if err != nil {
return fmt.Errorf("failed to read source: %w", err)
}
file.SourceID = s.ID
resolvedSrc = s
} else {
repoConfig.URL = gittypes.SanitizeURL(repoConfig.URL)
@@ -146,6 +154,7 @@ func (b *GitMethodStackBuilder) prepare(ctx context.Context, payload *StackPaylo
}
file.SourceID = src.ID
resolvedSrc = src
}
if err := workflows.SaveSourceStatus(tx, userContext, file.SourceID, nil); err != nil {
@@ -164,6 +173,7 @@ func (b *GitMethodStackBuilder) prepare(ctx context.Context, payload *StackPaylo
}
workflowID = wf.ID
b.resolvedSourceID = resolvedSrc.ID
return nil
}); err != nil {
@@ -175,35 +185,6 @@ func (b *GitMethodStackBuilder) prepare(ctx context.Context, payload *StackPaylo
return nil
}
// postDeploy enables the auto-update scheduler job for the stack if configured,
// and persists the resulting job ID back to the database.
func (b *GitMethodStackBuilder) postDeploy(ctx context.Context, stack *portainer.Stack) error {
if stack.AutoUpdate == nil || stack.AutoUpdate.Interval == "" {
return nil
}
jobID, err := deployments.StartAutoupdate(ctx, stack.ID,
stack.AutoUpdate.Interval,
b.scheduler,
b.stackDeployer,
b.dataStore,
b.gitService)
if err != nil {
return err
}
return b.dataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
s, err := tx.Stack().Read(stack.ID)
if err != nil {
return fmt.Errorf("Unable to retrieve the stack from the database: %w", err)
}
s.AutoUpdate.JobID = jobID
if err := tx.Stack().Update(s.ID, s); err != nil {
return fmt.Errorf("Unable to update the stack inside the database: %w", err)
}
return nil
})
func (b *GitMethodStackBuilder) postDeploy(_ context.Context, _ *portainer.Stack) error {
return b.sourceScheduler.Reconcile(b.resolvedSourceID)
}
@@ -5,8 +5,8 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/gitops/scheduling"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/scheduler"
"github.com/portainer/portainer/api/stacks/deployments"
)
@@ -20,14 +20,14 @@ func CreateSwarmStackGitBuilder(securityContext *security.RestrictedRequestConte
dataStore dataservices.DataStore,
fileService portainer.FileService,
gitService portainer.GitService,
scheduler *scheduler.Scheduler,
sourceScheduler *scheduling.SourceScheduler,
stackDeployer deployments.StackDeployer) *SwarmStackGitBuilder {
return &SwarmStackGitBuilder{
GitMethodStackBuilder: GitMethodStackBuilder{
StackBuilder: CreateStackBuilder(dataStore, fileService, stackDeployer),
gitService: gitService,
scheduler: scheduler,
StackBuilder: CreateStackBuilder(dataStore, fileService, stackDeployer),
gitService: gitService,
sourceScheduler: sourceScheduler,
},
SecurityContext: securityContext,
}
-1
View File
@@ -18,5 +18,4 @@ export const KubernetesDeployRequestMethods = Object.freeze({
export const RepositoryMechanismTypes = Object.freeze({
WEBHOOK: 'Webhook',
INTERVAL: 'Interval',
});
@@ -161,8 +161,6 @@ class KubernetesDeployController {
function automaticUpdatesLabel(repositoryAutomaticUpdates, repositoryMechanism) {
switch (repositoryAutomaticUpdates && repositoryMechanism) {
case RepositoryMechanismTypes.INTERVAL:
return 'polling';
case RepositoryMechanismTypes.WEBHOOK:
return 'webhook';
default:
@@ -101,8 +101,6 @@ describe('CreateStackForm - Webhook ID Integration', () => {
AdditionalFiles: [],
AutoUpdate: {
RepositoryAutomaticUpdates: true,
RepositoryMechanism: 'Webhook',
RepositoryFetchInterval: '',
ForcePullImage: false,
RepositoryAutomaticUpdatesForce: false,
},
@@ -29,8 +29,6 @@ describe('GitSection', () => {
initialValues: {
AutoUpdate: {
RepositoryAutomaticUpdates: true,
RepositoryMechanism: 'Interval' as const,
RepositoryFetchInterval: '5m',
ForcePullImage: false,
RepositoryAutomaticUpdatesForce: false,
},
@@ -214,7 +214,6 @@ function InnerForm({
})
}
baseWebhookUrl={baseEdgeStackWebhookUrl()}
errors={errors.autoUpdate}
/>
</FormSection>
@@ -1751,6 +1751,8 @@ export type PortainerAutoUpdateSettings = {
ForceUpdate?: boolean;
/**
* Auto update interval
* Deprecated: polling interval now lives on the associated Source (Source.Interval).
* Kept for DB backwards-compatibility only; new code must not read or write this field.
*/
Interval?: string;
/**
@@ -3012,6 +3014,7 @@ export type PortainerSource = {
git?: GittypesGitSource;
helm?: PortainerHelmConfig;
id?: number;
interval?: string;
lastSync?: number;
name?: string;
ownerID?: number;
@@ -4335,6 +4338,7 @@ export type SourcesGitAuthenticationUpdatePayload = {
export type SourcesGitSourceCreatePayload = {
administratorsOnly?: boolean;
authentication?: SourcesGitAuthenticationPayload;
interval?: string;
name?: string;
public?: boolean;
teamAccesses?: Array<number>;
@@ -4345,6 +4349,7 @@ export type SourcesGitSourceCreatePayload = {
export type SourcesGitSourceUpdatePayload = {
authentication?: SourcesGitAuthenticationUpdatePayload;
interval?: string;
name?: string;
tlsSkipVerify?: boolean;
url?: string;
@@ -4354,6 +4359,7 @@ export type SourcesSource = {
environments?: number;
error?: string;
id: number;
interval?: string;
lastSync?: number;
name: string;
status: WorkflowsStatus;
@@ -4381,6 +4387,7 @@ export type SourcesSourceDetail = {
environments?: number;
error?: string;
id: number;
interval?: string;
lastSync?: number;
name: string;
status: WorkflowsStatus;
@@ -1634,6 +1634,7 @@ export const zPortainerSource = z.object({
git: zGittypesGitSource.optional(),
helm: zPortainerHelmConfig.optional(),
id: z.int().optional(),
interval: z.string().optional(),
lastSync: z.int().optional(),
name: z.string().optional(),
ownerID: z.int().optional(),
@@ -2048,6 +2049,7 @@ export const zSourcesGitAuthenticationUpdatePayload = z.object({
export const zSourcesGitSourceCreatePayload = z.object({
administratorsOnly: z.boolean().optional(),
authentication: zSourcesGitAuthenticationPayload.optional(),
interval: z.string().optional(),
name: z.string().optional(),
public: z.boolean().optional(),
teamAccesses: z.array(z.int()).optional(),
@@ -2058,6 +2060,7 @@ export const zSourcesGitSourceCreatePayload = z.object({
export const zSourcesGitSourceUpdatePayload = z.object({
authentication: zSourcesGitAuthenticationUpdatePayload.optional(),
interval: z.string().optional(),
name: z.string().optional(),
tlsSkipVerify: z.boolean().optional(),
url: z.string().optional(),
@@ -3316,6 +3319,7 @@ export const zSourcesSource = z.object({
environments: z.int().optional(),
error: z.string().optional(),
id: z.int(),
interval: z.string().optional(),
lastSync: z.int().optional(),
name: z.string(),
status: zWorkflowsStatus,
@@ -3387,6 +3391,7 @@ export const zSourcesSourceDetail = z.object({
environments: z.int().optional(),
error: z.string().optional(),
id: z.int(),
interval: z.string().optional(),
lastSync: z.int().optional(),
name: z.string(),
status: zWorkflowsStatus,
@@ -1,5 +1,3 @@
import { FormikErrors } from 'formik';
import { AutoUpdateModel } from '@/react/portainer/gitops/types';
import { SwitchField } from '@@/form-components/SwitchField';
@@ -12,7 +10,6 @@ export function AutoUpdateFieldset({
onChange,
environmentType,
isForcePullVisible = true,
errors,
baseWebhookUrl,
webhookId,
webhooksDocs,
@@ -21,7 +18,6 @@ export function AutoUpdateFieldset({
onChange: (value: AutoUpdateModel) => void;
environmentType?: 'DOCKER' | 'KUBERNETES';
isForcePullVisible?: boolean;
errors?: FormikErrors<AutoUpdateModel>;
baseWebhookUrl: string;
webhookId: string;
webhooksDocs?: string;
@@ -71,7 +67,6 @@ export function AutoUpdateFieldset({
onChange={handleChange}
environmentType={environmentType}
showForcePullImage={isForcePullVisible}
errors={errors}
webhookDocs={webhooksDocs}
/>
)}
@@ -1,15 +1,10 @@
import { FormikErrors } from 'formik';
import { FeatureId } from '@/react/portainer/feature-flags/enums';
import { type AutoUpdateModel } from '@/react/portainer/gitops/types';
import { ButtonSelector } from '@@/form-components/ButtonSelector/ButtonSelector';
import { FormControl } from '@@/form-components/FormControl';
import { SwitchField } from '@@/form-components/SwitchField';
import { TextTip } from '@@/Tip/TextTip';
import { ForceDeploymentSwitch } from './ForceDeploymentSwitch';
import { IntervalField } from './IntervalField';
import { WebhookSettings } from './WebhookSettings';
export function AutoUpdateSettings({
@@ -17,7 +12,6 @@ export function AutoUpdateSettings({
onChange,
environmentType,
showForcePullImage,
errors,
baseWebhookUrl,
webhookId,
webhookDocs,
@@ -26,7 +20,6 @@ export function AutoUpdateSettings({
onChange: (value: Partial<AutoUpdateModel>) => void;
environmentType?: 'DOCKER' | 'KUBERNETES';
showForcePullImage: boolean;
errors?: FormikErrors<AutoUpdateModel>;
baseWebhookUrl: string;
webhookId: string;
webhookDocs?: string;
@@ -39,33 +32,11 @@ export function AutoUpdateSettings({
repository content, which may cause service interruption.
</TextTip>
<FormControl label="Mechanism">
<ButtonSelector
size="small"
options={[
{ value: 'Interval', label: 'Polling' },
{ value: 'Webhook', label: 'Webhook' },
]}
value={value.RepositoryMechanism || 'Interval'}
onChange={(value) => onChange({ RepositoryMechanism: value })}
/>
</FormControl>
{value.RepositoryMechanism === 'Webhook' && (
<WebhookSettings
baseUrl={baseWebhookUrl}
value={webhookId}
docsLink={webhookDocs}
/>
)}
{value.RepositoryMechanism === 'Interval' && (
<IntervalField
value={value.RepositoryFetchInterval || ''}
onChange={(value) => onChange({ RepositoryFetchInterval: value })}
errors={errors?.RepositoryFetchInterval}
/>
)}
<WebhookSettings
baseUrl={baseWebhookUrl}
value={webhookId}
docsLink={webhookDocs}
/>
{showForcePullImage && (
<div className="form-group">
@@ -1,6 +1,10 @@
export type AutoUpdateMechanism = 'Webhook' | 'Interval';
export interface AutoUpdateResponse {
/* Auto update interval */
/**
* Auto update interval
*
* @deprecated polling interval now lives on the associated Source (Source.Interval).
* Kept for API backwards-compatibility only; the UI never reads or writes this field.
*/
Interval: string;
/* A UUID generated from client */
@@ -15,8 +19,6 @@ export interface AutoUpdateResponse {
export type AutoUpdateModel = {
RepositoryAutomaticUpdates: boolean;
RepositoryMechanism: AutoUpdateMechanism;
RepositoryFetchInterval: string;
ForcePullImage: boolean;
RepositoryAutomaticUpdatesForce: boolean;
};
@@ -25,8 +27,6 @@ export function getDefaultAutoUpdateValues(): AutoUpdateModel {
return {
RepositoryAutomaticUpdates: false,
RepositoryAutomaticUpdatesForce: false,
RepositoryMechanism: 'Interval',
RepositoryFetchInterval: '5m',
ForcePullImage: false,
};
}
@@ -40,8 +40,6 @@ export function parseAutoUpdateResponse(
return {
RepositoryAutomaticUpdates: true,
RepositoryMechanism: response.Interval ? 'Interval' : 'Webhook',
RepositoryFetchInterval: response.Interval || '',
RepositoryAutomaticUpdatesForce: response.ForceUpdate,
ForcePullImage: response.ForcePullImage,
};
@@ -55,17 +53,13 @@ export function transformAutoUpdateViewModel(
return null;
}
if (viewModel.RepositoryMechanism === 'Webhook' && !webhookId) {
if (!webhookId) {
throw new Error('Webhook ID is required');
}
return {
Interval:
viewModel.RepositoryMechanism === 'Interval'
? viewModel.RepositoryFetchInterval
: '',
Webhook:
viewModel.RepositoryMechanism === 'Webhook' && webhookId ? webhookId : '',
Interval: '',
Webhook: webhookId,
ForceUpdate: viewModel.RepositoryAutomaticUpdatesForce,
ForcePullImage: viewModel.ForcePullImage,
};
@@ -1,27 +1,11 @@
import { string, boolean, object, SchemaOf, mixed } from 'yup';
import { string, boolean, object, SchemaOf } from 'yup';
import { AutoUpdateMechanism, AutoUpdateModel } from '../types';
import { intervalValidation } from './IntervalField';
import { AutoUpdateModel } from '../types';
export function autoUpdateValidation(): SchemaOf<AutoUpdateModel> {
return object({
RepositoryAutomaticUpdates: boolean().default(false),
RepositoryAutomaticUpdatesForce: boolean().default(false),
RepositoryMechanism: mixed<AutoUpdateMechanism>()
.oneOf(['Interval', 'Webhook'])
.when('RepositoryAutomaticUpdates', {
is: true,
then: string().required(),
})
.default('Interval'),
RepositoryFetchInterval: string()
.default('')
.when(['RepositoryAutomaticUpdates', 'RepositoryMechanism'], {
is: (autoUpdates: boolean, mechanism: AutoUpdateMechanism) =>
autoUpdates && mechanism === 'Interval',
then: intervalValidation(),
}),
RepositoryWebhookId: string().default(''),
ForcePullImage: boolean().default(false),
});
-1
View File
@@ -96,7 +96,6 @@ export function GitForm({
value={value.AutoUpdate}
onChange={(value) => handleChange({ AutoUpdate: value })}
isForcePullVisible={isForcePullVisible}
errors={errors.AutoUpdate as FormikErrors<GitFormModel['AutoUpdate']>}
webhooksDocs={webhooksDocs}
/>
)}
@@ -31,6 +31,10 @@ export function CreateForm({ steps }: Props) {
authentication: {
authEnabled: true,
},
polling: {
enabled: false,
interval: '',
},
connectionOk: false,
},
authorizedTeams: [],
@@ -52,6 +52,7 @@ const initialFormValues: FormValues = {
url: '',
connectionOk: false,
authentication: { authEnabled: false },
polling: { enabled: false, interval: '' },
},
authorizedTeams: [],
authorizedUsers: [],
@@ -65,6 +66,7 @@ const validFormValues: FormValues = {
url: 'https://github.com/org/repo.git',
connectionOk: true,
authentication: { authEnabled: false },
polling: { enabled: false, interval: '' },
},
authorizedTeams: [],
authorizedUsers: [],
@@ -4,6 +4,7 @@ import { Input } from '@@/form-components/Input';
import { FormControl } from '@@/form-components/FormControl';
import { SwitchField } from '@@/form-components/SwitchField';
import { IntervalField } from '../../components/IntervalField';
import { FormValues } from '../type';
import { Authentication } from './Authentication';
@@ -47,6 +48,24 @@ export function ConfigureGit() {
<Authentication />
<SwitchField
label="Enable polling"
labelClass="col-sm-3 col-lg-2"
name="polling-enabled"
checked={values.git.polling.enabled}
onChange={(value) => setFieldValue('git.polling.enabled', value)}
tooltip="When enabled, Portainer periodically fetches this repository to detect changes."
data-cy="source-polling-switch"
/>
{values.git.polling.enabled && (
<IntervalField
value={values.git.polling.interval}
onChange={(value) => setFieldValue('git.polling.interval', value)}
errors={errors.git?.polling?.interval}
/>
)}
<ConnectionTest />
</div>
);
@@ -18,6 +18,7 @@ const baseGitValues: FormValues['git'] = {
authentication: {
authEnabled: false,
},
polling: { enabled: false, interval: '' },
};
const invalidGitValues: FormValues['git'] = {
@@ -27,6 +28,7 @@ const invalidGitValues: FormValues['git'] = {
authentication: {
authEnabled: false,
},
polling: { enabled: false, interval: '' },
};
function renderConnectionTest(gitValues: FormValues['git']) {
@@ -8,6 +8,7 @@ import { formValuesToCreatePayload, gitFormValuesToTestPayload } from './type';
const baseGit = {
url: 'https://github.com/org/repo.git',
tlsSkipVerify: false,
polling: { enabled: false, interval: '' },
connectionOk: false,
};
@@ -87,6 +88,36 @@ describe('formValuesToCreatePayload', () => {
expect(payload.git.authentication).toBeUndefined();
});
it('sends the interval when polling is enabled', () => {
const payload = formValuesToCreatePayload({
...baseUAC,
name: 'my-source',
type: 'git',
git: {
...baseGit,
authentication: { authEnabled: false },
polling: { enabled: true, interval: '5m' },
},
});
expect(payload.git.interval).toBe('5m');
});
it('sends an empty interval when polling is disabled', () => {
const payload = formValuesToCreatePayload({
...baseUAC,
name: 'my-source',
type: 'git',
git: {
...baseGit,
authentication: { authEnabled: false },
polling: { enabled: false, interval: '5m' },
},
});
expect(payload.git.interval).toBe('');
});
it('does not include connectionOk in the create payload', () => {
const payload = formValuesToCreatePayload({
...baseUAC,
@@ -18,6 +18,10 @@ type GitFormValues = {
password?: string;
};
tlsSkipVerify?: boolean;
polling: {
enabled: boolean;
interval: string;
};
/** Mirrors the connection-test result; not sent in the create payload. */
connectionOk: boolean;
};
@@ -33,7 +37,7 @@ export type FormValues = AccessControlFormData & {
export function formValuesToCreatePayload({
name,
type,
git: { authentication, tlsSkipVerify, url },
git: { authentication, tlsSkipVerify, url, polling },
authorizedTeams,
authorizedUsers,
ownership,
@@ -49,6 +53,7 @@ export function formValuesToCreatePayload({
public: ownership === ResourceControlOwnership.PUBLIC,
teamAccesses: authorizedTeams,
userAccesses: authorizedUsers,
interval: polling.enabled ? polling.interval : '',
},
};
}
@@ -12,6 +12,7 @@ const validGitValues = {
tlsSkipVerify: false,
connectionOk: true,
authentication: baseAuth,
polling: { enabled: false, interval: '' },
};
describe('validateGitConnection (pick schema — no connectionOk)', () => {
@@ -119,6 +120,72 @@ describe('validationSchema git.authentication', () => {
});
});
describe('validationSchema git.polling', () => {
it('fails when polling is enabled without an interval', async () => {
const schema = validationSchema();
const result = await schema.isValid({
name: 'src',
type: 'git',
git: {
...validGitValues,
polling: { enabled: true, interval: '' },
},
authorizedTeams: [],
authorizedUsers: [],
ownership: ResourceControlOwnership.ADMINISTRATORS,
} satisfies FormValues);
expect(result).toBe(false);
});
it('fails when the interval is below the 1 minute minimum', async () => {
const schema = validationSchema();
const result = await schema.isValid({
name: 'src',
type: 'git',
git: {
...validGitValues,
polling: { enabled: true, interval: '30s' },
},
authorizedTeams: [],
authorizedUsers: [],
ownership: ResourceControlOwnership.ADMINISTRATORS,
} satisfies FormValues);
expect(result).toBe(false);
});
it('passes when polling is enabled with a valid interval', async () => {
const schema = validationSchema();
const result = await schema.isValid({
name: 'src',
type: 'git',
git: {
...validGitValues,
polling: { enabled: true, interval: '5m' },
},
authorizedTeams: [],
authorizedUsers: [],
ownership: ResourceControlOwnership.ADMINISTRATORS,
} satisfies FormValues);
expect(result).toBe(true);
});
it('passes when polling is disabled regardless of interval', async () => {
const schema = validationSchema();
const result = await schema.isValid({
name: 'src',
type: 'git',
git: {
...validGitValues,
polling: { enabled: false, interval: '' },
},
authorizedTeams: [],
authorizedUsers: [],
ownership: ResourceControlOwnership.ADMINISTRATORS,
} satisfies FormValues);
expect(result).toBe(true);
});
});
describe('validationSchema full git (requires connectionOk)', () => {
it('fails when connectionOk is false', async () => {
const schema = validationSchema();
@@ -5,6 +5,8 @@ import { stringEnumValues } from '@/types';
import { isValidUrl } from '@@/form-components/validate-url';
import { intervalValidation } from '../components/IntervalField';
import { FormValues, FormValueTypes } from './type';
export function validationSchema(): SchemaOf<FormValues> {
@@ -52,6 +54,13 @@ function validateGit(): SchemaOf<FormValues['git']> {
)
),
tlsSkipVerify: bool(),
polling: object({
enabled: bool().required().default(false),
interval: string().defined().when('enabled', {
is: true,
then: intervalValidation(),
}),
}),
connectionOk: bool()
.oneOf([true], 'The connection test must succeed before continuing.')
.required(),
@@ -0,0 +1,40 @@
import { RefreshCwIcon } from 'lucide-react';
import { useFormikContext } from 'formik';
import { Card } from '@@/primitives/Card';
import { SwitchField } from '@@/form-components/SwitchField';
import { IntervalField } from '../../../components/IntervalField';
import { SettingsFormValues } from './types';
export function EditPollingWidget() {
const { values, errors, setFieldValue } =
useFormikContext<SettingsFormValues>();
return (
<Card.Container>
<Card.Header
icon={RefreshCwIcon}
title="Polling"
subtitle="Periodically fetch this repository to detect changes"
/>
<Card.Body>
<SwitchField
label="Enable polling"
name="pollingEnabled"
checked={values.pollingEnabled}
onChange={(value) => setFieldValue('pollingEnabled', value)}
data-cy="source-polling-switch"
/>
{values.pollingEnabled && (
<IntervalField
value={values.interval}
onChange={(value) => setFieldValue('interval', value)}
errors={errors.interval}
/>
)}
</Card.Body>
</Card.Container>
);
}
@@ -13,6 +13,7 @@ import { useUpdateSourceMutation } from '../../../queries/useUpdateSourceMutatio
import { EditConnectionDetailsWidget } from './EditConnectionDetailsWidget';
import { EditAuthWidget } from './EditAuthWidget';
import { EditPollingWidget } from './EditPollingWidget';
import { TestConnectionWidget } from './TestConnectionWidget';
import { SettingsFormValues, validationSchema } from './types';
import { buildUpdatePayload } from './payload';
@@ -35,6 +36,8 @@ export function SettingsForm({ source, onCancel }: Props) {
authEnabled: !!source.connection.authentication,
username: source.connection.authentication?.username ?? '',
password: '',
pollingEnabled: !!source.interval,
interval: source.interval ?? '',
};
return (
@@ -63,6 +66,7 @@ export function SettingsForm({ source, onCancel }: Props) {
>
<EditConnectionDetailsWidget />
<EditAuthWidget />
<EditPollingWidget />
<TestConnectionWidget sourceId={source.id} />
<StickyFooter className="gap-4">
<Button
@@ -0,0 +1,55 @@
import { buildUpdatePayload } from './payload';
import { SettingsFormValues } from './types';
const baseValues: SettingsFormValues = {
name: 'my-source',
url: 'https://github.com/org/repo.git',
tlsSkipVerify: false,
authEnabled: false,
username: '',
password: '',
pollingEnabled: false,
interval: '',
};
describe('buildUpdatePayload interval handling', () => {
it('omits interval when polling settings are unchanged', () => {
const payload = buildUpdatePayload(baseValues, baseValues);
expect(payload.interval).toBeUndefined();
});
it('sends the interval when polling is newly enabled', () => {
const values: SettingsFormValues = {
...baseValues,
pollingEnabled: true,
interval: '5m',
};
const payload = buildUpdatePayload(values, baseValues);
expect(payload.interval).toBe('5m');
});
it('sends an empty interval when polling is disabled', () => {
const initialValues: SettingsFormValues = {
...baseValues,
pollingEnabled: true,
interval: '5m',
};
const values: SettingsFormValues = {
...initialValues,
pollingEnabled: false,
};
const payload = buildUpdatePayload(values, initialValues);
expect(payload.interval).toBe('');
});
it('sends the updated interval when the value changes while enabled', () => {
const initialValues: SettingsFormValues = {
...baseValues,
pollingEnabled: true,
interval: '5m',
};
const values: SettingsFormValues = { ...initialValues, interval: '10m' };
const payload = buildUpdatePayload(values, initialValues);
expect(payload.interval).toBe('10m');
});
});
@@ -11,6 +11,10 @@ export function buildUpdatePayload(
url: changed(values.url, initialValues.url),
tlsSkipVerify: changed(values.tlsSkipVerify, initialValues.tlsSkipVerify),
authentication: buildAuthenticationPayload(values, initialValues),
interval: changed(
values.pollingEnabled ? values.interval : '',
initialValues.interval
),
};
}
@@ -1,5 +1,7 @@
import { boolean as yupBoolean, object, string } from 'yup';
import { intervalValidation } from '../../../components/IntervalField';
export interface SettingsFormValues {
name: string;
url: string;
@@ -7,6 +9,8 @@ export interface SettingsFormValues {
authEnabled: boolean;
username: string;
password: string;
pollingEnabled: boolean;
interval: string;
}
export const validationSchema = object({
@@ -20,4 +24,11 @@ export const validationSchema = object({
otherwise: (schema) => schema.optional(),
}),
password: string().optional(),
pollingEnabled: yupBoolean().defined(),
interval: string()
.defined()
.when('pollingEnabled', {
is: true,
then: () => intervalValidation(),
}),
});
@@ -0,0 +1,37 @@
import { RefreshCwIcon } from 'lucide-react';
import { Card } from '@@/primitives/Card';
import { DetailField } from './DetailField';
interface Props {
interval?: string;
}
export function PollingWidget({ interval }: Props) {
return (
<Card.Container>
<Card.Header
icon={RefreshCwIcon}
title="Polling"
subtitle="Periodically fetch this repository to detect changes"
/>
<Card.Body>
<div className="grid grid-cols-2 gap-4">
<DetailField label="Status">
<span className="text-gray-6 th-dark:text-gray-5">
{interval ? 'Enabled' : 'Disabled'}
</span>
</DetailField>
{interval && (
<DetailField label="Interval">
<span className="text-gray-6 th-dark:text-gray-5">
{interval}
</span>
</DetailField>
)}
</div>
</Card.Body>
</Card.Container>
);
}
@@ -3,6 +3,7 @@ import { SourceDetail } from '../../queries/useSource';
import { ConnectionDetailsWidget } from './ConnectionDetailsWidget';
import { AuthWidget } from './AuthWidget';
import { AutoUpdateWidget } from './AutoUpdateWidget';
import { PollingWidget } from './PollingWidget';
import { SyncStatusWidget } from './SyncStatusWidget';
import { SettingsForm } from './EditForm/SettingsForm';
@@ -23,6 +24,7 @@ export function SettingsTab({ source, isEditing, onEditingChange }: Props) {
<>
<ConnectionDetailsWidget source={source} />
<AuthWidget auth={source?.connection.authentication} />
<PollingWidget interval={source.interval} />
<AutoUpdateWidget autoUpdate={source.autoUpdate} />
<SyncStatusWidget source={source} />
</>
-1
View File
@@ -6,7 +6,6 @@ import {
} from './AutoUpdateFieldset/utils';
export type {
AutoUpdateMechanism,
AutoUpdateModel,
AutoUpdateResponse,
} from './AutoUpdateFieldset/utils';