feat(gitops): tidy up git auth [BE-12666] (#2026)

This commit is contained in:
Devon Steenberg
2026-03-23 13:53:04 +13:00
committed by GitHub
parent f199d0882f
commit bd9c3c1593
29 changed files with 477 additions and 635 deletions
+10 -11
View File
@@ -4,13 +4,13 @@
Portainer maintains both Short-Term Support (STS) and Long-Term Support (LTS) versions in accordance with our official [Portainer Lifecycle Policy](https://docs.portainer.io/start/lifecycle). Portainer maintains both Short-Term Support (STS) and Long-Term Support (LTS) versions in accordance with our official [Portainer Lifecycle Policy](https://docs.portainer.io/start/lifecycle).
| Version Type | Support Status | | Version Type | Support Status |
| --- | --- | | ------------------------ | ------------------------------------------- |
| LTS (Long-Term Support) | Supported for critical security fixes | | LTS (Long-Term Support) | Supported for critical security fixes |
| STS (Short-Term Support) | Supported until the next STS or LTS release | | STS (Short-Term Support) | Supported until the next STS or LTS release |
| Legacy / EOL | Not supported | | Legacy / EOL | Not supported |
For a detailed breakdown of current versions and their specific End of Life (EOL) dates, For a detailed breakdown of current versions and their specific End of Life (EOL) dates,
please refer to the [Portainer Lifecycle Policy](https://docs.portainer.io/start/lifecycle). please refer to the [Portainer Lifecycle Policy](https://docs.portainer.io/start/lifecycle).
## Reporting a Vulnerability ## Reporting a Vulnerability
@@ -23,17 +23,17 @@ The Portainer team takes the security of our products seriously. If you believe
1. **Report**: You can report in one of two ways: 1. **Report**: You can report in one of two ways:
- **GitHub**: Use the **Report a vulnerability** button on the **Security** tab of this repository. - **GitHub**: Use the **Report a vulnerability** button on the **Security** tab of this repository.
- **Email**: Send your findings to security@portainer.io. - **Email**: Send your findings to security@portainer.io.
2. **Details**: To help us verify the issue, please include: 2. **Details**: To help us verify the issue, please include:
- A description of the vulnerability and its potential impact. - A description of the vulnerability and its potential impact.
- Step-by-step instructions to reproduce the issue (e.g. proof-of-concept code, scripts, or screenshots). - Step-by-step instructions to reproduce the issue (e.g. proof-of-concept code, scripts, or screenshots).
- The version of the software and the environment in which it was found. - The version of the software and the environment in which it was found.
3. **Acknowledge**: We will acknowledge receipt of your report and provide an initial assessment. 3. **Acknowledge**: We will acknowledge receipt of your report and provide an initial assessment.
@@ -51,7 +51,6 @@ If you follow the responsible disclosure process, we will:
- Give credit for the discovery (if desired) once the fix is public. - Give credit for the discovery (if desired) once the fix is public.
We will make every effort to promptly address any security weaknesses. Security advisories and fixes will be published through GitHub Security Advisories and other channels as needed. We will make every effort to promptly address any security weaknesses. Security advisories and fixes will be published through GitHub Security Advisories and other channels as needed.
Thank you for helping keep Portainer and our community secure. Thank you for helping keep Portainer and our community secure.
+90 -34
View File
@@ -16,7 +16,9 @@ import (
"github.com/portainer/portainer/api/logs" "github.com/portainer/portainer/api/logs"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/filemode"
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/segmentio/encoding/json" "github.com/segmentio/encoding/json"
) )
@@ -26,7 +28,7 @@ const (
visualStudioHostSuffix = ".visualstudio.com" visualStudioHostSuffix = ".visualstudio.com"
) )
func isAzureUrl(s string) bool { func IsAzureUrl(s string) bool {
return strings.Contains(s, azureDevOpsHost) || return strings.Contains(s, azureDevOpsHost) ||
strings.Contains(s, visualStudioHostSuffix) strings.Contains(s, visualStudioHostSuffix)
} }
@@ -73,7 +75,11 @@ func newHttpClientForAzure(insecureSkipVerify bool) *http.Client {
return httpsCli return httpsCli
} }
func (a *azureClient) download(ctx context.Context, destination string, opt cloneOption) error { func (a *azureClient) Download(ctx context.Context, destination string, opt *git.CloneOptions) error {
if opt == nil {
return errors.New("options cannot be nil")
}
zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, opt) zipFilepath, err := a.downloadZipFromAzureDevOps(ctx, opt)
if err != nil { if err != nil {
return errors.Wrap(err, "failed to download a zip file from Azure DevOps") return errors.Wrap(err, "failed to download a zip file from Azure DevOps")
@@ -91,13 +97,13 @@ func (a *azureClient) download(ctx context.Context, destination string, opt clon
return nil return nil
} }
func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneOption) (string, error) { func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt *git.CloneOptions) (string, error) {
config, err := parseUrl(opt.repositoryUrl) config, err := parseUrl(opt.URL)
if err != nil { if err != nil {
return "", errors.WithMessage(err, "failed to parse url") return "", errors.WithMessage(err, "failed to parse url")
} }
downloadUrl, err := a.buildDownloadUrl(config, opt.referenceName) downloadUrl, err := a.buildDownloadUrl(config, string(opt.ReferenceName))
if err != nil { if err != nil {
return "", errors.WithMessage(err, "failed to build download url") return "", errors.WithMessage(err, "failed to build download url")
} }
@@ -109,9 +115,18 @@ func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneO
defer logs.CloseAndLogErr(zipFile) defer logs.CloseAndLogErr(zipFile)
var basicAuth *githttp.BasicAuth
if opt.Auth != nil {
var ok bool
basicAuth, ok = opt.Auth.(*githttp.BasicAuth)
if !ok {
return "", errors.New("only basic auth is supported for azure")
}
}
req, err := http.NewRequestWithContext(ctx, "GET", downloadUrl, nil) req, err := http.NewRequestWithContext(ctx, "GET", downloadUrl, nil)
if opt.username != "" || opt.password != "" { if basicAuth != nil {
req.SetBasicAuth(opt.username, opt.password) req.SetBasicAuth(basicAuth.Username, basicAuth.Password)
} else if config.username != "" || config.password != "" { } else if config.username != "" || config.password != "" {
req.SetBasicAuth(config.username, config.password) req.SetBasicAuth(config.username, config.password)
} }
@@ -120,7 +135,7 @@ func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneO
return "", errors.WithMessage(err, "failed to create a new HTTP request") return "", errors.WithMessage(err, "failed to create a new HTTP request")
} }
client := newHttpClientForAzure(opt.tlsSkipVerify) client := newHttpClientForAzure(opt.InsecureSkipTLS)
defer client.CloseIdleConnections() defer client.CloseIdleConnections()
res, err := client.Do(req) res, err := client.Do(req)
@@ -145,8 +160,12 @@ func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneO
return zipFile.Name(), nil return zipFile.Name(), nil
} }
func (a *azureClient) latestCommitID(ctx context.Context, opt fetchOption) (string, error) { func (a *azureClient) LatestCommitID(ctx context.Context, repositoryUrl, referenceName string, opt *git.ListOptions) (string, error) {
rootItem, err := a.getRootItem(ctx, opt) if opt == nil {
return "", errors.New("options cannot be nil")
}
rootItem, err := a.getRootItem(ctx, repositoryUrl, referenceName, opt)
if err != nil { if err != nil {
return "", err return "", err
} }
@@ -154,20 +173,29 @@ func (a *azureClient) latestCommitID(ctx context.Context, opt fetchOption) (stri
return rootItem.CommitId, nil return rootItem.CommitId, nil
} }
func (a *azureClient) getRootItem(ctx context.Context, opt fetchOption) (*azureItem, error) { func (a *azureClient) getRootItem(ctx context.Context, repositoryUrl, referenceName string, opt *git.ListOptions) (*azureItem, error) {
config, err := parseUrl(opt.repositoryUrl) config, err := parseUrl(repositoryUrl)
if err != nil { if err != nil {
return nil, errors.WithMessage(err, "failed to parse url") return nil, errors.WithMessage(err, "failed to parse url")
} }
rootItemUrl, err := a.buildRootItemUrl(config, opt.referenceName) rootItemUrl, err := a.buildRootItemUrl(config, referenceName)
if err != nil { if err != nil {
return nil, errors.WithMessage(err, "failed to build azure root item url") return nil, errors.WithMessage(err, "failed to build azure root item url")
} }
var basicAuth *githttp.BasicAuth
if opt.Auth != nil {
var ok bool
basicAuth, ok = opt.Auth.(*githttp.BasicAuth)
if !ok {
return nil, errors.New("only basic auth is supported for azure")
}
}
req, err := http.NewRequestWithContext(ctx, "GET", rootItemUrl, nil) req, err := http.NewRequestWithContext(ctx, "GET", rootItemUrl, nil)
if opt.username != "" || opt.password != "" { if basicAuth != nil {
req.SetBasicAuth(opt.username, opt.password) req.SetBasicAuth(basicAuth.Username, basicAuth.Password)
} else if config.username != "" || config.password != "" { } else if config.username != "" || config.password != "" {
req.SetBasicAuth(config.username, config.password) req.SetBasicAuth(config.username, config.password)
} }
@@ -176,7 +204,7 @@ func (a *azureClient) getRootItem(ctx context.Context, opt fetchOption) (*azureI
return nil, errors.WithMessage(err, "failed to create a new HTTP request") return nil, errors.WithMessage(err, "failed to create a new HTTP request")
} }
client := newHttpClientForAzure(opt.tlsSkipVerify) client := newHttpClientForAzure(opt.InsecureSkipTLS)
defer client.CloseIdleConnections() defer client.CloseIdleConnections()
resp, err := client.Do(req) resp, err := client.Do(req)
@@ -239,8 +267,10 @@ func parseSshUrl(rawUrl string) (*azureOptions, error) {
}, nil }, nil
} }
const expectedAzureDevOpsHttpUrl = "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository" const (
const expectedVisualStudioHttpUrl = "https://organisation.visualstudio.com/project/_git/repository" expectedAzureDevOpsHttpUrl = "https://Organisation@dev.azure.com/Organisation/Project/_git/Repository"
expectedVisualStudioHttpUrl = "https://organisation.visualstudio.com/project/_git/repository"
)
func parseHttpUrl(rawUrl string) (*azureOptions, error) { func parseHttpUrl(rawUrl string) (*azureOptions, error) {
u, err := url.Parse(rawUrl) u, err := url.Parse(rawUrl)
@@ -283,7 +313,6 @@ func (a *azureClient) buildDownloadUrl(config *azureOptions, referenceName strin
url.PathEscape(config.project), url.PathEscape(config.project),
url.PathEscape(config.repository)) url.PathEscape(config.repository))
u, err := url.Parse(rawUrl) u, err := url.Parse(rawUrl)
if err != nil { if err != nil {
return "", errors.Wrapf(err, "failed to parse download url path %s", rawUrl) return "", errors.Wrapf(err, "failed to parse download url path %s", rawUrl)
} }
@@ -310,7 +339,6 @@ func (a *azureClient) buildRootItemUrl(config *azureOptions, referenceName strin
url.PathEscape(config.project), url.PathEscape(config.project),
url.PathEscape(config.repository)) url.PathEscape(config.repository))
u, err := url.Parse(rawUrl) u, err := url.Parse(rawUrl)
if err != nil { if err != nil {
return "", errors.Wrapf(err, "failed to parse root item url path %s", rawUrl) return "", errors.Wrapf(err, "failed to parse root item url path %s", rawUrl)
} }
@@ -335,7 +363,6 @@ func (a *azureClient) buildRefsUrl(config *azureOptions) (string, error) {
url.PathEscape(config.project), url.PathEscape(config.project),
url.PathEscape(config.repository)) url.PathEscape(config.repository))
u, err := url.Parse(rawUrl) u, err := url.Parse(rawUrl)
if err != nil { if err != nil {
return "", errors.Wrapf(err, "failed to parse list refs url path %s", rawUrl) return "", errors.Wrapf(err, "failed to parse list refs url path %s", rawUrl)
} }
@@ -357,7 +384,6 @@ func (a *azureClient) buildTreeUrl(config *azureOptions, rootObjectHash string)
url.PathEscape(rootObjectHash), url.PathEscape(rootObjectHash),
) )
u, err := url.Parse(rawUrl) u, err := url.Parse(rawUrl)
if err != nil { if err != nil {
return "", errors.Wrapf(err, "failed to parse list tree url path %s", rawUrl) return "", errors.Wrapf(err, "failed to parse list tree url path %s", rawUrl)
} }
@@ -400,8 +426,12 @@ func getVersionType(name string) string {
return "commit" return "commit"
} }
func (a *azureClient) listRefs(ctx context.Context, opt baseOption) ([]string, error) { func (a *azureClient) ListRefs(ctx context.Context, repositoryUrl string, opt *git.ListOptions) ([]string, error) {
config, err := parseUrl(opt.repositoryUrl) if opt == nil {
return nil, errors.New("options cannot be nil")
}
config, err := parseUrl(repositoryUrl)
if err != nil { if err != nil {
return nil, errors.WithMessage(err, "failed to parse url") return nil, errors.WithMessage(err, "failed to parse url")
} }
@@ -411,9 +441,18 @@ func (a *azureClient) listRefs(ctx context.Context, opt baseOption) ([]string, e
return nil, errors.WithMessage(err, "failed to build list refs url") return nil, errors.WithMessage(err, "failed to build list refs url")
} }
var basicAuth *githttp.BasicAuth
if opt.Auth != nil {
var ok bool
basicAuth, ok = opt.Auth.(*githttp.BasicAuth)
if !ok {
return nil, errors.New("only basic auth is supported for azure")
}
}
req, err := http.NewRequestWithContext(ctx, "GET", listRefsUrl, nil) req, err := http.NewRequestWithContext(ctx, "GET", listRefsUrl, nil)
if opt.username != "" || opt.password != "" { if basicAuth != nil {
req.SetBasicAuth(opt.username, opt.password) req.SetBasicAuth(basicAuth.Username, basicAuth.Password)
} else if config.username != "" || config.password != "" { } else if config.username != "" || config.password != "" {
req.SetBasicAuth(config.username, config.password) req.SetBasicAuth(config.username, config.password)
} }
@@ -422,7 +461,7 @@ func (a *azureClient) listRefs(ctx context.Context, opt baseOption) ([]string, e
return nil, errors.WithMessage(err, "failed to create a new HTTP request") return nil, errors.WithMessage(err, "failed to create a new HTTP request")
} }
client := newHttpClientForAzure(opt.tlsSkipVerify) client := newHttpClientForAzure(opt.InsecureSkipTLS)
defer client.CloseIdleConnections() defer client.CloseIdleConnections()
resp, err := client.Do(req) resp, err := client.Do(req)
@@ -459,13 +498,21 @@ func (a *azureClient) listRefs(ctx context.Context, opt baseOption) ([]string, e
} }
// listFiles list all filenames under the specific repository // listFiles list all filenames under the specific repository
func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string, error) { func (a *azureClient) ListFiles(ctx context.Context, dirOnly bool, opt *git.CloneOptions) ([]string, error) {
rootItem, err := a.getRootItem(ctx, opt) if opt == nil {
return nil, errors.New("options cannot be nil")
}
listOptions := &git.ListOptions{
Auth: opt.Auth,
InsecureSkipTLS: opt.InsecureSkipTLS,
}
rootItem, err := a.getRootItem(ctx, opt.URL, string(opt.ReferenceName), listOptions)
if err != nil { if err != nil {
return nil, err return nil, err
} }
config, err := parseUrl(opt.repositoryUrl) config, err := parseUrl(opt.URL)
if err != nil { if err != nil {
return nil, errors.WithMessage(err, "failed to parse url") return nil, errors.WithMessage(err, "failed to parse url")
} }
@@ -475,9 +522,18 @@ func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string,
return nil, errors.WithMessage(err, "failed to build list tree url") return nil, errors.WithMessage(err, "failed to build list tree url")
} }
var basicAuth *githttp.BasicAuth
if opt.Auth != nil {
var ok bool
basicAuth, ok = opt.Auth.(*githttp.BasicAuth)
if !ok {
return nil, errors.New("only basic auth is supported for azure")
}
}
req, err := http.NewRequestWithContext(ctx, "GET", listTreeUrl, nil) req, err := http.NewRequestWithContext(ctx, "GET", listTreeUrl, nil)
if opt.username != "" || opt.password != "" { if basicAuth != nil {
req.SetBasicAuth(opt.username, opt.password) req.SetBasicAuth(basicAuth.Username, basicAuth.Password)
} else if config.username != "" || config.password != "" { } else if config.username != "" || config.password != "" {
req.SetBasicAuth(config.username, config.password) req.SetBasicAuth(config.username, config.password)
} }
@@ -486,7 +542,7 @@ func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string,
return nil, errors.WithMessage(err, "failed to create a new HTTP request") return nil, errors.WithMessage(err, "failed to create a new HTTP request")
} }
client := newHttpClientForAzure(opt.tlsSkipVerify) client := newHttpClientForAzure(opt.InsecureSkipTLS)
defer client.CloseIdleConnections() defer client.CloseIdleConnections()
resp, err := client.Do(req) resp, err := client.Do(req)
@@ -518,7 +574,7 @@ func (a *azureClient) listFiles(ctx context.Context, opt fetchOption) ([]string,
for _, treeEntry := range tree.TreeEntries { for _, treeEntry := range tree.TreeEntries {
mode, _ := filemode.New(treeEntry.Mode) mode, _ := filemode.New(treeEntry.Mode)
isDir := filemode.Dir == mode isDir := filemode.Dir == mode
if opt.dirOnly == isDir { if dirOnly == isDir {
allPaths = append(allPaths, treeEntry.RelativePath) allPaths = append(allPaths, treeEntry.RelativePath)
} }
} }
+49 -63
View File
@@ -65,7 +65,6 @@ func TestService_ClonePublicRepository_Azure(t *testing.T) {
tt.args.referenceName, tt.args.referenceName,
"", "",
"", "",
gittypes.GitCredentialAuthType_Basic,
false, false,
) )
require.NoError(t, err) require.NoError(t, err)
@@ -88,7 +87,6 @@ func TestService_ClonePrivateRepository_Azure(t *testing.T) {
"refs/heads/main", "refs/heads/main",
"", "",
pat, pat,
gittypes.GitCredentialAuthType_Basic,
false, false,
) )
require.NoError(t, err) require.NoError(t, err)
@@ -106,7 +104,6 @@ func TestService_LatestCommitID_Azure(t *testing.T) {
"refs/heads/main", "refs/heads/main",
"", "",
pat, pat,
gittypes.GitCredentialAuthType_Basic,
false, false,
) )
require.NoError(t, err) require.NoError(t, err)
@@ -124,7 +121,6 @@ func TestService_ListRefs_Azure(t *testing.T) {
privateAzureRepoURL, privateAzureRepoURL,
username, username,
accessToken, accessToken,
gittypes.GitCredentialAuthType_Basic,
false, false,
false, false,
) )
@@ -140,10 +136,10 @@ func TestService_ListRefs_Azure_Concurrently(t *testing.T) {
service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond) service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond)
go func() { go func() {
_, _ = service.ListRefs(privateAzureRepoURL, username, accessToken, gittypes.GitCredentialAuthType_Basic, false, false) _, _ = service.ListRefs(privateAzureRepoURL, username, accessToken, false, false)
}() }()
_, err := service.ListRefs(privateAzureRepoURL, username, accessToken, gittypes.GitCredentialAuthType_Basic, false, false) _, err := service.ListRefs(privateAzureRepoURL, username, accessToken, false, false)
require.NoError(t, err) require.NoError(t, err)
time.Sleep(2 * time.Second) time.Sleep(2 * time.Second)
@@ -152,6 +148,14 @@ func TestService_ListRefs_Azure_Concurrently(t *testing.T) {
func TestService_ListFiles_Azure(t *testing.T) { func TestService_ListFiles_Azure(t *testing.T) {
ensureIntegrationTest(t) ensureIntegrationTest(t)
type args struct {
repositoryUrl string
referenceName string
username string
password string
extensions []string
}
type expectResult struct { type expectResult struct {
shouldFail bool shouldFail bool
err error err error
@@ -163,22 +167,19 @@ func TestService_ListFiles_Azure(t *testing.T) {
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME") username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
tests := []struct { tests := []struct {
name string name string
args fetchOption args args
extensions []string expect expectResult
expect expectResult
}{ }{
{ {
name: "list tree with real repository and head ref but incorrect credential", name: "list tree with real repository and head ref but incorrect credential",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateAzureRepoURL,
repositoryUrl: privateAzureRepoURL,
username: "test-username",
password: "test-token",
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: "test-username",
password: "test-token",
extensions: []string{},
}, },
extensions: []string{},
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
err: gittypes.ErrAuthenticationFailure, err: gittypes.ErrAuthenticationFailure,
@@ -186,15 +187,13 @@ func TestService_ListFiles_Azure(t *testing.T) {
}, },
{ {
name: "list tree with real repository and head ref but no credential", name: "list tree with real repository and head ref but no credential",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateAzureRepoURL,
repositoryUrl: privateAzureRepoURL,
username: "",
password: "",
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: "",
password: "",
extensions: []string{},
}, },
extensions: []string{},
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
err: gittypes.ErrAuthenticationFailure, err: gittypes.ErrAuthenticationFailure,
@@ -202,15 +201,13 @@ func TestService_ListFiles_Azure(t *testing.T) {
}, },
{ {
name: "list tree with real repository and head ref", name: "list tree with real repository and head ref",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateAzureRepoURL,
repositoryUrl: privateAzureRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: username,
password: accessToken,
extensions: []string{},
}, },
extensions: []string{},
expect: expectResult{ expect: expectResult{
err: nil, err: nil,
matchedCount: 19, matchedCount: 19,
@@ -218,15 +215,13 @@ func TestService_ListFiles_Azure(t *testing.T) {
}, },
{ {
name: "list tree with real repository and head ref and existing file extension", name: "list tree with real repository and head ref and existing file extension",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateAzureRepoURL,
repositoryUrl: privateAzureRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: username,
password: accessToken,
extensions: []string{"yml"},
}, },
extensions: []string{"yml"},
expect: expectResult{ expect: expectResult{
err: nil, err: nil,
matchedCount: 2, matchedCount: 2,
@@ -234,15 +229,13 @@ func TestService_ListFiles_Azure(t *testing.T) {
}, },
{ {
name: "list tree with real repository and head ref and non-existing file extension", name: "list tree with real repository and head ref and non-existing file extension",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateAzureRepoURL,
repositoryUrl: privateAzureRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: username,
password: accessToken,
extensions: []string{"hcl"},
}, },
extensions: []string{"hcl"},
expect: expectResult{ expect: expectResult{
err: nil, err: nil,
matchedCount: 2, matchedCount: 2,
@@ -250,30 +243,26 @@ func TestService_ListFiles_Azure(t *testing.T) {
}, },
{ {
name: "list tree with real repository but non-existing ref", name: "list tree with real repository but non-existing ref",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateAzureRepoURL,
repositoryUrl: privateAzureRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature", referenceName: "refs/fake/feature",
username: username,
password: accessToken,
extensions: []string{},
}, },
extensions: []string{},
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
}, },
}, },
{ {
name: "list tree with fake repository ", name: "list tree with fake repository ",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateAzureRepoURL + "fake",
repositoryUrl: privateAzureRepoURL + "fake",
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature", referenceName: "refs/fake/feature",
username: username,
password: accessToken,
extensions: []string{},
}, },
extensions: []string{},
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
err: gittypes.ErrIncorrectRepositoryURL, err: gittypes.ErrIncorrectRepositoryURL,
@@ -288,10 +277,9 @@ func TestService_ListFiles_Azure(t *testing.T) {
tt.args.referenceName, tt.args.referenceName,
tt.args.username, tt.args.username,
tt.args.password, tt.args.password,
gittypes.GitCredentialAuthType_Basic,
false, false,
false, false,
tt.extensions, tt.args.extensions,
false, false,
) )
@@ -323,7 +311,6 @@ func TestService_ListFiles_Azure_Concurrently(t *testing.T) {
"refs/heads/main", "refs/heads/main",
username, username,
accessToken, accessToken,
gittypes.GitCredentialAuthType_Basic,
false, false,
false, false,
[]string{}, []string{},
@@ -336,7 +323,6 @@ func TestService_ListFiles_Azure_Concurrently(t *testing.T) {
"refs/heads/main", "refs/heads/main",
username, username,
accessToken, accessToken,
gittypes.GitCredentialAuthType_Basic,
false, false,
false, false,
[]string{}, []string{},
+93 -75
View File
@@ -7,6 +7,9 @@ import (
"net/url" "net/url"
"testing" "testing"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
gittypes "github.com/portainer/portainer/api/git/types" gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/pkg/fips" "github.com/portainer/portainer/pkg/fips"
@@ -234,7 +237,7 @@ func Test_isAzureUrl(t *testing.T) {
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, isAzureUrl(tt.args.s)) assert.Equal(t, tt.want, IsAzureUrl(tt.args.s))
}) })
} }
} }
@@ -243,7 +246,9 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
fips.InitFIPS(false) fips.InitFIPS(false)
type args struct { type args struct {
options baseOption repositoryUrl string
username string
password string
} }
type basicAuth struct { type basicAuth struct {
username, password string username, password string
@@ -256,9 +261,7 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
{ {
name: "username, password embedded", name: "username, password embedded",
args: args{ args: args{
options: baseOption{ repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
},
}, },
want: &basicAuth{ want: &basicAuth{
username: "username", username: "username",
@@ -268,11 +271,9 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
{ {
name: "username, password embedded, clone options take precedence", name: "username, password embedded, clone options take precedence",
args: args{ args: args{
options: baseOption{ repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository",
repositoryUrl: "https://username:password@dev.azure.com/Organisation/Project/_git/Repository", username: "u",
username: "u", password: "p",
password: "p",
},
}, },
want: &basicAuth{ want: &basicAuth{
username: "u", username: "u",
@@ -282,9 +283,7 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
{ {
name: "no credentials", name: "no credentials",
args: args{ args: args{
options: baseOption{ repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
},
}, },
}, },
} }
@@ -303,10 +302,14 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) {
baseUrl: server.URL, baseUrl: server.URL,
} }
option := cloneOption{ option := &git.CloneOptions{
fetchOption: fetchOption{ URL: tt.args.repositoryUrl,
baseOption: tt.args.options, }
}, if tt.args.username != "" || tt.args.password != "" {
option.Auth = &githttp.BasicAuth{
Username: tt.args.username,
Password: tt.args.password,
}
} }
_, err := a.downloadZipFromAzureDevOps(context.Background(), option) _, err := a.downloadZipFromAzureDevOps(context.Background(), option)
require.Error(t, err) require.Error(t, err)
@@ -340,18 +343,21 @@ func Test_azureDownloader_latestCommitID(t *testing.T) {
a := &azureClient{baseUrl: server.URL} a := &azureClient{baseUrl: server.URL}
type args struct {
repositoryUrl string
referenceName string
}
tests := []struct { tests := []struct {
name string name string
args fetchOption args args
want string want string
wantErr bool wantErr bool
}{ }{
{ {
name: "should be able to parse response", name: "should be able to parse response",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository",
},
referenceName: "", referenceName: "",
}, },
want: "27104ad7549d9e66685e115a497533f18024be9c", want: "27104ad7549d9e66685e115a497533f18024be9c",
@@ -361,7 +367,7 @@ func Test_azureDownloader_latestCommitID(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
id, err := a.latestCommitID(context.Background(), tt.args) id, err := a.LatestCommitID(context.Background(), tt.args.repositoryUrl, tt.args.referenceName, &git.ListOptions{})
if (err != nil) != tt.wantErr { if (err != nil) != tt.wantErr {
t.Errorf("azureDownloader.latestCommitID() error = %v, wantErr %v", err, tt.wantErr) t.Errorf("azureDownloader.latestCommitID() error = %v, wantErr %v", err, tt.wantErr)
return return
@@ -375,22 +381,23 @@ type testRepoManager struct {
called bool called bool
} }
func (t *testRepoManager) download(_ context.Context, _ string, _ cloneOption) error { func (t *testRepoManager) Download(_ context.Context, _ string, _ *git.CloneOptions) error {
t.called = true t.called = true
return nil return nil
} }
func (t *testRepoManager) latestCommitID(_ context.Context, _ fetchOption) (string, error) { func (t *testRepoManager) LatestCommitID(_ context.Context, _, _ string, _ *git.ListOptions) (string, error) {
return "", nil return "", nil
} }
func (t *testRepoManager) listRefs(_ context.Context, _ baseOption) ([]string, error) { func (t *testRepoManager) ListRefs(_ context.Context, _ string, _ *git.ListOptions) ([]string, error) {
return nil, nil return nil, nil
} }
func (t *testRepoManager) listFiles(_ context.Context, _ fetchOption) ([]string, error) { func (t *testRepoManager) ListFiles(_ context.Context, _ bool, _ *git.CloneOptions) ([]string, error) {
return nil, nil return nil, nil
} }
func Test_cloneRepository_azure(t *testing.T) { func Test_cloneRepository_azure(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@@ -420,15 +427,7 @@ func Test_cloneRepository_azure(t *testing.T) {
git := &testRepoManager{} git := &testRepoManager{}
s := &Service{azure: azure, git: git} s := &Service{azure: azure, git: git}
err := s.cloneRepository("", cloneOption{ err := s.CloneRepository("", tt.url, "", "", "", false)
fetchOption: fetchOption{
baseOption: baseOption{
repositoryUrl: tt.url,
},
},
depth: 1,
})
require.NoError(t, err) require.NoError(t, err)
// if azure API is called, git isn't and vice versa // if azure API is called, git isn't and vice versa
@@ -443,6 +442,12 @@ func Test_listRefs_azure(t *testing.T) {
client := NewAzureClient() client := NewAzureClient()
type args struct {
repositoryUrl string
username string
password string
}
type expectResult struct { type expectResult struct {
err error err error
refsCount int refsCount int
@@ -453,12 +458,12 @@ func Test_listRefs_azure(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
args baseOption args args
expect expectResult expect expectResult
}{ }{
{ {
name: "list refs of a real repository", name: "list refs of a real repository",
args: baseOption{ args: args{
repositoryUrl: privateAzureRepoURL, repositoryUrl: privateAzureRepoURL,
username: username, username: username,
password: accessToken, password: accessToken,
@@ -470,7 +475,7 @@ func Test_listRefs_azure(t *testing.T) {
}, },
{ {
name: "list refs of a real repository with incorrect credential", name: "list refs of a real repository with incorrect credential",
args: baseOption{ args: args{
repositoryUrl: privateAzureRepoURL, repositoryUrl: privateAzureRepoURL,
username: "test-username", username: "test-username",
password: "test-token", password: "test-token",
@@ -481,7 +486,7 @@ func Test_listRefs_azure(t *testing.T) {
}, },
{ {
name: "list refs of a real repository without providing credential", name: "list refs of a real repository without providing credential",
args: baseOption{ args: args{
repositoryUrl: privateAzureRepoURL, repositoryUrl: privateAzureRepoURL,
username: "", username: "",
password: "", password: "",
@@ -492,7 +497,7 @@ func Test_listRefs_azure(t *testing.T) {
}, },
{ {
name: "list refs of a fake repository", name: "list refs of a fake repository",
args: baseOption{ args: args{
repositoryUrl: privateAzureRepoURL + "fake", repositoryUrl: privateAzureRepoURL + "fake",
username: username, username: username,
password: accessToken, password: accessToken,
@@ -505,7 +510,14 @@ func Test_listRefs_azure(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
refs, err := client.listRefs(context.TODO(), tt.args) option := &git.ListOptions{}
if tt.args.username != "" || tt.args.password != "" {
option.Auth = &githttp.BasicAuth{
Username: tt.args.username,
Password: tt.args.password,
}
}
refs, err := client.ListRefs(context.TODO(), tt.args.repositoryUrl, option)
if tt.expect.err == nil { if tt.expect.err == nil {
require.NoError(t, err) require.NoError(t, err)
if tt.expect.refsCount > 0 { if tt.expect.refsCount > 0 {
@@ -517,7 +529,6 @@ func Test_listRefs_azure(t *testing.T) {
} }
}) })
} }
} }
func Test_listFiles_azure(t *testing.T) { func Test_listFiles_azure(t *testing.T) {
@@ -525,6 +536,13 @@ func Test_listFiles_azure(t *testing.T) {
client := NewAzureClient() client := NewAzureClient()
type args struct {
repositoryUrl string
referenceName string
username string
password string
}
type expectResult struct { type expectResult struct {
shouldFail bool shouldFail bool
err error err error
@@ -535,18 +553,16 @@ func Test_listFiles_azure(t *testing.T) {
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME") username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
tests := []struct { tests := []struct {
name string name string
args fetchOption args args
expect expectResult expect expectResult
}{ }{
{ {
name: "list tree with real repository and head ref but incorrect credential", name: "list tree with real repository and head ref but incorrect credential",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateAzureRepoURL,
repositoryUrl: privateAzureRepoURL,
username: "test-username",
password: "test-token",
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: "test-username",
password: "test-token",
}, },
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
@@ -555,13 +571,11 @@ func Test_listFiles_azure(t *testing.T) {
}, },
{ {
name: "list tree with real repository and head ref but no credential", name: "list tree with real repository and head ref but no credential",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateAzureRepoURL,
repositoryUrl: privateAzureRepoURL,
username: "",
password: "",
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: "",
password: "",
}, },
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
@@ -570,13 +584,11 @@ func Test_listFiles_azure(t *testing.T) {
}, },
{ {
name: "list tree with real repository and head ref", name: "list tree with real repository and head ref",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateAzureRepoURL,
repositoryUrl: privateAzureRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: username,
password: accessToken,
}, },
expect: expectResult{ expect: expectResult{
err: nil, err: nil,
@@ -585,13 +597,11 @@ func Test_listFiles_azure(t *testing.T) {
}, },
{ {
name: "list tree with real repository but non-existing ref", name: "list tree with real repository but non-existing ref",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateAzureRepoURL,
repositoryUrl: privateAzureRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature", referenceName: "refs/fake/feature",
username: username,
password: accessToken,
}, },
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
@@ -599,13 +609,11 @@ func Test_listFiles_azure(t *testing.T) {
}, },
{ {
name: "list tree with fake repository ", name: "list tree with fake repository ",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateAzureRepoURL + "fake",
repositoryUrl: privateAzureRepoURL + "fake",
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature", referenceName: "refs/fake/feature",
username: username,
password: accessToken,
}, },
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
@@ -616,7 +624,17 @@ func Test_listFiles_azure(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
paths, err := client.listFiles(context.TODO(), tt.args) option := &git.CloneOptions{
URL: tt.args.repositoryUrl,
ReferenceName: plumbing.ReferenceName(tt.args.referenceName),
}
if tt.args.username != "" || tt.args.password != "" {
option.Auth = &githttp.BasicAuth{
Username: tt.args.username,
Password: tt.args.password,
}
}
paths, err := client.ListFiles(context.TODO(), false, option)
if tt.expect.shouldFail { if tt.expect.shouldFail {
require.Error(t, err) require.Error(t, err)
if tt.expect.err != nil { if tt.expect.err != nil {
-2
View File
@@ -19,7 +19,6 @@ type CloneOptions struct {
ReferenceName string ReferenceName string
Username string Username string
Password string Password string
AuthType gittypes.GitCredentialAuthType
// TLSSkipVerify skips SSL verification when cloning the Git repository // TLSSkipVerify skips SSL verification when cloning the Git repository
TLSSkipVerify bool `example:"false"` TLSSkipVerify bool `example:"false"`
} }
@@ -49,7 +48,6 @@ func CloneWithBackup(gitService portainer.GitService, fileService portainer.File
options.ReferenceName, options.ReferenceName,
options.Username, options.Username,
options.Password, options.Password,
options.AuthType,
options.TLSSkipVerify, options.TLSSkipVerify,
); err != nil { ); err != nil {
cleanUp = false cleanUp = false
+12 -89
View File
@@ -11,11 +11,8 @@ import (
"github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport"
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/storage/memory" "github.com/go-git/go-git/v5/storage/memory"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@@ -30,21 +27,8 @@ func NewGitClient(preserveGitDir bool) *gitClient {
} }
} }
func (c *gitClient) download(ctx context.Context, dst string, opt cloneOption) error { func (c *gitClient) Download(ctx context.Context, dst string, opt *git.CloneOptions) error {
gitOptions := git.CloneOptions{ _, err := git.PlainCloneContext(ctx, dst, false, opt)
URL: opt.repositoryUrl,
Depth: opt.depth,
InsecureSkipTLS: opt.tlsSkipVerify,
Auth: getAuth(opt.authType, opt.username, opt.password),
Tags: git.NoTags,
}
if opt.referenceName != "" {
gitOptions.ReferenceName = plumbing.ReferenceName(opt.referenceName)
}
_, err := git.PlainCloneContext(ctx, dst, false, &gitOptions)
if err != nil { if err != nil {
if err.Error() == "authentication required" { if err.Error() == "authentication required" {
return gittypes.ErrAuthenticationFailure return gittypes.ErrAuthenticationFailure
@@ -62,18 +46,13 @@ func (c *gitClient) download(ctx context.Context, dst string, opt cloneOption) e
return nil return nil
} }
func (c *gitClient) latestCommitID(ctx context.Context, opt fetchOption) (string, error) { func (c *gitClient) LatestCommitID(ctx context.Context, repositoryUrl, referenceName string, opt *git.ListOptions) (string, error) {
remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{ remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
Name: "origin", Name: "origin",
URLs: []string{opt.repositoryUrl}, URLs: []string{repositoryUrl},
}) })
listOptions := &git.ListOptions{ refs, err := remote.List(opt)
Auth: getAuth(opt.authType, opt.username, opt.password),
InsecureSkipTLS: opt.tlsSkipVerify,
}
refs, err := remote.List(listOptions)
if err != nil { if err != nil {
if err.Error() == "authentication required" { if err.Error() == "authentication required" {
return "", gittypes.ErrAuthenticationFailure return "", gittypes.ErrAuthenticationFailure
@@ -81,7 +60,6 @@ func (c *gitClient) latestCommitID(ctx context.Context, opt fetchOption) (string
return "", errors.Wrap(err, "failed to list repository refs") return "", errors.Wrap(err, "failed to list repository refs")
} }
referenceName := opt.referenceName
if referenceName == "" { if referenceName == "" {
for _, ref := range refs { for _, ref := range refs {
if strings.EqualFold(ref.Name().String(), "HEAD") { if strings.EqualFold(ref.Name().String(), "HEAD") {
@@ -96,60 +74,16 @@ func (c *gitClient) latestCommitID(ctx context.Context, opt fetchOption) (string
} }
} }
return "", errors.Errorf("could not find ref %q in the repository", opt.referenceName) return "", errors.Errorf("could not find ref %q in the repository", referenceName)
} }
func getAuth(authType gittypes.GitCredentialAuthType, username, password string) transport.AuthMethod { func (c *gitClient) ListRefs(ctx context.Context, repositoryUrl string, opt *git.ListOptions) ([]string, error) {
if password == "" {
return nil
}
switch authType {
case gittypes.GitCredentialAuthType_Basic:
return getBasicAuth(username, password)
case gittypes.GitCredentialAuthType_Token:
return getTokenAuth(password)
default:
log.Warn().Msg("unknown git credentials authorization type, defaulting to None")
return nil
}
}
func getBasicAuth(username, password string) *githttp.BasicAuth {
if password != "" {
if username == "" {
username = "token"
}
return &githttp.BasicAuth{
Username: username,
Password: password,
}
}
return nil
}
func getTokenAuth(token string) *githttp.TokenAuth {
if token != "" {
return &githttp.TokenAuth{
Token: token,
}
}
return nil
}
func (c *gitClient) listRefs(ctx context.Context, opt baseOption) ([]string, error) {
rem := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{ rem := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{
Name: "origin", Name: "origin",
URLs: []string{opt.repositoryUrl}, URLs: []string{repositoryUrl},
}) })
listOptions := &git.ListOptions{ refs, err := rem.List(opt)
Auth: getAuth(opt.authType, opt.username, opt.password),
InsecureSkipTLS: opt.tlsSkipVerify,
}
refs, err := rem.List(listOptions)
if err != nil { if err != nil {
return nil, checkGitError(err) return nil, checkGitError(err)
} }
@@ -166,19 +100,8 @@ func (c *gitClient) listRefs(ctx context.Context, opt baseOption) ([]string, err
} }
// listFiles list all filenames under the specific repository // listFiles list all filenames under the specific repository
func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, error) { func (c *gitClient) ListFiles(ctx context.Context, dirOnly bool, opt *git.CloneOptions) ([]string, error) {
cloneOption := &git.CloneOptions{ repo, err := git.Clone(memory.NewStorage(), nil, opt)
URL: opt.repositoryUrl,
NoCheckout: true,
Depth: 1,
SingleBranch: true,
ReferenceName: plumbing.ReferenceName(opt.referenceName),
Auth: getAuth(opt.authType, opt.username, opt.password),
InsecureSkipTLS: opt.tlsSkipVerify,
Tags: git.NoTags,
}
repo, err := git.Clone(memory.NewStorage(), nil, cloneOption)
if err != nil { if err != nil {
return nil, checkGitError(err) return nil, checkGitError(err)
} }
@@ -210,7 +133,7 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e
} }
isDir := entry.Mode == filemode.Dir isDir := entry.Mode == filemode.Dir
if opt.dirOnly == isDir { if dirOnly == isDir {
allPaths = append(allPaths, name) allPaths = append(allPaths, name)
} }
} }
+57 -76
View File
@@ -34,7 +34,6 @@ func TestService_ClonePrivateRepository_GitHub(t *testing.T) {
"refs/heads/main", "refs/heads/main",
username, username,
accessToken, accessToken,
gittypes.GitCredentialAuthType_Basic,
false, false,
) )
require.NoError(t, err) require.NoError(t, err)
@@ -54,7 +53,6 @@ func TestService_LatestCommitID_GitHub(t *testing.T) {
"refs/heads/main", "refs/heads/main",
username, username,
accessToken, accessToken,
gittypes.GitCredentialAuthType_Basic,
false, false,
) )
require.NoError(t, err) require.NoError(t, err)
@@ -69,7 +67,7 @@ func TestService_ListRefs_GitHub(t *testing.T) {
service := newService(context.TODO(), 0, 0) service := newService(context.TODO(), 0, 0)
repositoryUrl := privateGitRepoURL repositoryUrl := privateGitRepoURL
refs, err := service.ListRefs(repositoryUrl, username, accessToken, gittypes.GitCredentialAuthType_Basic, false, false) refs, err := service.ListRefs(repositoryUrl, username, accessToken, false, false)
require.NoError(t, err) require.NoError(t, err)
assert.GreaterOrEqual(t, len(refs), 1) assert.GreaterOrEqual(t, len(refs), 1)
} }
@@ -83,10 +81,10 @@ func TestService_ListRefs_Github_Concurrently(t *testing.T) {
repositoryUrl := privateGitRepoURL repositoryUrl := privateGitRepoURL
go func() { go func() {
_, _ = service.ListRefs(repositoryUrl, username, accessToken, gittypes.GitCredentialAuthType_Basic, false, false) _, _ = service.ListRefs(repositoryUrl, username, accessToken, false, false)
}() }()
_, err := service.ListRefs(repositoryUrl, username, accessToken, gittypes.GitCredentialAuthType_Basic, false, false) _, err := service.ListRefs(repositoryUrl, username, accessToken, false, false)
require.NoError(t, err) require.NoError(t, err)
time.Sleep(2 * time.Second) time.Sleep(2 * time.Second)
@@ -95,6 +93,14 @@ func TestService_ListRefs_Github_Concurrently(t *testing.T) {
func TestService_ListFiles_GitHub(t *testing.T) { func TestService_ListFiles_GitHub(t *testing.T) {
ensureIntegrationTest(t) ensureIntegrationTest(t)
type args struct {
repositoryUrl string
referenceName string
username string
password string
extensions []string
}
type expectResult struct { type expectResult struct {
shouldFail bool shouldFail bool
err error err error
@@ -105,22 +111,19 @@ func TestService_ListFiles_GitHub(t *testing.T) {
username := getRequiredValue(t, "GITHUB_USERNAME") username := getRequiredValue(t, "GITHUB_USERNAME")
tests := []struct { tests := []struct {
name string name string
args fetchOption args args
extensions []string expect expectResult
expect expectResult
}{ }{
{ {
name: "list tree with real repository and head ref but incorrect credential", name: "list tree with real repository and head ref but incorrect credential",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateGitRepoURL,
repositoryUrl: privateGitRepoURL,
username: "test-username",
password: "test-token",
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: "test-username",
password: "test-token",
extensions: []string{},
}, },
extensions: []string{},
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
err: gittypes.ErrAuthenticationFailure, err: gittypes.ErrAuthenticationFailure,
@@ -128,15 +131,13 @@ func TestService_ListFiles_GitHub(t *testing.T) {
}, },
{ {
name: "list tree with real repository and head ref but no credential", name: "list tree with real repository and head ref but no credential",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateGitRepoURL + "fake",
repositoryUrl: privateGitRepoURL + "fake",
username: "",
password: "",
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: "",
password: "",
extensions: []string{},
}, },
extensions: []string{},
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
err: gittypes.ErrAuthenticationFailure, err: gittypes.ErrAuthenticationFailure,
@@ -144,15 +145,13 @@ func TestService_ListFiles_GitHub(t *testing.T) {
}, },
{ {
name: "list tree with real repository and head ref", name: "list tree with real repository and head ref",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateGitRepoURL,
repositoryUrl: privateGitRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: username,
password: accessToken,
extensions: []string{},
}, },
extensions: []string{},
expect: expectResult{ expect: expectResult{
err: nil, err: nil,
matchedCount: 15, matchedCount: 15,
@@ -160,15 +159,13 @@ func TestService_ListFiles_GitHub(t *testing.T) {
}, },
{ {
name: "list tree with real repository and head ref and existing file extension", name: "list tree with real repository and head ref and existing file extension",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateGitRepoURL,
repositoryUrl: privateGitRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: username,
password: accessToken,
extensions: []string{"yml"},
}, },
extensions: []string{"yml"},
expect: expectResult{ expect: expectResult{
err: nil, err: nil,
matchedCount: 2, matchedCount: 2,
@@ -176,15 +173,13 @@ func TestService_ListFiles_GitHub(t *testing.T) {
}, },
{ {
name: "list tree with real repository and head ref and non-existing file extension", name: "list tree with real repository and head ref and non-existing file extension",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateGitRepoURL,
repositoryUrl: privateGitRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: username,
password: accessToken,
extensions: []string{"hcl"},
}, },
extensions: []string{"hcl"},
expect: expectResult{ expect: expectResult{
err: nil, err: nil,
matchedCount: 2, matchedCount: 2,
@@ -192,30 +187,26 @@ func TestService_ListFiles_GitHub(t *testing.T) {
}, },
{ {
name: "list tree with real repository but non-existing ref", name: "list tree with real repository but non-existing ref",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateGitRepoURL,
repositoryUrl: privateGitRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature", referenceName: "refs/fake/feature",
username: username,
password: accessToken,
extensions: []string{},
}, },
extensions: []string{},
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
}, },
}, },
{ {
name: "list tree with fake repository ", name: "list tree with fake repository ",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateGitRepoURL + "fake",
repositoryUrl: privateGitRepoURL + "fake",
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature", referenceName: "refs/fake/feature",
username: username,
password: accessToken,
extensions: []string{},
}, },
extensions: []string{},
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
err: gittypes.ErrIncorrectRepositoryURL, err: gittypes.ErrIncorrectRepositoryURL,
@@ -230,10 +221,9 @@ func TestService_ListFiles_GitHub(t *testing.T) {
tt.args.referenceName, tt.args.referenceName,
tt.args.username, tt.args.username,
tt.args.password, tt.args.password,
gittypes.GitCredentialAuthType_Basic,
false, false,
false, false,
tt.extensions, tt.args.extensions,
false, false,
) )
if tt.expect.shouldFail { if tt.expect.shouldFail {
@@ -265,7 +255,6 @@ func TestService_ListFiles_Github_Concurrently(t *testing.T) {
"refs/heads/main", "refs/heads/main",
username, username,
accessToken, accessToken,
gittypes.GitCredentialAuthType_Basic,
false, false,
false, false,
[]string{}, []string{},
@@ -278,7 +267,6 @@ func TestService_ListFiles_Github_Concurrently(t *testing.T) {
"refs/heads/main", "refs/heads/main",
username, username,
accessToken, accessToken,
gittypes.GitCredentialAuthType_Basic,
false, false,
false, false,
[]string{}, []string{},
@@ -297,7 +285,7 @@ func TestService_purgeCache_Github(t *testing.T) {
username := getRequiredValue(t, "GITHUB_USERNAME") username := getRequiredValue(t, "GITHUB_USERNAME")
service := NewService(context.TODO()) service := NewService(context.TODO())
_, err := service.ListRefs(repositoryUrl, username, accessToken, gittypes.GitCredentialAuthType_Basic, false, false) _, err := service.ListRefs(repositoryUrl, username, accessToken, false, false)
require.NoError(t, err) require.NoError(t, err)
_, err = service.ListFiles( _, err = service.ListFiles(
@@ -305,7 +293,6 @@ func TestService_purgeCache_Github(t *testing.T) {
"refs/heads/main", "refs/heads/main",
username, username,
accessToken, accessToken,
gittypes.GitCredentialAuthType_Basic,
false, false,
false, false,
[]string{}, []string{},
@@ -331,14 +318,13 @@ func TestService_purgeCacheByTTL_Github(t *testing.T) {
// 40*timeout is designed for giving enough time for ListRefs and ListFiles to cache the result // 40*timeout is designed for giving enough time for ListRefs and ListFiles to cache the result
service := newService(context.TODO(), 2, 40*timeout) service := newService(context.TODO(), 2, 40*timeout)
_, err := service.ListRefs(repositoryUrl, username, accessToken, gittypes.GitCredentialAuthType_Basic, false, false) _, err := service.ListRefs(repositoryUrl, username, accessToken, false, false)
require.NoError(t, err) require.NoError(t, err)
_, err = service.ListFiles( _, err = service.ListFiles(
repositoryUrl, repositoryUrl,
"refs/heads/main", "refs/heads/main",
username, username,
accessToken, accessToken,
gittypes.GitCredentialAuthType_Basic,
false, false,
false, false,
[]string{}, []string{},
@@ -375,12 +361,12 @@ func TestService_HardRefresh_ListRefs_GitHub(t *testing.T) {
service := newService(context.TODO(), 2, 0) service := newService(context.TODO(), 2, 0)
repositoryUrl := privateGitRepoURL repositoryUrl := privateGitRepoURL
refs, err := service.ListRefs(repositoryUrl, username, accessToken, gittypes.GitCredentialAuthType_Basic, false, false) refs, err := service.ListRefs(repositoryUrl, username, accessToken, false, false)
require.NoError(t, err) require.NoError(t, err)
assert.GreaterOrEqual(t, len(refs), 1) assert.GreaterOrEqual(t, len(refs), 1)
assert.Equal(t, 1, service.repoRefCache.Len()) assert.Equal(t, 1, service.repoRefCache.Len())
_, err = service.ListRefs(repositoryUrl, username, "fake-token", gittypes.GitCredentialAuthType_Basic, false, false) _, err = service.ListRefs(repositoryUrl, username, "fake-token", false, false)
require.Error(t, err) require.Error(t, err)
assert.Equal(t, 1, service.repoRefCache.Len()) assert.Equal(t, 1, service.repoRefCache.Len())
} }
@@ -393,7 +379,7 @@ func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
service := newService(context.TODO(), 2, 0) service := newService(context.TODO(), 2, 0)
repositoryUrl := privateGitRepoURL repositoryUrl := privateGitRepoURL
refs, err := service.ListRefs(repositoryUrl, username, accessToken, gittypes.GitCredentialAuthType_Basic, false, false) refs, err := service.ListRefs(repositoryUrl, username, accessToken, false, false)
require.NoError(t, err) require.NoError(t, err)
assert.GreaterOrEqual(t, len(refs), 1) assert.GreaterOrEqual(t, len(refs), 1)
assert.Equal(t, 1, service.repoRefCache.Len()) assert.Equal(t, 1, service.repoRefCache.Len())
@@ -403,7 +389,6 @@ func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
"refs/heads/main", "refs/heads/main",
username, username,
accessToken, accessToken,
gittypes.GitCredentialAuthType_Basic,
false, false,
false, false,
[]string{}, []string{},
@@ -418,7 +403,6 @@ func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
"refs/heads/test", "refs/heads/test",
username, username,
accessToken, accessToken,
gittypes.GitCredentialAuthType_Basic,
false, false,
false, false,
[]string{}, []string{},
@@ -428,11 +412,11 @@ func TestService_HardRefresh_ListRefs_And_RemoveAllCaches_GitHub(t *testing.T) {
assert.GreaterOrEqual(t, len(files), 1) assert.GreaterOrEqual(t, len(files), 1)
assert.Equal(t, 2, service.repoFileCache.Len()) assert.Equal(t, 2, service.repoFileCache.Len())
_, err = service.ListRefs(repositoryUrl, username, "fake-token", gittypes.GitCredentialAuthType_Basic, false, false) _, err = service.ListRefs(repositoryUrl, username, "fake-token", false, false)
require.Error(t, err) require.Error(t, err)
assert.Equal(t, 1, service.repoRefCache.Len()) assert.Equal(t, 1, service.repoRefCache.Len())
_, err = service.ListRefs(repositoryUrl, username, "fake-token", gittypes.GitCredentialAuthType_Basic, true, false) _, err = service.ListRefs(repositoryUrl, username, "fake-token", true, false)
require.Error(t, err) require.Error(t, err)
assert.Equal(t, 1, service.repoRefCache.Len()) assert.Equal(t, 1, service.repoRefCache.Len())
// The relevant file caches should be removed too // The relevant file caches should be removed too
@@ -451,7 +435,6 @@ func TestService_HardRefresh_ListFiles_GitHub(t *testing.T) {
"refs/heads/main", "refs/heads/main",
username, username,
accessToken, accessToken,
gittypes.GitCredentialAuthType_Basic,
false, false,
false, false,
[]string{}, []string{},
@@ -466,7 +449,6 @@ func TestService_HardRefresh_ListFiles_GitHub(t *testing.T) {
"refs/heads/main", "refs/heads/main",
username, username,
"fake-token", "fake-token",
gittypes.GitCredentialAuthType_Basic,
false, false,
true, true,
[]string{}, []string{},
@@ -495,7 +477,6 @@ func TestService_CloneRepository_TokenAuth(t *testing.T) {
"refs/heads/main", "refs/heads/main",
username, username,
accessToken, accessToken,
gittypes.GitCredentialAuthType_Token,
false, false,
) )
+65 -67
View File
@@ -10,7 +10,9 @@ import (
gittypes "github.com/portainer/portainer/api/git/types" gittypes "github.com/portainer/portainer/api/git/types"
"github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/object"
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -20,7 +22,7 @@ func setup(t *testing.T) string {
dir := t.TempDir() dir := t.TempDir()
bareRepoDir := filepath.Join(dir, "test-clone.git") bareRepoDir := filepath.Join(dir, "test-clone.git")
file, err := os.OpenFile("./testdata/test-clone-git-repo.tar.gz", os.O_RDONLY, 0755) file, err := os.OpenFile("./testdata/test-clone-git-repo.tar.gz", os.O_RDONLY, 0o755)
if err != nil { if err != nil {
t.Fatal(errors.Wrap(err, "failed to open an archive")) t.Fatal(errors.Wrap(err, "failed to open an archive"))
} }
@@ -39,7 +41,7 @@ func Test_ClonePublicRepository_Shallow(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
t.Logf("Cloning into %s", dir) t.Logf("Cloning into %s", dir)
err := service.CloneRepository(dir, repositoryURL, referenceName, "", "", gittypes.GitCredentialAuthType_Basic, false) err := service.CloneRepository(dir, repositoryURL, referenceName, "", "", false)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, 1, getCommitHistoryLength(t, dir), "cloned repo has incorrect depth") assert.Equal(t, 1, getCommitHistoryLength(t, dir), "cloned repo has incorrect depth")
} }
@@ -51,41 +53,18 @@ func Test_ClonePublicRepository_NoGitDirectory(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
t.Logf("Cloning into %s", dir) t.Logf("Cloning into %s", dir)
err := service.CloneRepository(dir, repositoryURL, referenceName, "", "", gittypes.GitCredentialAuthType_Basic, false) err := service.CloneRepository(dir, repositoryURL, referenceName, "", "", false)
require.NoError(t, err) require.NoError(t, err)
assert.NoDirExists(t, filepath.Join(dir, ".git")) assert.NoDirExists(t, filepath.Join(dir, ".git"))
} }
func Test_cloneRepository(t *testing.T) {
service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system.
repositoryURL := setup(t)
referenceName := "refs/heads/main"
dir := t.TempDir()
t.Logf("Cloning into %s", dir)
err := service.cloneRepository(dir, cloneOption{
fetchOption: fetchOption{
baseOption: baseOption{
repositoryUrl: repositoryURL,
},
referenceName: referenceName,
},
depth: 10,
})
require.NoError(t, err)
assert.Equal(t, 4, getCommitHistoryLength(t, dir), "cloned repo has incorrect depth")
}
func Test_latestCommitID(t *testing.T) { func Test_latestCommitID(t *testing.T) {
service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system. service := Service{git: NewGitClient(true)} // no need for http client since the test access the repo via file system.
repositoryURL := setup(t) repositoryURL := setup(t)
referenceName := "refs/heads/main" referenceName := "refs/heads/main"
id, err := service.LatestCommitID(repositoryURL, referenceName, "", "", gittypes.GitCredentialAuthType_Basic, false) id, err := service.LatestCommitID(repositoryURL, referenceName, "", "", false)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id) assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id)
@@ -96,7 +75,7 @@ func Test_ListRefs(t *testing.T) {
repositoryURL := setup(t) repositoryURL := setup(t)
fs, err := service.ListRefs(repositoryURL, "", "", gittypes.GitCredentialAuthType_Basic, false, false) fs, err := service.ListRefs(repositoryURL, "", "", false, false)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, []string{"refs/heads/main"}, fs) assert.Equal(t, []string{"refs/heads/main"}, fs)
@@ -113,7 +92,6 @@ func Test_ListFiles(t *testing.T) {
referenceName, referenceName,
"", "",
"", "",
gittypes.GitCredentialAuthType_Basic,
false, false,
false, false,
[]string{".yml"}, []string{".yml"},
@@ -154,6 +132,12 @@ func Test_listRefsPrivateRepository(t *testing.T) {
client := NewGitClient(false) client := NewGitClient(false)
type args struct {
repositoryUrl string
username string
password string
}
type expectResult struct { type expectResult struct {
err error err error
refsCount int refsCount int
@@ -161,12 +145,12 @@ func Test_listRefsPrivateRepository(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
args baseOption args args
expect expectResult expect expectResult
}{ }{
{ {
name: "list refs of a real private repository", name: "list refs of a real private repository",
args: baseOption{ args: args{
repositoryUrl: privateGitRepoURL, repositoryUrl: privateGitRepoURL,
username: username, username: username,
password: accessToken, password: accessToken,
@@ -178,7 +162,7 @@ func Test_listRefsPrivateRepository(t *testing.T) {
}, },
{ {
name: "list refs of a real private repository with incorrect credential", name: "list refs of a real private repository with incorrect credential",
args: baseOption{ args: args{
repositoryUrl: privateGitRepoURL, repositoryUrl: privateGitRepoURL,
username: "test-username", username: "test-username",
password: "test-token", password: "test-token",
@@ -189,7 +173,7 @@ func Test_listRefsPrivateRepository(t *testing.T) {
}, },
{ {
name: "list refs of a fake repository without providing credential", name: "list refs of a fake repository without providing credential",
args: baseOption{ args: args{
repositoryUrl: privateGitRepoURL + "fake", repositoryUrl: privateGitRepoURL + "fake",
username: "", username: "",
password: "", password: "",
@@ -200,7 +184,7 @@ func Test_listRefsPrivateRepository(t *testing.T) {
}, },
{ {
name: "list refs of a fake repository", name: "list refs of a fake repository",
args: baseOption{ args: args{
repositoryUrl: privateGitRepoURL + "fake", repositoryUrl: privateGitRepoURL + "fake",
username: username, username: username,
password: accessToken, password: accessToken,
@@ -213,7 +197,14 @@ func Test_listRefsPrivateRepository(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
refs, err := client.listRefs(context.TODO(), tt.args) option := &git.ListOptions{}
if tt.args.username != "" || tt.args.password != "" {
option.Auth = &githttp.BasicAuth{
Username: tt.args.username,
Password: tt.args.password,
}
}
refs, err := client.ListRefs(context.TODO(), tt.args.repositoryUrl, option)
if tt.expect.err == nil { if tt.expect.err == nil {
require.NoError(t, err) require.NoError(t, err)
if tt.expect.refsCount > 0 { if tt.expect.refsCount > 0 {
@@ -232,6 +223,13 @@ func Test_listFilesPrivateRepository(t *testing.T) {
client := NewGitClient(false) client := NewGitClient(false)
type args struct {
repositoryUrl string
referenceName string
username string
password string
}
type expectResult struct { type expectResult struct {
shouldFail bool shouldFail bool
err error err error
@@ -243,18 +241,16 @@ func Test_listFilesPrivateRepository(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
args fetchOption args args
expect expectResult expect expectResult
}{ }{
{ {
name: "list tree with real repository and head ref but incorrect credential", name: "list tree with real repository and head ref but incorrect credential",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateGitRepoURL,
repositoryUrl: privateGitRepoURL,
username: "test-username",
password: "test-token",
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: "test-username",
password: "test-token",
}, },
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
@@ -263,13 +259,11 @@ func Test_listFilesPrivateRepository(t *testing.T) {
}, },
{ {
name: "list tree with real repository and head ref but no credential", name: "list tree with real repository and head ref but no credential",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateGitRepoURL,
repositoryUrl: privateGitRepoURL,
username: "",
password: "",
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: "",
password: "",
}, },
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
@@ -278,13 +272,11 @@ func Test_listFilesPrivateRepository(t *testing.T) {
}, },
{ {
name: "list tree with real repository and head ref", name: "list tree with real repository and head ref",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateGitRepoURL,
repositoryUrl: privateGitRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/heads/main", referenceName: "refs/heads/main",
username: username,
password: accessToken,
}, },
expect: expectResult{ expect: expectResult{
err: nil, err: nil,
@@ -293,13 +285,11 @@ func Test_listFilesPrivateRepository(t *testing.T) {
}, },
{ {
name: "list tree with real repository but non-existing ref", name: "list tree with real repository but non-existing ref",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateGitRepoURL,
repositoryUrl: privateGitRepoURL,
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature", referenceName: "refs/fake/feature",
username: username,
password: accessToken,
}, },
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
@@ -307,13 +297,11 @@ func Test_listFilesPrivateRepository(t *testing.T) {
}, },
{ {
name: "list tree with fake repository ", name: "list tree with fake repository ",
args: fetchOption{ args: args{
baseOption: baseOption{ repositoryUrl: privateGitRepoURL + "fake",
repositoryUrl: privateGitRepoURL + "fake",
username: username,
password: accessToken,
},
referenceName: "refs/fake/feature", referenceName: "refs/fake/feature",
username: username,
password: accessToken,
}, },
expect: expectResult{ expect: expectResult{
shouldFail: true, shouldFail: true,
@@ -324,7 +312,17 @@ func Test_listFilesPrivateRepository(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
paths, err := client.listFiles(context.TODO(), tt.args) option := &git.CloneOptions{
URL: tt.args.repositoryUrl,
ReferenceName: plumbing.ReferenceName(tt.args.referenceName),
}
if tt.args.username != "" || tt.args.password != "" {
option.Auth = &githttp.BasicAuth{
Username: tt.args.username,
Password: tt.args.password,
}
}
paths, err := client.ListFiles(context.TODO(), false, option)
if tt.expect.shouldFail { if tt.expect.shouldFail {
require.Error(t, err) require.Error(t, err)
if tt.expect.err != nil { if tt.expect.err != nil {
+55 -85
View File
@@ -7,8 +7,10 @@ import (
"sync" "sync"
"time" "time"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
lru "github.com/hashicorp/golang-lru" lru "github.com/hashicorp/golang-lru"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"golang.org/x/sync/singleflight" "golang.org/x/sync/singleflight"
) )
@@ -18,40 +20,18 @@ const (
repositoryCacheTTL = 5 * time.Minute repositoryCacheTTL = 5 * time.Minute
) )
// baseOption provides a minimum group of information to operate a git repository, like git-remote type RepoManager interface {
type baseOption struct { Download(ctx context.Context, dst string, opt *git.CloneOptions) error
repositoryUrl string LatestCommitID(ctx context.Context, repositoryUrl, referenceName string, opt *git.ListOptions) (string, error)
username string ListRefs(ctx context.Context, repositoryUrl string, opt *git.ListOptions) ([]string, error)
password string ListFiles(ctx context.Context, dirOnly bool, opt *git.CloneOptions) ([]string, error)
authType gittypes.GitCredentialAuthType
tlsSkipVerify bool
}
// fetchOption allows to specify the reference name of the target repository
type fetchOption struct {
baseOption
referenceName string
dirOnly bool
}
// cloneOption allows to add a history truncated to the specified number of commits
type cloneOption struct {
fetchOption
depth int
}
type repoManager interface {
download(ctx context.Context, dst string, opt cloneOption) error
latestCommitID(ctx context.Context, opt fetchOption) (string, error)
listRefs(ctx context.Context, opt baseOption) ([]string, error)
listFiles(ctx context.Context, opt fetchOption) ([]string, error)
} }
// Service represents a service for managing Git. // Service represents a service for managing Git.
type Service struct { type Service struct {
shutdownCtx context.Context shutdownCtx context.Context
azure repoManager azure RepoManager
git repoManager git RepoManager
timerStopped bool timerStopped bool
mut sync.Mutex mut sync.Mutex
@@ -131,61 +111,47 @@ func (service *Service) CloneRepository(
referenceName, referenceName,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
tlsSkipVerify bool, tlsSkipVerify bool,
) error { ) error {
options := cloneOption{ gitOptions := &git.CloneOptions{
fetchOption: fetchOption{ URL: repositoryURL,
baseOption: baseOption{ Depth: 1,
repositoryUrl: repositoryURL, InsecureSkipTLS: tlsSkipVerify,
username: username, Auth: GetBasicAuth(username, password),
password: password, Tags: git.NoTags,
authType: authType,
tlsSkipVerify: tlsSkipVerify,
},
referenceName: referenceName,
},
depth: 1,
} }
return service.cloneRepository(destination, options) if referenceName != "" {
gitOptions.ReferenceName = plumbing.ReferenceName(referenceName)
}
return service.repoManager(repositoryURL).Download(context.TODO(), destination, gitOptions)
} }
func (service *Service) repoManager(options baseOption) repoManager { func (service *Service) repoManager(repositoryURL string) RepoManager {
repoManager := service.git repoManager := service.git
if isAzureUrl(options.repositoryUrl) { if IsAzureUrl(repositoryURL) {
repoManager = service.azure repoManager = service.azure
} }
return repoManager return repoManager
} }
func (service *Service) cloneRepository(destination string, options cloneOption) error {
return service.repoManager(options.baseOption).download(context.TODO(), destination, options)
}
// LatestCommitID returns SHA1 of the latest commit of the specified reference // LatestCommitID returns SHA1 of the latest commit of the specified reference
func (service *Service) LatestCommitID( func (service *Service) LatestCommitID(
repositoryURL, repositoryURL,
referenceName, referenceName,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
tlsSkipVerify bool, tlsSkipVerify bool,
) (string, error) { ) (string, error) {
options := fetchOption{ listOptions := &git.ListOptions{
baseOption: baseOption{ Auth: GetBasicAuth(username, password),
repositoryUrl: repositoryURL, InsecureSkipTLS: tlsSkipVerify,
username: username,
password: password,
authType: authType,
tlsSkipVerify: tlsSkipVerify,
},
referenceName: referenceName,
} }
return service.repoManager(options.baseOption).latestCommitID(context.TODO(), options) return service.repoManager(repositoryURL).LatestCommitID(context.TODO(), repositoryURL, referenceName, listOptions)
} }
// ListRefs will list target repository's references without cloning the repository // ListRefs will list target repository's references without cloning the repository
@@ -193,7 +159,6 @@ func (service *Service) ListRefs(
repositoryURL, repositoryURL,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
hardRefresh bool, hardRefresh bool,
tlsSkipVerify bool, tlsSkipVerify bool,
) ([]string, error) { ) ([]string, error) {
@@ -218,15 +183,12 @@ func (service *Service) ListRefs(
} }
} }
options := baseOption{ options := &git.ListOptions{
repositoryUrl: repositoryURL, Auth: GetBasicAuth(username, password),
username: username, InsecureSkipTLS: tlsSkipVerify,
password: password,
authType: authType,
tlsSkipVerify: tlsSkipVerify,
} }
refs, err := service.repoManager(options).listRefs(context.TODO(), options) refs, err := service.repoManager(repositoryURL).ListRefs(context.TODO(), repositoryURL, options)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -247,7 +209,6 @@ func (service *Service) ListFiles(
referenceName, referenceName,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
dirOnly, dirOnly,
hardRefresh bool, hardRefresh bool,
includedExts []string, includedExts []string,
@@ -259,7 +220,6 @@ func (service *Service) ListFiles(
username, username,
password, password,
strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(tlsSkipVerify),
strconv.Itoa(int(authType)),
strconv.FormatBool(dirOnly), strconv.FormatBool(dirOnly),
) )
@@ -269,7 +229,6 @@ func (service *Service) ListFiles(
referenceName, referenceName,
username, username,
password, password,
authType,
dirOnly, dirOnly,
hardRefresh, hardRefresh,
tlsSkipVerify, tlsSkipVerify,
@@ -284,7 +243,6 @@ func (service *Service) listFiles(
referenceName, referenceName,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
dirOnly, dirOnly,
hardRefresh bool, hardRefresh bool,
tlsSkipVerify bool, tlsSkipVerify bool,
@@ -295,7 +253,6 @@ func (service *Service) listFiles(
username, username,
password, password,
strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(tlsSkipVerify),
strconv.Itoa(int(authType)),
strconv.FormatBool(dirOnly), strconv.FormatBool(dirOnly),
) )
@@ -313,19 +270,18 @@ func (service *Service) listFiles(
} }
} }
options := fetchOption{ cloneOption := &git.CloneOptions{
baseOption: baseOption{ URL: repositoryURL,
repositoryUrl: repositoryURL, NoCheckout: true,
username: username, Depth: 1,
password: password, SingleBranch: true,
authType: authType, ReferenceName: plumbing.ReferenceName(referenceName),
tlsSkipVerify: tlsSkipVerify, Auth: GetBasicAuth(username, password),
}, InsecureSkipTLS: tlsSkipVerify,
referenceName: referenceName, Tags: git.NoTags,
dirOnly: dirOnly,
} }
files, err := service.repoManager(options.baseOption).listFiles(context.TODO(), options) files, err := service.repoManager(repositoryURL).ListFiles(context.TODO(), dirOnly, cloneOption)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -380,3 +336,17 @@ func filterFiles(paths []string, includedExts []string) []string {
return includedFiles return includedFiles
} }
func GetBasicAuth(username, password string) *githttp.BasicAuth {
if password != "" {
if username == "" {
username = "token"
}
return &githttp.BasicAuth{
Username: username,
Password: password,
}
}
return nil
}
+3 -11
View File
@@ -9,13 +9,6 @@ var (
ErrAuthenticationFailure = errors.New("authentication failed, please ensure that the git credentials are correct") ErrAuthenticationFailure = errors.New("authentication failed, please ensure that the git credentials are correct")
) )
type GitCredentialAuthType int
const (
GitCredentialAuthType_Basic GitCredentialAuthType = iota
GitCredentialAuthType_Token
)
// RepoConfig represents a configuration for a repo // RepoConfig represents a configuration for a repo
type RepoConfig struct { type RepoConfig struct {
// The repo url // The repo url
@@ -33,11 +26,10 @@ type RepoConfig struct {
} }
type GitAuthentication struct { type GitAuthentication struct {
Username string Username string
Password string Password string
AuthorizationType GitCredentialAuthType
// Git credentials identifier when the value is not 0 // Git credentials identifier when the value is not 0
// When the value is 0, Username, Password, and Authtype are set without using saved credential // When the value is 0, Username and Password are set without using saved credential
// This is introduced since 2.15.0 // This is introduced since 2.15.0
GitCredentialID int `example:"0"` GitCredentialID int `example:"0"`
} }
-5
View File
@@ -34,7 +34,6 @@ func UpdateGitObject(gitService portainer.GitService, objId string, gitConfig *g
gitConfig.ReferenceName, gitConfig.ReferenceName,
username, username,
password, password,
gittypes.GitCredentialAuthType_Basic,
gitConfig.TLSSkipVerify, gitConfig.TLSSkipVerify,
) )
if err != nil { if err != nil {
@@ -69,7 +68,6 @@ func UpdateGitObject(gitService portainer.GitService, objId string, gitConfig *g
cloneParams.auth = &gitAuth{ cloneParams.auth = &gitAuth{
username: username, username: username,
password: password, password: password,
authType: gitConfig.Authentication.AuthorizationType,
} }
} }
@@ -97,7 +95,6 @@ type cloneRepositoryParameters struct {
} }
type gitAuth struct { type gitAuth struct {
authType gittypes.GitCredentialAuthType
username string username string
password string password string
} }
@@ -110,7 +107,6 @@ func cloneGitRepository(gitService portainer.GitService, cloneParams *cloneRepos
cloneParams.ref, cloneParams.ref,
cloneParams.auth.username, cloneParams.auth.username,
cloneParams.auth.password, cloneParams.auth.password,
cloneParams.auth.authType,
cloneParams.tlsSkipVerify, cloneParams.tlsSkipVerify,
) )
} }
@@ -121,7 +117,6 @@ func cloneGitRepository(gitService portainer.GitService, cloneParams *cloneRepos
cloneParams.ref, cloneParams.ref,
"", "",
"", "",
gittypes.GitCredentialAuthType_Basic,
cloneParams.tlsSkipVerify, cloneParams.tlsSkipVerify,
) )
} }
-23
View File
@@ -1,23 +0,0 @@
package git
import (
gittypes "github.com/portainer/portainer/api/git/types"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/pkg/validate"
)
func ValidateRepoConfig(repoConfig *gittypes.RepoConfig) error {
if len(repoConfig.URL) == 0 || !validate.IsURL(repoConfig.URL) {
return httperrors.NewInvalidPayloadError("Invalid repository URL. Must correspond to a valid URL format")
}
return ValidateRepoAuthentication(repoConfig.Authentication)
}
func ValidateRepoAuthentication(auth *gittypes.GitAuthentication) error {
if auth != nil && len(auth.Password) == 0 && auth.GitCredentialID == 0 {
return httperrors.NewInvalidPayloadError("Invalid repository credentials. Password or GitCredentialID must be specified when authentication is enabled")
}
return nil
}
@@ -46,7 +46,6 @@ func (g *TestGitService) CloneRepository(
referenceName string, referenceName string,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
tlsSkipVerify bool, tlsSkipVerify bool,
) error { ) error {
time.Sleep(100 * time.Millisecond) time.Sleep(100 * time.Millisecond)
@@ -59,7 +58,6 @@ func (g *TestGitService) LatestCommitID(
referenceName, referenceName,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
tlsSkipVerify bool, tlsSkipVerify bool,
) (string, error) { ) (string, error) {
return "", nil return "", nil
@@ -84,7 +82,6 @@ func (g *InvalidTestGitService) CloneRepository(
refName, refName,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
tlsSkipVerify bool, tlsSkipVerify bool,
) error { ) error {
return errors.New("simulate network error") return errors.New("simulate network error")
@@ -95,7 +92,6 @@ func (g *InvalidTestGitService) LatestCommitID(
referenceName, referenceName,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
tlsSkipVerify bool, tlsSkipVerify bool,
) (string, error) { ) (string, error) {
return "", nil return "", nil
@@ -45,8 +45,6 @@ type customTemplateUpdatePayload struct {
// Password used in basic authentication or token used in token authentication. // Password used in basic authentication or token used in token authentication.
// Required when RepositoryAuthentication is true and RepositoryGitCredentialID is 0 // Required when RepositoryAuthentication is true and RepositoryGitCredentialID is 0
RepositoryPassword string `example:"myGitPassword"` RepositoryPassword string `example:"myGitPassword"`
// RepositoryAuthorizationType is the authorization type to use
RepositoryAuthorizationType gittypes.GitCredentialAuthType `example:"0"`
// GitCredentialID used to identify the bound git credential. Required when RepositoryAuthentication // GitCredentialID used to identify the bound git credential. Required when RepositoryAuthentication
// is true and RepositoryUsername/RepositoryPassword are not provided // is true and RepositoryUsername/RepositoryPassword are not provided
RepositoryGitCredentialID int `example:"0"` RepositoryGitCredentialID int `example:"0"`
@@ -184,15 +182,12 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
repositoryUsername := "" repositoryUsername := ""
repositoryPassword := "" repositoryPassword := ""
repositoryAuthType := gittypes.GitCredentialAuthType_Basic
if payload.RepositoryAuthentication { if payload.RepositoryAuthentication {
repositoryUsername = payload.RepositoryUsername repositoryUsername = payload.RepositoryUsername
repositoryPassword = payload.RepositoryPassword repositoryPassword = payload.RepositoryPassword
repositoryAuthType = payload.RepositoryAuthorizationType
gitConfig.Authentication = &gittypes.GitAuthentication{ gitConfig.Authentication = &gittypes.GitAuthentication{
Username: payload.RepositoryUsername, Username: payload.RepositoryUsername,
Password: payload.RepositoryPassword, Password: payload.RepositoryPassword,
AuthorizationType: payload.RepositoryAuthorizationType,
} }
} }
@@ -202,7 +197,6 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
ReferenceName: gitConfig.ReferenceName, ReferenceName: gitConfig.ReferenceName,
Username: repositoryUsername, Username: repositoryUsername,
Password: repositoryPassword, Password: repositoryPassword,
AuthType: repositoryAuthType,
TLSSkipVerify: gitConfig.TLSSkipVerify, TLSSkipVerify: gitConfig.TLSSkipVerify,
}) })
if err != nil { if err != nil {
@@ -216,7 +210,6 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
gitConfig.ReferenceName, gitConfig.ReferenceName,
repositoryUsername, repositoryUsername,
repositoryPassword, repositoryPassword,
repositoryAuthType,
gitConfig.TLSSkipVerify, gitConfig.TLSSkipVerify,
) )
if err != nil { if err != nil {
@@ -34,8 +34,6 @@ type edgeStackFromGitRepositoryPayload struct {
RepositoryUsername string `example:"myGitUsername"` RepositoryUsername string `example:"myGitUsername"`
// Password used in basic authentication. Required when RepositoryAuthentication is true. // Password used in basic authentication. Required when RepositoryAuthentication is true.
RepositoryPassword string `example:"myGitPassword"` RepositoryPassword string `example:"myGitPassword"`
// RepositoryAuthorizationType is the authorization type to use
RepositoryAuthorizationType gittypes.GitCredentialAuthType `example:"0"`
// Path to the Stack file inside the Git repository // Path to the Stack file inside the Git repository
FilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"` FilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
// List of identifiers of EdgeGroups // List of identifiers of EdgeGroups
@@ -128,9 +126,8 @@ func (handler *Handler) createEdgeStackFromGitRepository(r *http.Request, tx dat
if payload.RepositoryAuthentication { if payload.RepositoryAuthentication {
repoConfig.Authentication = &gittypes.GitAuthentication{ repoConfig.Authentication = &gittypes.GitAuthentication{
Username: payload.RepositoryUsername, Username: payload.RepositoryUsername,
Password: payload.RepositoryPassword, Password: payload.RepositoryPassword,
AuthorizationType: payload.RepositoryAuthorizationType,
} }
} }
@@ -152,11 +149,9 @@ func (handler *Handler) storeManifestFromGitRepository(tx dataservices.DataStore
projectPath = handler.FileService.GetEdgeStackProjectPath(stackFolder) projectPath = handler.FileService.GetEdgeStackProjectPath(stackFolder)
repositoryUsername := "" repositoryUsername := ""
repositoryPassword := "" repositoryPassword := ""
repositoryAuthType := gittypes.GitCredentialAuthType_Basic
if repositoryConfig.Authentication != nil && repositoryConfig.Authentication.Password != "" { if repositoryConfig.Authentication != nil && repositoryConfig.Authentication.Password != "" {
repositoryUsername = repositoryConfig.Authentication.Username repositoryUsername = repositoryConfig.Authentication.Username
repositoryPassword = repositoryConfig.Authentication.Password repositoryPassword = repositoryConfig.Authentication.Password
repositoryAuthType = repositoryConfig.Authentication.AuthorizationType
} }
if err := handler.GitService.CloneRepository( if err := handler.GitService.CloneRepository(
@@ -165,7 +160,6 @@ func (handler *Handler) storeManifestFromGitRepository(tx dataservices.DataStore
repositoryConfig.ReferenceName, repositoryConfig.ReferenceName,
repositoryUsername, repositoryUsername,
repositoryPassword, repositoryPassword,
repositoryAuthType,
repositoryConfig.TLSSkipVerify, repositoryConfig.TLSSkipVerify,
); err != nil { ); err != nil {
return "", "", "", err return "", "", "", err
@@ -18,11 +18,10 @@ type fileResponse struct {
} }
type repositoryFilePreviewPayload struct { type repositoryFilePreviewPayload struct {
Repository string `json:"repository" example:"https://github.com/openfaas/faas" validate:"required"` Repository string `json:"repository" example:"https://github.com/openfaas/faas" validate:"required"`
Reference string `json:"reference" example:"refs/heads/master"` Reference string `json:"reference" example:"refs/heads/master"`
Username string `json:"username" example:"myGitUsername"` Username string `json:"username" example:"myGitUsername"`
Password string `json:"password" example:"myGitPassword"` Password string `json:"password" example:"myGitPassword"`
AuthorizationType gittypes.GitCredentialAuthType `json:"authorizationType"`
// Path to file whose content will be read // Path to file whose content will be read
TargetFile string `json:"targetFile" example:"docker-compose.yml"` TargetFile string `json:"targetFile" example:"docker-compose.yml"`
// TLSSkipVerify skips SSL verification when cloning the Git repository // TLSSkipVerify skips SSL verification when cloning the Git repository
@@ -76,7 +75,6 @@ func (handler *Handler) gitOperationRepoFilePreview(w http.ResponseWriter, r *ht
payload.Reference, payload.Reference,
payload.Username, payload.Username,
payload.Password, payload.Password,
payload.AuthorizationType,
payload.TLSSkipVerify, payload.TLSSkipVerify,
) )
if err != nil { if err != nil {
+10 -13
View File
@@ -19,15 +19,14 @@ import (
) )
type stackGitUpdatePayload struct { type stackGitUpdatePayload struct {
AutoUpdate *portainer.AutoUpdateSettings AutoUpdate *portainer.AutoUpdateSettings
Env []portainer.Pair Env []portainer.Pair
Prune bool Prune bool
RepositoryReferenceName string RepositoryReferenceName string
RepositoryAuthentication bool RepositoryAuthentication bool
RepositoryUsername string RepositoryUsername string
RepositoryPassword string RepositoryPassword string
RepositoryAuthorizationType gittypes.GitCredentialAuthType TLSSkipVerify bool
TLSSkipVerify bool
} }
func (payload *stackGitUpdatePayload) Validate(r *http.Request) error { func (payload *stackGitUpdatePayload) Validate(r *http.Request) error {
@@ -169,9 +168,8 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *
} }
stack.GitConfig.Authentication = &gittypes.GitAuthentication{ stack.GitConfig.Authentication = &gittypes.GitAuthentication{
Username: payload.RepositoryUsername, Username: payload.RepositoryUsername,
Password: password, Password: password,
AuthorizationType: payload.RepositoryAuthorizationType,
} }
if _, err := handler.GitService.LatestCommitID( if _, err := handler.GitService.LatestCommitID(
@@ -179,7 +177,6 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *
stack.GitConfig.ReferenceName, stack.GitConfig.ReferenceName,
stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Username,
stack.GitConfig.Authentication.Password, stack.GitConfig.Authentication.Password,
stack.GitConfig.Authentication.AuthorizationType,
stack.GitConfig.TLSSkipVerify, stack.GitConfig.TLSSkipVerify,
); err != nil { ); err != nil {
return httperror.InternalServerError("Unable to fetch git repository", err) return httperror.InternalServerError("Unable to fetch git repository", err)
@@ -6,7 +6,6 @@ import (
portainer "github.com/portainer/portainer/api" portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/git" "github.com/portainer/portainer/api/git"
gittypes "github.com/portainer/portainer/api/git/types"
httperrors "github.com/portainer/portainer/api/http/errors" httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/http/security"
k "github.com/portainer/portainer/api/kubernetes" k "github.com/portainer/portainer/api/kubernetes"
@@ -20,13 +19,12 @@ import (
) )
type stackGitRedployPayload struct { type stackGitRedployPayload struct {
RepositoryReferenceName string RepositoryReferenceName string
RepositoryAuthentication bool RepositoryAuthentication bool
RepositoryUsername string RepositoryUsername string
RepositoryPassword string RepositoryPassword string
RepositoryAuthorizationType gittypes.GitCredentialAuthType Env []portainer.Pair
Env []portainer.Pair Prune bool
Prune bool
// RepullImageAndRedeploy indicates whether to force repulling images and redeploying the stack // RepullImageAndRedeploy indicates whether to force repulling images and redeploying the stack
RepullImageAndRedeploy bool RepullImageAndRedeploy bool
@@ -143,16 +141,13 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request)
repositoryUsername := "" repositoryUsername := ""
repositoryPassword := "" repositoryPassword := ""
repositoryAuthType := gittypes.GitCredentialAuthType_Basic
if payload.RepositoryAuthentication { if payload.RepositoryAuthentication {
repositoryPassword = payload.RepositoryPassword repositoryPassword = payload.RepositoryPassword
repositoryAuthType = payload.RepositoryAuthorizationType
// When the existing stack is using the custom username/password and the password is not updated, // When the existing stack is using the custom username/password and the password is not updated,
// the stack should keep using the saved username/password // the stack should keep using the saved username/password
if repositoryPassword == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil { if repositoryPassword == "" && stack.GitConfig != nil && stack.GitConfig.Authentication != nil {
repositoryPassword = stack.GitConfig.Authentication.Password repositoryPassword = stack.GitConfig.Authentication.Password
repositoryAuthType = stack.GitConfig.Authentication.AuthorizationType
} }
repositoryUsername = payload.RepositoryUsername repositoryUsername = payload.RepositoryUsername
} }
@@ -163,7 +158,6 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request)
ReferenceName: stack.GitConfig.ReferenceName, ReferenceName: stack.GitConfig.ReferenceName,
Username: repositoryUsername, Username: repositoryUsername,
Password: repositoryPassword, Password: repositoryPassword,
AuthType: repositoryAuthType,
TLSSkipVerify: stack.GitConfig.TLSSkipVerify, TLSSkipVerify: stack.GitConfig.TLSSkipVerify,
} }
@@ -178,7 +172,7 @@ func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request)
return err return err
} }
newHash, err := handler.GitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, repositoryUsername, repositoryPassword, repositoryAuthType, stack.GitConfig.TLSSkipVerify) newHash, err := handler.GitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, repositoryUsername, repositoryPassword, stack.GitConfig.TLSSkipVerify)
if err != nil { if err != nil {
return httperror.InternalServerError("Unable get latest commit id", errors.WithMessagef(err, "failed to fetch latest commit id of the stack %v", stack.ID)) return httperror.InternalServerError("Unable get latest commit id", errors.WithMessagef(err, "failed to fetch latest commit id of the stack %v", stack.ID))
} }
@@ -27,13 +27,12 @@ type kubernetesFileStackUpdatePayload struct {
} }
type kubernetesGitStackUpdatePayload struct { type kubernetesGitStackUpdatePayload struct {
RepositoryReferenceName string RepositoryReferenceName string
RepositoryAuthentication bool RepositoryAuthentication bool
RepositoryUsername string RepositoryUsername string
RepositoryPassword string RepositoryPassword string
RepositoryAuthorizationType gittypes.GitCredentialAuthType AutoUpdate *portainer.AutoUpdateSettings
AutoUpdate *portainer.AutoUpdateSettings TLSSkipVerify bool
TLSSkipVerify bool
} }
func (payload *kubernetesFileStackUpdatePayload) Validate(r *http.Request) error { func (payload *kubernetesFileStackUpdatePayload) Validate(r *http.Request) error {
@@ -77,9 +76,8 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer.
} }
stack.GitConfig.Authentication = &gittypes.GitAuthentication{ stack.GitConfig.Authentication = &gittypes.GitAuthentication{
Username: payload.RepositoryUsername, Username: payload.RepositoryUsername,
Password: password, Password: password,
AuthorizationType: payload.RepositoryAuthorizationType,
} }
if _, err := handler.GitService.LatestCommitID( if _, err := handler.GitService.LatestCommitID(
@@ -87,7 +85,6 @@ func (handler *Handler) updateKubernetesStack(r *http.Request, stack *portainer.
stack.GitConfig.ReferenceName, stack.GitConfig.ReferenceName,
stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Username,
stack.GitConfig.Authentication.Password, stack.GitConfig.Authentication.Password,
stack.GitConfig.Authentication.AuthorizationType,
stack.GitConfig.TLSSkipVerify, stack.GitConfig.TLSSkipVerify,
); err != nil { ); err != nil {
return httperror.InternalServerError("Unable to fetch git repository", err) return httperror.InternalServerError("Unable to fetch git repository", err)
@@ -5,7 +5,6 @@ import (
"slices" "slices"
portainer "github.com/portainer/portainer/api" portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
httperror "github.com/portainer/portainer/pkg/libhttp/error" httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request" "github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response" "github.com/portainer/portainer/pkg/libhttp/response"
@@ -78,7 +77,6 @@ func (handler *Handler) templateFile(w http.ResponseWriter, r *http.Request) *ht
"", "",
"", "",
"", "",
gittypes.GitCredentialAuthType_Basic,
false, false,
); err != nil { ); err != nil {
return httperror.InternalServerError("Unable to clone git repository", err) return httperror.InternalServerError("Unable to clone git repository", err)
@@ -17,7 +17,6 @@ import (
portainer "github.com/portainer/portainer/api" portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices" "github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/docker/client" "github.com/portainer/portainer/api/docker/client"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/proxy/factory/utils" "github.com/portainer/portainer/api/http/proxy/factory/utils"
"github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization" "github.com/portainer/portainer/api/internal/authorization"
@@ -449,7 +448,6 @@ func (transport *Transport) updateDefaultGitBranch(request *http.Request) error
"", "",
"", "",
"", "",
gittypes.GitCredentialAuthType_Basic,
false, false,
) )
if err != nil { if err != nil {
-5
View File
@@ -2,7 +2,6 @@ package testhelpers
import ( import (
portainer "github.com/portainer/portainer/api" portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
) )
type gitService struct { type gitService struct {
@@ -24,7 +23,6 @@ func (g *gitService) CloneRepository(
referenceName, referenceName,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
tlsSkipVerify bool, tlsSkipVerify bool,
) error { ) error {
return g.cloneErr return g.cloneErr
@@ -35,7 +33,6 @@ func (g *gitService) LatestCommitID(
referenceName, referenceName,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
tlsSkipVerify bool, tlsSkipVerify bool,
) (string, error) { ) (string, error) {
return g.id, nil return g.id, nil
@@ -45,7 +42,6 @@ func (g *gitService) ListRefs(
repositoryURL, repositoryURL,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
hardRefresh bool, hardRefresh bool,
tlsSkipVerify bool, tlsSkipVerify bool,
) ([]string, error) { ) ([]string, error) {
@@ -57,7 +53,6 @@ func (g *gitService) ListFiles(
referenceName, referenceName,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
dirOnly, dirOnly,
hardRefresh bool, hardRefresh bool,
includedExts []string, includedExts []string,
-4
View File
@@ -1647,7 +1647,6 @@ type (
referenceName, referenceName,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
tlsSkipVerify bool, tlsSkipVerify bool,
) error ) error
LatestCommitID( LatestCommitID(
@@ -1655,14 +1654,12 @@ type (
referenceName, referenceName,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
tlsSkipVerify bool, tlsSkipVerify bool,
) (string, error) ) (string, error)
ListRefs( ListRefs(
repositoryURL, repositoryURL,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
hardRefresh bool, hardRefresh bool,
tlsSkipVerify bool, tlsSkipVerify bool,
) ([]string, error) ) ([]string, error)
@@ -1671,7 +1668,6 @@ type (
referenceName, referenceName,
username, username,
password string, password string,
authType gittypes.GitCredentialAuthType,
dirOnly, dirOnly,
hardRefresh bool, hardRefresh bool,
includeExts []string, includeExts []string,
-4
View File
@@ -19,11 +19,9 @@ var (
func DownloadGitRepository(config gittypes.RepoConfig, gitService portainer.GitService, getProjectPath func() string) (string, error) { func DownloadGitRepository(config gittypes.RepoConfig, gitService portainer.GitService, getProjectPath func() string) (string, error) {
username := "" username := ""
password := "" password := ""
authType := gittypes.GitCredentialAuthType_Basic
if config.Authentication != nil { if config.Authentication != nil {
username = config.Authentication.Username username = config.Authentication.Username
password = config.Authentication.Password password = config.Authentication.Password
authType = config.Authentication.AuthorizationType
} }
projectPath := getProjectPath() projectPath := getProjectPath()
@@ -33,7 +31,6 @@ func DownloadGitRepository(config gittypes.RepoConfig, gitService portainer.GitS
config.ReferenceName, config.ReferenceName,
username, username,
password, password,
authType,
config.TLSSkipVerify, config.TLSSkipVerify,
) )
if err != nil { if err != nil {
@@ -51,7 +48,6 @@ func DownloadGitRepository(config gittypes.RepoConfig, gitService portainer.GitS
config.ReferenceName, config.ReferenceName,
username, username,
password, password,
authType,
config.TLSSkipVerify, config.TLSSkipVerify,
) )
if err != nil { if err != nil {
-2
View File
@@ -4,7 +4,6 @@ import (
"time" "time"
portainer "github.com/portainer/portainer/api" portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
) )
type InstallOptions struct { type InstallOptions struct {
@@ -30,7 +29,6 @@ type InstallOptions struct {
CreateNamespace bool CreateNamespace bool
// GitOps related options // GitOps related options
GitConfig *gittypes.RepoConfig
AutoUpdate *portainer.AutoUpdateSettings AutoUpdate *portainer.AutoUpdateSettings
// StackID is the ID of the Portainer stack associated with this release // StackID is the ID of the Portainer stack associated with this release
+4 -5
View File
@@ -11,7 +11,6 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
portainer "github.com/portainer/portainer/api" portainer "github.com/portainer/portainer/api"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/pkg/libhelm/options" "github.com/portainer/portainer/pkg/libhelm/options"
"github.com/portainer/portainer/pkg/libhelm/release" "github.com/portainer/portainer/pkg/libhelm/release"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
@@ -195,7 +194,7 @@ func ensureHelmDirectoriesExist(settings *cli.EnvSettings) error {
} }
if _, err := os.Stat(dir); os.IsNotExist(err) { if _, err := os.Stat(dir); os.IsNotExist(err) {
if err := os.MkdirAll(dir, 0700); err != nil { if err := os.MkdirAll(dir, 0o700); err != nil {
log.Error(). log.Error().
Str("context", "helm_sdk_dirs"). Str("context", "helm_sdk_dirs").
Str("directory", dir). Str("directory", dir).
@@ -211,7 +210,7 @@ func ensureHelmDirectoriesExist(settings *cli.EnvSettings) error {
if _, err := os.Stat(settings.RegistryConfig); os.IsNotExist(err) { if _, err := os.Stat(settings.RegistryConfig); os.IsNotExist(err) {
// Create the directory if it doesn't exist // Create the directory if it doesn't exist
dir := filepath.Dir(settings.RegistryConfig) dir := filepath.Dir(settings.RegistryConfig)
if err := os.MkdirAll(dir, 0700); err != nil { if err := os.MkdirAll(dir, 0o700); err != nil {
log.Error(). log.Error().
Str("context", "helm_sdk_dirs"). Str("context", "helm_sdk_dirs").
Str("directory", dir). Str("directory", dir).
@@ -237,7 +236,7 @@ func ensureHelmDirectoriesExist(settings *cli.EnvSettings) error {
if _, err := os.Stat(settings.RepositoryConfig); os.IsNotExist(err) { if _, err := os.Stat(settings.RepositoryConfig); os.IsNotExist(err) {
// Create an empty repository config file with default yaml structure // Create an empty repository config file with default yaml structure
f := repo.NewFile() f := repo.NewFile()
if err := f.WriteFile(settings.RepositoryConfig, 0644); err != nil { if err := f.WriteFile(settings.RepositoryConfig, 0o644); err != nil {
log.Error(). log.Error().
Str("context", "helm_sdk_dirs"). Str("context", "helm_sdk_dirs").
Str("file", settings.RepositoryConfig). Str("file", settings.RepositoryConfig).
@@ -258,7 +257,7 @@ func ensureHelmDirectoriesExist(settings *cli.EnvSettings) error {
// appendChartReferenceAnnotations encodes chart reference values for safe storage in Helm labels. // appendChartReferenceAnnotations encodes chart reference values for safe storage in Helm labels.
// It creates a new map with encoded values for specific chart reference labels. // It creates a new map with encoded values for specific chart reference labels.
// Preserves existing labels and handles edge cases gracefully. // Preserves existing labels and handles edge cases gracefully.
func appendChartReferenceAnnotations(chartPath, repoURL string, registryID int, stackID int, gitConfig *gittypes.RepoConfig, autoUpdateSettings *portainer.AutoUpdateSettings, existingAnnotations map[string]string) map[string]string { func appendChartReferenceAnnotations(chartPath, repoURL string, registryID int, stackID int, autoUpdateSettings *portainer.AutoUpdateSettings, existingAnnotations map[string]string) map[string]string {
// Copy existing annotations // Copy existing annotations
annotations := make(map[string]string) annotations := make(map[string]string)
maps.Copy(annotations, existingAnnotations) maps.Copy(annotations, existingAnnotations)
+4 -4
View File
@@ -74,7 +74,7 @@ func TestAppendChartReferenceAnnotations(t *testing.T) {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
result := appendChartReferenceAnnotations( result := appendChartReferenceAnnotations(
tt.chartPath, tt.repoURL, tt.registryID, tt.stackID, tt.chartPath, tt.repoURL, tt.registryID, tt.stackID,
nil, nil, tt.existing, nil, tt.existing,
) )
assert.Equal(t, tt.want, result) assert.Equal(t, tt.want, result)
@@ -85,18 +85,18 @@ func TestAppendChartReferenceAnnotations(t *testing.T) {
func TestAppendChartReferenceAnnotations_RepoURLLogic(t *testing.T) { func TestAppendChartReferenceAnnotations_RepoURLLogic(t *testing.T) {
t.Run("repoURL only added when registryID is zero", func(t *testing.T) { t.Run("repoURL only added when registryID is zero", func(t *testing.T) {
// With registry ID - no repoURL // With registry ID - no repoURL
result := appendChartReferenceAnnotations("chart", "url", 5, 0, nil, nil, nil) result := appendChartReferenceAnnotations("chart", "url", 5, 0, nil, nil)
_, hasRepoURL := result[RepoURLAnnotation] _, hasRepoURL := result[RepoURLAnnotation]
assert.False(t, hasRepoURL) assert.False(t, hasRepoURL)
// Without registry ID - includes repoURL // Without registry ID - includes repoURL
result = appendChartReferenceAnnotations("chart", "url", 0, 0, nil, nil, nil) result = appendChartReferenceAnnotations("chart", "url", 0, 0, nil, nil)
assert.Equal(t, "url", result[RepoURLAnnotation]) assert.Equal(t, "url", result[RepoURLAnnotation])
}) })
t.Run("does not mutate existing map", func(t *testing.T) { t.Run("does not mutate existing map", func(t *testing.T) {
existing := map[string]string{"key": "value"} existing := map[string]string{"key": "value"}
appendChartReferenceAnnotations("chart", "", 0, 0, nil, nil, existing) appendChartReferenceAnnotations("chart", "", 0, 0, nil, existing)
assert.Equal(t, map[string]string{"key": "value"}, existing) assert.Equal(t, map[string]string{"key": "value"}, existing)
}) })
} }
+1 -1
View File
@@ -100,7 +100,7 @@ func (hspm *HelmSDKPackageManager) install(installOpts options.InstallOptions) (
if installOpts.Registry != nil { if installOpts.Registry != nil {
registryID = int(installOpts.Registry.ID) registryID = int(installOpts.Registry.ID)
} }
chart.Metadata.Annotations = appendChartReferenceAnnotations(installOpts.Chart, installOpts.Repo, registryID, installOpts.StackID, installOpts.GitConfig, installOpts.AutoUpdate, chart.Metadata.Annotations) chart.Metadata.Annotations = appendChartReferenceAnnotations(installOpts.Chart, installOpts.Repo, registryID, installOpts.StackID, installOpts.AutoUpdate, chart.Metadata.Annotations)
// Run the installation // Run the installation
log.Info(). log.Info().
+1 -1
View File
@@ -124,7 +124,7 @@ func (hspm *HelmSDKPackageManager) Upgrade(upgradeOpts options.InstallOptions) (
if upgradeOpts.Registry != nil { if upgradeOpts.Registry != nil {
registryID = int(upgradeOpts.Registry.ID) registryID = int(upgradeOpts.Registry.ID)
} }
chart.Metadata.Annotations = appendChartReferenceAnnotations(upgradeOpts.Chart, upgradeOpts.Repo, registryID, upgradeOpts.StackID, upgradeOpts.GitConfig, upgradeOpts.AutoUpdate, chart.Metadata.Annotations) chart.Metadata.Annotations = appendChartReferenceAnnotations(upgradeOpts.Chart, upgradeOpts.Repo, registryID, upgradeOpts.StackID, upgradeOpts.AutoUpdate, chart.Metadata.Annotations)
log.Info(). log.Info().
Str("context", "HelmClient"). Str("context", "HelmClient").