homelab/opengist
1
0
Fork 0
mirror of https://github.com/thomiceli/opengist.git synced 2026-08-07 07:14:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Improve git /init with correlation token (#751)

Browse Source
This commit is contained in:
Thomas
2026-06-29 23:19:22 +07:00
committed by GitHub
parent c5d7832696
commit 0f942c8af8
11 changed files with 718 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/db/db.go
+7
View File
@@ -160,6 +160,12 @@ func Setup(dbUri string) error {
return err
}
if db.Migrator().HasTable(&GistInitQueue{}) {
if err = db.Where("1 = 1").Delete(&GistInitQueue{}).Error; err != nil {
return err
}
}
if err = db.AutoMigrate(&User{}, &Gist{}, &SSHKey{}, &AdminSetting{}, &Invitation{}, &WebAuthnCredential{}, &TOTP{}, &GistTopic{}, &GistLanguage{}, &GistInitQueue{}, &AccessToken{}, &ActionLock{}); err != nil {
return err
}
@@ -225,6 +231,7 @@ func setupSQLite(dbInfo databaseInfo) error {
u.Scheme = "file"
q := u.Query()
q.Set("_pragma", "foreign_keys(1)")
q.Add("_pragma", "busy_timeout(5000)")
q.Set("_journal_mode", journalMode)
u.RawQuery = q.Encode()
dsn = u.String()
internal/db/gist_init_queue.go
+59 -14
View File
@@ -1,34 +1,79 @@
package db
import "errors"
var ErrInitGistAlreadyConsumed = errors.New("init gist already consumed")
// GistInitQueue tracks gists created by the "git push .../init" flow between the
// two HTTP requests git performs (info/refs then git-receive-pack). Each entry
// carries a Token that correlates those two requests so the receive-pack step
// targets the exact gist created by its matching info/refs step, instead of
// guessing from a per-user FIFO.
type GistInitQueue struct {
GistID uint `gorm:"primaryKey"`
Gist Gist `gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE;foreignKey:GistID"`
UserID uint `gorm:"primaryKey"`
User User `gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE;foreignKey:UserID"`
GistID uint `gorm:"primaryKey"`
Gist Gist `gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE;foreignKey:GistID"`
UserID uint `gorm:"primaryKey"`
User User `gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE;foreignKey:UserID"`
Token string `gorm:"size:32;uniqueIndex"`
}
func GetInitGistInQueueForUser(userID uint) (*Gist, error) {
func AddInitGistToQueue(gistID uint, userID uint, token string) error {
queue := &GistInitQueue{
GistID: gistID,
UserID: userID,
Token: token,
}
return db.Create(&queue).Error
}
func GetInitGistByToken(token string) (*Gist, error) {
queue := new(GistInitQueue)
err := db.Preload("Gist").Preload("Gist.User").
Where("user_id = ?", userID).
Order("gist_id asc").
Where("token = ?", token).
First(&queue).Error
if err != nil {
return nil, err
}
return &queue.Gist, nil
}
err = db.Delete(&queue).Error
if err != nil {
func PopInitGistByToken(token string) (*Gist, error) {
queue := new(GistInitQueue)
if err := db.Preload("Gist").Preload("Gist.User").
Where("token = ?", token).
First(&queue).Error; err != nil {
return nil, err
}
res := db.Where("token = ?", token).Delete(&GistInitQueue{})
if res.Error != nil {
return nil, res.Error
}
if res.RowsAffected == 0 {
return nil, ErrInitGistAlreadyConsumed
}
return &queue.Gist, nil
}
func AddInitGistToQueue(gistID uint, userID uint) error {
queue := &GistInitQueue{
GistID: gistID,
UserID: userID,
func PopInitGistForUser(userID uint) (*Gist, error) {
for {
queue := new(GistInitQueue)
if err := db.Preload("Gist").Preload("Gist.User").
Where("user_id = ?", userID).
Order("gist_id asc").
First(&queue).Error; err != nil {
return nil, err
}
res := db.Where("gist_id = ? AND user_id = ?", queue.GistID, userID).
Delete(&GistInitQueue{})
if res.Error != nil {
return nil, res.Error
}
if res.RowsAffected > 0 {
return &queue.Gist, nil
}
// Lost the race for this entry; try the next oldest one.
}
return db.Create(&queue).Error
}
internal/web/context/store.go
+6 -1
View File
@@ -4,6 +4,7 @@ import (
"net/http"
"path/filepath"
"strings"
"sync"
"github.com/gorilla/sessions"
"github.com/markbates/goth/gothic"
@@ -11,6 +12,8 @@ import (
"github.com/thomiceli/opengist/internal/session"
)
var gothicStoreOnce sync.Once
type Store struct {
sessionsPath string
@@ -27,7 +30,9 @@ func NewStore(sessionsPath string) *Store {
s.UserStore = sessions.NewFilesystemStore(s.sessionsPath, config.SecretKey, encryptKey)
s.UserStore.MaxLength(10 * 1024)
hardenCookie(s.UserStore.Options)
gothic.Store = s.UserStore
gothicStoreOnce.Do(func() {
gothic.Store = s.UserStore
})
return s
}
internal/web/handlers/git/access.go
+81
View File
@@ -0,0 +1,81 @@
package git
import (
"strings"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/db"
"github.com/thomiceli/opengist/internal/web/context"
)
// handlePull authorizes a clone/pull of an existing gist before serving it.
func handlePull(ctx *context.Context, route *gitRoute, gist *db.Gist, gistExists bool, username, password string) error {
log.Debug().Msg("Detected git pull operation")
if !gistExists {
log.Debug().Str("authUsername", username).Msg("Pulling unknown gist")
return ctx.PlainText(404, "Check your credentials or make sure you have access to the Gist")
}
// For a non-private gist (reached here only when unauthenticated access is
// disabled) any valid account may pull; for a private gist the password is
// checked against the gist owner.
userToCheck := gist.User.Username
if gist.Private != db.PrivateVisibility {
log.Debug().Str("authUsername", username).Msg("Pulling non-private gist with authenticated access")
userToCheck = username
} else {
log.Debug().Str("authUsername", username).Str("gistOwner", gist.User.Username).Msg("Pulling private gist")
}
if user, err := authOrFail(ctx, userToCheck, password, 404, "Check your credentials or make sure you have access to the Gist"); user == nil {
return err
}
log.Debug().Str("authUsername", username).Msg("Pulling gist")
return route.handler(ctx)
}
// handlePush authorizes a push to an existing gist, or creates a new gist when
// the user pushes to /<user>/<name>.
func handlePush(ctx *context.Context, route *gitRoute, gist *db.Gist, gistExists bool, username, password string) error {
log.Debug().Msg("Detected git push operation")
if gistExists {
log.Debug().Str("authUsername", username).Str("gistOwner", gist.User.Username).Msg("Pushing to existing gist")
if user, err := authOrFail(ctx, gist.User.Username, password, 404, "Check your credentials or make sure you have access to the Gist"); user == nil {
return err
}
if gist.Archived {
log.Debug().Str("authUsername", username).Msg("Pushing to archived gist")
return ctx.PlainText(403, "This gist is archived and is read-only")
}
log.Debug().Str("authUsername", username).Msg("Pushing gist")
return route.handler(ctx)
}
// The gist does not exist: the user creates it by pushing to /<user>/<name>.
log.Debug().Str("authUsername", username).Msg("Creating new gist by pushing")
user, err := authOrFail(ctx, username, password, 404, "Check your credentials or make sure you have access to the Gist")
if user == nil {
return err
}
urlPath := ctx.Request().URL.Path
pathParts := strings.Split(strings.Trim(urlPath, "/"), "/")
if pathParts[0] != username || len(pathParts) != 4 {
log.Debug().Str("authUsername", username).Any("path", pathParts).Msg("Invalid URL format for push operation")
return ctx.PlainText(401, "Invalid URL format for push operation")
}
log.Debug().Str("authUsername", username).Msg("Valid URL format for push operation")
gist, err = createGist(user, pathParts[1])
if err != nil {
return ctx.ErrorRes(500, "Cannot create gist", err)
}
log.Debug().Str("authUsername", username).Str("url", urlPath).Msg("Gist created")
setGistContext(ctx, gist)
return route.handler(ctx)
}
internal/web/handlers/git/auth.go
+72
View File
@@ -0,0 +1,72 @@
package git
import (
"encoding/base64"
"errors"
"strings"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/auth"
"github.com/thomiceli/opengist/internal/db"
"github.com/thomiceli/opengist/internal/web/context"
)
// authOrFail authenticates the given credentials. On success it returns the
// user. On failure it writes the appropriate HTTP response and returns a nil
// user, so callers stop with:
//
// user, err := authOrFail(...)
// if user == nil {
// return err
// }
//
// `err` is nil for an already-written invalid-credentials response and a
// renderable error for an internal authentication failure.
func authOrFail(ctx *context.Context, username, password string, invalidCode int, invalidMsg string) (*db.User, error) {
user, err := auth.TryAuthentication(username, password)
if err == nil {
return user, nil
}
var authErr auth.AuthError
if errors.As(err, &authErr) {
log.Warn().Msg("Invalid HTTP authentication attempt from " + ctx.RealIP())
return nil, ctx.PlainText(invalidCode, invalidMsg)
}
return nil, ctx.ErrorRes(500, "Authentication system error", nil)
}
func basicAuth(ctx *context.Context) error {
ctx.Response().Header().Set("WWW-Authenticate", `Basic realm="."`)
return ctx.PlainText(401, "Requires authentication")
}
func parseAuthHeader(ctx *context.Context) (string, string, error) {
authHeader := ctx.Request().Header.Get("Authorization")
if authHeader == "" {
return "", "", errors.New("no auth header")
}
authFields := strings.Fields(authHeader)
if len(authFields) != 2 || authFields[0] != "Basic" {
return "", "", errors.New("invalid auth header")
}
authUsername, authPassword, err := basicAuthDecode(authFields[1])
if err != nil {
log.Error().Err(err).Msg("Cannot decode basic auth header")
return "", "", err
}
return authUsername, authPassword, nil
}
func basicAuthDecode(encoded string) (string, string, error) {
s, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return "", "", err
}
auth := strings.SplitN(string(s), ":", 2)
return auth[0], auth[1], nil
}
internal/web/handlers/git/dumb.go
+64
View File
@@ -0,0 +1,64 @@
package git
import (
"fmt"
"net/http"
"os"
"path"
"strings"
"time"
"github.com/thomiceli/opengist/internal/web/context"
)
func textFile(ctx *context.Context) error {
noCacheHeaders(ctx)
return sendFile(ctx, "text/plain")
}
func infoPacks(ctx *context.Context) error {
cacheHeadersForever(ctx)
return sendFile(ctx, "text/plain; charset=utf-8")
}
func looseObject(ctx *context.Context) error {
cacheHeadersForever(ctx)
return sendFile(ctx, "application/x-git-loose-object")
}
func packFile(ctx *context.Context) error {
cacheHeadersForever(ctx)
return sendFile(ctx, "application/x-git-packed-objects")
}
func idxFile(ctx *context.Context) error {
cacheHeadersForever(ctx)
return sendFile(ctx, "application/x-git-packed-objects-toc")
}
func sendFile(ctx *context.Context, contentType string) error {
gitFile := "/" + strings.Join(strings.Split(ctx.Request().URL.Path, "/")[3:], "/")
gitFile = path.Join(ctx.GetData("repositoryPath").(string), gitFile)
fi, err := os.Stat(gitFile)
if os.IsNotExist(err) {
return ctx.ErrorRes(404, "File not found", nil)
}
ctx.Response().Header().Set("Content-Type", contentType)
ctx.Response().Header().Set("Content-Length", fmt.Sprintf("%d", fi.Size()))
ctx.Response().Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))
return ctx.File(gitFile)
}
func noCacheHeaders(ctx *context.Context) {
ctx.Response().Header().Set("Expires", "Thu, 01 Jan 1970 00:00:00 UTC")
ctx.Response().Header().Set("Pragma", "no-cache")
ctx.Response().Header().Set("Cache-Control", "no-cache, max-age=0, must-revalidate")
}
func cacheHeadersForever(ctx *context.Context) {
now := time.Now().Unix()
expires := now + 31536000
ctx.Response().Header().Set("Date", fmt.Sprintf("%d", now))
ctx.Response().Header().Set("Expires", fmt.Sprintf("%d", expires))
ctx.Response().Header().Set("Cache-Control", "public, max-age=31536000")
}
internal/web/handlers/git/http.go
+34 -355
View File
@@ -1,21 +1,9 @@
package git
import (
"bytes"
"compress/gzip"
"encoding/base64"
"errors"
"fmt"
"net/http"
"os"
"os/exec"
"path"
"regexp"
"strconv"
"strings"
"time"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/auth"
"github.com/thomiceli/opengist/internal/db"
@@ -24,11 +12,15 @@ import (
"github.com/thomiceli/opengist/internal/web/handlers"
)
var routes = []struct {
// gitRoute maps a Git smart/dumb HTTP URL (matched as a regexp) and method to
// the handler that serves it.
type gitRoute struct {
gitUrl string
method string
handler func(ctx *context.Context) error
}{
}
var routes = []gitRoute{
{"(.*?)/git-upload-pack$", "POST", uploadPack},
{"(.*?)/git-receive-pack$", "POST", receivePack},
{"(.*?)/info/refs$", "GET", infoRefs},
@@ -42,37 +34,39 @@ var routes = []struct {
{"(.*?)/objects/pack/pack-[0-9a-f]{40}\\.idx$", "GET", idxFile},
}
// GitHttp is the entry point for all Git-over-HTTP requests. It resolves the
// matching route, classifies the request (init / pull / push) and dispatches to
// the relevant handler once access has been authorized.
func GitHttp(ctx *context.Context) error {
route := findMatchingRoute(ctx)
if route == nil {
return ctx.NotFound("Gist not found") // regular 404 for non-git routes
}
gist := ctx.GetData("gist").(*db.Gist)
gistExists := gist.ID != 0
isInitRoute := strings.HasPrefix(ctx.Request().URL.Path, "/init/info/refs")
isInitRouteReceive := strings.HasPrefix(ctx.Request().URL.Path, "/init/git-receive-pack")
initKind, initToken := classifyInitRequest(ctx.Request().URL.Path)
isInfoRefs := strings.HasSuffix(route.gitUrl, "/info/refs$")
isPull := ctx.QueryParam("service") == "git-upload-pack" ||
strings.HasSuffix(ctx.Request().URL.Path, "git-upload-pack") && !isInfoRefs
isPush := ctx.QueryParam("service") == "git-receive-pack" ||
strings.HasSuffix(ctx.Request().URL.Path, "git-receive-pack") && !isInfoRefs
repositoryPath := git.RepositoryPath(gist.User.Username, gist.Uuid)
ctx.SetData("repositoryPath", repositoryPath)
ctx.SetData("repositoryPath", git.RepositoryPath(gist.User.Username, gist.Uuid))
allow, err := auth.ShouldAllowUnauthenticatedGistAccess(handlers.ContextAuthInfo{Context: ctx}, true)
if err != nil {
log.Fatal().Err(err).Msg("Cannot check if unauthenticated access is allowed")
}
// No need to authenticate if the user wants
// to clone/pull ; a non-private gist ; that exists ; where unauthenticated access is allowed in the instance
// No need to authenticate if the user wants to clone/pull ; a non-private
// gist ; that exists ; where unauthenticated access is allowed in the instance
if isPull && gist.Private != db.PrivateVisibility && gistExists && allow {
return route.handler(ctx)
}
// Else we need to authenticate the user, that include other cases:
// Every other case needs credentials:
// - user wants to push the gist
// - user wants to clone/pull a private gist
// - user wants to clone/pull a non-private gist but unauthenticated access is not allowed
@@ -83,350 +77,35 @@ func GitHttp(ctx *context.Context) error {
return basicAuth(ctx)
}
// if the user wants to create a gist via the /init route
if isInitRoute || isInitRouteReceive {
var user *db.User
// check if the user has a valid account on opengist to push a gist
user, err = auth.TryAuthentication(authUsername, authPassword)
if err != nil {
var authErr auth.AuthError
if errors.As(err, &authErr) {
log.Warn().Msg("Invalid HTTP authentication attempt from " + ctx.RealIP())
return ctx.PlainText(401, "Invalid credentials")
}
return ctx.ErrorRes(500, "Authentication system error", nil)
}
if isInitRoute {
gist, err = createGist(user, "")
if err != nil {
return ctx.ErrorRes(500, "Cannot create gist", err)
}
err = db.AddInitGistToQueue(gist.ID, user.ID)
if err != nil {
return ctx.ErrorRes(500, "Cannot add inited gist to the queue", err)
}
ctx.SetData("gist", gist)
return route.handler(ctx)
} else {
gist, err = db.GetInitGistInQueueForUser(user.ID)
if err != nil {
return ctx.ErrorRes(500, "Cannot retrieve inited gist from the queue", err)
}
ctx.SetData("gist", gist)
ctx.SetData("repositoryPath", git.RepositoryPath(gist.User.Username, gist.Uuid))
return route.handler(ctx)
}
}
// if clone/pull
// check if the gist exists and if the credentials are valid
if isPull {
log.Debug().Msg("Detected git pull operation")
if !gistExists {
log.Debug().Str("authUsername", authUsername).Msg("Pulling unknown gist")
return ctx.PlainText(404, "Check your credentials or make sure you have access to the Gist")
}
var userToCheckPermissions string
// if the user is trying to clone/pull a non-private gist while unauthenticated access is not allowed,
// check if the user has a valid account
if gist.Private != db.PrivateVisibility {
log.Debug().Str("authUsername", authUsername).Msg("Pulling non-private gist with authenticated access")
userToCheckPermissions = authUsername
} else { // else just check the password against the gist owner
log.Debug().Str("authUsername", authUsername).Str("gistOwner", gist.User.Username).Msg("Pulling private gist")
userToCheckPermissions = gist.User.Username
}
if _, err = auth.TryAuthentication(userToCheckPermissions, authPassword); err != nil {
var authErr auth.AuthError
if errors.As(err, &authErr) {
log.Warn().Msg("Invalid HTTP authentication attempt from " + ctx.RealIP())
return ctx.PlainText(404, "Check your credentials or make sure you have access to the Gist")
}
return ctx.ErrorRes(500, "Authentication system error", nil)
}
log.Debug().Str("authUsername", authUsername).Msg("Pulling gist")
switch {
case initKind != initNone:
return handleInit(ctx, route, initKind, initToken, authUsername, authPassword)
case isPull:
return handlePull(ctx, route, gist, gistExists, authUsername, authPassword)
case isPush:
return handlePush(ctx, route, gist, gistExists, authUsername, authPassword)
default:
return route.handler(ctx)
}
if isPush {
log.Debug().Msg("Detected git push operation")
// if gist exists, check if the credentials are valid and if the user is the gist owner
if gistExists {
log.Debug().Str("authUsername", authUsername).Str("gistOwner", gist.User.Username).Msg("Pushing to existing gist")
if _, err = auth.TryAuthentication(gist.User.Username, authPassword); err != nil {
var authErr auth.AuthError
if errors.As(err, &authErr) {
log.Warn().Msg("Invalid HTTP authentication attempt from " + ctx.RealIP())
return ctx.PlainText(404, "Check your credentials or make sure you have access to the Gist")
}
return ctx.ErrorRes(500, "Authentication system error", nil)
}
if gist.Archived {
log.Debug().Str("authUsername", authUsername).Msg("Pushing to archived gist")
return ctx.PlainText(403, "This gist is archived and is read-only")
}
log.Debug().Str("authUsername", authUsername).Msg("Pushing gist")
return route.handler(ctx)
} else { // if the gist does not exist, check if the user has a valid account on opengist to push a gist and create it
log.Debug().Str("authUsername", authUsername).Msg("Creating new gist by pushing")
var user *db.User
if user, err = auth.TryAuthentication(authUsername, authPassword); err != nil {
var authErr auth.AuthError
if errors.As(err, &authErr) {
log.Warn().Msg("Invalid HTTP authentication attempt from " + ctx.RealIP())
return ctx.PlainText(404, "Check your credentials or make sure you have access to the Gist")
}
return ctx.ErrorRes(500, "Authentication system error", nil)
}
urlPath := ctx.Request().URL.Path
pathParts := strings.Split(strings.Trim(urlPath, "/"), "/")
if pathParts[0] == authUsername && len(pathParts) == 4 {
log.Debug().Str("authUsername", authUsername).Msg("Valid URL format for push operation")
gist, err = createGist(user, pathParts[1])
if err != nil {
return ctx.ErrorRes(500, "Cannot create gist", err)
}
log.Debug().Str("authUsername", authUsername).Str("url", urlPath).Msg("Gist created")
ctx.SetData("gist", gist)
ctx.SetData("repositoryPath", git.RepositoryPath(gist.User.Username, gist.Uuid))
} else {
log.Debug().Str("authUsername", authUsername).Any("path", pathParts).Msg("Invalid URL format for push operation")
return ctx.PlainText(401, "Invalid URL format for push operation")
}
return route.handler(ctx)
}
}
return route.handler(ctx)
}
func findMatchingRoute(ctx *context.Context) *struct {
gitUrl string
method string
handler func(ctx *context.Context) error
} {
for _, route := range routes {
// setGistContext points the request at a specific gist and its repository path,
// overriding whatever the soft-init middleware put in place.
func setGistContext(ctx *context.Context, gist *db.Gist) {
ctx.SetData("gist", gist)
ctx.SetData("repositoryPath", git.RepositoryPath(gist.User.Username, gist.Uuid))
}
func findMatchingRoute(ctx *context.Context) *gitRoute {
for i := range routes {
route := &routes[i]
matched, _ := regexp.MatchString(route.gitUrl, ctx.Request().URL.Path)
if ctx.Request().Method == route.method && matched {
if !strings.HasPrefix(ctx.Request().Header.Get("User-Agent"), "git/") {
continue
}
return &route
return route
}
}
return nil
}
func createGist(user *db.User, url string) (*db.Gist, error) {
gist := new(db.Gist)
gist.UserID = user.ID
gist.User = *user
uuidGist, err := uuid.NewRandom()
if err != nil {
return nil, err
}
gist.Uuid = strings.ReplaceAll(uuidGist.String(), "-", "")
gist.Title = "gist:" + gist.Uuid
if url != "" {
gist.URL = strings.TrimSuffix(url, ".git")
gist.Title = strings.TrimSuffix(url, ".git")
}
if err := gist.InitRepository(); err != nil {
return nil, err
}
if err := gist.Create(); err != nil {
return nil, err
}
return gist, nil
}
func uploadPack(ctx *context.Context) error {
return pack(ctx, "upload-pack")
}
func receivePack(ctx *context.Context) error {
return pack(ctx, "receive-pack")
}
func pack(ctx *context.Context, serviceType string) error {
noCacheHeaders(ctx)
defer ctx.Request().Body.Close()
if ctx.Request().Header.Get("Content-Type") != "application/x-git-"+serviceType+"-request" {
return ctx.ErrorRes(401, "Git client unsupported", nil)
}
ctx.Response().Header().Set("Content-Type", "application/x-git-"+serviceType+"-result")
var err error
reqBody := ctx.Request().Body
if ctx.Request().Header.Get("Content-Encoding") == "gzip" {
reqBody, err = gzip.NewReader(reqBody)
if err != nil {
return ctx.ErrorRes(500, "Cannot create gzip reader", err)
}
}
repositoryPath := ctx.GetData("repositoryPath").(string)
gist := ctx.GetData("gist").(*db.Gist)
var stderr bytes.Buffer
cmd := exec.Command("git", serviceType, "--stateless-rpc", repositoryPath)
cmd.Dir = repositoryPath
cmd.Stdin = reqBody
cmd.Stdout = ctx.Response().Writer
cmd.Stderr = &stderr
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, "OPENGIST_REPOSITORY_URL_INTERNAL="+git.RepositoryUrl(ctx, gist.User.Username, gist.Identifier()))
cmd.Env = append(cmd.Env, "OPENGIST_REPOSITORY_ID="+strconv.Itoa(int(gist.ID)))
if err = cmd.Run(); err != nil {
return ctx.ErrorRes(500, "Cannot run git "+serviceType+" ; "+stderr.String(), err)
}
return nil
}
func infoRefs(ctx *context.Context) error {
noCacheHeaders(ctx)
var service string
gist := ctx.GetData("gist").(*db.Gist)
serviceType := ctx.QueryParam("service")
if strings.HasPrefix(serviceType, "git-") {
service = strings.TrimPrefix(serviceType, "git-")
}
if service != "upload-pack" && service != "receive-pack" {
if err := gist.UpdateServerInfo(); err != nil {
return ctx.ErrorRes(500, "Cannot update server info", err)
}
return sendFile(ctx, "text/plain; charset=utf-8")
}
refs, err := gist.RPC(service)
if err != nil {
return ctx.ErrorRes(500, "Cannot run git "+service, err)
}
ctx.Response().Header().Set("Content-Type", "application/x-git-"+service+"-advertisement")
ctx.Response().WriteHeader(200)
_, _ = ctx.Response().Write(packetWrite("# service=git-" + service + "\n"))
_, _ = ctx.Response().Write([]byte("0000"))
_, _ = ctx.Response().Write(refs)
return nil
}
func textFile(ctx *context.Context) error {
noCacheHeaders(ctx)
return sendFile(ctx, "text/plain")
}
func infoPacks(ctx *context.Context) error {
cacheHeadersForever(ctx)
return sendFile(ctx, "text/plain; charset=utf-8")
}
func looseObject(ctx *context.Context) error {
cacheHeadersForever(ctx)
return sendFile(ctx, "application/x-git-loose-object")
}
func packFile(ctx *context.Context) error {
cacheHeadersForever(ctx)
return sendFile(ctx, "application/x-git-packed-objects")
}
func idxFile(ctx *context.Context) error {
cacheHeadersForever(ctx)
return sendFile(ctx, "application/x-git-packed-objects-toc")
}
func noCacheHeaders(ctx *context.Context) {
ctx.Response().Header().Set("Expires", "Thu, 01 Jan 1970 00:00:00 UTC")
ctx.Response().Header().Set("Pragma", "no-cache")
ctx.Response().Header().Set("Cache-Control", "no-cache, max-age=0, must-revalidate")
}
func cacheHeadersForever(ctx *context.Context) {
now := time.Now().Unix()
expires := now + 31536000
ctx.Response().Header().Set("Date", fmt.Sprintf("%d", now))
ctx.Response().Header().Set("Expires", fmt.Sprintf("%d", expires))
ctx.Response().Header().Set("Cache-Control", "public, max-age=31536000")
}
func basicAuth(ctx *context.Context) error {
ctx.Response().Header().Set("WWW-Authenticate", `Basic realm="."`)
return ctx.PlainText(401, "Requires authentication")
}
func parseAuthHeader(ctx *context.Context) (string, string, error) {
authHeader := ctx.Request().Header.Get("Authorization")
if authHeader == "" {
return "", "", errors.New("no auth header")
}
authFields := strings.Fields(authHeader)
if len(authFields) != 2 || authFields[0] != "Basic" {
return "", "", errors.New("invalid auth header")
}
authUsername, authPassword, err := basicAuthDecode(authFields[1])
if err != nil {
log.Error().Err(err).Msg("Cannot decode basic auth header")
return "", "", err
}
return authUsername, authPassword, nil
}
func basicAuthDecode(encoded string) (string, string, error) {
s, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return "", "", err
}
auth := strings.SplitN(string(s), ":", 2)
return auth[0], auth[1], nil
}
func sendFile(ctx *context.Context, contentType string) error {
gitFile := "/" + strings.Join(strings.Split(ctx.Request().URL.Path, "/")[3:], "/")
gitFile = path.Join(ctx.GetData("repositoryPath").(string), gitFile)
fi, err := os.Stat(gitFile)
if os.IsNotExist(err) {
return ctx.ErrorRes(404, "File not found", nil)
}
ctx.Response().Header().Set("Content-Type", contentType)
ctx.Response().Header().Set("Content-Length", fmt.Sprintf("%d", fi.Size()))
ctx.Response().Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))
return ctx.File(gitFile)
}
func packetWrite(str string) []byte {
s := strconv.FormatInt(int64(len(str)+4), 16)
if len(s)%4 != 0 {
s = strings.Repeat("0", 4-len(s)%4) + s
}
return []byte(s + str)
}
internal/web/handlers/git/http_test.go
+100
View File
@@ -1,10 +1,12 @@
package git_test
import (
"fmt"
"net/url"
"os"
"os/exec"
"path/filepath"
"sync"
"testing"
"github.com/stretchr/testify/require"
@@ -212,6 +214,104 @@ func TestGitPushArchived(t *testing.T) {
require.NoError(t, gitPush(dest, "afterunarchive.txt", "content"))
}
// gitInitAndPushTo initializes a fresh repo with a single file and pushes it to
// remotePath (e.g. "/init") as creds. It returns whatever git push returns.
func gitInitAndPushTo(baseUrl, creds, remotePath, filename, content string, destDir string) error {
if err := exec.Command("git", "init", "--initial-branch=master", destDir).Run(); err != nil {
return err
}
remote := "http://" + creds + "@" + baseUrl[len("http://"):] + remotePath
if err := exec.Command("git", "-C", destDir, "remote", "add", "origin", remote).Run(); err != nil {
return err
}
if err := os.WriteFile(filepath.Join(destDir, filename), []byte(content), 0644); err != nil {
return err
}
if err := exec.Command("git", "-C", destDir, "add", ".").Run(); err != nil {
return err
}
if err := exec.Command("git", "-C", destDir, "commit", "-m", "init").Run(); err != nil {
return err
}
return exec.Command("git", "-C", destDir, "push", "origin", "master").Run()
}
// TestGitInitPushParallel hammers the /init create-by-push flow concurrently.
// Before the per-push correlation token, the two HTTP requests git makes
// (info/refs then git-receive-pack) were matched via a per-user FIFO queue, so
// interleaved pushes desynced it: pushes 500'd or content landed in the wrong
// gist. This asserts every parallel push lands in its own gist with its own
// content, and that the queue drains to empty.
func TestGitInitPushParallel(t *testing.T) {
s := webtest.Setup(t)
defer webtest.Teardown(t)
baseUrl := s.StartHttpServer(t)
s.Register(t, "thomas")
const n = 10
creds := "thomas:thomas"
// Each push writes a distinct file/content so we can detect misrouting.
want := make(map[string]string, n)
for i := 0; i < n; i++ {
want[fmt.Sprintf("file-%d.txt", i)] = fmt.Sprintf("content-%d", i)
}
// Pre-create the temp dirs on the main goroutine, then push from all of them
// at once.
dirs := make([]string, n)
for i := 0; i < n; i++ {
dirs[i] = t.TempDir()
}
var wg sync.WaitGroup
errs := make([]error, n)
for i := 0; i < n; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
errs[i] = gitInitAndPushTo(baseUrl, creds, "/init",
fmt.Sprintf("file-%d.txt", i), fmt.Sprintf("content-%d", i), dirs[i])
}(i)
}
wg.Wait()
for i, err := range errs {
require.NoErrorf(t, err, "parallel /init push %d failed", i)
}
user, err := db.GetUserByUsername("thomas")
require.NoError(t, err)
// Exactly n gists were created, one per push.
count, err := db.CountAllGistsFromUser(user.ID, user.ID)
require.NoError(t, err)
require.EqualValues(t, n, count, "expected one gist per parallel push")
// Every pushed file/content is present exactly once across the gists, proving
// no push's content was routed into another push's gist.
gists, err := db.GetAllGistsOfUser(user.ID, nil, 0, "created", "desc", 100, 100)
require.NoError(t, err)
got := make(map[string]string, n)
for _, g := range gists {
files, _, err := g.Files("HEAD", false)
require.NoError(t, err)
require.Len(t, files, 1, "each init gist should contain exactly one file")
f := files[0]
_, dup := got[f.Filename]
require.Falsef(t, dup, "file %q appeared in more than one gist", f.Filename)
got[f.Filename] = f.Content
}
require.Equal(t, want, got, "pushed files/content did not map one-to-one onto gists")
// The init queue must drain completely, leaving no stale correlation entries.
queued, err := db.CountAll(&db.GistInitQueue{})
require.NoError(t, err)
require.EqualValues(t, 0, queued, "init queue should be empty after all pushes complete")
}
func TestGitCreatePush(t *testing.T) {
s := webtest.Setup(t)
defer webtest.Teardown(t)
internal/web/handlers/git/init.go
+158
View File
@@ -0,0 +1,158 @@
package git
import (
"strings"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
"github.com/thomiceli/opengist/internal/db"
"github.com/thomiceli/opengist/internal/web/context"
)
type initKind int
const (
initNone initKind = iota // not an /init request
initStart // GET /init/info/refs (create gist, redirect)
initToken2InfoRefs // GET /init/<token>/info/refs
initToken2Receive // POST /init/<token>/git-receive-pack
initLegacyReceive // POST /init/git-receive-pack (no token)
)
// classifyInitRequest inspects the request path and reports which leg of the
// /init push flow it belongs to, returning the correlation token when present.
func classifyInitRequest(urlPath string) (initKind, string) {
parts := strings.Split(strings.Trim(urlPath, "/"), "/")
if len(parts) < 2 || parts[0] != "init" {
return initNone, ""
}
switch parts[1] {
case "info": // /init/info/refs
return initStart, ""
case "git-receive-pack": // /init/git-receive-pack (legacy, no token)
return initLegacyReceive, ""
default: // /init/<token>/...
token := parts[1]
tail := strings.Join(parts[2:], "/")
switch {
case strings.HasPrefix(tail, "info/refs"):
return initToken2InfoRefs, token
case tail == "git-receive-pack":
return initToken2Receive, token
default:
return initNone, ""
}
}
}
// handleInit serves a request belonging to the "git push .../init" flow, after
// the caller has been authenticated.
func handleInit(ctx *context.Context, route *gitRoute, kind initKind, token, username, password string) error {
user, err := authOrFail(ctx, username, password, 401, "Invalid credentials")
if user == nil {
return err
}
switch kind {
case initStart:
return initStartPush(ctx, user)
case initToken2InfoRefs:
return initServeToken(ctx, route, user, token, false)
case initToken2Receive:
return initServeToken(ctx, route, user, token, true)
default: // initLegacyReceive
// git client that did not follow the correlation redirect: fall back to
// the per-user queue, popping the oldest pending init gist atomically.
gist, err := db.PopInitGistForUser(user.ID)
if err != nil {
return ctx.ErrorRes(500, "Cannot retrieve inited gist from the queue", err)
}
setGistContext(ctx, gist)
return route.handler(ctx)
}
}
// initStartPush handles the first request of a push to /init: it creates the
// gist and hands the client a per-push token via a redirect. git follows the
// redirect on this initial request (http.followRedirects=initial, the default)
// and uses the redirected URL as the base for the git-receive-pack POST, so that
// POST is unambiguously tied to this gist.
func initStartPush(ctx *context.Context, user *db.User) error {
gist, err := createGist(user, "")
if err != nil {
return ctx.ErrorRes(500, "Cannot create gist", err)
}
token, err := newInitToken()
if err != nil {
return ctx.ErrorRes(500, "Cannot create init token", err)
}
if err = db.AddInitGistToQueue(gist.ID, user.ID, token); err != nil {
return ctx.ErrorRes(500, "Cannot add inited gist to the queue", err)
}
// Relative redirect so it stays correct behind a reverse-proxy subpath.
noCacheHeaders(ctx)
return ctx.Redirect(302, "../"+token+"/info/refs?service=git-receive-pack")
}
// initServeToken resolves the gist for a token-carrying second-leg request and
// dispatches to the underlying handler. The info/refs leg looks it up without
// consuming the queue entry; the receive-pack leg atomically consumes it.
func initServeToken(ctx *context.Context, route *gitRoute, user *db.User, token string, consume bool) error {
var gist *db.Gist
var err error
if consume {
gist, err = db.PopInitGistByToken(token)
} else {
gist, err = db.GetInitGistByToken(token)
}
if err != nil {
return ctx.PlainText(404, "Unknown or expired init token")
}
if gist.UserID != user.ID {
log.Warn().Msg("Init token user mismatch from " + ctx.RealIP())
return ctx.PlainText(404, "Unknown or expired init token")
}
setGistContext(ctx, gist)
return route.handler(ctx)
}
// newInitToken returns a random, URL-safe token used to correlate the two HTTP
// requests of a single push to /init.
func newInitToken() (string, error) {
u, err := uuid.NewRandom()
if err != nil {
return "", err
}
return strings.ReplaceAll(u.String(), "-", ""), nil
}
func createGist(user *db.User, url string) (*db.Gist, error) {
gist := new(db.Gist)
gist.UserID = user.ID
gist.User = *user
uuidGist, err := uuid.NewRandom()
if err != nil {
return nil, err
}
gist.Uuid = strings.ReplaceAll(uuidGist.String(), "-", "")
gist.Title = "gist:" + gist.Uuid
if url != "" {
gist.URL = strings.TrimSuffix(url, ".git")
gist.Title = strings.TrimSuffix(url, ".git")
}
if err := gist.InitRepository(); err != nil {
return nil, err
}
if err := gist.Create(); err != nil {
return nil, err
}
return gist, nil
}
internal/web/handlers/git/pack.go
+111
View File
@@ -0,0 +1,111 @@
package git
import (
"bytes"
"compress/gzip"
"os"
"os/exec"
"strconv"
"strings"
"github.com/thomiceli/opengist/internal/db"
"github.com/thomiceli/opengist/internal/git"
"github.com/thomiceli/opengist/internal/web/context"
)
func uploadPack(ctx *context.Context) error {
return pack(ctx, "upload-pack")
}
func receivePack(ctx *context.Context) error {
return pack(ctx, "receive-pack")
}
func pack(ctx *context.Context, serviceType string) error {
noCacheHeaders(ctx)
defer ctx.Request().Body.Close()
if ctx.Request().Header.Get("Content-Type") != "application/x-git-"+serviceType+"-request" {
return ctx.ErrorRes(401, "Git client unsupported", nil)
}
ctx.Response().Header().Set("Content-Type", "application/x-git-"+serviceType+"-result")
var err error
reqBody := ctx.Request().Body
if ctx.Request().Header.Get("Content-Encoding") == "gzip" {
reqBody, err = gzip.NewReader(reqBody)
if err != nil {
return ctx.ErrorRes(500, "Cannot create gzip reader", err)
}
}
repositoryPath := ctx.GetData("repositoryPath").(string)
gist := ctx.GetData("gist").(*db.Gist)
// Guard against a stale/desynced reference pointing at a repository that no
// longer exists on disk (e.g. an init gist whose empty repo was cleaned up).
// Without this, git would fail with an opaque "chdir ...: no such file or
// directory" and 500 the push.
if fi, err := os.Stat(repositoryPath); err != nil || !fi.IsDir() {
return ctx.ErrorRes(404, "Repository not found", err)
}
var stderr bytes.Buffer
cmd := exec.Command("git", serviceType, "--stateless-rpc", repositoryPath)
cmd.Dir = repositoryPath
cmd.Stdin = reqBody
cmd.Stdout = ctx.Response().Writer
cmd.Stderr = &stderr
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, "OPENGIST_REPOSITORY_URL_INTERNAL="+git.RepositoryUrl(ctx, gist.User.Username, gist.Identifier()))
cmd.Env = append(cmd.Env, "OPENGIST_REPOSITORY_ID="+strconv.Itoa(int(gist.ID)))
if err = cmd.Run(); err != nil {
return ctx.ErrorRes(500, "Cannot run git "+serviceType+" ; "+stderr.String(), err)
}
return nil
}
func infoRefs(ctx *context.Context) error {
noCacheHeaders(ctx)
var service string
gist := ctx.GetData("gist").(*db.Gist)
serviceType := ctx.QueryParam("service")
if strings.HasPrefix(serviceType, "git-") {
service = strings.TrimPrefix(serviceType, "git-")
}
if service != "upload-pack" && service != "receive-pack" {
if err := gist.UpdateServerInfo(); err != nil {
return ctx.ErrorRes(500, "Cannot update server info", err)
}
return sendFile(ctx, "text/plain; charset=utf-8")
}
refs, err := gist.RPC(service)
if err != nil {
return ctx.ErrorRes(500, "Cannot run git "+service, err)
}
ctx.Response().Header().Set("Content-Type", "application/x-git-"+service+"-advertisement")
ctx.Response().WriteHeader(200)
_, _ = ctx.Response().Write(packetWrite("# service=git-" + service + "\n"))
_, _ = ctx.Response().Write([]byte("0000"))
_, _ = ctx.Response().Write(refs)
return nil
}
func packetWrite(str string) []byte {
s := strconv.FormatInt(int64(len(str)+4), 16)
if len(s)%4 != 0 {
s = strings.Repeat("0", 4-len(s)%4) + s
}
return []byte(s + str)
}
internal/web/test/server.go
+26 -5
View File
@@ -9,6 +9,7 @@ import (
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"github.com/gorilla/schema"
@@ -34,7 +35,25 @@ func init() {
type Server struct {
server *server.Server
SessionCookie string
contextData echo.Map
// contextData captures the last request's context data for assertion
// helpers. It is written by a middleware on every request, so concurrent
// requests (e.g. parallel pushes) must serialize access via mu.
mu sync.Mutex
contextData echo.Map
}
func (s *Server) setContextData(data echo.Map) {
s.mu.Lock()
defer s.mu.Unlock()
s.contextData = data
}
func (s *Server) ContextData(key string) (any, bool) {
s.mu.Lock()
defer s.mu.Unlock()
v, ok := s.contextData[key]
return v, ok
}
func (s *Server) Request(t *testing.T, method, uri string, data interface{}, expectedCode int) *http.Response {
@@ -118,15 +137,17 @@ func (s *Server) StartHttpServer(t *testing.T) string {
func (s *Server) User() *db.User {
s.Request(nil, "GET", "/", nil, 0)
if user, ok := s.contextData["userLogged"].(*db.User); ok {
return user
if v, ok := s.ContextData("userLogged"); ok {
if user, ok := v.(*db.User); ok {
return user
}
}
return nil
}
func (s *Server) TestCtxData(t *testing.T, expected echo.Map) {
for key, expectedValue := range expected {
actualValue, exists := s.contextData[key]
actualValue, exists := s.ContextData(key)
require.True(t, exists, "Key %q not found in context data", key)
require.Equal(t, expectedValue, actualValue, "Context data mismatch for key %q", key)
}
@@ -277,7 +298,7 @@ func Setup(t *testing.T) *Server {
return func(c echo.Context) error {
err := next(c)
if data, ok := c.Request().Context().Value(context.DataKeyStr).(echo.Map); ok {
s.contextData = data
s.setContextData(data)
}
return err
}