diff --git a/internal/db/gist.go b/internal/db/gist.go index 11dc428..59ce691 100644 --- a/internal/db/gist.go +++ b/internal/db/gist.go @@ -414,14 +414,14 @@ func (gist *Gist) DeleteRepository() error { return git.DeleteRepository(gist.User.Username, gist.Uuid) } -func (gist *Gist) Files(revision string, truncate bool) ([]*git.File, error) { - filesCat, err := git.CatFileBatch(gist.User.Username, gist.Uuid, revision, truncate) +func (gist *Gist) Files(revision string, truncate bool) ([]*git.File, bool, error) { + filesCat, gistTruncated, err := git.CatFileBatch(gist.User.Username, gist.Uuid, revision, truncate) if err != nil { // if the revision or the file do not exist if exiterr, ok := err.(*exec.ExitError); ok && exiterr.ExitCode() == 128 { - return nil, &git.RevisionNotFoundError{} + return nil, false, &git.RevisionNotFoundError{} } - return nil, err + return nil, false, err } var files []*git.File @@ -442,7 +442,7 @@ func (gist *Gist) Files(revision string, truncate bool) ([]*git.File, error) { MimeType: git.DetectMimeType([]byte(shortContent), filepath.Ext(fileCat.Name)), }) } - return files, err + return files, gistTruncated, err } func (gist *Gist) File(revision string, filename string, truncate bool) (*git.File, error) { @@ -613,7 +613,7 @@ func (gist *Gist) Identifier() string { } func (gist *Gist) GetLanguagesFromFiles() ([]string, error) { - files, err := gist.Files("HEAD", true) + files, _, err := gist.Files("HEAD", true) if err != nil { return nil, err } @@ -694,7 +694,7 @@ func (gist *Gist) UpdateLanguages() { } func (gist *Gist) ToDTO() (*GistDTO, error) { - files, err := gist.Files("HEAD", false) + files, _, err := gist.Files("HEAD", false) if err != nil { return nil, err } @@ -786,7 +786,7 @@ func (dto *GistDTO) TopicStrToSlice() []GistTopic { // -- Index -- // func (gist *Gist) ToIndexedGist() (*index.Gist, error) { - files, err := gist.Files("HEAD", true) + files, _, err := gist.Files("HEAD", true) if err != nil { return nil, err } diff --git a/internal/git/commands.go b/internal/git/commands.go index 15e53de..439c1c0 100644 --- a/internal/git/commands.go +++ b/internal/git/commands.go @@ -132,21 +132,36 @@ type catFileBatch struct { Truncated bool } -func CatFileBatch(user string, gist string, revision string, truncate bool) ([]*catFileBatch, error) { +func CatFileBatch(user string, gist string, revision string, truncate bool) ([]*catFileBatch, bool, error) { repositoryPath := RepositoryPath(user, gist) + maxFiles := 50 lsTreeCmd := exec.Command("git", "ls-tree", "-l", revision) lsTreeCmd.Dir = repositoryPath - lsTreeOutput, err := lsTreeCmd.Output() + + var lsTreeStderr bytes.Buffer + lsTreeCmd.Stderr = &lsTreeStderr + + lsTreeStdout, err := lsTreeCmd.StdoutPipe() if err != nil { - return nil, err + return nil, false, err + } + if err = lsTreeCmd.Start(); err != nil { + return nil, false, err } fileMap := make([]*catFileBatch, 0) + gistTruncated := false - lines := strings.Split(string(lsTreeOutput), "\n") - for _, line := range lines { - fields := strings.Fields(line) + scanner := bufio.NewScanner(lsTreeStdout) + scanner.Buffer(make([]byte, 64*1024), 1024*1024) + for scanner.Scan() { + if truncate && len(fileMap) >= maxFiles { + gistTruncated = true + break + } + + fields := strings.Fields(scanner.Text()) if len(fields) < 4 { continue // Skip lines that don't have enough fields } @@ -164,19 +179,33 @@ func CatFileBatch(user string, gist string, revision string, truncate bool) ([]* Name: convertOctalToUTF8(name), }) } + scanErr := scanner.Err() + + // Closing the read end before git is done writing causes git's next write + // to fail (SIGPIPE on Unix, broken-pipe error on Windows). That shows up as + // a non-zero exit from Wait, but it's expected when we stop early — so we + // only treat the Wait error as real if git actually printed something to stderr. + _ = lsTreeStdout.Close() + waitErr := lsTreeCmd.Wait() + if scanErr != nil { + return nil, false, scanErr + } + if waitErr != nil && lsTreeStderr.Len() > 0 { + return nil, false, waitErr + } catFileCmd := exec.Command("git", "cat-file", "--batch") catFileCmd.Dir = repositoryPath stdin, err := catFileCmd.StdinPipe() if err != nil { - return nil, err + return nil, false, err } stdout, err := catFileCmd.StdoutPipe() if err != nil { - return nil, err + return nil, false, err } if err = catFileCmd.Start(); err != nil { - return nil, err + return nil, false, err } reader := bufio.NewReader(stdout) @@ -184,12 +213,12 @@ func CatFileBatch(user string, gist string, revision string, truncate bool) ([]* for _, file := range fileMap { _, err = stdin.Write([]byte(file.Hash + "\n")) if err != nil { - return nil, err + return nil, false, err } header, err := reader.ReadString('\n') if err != nil { - return nil, err + return nil, false, err } parts := strings.Fields(header) @@ -199,7 +228,7 @@ func CatFileBatch(user string, gist string, revision string, truncate bool) ([]* size, err := strconv.ParseUint(parts[2], 10, 64) if err != nil { - return nil, err + return nil, false, err } // Don't truncate Jupyter notebooks @@ -215,7 +244,7 @@ func CatFileBatch(user string, gist string, revision string, truncate bool) ([]* // Read exactly size bytes from header, or the max allowed if truncated content := make([]byte, sizeToRead) if _, err = io.ReadFull(reader, content); err != nil { - return nil, err + return nil, false, err } file.Content = string(content) @@ -223,26 +252,26 @@ func CatFileBatch(user string, gist string, revision string, truncate bool) ([]* if truncate && size > truncateLimit { // skip other bytes if truncated if _, err = reader.Discard(int(size - truncateLimit)); err != nil { - return nil, err + return nil, false, err } file.Truncated = true } // Read the blank line following the content if _, err := reader.ReadByte(); err != nil { - return nil, err + return nil, false, err } } if err = stdin.Close(); err != nil { - return nil, err + return nil, false, err } if err = catFileCmd.Wait(); err != nil { - return nil, err + return nil, false, err } - return fileMap, nil + return fileMap, gistTruncated, nil } func GetFileContent(user string, gist string, revision string, filename string, truncate bool) (string, bool, error) { diff --git a/internal/i18n/locales/en-US.yml b/internal/i18n/locales/en-US.yml index 5e4d7ae..5014843 100644 --- a/internal/i18n/locales/en-US.yml +++ b/internal/i18n/locales/en-US.yml @@ -23,6 +23,7 @@ gist.header.download-zip: Download ZIP gist.raw: Raw gist.file-truncated: This file has been truncated. +gist.files-truncated: Not all files in this gist are not displayed. Clone or download the gist to see them all. gist.file-raw: This file can't be rendered. gist.file-binary-edit: This file is binary. gist.watch-full-file: View the full file. diff --git a/internal/web/handlers/gist/create.go b/internal/web/handlers/gist/create.go index 57bafdd..243d54a 100644 --- a/internal/web/handlers/gist/create.go +++ b/internal/web/handlers/gist/create.go @@ -127,7 +127,7 @@ func ProcessCreate(ctx *context.Context) error { if isCreate { return ctx.HtmlWithCode(400, "create.html") } else { - files, err := gist.Files("HEAD", false) + files, _, err := gist.Files("HEAD", false) if err != nil { return ctx.ErrorRes(500, "Error fetching files", err) } diff --git a/internal/web/handlers/gist/create_test.go b/internal/web/handlers/gist/create_test.go index 8706f70..7a34dd0 100644 --- a/internal/web/handlers/gist/create_test.go +++ b/internal/web/handlers/gist/create_test.go @@ -494,7 +494,7 @@ func TestGistCreation(t *testing.T) { // Verify files if specified if len(tt.expectedFileNames) > 0 { - files, err := gist.Files("HEAD", false) + files, _, err := gist.Files("HEAD", false) require.NoError(t, err, "Failed to get gist files") require.Len(t, files, len(tt.expectedFileNames), "File count mismatch") diff --git a/internal/web/handlers/gist/download.go b/internal/web/handlers/gist/download.go index 946b04b..6d3baba 100644 --- a/internal/web/handlers/gist/download.go +++ b/internal/web/handlers/gist/download.go @@ -67,7 +67,7 @@ func DownloadZip(ctx *context.Context) error { gist := ctx.GetData("gist").(*db.Gist) revision := ctx.Param("revision") - files, err := gist.Files(revision, false) + files, _, err := gist.Files(revision, false) if err != nil { return ctx.ErrorRes(500, "Error fetching files from repository", err) } diff --git a/internal/web/handlers/gist/fork_test.go b/internal/web/handlers/gist/fork_test.go index 503eafb..9c95eba 100644 --- a/internal/web/handlers/gist/fork_test.go +++ b/internal/web/handlers/gist/fork_test.go @@ -29,10 +29,10 @@ func TestFork(t *testing.T) { require.Equal(t, gist.Private, forkedGist.Private) require.Equal(t, gist.ID, forkedGist.ForkedID) - forkedFiles, err := forkedGist.Files("HEAD", false) + forkedFiles, _, err := forkedGist.Files("HEAD", false) require.NoError(t, err) - gistFiles, err := gist.Files("HEAD", false) + gistFiles, _, err := gist.Files("HEAD", false) require.NoError(t, err) for i, file := range gistFiles { diff --git a/internal/web/handlers/gist/gist.go b/internal/web/handlers/gist/gist.go index bd297a4..63d9296 100644 --- a/internal/web/handlers/gist/gist.go +++ b/internal/web/handlers/gist/gist.go @@ -28,7 +28,7 @@ func GistIndex(ctx *context.Context) error { revision = "HEAD" } - files, err := gist.Files(revision, true) + files, hasMoreFiles, err := gist.Files(revision, true) if _, ok := err.(*git.RevisionNotFoundError); ok { return ctx.NotFound("Revision not found") } else if err != nil { @@ -40,6 +40,7 @@ func GistIndex(ctx *context.Context) error { ctx.SetData("page", "code") ctx.SetData("commit", revision) ctx.SetData("files", renderedFiles) + ctx.SetData("hasMoreFiles", hasMoreFiles) ctx.SetData("revision", revision) ctx.SetData("htmlTitle", gist.Title) return ctx.Html("gist.html") @@ -47,13 +48,14 @@ func GistIndex(ctx *context.Context) error { func GistJson(ctx *context.Context) error { gist := ctx.GetData("gist").(*db.Gist) - files, err := gist.Files("HEAD", true) + files, hasMoreFiles, err := gist.Files("HEAD", true) if err != nil { return ctx.ErrorRes(500, "Error fetching files", err) } renderedFiles := render.RenderFiles(files) ctx.SetData("files", renderedFiles) + ctx.SetData("hasMoreFiles", hasMoreFiles) topics, err := gist.GetTopics() if err != nil { @@ -104,13 +106,14 @@ func GistJs(ctx *context.Context) error { } gist := ctx.GetData("gist").(*db.Gist) - files, err := gist.Files("HEAD", true) + files, hasMoreFiles, err := gist.Files("HEAD", true) if err != nil { return ctx.ErrorRes(500, "Error fetching files", err) } renderedFiles := render.RenderFiles(files) ctx.SetData("files", renderedFiles) + ctx.SetData("hasMoreFiles", hasMoreFiles) htmlbuf := bytes.Buffer{} w := bufio.NewWriter(&htmlbuf) diff --git a/internal/web/handlers/gist/gist_test.go b/internal/web/handlers/gist/gist_test.go index ae36c21..72689d6 100644 --- a/internal/web/handlers/gist/gist_test.go +++ b/internal/web/handlers/gist/gist_test.go @@ -81,7 +81,7 @@ func TestGistIndex(t *testing.T) { "content": {"updated content"}, }, 302) - files, err := gist.Files("HEAD", false) + files, _, err := gist.Files("HEAD", false) require.NoError(t, err) found := false for _, f := range files { @@ -96,7 +96,7 @@ func TestGistIndex(t *testing.T) { require.NoError(t, err) require.Len(t, commits, 2) - filesOld, err := gist.Files(commits[1].Hash, false) + filesOld, _, err := gist.Files(commits[1].Hash, false) require.NoError(t, err) for _, f := range filesOld { if f.Filename == "file.txt" { diff --git a/templates/pages/gist.html b/templates/pages/gist.html index 3f94879..47373ec 100644 --- a/templates/pages/gist.html +++ b/templates/pages/gist.html @@ -2,6 +2,11 @@ {{ template "gist_header" .}} {{ if .files }}
+ {{ if .hasMoreFiles }} +
+ {{ .locale.Tr "gist.files-truncated" }} +
+ {{ end }} {{ range $file := .files }}