Limit display if there is too much files in one gist (#701)

Signed-off-by: Thomas Miceli <tho.miceli@gmail.com>

internal/db/gist.go

@@ -414,14 +414,14 @@ func (gist *Gist) DeleteRepository() error {
 	return git.DeleteRepository(gist.User.Username, gist.Uuid)
 }
 
-func (gist *Gist) Files(revision string, truncate bool) ([]*git.File, error) {
-	filesCat, err := git.CatFileBatch(gist.User.Username, gist.Uuid, revision, truncate)
+func (gist *Gist) Files(revision string, truncate bool) ([]*git.File, bool, error) {
+	filesCat, gistTruncated, err := git.CatFileBatch(gist.User.Username, gist.Uuid, revision, truncate)
 	if err != nil {
 		// if the revision or the file do not exist
 		if exiterr, ok := err.(*exec.ExitError); ok && exiterr.ExitCode() == 128 {
-			return nil, &git.RevisionNotFoundError{}
+			return nil, false, &git.RevisionNotFoundError{}
 		}
-		return nil, err
+		return nil, false, err
 	}
 
 	var files []*git.File
@@ -442,7 +442,7 @@ func (gist *Gist) Files(revision string, truncate bool) ([]*git.File, error) {
 			MimeType:  git.DetectMimeType([]byte(shortContent), filepath.Ext(fileCat.Name)),
 		})
 	}
-	return files, err
+	return files, gistTruncated, err
 }
 
 func (gist *Gist) File(revision string, filename string, truncate bool) (*git.File, error) {
@@ -613,7 +613,7 @@ func (gist *Gist) Identifier() string {
 }
 
 func (gist *Gist) GetLanguagesFromFiles() ([]string, error) {
-	files, err := gist.Files("HEAD", true)
+	files, _, err := gist.Files("HEAD", true)
 	if err != nil {
 		return nil, err
 	}
@@ -694,7 +694,7 @@ func (gist *Gist) UpdateLanguages() {
 }
 
 func (gist *Gist) ToDTO() (*GistDTO, error) {
-	files, err := gist.Files("HEAD", false)
+	files, _, err := gist.Files("HEAD", false)
 	if err != nil {
 		return nil, err
 	}
@@ -786,7 +786,7 @@ func (dto *GistDTO) TopicStrToSlice() []GistTopic {
 // -- Index -- //
 
 func (gist *Gist) ToIndexedGist() (*index.Gist, error) {
-	files, err := gist.Files("HEAD", true)
+	files, _, err := gist.Files("HEAD", true)
 	if err != nil {
 		return nil, err
 	}

internal/git/commands.go

@@ -132,21 +132,36 @@ type catFileBatch struct {
 	Truncated           bool
 }
 
-func CatFileBatch(user string, gist string, revision string, truncate bool) ([]*catFileBatch, error) {
+func CatFileBatch(user string, gist string, revision string, truncate bool) ([]*catFileBatch, bool, error) {
 	repositoryPath := RepositoryPath(user, gist)
+	maxFiles := 50
 
 	lsTreeCmd := exec.Command("git", "ls-tree", "-l", revision)
 	lsTreeCmd.Dir = repositoryPath
-	lsTreeOutput, err := lsTreeCmd.Output()
+
+	var lsTreeStderr bytes.Buffer
+	lsTreeCmd.Stderr = &lsTreeStderr
+
+	lsTreeStdout, err := lsTreeCmd.StdoutPipe()
 	if err != nil {
-		return nil, err
+		return nil, false, err
+	}
+	if err = lsTreeCmd.Start(); err != nil {
+		return nil, false, err
 	}
 
 	fileMap := make([]*catFileBatch, 0)
+	gistTruncated := false
 
-	lines := strings.Split(string(lsTreeOutput), "\n")
-	for _, line := range lines {
-		fields := strings.Fields(line)
+	scanner := bufio.NewScanner(lsTreeStdout)
+	scanner.Buffer(make([]byte, 64*1024), 1024*1024)
+	for scanner.Scan() {
+		if truncate && len(fileMap) >= maxFiles {
+			gistTruncated = true
+			break
+		}
+
+		fields := strings.Fields(scanner.Text())
 		if len(fields) < 4 {
 			continue // Skip lines that don't have enough fields
 		}
@@ -164,19 +179,33 @@ func CatFileBatch(user string, gist string, revision string, truncate bool) ([]*
 			Name: convertOctalToUTF8(name),
 		})
 	}
+	scanErr := scanner.Err()
+
+	// Closing the read end before git is done writing causes git's next write
+	// to fail (SIGPIPE on Unix, broken-pipe error on Windows). That shows up as
+	// a non-zero exit from Wait, but it's expected when we stop early — so we
+	// only treat the Wait error as real if git actually printed something to stderr.
+	_ = lsTreeStdout.Close()
+	waitErr := lsTreeCmd.Wait()
+	if scanErr != nil {
+		return nil, false, scanErr
+	}
+	if waitErr != nil && lsTreeStderr.Len() > 0 {
+		return nil, false, waitErr
+	}
 
 	catFileCmd := exec.Command("git", "cat-file", "--batch")
 	catFileCmd.Dir = repositoryPath
 	stdin, err := catFileCmd.StdinPipe()
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	stdout, err := catFileCmd.StdoutPipe()
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	if err = catFileCmd.Start(); err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	reader := bufio.NewReader(stdout)
@@ -184,12 +213,12 @@ func CatFileBatch(user string, gist string, revision string, truncate bool) ([]*
 	for _, file := range fileMap {
 		_, err = stdin.Write([]byte(file.Hash + "\n"))
 		if err != nil {
-			return nil, err
+			return nil, false, err
 		}
 
 		header, err := reader.ReadString('\n')
 		if err != nil {
-			return nil, err
+			return nil, false, err
 		}
 
 		parts := strings.Fields(header)
@@ -199,7 +228,7 @@ func CatFileBatch(user string, gist string, revision string, truncate bool) ([]*
 
 		size, err := strconv.ParseUint(parts[2], 10, 64)
 		if err != nil {
-			return nil, err
+			return nil, false, err
 		}
 
 		// Don't truncate Jupyter notebooks
@@ -215,7 +244,7 @@ func CatFileBatch(user string, gist string, revision string, truncate bool) ([]*
 		// Read exactly size bytes from header, or the max allowed if truncated
 		content := make([]byte, sizeToRead)
 		if _, err = io.ReadFull(reader, content); err != nil {
-			return nil, err
+			return nil, false, err
 		}
 
 		file.Content = string(content)
@@ -223,26 +252,26 @@ func CatFileBatch(user string, gist string, revision string, truncate bool) ([]*
 		if truncate && size > truncateLimit {
 			// skip other bytes if truncated
 			if _, err = reader.Discard(int(size - truncateLimit)); err != nil {
-				return nil, err
+				return nil, false, err
 			}
 			file.Truncated = true
 		}
 
 		// Read the blank line following the content
 		if _, err := reader.ReadByte(); err != nil {
-			return nil, err
+			return nil, false, err
 		}
 	}
 
 	if err = stdin.Close(); err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	if err = catFileCmd.Wait(); err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
-	return fileMap, nil
+	return fileMap, gistTruncated, nil
 }
 
 func GetFileContent(user string, gist string, revision string, filename string, truncate bool) (string, bool, error) {

internal/i18n/locales/en-US.yml

@@ -23,6 +23,7 @@ gist.header.download-zip: Download ZIP
 
 gist.raw: Raw
 gist.file-truncated: This file has been truncated.
+gist.files-truncated: Not all files in this gist are not displayed. Clone or download the gist to see them all.
 gist.file-raw: This file can't be rendered.
 gist.file-binary-edit: This file is binary.
 gist.watch-full-file: View the full file.

internal/web/handlers/gist/create.go

@@ -127,7 +127,7 @@ func ProcessCreate(ctx *context.Context) error {
 		if isCreate {
 			return ctx.HtmlWithCode(400, "create.html")
 		} else {
-			files, err := gist.Files("HEAD", false)
+			files, _, err := gist.Files("HEAD", false)
 			if err != nil {
 				return ctx.ErrorRes(500, "Error fetching files", err)
 			}

internal/web/handlers/gist/create_test.go

@@ -494,7 +494,7 @@ func TestGistCreation(t *testing.T) {
 
 				// Verify files if specified
 				if len(tt.expectedFileNames) > 0 {
-					files, err := gist.Files("HEAD", false)
+					files, _, err := gist.Files("HEAD", false)
 					require.NoError(t, err, "Failed to get gist files")
 					require.Len(t, files, len(tt.expectedFileNames), "File count mismatch")
 

internal/web/handlers/gist/download.go

@@ -67,7 +67,7 @@ func DownloadZip(ctx *context.Context) error {
 	gist := ctx.GetData("gist").(*db.Gist)
 	revision := ctx.Param("revision")
 
-	files, err := gist.Files(revision, false)
+	files, _, err := gist.Files(revision, false)
 	if err != nil {
 		return ctx.ErrorRes(500, "Error fetching files from repository", err)
 	}

internal/web/handlers/gist/fork_test.go

@@ -29,10 +29,10 @@ func TestFork(t *testing.T) {
 		require.Equal(t, gist.Private, forkedGist.Private)
 		require.Equal(t, gist.ID, forkedGist.ForkedID)
 
-		forkedFiles, err := forkedGist.Files("HEAD", false)
+		forkedFiles, _, err := forkedGist.Files("HEAD", false)
 		require.NoError(t, err)
 
-		gistFiles, err := gist.Files("HEAD", false)
+		gistFiles, _, err := gist.Files("HEAD", false)
 		require.NoError(t, err)
 
 		for i, file := range gistFiles {

internal/web/handlers/gist/gist.go

@@ -28,7 +28,7 @@ func GistIndex(ctx *context.Context) error {
 		revision = "HEAD"
 	}
 
-	files, err := gist.Files(revision, true)
+	files, hasMoreFiles, err := gist.Files(revision, true)
 	if _, ok := err.(*git.RevisionNotFoundError); ok {
 		return ctx.NotFound("Revision not found")
 	} else if err != nil {
@@ -40,6 +40,7 @@ func GistIndex(ctx *context.Context) error {
 	ctx.SetData("page", "code")
 	ctx.SetData("commit", revision)
 	ctx.SetData("files", renderedFiles)
+	ctx.SetData("hasMoreFiles", hasMoreFiles)
 	ctx.SetData("revision", revision)
 	ctx.SetData("htmlTitle", gist.Title)
 	return ctx.Html("gist.html")
@@ -47,13 +48,14 @@ func GistIndex(ctx *context.Context) error {
 
 func GistJson(ctx *context.Context) error {
 	gist := ctx.GetData("gist").(*db.Gist)
-	files, err := gist.Files("HEAD", true)
+	files, hasMoreFiles, err := gist.Files("HEAD", true)
 	if err != nil {
 		return ctx.ErrorRes(500, "Error fetching files", err)
 	}
 
 	renderedFiles := render.RenderFiles(files)
 	ctx.SetData("files", renderedFiles)
+	ctx.SetData("hasMoreFiles", hasMoreFiles)
 
 	topics, err := gist.GetTopics()
 	if err != nil {
@@ -104,13 +106,14 @@ func GistJs(ctx *context.Context) error {
 	}
 
 	gist := ctx.GetData("gist").(*db.Gist)
-	files, err := gist.Files("HEAD", true)
+	files, hasMoreFiles, err := gist.Files("HEAD", true)
 	if err != nil {
 		return ctx.ErrorRes(500, "Error fetching files", err)
 	}
 
 	renderedFiles := render.RenderFiles(files)
 	ctx.SetData("files", renderedFiles)
+	ctx.SetData("hasMoreFiles", hasMoreFiles)
 
 	htmlbuf := bytes.Buffer{}
 	w := bufio.NewWriter(&htmlbuf)

internal/web/handlers/gist/gist_test.go

@@ -81,7 +81,7 @@ func TestGistIndex(t *testing.T) {
 			"content": {"updated content"},
 		}, 302)
 
-		files, err := gist.Files("HEAD", false)
+		files, _, err := gist.Files("HEAD", false)
 		require.NoError(t, err)
 		found := false
 		for _, f := range files {
@@ -96,7 +96,7 @@ func TestGistIndex(t *testing.T) {
 		require.NoError(t, err)
 		require.Len(t, commits, 2)
 
-		filesOld, err := gist.Files(commits[1].Hash, false)
+		filesOld, _, err := gist.Files(commits[1].Hash, false)
 		require.NoError(t, err)
 		for _, f := range filesOld {
 			if f.Filename == "file.txt" {

templates/pages/gist.html

@@ -2,6 +2,11 @@
 {{ template "gist_header" .}}
     {{ if .files }}
         <div class="grid gap-y-4">
+        {{ if .hasMoreFiles }}
+        <div class="rounded-md border border-1 border-yellow-300 dark:border-yellow-700 bg-yellow-50 dark:bg-yellow-900/30 px-4 py-2 text-sm text-yellow-900 dark:text-yellow-200">
+            {{ .locale.Tr "gist.files-truncated" }}
+        </div>
+        {{ end }}
         {{ range $file := .files }}
         <div class="rounded-md border border-1 border-gray-200 dark:border-gray-700 overflow-auto" data-file="{{ $file.Filename }}">
             <div class="border-b-1 border-gray-200 dark:border-gray-700 bg-gray-50 dark:bg-gray-800 my-auto block">