Files
cocos/internal/zip.go
T
Sammy Kerata Oina b44780df95
CI / lint (push) Has been cancelled
CI / test (agent) (push) Has been cancelled
CI / test (cli) (push) Has been cancelled
CI / test (cmd) (push) Has been cancelled
CI / test (internal) (push) Has been cancelled
CI / test (manager, true) (push) Has been cancelled
CI / test (pkg) (push) Has been cancelled
CI / upload-coverage (push) Has been cancelled
NOISSUE - Enhance OCI image extraction to return algorithm and requirements paths, and add deferred cleanup for temporary files (#586)
* feat: Enhance OCI image extraction to return algorithm and requirements paths, and add deferred cleanup for temporary files.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: implement deterministic zipping and enhance checksum verification for resources

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Update component build sources, add gRPC health checks to the CVM server, and refine algorithm argument handling and documentation.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* docs: Update remote resources testing guide with `sudo` for KBS, algorithm result saving, `requirements.txt`, and `algo-args` for RVPS.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* refactor: Explicitly ignore `stderr.Write` return values and add minor whitespace in tests.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: add comprehensive error path and edge case tests for file, zip, OCI, and agent components.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Add mutexes for thread-safe algorithm execution and expand recognized data file extensions to include common archive formats.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Add OCI extraction tests for Python algorithms and multi-layer datasets, refactor algorithm execution for testability, and enhance algorithm stop and error handling tests.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: Add error assertions to OCI extraction test helpers and remove an unused mock exec command.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: Improve error handling test coverage for algorithm execution and OCI resource extraction.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* fix: Improve algorithm process termination, enhance computation error handling, and add concurrency safety to agent service.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

---------

Signed-off-by: Sammy Oina <sammyoina@gmail.com>
2026-03-27 14:23:52 +01:00

194 lines
3.5 KiB
Go

// Copyright (c) Ultraviolet
// SPDX-License-Identifier: Apache-2.0
package internal
import (
"archive/zip"
"bytes"
"io"
"os"
"path/filepath"
"sort"
"time"
)
func ZipDirectoryToMemory(sourceDir string) ([]byte, error) {
buf := new(bytes.Buffer)
zipWriter := zip.NewWriter(buf)
var files []string
err := filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
files = append(files, path)
}
return nil
})
if err != nil {
return nil, err
}
sort.Strings(files)
for _, path := range files {
info, err := os.Stat(path)
if err != nil {
return nil, err
}
relPath, err := filepath.Rel(sourceDir, path)
if err != nil {
return nil, err
}
zipHeader, err := zip.FileInfoHeader(info)
if err != nil {
return nil, err
}
zipHeader.Name = relPath
zipHeader.Modified = time.Unix(0, 0) // Deterministic timestamp
zipWriterEntry, err := zipWriter.CreateHeader(zipHeader)
if err != nil {
return nil, err
}
fileToZip, err := os.Open(path)
if err != nil {
return nil, err
}
defer fileToZip.Close()
_, err = io.Copy(zipWriterEntry, fileToZip)
if err != nil {
return nil, err
}
}
if err := zipWriter.Close(); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func ZipDirectoryToTempFile(sourceDir string) (*os.File, error) {
tmpFile, err := os.CreateTemp("", "dataset*.zip")
if err != nil {
return nil, err
}
zipWriter := zip.NewWriter(tmpFile)
var files []string
err = filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
files = append(files, path)
}
return nil
})
if err != nil {
zipWriter.Close()
return nil, err
}
sort.Strings(files)
for _, path := range files {
info, err := os.Stat(path)
if err != nil {
zipWriter.Close()
return nil, err
}
relPath, err := filepath.Rel(sourceDir, path)
if err != nil {
zipWriter.Close()
return nil, err
}
zipHeader, err := zip.FileInfoHeader(info)
if err != nil {
zipWriter.Close()
return nil, err
}
zipHeader.Name = relPath
zipHeader.Modified = time.Unix(0, 0) // Deterministic timestamp
zipWriterEntry, err := zipWriter.CreateHeader(zipHeader)
if err != nil {
zipWriter.Close()
return nil, err
}
fileToZip, err := os.Open(path)
if err != nil {
zipWriter.Close()
return nil, err
}
defer fileToZip.Close()
_, err = io.Copy(zipWriterEntry, fileToZip)
if err != nil {
zipWriter.Close()
return nil, err
}
}
if err := zipWriter.Close(); err != nil {
return nil, err
}
if _, err := tmpFile.Seek(0, 0); err != nil {
return nil, err
}
return tmpFile, nil
}
func UnzipFromMemory(zipData []byte, targetDir string) error {
reader := bytes.NewReader(zipData)
zipReader, err := zip.NewReader(reader, int64(len(zipData)))
if err != nil {
return err
}
for _, file := range zipReader.File {
filePath := filepath.Join(targetDir, file.Name)
if file.FileInfo().IsDir() {
if err := os.MkdirAll(filePath, os.ModePerm); err != nil {
return err
}
continue
}
if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
return err
}
srcFile, err := file.Open()
if err != nil {
return err
}
defer srcFile.Close()
dstFile, err := os.Create(filePath)
if err != nil {
return err
}
defer dstFile.Close()
if _, err := io.Copy(dstFile, srcFile); err != nil {
return err
}
}
return nil
}