mirror of
https://github.com/ultravioletrs/cocos.git
synced 2026-06-22 20:00:18 +00:00
b44780df95
CI / lint (push) Has been cancelled
CI / test (agent) (push) Has been cancelled
CI / test (cli) (push) Has been cancelled
CI / test (cmd) (push) Has been cancelled
CI / test (internal) (push) Has been cancelled
CI / test (manager, true) (push) Has been cancelled
CI / test (pkg) (push) Has been cancelled
CI / upload-coverage (push) Has been cancelled
* feat: Enhance OCI image extraction to return algorithm and requirements paths, and add deferred cleanup for temporary files. Signed-off-by: Sammy Oina <sammyoina@gmail.com> * feat: implement deterministic zipping and enhance checksum verification for resources Signed-off-by: Sammy Oina <sammyoina@gmail.com> * feat: Update component build sources, add gRPC health checks to the CVM server, and refine algorithm argument handling and documentation. Signed-off-by: Sammy Oina <sammyoina@gmail.com> * docs: Update remote resources testing guide with `sudo` for KBS, algorithm result saving, `requirements.txt`, and `algo-args` for RVPS. Signed-off-by: Sammy Oina <sammyoina@gmail.com> * refactor: Explicitly ignore `stderr.Write` return values and add minor whitespace in tests. Signed-off-by: Sammy Oina <sammyoina@gmail.com> * test: add comprehensive error path and edge case tests for file, zip, OCI, and agent components. Signed-off-by: Sammy Oina <sammyoina@gmail.com> * feat: Add mutexes for thread-safe algorithm execution and expand recognized data file extensions to include common archive formats. Signed-off-by: Sammy Oina <sammyoina@gmail.com> * feat: Add OCI extraction tests for Python algorithms and multi-layer datasets, refactor algorithm execution for testability, and enhance algorithm stop and error handling tests. Signed-off-by: Sammy Oina <sammyoina@gmail.com> * test: Add error assertions to OCI extraction test helpers and remove an unused mock exec command. Signed-off-by: Sammy Oina <sammyoina@gmail.com> * test: Improve error handling test coverage for algorithm execution and OCI resource extraction. Signed-off-by: Sammy Oina <sammyoina@gmail.com> * fix: Improve algorithm process termination, enhance computation error handling, and add concurrency safety to agent service. Signed-off-by: Sammy Oina <sammyoina@gmail.com> --------- Signed-off-by: Sammy Oina <sammyoina@gmail.com>
194 lines
3.5 KiB
Go
194 lines
3.5 KiB
Go
// Copyright (c) Ultraviolet
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
package internal
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"time"
|
|
)
|
|
|
|
func ZipDirectoryToMemory(sourceDir string) ([]byte, error) {
|
|
buf := new(bytes.Buffer)
|
|
zipWriter := zip.NewWriter(buf)
|
|
|
|
var files []string
|
|
err := filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !info.IsDir() {
|
|
files = append(files, path)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
sort.Strings(files)
|
|
|
|
for _, path := range files {
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
relPath, err := filepath.Rel(sourceDir, path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
zipHeader, err := zip.FileInfoHeader(info)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
zipHeader.Name = relPath
|
|
zipHeader.Modified = time.Unix(0, 0) // Deterministic timestamp
|
|
|
|
zipWriterEntry, err := zipWriter.CreateHeader(zipHeader)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
fileToZip, err := os.Open(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer fileToZip.Close()
|
|
|
|
_, err = io.Copy(zipWriterEntry, fileToZip)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if err := zipWriter.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return buf.Bytes(), nil
|
|
}
|
|
|
|
func ZipDirectoryToTempFile(sourceDir string) (*os.File, error) {
|
|
tmpFile, err := os.CreateTemp("", "dataset*.zip")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
zipWriter := zip.NewWriter(tmpFile)
|
|
|
|
var files []string
|
|
err = filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !info.IsDir() {
|
|
files = append(files, path)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
zipWriter.Close()
|
|
return nil, err
|
|
}
|
|
|
|
sort.Strings(files)
|
|
|
|
for _, path := range files {
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
zipWriter.Close()
|
|
return nil, err
|
|
}
|
|
|
|
relPath, err := filepath.Rel(sourceDir, path)
|
|
if err != nil {
|
|
zipWriter.Close()
|
|
return nil, err
|
|
}
|
|
|
|
zipHeader, err := zip.FileInfoHeader(info)
|
|
if err != nil {
|
|
zipWriter.Close()
|
|
return nil, err
|
|
}
|
|
zipHeader.Name = relPath
|
|
zipHeader.Modified = time.Unix(0, 0) // Deterministic timestamp
|
|
|
|
zipWriterEntry, err := zipWriter.CreateHeader(zipHeader)
|
|
if err != nil {
|
|
zipWriter.Close()
|
|
return nil, err
|
|
}
|
|
|
|
fileToZip, err := os.Open(path)
|
|
if err != nil {
|
|
zipWriter.Close()
|
|
return nil, err
|
|
}
|
|
defer fileToZip.Close()
|
|
|
|
_, err = io.Copy(zipWriterEntry, fileToZip)
|
|
if err != nil {
|
|
zipWriter.Close()
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if err := zipWriter.Close(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if _, err := tmpFile.Seek(0, 0); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return tmpFile, nil
|
|
}
|
|
|
|
func UnzipFromMemory(zipData []byte, targetDir string) error {
|
|
reader := bytes.NewReader(zipData)
|
|
zipReader, err := zip.NewReader(reader, int64(len(zipData)))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, file := range zipReader.File {
|
|
filePath := filepath.Join(targetDir, file.Name)
|
|
|
|
if file.FileInfo().IsDir() {
|
|
if err := os.MkdirAll(filePath, os.ModePerm); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
|
|
if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
|
|
return err
|
|
}
|
|
|
|
srcFile, err := file.Open()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer srcFile.Close()
|
|
|
|
dstFile, err := os.Create(filePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer dstFile.Close()
|
|
|
|
if _, err := io.Copy(dstFile, srcFile); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|