Files
Sammy Kerata Oina b44780df95
CI / lint (push) Has been cancelled
CI / test (agent) (push) Has been cancelled
CI / test (cli) (push) Has been cancelled
CI / test (cmd) (push) Has been cancelled
CI / test (internal) (push) Has been cancelled
CI / test (manager, true) (push) Has been cancelled
CI / test (pkg) (push) Has been cancelled
CI / upload-coverage (push) Has been cancelled
NOISSUE - Enhance OCI image extraction to return algorithm and requirements paths, and add deferred cleanup for temporary files (#586)
* feat: Enhance OCI image extraction to return algorithm and requirements paths, and add deferred cleanup for temporary files.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: implement deterministic zipping and enhance checksum verification for resources

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Update component build sources, add gRPC health checks to the CVM server, and refine algorithm argument handling and documentation.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* docs: Update remote resources testing guide with `sudo` for KBS, algorithm result saving, `requirements.txt`, and `algo-args` for RVPS.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* refactor: Explicitly ignore `stderr.Write` return values and add minor whitespace in tests.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: add comprehensive error path and edge case tests for file, zip, OCI, and agent components.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Add mutexes for thread-safe algorithm execution and expand recognized data file extensions to include common archive formats.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Add OCI extraction tests for Python algorithms and multi-layer datasets, refactor algorithm execution for testability, and enhance algorithm stop and error handling tests.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: Add error assertions to OCI extraction test helpers and remove an unused mock exec command.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: Improve error handling test coverage for algorithm execution and OCI resource extraction.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* fix: Improve algorithm process termination, enhance computation error handling, and add concurrency safety to agent service.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

---------

Signed-off-by: Sammy Oina <sammyoina@gmail.com>
2026-03-27 14:23:52 +01:00

442 lines
11 KiB
Go

// Copyright (c) Ultraviolet
// SPDX-License-Identifier: Apache-2.0
package internal
import (
"archive/zip"
"bytes"
"io"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestZipDirectoryToMemory(t *testing.T) {
tempDir, err := os.MkdirTemp("", "zip_test")
if err != nil {
t.Fatalf("Failed to create temp directory: %v", err)
}
defer os.RemoveAll(tempDir)
testFiles := map[string]string{
"file1.txt": "Content of file 1",
"file2.txt": "Content of file 2",
"subdir/file3.txt": "Content of file 3 in subdirectory",
}
for path, content := range testFiles {
fullPath := filepath.Join(tempDir, path)
err := os.MkdirAll(filepath.Dir(fullPath), 0o755)
if err != nil {
t.Fatalf("Failed to create directory: %v", err)
}
err = os.WriteFile(fullPath, []byte(content), 0o644)
if err != nil {
t.Fatalf("Failed to write test file: %v", err)
}
}
zipData, err := ZipDirectoryToMemory(tempDir)
if err != nil {
t.Fatalf("ZipDirectoryToMemory failed: %v", err)
}
if len(zipData) == 0 {
t.Error("Zip data is empty")
}
unzipDir, err := os.MkdirTemp("", "unzip_test")
if err != nil {
t.Fatalf("Failed to create temp directory for unzip: %v", err)
}
defer os.RemoveAll(unzipDir)
err = UnzipFromMemory(zipData, unzipDir)
if err != nil {
t.Fatalf("UnzipFromMemory failed: %v", err)
}
for path, expectedContent := range testFiles {
fullPath := filepath.Join(unzipDir, path)
content, err := os.ReadFile(fullPath)
if err != nil {
t.Errorf("Failed to read unzipped file %s: %v", path, err)
continue
}
if string(content) != expectedContent {
t.Errorf("Content mismatch for file %s. Expected: %s, Got: %s", path, expectedContent, string(content))
}
}
}
func TestZipDirectoryToMemory_EmptyDirectory(t *testing.T) {
tempDir, err := os.MkdirTemp("", "empty_zip_test")
if err != nil {
t.Fatalf("Failed to create temp directory: %v", err)
}
defer os.RemoveAll(tempDir)
zipData, err := ZipDirectoryToMemory(tempDir)
if err != nil {
t.Fatalf("ZipDirectoryToMemory failed on empty directory: %v", err)
}
if len(zipData) == 0 {
t.Error("Zip data is empty for an empty directory")
}
}
func TestUnzipFromMemory_InvalidZipData(t *testing.T) {
invalidZipData := []byte("This is not a valid zip file")
tempDir, err := os.MkdirTemp("", "invalid_unzip_test")
if err != nil {
t.Fatalf("Failed to create temp directory: %v", err)
}
defer os.RemoveAll(tempDir)
err = UnzipFromMemory(invalidZipData, tempDir)
if err == nil {
t.Error("UnzipFromMemory should fail with invalid zip data")
}
}
func TestZipDirectoryToMemory_NonExistentDirectory(t *testing.T) {
nonExistentDir := "/path/to/non/existent/directory"
_, err := ZipDirectoryToMemory(nonExistentDir)
if err == nil {
t.Error("ZipDirectoryToMemory should fail with non-existent directory")
}
}
func TestZipDirectoryToTempFile(t *testing.T) {
tests := []struct {
name string
setupFiles map[string]string // map of relative path to content
expectError bool
}{
{
name: "single file",
setupFiles: map[string]string{
"test.txt": "hello world",
},
expectError: false,
},
{
name: "multiple files in root",
setupFiles: map[string]string{
"test1.txt": "content1",
"test2.txt": "content2",
"test3.txt": "content3",
},
expectError: false,
},
{
name: "nested directory structure",
setupFiles: map[string]string{
"file1.txt": "root file",
"dir1/file2.txt": "nested file",
"dir1/dir2/file3.txt": "deeply nested file",
"dir1/dir2/dir3/file4.txt": "very deeply nested file",
},
expectError: false,
},
{
name: "empty directory",
setupFiles: map[string]string{},
expectError: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sourceDir, err := os.MkdirTemp("", "source")
if err != nil {
t.Fatalf("Failed to create temp source directory: %v", err)
}
defer os.RemoveAll(sourceDir)
for relPath, content := range tt.setupFiles {
fullPath := filepath.Join(sourceDir, relPath)
dir := filepath.Dir(fullPath)
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatalf("Failed to create directory %s: %v", dir, err)
}
if err := os.WriteFile(fullPath, []byte(content), 0o644); err != nil {
t.Fatalf("Failed to write file %s: %v", fullPath, err)
}
}
zipFile, err := ZipDirectoryToTempFile(sourceDir)
if err != nil {
if !tt.expectError {
t.Fatalf("Unexpected error: %v", err)
}
return
}
defer os.Remove(zipFile.Name())
defer zipFile.Close()
if tt.expectError {
t.Fatal("Expected error but got none")
}
zipReader, err := zip.OpenReader(zipFile.Name())
if err != nil {
t.Fatalf("Failed to open zip file: %v", err)
}
defer zipReader.Close()
expectedFiles := make(map[string]string)
for path, content := range tt.setupFiles {
expectedFiles[filepath.ToSlash(path)] = content
}
for _, file := range zipReader.File {
expectedContent, exists := expectedFiles[file.Name]
if !exists {
t.Errorf("Unexpected file in zip: %s", file.Name)
continue
}
rc, err := file.Open()
if err != nil {
t.Errorf("Failed to open file in zip %s: %v", file.Name, err)
continue
}
content, err := io.ReadAll(rc)
rc.Close()
if err != nil {
t.Errorf("Failed to read file in zip %s: %v", file.Name, err)
continue
}
if string(content) != expectedContent {
t.Errorf("File %s content mismatch: got %s, want %s", file.Name, content, expectedContent)
}
delete(expectedFiles, file.Name)
}
for path := range expectedFiles {
t.Errorf("Missing file in zip: %s", path)
}
})
}
}
func TestUnzipFromMemory_ErrorPaths(t *testing.T) {
tempDir, err := os.MkdirTemp("", "unzip_error_test")
if err != nil {
t.Fatalf("Failed to create temp directory: %v", err)
}
defer os.RemoveAll(tempDir)
// Create a valid zip with one file
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
f, err := zw.Create("file.txt")
require.NoError(t, err)
_, err = f.Write([]byte("content"))
require.NoError(t, err)
require.NoError(t, zw.Close())
t.Run("mkdir failure", func(t *testing.T) {
// Create a file where a directory should be
blockedDir := filepath.Join(tempDir, "subdir")
require.NoError(t, os.WriteFile(blockedDir, []byte("blocked"), 0o644))
defer os.Remove(blockedDir)
// Create a zip that tries to create a file in that subdir
var buf2 bytes.Buffer
zw2 := zip.NewWriter(&buf2)
_, err := zw2.Create("subdir/file.txt")
require.NoError(t, err)
require.NoError(t, zw2.Close())
err = UnzipFromMemory(buf2.Bytes(), tempDir)
assert.Error(t, err)
})
t.Run("create file failure", func(t *testing.T) {
// Create a directory where a file should be
blockedFile := filepath.Join(tempDir, "blocked_file.txt")
require.NoError(t, os.MkdirAll(blockedFile, 0o755))
defer os.RemoveAll(blockedFile)
var buf2 bytes.Buffer
zw2 := zip.NewWriter(&buf2)
_, err := zw2.Create("blocked_file.txt")
require.NoError(t, err)
require.NoError(t, zw2.Close())
err = UnzipFromMemory(buf2.Bytes(), tempDir)
assert.Error(t, err)
})
}
func TestZipDirectoryToMemory_ErrorPaths(t *testing.T) {
t.Run("non-existent directory", func(t *testing.T) {
_, err := ZipDirectoryToMemory("/non/existent/path")
assert.Error(t, err)
})
}
func TestZipDirectoryToTempFile_ErrorPaths(t *testing.T) {
t.Run("non-existent directory", func(t *testing.T) {
_, err := ZipDirectoryToTempFile("/non/existent/path")
assert.Error(t, err)
})
t.Run("empty source path", func(t *testing.T) {
_, err := ZipDirectoryToTempFile("")
assert.Error(t, err)
})
}
func TestZipDirectoryToMemory_NotADirectory(t *testing.T) {
tempFile, err := os.CreateTemp("", "not_a_dir")
require.NoError(t, err)
defer os.Remove(tempFile.Name())
tempFile.Close()
_, err = ZipDirectoryToMemory(tempFile.Name())
// filepath.Walk on a file succeeds and visits only that file
assert.NoError(t, err)
}
func TestZipDirectoryToTempFile_NotADirectory(t *testing.T) {
tempFile, err := os.CreateTemp("", "not_a_dir_tempfile")
require.NoError(t, err)
defer os.Remove(tempFile.Name())
tempFile.Close()
zf, err := ZipDirectoryToTempFile(tempFile.Name())
assert.NoError(t, err)
if err == nil {
zf.Close()
os.Remove(zf.Name())
}
}
func TestZipDirectoryToMemory_OpenError(t *testing.T) {
tempDir, err := os.MkdirTemp("", "open_err_test")
require.NoError(t, err)
defer os.RemoveAll(tempDir)
file := filepath.Join(tempDir, "unreadable.txt")
require.NoError(t, os.WriteFile(file, []byte("test"), 0o000))
_, err = ZipDirectoryToMemory(tempDir)
assert.Error(t, err)
}
func TestZipDirectoryToTempFile_InvalidInput(t *testing.T) {
tests := []struct {
name string
sourceDir string
}{
{
name: "non-existent directory",
sourceDir: "/path/that/does/not/exist",
},
{
name: "empty path",
sourceDir: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := ZipDirectoryToTempFile(tt.sourceDir)
if err == nil {
t.Error("Expected error but got none")
}
})
}
}
func TestZipDirectoryToTempFile_InternalErrorPaths(t *testing.T) {
t.Run("unreadable file in directory", func(t *testing.T) {
tempDir, err := os.MkdirTemp("", "unreadable_zip_temp")
require.NoError(t, err)
defer os.RemoveAll(tempDir)
file := filepath.Join(tempDir, "unreadable.txt")
require.NoError(t, os.WriteFile(file, []byte("test"), 0o000))
_, err = ZipDirectoryToTempFile(tempDir)
assert.Error(t, err)
})
}
func TestUnzipFromMemory_MoreEdgeCases(t *testing.T) {
t.Run("empty zip data", func(t *testing.T) {
err := UnzipFromMemory([]byte{}, t.TempDir())
assert.Error(t, err)
})
t.Run("zip with absolute paths", func(t *testing.T) {
// This tests if UnzipFromMemory handles files with names like "/tmp/hacker.txt"
// zip.NewReader usually doesn't allow absolute paths easily, but let's see.
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
_, err := zw.Create("/tmp/test.txt")
require.NoError(t, err)
require.NoError(t, zw.Close())
tempDir := t.TempDir()
err = UnzipFromMemory(buf.Bytes(), tempDir)
assert.NoError(t, err)
// It should be joined with tempDir, not written to /tmp/test.txt
expectedPath := filepath.Join(tempDir, "/tmp/test.txt")
_, err = os.Stat(expectedPath)
assert.NoError(t, err)
})
}
func TestUnzipFromMemory_InvalidReader(t *testing.T) {
err := UnzipFromMemory([]byte("invalid"), t.TempDir())
assert.Error(t, err)
}
func TestUnzipFromMemory_FileCreateError(t *testing.T) {
tempDir := t.TempDir()
// Create a zip with one file
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
_, err := zw.Create("file.txt")
require.NoError(t, err)
require.NoError(t, zw.Close())
// Create a directory where the file should be
blockedFile := filepath.Join(tempDir, "file.txt")
require.NoError(t, os.MkdirAll(blockedFile, 0o755))
err = UnzipFromMemory(buf.Bytes(), tempDir)
assert.Error(t, err)
}
func TestUnzipFromMemory_DirCreateError(t *testing.T) {
tempDir := t.TempDir()
// Create a zip with one directory entry
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
_, err := zw.Create("subdir/")
require.NoError(t, err)
require.NoError(t, zw.Close())
// Create a file where the directory should be
blockedDir := filepath.Join(tempDir, "subdir")
require.NoError(t, os.WriteFile(blockedDir, []byte("blocked"), 0o644))
err = UnzipFromMemory(buf.Bytes(), tempDir)
assert.Error(t, err)
}