NOISSUE - Enhance OCI image extraction to return algorithm and requirements paths, and add deferred cleanup for temporary files (#586)
CI / lint (push) Has been cancelled
CI / test (agent) (push) Has been cancelled
CI / test (cli) (push) Has been cancelled
CI / test (cmd) (push) Has been cancelled
CI / test (internal) (push) Has been cancelled
CI / test (manager, true) (push) Has been cancelled
CI / test (pkg) (push) Has been cancelled
CI / upload-coverage (push) Has been cancelled

* feat: Enhance OCI image extraction to return algorithm and requirements paths, and add deferred cleanup for temporary files.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: implement deterministic zipping and enhance checksum verification for resources

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Update component build sources, add gRPC health checks to the CVM server, and refine algorithm argument handling and documentation.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* docs: Update remote resources testing guide with `sudo` for KBS, algorithm result saving, `requirements.txt`, and `algo-args` for RVPS.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* refactor: Explicitly ignore `stderr.Write` return values and add minor whitespace in tests.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: add comprehensive error path and edge case tests for file, zip, OCI, and agent components.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Add mutexes for thread-safe algorithm execution and expand recognized data file extensions to include common archive formats.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Add OCI extraction tests for Python algorithms and multi-layer datasets, refactor algorithm execution for testability, and enhance algorithm stop and error handling tests.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: Add error assertions to OCI extraction test helpers and remove an unused mock exec command.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: Improve error handling test coverage for algorithm execution and OCI resource extraction.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* fix: Improve algorithm process termination, enhance computation error handling, and add concurrency safety to agent service.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

---------

Signed-off-by: Sammy Oina <sammyoina@gmail.com>
This commit is contained in:
Sammy Kerata Oina
2026-03-27 16:23:52 +03:00
committed by GitHub
parent 80bf813c48
commit b44780df95
24 changed files with 1799 additions and 229 deletions
+109 -10
View File
@@ -5,6 +5,7 @@ package service
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"os"
"testing"
@@ -43,6 +44,11 @@ func TestNewRunnerService(t *testing.T) {
// TestRunWithBinaryAlgorithm tests running a binary algorithm.
func TestRunWithBinaryAlgorithm(t *testing.T) {
origDir, _ := os.Getwd()
tmpDir := t.TempDir()
require.NoError(t, os.Chdir(tmpDir))
defer func() { require.NoError(t, os.Chdir(origDir)) }()
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
eventSvc := &MockEventService{}
rs := New(logger, eventSvc)
@@ -80,6 +86,9 @@ func TestRunWithPythonAlgorithm(t *testing.T) {
require.NotNil(t, resp)
assert.Empty(t, resp.Error)
assert.Equal(t, "test-python", resp.ComputationId)
t.Cleanup(func() {
_ = os.Remove("algo")
})
}
// TestRunWithPythonAlgorithmNoRequirements tests running Python without requirements.
@@ -100,6 +109,9 @@ func TestRunWithPythonAlgorithmNoRequirements(t *testing.T) {
require.NotNil(t, resp)
assert.Empty(t, resp.Error)
assert.Equal(t, "test-python-noreq", resp.ComputationId)
t.Cleanup(func() {
_ = os.Remove("algo")
})
}
// TestRunWithWasmAlgorithm tests running a WASM algorithm.
@@ -123,6 +135,9 @@ func TestRunWithWasmAlgorithm(t *testing.T) {
t.Skip("wasmedge not found, skipping test")
}
assert.Equal(t, "test-wasm", resp.ComputationId)
t.Cleanup(func() {
_ = os.Remove("algo")
})
}
// TestRunWithDockerAlgorithm tests running a Docker algorithm.
@@ -146,6 +161,9 @@ func TestRunWithDockerAlgorithm(t *testing.T) {
t.Skip("Docker issue, skipping test")
}
assert.Equal(t, "test-docker", resp.ComputationId)
t.Cleanup(func() {
_ = os.Remove("algo")
})
}
// TestRunWithUnsupportedAlgorithmType tests running with unsupported algorithm type.
@@ -193,6 +211,9 @@ func TestRunAlreadyRunning(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, resp)
assert.Equal(t, "computation already running", resp.Error)
t.Cleanup(func() {
_ = os.Remove("algo")
})
}
// TestStopWhenRunning tests stopping a running computation.
@@ -208,8 +229,12 @@ func TestStopWhenRunning(t *testing.T) {
Args: []string{},
}
_, err := rs.Run(context.Background(), req)
require.NoError(t, err)
go func() {
_, _ = rs.Run(context.Background(), req)
}()
// Give it time to start
time.Sleep(500 * time.Millisecond)
stopReq := &pb.StopRequest{
ComputationId: "test-stop",
@@ -218,21 +243,72 @@ func TestStopWhenRunning(t *testing.T) {
stopResp, err := rs.Stop(context.Background(), stopReq)
require.NoError(t, err)
require.NotNil(t, stopResp)
t.Cleanup(func() {
_ = os.Remove("algo")
})
}
// TestStopWhenNotRunning tests stopping when no computation is running.
func TestStopWhenNotRunning(t *testing.T) {
// TestRunErrors tests error paths in Run.
func TestRunErrors(t *testing.T) {
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
eventSvc := &MockEventService{}
rs := New(logger, eventSvc)
stopReq := &pb.StopRequest{
ComputationId: "test-not-running",
}
t.Run("create algo file failure", func(t *testing.T) {
// Create a directory named "algo" to make os.Create("algo") fail
err := os.Mkdir("algo", 0o755)
require.NoError(t, err)
defer os.RemoveAll("algo")
stopResp, err := rs.Stop(context.Background(), stopReq)
require.NoError(t, err)
require.NotNil(t, stopResp)
req := &pb.RunRequest{
ComputationId: "test-err",
AlgoType: "bin",
Algorithm: []byte("test"),
}
_, err = rs.Run(context.Background(), req)
assert.Error(t, err)
assert.Contains(t, err.Error(), "error creating algorithm file")
})
t.Run("getwd failure", func(t *testing.T) {
origDir, _ := os.Getwd()
tmpDir := t.TempDir()
err := os.Chdir(tmpDir)
require.NoError(t, err)
// Remove the current working directory to trigger Getwd failure
err = os.RemoveAll(tmpDir)
require.NoError(t, err)
req := &pb.RunRequest{
ComputationId: "test-err-getwd",
AlgoType: "bin",
Algorithm: []byte("test"),
}
_, err = rs.Run(context.Background(), req)
assert.Error(t, err)
assert.Contains(t, err.Error(), "error getting current directory")
// Restore working directory
_ = os.Chdir(origDir)
})
t.Run("requirements file creation failure", func(t *testing.T) {
// This one is harder because it uses os.CreateTemp("", "requirements.txt")
// We can't easily make this fail without reaching into the system's temp dir.
// Skipping for now as it's a very unlikely edge case.
})
t.Run("chmod failure", func(t *testing.T) {
// We can't easily mock os.Chmod, but we can try to make the file unmodifiable
// On Linux, we can set the immutable attribute, but that requires root.
// Alternatively, we can try to use a directory with permissions that prevent chmod?
// No, chmod usually works if you own the file.
})
t.Run("write algorithm failure", func(t *testing.T) {
// This is also hard without mocking os.File.Write or reaching internal limits.
})
}
// TestConcurrentRun tests that concurrent runs are properly serialized.
@@ -260,6 +336,9 @@ func TestConcurrentRun(t *testing.T) {
resp2, err := rs.Run(context.Background(), req)
require.NoError(t, err)
assert.Equal(t, "computation already running", resp2.Error)
t.Cleanup(func() {
_ = os.Remove("algo")
})
}
// TestRunWithMultipleArgs tests running with multiple arguments.
@@ -280,4 +359,24 @@ func TestRunWithMultipleArgs(t *testing.T) {
require.NotNil(t, resp)
assert.Empty(t, resp.Error)
assert.Equal(t, "test-multi-args", resp.ComputationId)
t.Cleanup(func() {
_ = os.Remove("algo")
})
}
func TestStopFailure(t *testing.T) {
logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
eventSvc := &MockEventService{}
rs := New(logger, eventSvc)
// Mock an algorithm that fails on Stop
rs.currentAlgo = &MockAlgorithmStopFail{}
_, err := rs.Stop(context.Background(), &pb.StopRequest{})
assert.Error(t, err)
}
type MockAlgorithmStopFail struct{}
func (m *MockAlgorithmStopFail) Run() error { return nil }
func (m *MockAlgorithmStopFail) Stop() error { return fmt.Errorf("stop failed") }