Files
cocos/agent/algorithm/python/python.go
T
Sammy Kerata Oina b44780df95
CI / lint (push) Has been cancelled
CI / test (agent) (push) Has been cancelled
CI / test (cli) (push) Has been cancelled
CI / test (cmd) (push) Has been cancelled
CI / test (internal) (push) Has been cancelled
CI / test (manager, true) (push) Has been cancelled
CI / test (pkg) (push) Has been cancelled
CI / upload-coverage (push) Has been cancelled
NOISSUE - Enhance OCI image extraction to return algorithm and requirements paths, and add deferred cleanup for temporary files (#586)
* feat: Enhance OCI image extraction to return algorithm and requirements paths, and add deferred cleanup for temporary files.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: implement deterministic zipping and enhance checksum verification for resources

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Update component build sources, add gRPC health checks to the CVM server, and refine algorithm argument handling and documentation.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* docs: Update remote resources testing guide with `sudo` for KBS, algorithm result saving, `requirements.txt`, and `algo-args` for RVPS.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* refactor: Explicitly ignore `stderr.Write` return values and add minor whitespace in tests.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: add comprehensive error path and edge case tests for file, zip, OCI, and agent components.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Add mutexes for thread-safe algorithm execution and expand recognized data file extensions to include common archive formats.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Add OCI extraction tests for Python algorithms and multi-layer datasets, refactor algorithm execution for testability, and enhance algorithm stop and error handling tests.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: Add error assertions to OCI extraction test helpers and remove an unused mock exec command.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: Improve error handling test coverage for algorithm execution and OCI resource extraction.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* fix: Improve algorithm process termination, enhance computation error handling, and add concurrency safety to agent service.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

---------

Signed-off-by: Sammy Oina <sammyoina@gmail.com>
2026-03-27 14:23:52 +01:00

134 lines
3.2 KiB
Go

// Copyright (c) Ultraviolet
// SPDX-License-Identifier: Apache-2.0
package python
import (
"context"
"errors"
"fmt"
"io"
"log/slog"
"os"
"os/exec"
"path/filepath"
"sync"
"github.com/ultravioletrs/cocos/agent/algorithm"
"github.com/ultravioletrs/cocos/agent/algorithm/logging"
"github.com/ultravioletrs/cocos/agent/events"
"google.golang.org/grpc/metadata"
)
const (
PyRuntime = "python3"
PyRuntimeKey = "python_runtime"
)
func PythonRunTimeToContext(ctx context.Context, runtime string) context.Context {
return metadata.AppendToOutgoingContext(ctx, PyRuntimeKey, runtime)
}
func PythonRunTimeFromContext(ctx context.Context) string {
return metadata.ValueFromIncomingContext(ctx, PyRuntimeKey)[0]
}
var _ algorithm.Algorithm = (*python)(nil)
type python struct {
algoFile string
stderr io.Writer
stdout io.Writer
runtime string
requirementsFile string
args []string
cmd *exec.Cmd
mu sync.Mutex
}
func NewAlgorithm(logger *slog.Logger, eventsSvc events.Service, runtime, requirementsFile, algoFile string, args []string, cmpID string) algorithm.Algorithm {
p := &python{
algoFile: algoFile,
stderr: &logging.Stderr{Logger: logger, EventSvc: eventsSvc, CmpID: cmpID},
stdout: &logging.Stdout{Logger: logger},
requirementsFile: requirementsFile,
args: args,
}
if runtime != "" {
p.runtime = runtime
} else {
p.runtime = PyRuntime
}
return p
}
func (p *python) Run() error {
venvPath := "venv"
defer func() {
if err := os.RemoveAll(venvPath); err != nil {
_, _ = p.stderr.Write([]byte(fmt.Sprintf("error removing virtual environment: %v\n", err)))
}
}()
createVenvCmd := exec.Command(p.runtime, "-m", "venv", venvPath)
createVenvCmd.Stderr = p.stderr
createVenvCmd.Stdout = p.stdout
if err := createVenvCmd.Run(); err != nil {
return fmt.Errorf("error creating virtual environment: %v", err)
}
pythonPath := filepath.Join(venvPath, "bin", "python")
updatePipCmd := exec.Command(pythonPath, "-m", "pip", "install", "--upgrade", "pip", "setuptools", "wheel")
updatePipCmd.Stderr = p.stderr
updatePipCmd.Stdout = p.stdout
if err := updatePipCmd.Run(); err != nil {
return fmt.Errorf("error updating pip, setuptools and wheel: %v", err)
}
if p.requirementsFile != "" {
rcmd := exec.Command(pythonPath, "-m", "pip", "install", "-r", p.requirementsFile)
rcmd.Stderr = p.stderr
rcmd.Stdout = p.stdout
if err := rcmd.Run(); err != nil {
return fmt.Errorf("error installing requirements: %v", err)
}
}
args := append([]string{p.algoFile}, p.args...)
p.mu.Lock()
p.cmd = exec.Command(pythonPath, args...)
p.cmd.Stderr = p.stderr
p.cmd.Stdout = p.stdout
if err := p.cmd.Start(); err != nil {
p.mu.Unlock()
return fmt.Errorf("error starting algorithm: %v", err)
}
p.mu.Unlock()
if err := p.cmd.Wait(); err != nil {
return fmt.Errorf("algorithm execution error: %v", err)
}
return nil
}
func (p *python) Stop() error {
p.mu.Lock()
defer p.mu.Unlock()
if p.cmd == nil {
return nil
}
if p.cmd.Process == nil {
return nil
}
if err := p.cmd.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) {
return fmt.Errorf("error stopping algorithm: %v", err)
}
return nil
}