mirror of
https://github.com/ultravioletrs/cocos.git
synced 2026-08-07 23:31:55 +00:00
NOISSUE - Enable WASM Support and FileSystem Support (#189)
* feat(algorithm): Add wasm as an algo type Signed-off-by: Rodney Osodo <socials@rodneyosodo.com> * feat(algorithm): Use filesystem to store results Move from unix socket for results storage to filesystem * test: test new filesystem changes Signed-off-by: Rodney Osodo <socials@rodneyosodo.com> * refactor(files): rename resultFile to resultsFilePath * feat(wasm-runtime): change from wasmtime to wasmedge Wasmedge enables easier directory mapping to get results Signed-off-by: Rodney Osodo <socials@rodneyosodo.com> * feat(algorithm): send results as zipped directory Create a new function to zip the results directory and send it back to the user * fix(wasm): runtime argument Fix the directory mapping for wasm runtime arguments Signed-off-by: Rodney Osodo <socials@rodneyosodo.com> * fix(errors): provide useful error message * chore(gitignore): add results zip to gitignore * feat(filesystem): Enable storing results on filesystem for python algos * refactor: revert to upstream cocos repo Signed-off-by: Rodney Osodo <socials@rodneyosodo.com> * fix: remove AddDataset from algorithm interface * fix: agent to handle results zipping * test: test zipping directories * refactor(agent): Handle file operations from agent * test: run test inside eos Signed-off-by: Rodney Osodo <socials@rodneyosodo.com> * refactor(test): Document and test algos are running Document steps on running the 2 python exampls and ensure they are running on eos Signed-off-by: Rodney Osodo <socials@rodneyosodo.com> * fix: remove witheDataset option * test: test without dataset argument Signed-off-by: Rodney Osodo <socials@rodneyosodo.com> --------- Signed-off-by: Rodney Osodo <socials@rodneyosodo.com>
This commit is contained in:
@@ -13,12 +13,10 @@ import (
|
||||
|
||||
"github.com/ultravioletrs/cocos/agent/algorithm"
|
||||
"github.com/ultravioletrs/cocos/agent/events"
|
||||
"github.com/ultravioletrs/cocos/pkg/socket"
|
||||
"google.golang.org/grpc/metadata"
|
||||
)
|
||||
|
||||
const (
|
||||
socketPath = "unix_socket"
|
||||
PyRuntime = "python3"
|
||||
pyRuntimeKey = "python_runtime"
|
||||
)
|
||||
@@ -35,18 +33,15 @@ var _ algorithm.Algorithm = (*python)(nil)
|
||||
|
||||
type python struct {
|
||||
algoFile string
|
||||
datasets []string
|
||||
logger *slog.Logger
|
||||
stderr io.Writer
|
||||
stdout io.Writer
|
||||
runtime string
|
||||
requirementsFile string
|
||||
}
|
||||
|
||||
func New(logger *slog.Logger, eventsSvc events.Service, runtime, requirementsFile, algoFile string) algorithm.Algorithm {
|
||||
func NewAlgorithm(logger *slog.Logger, eventsSvc events.Service, runtime, requirementsFile, algoFile string) algorithm.Algorithm {
|
||||
p := &python{
|
||||
algoFile: algoFile,
|
||||
logger: logger,
|
||||
stderr: &algorithm.Stderr{Logger: logger, EventSvc: eventsSvc},
|
||||
stdout: &algorithm.Stdout{Logger: logger},
|
||||
requirementsFile: requirementsFile,
|
||||
@@ -59,17 +54,13 @@ func New(logger *slog.Logger, eventsSvc events.Service, runtime, requirementsFil
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *python) AddDataset(dataset string) {
|
||||
p.datasets = append(p.datasets, dataset)
|
||||
}
|
||||
|
||||
func (p *python) Run() ([]byte, error) {
|
||||
func (p *python) Run() error {
|
||||
venvPath := "venv"
|
||||
createVenvCmd := exec.Command(p.runtime, "-m", "venv", venvPath)
|
||||
createVenvCmd.Stderr = p.stderr
|
||||
createVenvCmd.Stdout = p.stdout
|
||||
if err := createVenvCmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("error creating virtual environment: %v", err)
|
||||
return fmt.Errorf("error creating virtual environment: %v", err)
|
||||
}
|
||||
|
||||
pythonPath := filepath.Join(venvPath, "bin", "python")
|
||||
@@ -79,48 +70,25 @@ func (p *python) Run() ([]byte, error) {
|
||||
rcmd.Stderr = p.stderr
|
||||
rcmd.Stdout = p.stdout
|
||||
if err := rcmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("error installing requirements: %v", err)
|
||||
return fmt.Errorf("error installing requirements: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
defer os.Remove(p.algoFile)
|
||||
defer func() {
|
||||
for _, file := range p.datasets {
|
||||
os.Remove(file)
|
||||
}
|
||||
}()
|
||||
defer os.RemoveAll(venvPath)
|
||||
|
||||
listener, err := socket.StartUnixSocketServer(socketPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating stdout pipe: %v", err)
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
dataChannel := make(chan []byte)
|
||||
errorChannel := make(chan error)
|
||||
|
||||
var result []byte
|
||||
|
||||
go socket.AcceptConnection(listener, dataChannel, errorChannel)
|
||||
|
||||
args := append([]string{p.algoFile, socketPath}, p.datasets...)
|
||||
cmd := exec.Command(pythonPath, args...)
|
||||
cmd := exec.Command(pythonPath, p.algoFile)
|
||||
cmd.Stderr = p.stderr
|
||||
cmd.Stdout = p.stdout
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("error starting algorithm: %v", err)
|
||||
return fmt.Errorf("error starting algorithm: %v", err)
|
||||
}
|
||||
|
||||
if err := cmd.Wait(); err != nil {
|
||||
return nil, fmt.Errorf("algorithm execution error: %v", err)
|
||||
return fmt.Errorf("algorithm execution error: %v", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case result = <-dataChannel:
|
||||
return result, nil
|
||||
case err = <-errorChannel:
|
||||
return nil, fmt.Errorf("error receiving data: %v", err)
|
||||
if err := os.RemoveAll(venvPath); err != nil {
|
||||
return fmt.Errorf("error removing virtual environment: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user