mirror of
https://github.com/ultravioletrs/cocos.git
synced 2026-06-23 04:10:25 +00:00
COCOS-165 - Add Docker support (#180)
* add docker support * add copyright clause * rebase docker support * address blank lines * update manual tests to include docker * fix algo test * fix docker command * add docker doc * fix AddDataset method * fixed lin_reg.py * rebsed docker implementation * fix NewAlgorithm error * change docker README.md based on rebase * fix docker README * fix docker.go gofumpt * add option for datasets and results mount * edit README for docker * make docker container run command a part of docker image * remove unused code * make /cocos the default directory * updated documentation * removed docker dir * rebased docker
This commit is contained in:
Danko Miladinovic
committed by
GitHub
GitHub
parent
f906593492
commit
ee8370406c
@@ -14,10 +14,12 @@ const (
|
||||
AlgoTypeBin AlgorithType = "bin"
|
||||
AlgoTypePython AlgorithType = "python"
|
||||
AlgoTypeWasm AlgorithType = "wasm"
|
||||
AlgoTypeDocker AlgorithType = "docker"
|
||||
AlgoTypeKey = "algo_type"
|
||||
|
||||
ResultsDir = "results"
|
||||
DatasetsDir = "datasets"
|
||||
ResultsDir = "results"
|
||||
DatasetsDir = "datasets"
|
||||
AlgoWorkingDir = "/cocos"
|
||||
)
|
||||
|
||||
func AlgorithmTypeToContext(ctx context.Context, algoType string) context.Context {
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
// Copyright (c) Ultraviolet
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package docker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/docker/docker/api/types/container"
|
||||
"github.com/docker/docker/api/types/image"
|
||||
"github.com/docker/docker/api/types/mount"
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/ultravioletrs/cocos/agent/algorithm"
|
||||
"github.com/ultravioletrs/cocos/agent/events"
|
||||
)
|
||||
|
||||
const (
|
||||
containerName = "agent_container"
|
||||
datasetsMountPath = "/cocos/datasets"
|
||||
resultsMountPath = "/cocos/results"
|
||||
)
|
||||
|
||||
var _ algorithm.Algorithm = (*docker)(nil)
|
||||
|
||||
type docker struct {
|
||||
algoFile string
|
||||
logger *slog.Logger
|
||||
stderr io.Writer
|
||||
stdout io.Writer
|
||||
}
|
||||
|
||||
func NewAlgorithm(logger *slog.Logger, eventsSvc events.Service, algoFile string) algorithm.Algorithm {
|
||||
d := &docker{
|
||||
algoFile: algoFile,
|
||||
logger: logger,
|
||||
stderr: &algorithm.Stderr{Logger: logger, EventSvc: eventsSvc},
|
||||
stdout: &algorithm.Stdout{Logger: logger},
|
||||
}
|
||||
|
||||
return d
|
||||
}
|
||||
|
||||
func (d *docker) Run() error {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create a new Docker client.
|
||||
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create a new Docker client: %v", err)
|
||||
}
|
||||
|
||||
// Open the Docker image tar file.
|
||||
imageFile, err := os.Open(d.algoFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not open Docker image: %v", err)
|
||||
}
|
||||
defer imageFile.Close()
|
||||
|
||||
// Load the Docker image from the tar file.
|
||||
resp, err := cli.ImageLoad(ctx, imageFile, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not load Docker image from file: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// List the loaded images to get the image ID.
|
||||
images, err := cli.ImageList(ctx, image.ListOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get the Docker image list: %v", err)
|
||||
}
|
||||
|
||||
var imageID string = ""
|
||||
var dockerImageName string = ""
|
||||
for _, image := range images {
|
||||
for _, tag := range image.RepoTags {
|
||||
imageID = image.ID
|
||||
dockerImageName = tag
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if imageID == "" {
|
||||
return fmt.Errorf("could not find image ID")
|
||||
}
|
||||
|
||||
// Create and start the container.
|
||||
respContainer, err := cli.ContainerCreate(ctx, &container.Config{
|
||||
Image: dockerImageName,
|
||||
}, &container.HostConfig{
|
||||
Mounts: []mount.Mount{
|
||||
{
|
||||
Type: mount.TypeBind,
|
||||
Source: path.Join(algorithm.AlgoWorkingDir, algorithm.DatasetsDir),
|
||||
Target: datasetsMountPath,
|
||||
},
|
||||
{
|
||||
Type: mount.TypeBind,
|
||||
Source: path.Join(algorithm.AlgoWorkingDir, algorithm.ResultsDir),
|
||||
Target: resultsMountPath,
|
||||
},
|
||||
},
|
||||
}, nil, nil, containerName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create a Docker container: %v", err)
|
||||
}
|
||||
|
||||
if err := cli.ContainerStart(ctx, respContainer.ID, container.StartOptions{}); err != nil {
|
||||
return fmt.Errorf("could not start a Docker container: %v", err)
|
||||
}
|
||||
|
||||
statusCh, errCh := cli.ContainerWait(ctx, respContainer.ID, container.WaitConditionNotRunning)
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not wait for a Docker container: %v", err)
|
||||
}
|
||||
case <-statusCh:
|
||||
}
|
||||
|
||||
stdout, err := cli.ContainerLogs(ctx, respContainer.ID, container.LogsOptions{ShowStdout: true})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not read stdout from the container: %v", err)
|
||||
}
|
||||
defer stdout.Close()
|
||||
|
||||
err = writeToOut(stdout, d.stdout)
|
||||
if err != nil {
|
||||
d.logger.Warn(fmt.Sprintf("could not write to stdout: %v", err))
|
||||
}
|
||||
|
||||
stderr, err := cli.ContainerLogs(ctx, respContainer.ID, container.LogsOptions{ShowStderr: true})
|
||||
if err != nil {
|
||||
d.logger.Warn(fmt.Sprintf("could not read stderr from the container: %v", err))
|
||||
}
|
||||
defer stderr.Close()
|
||||
|
||||
err = writeToOut(stderr, d.stderr)
|
||||
if err != nil {
|
||||
d.logger.Warn(fmt.Sprintf("could not write to stderr: %v", err))
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err = cli.ContainerRemove(ctx, respContainer.ID, container.RemoveOptions{Force: true}); err != nil {
|
||||
d.logger.Warn(fmt.Sprintf("error could not remove container: %v", err))
|
||||
}
|
||||
|
||||
if _, err := cli.ImageRemove(ctx, imageID, image.RemoveOptions{Force: true}); err != nil {
|
||||
d.logger.Warn(fmt.Sprintf("error could not remove image: %v", err))
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeToOut(readCloser io.ReadCloser, ioWriter io.Writer) error {
|
||||
content, err := io.ReadAll(readCloser)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not convert content from the container: %v", err)
|
||||
}
|
||||
|
||||
if _, err := ioWriter.Write(content); err != nil {
|
||||
return fmt.Errorf("could not write to output: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
+4
-1
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/google/go-sev-guest/client"
|
||||
"github.com/ultravioletrs/cocos/agent/algorithm"
|
||||
"github.com/ultravioletrs/cocos/agent/algorithm/binary"
|
||||
"github.com/ultravioletrs/cocos/agent/algorithm/docker"
|
||||
"github.com/ultravioletrs/cocos/agent/algorithm/python"
|
||||
"github.com/ultravioletrs/cocos/agent/algorithm/wasm"
|
||||
"github.com/ultravioletrs/cocos/agent/events"
|
||||
@@ -115,7 +116,7 @@ func (as *agentService) Algo(ctx context.Context, algo Algorithm) error {
|
||||
return ErrHashMismatch
|
||||
}
|
||||
|
||||
f, err := os.CreateTemp("", "algorithm")
|
||||
f, err := os.Create("algorithm")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating algorithm file: %v", err)
|
||||
}
|
||||
@@ -160,6 +161,8 @@ func (as *agentService) Algo(ctx context.Context, algo Algorithm) error {
|
||||
as.algorithm = python.NewAlgorithm(as.sm.logger, as.eventSvc, runtime, requirementsFile, f.Name())
|
||||
case string(algorithm.AlgoTypeWasm):
|
||||
as.algorithm = wasm.NewAlgorithm(as.sm.logger, as.eventSvc, f.Name())
|
||||
case string(algorithm.AlgoTypeDocker):
|
||||
as.algorithm = docker.NewAlgorithm(as.sm.logger, as.eventSvc, f.Name())
|
||||
}
|
||||
|
||||
if err := os.Mkdir(algorithm.DatasetsDir, 0o755); err != nil {
|
||||
|
||||
@@ -23,16 +23,29 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Microsoft/go-winio v0.6.1 // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/go-connections v0.5.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
|
||||
go.opentelemetry.io/otel v1.28.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.28.0 // indirect
|
||||
golang.org/x/mod v0.19.0 // indirect
|
||||
golang.org/x/tools v0.23.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/docker/docker v27.1.0+incompatible
|
||||
github.com/go-kit/log v0.2.1 // indirect
|
||||
github.com/go-logfmt/logfmt v0.6.0 // indirect
|
||||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
@@ -55,7 +68,7 @@ require (
|
||||
go.opentelemetry.io/otel/metric v1.28.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/net v0.26.0 // indirect
|
||||
golang.org/x/net v0.27.0 // indirect
|
||||
golang.org/x/sys v0.22.0 // indirect
|
||||
golang.org/x/term v0.22.0
|
||||
golang.org/x/text v0.16.0 // indirect
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
|
||||
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
|
||||
github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE=
|
||||
github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
|
||||
github.com/absmach/magistrala v0.14.1-0.20240709113739-04c359462746 h1:Tj567KeGVygjTsSCxn4++skKiz9GkPugM1KMdIFxvfw=
|
||||
@@ -15,6 +17,16 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/digitalocean/go-libvirt v0.0.0-20240709142323-d8406205c752 h1:NI7XEcHzWVvBfVjSVK6Qk4wmrUfoyQxCNpBjrHelZFk=
|
||||
github.com/digitalocean/go-libvirt v0.0.0-20240709142323-d8406205c752/go.mod h1:/Ok8PA2qi/ve0Py38+oL+VxoYmlowigYRyLEODRYdgc=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/docker/docker v27.1.0+incompatible h1:rEHVQc4GZ0MIQKifQPHSFGV/dVgaZafgRf8fCPtDYBs=
|
||||
github.com/docker/docker v27.1.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
|
||||
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-kit/kit v0.13.0 h1:OoneCcHKHQ03LfBpoQCUfCluwd2Vt3ohz+kvbJneZAU=
|
||||
github.com/go-kit/kit v0.13.0/go.mod h1:phqEHMMUbyrCFCTgH48JueqrM3md2HcAZ8N3XE4FKDg=
|
||||
github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
|
||||
@@ -28,6 +40,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
|
||||
github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
@@ -45,6 +59,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
@@ -53,6 +69,12 @@ github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U
|
||||
github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA=
|
||||
github.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ=
|
||||
github.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnENvE+SE=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
|
||||
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
|
||||
github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
|
||||
github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
@@ -78,8 +100,12 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g=
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg=
|
||||
go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
|
||||
go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY=
|
||||
@@ -96,19 +122,50 @@ go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeX
|
||||
go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
|
||||
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
|
||||
golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
|
||||
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
|
||||
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
|
||||
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
|
||||
golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
|
||||
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
|
||||
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA=
|
||||
|
||||
@@ -40,6 +40,9 @@ CONFIG_TMPFS_POSIX_ACL=y
|
||||
CONFIG_UNWINDER_FRAME_POINTER=y
|
||||
CONFIG_VSOCKETS=y
|
||||
CONFIG_VIRTIO_VSOCKETS=y
|
||||
CONFIG_NF_TABLES=y
|
||||
CONFIG_BPF_SYSCALL=y
|
||||
CONFIG_CGROUP_BPF=y
|
||||
|
||||
###
|
||||
# AMD SEV-SNP
|
||||
|
||||
@@ -1,5 +1,16 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Change the docker.service file to allow the Docker to run in RAM
|
||||
mkdir -p /etc/systemd/system/docker.service.d
|
||||
|
||||
# Create or overwrite the override.conf file with the new Environment variable
|
||||
tee /etc/systemd/system/docker.service.d/override.conf > /dev/null <<EOF
|
||||
[Service]
|
||||
Environment=DOCKER_RAMDISK=true
|
||||
EOF
|
||||
|
||||
systemctl daemon-reload
|
||||
|
||||
NUM_OF_PERMITED_IFACE=1
|
||||
|
||||
NUM_OF_IFACE=$(ip route | grep -Eo 'dev [a-z0-9]+' | awk '{ print $2 }' | sort | uniq | wc -l)
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
[Unit]
|
||||
Description=Cocos AI agent
|
||||
After=network.target
|
||||
Before=docker.service
|
||||
|
||||
[Service]
|
||||
WorkingDirectory=/cocos
|
||||
StandardOutput=file:/var/log/cocos/agent.stdout
|
||||
StandardError=file:/var/log/cocos/agent.stderr
|
||||
|
||||
|
||||
@@ -205,28 +205,28 @@ func (p *ProgressBar) renderProgressBar() error {
|
||||
}
|
||||
|
||||
if err := p.clearProgressBar(); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to clear progress bar: %v", err)
|
||||
}
|
||||
|
||||
// The progress bar starts with the description.
|
||||
if _, err := builder.WriteString(p.description); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to add description: %v", err)
|
||||
}
|
||||
|
||||
// Add dots to progress bar.
|
||||
if _, err := builder.WriteString(progressBarDots); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to add dots: %v", err)
|
||||
}
|
||||
|
||||
// Add uploaded percentage.
|
||||
strCurrentUploadPercentage := fmt.Sprintf("%4d%% ", p.currentUploadPercentage)
|
||||
if _, err := builder.WriteString(strCurrentUploadPercentage); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to add upload percentage bracket: %v", err)
|
||||
}
|
||||
|
||||
// Add letf bracket and space to progress bar.
|
||||
if _, err := builder.WriteString(leftBracket); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to add left bracket: %v", err)
|
||||
}
|
||||
|
||||
progressWidth := width - builder.Len() - len(rightBracket+" ")
|
||||
@@ -242,12 +242,12 @@ func (p *ProgressBar) renderProgressBar() error {
|
||||
|
||||
// Add progress to the progress bar.
|
||||
if _, err := builder.WriteString(progress); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to add progress strings to padding: %v", err)
|
||||
}
|
||||
|
||||
// Add head to progress bar.
|
||||
if _, err := builder.WriteString(head); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to add head to padding: %v", err)
|
||||
}
|
||||
|
||||
// Add padding to end of bar.
|
||||
@@ -255,17 +255,17 @@ func (p *ProgressBar) renderProgressBar() error {
|
||||
|
||||
// Add padding to progress bar.
|
||||
if _, err := builder.WriteString(padding); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to add padding: %v", err)
|
||||
}
|
||||
|
||||
// Add right bracket to progress bar.
|
||||
if _, err := builder.WriteString(rightBracket); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to add right bracket: %v", err)
|
||||
}
|
||||
|
||||
// Write progress bar.
|
||||
if _, err := io.WriteString(os.Stdout, builder.String()); err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to write string: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -30,7 +30,7 @@ var (
|
||||
algoPath = "../../test/manual/algo/lin_reg.py"
|
||||
dataPath = "../../test/manual/data/iris.csv"
|
||||
|
||||
errInappropriateIoctl = errors.New("inappropriate ioctl for device")
|
||||
errInappropriateIoctl = errors.New("failed to get terminal width: inappropriate ioctl for device")
|
||||
)
|
||||
|
||||
func TestAlgo(t *testing.T) {
|
||||
|
||||
@@ -60,7 +60,7 @@ cd ../..
|
||||
# the host data information. To add the host data to the .json file that contains
|
||||
# the information about the platform, run CLI with the host data in base64 format
|
||||
# and the path of the backend_info.json file.:
|
||||
./build/cocos-cli backend measurement '<host-data>' '<backend_info.json>'
|
||||
./build/cocos-cli backend hostdata '<host-data>' '<backend_info.json>'
|
||||
|
||||
# For attested TLS, also define the path to the backend_info.json that contains reference values for the fields of the attestation report
|
||||
export AGENT_GRPC_MANIFEST=./scripts/backend_info/backend_info.json
|
||||
@@ -77,6 +77,10 @@ export AGENT_GRPC_ATTESTED_TLS=true
|
||||
./build/cocos-cli algo test/manual/algo/lin_reg.py <private_key_file_path> -a python -r test/manual/algo/requirements.py
|
||||
# 2023/09/21 10:43:53 Uploading algorithm binary: test/manual/algo/lin_reg.bin
|
||||
|
||||
# In order to run the Docker image, run the CLI program with the algorithm docker option
|
||||
go run ./cmd/cli/main.go algo -a docker <path_to_docker_image.tar> <private_key_file_path>
|
||||
# 2023/09/21 10:43:53 Uploading algorithm binary: <path_to_docker_image.tar>
|
||||
|
||||
# Run the CLI program with dataset input
|
||||
./build/cocos-cli data test/manual/data/iris.csv <private_key_file_path>
|
||||
# 2023/09/21 10:45:25 Uploading dataset CSV: test/manual/data/iris.csv
|
||||
@@ -87,7 +91,7 @@ export AGENT_GRPC_ATTESTED_TLS=true
|
||||
# 2023/09/21 10:45:40 Computation result retrieved and saved successfully!
|
||||
```
|
||||
|
||||
Now there is a `result.bin` file in the current working directory. The file holds the trained logistic regression model. To test the model, run
|
||||
Now there is a `result.zip` file in the current working directory. The file holds the trained logistic regression model. To test the model, run
|
||||
|
||||
```sh
|
||||
python ./test/manual/algo/lin_reg.py predict results.zip ./test/manual/data
|
||||
|
||||
@@ -32,7 +32,7 @@ func main() {
|
||||
pubKeyFile := os.Args[3]
|
||||
attestedTLSParam, err := strconv.ParseBool(os.Args[4])
|
||||
if err != nil {
|
||||
log.Fatalf("usage: %s <data-path> <algo-path> <attested-tls-bool>, <attested-tls-bool> must be a bool value", os.Args[0])
|
||||
log.Fatalf("usage: %s <data-path> <algo-path> <public-key-path> <attested-tls-bool>, <attested-tls-bool> must be a bool value", os.Args[0])
|
||||
}
|
||||
attestedTLS := attestedTLSParam
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Algorithm
|
||||
|
||||
Agent accepts binaries programs, python scripts, and wasm files. It runs them in a sandboxed environment and returns the output.
|
||||
Agent accepts binaries programs, python scripts, Docker images and wasm files. It runs them in a sandboxed environment and returns the output.
|
||||
|
||||
## Python Example
|
||||
|
||||
@@ -38,7 +38,7 @@ python3 test/manual/algo/lin_reg.py predict results.zip test/manual/data
|
||||
|
||||
This will make inference on the results of the linear regression model.
|
||||
|
||||
To run the examples in the agent, you can use the following command:
|
||||
To run the examples in the secure VM (SVM) by the Agent, you can use the following command:
|
||||
|
||||
```bash
|
||||
go run ./test/computations/main.go ./test/manual/algo/lin_reg.py public.pem false ./test/manual/data/iris.csv
|
||||
@@ -96,6 +96,78 @@ For addition example, you can use the following command:
|
||||
./build/cocos-cli result ./private.pem
|
||||
```
|
||||
|
||||
## Docker Example
|
||||
|
||||
Here we will use the docker with the linear regression example (`lin_reg.py`). Throughout the example, we assume that our current working directory is the directory in which the `cocos` repository is cloned. For example:
|
||||
```bash
|
||||
# ls
|
||||
cocos
|
||||
```
|
||||
|
||||
The docker image must have a `cocos` directory containing the `datasets` and `results` directories. The Agent will run this image inside the SVM and will mount the datasets and results onto the `/cocos/datasets` and `/cocos/results` directories inside the image. The docker image must also contain the command that will be run when the docker container is run.
|
||||
|
||||
The first step is to create a docker file. Use your favorite editor to create a file named `Dockerfile` in the current working directory and write in it the following code:
|
||||
|
||||
```bash
|
||||
FROM python:3.9-slim
|
||||
|
||||
# set the working directory in the container
|
||||
WORKDIR /cocos
|
||||
RUN mkdir /cocos/results
|
||||
RUN mkdir /cocos/datasets
|
||||
|
||||
COPY ./cocos/test/manual/algo/requirements.txt /cocos/requirements.txt
|
||||
COPY ./cocos/test/manual/algo/lin_reg.py /cocos/lin_reg.py
|
||||
|
||||
# install dependencies
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
# command to be run when the docker container is started
|
||||
CMD ["python3", "/cocos/lin_reg.py"]
|
||||
```
|
||||
|
||||
Next, run the build command and then save the docker image as a `tar` file.
|
||||
```bash
|
||||
docker build -t linreg .
|
||||
docker save linreg > linreg.tar
|
||||
```
|
||||
|
||||
In another window, you can run the following command:
|
||||
|
||||
```bash
|
||||
sudo MANAGER_QEMU_SMP_MAXCPUS=4 MANAGER_GRPC_URL=localhost:7001 MANAGER_LOG_LEVEL=debug MANAGER_QEMU_USE_SUDO=false MANAGER_QEMU_ENABLE_SEV=false MANAGER_QEMU_SEV_CBITPOS=51 MANAGER_QEMU_ENABLE_SEV_SNP=false MANAGER_QEMU_OVMF_CODE_FILE=/usr/share/edk2/x64/OVMF_CODE.fd MANAGER_QEMU_OVMF_VARS_FILE=/usr/share/edk2/x64/OVMF_VARS.fd go run main.go
|
||||
```
|
||||
|
||||
This command is run from the [manager main directory](../../../cmd/manager/). This will start the manager. Make sure you have already built the [qemu image](../../../hal/linux/README.md).
|
||||
|
||||
In another window, specify what kind of algorithm you want the Agent to run (docker):
|
||||
|
||||
```bash
|
||||
./cocos/build/cocos-cli algo ./linreg.tar ./cocos/private.pem -a docker
|
||||
```
|
||||
|
||||
make sure you have built the cocos-cli. This will upload the docker image.
|
||||
|
||||
Next we need to upload the dataset
|
||||
|
||||
```bash
|
||||
./cocos/build/cocos-cli data ./cocos/test/manual/data/iris.csv ./cocos/private.pem
|
||||
```
|
||||
|
||||
After some time when the results are ready, you can run the following command to get the results:
|
||||
|
||||
```bash
|
||||
./cocos/build/cocos-cli results ./cocos/private.pem
|
||||
```
|
||||
|
||||
This will return the results of the algorithm.
|
||||
|
||||
To make inference on the results, you can use the following command:
|
||||
|
||||
```bash
|
||||
python3 ./cocos/test/manual/algo/lin_reg.py predict result.zip ./cocos/test/manual/data
|
||||
```
|
||||
|
||||
## Wasm Example
|
||||
|
||||
More information on how to run wasm files can be found [here](https://github.com/ultravioletrs/ai/tree/main/burn-algorithms).
|
||||
|
||||
Reference in New Issue
Block a user