NOISSUE - Enhance OCI image extraction to return algorithm and requirements paths, and add deferred cleanup for temporary files (#586)

* feat: Enhance OCI image extraction to return algorithm and requirements paths, and add deferred cleanup for temporary files.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: implement deterministic zipping and enhance checksum verification for resources

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Update component build sources, add gRPC health checks to the CVM server, and refine algorithm argument handling and documentation.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* docs: Update remote resources testing guide with `sudo` for KBS, algorithm result saving, `requirements.txt`, and `algo-args` for RVPS.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* refactor: Explicitly ignore `stderr.Write` return values and add minor whitespace in tests.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: add comprehensive error path and edge case tests for file, zip, OCI, and agent components.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Add mutexes for thread-safe algorithm execution and expand recognized data file extensions to include common archive formats.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* feat: Add OCI extraction tests for Python algorithms and multi-layer datasets, refactor algorithm execution for testability, and enhance algorithm stop and error handling tests.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: Add error assertions to OCI extraction test helpers and remove an unused mock exec command.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* test: Improve error handling test coverage for algorithm execution and OCI resource extraction.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

* fix: Improve algorithm process termination, enhance computation error handling, and add concurrency safety to agent service.

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

---------

Signed-off-by: Sammy Oina <sammyoina@gmail.com>

test/cvms/README.md

@@ -40,6 +40,11 @@ The service is configured using environment variables from the following table.
 | `-algo-kbs-path` | Algorithm KBS resource path (e.g., 'default/key/algo-key') |
 | `-dataset-source-urls` | Comma-separated dataset source URLs |
 | `-dataset-kbs-paths` | Comma-separated dataset KBS resource paths |
+| `-algo-type` | Algorithm execution type (binary, python, docker, etc.) |
+| `-algo-args` | Comma-separated algorithm arguments |
+| `-algo-hash` | Expected SHA3-256 hash of decrypted algorithm (hex) |
+| `-dataset-hash` | Expected SHA3-256 hash of decrypted dataset (hex) |
+| `-dataset-decompress` | Whether to decompress datasets (true,false) |
 
 ### Optional Flags
 
@@ -114,11 +119,12 @@ go run ./test/cvms/main.go \
 
 ## Notes
 
-- **Either** `-algo-path` **OR** (`-algo-source-url` AND `-algo-kbs-path`) must be provided
-- When using remote datasets, `-dataset-source-urls` and `-dataset-kbs-paths` must have the same number of comma-separated values
-- The `-kbs-url` flag should be provided when using any remote resources
-- For remote resources, the hash values in the manifest are currently placeholders (all zeros). In production, these should be the actual hashes of the **decrypted** data
-- See [TESTING_REMOTE_RESOURCES.md](../TESTING_REMOTE_RESOURCES.md) for a complete guide on testing remote resource downloads with KBS attestation
+- **Either** `-algo-path` **OR** (`-algo-source-url` AND `-algo-kbs-path`) must be provided.
+- When using remote datasets, `-dataset-source-urls` and `-dataset-kbs-paths` must have the same number of comma-separated values.
+- The `-kbs-url` flag should be provided when using any remote resources.
+- **Checksum Verification**: For remote resources, you must provide the actual SHA3-256 hash of the **decrypted plaintext** content via `-algo-hash` and `-dataset-hash`. The Agent will verify this hash after downloading and decrypting the resource.
+- **Calculating Hashes**: Use `cocos-cli checksum <path>` on your local source files (or directories) to generate the correct hash for the manifest.
+- See [TESTING_REMOTE_RESOURCES.md](../../agent/TESTING_REMOTE_RESOURCES.md) for a complete guide on testing remote resource downloads with KBS attestation.
 
 ## Architecture
 

test/cvms/main.go

@@ -140,13 +140,15 @@ func (s *svc) Run(ctx context.Context, ipAddress string, sendMessage cvmsgrpc.Se
 				s.logger.Error(fmt.Sprintf("data file does not exist: %s", dataPath))
 				return
 			}
-			dataHash, err := internal.Checksum(dataPath)
+			dataHash, err := internal.ChecksumHex(dataPath)
 			if err != nil {
 				s.logger.Error(fmt.Sprintf("failed to calculate checksum: %s", err))
 				return
 			}
+			s.logger.Info("local dataset checksum", "path", dataPath, "hash", dataHash)
 
-			datasets = append(datasets, &cvms.Dataset{Hash: dataHash[:], UserKey: pubPem.Bytes})
+			hashBytes, _ := hex.DecodeString(dataHash)
+			datasets = append(datasets, &cvms.Dataset{Hash: hashBytes, UserKey: pubPem.Bytes})
 		}
 	}
 
@@ -170,11 +172,16 @@ func (s *svc) Run(ctx context.Context, ipAddress string, sendMessage cvmsgrpc.Se
 			algoHashBytes = make([]byte, 32)
 		}
 
+		var algoArgs []string
+		if algoArgsString != "" {
+			algoArgs = strings.Split(algoArgsString, ",")
+		}
+
 		algorithm = &cvms.Algorithm{
 			Hash:     algoHashBytes,
 			UserKey:  pubPem.Bytes,
 			AlgoType: algoType,
-			AlgoArgs: strings.Split(algoArgsString, ","),
+			AlgoArgs: algoArgs,
 			Source: &cvms.Source{
 				Type:            "oci-image",
 				Url:             algoSourceURL,
@@ -184,16 +191,25 @@ func (s *svc) Run(ctx context.Context, ipAddress string, sendMessage cvmsgrpc.Se
 		}
 	} else {
 		// Direct upload mode - use local file
-		if algoPath == "" {
-			s.logger.Error("algorithm path is required when not using remote source")
-			return
-		}
-		algoHash, err := internal.Checksum(algoPath)
+		fileHash, err := internal.ChecksumHex(algoPath)
 		if err != nil {
 			s.logger.Error(fmt.Sprintf("failed to calculate checksum: %s", err))
 			return
 		}
-		algorithm = &cvms.Algorithm{Hash: algoHash[:], UserKey: pubPem.Bytes}
+		s.logger.Info("local algorithm checksum", "path", algoPath, "hash", fileHash)
+
+		var algoArgs []string
+		if algoArgsString != "" {
+			algoArgs = strings.Split(algoArgsString, ",")
+		}
+
+		hashBytes, _ := hex.DecodeString(fileHash)
+		algorithm = &cvms.Algorithm{
+			Hash:     hashBytes,
+			UserKey:  pubPem.Bytes,
+			AlgoType: algoType,
+			AlgoArgs: algoArgs,
+		}
 	}
 
 	// Build KBS config