mirror of
https://github.com/portainer/portainer.git
synced 2026-08-07 11:44:49 +00:00
feat(alerts): removal of snapshot reliance [R8S-902] (#1994)
This commit is contained in:
Steven Kang
committed by
Robbie Cowan
Robbie Cowan
parent
b5bc5f65ad
commit
ac1e333dde
@@ -65,7 +65,6 @@ require (
|
||||
k8s.io/cli-runtime v0.35.1
|
||||
k8s.io/client-go v0.35.1
|
||||
k8s.io/kubectl v0.35.1
|
||||
k8s.io/kubelet v0.35.0
|
||||
k8s.io/metrics v0.35.1
|
||||
oras.land/oras-go/v2 v2.6.0
|
||||
software.sslmate.com/src/go-pkcs12 v0.0.0-20210415151418-c5206de65a78
|
||||
|
||||
@@ -1107,8 +1107,6 @@ k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZ
|
||||
k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ=
|
||||
k8s.io/kubectl v0.35.1 h1:zP3Er8C5i1dcAFUMh9Eva0kVvZHptXIn/+8NtRWMxwg=
|
||||
k8s.io/kubectl v0.35.1/go.mod h1:cQ2uAPs5IO/kx8R5s5J3Ihv3VCYwrx0obCXum0CvnXo=
|
||||
k8s.io/kubelet v0.35.0 h1:8cgJHCBCKLYuuQ7/Pxb/qWbJfX1LXIw7790ce9xHq7c=
|
||||
k8s.io/kubelet v0.35.0/go.mod h1:ciRzAXn7C4z5iB7FhG1L2CGPPXLTVCABDlbXt/Zz8YA=
|
||||
k8s.io/metrics v0.35.1 h1:MUcrUcWlq81XiripkydzCGsY9zQawDXfP9IICNNcVVw=
|
||||
k8s.io/metrics v0.35.1/go.mod h1:9x7xWOAOiWzHA0vaqLgSE4PXF3vyT5ts5XIbx8OSjiI=
|
||||
k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck=
|
||||
|
||||
@@ -5,9 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -22,7 +20,6 @@ import (
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
|
||||
)
|
||||
|
||||
func CreateKubernetesSnapshot(cli *kubernetes.Clientset) (*portainer.KubernetesSnapshot, error) {
|
||||
@@ -70,45 +67,9 @@ func kubernetesSnapshotNodes(snapshot *portainer.KubernetesSnapshot, cli kuberne
|
||||
snapshot.TotalMemory = totalMemory
|
||||
snapshot.NodeCount = len(nodeList.Items)
|
||||
|
||||
// Collect performance metrics if we have a real client, otherwise use zero values
|
||||
if clientset, ok := cli.(*kubernetes.Clientset); ok {
|
||||
kubernetesSnapshotPerformanceMetricsWithClient(nodeList, clientset, snapshot)
|
||||
} else {
|
||||
snapshot.PerformanceMetrics = &portainer.PerformanceMetrics{
|
||||
CPUUsage: 0,
|
||||
MemoryUsage: 0,
|
||||
NetworkUsage: 0,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func kubernetesSnapshotPerformanceMetricsWithClient(
|
||||
nodeList *corev1.NodeList,
|
||||
cli *kubernetes.Clientset,
|
||||
snapshot *portainer.KubernetesSnapshot,
|
||||
) {
|
||||
performanceMetrics := &portainer.PerformanceMetrics{
|
||||
CPUUsage: 0,
|
||||
MemoryUsage: 0,
|
||||
NetworkUsage: 0,
|
||||
}
|
||||
|
||||
for _, node := range nodeList.Items {
|
||||
nodeMetrics, err := kubernetesSnapshotNodePerformanceMetrics(cli, node, nil)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msgf("failed to snapshot performance metrics for node %s", node.Name)
|
||||
continue
|
||||
}
|
||||
if nodeMetrics != nil {
|
||||
performanceMetrics.CPUUsage += nodeMetrics.CPUUsage
|
||||
performanceMetrics.MemoryUsage += nodeMetrics.MemoryUsage
|
||||
performanceMetrics.NetworkUsage += nodeMetrics.NetworkUsage
|
||||
}
|
||||
}
|
||||
snapshot.PerformanceMetrics = performanceMetrics
|
||||
}
|
||||
|
||||
// KubernetesSnapshotDiagnostics returns the diagnostics data for the agent
|
||||
func KubernetesSnapshotDiagnostics(cli *kubernetes.Clientset, edgeKey string) (*portainer.DiagnosticsData, error) {
|
||||
podID := os.Getenv("HOSTNAME")
|
||||
@@ -166,58 +127,6 @@ func kubernetesSnapshotPodErrorLogs(snapshot *portainer.KubernetesSnapshot, cli
|
||||
return nil
|
||||
}
|
||||
|
||||
func kubernetesSnapshotNodePerformanceMetrics(cli *kubernetes.Clientset, node corev1.Node, _ *portainer.PerformanceMetrics) (*portainer.PerformanceMetrics, error) {
|
||||
result := cli.RESTClient().Get().AbsPath(fmt.Sprintf("/api/v1/nodes/%s/proxy/stats/summary", node.Name)).Do(context.TODO())
|
||||
if result.Error() != nil {
|
||||
return nil, fmt.Errorf("failed to get node performance metrics: %w", result.Error())
|
||||
}
|
||||
|
||||
raw, err := result.Raw()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get node performance metrics: %w", err)
|
||||
}
|
||||
|
||||
stats := statsapi.Summary{}
|
||||
err = json.Unmarshal(raw, &stats)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal node performance metrics: %w", err)
|
||||
}
|
||||
|
||||
nodeStats := stats.Node
|
||||
metrics := calculateNodeMetrics(nodeStats, node)
|
||||
return metrics, nil
|
||||
}
|
||||
|
||||
// calculateNodeMetrics calculates performance metrics from node stats - extracted for testability
|
||||
func calculateNodeMetrics(nodeStats statsapi.NodeStats, node corev1.Node) *portainer.PerformanceMetrics {
|
||||
if reflect.DeepEqual(nodeStats, statsapi.NodeStats{}) {
|
||||
return nil
|
||||
}
|
||||
|
||||
metrics := &portainer.PerformanceMetrics{}
|
||||
|
||||
// Calculate CPU usage percentage
|
||||
if nodeStats.CPU != nil && nodeStats.CPU.UsageNanoCores != nil {
|
||||
totalCapacityNanoCores := node.Status.Capacity.Cpu().Value() * 1_000_000_000
|
||||
metrics.CPUUsage = math.Round(float64(*nodeStats.CPU.UsageNanoCores) / float64(totalCapacityNanoCores) * 100)
|
||||
}
|
||||
|
||||
// Calculate Memory usage percentage
|
||||
if nodeStats.Memory != nil && nodeStats.Memory.WorkingSetBytes != nil {
|
||||
totalCapacityBytes := node.Status.Capacity.Memory().Value()
|
||||
metrics.MemoryUsage = math.Round(float64(*nodeStats.Memory.WorkingSetBytes) / float64(totalCapacityBytes) * 100)
|
||||
}
|
||||
|
||||
// Calculate Network usage in MB
|
||||
if nodeStats.Network != nil && nodeStats.Network.RxBytes != nil && nodeStats.Network.TxBytes != nil {
|
||||
totalBytes := float64(*nodeStats.Network.RxBytes) + float64(*nodeStats.Network.TxBytes)
|
||||
const bytesToMB = 1024 * 1024
|
||||
metrics.NetworkUsage = math.Round(totalBytes / bytesToMB)
|
||||
}
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
func filterLogsByPattern(logBytes []byte, patterns []string) []map[string]string {
|
||||
logs := []map[string]string{}
|
||||
for line := range strings.SplitSeq(strings.TrimSpace(string(logBytes)), "\n") {
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
kfake "k8s.io/client-go/kubernetes/fake"
|
||||
ktesting "k8s.io/client-go/testing"
|
||||
statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
|
||||
)
|
||||
|
||||
func TestKubernetesSnapshotNodes(t *testing.T) {
|
||||
@@ -76,7 +75,7 @@ func TestKubernetesSnapshotNodes(t *testing.T) {
|
||||
require.Equal(t, 3, snapshot.NodeCount) // 3 nodes
|
||||
require.Equal(t, int64(12), snapshot.TotalCPU) // 6 + 4 + 2 = 12 CPUs
|
||||
require.Equal(t, int64(25769803776), snapshot.TotalMemory) // 12GB + 8GB + 4GB = 24GB in bytes
|
||||
require.NotNil(t, snapshot.PerformanceMetrics) // Performance metrics should be initialized
|
||||
require.Nil(t, snapshot.PerformanceMetrics) // Performance metrics are no longer collected server-side
|
||||
|
||||
t.Logf("kubernetesSnapshotNodes test result: Nodes=%d, CPUs=%d, Memory=%d bytes",
|
||||
snapshot.NodeCount, snapshot.TotalCPU, snapshot.TotalMemory)
|
||||
@@ -150,7 +149,7 @@ func TestCreateKubernetesSnapshotIntegration(t *testing.T) {
|
||||
require.Equal(t, 2, snapshot.NodeCount)
|
||||
require.Equal(t, int64(12), snapshot.TotalCPU) // 8 + 4 = 12 CPUs
|
||||
require.Equal(t, int64(25769803776), snapshot.TotalMemory) // 16GB + 8GB = 24GB in bytes
|
||||
require.NotNil(t, snapshot.PerformanceMetrics)
|
||||
require.Nil(t, snapshot.PerformanceMetrics)
|
||||
|
||||
// Manually set the version to complete the integration test
|
||||
snapshot.KubernetesVersion = serverInfo.GitVersion
|
||||
@@ -213,7 +212,7 @@ func TestKubernetesSnapshotNodesSingleNode(t *testing.T) {
|
||||
require.Equal(t, 1, snapshot.NodeCount)
|
||||
require.Equal(t, int64(1), snapshot.TotalCPU)
|
||||
require.Equal(t, int64(1073741824), snapshot.TotalMemory) // 1GB in bytes
|
||||
require.NotNil(t, snapshot.PerformanceMetrics)
|
||||
require.Nil(t, snapshot.PerformanceMetrics)
|
||||
|
||||
t.Logf("Single node test result: Nodes=%d, CPUs=%d, Memory=%d bytes",
|
||||
snapshot.NodeCount, snapshot.TotalCPU, snapshot.TotalMemory)
|
||||
@@ -246,85 +245,7 @@ func TestKubernetesSnapshotNodesZeroResources(t *testing.T) {
|
||||
require.Equal(t, 1, snapshot.NodeCount)
|
||||
require.Equal(t, int64(0), snapshot.TotalCPU)
|
||||
require.Equal(t, int64(0), snapshot.TotalMemory)
|
||||
require.NotNil(t, snapshot.PerformanceMetrics)
|
||||
require.Nil(t, snapshot.PerformanceMetrics)
|
||||
|
||||
t.Log("Zero resources test passed - handles edge case correctly")
|
||||
}
|
||||
|
||||
func TestCalculateNodeMetrics(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Create a test node with specific capacity
|
||||
node := corev1.Node{
|
||||
Status: corev1.NodeStatus{
|
||||
Capacity: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("4"), // 4 CPU cores
|
||||
corev1.ResourceMemory: resource.MustParse("8Gi"), // 8GB memory
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("CalculatesCorrectCPUPercentage", func(t *testing.T) {
|
||||
usageNanoCores := uint64(2_000_000_000) // 2 cores worth of nanocores
|
||||
nodeStats := statsapi.NodeStats{
|
||||
CPU: &statsapi.CPUStats{
|
||||
UsageNanoCores: &usageNanoCores,
|
||||
},
|
||||
}
|
||||
|
||||
metrics := calculateNodeMetrics(nodeStats, node)
|
||||
require.NotNil(t, metrics)
|
||||
require.Equal(t, 50, int(metrics.CPUUsage)) // 2/4 = 50%
|
||||
})
|
||||
|
||||
t.Run("CalculatesCorrectMemoryPercentage", func(t *testing.T) {
|
||||
workingSetBytes := uint64(4 * 1024 * 1024 * 1024) // 4GB
|
||||
nodeStats := statsapi.NodeStats{
|
||||
Memory: &statsapi.MemoryStats{
|
||||
WorkingSetBytes: &workingSetBytes,
|
||||
},
|
||||
}
|
||||
|
||||
metrics := calculateNodeMetrics(nodeStats, node)
|
||||
require.NotNil(t, metrics)
|
||||
require.Equal(t, 50, int(metrics.MemoryUsage)) // 4GB/8GB = 50%
|
||||
})
|
||||
|
||||
t.Run("CalculatesCorrectNetworkUsage", func(t *testing.T) {
|
||||
rxBytes := uint64(1024 * 1024 * 1024) // 1GB
|
||||
txBytes := uint64(1024 * 1024 * 1024) // 1GB
|
||||
nodeStats := statsapi.NodeStats{
|
||||
Network: &statsapi.NetworkStats{
|
||||
InterfaceStats: statsapi.InterfaceStats{
|
||||
RxBytes: &rxBytes,
|
||||
TxBytes: &txBytes,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
metrics := calculateNodeMetrics(nodeStats, node)
|
||||
require.NotNil(t, metrics)
|
||||
require.Equal(t, 2048, int(metrics.NetworkUsage)) // 2GB = 2048MB
|
||||
})
|
||||
|
||||
t.Run("HandlesEmptyStats", func(t *testing.T) {
|
||||
nodeStats := statsapi.NodeStats{}
|
||||
metrics := calculateNodeMetrics(nodeStats, node)
|
||||
require.Nil(t, metrics)
|
||||
})
|
||||
|
||||
t.Run("HandlesPartialStats", func(t *testing.T) {
|
||||
usageNanoCores := uint64(1_000_000_000) // 1 core
|
||||
nodeStats := statsapi.NodeStats{
|
||||
CPU: &statsapi.CPUStats{
|
||||
UsageNanoCores: &usageNanoCores,
|
||||
},
|
||||
// Memory and Network are nil
|
||||
}
|
||||
|
||||
metrics := calculateNodeMetrics(nodeStats, node)
|
||||
require.NotNil(t, metrics)
|
||||
require.Equal(t, 25, int(metrics.CPUUsage)) // 1/4 = 25%
|
||||
require.Equal(t, 0, int(metrics.MemoryUsage))
|
||||
require.Equal(t, 0, int(metrics.NetworkUsage))
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user