From 3062410b42e61a2ae8ffe40fe2e92d16a00d7484 Mon Sep 17 00:00:00 2001 From: Hiram Chirino Date: Wed, 1 Jul 2026 12:07:51 -0400 Subject: [PATCH] feat(k8s): group containers by owner chain (#4811) Signed-off-by: Hiram Chirino --- assets/auto-imports.d.ts | 8 +- assets/components/K8sMenu.vue | 8 +- assets/composable/eventStreams.ts | 6 +- assets/pages/owner/[name].vue | 9 +- assets/stores/k8s.spec.ts | 149 +++++++++++++++ assets/stores/k8s.ts | 101 ++++++---- docs/guide/k8s.md | 6 + examples/k8s.dozzle.yml | 6 + internal/k8s/client.go | 307 ++++++++++++++++++++++++++++-- internal/k8s/client_test.go | 293 ++++++++++++++++++++++++++++ 10 files changed, 833 insertions(+), 60 deletions(-) create mode 100644 assets/stores/k8s.spec.ts create mode 100644 internal/k8s/client_test.go diff --git a/assets/auto-imports.d.ts b/assets/auto-imports.d.ts index c6b6db12..72404e5a 100644 --- a/assets/auto-imports.d.ts +++ b/assets/auto-imports.d.ts @@ -66,8 +66,10 @@ declare global { const getCurrentScope: typeof import('vue').getCurrentScope const getCurrentWatcher: typeof import('vue').getCurrentWatcher const getDeep: typeof import('./utils/index').getDeep + const getK8sOwnerRefs: typeof import('./stores/k8s').getK8sOwnerRefs const globalShowPopup: typeof import('./composable/popup').globalShowPopup const groupContainers: typeof import('./stores/settings').groupContainers + const groupK8sOwners: typeof import('./stores/k8s').groupK8sOwners const h: typeof import('vue').h const hashCode: typeof import('./utils/index').hashCode const highlightSubstringInHtml: typeof import('./utils/index').highlightSubstringInHtml @@ -117,6 +119,7 @@ declare global { const onUnmounted: typeof import('vue').onUnmounted const onUpdated: typeof import('vue').onUpdated const onWatcherCleanup: typeof import('vue').onWatcherCleanup + const ownerMembershipLabel: typeof import('./stores/k8s').ownerMembershipLabel const parseMessage: typeof import('./composable/loadBetween').parseMessage const pausableWatch: typeof import('@vueuse/core').pausableWatch const persistentVisibleKeysForContainer: typeof import('./composable/storage').persistentVisibleKeysForContainer @@ -435,7 +438,7 @@ declare global { export type { Host } from './stores/hosts' import('./stores/hosts') // @ts-ignore - export type { K8sNamespace, K8sOwner } from './stores/k8s' + export type { K8sNamespace, K8sOwner, K8sOwnerRef } from './stores/k8s' import('./stores/k8s') // @ts-ignore export type { Settings } from './stores/settings' @@ -506,8 +509,10 @@ declare module 'vue' { readonly getCurrentScope: UnwrapRef readonly getCurrentWatcher: UnwrapRef readonly getDeep: UnwrapRef + readonly getK8sOwnerRefs: UnwrapRef readonly globalShowPopup: UnwrapRef readonly groupContainers: UnwrapRef + readonly groupK8sOwners: UnwrapRef readonly h: UnwrapRef readonly hashCode: UnwrapRef readonly hourStyle: UnwrapRef @@ -556,6 +561,7 @@ declare module 'vue' { readonly onUnmounted: UnwrapRef readonly onUpdated: UnwrapRef readonly onWatcherCleanup: UnwrapRef + readonly ownerMembershipLabel: UnwrapRef readonly parseMessage: UnwrapRef readonly pausableWatch: UnwrapRef readonly persistentVisibleKeysForContainer: UnwrapRef diff --git a/assets/components/K8sMenu.vue b/assets/components/K8sMenu.vue index 9e81618b..164bef1f 100644 --- a/assets/components/K8sMenu.vue +++ b/assets/components/K8sMenu.vue @@ -78,8 +78,8 @@
    -
  • - +
  • +
    {{ owner.kind }}/{{ owner.name }}
     @@ -95,8 +95,8 @@ {{ $t("label.owners") }} ({{ ownersWithoutNamespace.length }})
      -
    • - +
    • +
      {{ owner.kind }}/{{ owner.name }}
       diff --git a/assets/composable/eventStreams.ts b/assets/composable/eventStreams.ts index ddad3554..411c1b41 100644 --- a/assets/composable/eventStreams.ts +++ b/assets/composable/eventStreams.ts @@ -57,12 +57,12 @@ export function useServiceStream(service: Ref): LogStreamSource { } export function useNamespaceStream(namespace: Ref<{ name: string }>): LogStreamSource { - const labels = computed(() => `namespace:${namespace.value.name}`); + const labels = computed(() => `@k8s.namespace:${namespace.value.name}`); return useLogStream(computed(() => `/api/labels/${labels.value}/logs/stream`)); } -export function useOwnerStream(owner: Ref<{ name: string; kind: string }>): LogStreamSource { - const labels = computed(() => `owner.kind:${owner.value.kind},owner.name:${owner.value.name}`); +export function useOwnerStream(owner: Ref<{ label: string }>): LogStreamSource { + const labels = computed(() => `${owner.value.label}:true`); return useLogStream(computed(() => `/api/labels/${labels.value}/logs/stream`)); } diff --git a/assets/pages/owner/[name].vue b/assets/pages/owner/[name].vue index 3142b8a3..111e0d1c 100644 --- a/assets/pages/owner/[name].vue +++ b/assets/pages/owner/[name].vue @@ -14,7 +14,14 @@ const { pinnedLogs } = storeToRefs(pinnedLogsStore); const k8sStore = useK8sStore(); const { owners } = storeToRefs(k8sStore); -const owner = computed(() => owners.value.find((o) => o.name === route.params.name)); +const ownerKey = computed(() => { + try { + return decodeURIComponent(String(route.params.name)); + } catch { + return String(route.params.name); + } +}); +const owner = computed(() => owners.value.find((o) => o.key === ownerKey.value)); watchEffect(() => { if (ready.value) { diff --git a/assets/stores/k8s.spec.ts b/assets/stores/k8s.spec.ts new file mode 100644 index 00000000..6a0b9fdf --- /dev/null +++ b/assets/stores/k8s.spec.ts @@ -0,0 +1,149 @@ +import { describe, expect, test, vi } from "vitest"; + +import { Container } from "@/models/Container"; +import { getK8sOwnerRefs, groupK8sOwners, ownerMembershipLabel } from "./k8s"; + +vi.mock("@/stores/config", () => ({ + __esModule: true, + default: { base: "", hosts: [{ name: "localhost", id: "localhost" }] }, + withBase: (path: string) => path, +})); + +function makeContainer(id: string, labels: Record) { + return new Container( + id, + new Date(), + new Date(), + new Date(), + "image", + id, + "command", + "localhost", + labels, + "running", + 0, + 0, + [], + ); +} + +describe("getK8sOwnerRefs", () => { + test("parses indexed owner-chain labels", () => { + const container = makeContainer("api", { + namespace: "default", + "@k8s.owner.count": "2", + "@k8s.owner.0.kind": "ReplicaSet", + "@k8s.owner.0.namespace": "default", + "@k8s.owner.0.name": "api-6f88b977f4", + "@k8s.owner.0.key": "ReplicaSet~default~api-6f88b977f4", + "@k8s.owner.1.kind": "Deployment", + "@k8s.owner.1.namespace": "default", + "@k8s.owner.1.name": "api", + "@k8s.owner.1.key": "Deployment~default~api", + }); + + expect(getK8sOwnerRefs(container)).toEqual([ + { + key: "ReplicaSet~default~api-6f88b977f4", + label: ownerMembershipLabel("ReplicaSet~default~api-6f88b977f4"), + kind: "ReplicaSet", + name: "api-6f88b977f4", + namespace: "default", + }, + { + key: "Deployment~default~api", + label: ownerMembershipLabel("Deployment~default~api"), + kind: "Deployment", + name: "api", + namespace: "default", + }, + ]); + }); + + test("falls back to legacy immediate owner labels", () => { + const container = makeContainer("api", { + namespace: "default", + "owner.kind": "ReplicaSet", + "owner.name": "api-6f88b977f4", + }); + + expect(getK8sOwnerRefs(container)).toEqual([ + { + key: "ReplicaSet~default~api-6f88b977f4", + label: ownerMembershipLabel("ReplicaSet~default~api-6f88b977f4"), + kind: "ReplicaSet", + name: "api-6f88b977f4", + namespace: "default", + }, + ]); + }); + + test("uses kind for display and type for CRD identity", () => { + const container = makeContainer("rollout-api", { + namespace: "default", + "@k8s.owner.count": "1", + "@k8s.owner.0.kind": "Rollout", + "@k8s.owner.0.namespace": "default", + "@k8s.owner.0.name": "api", + "@k8s.owner.0.key": "argoproj.io~v1alpha1~Rollout~default~api", + }); + + expect(getK8sOwnerRefs(container)).toEqual([ + { + key: "argoproj.io~v1alpha1~Rollout~default~api", + label: ownerMembershipLabel("argoproj.io~v1alpha1~Rollout~default~api"), + kind: "Rollout", + name: "api", + namespace: "default", + }, + ]); + }); +}); + +describe("groupK8sOwners", () => { + test("groups a container under every owner in its chain", () => { + const container = makeContainer("api", { + namespace: "default", + "@k8s.owner.count": "2", + "@k8s.owner.0.kind": "ReplicaSet", + "@k8s.owner.0.namespace": "default", + "@k8s.owner.0.name": "api-6f88b977f4", + "@k8s.owner.0.key": "ReplicaSet~default~api-6f88b977f4", + "@k8s.owner.1.kind": "Deployment", + "@k8s.owner.1.namespace": "default", + "@k8s.owner.1.name": "api", + "@k8s.owner.1.key": "Deployment~default~api", + }); + + const owners = groupK8sOwners([container]); + + expect(owners.map((owner) => owner.key).sort()).toEqual([ + "Deployment~default~api", + "ReplicaSet~default~api-6f88b977f4", + ]); + expect(owners.every((owner) => owner.containers.length === 1)).toBe(true); + }); + + test("keeps same-name owners in different namespaces separate", () => { + const owners = groupK8sOwners([ + makeContainer("default-api", { + namespace: "default", + "@k8s.owner.count": "1", + "@k8s.owner.0.kind": "Deployment", + "@k8s.owner.0.namespace": "default", + "@k8s.owner.0.name": "api", + "@k8s.owner.0.key": "Deployment~default~api", + }), + makeContainer("prod-api", { + namespace: "prod", + "@k8s.owner.count": "1", + "@k8s.owner.0.kind": "Deployment", + "@k8s.owner.0.namespace": "prod", + "@k8s.owner.0.name": "api", + "@k8s.owner.0.key": "Deployment~prod~api", + }), + ]); + + expect(owners.map((owner) => owner.key).sort()).toEqual(["Deployment~default~api", "Deployment~prod~api"]); + }); +}); diff --git a/assets/stores/k8s.ts b/assets/stores/k8s.ts index b25d4e59..304026d4 100644 --- a/assets/stores/k8s.ts +++ b/assets/stores/k8s.ts @@ -2,6 +2,14 @@ import { acceptHMRUpdate, defineStore } from "pinia"; import { Container, GroupedContainers } from "@/models/Container"; +export type K8sOwnerRef = { + key: string; + label: string; + kind: string; + name: string; + namespace?: string; +}; + export class K8sNamespace { constructor( public readonly name: string, @@ -22,6 +30,9 @@ export class K8sOwner { constructor( public readonly name: string, public readonly kind: string, + public readonly namespaceName: string | undefined, + public readonly key: string, + public readonly label: string, public readonly containers: Container[], ) {} @@ -32,6 +43,57 @@ export class K8sOwner { } } +export function ownerMembershipLabel(key: string) { + const bytes = new TextEncoder().encode(key); + let binary = ""; + for (const byte of bytes) { + binary += String.fromCharCode(byte); + } + return `@k8s.owner.key.${btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "")}`; +} + +export function getK8sOwnerRefs(container: Container): K8sOwnerRef[] { + const count = Number(container.labels["@k8s.owner.count"] ?? container.labels["k8s.owner.count"] ?? 0); + if (count > 0) { + const owners: K8sOwnerRef[] = []; + for (let i = 0; i < count; i++) { + const syntheticPrefix = `@k8s.owner.${i}.`; + const legacyPrefix = `k8s.owner.${i}.`; + const labelValue = (key: string) => + container.labels[`${syntheticPrefix}${key}`] ?? container.labels[`${legacyPrefix}${key}`]; + const kind = labelValue("kind"); + const name = labelValue("name"); + const namespace = labelValue("namespace"); + if (!kind || !name) continue; + const key = labelValue("key") ?? `${kind}~${namespace ?? ""}~${name}`; + owners.push({ key, label: ownerMembershipLabel(key), kind, name, namespace }); + } + return owners; + } + + // "~" matches the backend owner-key delimiter: URL-safe and invalid in Kubernetes names. + const kind = container.labels["owner.kind"]; + const name = container.labels["owner.name"]; + if (!kind || !name) return []; + + const namespace = container.labels["namespace"]; + const key = container.labels["owner.key"] ?? `${kind}~${namespace ?? ""}~${name}`; + return [{ key, label: ownerMembershipLabel(key), kind, name, namespace }]; +} + +export function groupK8sOwners(containers: Container[]) { + const ownerGroups: Record = {}; + for (const container of containers) { + for (const owner of getK8sOwnerRefs(container)) { + ownerGroups[owner.key] ||= { owner, containers: [] }; + ownerGroups[owner.key].containers.push(container); + } + } + return Object.values(ownerGroups).map(({ owner, containers }) => { + return new K8sOwner(owner.name, owner.kind, owner.namespace, owner.key, owner.label, containers); + }); +} + export const useK8sStore = defineStore("k8s", () => { const containerStore = useContainerStore(); const { containers } = storeToRefs(containerStore) as unknown as { containers: Ref }; @@ -50,24 +112,7 @@ export const useK8sStore = defineStore("k8s", () => { const newNamespaces: K8sNamespace[] = []; for (const [name, containers] of Object.entries(namespacedContainers)) { - const ownerGroups: Record = {}; - - for (const container of containers) { - const ownerKind = container.labels["owner.kind"]; - const ownerName = container.labels["owner.name"]; - - if (ownerKind === undefined || ownerName === undefined) continue; - const key = `${ownerKind}:${ownerName}`; - ownerGroups[key] ||= []; - ownerGroups[key].push(container); - } - - const newOwners: K8sOwner[] = []; - - for (const [key, containers] of Object.entries(ownerGroups)) { - const [kind, name] = key.split(":"); - newOwners.push(new K8sOwner(name, kind, containers)); - } + const newOwners = groupK8sOwners(containers); if (newOwners.length === 0) continue; @@ -75,7 +120,7 @@ export const useK8sStore = defineStore("k8s", () => { new K8sNamespace( name, containers, - newOwners.sort((a, b) => a.name.localeCompare(b.name)), + newOwners.sort((a, b) => a.key.localeCompare(b.key)), ), ); } @@ -83,33 +128,23 @@ export const useK8sStore = defineStore("k8s", () => { }); const owners = computed(() => { - const ownerGroups: Record = {}; + const containersWithoutNamespace: Container[] = []; for (const container of runningContainers.value) { - const ownerKind = container.labels["owner.kind"]; - const ownerName = container.labels["owner.name"]; const namespace = container.labels["namespace"]; - - if (ownerKind === undefined || ownerName === undefined) continue; if (namespace) { // Skip containers that are already part of a namespace const hasNamespace = namespaces.value.some((ns) => ns.name === namespace); if (hasNamespace) continue; } - - const key = `${ownerKind}:${ownerName}`; - ownerGroups[key] ||= []; - ownerGroups[key].push(container); + containersWithoutNamespace.push(container); } const ownersWithNamespace = namespaces.value.flatMap((ns) => ns.owners); - const ownersWithoutNamespace = Object.entries(ownerGroups).map(([key, containers]) => { - const [kind, name] = key.split(":"); - return new K8sOwner(name, kind, containers); - }); + const ownersWithoutNamespace = groupK8sOwners(containersWithoutNamespace); - return [...ownersWithNamespace, ...ownersWithoutNamespace].sort((a, b) => a.name.localeCompare(b.name)); + return [...ownersWithNamespace, ...ownersWithoutNamespace].sort((a, b) => a.key.localeCompare(b.key)); }); const customGroups = computed(() => { diff --git a/docs/guide/k8s.md b/docs/guide/k8s.md index 7122a17e..4a7256a2 100644 --- a/docs/guide/k8s.md +++ b/docs/guide/k8s.md @@ -26,6 +26,12 @@ rules: - apiGroups: [""] resources: ["pods", "pods/log", "nodes"] verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["deployments", "replicasets", "daemonsets", "statefulsets"] + verbs: ["get"] + - apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get"] - apiGroups: ["metrics.k8s.io"] resources: ["pods"] verbs: ["get", "list"] diff --git a/examples/k8s.dozzle.yml b/examples/k8s.dozzle.yml index c8c6ba1f..92e090e4 100644 --- a/examples/k8s.dozzle.yml +++ b/examples/k8s.dozzle.yml @@ -13,6 +13,12 @@ rules: - apiGroups: [""] resources: ["pods", "pods/log", "nodes"] verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["deployments", "replicasets", "daemonsets", "statefulsets"] + verbs: ["get"] + - apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["get"] - apiGroups: ["metrics.k8s.io"] resources: ["pods"] verbs: ["get", "list"] diff --git a/internal/k8s/client.go b/internal/k8s/client.go index b2e47c4d..d48c6cd4 100644 --- a/internal/k8s/client.go +++ b/internal/k8s/client.go @@ -2,9 +2,11 @@ package k8s import ( "context" + "encoding/base64" "errors" "fmt" "io" + "regexp" "strings" "sync" "time" @@ -14,8 +16,14 @@ import ( "github.com/amir20/dozzle/internal/container" "github.com/amir20/dozzle/internal/utils" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + "k8s.io/client-go/discovery/cached/memory" + "k8s.io/client-go/dynamic" "github.com/rs/zerolog/log" @@ -25,15 +33,20 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/remotecommand" ) type K8sClient struct { - Clientset *kubernetes.Clientset - namespace []string - config *rest.Config - host container.Host + Clientset kubernetes.Interface + DynamicClient dynamic.Interface + restMapper meta.RESTMapper + namespace []string + config *rest.Config + host container.Host + ownerCacheMu sync.Mutex + ownerCache map[string]ownerLookupResult } func NewK8sClient(namespace []string) (*K8sClient, error) { @@ -68,6 +81,14 @@ func NewK8sClient(namespace []string) (*K8sClient, error) { if err != nil { return nil, err } + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return nil, err + } + discoveryClient, err := discovery.NewDiscoveryClientForConfig(config) + if err != nil { + return nil, err + } nodes, err := clientset.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) if err != nil { @@ -79,17 +100,35 @@ func NewK8sClient(namespace []string) (*K8sClient, error) { node := nodes.Items[0] return &K8sClient{ - Clientset: clientset, - namespace: namespace, - config: config, + Clientset: clientset, + DynamicClient: dynamicClient, + restMapper: restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(discoveryClient)), + namespace: namespace, + config: config, host: container.Host{ ID: node.Status.NodeInfo.MachineID, Name: node.Name, }, + ownerCache: make(map[string]ownerLookupResult), }, nil } -func podToContainers(pod *corev1.Pod) []container.Container { +type k8sOwner struct { + APIVersion string + Kind string + Namespace string + Name string + UID string + TypeKey string + Key string +} + +type ownerLookupResult struct { + ownerReferences []metav1.OwnerReference + found bool +} + +func (k *K8sClient) podToContainers(ctx context.Context, pod *corev1.Pod) []container.Container { started := time.Time{} if pod.Status.StartTime != nil { started = pod.Status.StartTime.Time @@ -101,12 +140,32 @@ func podToContainers(pod *corev1.Pod) []container.Container { labels[k] = v } labels["namespace"] = pod.Namespace + labels["@k8s.namespace"] = pod.Namespace - // Add owner reference if present - if len(pod.OwnerReferences) > 0 { - owner := pod.OwnerReferences[0] - labels["owner.kind"] = owner.Kind - labels["owner.name"] = owner.Name + owners := k.resolveOwnerChain(ctx, pod.Namespace, pod.OwnerReferences) + if len(owners) > 0 { + labels["owner.kind"] = owners[0].Kind + labels["owner.name"] = owners[0].Name + labels["owner.key"] = owners[0].Key + labels["k8s.owner.count"] = fmt.Sprintf("%d", len(owners)) + labels["@k8s.owner.count"] = fmt.Sprintf("%d", len(owners)) + } + for i, owner := range owners { + prefix := fmt.Sprintf("k8s.owner.%d.", i) + syntheticPrefix := fmt.Sprintf("@k8s.owner.%d.", i) + labels[prefix+"apiVersion"] = owner.APIVersion + labels[prefix+"kind"] = owner.Kind + labels[prefix+"namespace"] = owner.Namespace + labels[prefix+"name"] = owner.Name + labels[prefix+"uid"] = owner.UID + labels[prefix+"key"] = owner.Key + labels[syntheticPrefix+"apiVersion"] = owner.APIVersion + labels[syntheticPrefix+"kind"] = owner.Kind + labels[syntheticPrefix+"namespace"] = owner.Namespace + labels[syntheticPrefix+"name"] = owner.Name + labels[syntheticPrefix+"uid"] = owner.UID + labels[syntheticPrefix+"key"] = owner.Key + labels[ownerMembershipLabel(owner.Key)] = "true" } var containers []container.Container @@ -129,10 +188,217 @@ func podToContainers(pod *corev1.Pod) []container.Container { return containers } +func (k *K8sClient) resolveOwnerChain(ctx context.Context, namespace string, refs []metav1.OwnerReference) []k8sOwner { + owners := make([]k8sOwner, 0) + seen := make(map[string]struct{}) + + for len(refs) > 0 { + ref := ownerReferenceToFollow(refs) + if isNodeOwnerReference(ref) { + break + } + owner := newK8sOwner(namespace, ref) + if _, ok := seen[owner.cacheKey()]; ok { + break + } + seen[owner.cacheKey()] = struct{}{} + owners = append(owners, owner) + + next, ok := k.lookupOwnerReferences(ctx, owner) + if !ok { + break + } + refs = next + } + + return owners +} + +func ownerReferenceToFollow(refs []metav1.OwnerReference) metav1.OwnerReference { + for _, ref := range refs { + if ref.Controller != nil && *ref.Controller { + return ref + } + } + return refs[0] +} + +func isNodeOwnerReference(ref metav1.OwnerReference) bool { + return ref.APIVersion == "v1" && ref.Kind == "Node" +} + +func newK8sOwner(namespace string, ref metav1.OwnerReference) k8sOwner { + typeKey := ownerTypeKey(ref.APIVersion, ref.Kind) + // "~" is URL-safe and not allowed in Kubernetes resource names/namespaces, + // so owner route keys stay readable without colliding with real names. + key := fmt.Sprintf("%s~%s~%s", typeKey, namespace, ref.Name) + return k8sOwner{ + APIVersion: ref.APIVersion, + Kind: ref.Kind, + Namespace: namespace, + Name: ref.Name, + UID: string(ref.UID), + TypeKey: typeKey, + Key: key, + } +} + +func (o k8sOwner) cacheKey() string { + return fmt.Sprintf("%s/%s/%s/%s/%s", o.APIVersion, o.Kind, o.Namespace, o.Name, o.UID) +} + +func ownerMembershipLabel(key string) string { + return "@k8s.owner.key." + base64.RawURLEncoding.EncodeToString([]byte(key)) +} + +func ownerTypeKey(apiVersion, kind string) string { + if isKnownK8sOwnerType(apiVersion, kind) { + return kind + } + return strings.ReplaceAll(apiVersion, "/", "~") + "~" + kind +} + +func isKnownK8sOwnerType(apiVersion, kind string) bool { + switch apiVersion + "/" + kind { + case "apps/v1/Deployment", + "apps/v1/ReplicaSet", + "apps/v1/DaemonSet", + "apps/v1/StatefulSet", + "batch/v1/Job", + "batch/v1/CronJob", + "v1/Pod", + "v1/Service", + "v1/ConfigMap", + "v1/Secret": + return true + default: + return false + } +} + +func (k *K8sClient) lookupOwnerReferences(ctx context.Context, owner k8sOwner) ([]metav1.OwnerReference, bool) { + cacheKey := owner.cacheKey() + k.ownerCacheMu.Lock() + if k.ownerCache == nil { + k.ownerCache = make(map[string]ownerLookupResult) + } + if result, ok := k.ownerCache[cacheKey]; ok { + k.ownerCacheMu.Unlock() + return result.ownerReferences, result.found + } + k.ownerCacheMu.Unlock() + + refs, ok, cacheable := k.fetchOwnerReferences(ctx, owner) + + if cacheable { + k.ownerCacheMu.Lock() + k.ownerCache[cacheKey] = ownerLookupResult{ownerReferences: refs, found: ok} + k.ownerCacheMu.Unlock() + } + + return refs, ok +} + +func (k *K8sClient) fetchOwnerReferences(ctx context.Context, owner k8sOwner) ([]metav1.OwnerReference, bool, bool) { + if k.DynamicClient == nil || k.restMapper == nil { + return nil, false, false + } + + groupVersion, err := schema.ParseGroupVersion(owner.APIVersion) + if err != nil { + log.Debug().Err(err).Str("owner", owner.Key).Msg("failed to parse owner apiVersion") + return nil, false, false + } + + mapping, err := k.restMapper.RESTMapping(groupVersion.WithKind(owner.Kind).GroupKind(), groupVersion.Version) + if err != nil { + log.Debug().Err(err).Str("owner", owner.Key).Msg("failed to map owner resource") + return nil, false, false + } + + var resource dynamic.ResourceInterface + if mapping.Scope.Name() != meta.RESTScopeNameRoot { + resource = k.DynamicClient.Resource(mapping.Resource).Namespace(owner.Namespace) + } else { + resource = k.DynamicClient.Resource(mapping.Resource) + } + + obj, err := resource.Get(ctx, owner.Name, metav1.GetOptions{}) + if err != nil { + log.Debug().Err(err).Str("owner", owner.Key).Msg("failed to fetch owner resource") + if ctx.Err() != nil { + return nil, false, false + } + return nil, false, apierrors.IsNotFound(err) || apierrors.IsForbidden(err) + } + + return obj.GetOwnerReferences(), true, true +} + +func splitK8sFilters(labels container.ContainerLabels) (container.ContainerLabels, container.ContainerLabels) { + podLabels := make(container.ContainerLabels) + metadataLabels := make(container.ContainerLabels) + for key, values := range labels { + if isK8sMetadataLabel(key) || !isValidK8sLabelKey(key) { + metadataLabels[key] = values + } else { + podLabels[key] = values + } + } + return podLabels, metadataLabels +} + +func isK8sMetadataLabel(key string) bool { + if strings.HasPrefix(key, "@k8s.") { + return true + } + return key == "namespace" || + key == "owner.kind" || + key == "owner.name" || + key == "owner.key" || + strings.HasPrefix(key, "k8s.owner.") +} + +var ( + k8sLabelNamePattern = regexp.MustCompile(`^[A-Za-z0-9]([A-Za-z0-9_.-]{0,61}[A-Za-z0-9])?$`) + k8sLabelPrefixPattern = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`) +) + +func isValidK8sLabelKey(key string) bool { + prefix, name, hasPrefix := strings.Cut(key, "/") + if !hasPrefix { + name = prefix + } else if len(prefix) == 0 || len(prefix) > 253 || !k8sLabelPrefixPattern.MatchString(prefix) { + return false + } + return len(name) <= 63 && k8sLabelNamePattern.MatchString(name) +} + +func matchesContainerLabels(labels map[string]string, filters container.ContainerLabels) bool { + for key, values := range filters { + value, ok := labels[key] + if !ok { + return false + } + matched := false + for _, expected := range values { + if value == expected { + matched = true + break + } + } + if !matched { + return false + } + } + return true +} + func (k *K8sClient) ListContainers(ctx context.Context, labels container.ContainerLabels) ([]container.Container, error) { + podLabels, metadataLabels := splitK8sFilters(labels) selector := "" - if labels.Exists() { - for key, values := range labels { + if podLabels.Exists() { + for key, values := range podLabels { for _, value := range values { if selector != "" { selector += "," @@ -149,7 +415,12 @@ func (k *K8sClient) ListContainers(ctx context.Context, labels container.Contain } var containers []container.Container for _, pod := range pods.Items { - containers = append(containers, podToContainers(&pod)...) + for _, c := range k.podToContainers(ctx, &pod) { + if metadataLabels.Exists() && !matchesContainerLabels(c.Labels, metadataLabels) { + continue + } + containers = append(containers, c) + } } return lo.T2[[]container.Container, error](containers, nil) }) @@ -201,7 +472,7 @@ func (k *K8sClient) FindContainer(ctx context.Context, id string) (container.Con return container.Container{}, err } - for _, c := range podToContainers(pod) { + for _, c := range k.podToContainers(ctx, pod) { if c.ID == id { return c, nil } @@ -274,7 +545,7 @@ func (k *K8sClient) ContainerEvents(ctx context.Context, ch chan<- container.Con name = "update" } - for _, c := range podToContainers(pod) { + for _, c := range k.podToContainers(ctx, pod) { ch <- container.ContainerEvent{ Name: name, ActorID: c.ID, diff --git a/internal/k8s/client_test.go b/internal/k8s/client_test.go new file mode 100644 index 00000000..2c970501 --- /dev/null +++ b/internal/k8s/client_test.go @@ -0,0 +1,293 @@ +package k8s + +import ( + "context" + "errors" + "testing" + + "github.com/amir20/dozzle/internal/container" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8sfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +func TestPodToContainersAddsOwnerChainLabels(t *testing.T) { + client := newTestK8sClient(t, + &appsv1.ReplicaSet{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "ReplicaSet"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "api-6f88b977f4", + UID: types.UID("rs-uid"), + OwnerReferences: []metav1.OwnerReference{ + {APIVersion: "apps/v1", Kind: "Deployment", Name: "api", UID: types.UID("deploy-uid")}, + }, + }, + }, + &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "api", UID: types.UID("deploy-uid")}, + }, + ) + + containers := client.podToContainers(t.Context(), podWithOwner()) + require.Len(t, containers, 1) + + labels := containers[0].Labels + assert.Equal(t, "default", labels["namespace"]) + assert.Equal(t, "default", labels["@k8s.namespace"]) + assert.Equal(t, "ReplicaSet", labels["owner.kind"]) + assert.Equal(t, "api-6f88b977f4", labels["owner.name"]) + assert.Equal(t, "ReplicaSet~default~api-6f88b977f4", labels["owner.key"]) + assert.Equal(t, "2", labels["k8s.owner.count"]) + assert.Equal(t, "2", labels["@k8s.owner.count"]) + + assert.Equal(t, "ReplicaSet", labels["k8s.owner.0.kind"]) + assert.Equal(t, "ReplicaSet", labels["@k8s.owner.0.kind"]) + assert.Equal(t, "api-6f88b977f4", labels["k8s.owner.0.name"]) + assert.Equal(t, "ReplicaSet~default~api-6f88b977f4", labels["k8s.owner.0.key"]) + assert.Equal(t, "Deployment", labels["k8s.owner.1.kind"]) + assert.Equal(t, "Deployment", labels["@k8s.owner.1.kind"]) + assert.Equal(t, "api", labels["k8s.owner.1.name"]) + assert.Equal(t, "Deployment~default~api", labels["k8s.owner.1.key"]) + assert.Equal(t, "true", labels[ownerMembershipLabel("ReplicaSet~default~api-6f88b977f4")]) + assert.Equal(t, "true", labels[ownerMembershipLabel("Deployment~default~api")]) +} + +func TestPodToContainersStopsOwnerChainWhenOwnerCannotBeFetched(t *testing.T) { + client := newTestK8sClient(t) + + containers := client.podToContainers(t.Context(), podWithOwner()) + require.Len(t, containers, 1) + + labels := containers[0].Labels + assert.Equal(t, "1", labels["k8s.owner.count"]) + assert.Equal(t, "ReplicaSet", labels["k8s.owner.0.kind"]) + assert.Equal(t, "api-6f88b977f4", labels["k8s.owner.0.name"]) + assert.Empty(t, labels["k8s.owner.1.kind"]) +} + +func TestPodToContainersDoesNotAddNodeOwner(t *testing.T) { + client := newTestK8sClient(t) + pod := podWithOwner() + pod.OwnerReferences = []metav1.OwnerReference{ + {APIVersion: "v1", Kind: "Node", Name: "node-1", UID: types.UID("node-uid")}, + } + + containers := client.podToContainers(t.Context(), pod) + require.Len(t, containers, 1) + + labels := containers[0].Labels + assert.Empty(t, labels["owner.kind"]) + assert.Empty(t, labels["k8s.owner.count"]) + assert.Empty(t, labels["@k8s.owner.count"]) +} + +func TestPodToContainersStopsBeforeNodeOwnerInChain(t *testing.T) { + client := newTestK8sClient(t, + &appsv1.ReplicaSet{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "ReplicaSet"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "api-6f88b977f4", + UID: types.UID("rs-uid"), + OwnerReferences: []metav1.OwnerReference{ + {APIVersion: "v1", Kind: "Node", Name: "node-1", UID: types.UID("node-uid")}, + }, + }, + }, + ) + + containers := client.podToContainers(t.Context(), podWithOwner()) + require.Len(t, containers, 1) + + labels := containers[0].Labels + assert.Equal(t, "1", labels["k8s.owner.count"]) + assert.Equal(t, "ReplicaSet", labels["k8s.owner.0.kind"]) + assert.Empty(t, labels["k8s.owner.1.kind"]) +} + +func TestListContainersAppliesSyntheticOwnerFiltersAfterPodList(t *testing.T) { + client := newTestK8sClient(t, + &appsv1.ReplicaSet{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "ReplicaSet"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "api-6f88b977f4", + UID: types.UID("rs-uid"), + OwnerReferences: []metav1.OwnerReference{ + {APIVersion: "apps/v1", Kind: "Deployment", Name: "api", UID: types.UID("deploy-uid")}, + }, + }, + }, + &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "api", UID: types.UID("deploy-uid")}, + }, + ) + client.Clientset = k8sfake.NewSimpleClientset(podWithOwner()) + + containers, err := client.ListContainers(t.Context(), container.ContainerLabels{ + "app": {"api"}, + ownerMembershipLabel("Deployment~default~api"): {"true"}, + }) + + require.NoError(t, err) + require.Len(t, containers, 1) + assert.Equal(t, "default:api-6f88b977f4-pod:api", containers[0].ID) +} + +func TestSplitK8sFiltersKeepsInvalidSyntheticKeysOutOfPodSelector(t *testing.T) { + podLabels, metadataLabels := splitK8sFilters(container.ContainerLabels{ + "app": {"api"}, + "team.example.com/component": {"backend"}, + "@k8s.namespace": {"default"}, + "@k8s.owner.key.abc123": {"true"}, + "not:a:kubernetes:label:key": {"value"}, + "k8s.owner.key.legacyabc123": {"true"}, + }) + + assert.Equal(t, container.ContainerLabels{ + "app": {"api"}, + "team.example.com/component": {"backend"}, + }, podLabels) + assert.Equal(t, container.ContainerLabels{ + "@k8s.namespace": {"default"}, + "@k8s.owner.key.abc123": {"true"}, + "not:a:kubernetes:label:key": {"value"}, + "k8s.owner.key.legacyabc123": {"true"}, + }, metadataLabels) +} + +func TestOwnerTypeKeyUsesFullAPINameForUnknownTypes(t *testing.T) { + assert.Equal(t, "Deployment", ownerTypeKey("apps/v1", "Deployment")) + assert.Equal(t, "argoproj.io~v1alpha1~Rollout", ownerTypeKey("argoproj.io/v1alpha1", "Rollout")) + + ref := metav1.OwnerReference{APIVersion: "argoproj.io/v1alpha1", Kind: "Rollout", Name: "api"} + assert.Equal(t, "argoproj.io~v1alpha1~Rollout~default~api", newK8sOwner("default", ref).Key) +} + +func TestOwnerReferenceToFollowPrefersController(t *testing.T) { + controller := true + + ref := ownerReferenceToFollow([]metav1.OwnerReference{ + {Kind: "ConfigMap", Name: "sidecar-config"}, + {Kind: "ReplicaSet", Name: "api-6f88b977f4", Controller: &controller}, + }) + + assert.Equal(t, "ReplicaSet", ref.Kind) + assert.Equal(t, "api-6f88b977f4", ref.Name) +} + +func TestLookupOwnerReferencesDoesNotCacheTransientFailures(t *testing.T) { + client := newTestK8sClient(t) + dynamicClient := client.DynamicClient.(*dynamicfake.FakeDynamicClient) + calls := 0 + dynamicClient.PrependReactor("get", "replicasets", func(action k8stesting.Action) (bool, runtime.Object, error) { + calls++ + return true, nil, context.Canceled + }) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + _, ok := client.lookupOwnerReferences(ctx, replicaSetOwner()) + assert.False(t, ok) + + _, ok = client.lookupOwnerReferences(ctx, replicaSetOwner()) + assert.False(t, ok) + assert.Equal(t, 2, calls) +} + +func TestLookupOwnerReferencesCachesRealNegatives(t *testing.T) { + tests := []struct { + name string + err error + }{ + { + name: "forbidden", + err: apierrors.NewForbidden(schema.GroupResource{Group: "apps", Resource: "replicasets"}, "api-6f88b977f4", errors.New("denied")), + }, + { + name: "not found", + err: apierrors.NewNotFound(schema.GroupResource{Group: "apps", Resource: "replicasets"}, "api-6f88b977f4"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := newTestK8sClient(t) + dynamicClient := client.DynamicClient.(*dynamicfake.FakeDynamicClient) + calls := 0 + dynamicClient.PrependReactor("get", "replicasets", func(action k8stesting.Action) (bool, runtime.Object, error) { + calls++ + return true, nil, tt.err + }) + + _, ok := client.lookupOwnerReferences(t.Context(), replicaSetOwner()) + assert.False(t, ok) + + _, ok = client.lookupOwnerReferences(t.Context(), replicaSetOwner()) + assert.False(t, ok) + assert.Equal(t, 1, calls) + }) + } +} + +func podWithOwner() *corev1.Pod { + return &corev1.Pod{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "api-6f88b977f4-pod", + Labels: map[string]string{"app": "api"}, + OwnerReferences: []metav1.OwnerReference{ + {APIVersion: "apps/v1", Kind: "ReplicaSet", Name: "api-6f88b977f4", UID: types.UID("rs-uid")}, + }, + }, + Spec: corev1.PodSpec{ + NodeName: "node-1", + Containers: []corev1.Container{ + {Name: "api", Image: "example/api:latest"}, + }, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } +} + +func replicaSetOwner() k8sOwner { + return newK8sOwner("default", metav1.OwnerReference{ + APIVersion: "apps/v1", + Kind: "ReplicaSet", + Name: "api-6f88b977f4", + UID: types.UID("rs-uid"), + }) +} + +func newTestK8sClient(t *testing.T, objects ...runtime.Object) *K8sClient { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, appsv1.AddToScheme(scheme)) + + mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{appsv1.SchemeGroupVersion, corev1.SchemeGroupVersion}) + mapper.Add(appsv1.SchemeGroupVersion.WithKind("ReplicaSet"), meta.RESTScopeNamespace) + mapper.Add(appsv1.SchemeGroupVersion.WithKind("Deployment"), meta.RESTScopeNamespace) + + return &K8sClient{ + Clientset: k8sfake.NewSimpleClientset(), + DynamicClient: dynamicfake.NewSimpleDynamicClient(scheme, objects...), + restMapper: mapper, + namespace: []string{"default"}, + ownerCache: make(map[string]ownerLookupResult), + } +}