kubelet gc 源码分析

代码 kubernetes 1.26.15

问题

混部机子批量节点NotReady(十几个,丫的重大故障),报错为:

意思就是 rpc 超了,节点下有太多 PodSandBox,crictl ps -a 一看有1400多个。。。大量exited的容器没有被删掉,累积起来超过了rpc限制。

PodSandBox 泄漏,crictl pods 可以看到大量同名但是 pod id不同的sanbox,几个月了kubelet并不主动删除

crictl pods
crictl inspectp <pod id>
crictl ps -a | grep <pod-id>
crictl logs <container-id>

kubelet通过cri和containerd进行交互。crictl也可以通过cri规范和containerd交互
crictl 是 CRI(规范) 兼容的容器运行时命令行接口,可以使用它来检查和调试 k8s node节点上的容器运行时和应用程序。

kubernetes 垃圾回收(Garbage Collection)机制由kubelet完成,kubelet定期清理不再使用的容器和镜像,每分钟进行一次容器的GC,每五分钟进行一次镜像的GC

代码逻辑

1. 开始GC

pkg/kubelet/kubelet.go:1352,开始GC func (kl *Kubelet) StartGarbageCollection()

pkg/kubelet/kuberuntime/kuberuntime_gc.go:409

// GarbageCollect removes dead containers using the specified container gc policy.
// Note that gc policy is not applied to sandboxes. Sandboxes are only removed when they are
// not ready and containing no containers.
//
// GarbageCollect consists of the following steps:
// * gets evictable containers which are not active and created more than gcPolicy.MinAge ago.
// * removes oldest dead containers for each pod by enforcing gcPolicy.MaxPerPodContainer.
// * removes oldest dead containers by enforcing gcPolicy.MaxContainers.
// * gets evictable sandboxes which are not ready and contains no containers.
// * removes evictable sandboxes.
func (cgc *containerGC) GarbageCollect(ctx context.Context, gcPolicy kubecontainer.GCPolicy, allSourcesReady bool, evictNonDeletedPods bool) error {
	errors := []error{}
	// Remove evictable containers
	if err := cgc.evictContainers(ctx, gcPolicy, allSourcesReady, evictNonDeletedPods); err != nil {
		errors = append(errors, err)
	}

	// Remove sandboxes with zero containers
	if err := cgc.evictSandboxes(ctx, evictNonDeletedPods); err != nil {
		errors = append(errors, err)
	}

	// Remove pod sandbox log directory
	if err := cgc.evictPodLogsDirectories(ctx, allSourcesReady); err != nil {
		errors = append(errors, err)
	}
	return utilerrors.NewAggregate(errors)
}

2. 驱逐容器 evictContainers

  1. 获取 evictUnits pkg/kubelet/kuberuntime/kuberuntime_gc.go:187
    列出所有容器,容器中状态为 ContainerState_CONTAINER_RUNNING 和 container.CreatedAt 小于 minAge 直接跳过。
    其余添加到 evictUnits
map[evictUnit][]containerGCInfo

// evictUnit is considered for eviction as units of (UID, container name) pair.  
type evictUnit struct {  
    // UID of the pod.  
    uid types.UID  
    // Name of the container in the pod.  
    name string  
}

// containerGCInfo is the internal information kept for containers being considered for GC.
type containerGCInfo struct {
	// The ID of the container.
	id string
	// The name of the container.
	name string
	// Creation time for the container.
	createTime time.Time
	// If true, the container is in unknown state. Garbage collector should try
	// to stop containers before removal.
	unknown bool
}
  1. 删除容器逻辑
// evict all containers that are evictable
func (cgc *containerGC) evictContainers(ctx context.Context, gcPolicy kubecontainer.GCPolicy, allSourcesReady bool, evictNonDeletedPods bool) error {
	// Separate containers by evict units.
	evictUnits, err := cgc.evictableContainers(ctx, gcPolicy.MinAge)
	if err != nil {
		return err
	}

	// Remove deleted pod containers if all sources are ready.
	// 如果pod已经不存在了,那么就删除其中的所有容器。
	if allSourcesReady {
		for key, unit := range evictUnits {
			if cgc.podStateProvider.ShouldPodContentBeRemoved(key.uid) || (evictNonDeletedPods && cgc.podStateProvider.ShouldPodRuntimeBeRemoved(key.uid)) {
				cgc.removeOldestN(ctx, unit, len(unit)) // Remove all.
				delete(evictUnits, key)
			}
		}
	}

	// Enforce max containers per evict unit.
	// 执行 GC 策略,保证每个 POD 最多只能保存 MaxPerPodContainer 个已经退出的容器
	if gcPolicy.MaxPerPodContainer >= 0 {
		cgc.enforceMaxContainersPerEvictUnit(ctx, evictUnits, gcPolicy.MaxPerPodContainer)
	}

	// Enforce max total number of containers.
	// 执行 GC 策略,保证节点上最多有 MaxContainers 个已经退出的容器
	if gcPolicy.MaxContainers >= 0 && evictUnits.NumContainers() > gcPolicy.MaxContainers {
		// Leave an equal number of containers per evict unit (min: 1).
		numContainersPerEvictUnit := gcPolicy.MaxContainers / evictUnits.NumEvictUnits()
		if numContainersPerEvictUnit < 1 {
			numContainersPerEvictUnit = 1
		}
		cgc.enforceMaxContainersPerEvictUnit(ctx, evictUnits, numContainersPerEvictUnit)

		// If we still need to evict, evict oldest first.
		numContainers := evictUnits.NumContainers()
		if numContainers > gcPolicy.MaxContainers {
			flattened := make([]containerGCInfo, 0, numContainers)
			for key := range evictUnits {
				flattened = append(flattened, evictUnits[key]...)
			}
			sort.Sort(byCreated(flattened))

			cgc.removeOldestN(ctx, flattened, numContainers-gcPolicy.MaxContainers)
		}
	}
	return nil
}
  1. 移除该pod uid下的所有容器
    pkg/kubelet/kuberuntime/kuberuntime_gc.go:126
// removeOldestN removes the oldest toRemove containers and returns the resulting slice.
func (cgc *containerGC) removeOldestN(ctx context.Context, containers []containerGCInfo, toRemove int) []containerGCInfo {
	// Remove from oldest to newest (last to first).
	numToKeep := len(containers) - toRemove
	if numToKeep > 0 {
		sort.Sort(byCreated(containers))
	}
	for i := len(containers) - 1; i >= numToKeep; i-- {
		if containers[i].unknown {
			// Containers in known state could be running, we should try
			// to stop it before removal.
			id := kubecontainer.ContainerID{
				Type: cgc.manager.runtimeName,
				ID:   containers[i].id,
			}
			message := "Container is in unknown state, try killing it before removal"
			if err := cgc.manager.killContainer(ctx, nil, id, containers[i].name, message, reasonUnknown, nil); err != nil {
				klog.ErrorS(err, "Failed to stop container", "containerID", containers[i].id)
				continue
			}
		}
		if err := cgc.manager.removeContainer(ctx, containers[i].id); err != nil {
			klog.ErrorS(err, "Failed to remove container", "containerID", containers[i].id)
		}
	}

	// Assume we removed the containers so that we're not too aggressive.
	return containers[:numToKeep]
}

3. 驱逐sandbox evictSandboxes

pkg/kubelet/kuberuntime/kuberuntime_gc.go:276
移除所有可驱逐的沙箱。可驱逐的沙箱必须满足以下要求: 1.未处于就绪状态2.不包含任何容器。3.属于不存在的 (即,已经移除的) pod,或者不是该pod的最近创建的沙箱。

原因分析

目前现象是 crictl pods 可以看到大量同名但是 pod id不同的sanbox。 根据 3 点要求

  1. sanbox notReady 满足
  2. 不包容任何容器 不满足
  3. 不是该pod的最近创建的沙箱 满足

因此sandbox 删不掉的原因是 sandbox下的容器未被删除

容器异常退出后,根据重启策略 restartPolicy: Always pod 会不断重启,直到 超过时限失败。

Pod 的垃圾收集

https://kubernetes.io/zh-cn/docs/concepts/workloads/pods/pod-lifecycle/#pod-garbage-collection

对于已失败的 Pod 而言,对应的 API 对象仍然会保留在集群的 API 服务器上, 直到用户或者控制器进程显式地将其删除。

Pod 的垃圾收集器(PodGC)是控制平面的控制器,它会在 Pod 个数超出所配置的阈值 (根据 kube-controller-manager 的 terminated-pod-gc-threshold 设置 默认值:12500)时删除已终止的 Pod(阶段值为 Succeeded 或 Failed)。 这一行为会避免随着时间演进不断创建和终止 Pod 而引起的资源泄露问题。

容器什么时候删除

上面是pod纬度,但是我们的现象是容器删不掉,所以并不是原因,继续看代码 😢

经过大佬的实验验证,对于失败的 容器,只会保留一个失败的现场,多余的会GC掉,和 问题现场一致

容器 GC 虽然有利于空间和性能,但是删除容器也会导致错误现场被清理,不利于 debug 和错误定位,因此不建议把所有退出的容器都删除。
cmd/kubelet/app/options/options.go:183

// Maximum number of old instances of containers to retain globally.  Each container takes up some disk space. To disable, set to a negative number.
// 我们可以设置这个值兜底
MaxContainerCount:       -1,
MinimumGCAge:   metav1.Duration{Duration: 0},
// 每个 container 最终可以保存多少个已经结束的容器,默认是 1,设置为负数表示不做限制
MaxPerPodContainerCount: 1,

再看上面容器GC代码

// 如果pod已经不存在了,那么就删除其中的所有容器。
....

// 执行 GC 策略,保证每个 POD 最多只能保存 MaxPerPodContainerCount 个已经退出的容器
// MaxPerPodContainerCount 默认值为1,对应保留一个失败的现场
if gcPolicy.MaxPerPodContainer >= 0 {
	cgc.enforceMaxContainersPerEvictUnit(ctx, evictUnits, gcPolicy.MaxPerPodContainer)
}

// 保证节点上最多有 MaxContainerCount 个已经退出的容器
// MaxContainerCount 默认值为 -1 不限制,我们可以设置一个兜底
if gcPolicy.MaxContainers >= 0 && evictUnits.NumContainers() > gcPolicy.MaxContainers {
......
}

总结,容器失败,会保留一个现场不GC,导致越来越多失败的容器存在,最后容器过多,导致rpc传输超过限制,整个节点崩掉

解决方案

粗暴手删

  1. crictl 超出限制,不能正常工作时
#!/bin/bash

# 列出所有在 k8s.io 命名空间下的容器
containers=$(ctr -n k8s.io c list -q)

# 遍历容器 ID 并删除每一个容器
for container in $containers; do
    echo "Deleting container: $container"
    ctr -n k8s.io c rm "$container"
done

echo "All containers have been removed."

systemctl restart containerd
systemctl restart kubelet
  1. crictl 可以正常工作,删除失败容器,sandbox会1min后,自动gc
#!/bin/bash

# 获取所有Exited状态的容器ID
exited_containers=$(crictl ps -a | grep Exited | grep months | awk '{print $1}')

# 检查是否有Exited容器需要删除
if [ -z "$exited_containers" ]; then
    echo "没有找到任何处于Exited状态的容器。"
else
    # 遍历所有Exited状态的容器ID,并删除它们
    for container in $exited_containers; do
        echo "正在删除容器: $container"
        crictl rm $container
        if [ $? -eq 0 ]; then
            echo "容器 $container 已成功删除。"
        else
            echo "删除容器 $container 失败。"
        fi
    done
fi

优雅解决

  • 配置 maximum-dead-containers 兜底,默认-1,节点虽然限制每一个容器的失败实例为1,但是总的失败实例不做限制。
  • 使用operator 或则 npd 进行监控,太多,则和诊断中心联动删除(倒序删除最老的50个exited,滚动删除)

grpc ??

问题的本质是 grpc 超标,我们是否可以直接改 grpc 的 received message larger than max (4198720 vs. 4194304)
让我们看一下 containerd 的源码

kubelet 与 cri server 交互 pkg/cri/server/sandbox_list.go:29

func (c *criService) ListPodSandbox(ctx context.Context, r *runtime.ListPodSandboxRequest) (*runtime.ListPodSandboxResponse, error)

pkg/cri/cri.go:100 s, err := server.NewCRIService(c, client)
client 是New返回一个新的containerd客户端,该客户端连接到地址提供的containerd实例,代码很简单,如果 address!="" 设置 grpc 大小为 16m,如果为空,grpc 大小为默认值 4m

// New returns a new containerd client that is connected to the containerd
// instance provided by address
func New(address string, opts ...ClientOpt) (*Client, error) {
// .......
	c := &Client{  
	    defaultns: copts.defaultns,  
	}
// .......
	if address != "" {
// .......
		gopts := []grpc.DialOption{
			grpc.WithBlock(),
			grpc.WithTransportCredentials(insecure.NewCredentials()),
			grpc.FailOnNonTempDialError(true),
			grpc.WithConnectParams(connParams),
			grpc.WithContextDialer(dialer.ContextDialer),
			grpc.WithReturnConnectionError(),
		}
		if len(copts.dialOptions) > 0 {
			gopts = copts.dialOptions
		}
		// 设置 grpc 最大值 16m
		gopts = append(gopts, grpc.WithDefaultCallOptions(
			grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize),
			grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)))
//........

		connector := func() (*grpc.ClientConn, error) {
			ctx, cancel := context.WithTimeout(context.Background(), copts.timeout)
			defer cancel()
			conn, err := grpc.DialContext(ctx, dialer.DialAddress(address), gopts...)
			if err != nil {
				return nil, fmt.Errorf("failed to dial %q: %w", address, err)
			}
			return conn, nil
		}
		conn, err := connector()
		if err != nil {
			return nil, err
		}
		c.conn, c.connector = conn, connector
	}
//........
	return c, nil
}

但是在 pkg/cri/cri.go:62 初始化 cri 插件时,address 为空,grpc 大小为默认值 4m

client, err := containerd.New(
	"",
	containerd.WithDefaultNamespace(constants.K8sContainerdNamespace),
	containerd.WithDefaultPlatform(platforms.Default()),
	containerd.WithServices(servicesOpts...),
)

contianerd 相关issue

社区目前的方案就是设置 maximum-dead-containers 兜底
https://github.com/kubernetes/kubernetes/issues/63858

最终方案

  • 配置 pod status NotReady > 50 电话告警
    increase(problem_counter{app="ops.paas.npd",reason="lots of pods notReady"}[60m]) > 0
  • 配置 maximum-dead-containers=200

后续改进

死亡容器保持一个不删,只是原因,后续发现sandbox 的 GC 速度很慢 (看日志 GC 一个sandbox 5s 左右)
removeSandBox 会调用 stopSandBox,if sandbox.NetNS != nil 会 teardownPodNetwork ,这里会和 cni 插件交互,因为 cni-adaptor 重复删除网络又报错,GC 就失败了,极大影响 GC 效率,后续需要对 cni 插件进行优化

删除网络操作

cni 删除操作,因改为尽量删除
https://github.com/containernetworking/plugins/issues/210
vendor/github.com/containerd/go-cni/cni.go:234

// Remove removes the network config from the namespace
func (c *libcni) Remove(ctx context.Context, id string, path string, opts ...NamespaceOpts) error {
	if err := c.Status(); err != nil {
		return err
	}
	ns, err := newNamespace(id, path, opts...)
	if err != nil {
		return err
	}
	for _, network := range c.Networks() {
		if err := network.Remove(ctx, ns); err != nil {
			// Based on CNI spec v0.7.0, empty network namespace is allowed to
			// do best effort cleanup. However, it is not handled consistently
			// right now:
			// https://github.com/containernetworking/plugins/issues/210
			// TODO(random-liu): Remove the error handling when the issue is
			// fixed and the CNI spec v0.6.0 support is deprecated.
			// NOTE(claudiub): Some CNIs could return a "not found" error, which could mean that
			// it was already deleted.
			if (path == "" && strings.Contains(err.Error(), "no such file or directory")) || strings.Contains(err.Error(), "not found") {
				continue
			}
			return err
		}
	}
	return nil
}

热门相关:迷途   孤单时能弥补的大男人   你老公都不知道你那轻浮的裤裆   和公司女职员做爱的丈夫   威震八荒