Docs: Containerd

Sun, 22 May 2022 15:05:21 +0800

安装部署

在githbub官方地址：https://github.com/containerd/containerd/releases，选择需要的版本下载。

containerd-{{ containerd_version }}、cri-containerd-{{ containerd_version }} 有以上两种二进制包，“cri_containerd"区别于"containerd-"，在于它集合了"runc"等工具。 ”{{ containerd_version }}“是containerd版本，以下均以"1.5.11"为示例。

下载二进制

https://github.com/containerd/containerd/releases/download/v{{ containerd_version }}/cri-containerd-cni-{{ containerd_version }}-linux-amd64.tar.gz

解压部署

tar xzvf cri-containerd-cni-{{ containerd_version }}-linux-amd64.tar.gz /

解压”/“目录，二进制路径“/usr/local/bin”

“containerd-“与"cri-containerd-“安装包内容

“containerd-*”

bin/
bin/containerd-shim-runc-v2
bin/containerd-shim
bin/ctr
bin/containerd-shim-runc-v1
bin/containerd

“cri-containerd-*”

etc/
etc/cni/
etc/cni/net.d/
etc/cni/net.d/10-containerd-net.conflist
etc/systemd/
etc/systemd/system/
etc/systemd/system/containerd.service
etc/crictl.yaml
usr/
usr/local/
usr/local/bin/
usr/local/bin/containerd-shim-runc-v2
usr/local/bin/containerd-shim
usr/local/bin/crictl
usr/local/bin/ctr
usr/local/bin/containerd-shim-runc-v1
usr/local/bin/containerd
usr/local/bin/ctd-decoder
usr/local/bin/critest
usr/local/bin/containerd-stress
usr/local/sbin/
usr/local/sbin/runc
opt/
opt/cni/
opt/cni/bin/
opt/cni/bin/tuning
opt/cni/bin/vrf
opt/cni/bin/flannel
opt/cni/bin/loopback
opt/cni/bin/portmap
opt/cni/bin/ptp
opt/cni/bin/ipvlan
opt/cni/bin/host-device
opt/cni/bin/macvlan
opt/cni/bin/host-local
opt/cni/bin/firewall
opt/cni/bin/bandwidth
opt/cni/bin/sbr
opt/cni/bin/vlan
opt/cni/bin/static
opt/cni/bin/bridge
opt/cni/bin/dhcp
opt/containerd/
opt/containerd/cluster/
opt/containerd/cluster/version
opt/containerd/cluster/gce/
opt/containerd/cluster/gce/cni.template
opt/containerd/cluster/gce/configure.sh
opt/containerd/cluster/gce/env
opt/containerd/cluster/gce/cloud-init/
opt/containerd/cluster/gce/cloud-init/master.yaml
opt/containerd/cluster/gce/cloud-init/node.yaml

示例配置

disabled_plugins = [
 "io.containerd.snapshotter.v1.aufs",
 "io.containerd.snapshotter.v1.btrfs",
 "io.containerd.snapshotter.v1.devmapper",
 "io.containerd.snapshotter.v1.native",
 "io.containerd.snapshotter.v1.zfs"
]
imports = []
oom_score = 0
plugin_dir = "/data/containerd/plugins"
required_plugins = []
root = "/data/containerd"
state = "/run/containerd"
version = 2

[cgroup]
 path = ""

[debug]
 address = "/run/containerd/debug.sock"
 format = ""
 gid = 0
 level = ""
 uid = 0

[grpc]
 address = "/var/run/containerd/containerd.sock"
 gid = 0
 max_recv_message_size = 16777216
 max_send_message_size = 16777216
 tcp_address = ""
 tcp_tls_cert = ""
 tcp_tls_key = ""
 uid = 0

[metrics]
 address = "127.0.0.1:1338"
 grpc_histogram = false

[plugins]
 [plugins."io.containerd.gc.v1.scheduler"]
 deletion_threshold = 0
 mutation_threshold = 100
 pause_threshold = 0.02
 schedule_delay = "0s"
 startup_delay = "100ms"
 [plugins."io.containerd.grpc.v1.cri"]
 disable_apparmor = false
 disable_cgroup = false
 disable_hugetlb_controller = true
 disable_proc_mount = false
 disable_tcp_service = true
 enable_selinux = false
 enable_tls_streaming = false
 ignore_image_defined_volumes = false
 max_concurrent_downloads = 3
 max_container_log_line_size = 16384
 netns_mounts_under_state_dir = false
 restrict_oom_score_adj = false
 sandbox_image = "registry.cn-hangzhou.aliyuncs.com/kube-image-repo/pause:3.5-amd64"
 selinux_category_range = 1024
 stats_collect_period = 10
 stream_idle_timeout = "30m"
 stream_server_address = "127.0.0.1"
 stream_server_port = "1339"
 tolerate_missing_hugetlb_controller = true
 unset_seccomp_profile = ""
 [plugins."io.containerd.grpc.v1.cri".cni]
 bin_dir = "/opt/cni/bin"
 conf_dir = "/etc/cni/net.d"
 max_conf_num = 1
 [plugins."io.containerd.grpc.v1.cri".containerd]
 default_runtime_name = "runc"
 disable_snapshot_annotations = true
 discard_unpacked_layers = false
 no_pivot = false
 snapshotter = "overlayfs"
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
 base_runtime_spec = ""
 container_annotations = []
 pod_annotations = []
 privileged_without_host_devices = false
 runtime_type = "io.containerd.runc.v2"
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
 SystemdCgroup = true
 [plugins."io.containerd.grpc.v1.cri".image_decryption]
 key_model = "node"
 [plugins."io.containerd.grpc.v1.cri".registry]
 config_path = "/etc/containerd/certs.d"
 [plugins."io.containerd.internal.v1.opt"]
 path = "/opt/containerd"
 [plugins."io.containerd.internal.v1.restart"]
 interval = "10s"
 [plugins."io.containerd.metadata.v1.bolt"]
 content_sharing_policy = "shared"
 [plugins."io.containerd.monitor.v1.cgroups"]
 no_prometheus = false
 [plugins."io.containerd.runtime.v2.task"]
 platforms = ["linux/amd64"]
 [plugins."io.containerd.service.v1.diff-service"]
 default = ["walking"]
 [plugins."io.containerd.snapshotter.v1.overlayfs"]
 root_path = ""

[proxy_plugins]

[stream_processors]
 [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
 accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
 args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
 env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
 path = "ctd-decoder"
 returns = "application/vnd.oci.image.layer.v1.tar"
 [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
 accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
 args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
 env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
 path = "ctd-decoder"
 returns = "application/vnd.oci.image.layer.v1.tar+gzip"

[timeouts]
 "io.containerd.timeout.shim.cleanup" = "5s"
 "io.containerd.timeout.shim.load" = "5s"
 "io.containerd.timeout.shim.shutdown" = "3s"
 "io.containerd.timeout.task.state" = "2s"

[ttrpc]
 address = "/run/containerd/containerd.sock.ttrpc"
 gid = 0
 uid = 0

配置数据结构

github.com/containerd/containerd/services/server/config/config.go

// 对应 "/etc/containerd/config.toml" 配置的数据结构
type Config struct {
// 配置文件版本，使用版本2
Version int `toml:"version"`
// 用于保存持久化数据的目录,包括content,snapshot,metadata和runtime
Root string `toml:"root"`
// 用于存放containerd的临时数据，如：套接字、PID、运行时状态、装入点和其他在重新启动之间不能持久存在的插件数据
State string `toml:"state"`
// TODO; 插件存放路径
PluginDir string `toml:"plugin_dir"`
// 配置containerd grpc服务监听地址、数据接收大小等
GRPC GRPCConfig `toml:"grpc"`
// 一种适用于低内存环境的GRPC
TTRPC TTRPCConfig `toml:"ttrpc"`
// 开启pprof，配合 ctr pprof 使用
Debug Debug `toml:"debug"`
// 开启 /v1/metrics 接口
Metrics MetricsConfig `toml:"metrics"`
// 禁用某些插件，启动时不做初始化
DisabledPlugins []string `toml:"disabled_plugins"`
// 必须加载成功的插件
RequiredPlugins []string `toml:"required_plugins"`
// 各插件的配置
Plugins map[string]toml.Tree `toml:"plugins"`
// 更改各个容器值 /proc/%d/oom_score_adj
OOMScore int `toml:"oom_score"`
// Cgroup specifies cgroup information for the containerd daemon process
// TODO;
Cgroup CgroupConfig `toml:"cgroup"`
// ProxyPlugins configures plugins which are communicated to over GRPC
// TODO;
ProxyPlugins map[string]ProxyPlugin `toml:"proxy_plugins"`
// 控制shim的几个超时设置
Timeouts map[string]string `toml:"timeouts"`
// 其他路径的配置文件，一同合并至主配置
Imports []string `toml:"imports"`
// TODO；用于镜像解密
// https://github.com/containerd/containerd/blob/main/docs/cri/decryption.md
StreamProcessors map[string]StreamProcessor `toml:"stream_processors"`
}
type GRPCConfig struct {
// 创建grpc在本地unix socket路径地址
Address string `toml:"address"`
// 创建grpc在tcp监听的地址与tls配置
TCPAddress string `toml:"tcp_address"`
TCPTLSCert string `toml:"tcp_tls_cert"`
TCPTLSKey string `toml:"tcp_tls_key"`
// 本地unix socket的用户与组id
UID int `toml:"uid"`
GID int `toml:"gid"`
// 设置grpc服务最大接收与发送字节大小，默认16M
MaxRecvMsgSize int `toml:"max_recv_message_size"`
MaxSendMsgSize int `toml:"max_send_message_size"`
}
type TTRPCConfig struct {
// 同GRPCConfig
Address string `toml:"address"`
UID int `toml:"uid"`
GID int `toml:"gid"`
}
type Debug struct {
// 支持unix socket地址或者ip:port格式
Address string `toml:"address"`
UID int `toml:"uid"`
GID int `toml:"gid"`
Level string `toml:"level"`
// Format represents the logging format
Format string `toml:"format"`
}
type MetricsConfig struct {
// 这里配置的是ip:port地址，非unix socket，默认使用：127.0.0.1:1338
Address string `toml:"address"`
// 开启 grpc_prometheus.EnableHandlingTimeHistogram() 用于评估rpc的请求延迟
GRPCHistogram bool `toml:"grpc_histogram"`
}
// CgroupConfig provides cgroup configuration
type CgroupConfig struct {
Path string `toml:"path"`
}
// ProxyPlugin provides a proxy plugin configuration
type ProxyPlugin struct {
Type string `toml:"type"`
Address string `toml:"address"`
}

内置插件

github.com/containerd/containerd/plugin/plugin.go

这里 plugins 是一个 map[string]toml.Tree(或map[string]interface{})结构

类型	ID	代码路径	用途
io.containerd.internal.v1	restart	runtime/restart/monitor/monitor.go	根据"config.interval"配置定时检查存在配置了"restart"标签容器状态是否符合预期
io.containerd.internal.v1	opt	services/opt/service.go	创建"config.path"配置路径下"bin"与"lib"目录，同时把"lib"加入"LD_LIBRARY_PATH"变量
io.containerd.runtime.v1	linux	runtime/v1/linux/runtime.go	废弃！runtime配置，均使用"runc.v2"版本，由于代码存在依赖，该插件还是不能被禁用
io.containerd.runtime.v2	task	runtime/v2/manager.go	TODO; 数据均生成在”/run/containerd"路径下
io.containerd.grpc.v1	cri	pkg/cri/cri.go	TODO; 创建cri服务
io.containerd.grpc.v1	tasks	services/tasks/service.go	gRPC接口：api/services/tasks/v1，服务实例：tasks-service
io.containerd.service.v1	tasks-service	services/tasks/local.go	Tasks服务实现
io.containerd.grpc.v1	snapshots	services/snapshots/service.go	gRPC接口：api/services/snapshots/v1，服务实例：snapshots-service
io.containerd.service.v1	snapshots-service	services/snapshots/snapshotters.go	TODO
io.containerd.grpc.v1	images	services/images/service.go	gRPC接口：api/services/images/tasks/v1，服务实例：images-service
io.containerd.service.v1	images-service	services/images/local.go	TODO
io.containerd.grpc.v1	healthcheck	services/healthcheck/service.go	gRPC接口：google.golang.org/grpc/health
io.containerd.grpc.v1	content	services/content/service.go	gRPC接口：api/services/content/v1，服务实例：snapshots-service
io.containerd.service.v1	content-service	services/content/store.go	TODO
io.containerd.grpc.v1	diff	services/diff/service.go	api/services/diff/v1，服务实例：diff-service
io.containerd.service.v1	diff-service	services/diff/local.go	TODO
io.containerd.grpc.v1	introspection	services/introspection/service.go	gRPC接口：api/services/introspection/v1，服务实例：introspection-service
io.containerd.service.v1	introspection-service	services/introspection/local.go	TODO
io.containerd.grpc.v1	namespace	services/namespace/service.go	gRPC接口：api/services/namespace/v1，服务实例：namespace-service
io.containerd.service.v1	namespace-service	services/namespace/local.go	TODO
io.containerd.grpc.v1	leases	services/leases/service.go	gRPC接口：api/services/leases/v1，服务实例：leases-service
io.containerd.service.v1	leases-service	services/leases/local.go	TODO
io.containerd.grpc.v1	version	services/version/service.go	gRPC接口：api/services/version/v1，服务实例：leases-service
io.containerd.grpc.v1	containers	services/containers/service.go	gRPC接口：api/services/containers/v1，服务实例：containers-service
io.containerd.service.v1	containers-service	services/containers/local.go	TODO
io.containerd.grpc.v1	events	services/events/service.go	gRPC接口：api/services/events/v1
io.containerd.snapshotter.v1	btrfs	snapshots/btrfs/plugin/plugin.go	TDO
io.containerd.snapshotter.v1	native	snapshots/native/plugin/plugin.go	TDO
io.containerd.snapshotter.v1	windows-lcow	snapshots/lcow/lcow.go	TDO
io.containerd.snapshotter.v1	devmapper	snapshots/devmapper/plugin/plugin.go	TDO
io.containerd.snapshotter.v1	windows	snapshots/windows/windows.go	TDO
io.containerd.snapshotter.v1	overlayfs	snapshots/overlay/plugin/plugin.go	TDO
io.containerd.monitor.v1	cgroups	metrics/cgroups/cgroups.go	容器在cgroup中的性能数据暴露至配置的"Metrics"服务,否则仅有containerd的grpc调用数据
io.containerd.differ.v1	windows-lcow	diff/lcow/lcow.go	TODO
io.containerd.differ.v1	walking	diff/walking/plugin/plugin.go	TODO
io.containerd.differ.v1	windows	diff/windows/windows.go	TODO
io.containerd.metadata.v1	bolt	services/server/server.go	TODO; 管理meta.db文件
io.containerd.content.v1	content	services/server/server.go	TODO; 管理本地"ingest"存储
io.containerd.gc.v1	scheduler	gc/scheduler/scheduler.go	TODO;

io.containerd.internal.v1.restart

// Config for the restart monitor
type Config struct {
// Interval for how long to wait to check for state changes
Interval duration `toml:"interval"`
}

io.containerd.internal.v1.opt

// Config for the opt manager
type Config struct {
// Path for the opt directory
Path string `toml:"path"`
}

io.containerd.runtime.v1.linux

废弃！无需更改配置，保持默认即可，目前1.5.x代码里面还有引用，会在以后的版本移除。

RuntimeLinuxV1 = "io.containerd.runtime.v1.linux"
RuntimeRuncV1 = "io.containerd.runc.v1"
RuntimeRuncV2 = "io.containerd.runc.v2"

type Config struct {
// 使用shim的二进制文件名，默认为：containerd-shim
Shim string `toml:"shim"`
// 配合shim的runtime二进制文件名，默认为：runc
Runtime string `toml:"runtime"`
RuntimeRoot string `toml:"runtime_root"`
NoShim bool `toml:"no_shim"`
ShimDebug bool `toml:"shim_debug"`
}

io.containerd.runtime.v2.task

// Config for the v2 runtime
type Config struct {
// Supported platforms
Platforms []string `toml:"platforms"`
}

io.containerd.grpc.v1.cri

pkg/cri/config/config.go

PluginConfig

// PluginConfig contains toml config related to CRI plugin,
// it is a subset of Config.
type PluginConfig struct {
// ContainerdConfig contains config related to containerd
ContainerdConfig `toml:"containerd" json:"containerd"`
// cni相关
CniConfig `toml:"cni" json:"cni"`
// 镜像中心相关
Registry Registry `toml:"registry" json:"registry"`
// 镜像解密相关
ImageDecryption `toml:"image_decryption" json:"imageDecryption"`
// TODO; 不把cri tcp服务注册至"config.grpc.address"配置的地址
DisableTCPService bool `toml:"disable_tcp_service" json:"disableTCPService"`
// 配置本地http服务提供websocket，用于容器的exec, attach, portforward功能
// tcp连接均发生在127.0.0.1
StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress"`
// 可以配置为0，由containerd判断使用空闲端口
StreamServerPort string `toml:"stream_server_port" json:"streamServerPort"`
// 配置最大的空闲时间，格式使用golang duration语法，如：10m, 30s等
// https://golang.org/pkg/time/#ParseDuration
StreamIdleTimeout string `toml:"stream_idle_timeout" json:"streamIdleTimeout"`
// TODO;
// EnableSelinux indicates to enable the selinux support.
EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux"`
// SelinuxCategoryRange allows the upper bound on the category range to be set.
// If not specified or set to 0, defaults to 1024 from the selinux package.
SelinuxCategoryRange int `toml:"selinux_category_range" json:"selinuxCategoryRange"`
// pause镜像的地址
SandboxImage string `toml:"sandbox_image" json:"sandboxImage"`
// StatsCollectPeriod is the period (in seconds) of snapshots stats collection.
StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod"`
// 开启systemd的cgroup支持，仅使用"io.containerd.runtime.v1.linux"下有效，已被废弃
// 是在：plugins."io.containerd.grpc.v1.cri" 下的 "systemd_cgroup"
// 由：plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options 下的 SystemdCgroup 代替
SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup"`
// 是否为"StreamServerAddress"配置tls证实，以使用https
EnableTLSStreaming bool `toml:"enable_tls_streaming" json:"enableTLSStreaming"`
X509KeyPairStreaming `toml:"x509_key_pair_streaming" json:"x509KeyPairStreaming"`
// 容器日志单行最大大小，超过该尺寸则切割为多行
// oci默认为4096字节，containerd默认设置为16384
MaxContainerLogLineSize int `toml:"max_container_log_line_size" json:"maxContainerLogSize"`
// TODO; 关闭cgroup的支持，一般当containerd以非root用户运行使用
DisableCgroup bool `toml:"disable_cgroup" json:"disableCgroup"`
// TODO; 关闭apparmor的支持，一般当containerd以非root用户运行使用
DisableApparmor bool `toml:"disable_apparmor" json:"disableApparmor"`
// TODO; 关闭调整oom_score的支持，一般当containerd以非root用户运行使用
RestrictOOMScoreAdj bool `toml:"restrict_oom_score_adj" json:"restrictOOMScoreAdj"`
// 限制最大并行数对每个镜像下载，通过semaphore实现
MaxConcurrentDownloads int `toml:"max_concurrent_downloads" json:"maxConcurrentDownloads"`
// TODO; 在k8s<=1.11版本必须设为'true'
// cri-api/pkg/apis/runtime/v1alpha2/api.proto -> masked_paths
DisableProcMount bool `toml:"disable_proc_mount" json:"disableProcMount"`
// UnsetSeccompProfile is the profile containerd/cri will use If the provided seccomp profile is
// unset (`""`) for a container (default is `unconfined`)
UnsetSeccompProfile string `toml:"unset_seccomp_profile" json:"unsetSeccompProfile"`
// TolerateMissingHugetlbController if set to false will error out on create/update
// container requests with huge page limits if the cgroup controller for hugepages is not present.
// This helps with supporting Kubernetes <=1.18 out of the box. (default is `true`)
// TODO;
TolerateMissingHugetlbController bool `toml:"tolerate_missing_hugetlb_controller" json:"tolerateMissingHugetlbController"`
// DisableHugetlbController indicates to silently disable the hugetlb controller, even when it is
// present in /sys/fs/cgroup/cgroup.controllers.
// This helps with running rootless mode + cgroup v2 + systemd but without hugetlb delegation.
DisableHugetlbController bool `toml:"disable_hugetlb_controller" json:"disableHugetlbController"`
// IgnoreImageDefinedVolumes ignores volumes defined by the image. Useful for better resource
// isolation, security and early detection of issues in the mount configuration when using
// ReadOnlyRootFilesystem since containers won't silently mount a temporary volume.
IgnoreImageDefinedVolumes bool `toml:"ignore_image_defined_volumes" json:"ignoreImageDefinedVolumes"`
// NetNSMountsUnderStateDir places all mounts for network namespaces under StateDir/netns instead
// of being placed under the hardcoded directory /var/run/netns. Changing this setting requires
// that all containers are deleted.
NetNSMountsUnderStateDir bool `toml:"netns_mounts_under_state_dir" json:"netnsMountsUnderStateDir"`
}

ContainerdConfig

type ContainerdConfig struct {
// 使用哪个"io.containerd.snapshotter.v1"插件id
Snapshotter string `toml:"snapshotter" json:"snapshotter"`
// 默认使用的容器运行时，名称必须配置在"runtimes"里
DefaultRuntimeName string `toml:"default_runtime_name" json:"defaultRuntimeName"`
// 废弃！使用"DefaultRuntimeName"代替
DefaultRuntime Runtime `toml:"default_runtime" json:"defaultRuntime"`
// 废弃！在"Runtimes"里使用"untrusted"代替
UntrustedWorkloadRuntime Runtime `toml:"untrusted_workload_runtime" json:"untrustedWorkloadRuntime"`
// Runtimes is a map from CRI RuntimeHandler strings, which specify types of runtime
// configurations, to the matching configurations.
Runtimes map[string]Runtime `toml:"runtimes" json:"runtimes"`
// NoPivot disables pivot-root (linux only), required when running a container in a RamDisk with runc
// This only works for runtime type "io.containerd.runtime.v1.linux".
NoPivot bool `toml:"no_pivot" json:"noPivot"`
// DisableSnapshotAnnotations disables to pass additional annotations (image
// related information) to snapshotters. These annotations are required by
// stargz snapshotter (https://github.com/containerd/stargz-snapshotter).
DisableSnapshotAnnotations bool `toml:"disable_snapshot_annotations" json:"disableSnapshotAnnotations"`
// DiscardUnpackedLayers is a boolean flag to specify whether to allow GC to
// remove layers from the content store after successfully unpacking these
// layers to the snapshotter.
DiscardUnpackedLayers bool `toml:"discard_unpacked_layers" json:"discardUnpackedLayers"`
}
// Runtime struct to contain the type(ID), engine, and root variables for a default runtime
// and a runtime for untrusted worload.
type Runtime struct {
// 运行时使用的类型，如："io.containerd.runtime.v1.linux","io.containerd.runc.v1",io.containerd.runc.v2"（推荐）
Type string `toml:"runtime_type" json:"runtimeType"`
// 废弃！仅支持类型为"io.containerd.runtime.v1.linux"
Engine string `toml:"runtime_engine" json:"runtimeEngine"`
// PodAnnotations is a list of pod annotations passed to both pod sandbox as well as
// container OCI annotations.
PodAnnotations []string `toml:"pod_annotations" json:"PodAnnotations"`
// ContainerAnnotations is a list of container annotations passed through to the OCI config of the containers.
// Container annotations in CRI are usually generated by other Kubernetes node components (i.e., not users).
// Currently, only device plugins populate the annotations.
ContainerAnnotations []string `toml:"container_annotations" json:"ContainerAnnotations"`
// 废弃！仅支持类型为"io.containerd.runtime.v1.linux"
Root string `toml:"runtime_root" json:"runtimeRoot"`
// Options are config options for the runtime.
// If options is loaded from toml config, it will be map[string]interface{}.
// Options can be converted into toml.Tree using toml.TreeFromMap().
// Using options type as map[string]interface{} helps in correctly marshaling options from Go to JSON.
Options map[string]interface{} `toml:"options" json:"options"`
// PrivilegedWithoutHostDevices overloads the default behaviour for adding host devices to the
// runtime spec when the container is privileged. Defaults to false.
PrivilegedWithoutHostDevices bool `toml:"privileged_without_host_devices" json:"privileged_without_host_devices"`
// BaseRuntimeSpec is a json file with OCI spec to use as base spec that all container's will be created from.
BaseRuntimeSpec string `toml:"base_runtime_spec" json:"baseRuntimeSpec"`
}

CniConfig

type CniConfig struct {
// cni相关二进制存放目录
NetworkPluginBinDir string `toml:"bin_dir" json:"binDir"`
// cni配置存放目录
NetworkPluginConfDir string `toml:"conf_dir" json:"confDir"`
// 加载配置，默认仅允许加载1个配置，支持后缀：".conf", ".conflist", ".json"
// 后缀判断：github.com/containernetworking/cni/libcni/config.go
NetworkPluginMaxConfNum int `toml:"max_conf_num" json:"maxConfNum"`
// cni的模版，语法使用go template，根据kubelet配置自定生成配置，已被废弃，不建议使用
NetworkPluginConfTemplate string `toml:"conf_template" json:"confTemplate"`
}

ImageDecryption

// 用于对加密的镜像如何解密
type ImageDecryption struct {
// cri获取解密key的方式，与"stream_processors"配置配合使用，官方文档：
// https://github.com/containerd/cri/tree/master/docs/config.md
// https://github.com/containerd/cri/tree/master/docs/decryption.md
KeyModel string `toml:"key_model" json:"keyModel"`
}

Registry

// Registry is registry settings configured
type Registry struct {
// ConfigPath is a path to the root directory containing registry-specific
// configurations.
// If ConfigPath is set, the rest of the registry specific options are ignored.
ConfigPath string `toml:"config_path" json:"configPath"`
// Mirrors are namespace to mirror mapping for all namespaces.
// This option will not be used when ConfigPath is provided.
// DEPRECATED: Use ConfigPath instead. Remove in containerd 1.7.
Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors"`
// Configs are configs for each registry.
// The key is the domain name or IP of the registry.
// This option will be fully deprecated for ConfigPath in the future.
Configs map[string]RegistryConfig `toml:"configs" json:"configs"`
// Auths are registry endpoint to auth config mapping. The registry endpoint must
// be a valid url with host specified.
// DEPRECATED: Use ConfigPath instead. Remove in containerd 1.6.
Auths map[string]AuthConfig `toml:"auths" json:"auths"`
// Headers adds additional HTTP headers that get sent to all registries
Headers map[string][]string `toml:"headers" json:"headers"`
}

ConfigPath

remotes/docker/config/hosts.go

struct {
HostFileConfig
// Server specifies the default server. When `host` is
// also specified, those hosts are tried first.
Server string `toml:"server"`
// HostConfigs store the per-host configuration
HostConfigs map[string]hostFileConfig `toml:"host"`
}
type hostFileConfig struct {
// Capabilities determine what operations a host is
// capable of performing. Allowed values
// - pull
// - resolve
// - push
Capabilities []string `toml:"capabilities"`
// CACert can be a string or an array of strings
CACert interface{} `toml:"ca"`
// TODO: Make this an array (two key types, one for pairs (multiple files), one for single file?)
Client interface{} `toml:"client"`
SkipVerify *bool `toml:"skip_verify"`
Header map[string]interface{} `toml:"header"`
// API (default: "docker")
// API Version (default: "v2")
// Credentials: helper? name? username? alternate domain? token?
}

io.containerd.snapshotter.v1.btrfs

// Config represents configuration for the btrfs plugin.
type Config struct {
// Root directory for the plugin
RootPath string `toml:"root_path"`
}

io.containerd.snapshotter.v1.native

// Config represents configuration for the native plugin.
type Config struct {
// Root directory for the plugin
RootPath string `toml:"root_path"`
}

io.containerd.snapshotter.v1.devmapper

// Config represents device mapper configuration loaded from file.
// Size units can be specified in human-readable string format (like "32KIB", "32GB", "32Tb")
type Config struct {
// Device snapshotter root directory for metadata
RootPath string `toml:"root_path"`
// Name for 'thin-pool' device to be used by snapshotter (without /dev/mapper/ prefix)
PoolName string `toml:"pool_name"`
// Defines how much space to allocate when creating base image for container
BaseImageSize string `toml:"base_image_size"`
BaseImageSizeBytes uint64 `toml:"-"`
// Flag to async remove device using Cleanup() callback in snapshots GC
AsyncRemove bool `toml:"async_remove"`
}

io.containerd.snapshotter.v1.overlayfs

// Config represents configuration for the overlay plugin.
type Config struct {
// Root directory for the plugin
RootPath string `toml:"root_path"`
}

io.containerd.monitor.v1.cgroups

// Config for the cgroups monitor
type Config struct {
NoPrometheus bool `toml:"no_prometheus"`
}

io.containerd.metadata.v1.bolt

// BoltConfig defines the configuration values for the bolt plugin, which is
// loaded here, rather than back registered in the metadata package.
type BoltConfig struct {
 // ContentSharingPolicy sets the sharing policy for content between
 // namespaces.
 //
 // The default mode "shared" will make blobs available in all
 // namespaces once it is pulled into any namespace. The blob will be pulled
 // into the namespace if a writer is opened with the "Expected" digest that
 // is already present in the backend.
 //
 // The alternative mode, "isolated" requires that clients prove they have
 // access to the content by providing all of the content to the ingest
 // before the blob is added to the namespace.
 //
 // Both modes share backing data, while "shared" will reduce total
 // bandwidth across namespaces, at the cost of allowing access to any blob
 // just by knowing its digest.
 ContentSharingPolicy string `toml:"content_sharing_policy"`
}

io.containerd.gc.v1.scheduler

// config configures the garbage collection policies.
type config struct {
 // PauseThreshold represents the maximum amount of time garbage
 // collection should be scheduled based on the average pause time.
 // For example, a value of 0.02 means that scheduled garbage collection
 // pauses should present at most 2% of real time,
 // or 20ms of every second.
 //
 // A maximum value of .5 is enforced to prevent over scheduling of the
 // garbage collector, trigger options are available to run in a more
 // predictable time frame after mutation.
 //
 // Default is 0.02
 PauseThreshold float64 `toml:"pause_threshold"`

 // DeletionThreshold is used to guarantee that a garbage collection is
 // scheduled after configured number of deletions have occurred
 // since the previous garbage collection. A value of 0 indicates that
 // garbage collection will not be triggered by deletion count.
 //
 // Default 0
 DeletionThreshold int `toml:"deletion_threshold"`

 // MutationThreshold is used to guarantee that a garbage collection is
 // run after a configured number of database mutations have occurred
 // since the previous garbage collection. A value of 0 indicates that
 // garbage collection will only be run after a manual trigger or
 // deletion. Unlike the deletion threshold, the mutation threshold does
 // not cause scheduling of a garbage collection, but ensures GC is run
 // at the next scheduled GC.
 //
 // Default 100
 MutationThreshold int `toml:"mutation_threshold"`

 // ScheduleDelay is the duration in the future to schedule a garbage
 // collection triggered manually or by exceeding the configured
 // threshold for deletion or mutation. A zero value will immediately
 // schedule. Use suffix "ms" for millisecond and "s" for second.
 //
 // Default is "0ms"
 ScheduleDelay duration `toml:"schedule_delay"`

 // StartupDelay is the delay duration to do an initial garbage
 // collection after startup. The initial garbage collection is used to
 // set the base for pause threshold and should be scheduled in the
 // future to avoid slowing down other startup processes. Use suffix
 // "ms" for millisecond and "s" for second.
 //
 // Default is "100ms"
 StartupDelay duration `toml:"startup_delay"`
}

插件使用示例

容器自动重启

需要使用到"io.containerd.internal.v1.restart"插件，配置容器必须维持在"running"状态，也即是自动启动

ctr -n k8s.io c label 672bd8283f5c5fbbc22f9c31391132171503ef79b9eb3ca4f2652463d260ce77 containerd.io/restart.status=running
io.kubernetes.pod.name=etcd-192.168.31.201,io.kubernetes.container.name=etcd,io.cri-containerd.kind=container,containerd.io/restart.status=running,io.kubernetes.pod.uid=24c002bb538c36d9833eb09e226b1bba,io.kubernetes.pod.namespace=kube-system

plugin配置

toml

[plugins]
 [plugins."io.containerd.gc.v1.scheduler"]
 deletion_threshold = 0
 mutation_threshold = 100

 [plugins."io.containerd.grpc.v1.cri"]
 disable_apparmor = false
 disable_cgroup = false

 [plugins."io.containerd.grpc.v1.cri".cni]
 bin_dir = "/opt/cni/bin"
 conf_dir = "/etc/cni/net.d"

json

{
 "plubins": {
 "io.containerd.gc.v1.scheduler": {
 "deletion_threshold": 0,
 "mutation_threshold": 100
 },
 "io.containerd.grpc.v1.cri": {
 "disable_apparmor": false,
 "disable_cgroup": false,
 "cni": {
 "bin_dir": "/opt/cni/bin",
 "conf_dir": "/etc/cni/net.d"
 }
 }
 }
}

ctr plugins ls

// URI returns the full plugin URI
func (r *Registration) URI() string {
 return fmt.Sprintf("%s.%s", r.Type, r.ID)
}

首先需要明白go1.8新增的plugin特性，

FAQs

ctr --namespace k8s.io images pull --hosts-dir /etc/containerd/certs.d/ registry.cn-hangzhou.aliyuncs.com/kube-image-repo/pause:3.5-amd64 --http-dump

OPSaid – 容器运行时