X Tutup
Skip to content

Commit 55893b9

Browse files
crosbymichaelcrosbymichael
committed
Add CNI conf based on runtime class
Signed-off-by: Michael Crosby <michael@thepasture.io>
1 parent 7ddf5e5 commit 55893b9

File tree

12 files changed

+187
-63
lines changed

12 files changed

+187
-63
lines changed

‎docs/cri/config.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,18 @@ version = 2
175175
# Still running containers and restarted containers will still be using the original spec from which that container was created.
176176
base_runtime_spec = ""
177177

178+
# conf_dir is the directory in which the admin places a CNI conf.
179+
# this allows a different CNI conf for the network stack when a different runtime is being used.
180+
cni_conf_dir = "/etc/cni/net.d"
181+
182+
# cni_max_conf_num specifies the maximum number of CNI plugin config files to
183+
# load from the CNI config directory. By default, only 1 CNI plugin config
184+
# file will be loaded. If you want to load multiple CNI plugin config files
185+
# set max_conf_num to the number desired. Setting cni_max_config_num to 0 is
186+
# interpreted as no limit is desired and will result in all CNI plugin
187+
# config files being loaded from the CNI config directory.
188+
cni_max_conf_num = 1
189+
178190
# 'plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options' is options specific to
179191
# "io.containerd.runc.v1" and "io.containerd.runc.v2". Its corresponding options type is:
180192
# https://github.com/containerd/containerd/blob/v1.3.2/runtime/v2/runc/options/oci.pb.go#L26 .

‎pkg/cri/config/config.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,12 @@ type Runtime struct {
5656
PrivilegedWithoutHostDevices bool `toml:"privileged_without_host_devices" json:"privileged_without_host_devices"`
5757
// BaseRuntimeSpec is a json file with OCI spec to use as base spec that all container's will be created from.
5858
BaseRuntimeSpec string `toml:"base_runtime_spec" json:"baseRuntimeSpec"`
59+
// NetworkPluginConfDir is a directory containing the CNI network information for the runtime class.
60+
NetworkPluginConfDir string `toml:"cni_conf_dir" json:"cniConfDir"`
61+
// NetworkPluginMaxConfNum is the max number of plugin config files that will
62+
// be loaded from the cni config directory by go-cni. Set the value to 0 to
63+
// load all config files (no arbitrary limit). The legacy default value is 1.
64+
NetworkPluginMaxConfNum int `toml:"cni_max_conf_num" json:"cniMaxConfNum"`
5965
}
6066

6167
// ContainerdConfig contains toml config related to containerd

‎pkg/cri/server/sandbox_run.go‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
containerdio "github.com/containerd/containerd/cio"
2828
"github.com/containerd/containerd/errdefs"
2929
"github.com/containerd/containerd/log"
30+
"github.com/containerd/containerd/snapshots"
3031
cni "github.com/containerd/go-cni"
3132
"github.com/containerd/nri"
3233
v1 "github.com/containerd/nri/types/v1"
@@ -45,7 +46,6 @@ import (
4546
"github.com/containerd/containerd/pkg/cri/util"
4647
ctrdutil "github.com/containerd/containerd/pkg/cri/util"
4748
"github.com/containerd/containerd/pkg/netns"
48-
"github.com/containerd/containerd/snapshots"
4949
selinux "github.com/opencontainers/selinux/go-selinux"
5050
)
5151

@@ -204,7 +204,6 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
204204
if err != nil {
205205
return nil, errors.Wrap(err, "failed to generate runtime options")
206206
}
207-
208207
snapshotterOpt := snapshots.WithLabels(snapshots.FilterInheritedLabels(config.Annotations))
209208
opts := []containerd.NewContainerOpts{
210209
containerd.WithSnapshotter(c.config.ContainerdConfig.Snapshotter),
@@ -349,14 +348,30 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
349348
return &runtime.RunPodSandboxResponse{PodSandboxId: id}, nil
350349
}
351350

351+
// getNetworkPlugin returns the network plugin to be used by the runtime class
352+
// defaults to the global CNI options in the CRI config
353+
func (c *criService) getNetworkPlugin(runtimeClass string) cni.CNI {
354+
if c.netPlugin == nil {
355+
return nil
356+
}
357+
i, ok := c.netPlugin[runtimeClass]
358+
if !ok {
359+
if i, ok = c.netPlugin[defaultNetworkPlugin]; !ok {
360+
return nil
361+
}
362+
}
363+
return i
364+
}
365+
352366
// setupPodNetwork setups up the network for a pod
353367
func (c *criService) setupPodNetwork(ctx context.Context, sandbox *sandboxstore.Sandbox) error {
354368
var (
355-
id = sandbox.ID
356-
config = sandbox.Config
357-
path = sandbox.NetNSPath
369+
id = sandbox.ID
370+
config = sandbox.Config
371+
path = sandbox.NetNSPath
372+
netPlugin = c.getNetworkPlugin(sandbox.RuntimeHandler)
358373
)
359-
if c.netPlugin == nil {
374+
if netPlugin == nil {
360375
return errors.New("cni config not initialized")
361376
}
362377

@@ -365,7 +380,7 @@ func (c *criService) setupPodNetwork(ctx context.Context, sandbox *sandboxstore.
365380
return errors.Wrap(err, "get cni namespace options")
366381
}
367382

368-
result, err := c.netPlugin.Setup(ctx, id, path, opts...)
383+
result, err := netPlugin.Setup(ctx, id, path, opts...)
369384
if err != nil {
370385
return err
371386
}

‎pkg/cri/server/sandbox_stop.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,8 @@ func (c *criService) waitSandboxStop(ctx context.Context, sandbox sandboxstore.S
165165

166166
// teardownPodNetwork removes the network from the pod
167167
func (c *criService) teardownPodNetwork(ctx context.Context, sandbox sandboxstore.Sandbox) error {
168-
if c.netPlugin == nil {
168+
netPlugin := c.getNetworkPlugin(sandbox.RuntimeHandler)
169+
if netPlugin == nil {
169170
return errors.New("cni config not initialized")
170171
}
171172

@@ -179,7 +180,7 @@ func (c *criService) teardownPodNetwork(ctx context.Context, sandbox sandboxstor
179180
return errors.Wrap(err, "get cni namespace options")
180181
}
181182

182-
return c.netPlugin.Remove(ctx, id, path, opts...)
183+
return netPlugin.Remove(ctx, id, path, opts...)
183184
}
184185

185186
// cleanupUnknownSandbox cleanup stopped sandbox in unknown state.

‎pkg/cri/server/service.go‎

Lines changed: 39 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"net/http"
2424
"os"
2525
"path/filepath"
26+
"sync"
2627
"time"
2728

2829
"github.com/containerd/containerd"
@@ -49,6 +50,9 @@ import (
4950
"github.com/containerd/containerd/pkg/registrar"
5051
)
5152

53+
// defaultNetworkPlugin is used for the default CNI configuration
54+
const defaultNetworkPlugin = "default"
55+
5256
// grpcServices are all the grpc services provided by cri containerd.
5357
type grpcServices interface {
5458
runtime.RuntimeServiceServer
@@ -92,7 +96,7 @@ type criService struct {
9296
// snapshotStore stores information of all snapshots.
9397
snapshotStore *snapshotstore.Store
9498
// netPlugin is used to setup and teardown network when run/stop pod sandbox.
95-
netPlugin cni.CNI
99+
netPlugin map[string]cni.CNI
96100
// client is an instance of the containerd client
97101
client *containerd.Client
98102
// streamServer is the streaming server serves container streaming request.
@@ -104,7 +108,7 @@ type criService struct {
104108
initialized atomic.Bool
105109
// cniNetConfMonitor is used to reload cni network conf if there is
106110
// any valid fs change events from cni network conf dir.
107-
cniNetConfMonitor *cniNetConfSyncer
111+
cniNetConfMonitor map[string]*cniNetConfSyncer
108112
// baseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec`
109113
baseOCISpecs map[string]*oci.Spec
110114
// allCaps is the list of the capabilities.
@@ -127,6 +131,7 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi
127131
sandboxNameIndex: registrar.NewRegistrar(),
128132
containerNameIndex: registrar.NewRegistrar(),
129133
initialized: atomic.NewBool(false),
134+
netPlugin: make(map[string]cni.CNI),
130135
}
131136

132137
if client.SnapshotService(c.config.ContainerdConfig.Snapshotter) == nil {
@@ -148,9 +153,21 @@ func NewCRIService(config criconfig.Config, client *containerd.Client) (CRIServi
148153

149154
c.eventMonitor = newEventMonitor(c)
150155

151-
c.cniNetConfMonitor, err = newCNINetConfSyncer(c.config.NetworkPluginConfDir, c.netPlugin, c.cniLoadOptions())
152-
if err != nil {
153-
return nil, errors.Wrap(err, "failed to create cni conf monitor")
156+
c.cniNetConfMonitor = make(map[string]*cniNetConfSyncer)
157+
for name, i := range c.netPlugin {
158+
path := c.config.NetworkPluginConfDir
159+
if name != defaultNetworkPlugin {
160+
if rc, ok := c.config.Runtimes[name]; ok {
161+
path = rc.NetworkPluginConfDir
162+
}
163+
}
164+
if path != "" {
165+
m, err := newCNINetConfSyncer(path, i, c.cniLoadOptions())
166+
if err != nil {
167+
return nil, errors.Wrapf(err, "failed to create cni conf monitor for %s", name)
168+
}
169+
c.cniNetConfMonitor[name] = m
170+
}
154171
}
155172

156173
// Preload base OCI specs
@@ -200,12 +217,20 @@ func (c *criService) Run() error {
200217
)
201218
snapshotsSyncer.start()
202219

203-
// Start CNI network conf syncer
204-
logrus.Info("Start cni network conf syncer")
205-
cniNetConfMonitorErrCh := make(chan error, 1)
220+
// Start CNI network conf syncers
221+
cniNetConfMonitorErrCh := make(chan error, len(c.cniNetConfMonitor))
222+
var netSyncGroup sync.WaitGroup
223+
for name, h := range c.cniNetConfMonitor {
224+
netSyncGroup.Add(1)
225+
logrus.Infof("Start cni network conf syncer for %s", name)
226+
go func(h *cniNetConfSyncer) {
227+
cniNetConfMonitorErrCh <- h.syncLoop()
228+
netSyncGroup.Done()
229+
}(h)
230+
}
206231
go func() {
207-
defer close(cniNetConfMonitorErrCh)
208-
cniNetConfMonitorErrCh <- c.cniNetConfMonitor.syncLoop()
232+
netSyncGroup.Wait()
233+
close(cniNetConfMonitorErrCh)
209234
}()
210235

211236
// Start streaming server.
@@ -272,8 +297,10 @@ func (c *criService) Run() error {
272297
// TODO(random-liu): Make close synchronous.
273298
func (c *criService) Close() error {
274299
logrus.Info("Stop CRI service")
275-
if err := c.cniNetConfMonitor.stop(); err != nil {
276-
logrus.WithError(err).Error("failed to stop cni network conf monitor")
300+
for name, h := range c.cniNetConfMonitor {
301+
if err := h.stop(); err != nil {
302+
logrus.WithError(err).Errorf("failed to stop cni network conf monitor for %s", name)
303+
}
277304
}
278305
c.eventMonitor.stop()
279306
if err := c.streamServer.Stop(); err != nil {

‎pkg/cri/server/service_linux.go‎

Lines changed: 30 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,7 @@ import (
3030
const networkAttachCount = 2
3131

3232
// initPlatform handles linux specific initialization for the CRI service.
33-
func (c *criService) initPlatform() error {
34-
var err error
35-
33+
func (c *criService) initPlatform() (err error) {
3634
if userns.RunningInUserNS() {
3735
if !(c.config.DisableCgroup && !c.apparmorEnabled() && c.config.RestrictOOMScoreAdj) {
3836
logrus.Warn("Running containerd in a user namespace typically requires disable_cgroup, disable_apparmor, restrict_oom_score_adj set to be true")
@@ -50,16 +48,35 @@ func (c *criService) initPlatform() error {
5048
selinux.SetDisabled()
5149
}
5250

53-
// Pod needs to attach to at least loopback network and a non host network,
54-
// hence networkAttachCount is 2. If there are more network configs the
55-
// pod will be attached to all the networks but we will only use the ip
56-
// of the default network interface as the pod IP.
57-
c.netPlugin, err = cni.New(cni.WithMinNetworkCount(networkAttachCount),
58-
cni.WithPluginConfDir(c.config.NetworkPluginConfDir),
59-
cni.WithPluginMaxConfNum(c.config.NetworkPluginMaxConfNum),
60-
cni.WithPluginDir([]string{c.config.NetworkPluginBinDir}))
61-
if err != nil {
62-
return errors.Wrap(err, "failed to initialize cni")
51+
pluginDirs := map[string]string{
52+
defaultNetworkPlugin: c.config.NetworkPluginConfDir,
53+
}
54+
for name, conf := range c.config.Runtimes {
55+
if conf.NetworkPluginConfDir != "" {
56+
pluginDirs[name] = conf.NetworkPluginConfDir
57+
}
58+
}
59+
60+
c.netPlugin = make(map[string]cni.CNI)
61+
for name, dir := range pluginDirs {
62+
max := c.config.NetworkPluginMaxConfNum
63+
if name != defaultNetworkPlugin {
64+
if m := c.config.Runtimes[name].NetworkPluginMaxConfNum; m != 0 {
65+
max = m
66+
}
67+
}
68+
// Pod needs to attach to at least loopback network and a non host network,
69+
// hence networkAttachCount is 2. If there are more network configs the
70+
// pod will be attached to all the networks but we will only use the ip
71+
// of the default network interface as the pod IP.
72+
i, err := cni.New(cni.WithMinNetworkCount(networkAttachCount),
73+
cni.WithPluginConfDir(dir),
74+
cni.WithPluginMaxConfNum(max),
75+
cni.WithPluginDir([]string{c.config.NetworkPluginBinDir}))
76+
if err != nil {
77+
return errors.Wrap(err, "failed to initialize cni")
78+
}
79+
c.netPlugin[name] = i
6380
}
6481

6582
if c.allCaps == nil {

‎pkg/cri/server/service_test.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"testing"
2424

2525
"github.com/containerd/containerd/oci"
26+
"github.com/containerd/go-cni"
2627
"github.com/stretchr/testify/assert"
2728
"github.com/stretchr/testify/require"
2829

@@ -66,7 +67,9 @@ func newTestCRIService() *criService {
6667
sandboxNameIndex: registrar.NewRegistrar(),
6768
containerStore: containerstore.NewStore(labels),
6869
containerNameIndex: registrar.NewRegistrar(),
69-
netPlugin: servertesting.NewFakeCNIPlugin(),
70+
netPlugin: map[string]cni.CNI{
71+
defaultNetworkPlugin: servertesting.NewFakeCNIPlugin(),
72+
},
7073
}
7174
}
7275

‎pkg/cri/server/service_windows.go‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,18 +27,36 @@ const windowsNetworkAttachCount = 1
2727

2828
// initPlatform handles linux specific initialization for the CRI service.
2929
func (c *criService) initPlatform() error {
30-
var err error
31-
// For windows, the loopback network is added as default.
32-
// There is no need to explicitly add one hence networkAttachCount is 1.
33-
// If there are more network configs the pod will be attached to all the
34-
// networks but we will only use the ip of the default network interface
35-
// as the pod IP.
36-
c.netPlugin, err = cni.New(cni.WithMinNetworkCount(windowsNetworkAttachCount),
37-
cni.WithPluginConfDir(c.config.NetworkPluginConfDir),
38-
cni.WithPluginMaxConfNum(c.config.NetworkPluginMaxConfNum),
39-
cni.WithPluginDir([]string{c.config.NetworkPluginBinDir}))
40-
if err != nil {
41-
return errors.Wrap(err, "failed to initialize cni")
30+
pluginDirs := map[string]string{
31+
defaultNetworkPlugin: c.config.NetworkPluginConfDir,
32+
}
33+
for name, conf := range c.config.Runtimes {
34+
if conf.NetworkPluginConfDir != "" {
35+
pluginDirs[name] = conf.NetworkPluginConfDir
36+
}
37+
}
38+
39+
c.netPlugin = make(map[string]cni.CNI)
40+
for name, dir := range pluginDirs {
41+
max := c.config.NetworkPluginMaxConfNum
42+
if name != defaultNetworkPlugin {
43+
if m := c.config.Runtimes[name].NetworkPluginMaxConfNum; m != 0 {
44+
max = m
45+
}
46+
}
47+
// For windows, the loopback network is added as default.
48+
// There is no need to explicitly add one hence networkAttachCount is 1.
49+
// If there are more network configs the pod will be attached to all the
50+
// networks but we will only use the ip of the default network interface
51+
// as the pod IP.
52+
i, err := cni.New(cni.WithMinNetworkCount(windowsNetworkAttachCount),
53+
cni.WithPluginConfDir(dir),
54+
cni.WithPluginMaxConfNum(max),
55+
cni.WithPluginDir([]string{c.config.NetworkPluginBinDir}))
56+
if err != nil {
57+
return errors.Wrap(err, "failed to initialize cni")
58+
}
59+
c.netPlugin[name] = i
4260
}
4361

4462
return nil

0 commit comments

Comments
 (0)
X Tutup