X Tutup
Skip to content

Commit f76eefd

Browse files
crosbymichaelcrosbymichael
authored
Merge pull request containerd#3574 from mxpv/cfg
Support config imports
2 parents 1eaf601 + 01f7265 commit f76eefd

File tree

13 files changed

+1230
-45
lines changed

13 files changed

+1230
-45
lines changed

‎cmd/containerd/command/config.go‎

Lines changed: 54 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,50 @@ func (c *Config) WriteTo(w io.Writer) (int64, error) {
4040
return 0, toml.NewEncoder(w).Encode(c)
4141
}
4242

43+
func outputConfig(cfg *srvconfig.Config) error {
44+
config := &Config{
45+
Config: cfg,
46+
}
47+
48+
plugins, err := server.LoadPlugins(gocontext.Background(), config.Config)
49+
if err != nil {
50+
return err
51+
}
52+
if len(plugins) != 0 {
53+
config.Plugins = make(map[string]interface{})
54+
for _, p := range plugins {
55+
if p.Config == nil {
56+
continue
57+
}
58+
59+
pc, err := config.Decode(p)
60+
if err != nil {
61+
return err
62+
}
63+
64+
config.Plugins[p.URI()] = pc
65+
}
66+
}
67+
68+
timeouts := timeout.All()
69+
config.Timeouts = make(map[string]string)
70+
for k, v := range timeouts {
71+
config.Timeouts[k] = v.String()
72+
}
73+
74+
// for the time being, keep the defaultConfig's version set at 1 so that
75+
// when a config without a version is loaded from disk and has no version
76+
// set, we assume it's a v1 config. But when generating new configs via
77+
// this command, generate the v2 config
78+
config.Config.Version = 2
79+
80+
// remove overridden Plugins type to avoid duplication in output
81+
config.Config.Plugins = nil
82+
83+
_, err = config.WriteTo(os.Stdout)
84+
return err
85+
}
86+
4387
var configCommand = cli.Command{
4488
Name: "config",
4589
Usage: "information on the containerd config",
@@ -48,35 +92,19 @@ var configCommand = cli.Command{
4892
Name: "default",
4993
Usage: "see the output of the default config",
5094
Action: func(context *cli.Context) error {
51-
config := &Config{
52-
Config: defaultConfig(),
53-
}
54-
// for the time being, keep the defaultConfig's version set at 1 so that
55-
// when a config without a version is loaded from disk and has no version
56-
// set, we assume it's a v1 config. But when generating new configs via
57-
// this command, generate the v2 config
58-
config.Config.Version = 2
59-
plugins, err := server.LoadPlugins(gocontext.Background(), config.Config)
60-
if err != nil {
95+
return outputConfig(defaultConfig())
96+
},
97+
},
98+
{
99+
Name: "dump",
100+
Usage: "see the output of the final main config with imported in subconfig files",
101+
Action: func(context *cli.Context) error {
102+
config := defaultConfig()
103+
if err := srvconfig.LoadConfig(context.GlobalString("config"), config); err != nil && !os.IsNotExist(err) {
61104
return err
62105
}
63-
if len(plugins) != 0 {
64-
config.Plugins = make(map[string]interface{})
65-
for _, p := range plugins {
66-
if p.Config == nil {
67-
continue
68-
}
69-
config.Plugins[p.URI()] = p.Config
70-
}
71-
}
72-
timeouts := timeout.All()
73-
config.Timeouts = make(map[string]string)
74-
for k, v := range timeouts {
75-
config.Timeouts[k] = v.String()
76-
}
77106

78-
_, err = config.WriteTo(os.Stdout)
79-
return err
107+
return outputConfig(config)
80108
},
81109
},
82110
},

‎docs/man/containerd-config.toml.5.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,14 @@ settings.
3232
**oom_score**
3333
: The out of memory (OOM) score applied to the containerd daemon process (Default: 0)
3434

35+
**imports**
36+
: Imports is a list of additional configuration files to include.
37+
This allows to split the main configuration file and keep some sections
38+
separately (for example vendors may keep a custom runtime configuration in a
39+
separate file without modifying the main `config.toml`).
40+
Imported files will overwrite simple fields like `int` or
41+
`string` (if not empty) and will append `array` and `map` fields.
42+
3543
**[grpc]**
3644
: Section for gRPC socket listener settings. Contains three properties:
3745
- **address** (Default: "/run/containerd/containerd.sock")
@@ -82,6 +90,7 @@ The following is a complete **config.toml** default configuration example:
8290
root = "/var/lib/containerd"
8391
state = "/run/containerd"
8492
oom_score = 0
93+
imports = ["/etc/containerd/runtime_*.toml", "./debug.toml"]
8594
8695
[grpc]
8796
address = "/run/containerd/containerd.sock"

‎docs/stream_processors.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,20 +21,20 @@ pipe's path set as the value of the environment variable `STREAM_PROCESSOR_PIPE`
2121
## Configuration
2222

2323
To configure stream processors for containerd, entries in the config file need to be made.
24-
The `stream_processors` field is an array so that users can chain together multiple processors
24+
The `stream_processors` field is a map so that users can chain together multiple processors
2525
to mutate content streams.
2626

2727
Processor Fields:
2828

29-
* `id` - ID of the processor, used for passing a specific payload to the processor.
29+
* Key - ID of the processor, used for passing a specific payload to the processor.
3030
* `accepts` - Accepted media-types for the processor that it can handle.
3131
* `returns` - The media-type that the processor returns.
3232
* `path` - Path to the processor binary.
3333
* `args` - Arguments passed to the processor binary.
3434

3535
```toml
36-
[[stream_processors]]
37-
id = "io.containerd.processor.v1.pigz"
36+
[stream_processors]
37+
[stream_processors."io.containerd.processor.v1.pigz"]
3838
accepts = ["application/vnd.docker.image.rootfs.diff.tar.gzip"]
3939
returns = "application/vnd.oci.image.layer.v1.tar"
4040
path = "unpigz"

‎services/server/config/config.go‎

Lines changed: 116 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,15 @@
1717
package config
1818

1919
import (
20+
"path/filepath"
2021
"strings"
2122

2223
"github.com/BurntSushi/toml"
24+
"github.com/imdario/mergo"
25+
"github.com/pkg/errors"
26+
2327
"github.com/containerd/containerd/errdefs"
2428
"github.com/containerd/containerd/plugin"
25-
"github.com/pkg/errors"
2629
)
2730

2831
// Config provides containerd configuration data for the server
@@ -59,16 +62,14 @@ type Config struct {
5962
ProxyPlugins map[string]ProxyPlugin `toml:"proxy_plugins"`
6063
// Timeouts specified as a duration
6164
Timeouts map[string]string `toml:"timeouts"`
65+
// Imports are additional file path list to config files that can overwrite main config file fields
66+
Imports []string `toml:"imports"`
6267

63-
StreamProcessors []StreamProcessor `toml:"stream_processors"`
64-
65-
md toml.MetaData
68+
StreamProcessors map[string]StreamProcessor `toml:"stream_processors"`
6669
}
6770

6871
// StreamProcessor provides configuration for diff content processors
6972
type StreamProcessor struct {
70-
// ID of the processor, also used to fetch the specific payload
71-
ID string `toml:"id"`
7273
// Accepts specific media-types
7374
Accepts []string `toml:"accepts"`
7475
// Returns the media-type
@@ -202,23 +203,125 @@ func (c *Config) Decode(p *plugin.Registration) (interface{}, error) {
202203
if !ok {
203204
return p.Config, nil
204205
}
205-
if err := c.md.PrimitiveDecode(data, p.Config); err != nil {
206+
if err := toml.PrimitiveDecode(data, p.Config); err != nil {
206207
return nil, err
207208
}
208209
return p.Config, nil
209210
}
210211

211212
// LoadConfig loads the containerd server config from the provided path
212-
func LoadConfig(path string, v *Config) error {
213-
if v == nil {
214-
return errors.Wrapf(errdefs.ErrInvalidArgument, "argument v must not be nil")
213+
func LoadConfig(path string, out *Config) error {
214+
if out == nil {
215+
return errors.Wrapf(errdefs.ErrInvalidArgument, "argument out must not be nil")
216+
}
217+
218+
var (
219+
loaded = map[string]bool{}
220+
pending = []string{path}
221+
)
222+
223+
for len(pending) > 0 {
224+
path, pending = pending[0], pending[1:]
225+
226+
// Check if a file at the given path already loaded to prevent circular imports
227+
if _, ok := loaded[path]; ok {
228+
continue
229+
}
230+
231+
config, err := loadConfigFile(path)
232+
if err != nil {
233+
return err
234+
}
235+
236+
if err := mergeConfig(out, config); err != nil {
237+
return err
238+
}
239+
240+
imports, err := resolveImports(path, config.Imports)
241+
if err != nil {
242+
return err
243+
}
244+
245+
loaded[path] = true
246+
pending = append(pending, imports...)
247+
}
248+
249+
// Fix up the list of config files loaded
250+
out.Imports = []string{}
251+
for path := range loaded {
252+
out.Imports = append(out.Imports, path)
253+
}
254+
255+
return out.ValidateV2()
256+
}
257+
258+
// loadConfigFile decodes a TOML file at the given path
259+
func loadConfigFile(path string) (*Config, error) {
260+
config := &Config{}
261+
_, err := toml.DecodeFile(path, &config)
262+
if err != nil {
263+
return nil, err
215264
}
216-
md, err := toml.DecodeFile(path, v)
265+
return config, nil
266+
}
267+
268+
// resolveImports resolves import strings list to absolute paths list:
269+
// - If path contains *, glob pattern matching applied
270+
// - Non abs path is relative to parent config file directory
271+
// - Abs paths returned as is
272+
func resolveImports(parent string, imports []string) ([]string, error) {
273+
var out []string
274+
275+
for _, path := range imports {
276+
if strings.Contains(path, "*") {
277+
matches, err := filepath.Glob(path)
278+
if err != nil {
279+
return nil, err
280+
}
281+
282+
out = append(out, matches...)
283+
} else {
284+
path = filepath.Clean(path)
285+
if !filepath.IsAbs(path) {
286+
path = filepath.Join(filepath.Dir(parent), path)
287+
}
288+
289+
out = append(out, path)
290+
}
291+
}
292+
293+
return out, nil
294+
}
295+
296+
// mergeConfig merges Config structs with the following rules:
297+
// 'to' 'from' 'result'
298+
// "" "value" "value"
299+
// "value" "" "value"
300+
// 1 0 1
301+
// 0 1 1
302+
// []{"1"} []{"2"} []{"1","2"}
303+
// []{"1"} []{} []{"1"}
304+
// Maps merged by keys, but values are replaced entirely.
305+
func mergeConfig(to, from *Config) error {
306+
err := mergo.Merge(to, from, mergo.WithOverride, mergo.WithAppendSlice)
217307
if err != nil {
218308
return err
219309
}
220-
v.md = md
221-
return v.ValidateV2()
310+
311+
// Replace entire sections instead of merging map's values.
312+
for k, v := range from.Plugins {
313+
to.Plugins[k] = v
314+
}
315+
316+
for k, v := range from.StreamProcessors {
317+
to.StreamProcessors[k] = v
318+
}
319+
320+
for k, v := range from.ProxyPlugins {
321+
to.ProxyPlugins[k] = v
322+
}
323+
324+
return nil
222325
}
223326

224327
// V1DisabledFilter matches based on ID

0 commit comments

Comments
 (0)
X Tutup