X Tutup
Skip to content

Commit d7e1b25

Browse files
kevparkevpar
committed
Allow explicit configuration of TTRPC address
Previously the TTRPC address was generated as "<GRPC address>.ttrpc". This change now allows explicit configuration of the TTRPC address, with the default still being the old format if no value is specified. As part of this change, a new configuration section is added for TTRPC listener options. Signed-off-by: Kevin Parsons <kevpar@microsoft.com>
1 parent bd46ea5 commit d7e1b25

File tree

11 files changed

+73
-49
lines changed

11 files changed

+73
-49
lines changed

‎cmd/containerd/command/main.go‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -148,13 +148,16 @@ func App() *cli.App {
148148
for _, w := range warnings {
149149
log.G(ctx).WithError(w).Warn("cleanup temp mount")
150150
}
151-
var (
152-
address = config.GRPC.Address
153-
ttrpcAddress = fmt.Sprintf("%s.ttrpc", config.GRPC.Address)
154-
)
155-
if address == "" {
151+
152+
if config.GRPC.Address == "" {
156153
return errors.Wrap(errdefs.ErrInvalidArgument, "grpc address cannot be empty")
157154
}
155+
if config.TTRPC.Address == "" {
156+
// If TTRPC was not explicitly configured, use defaults based on GRPC.
157+
config.TTRPC.Address = fmt.Sprintf("%s.ttrpc", config.GRPC.Address)
158+
config.TTRPC.UID = config.GRPC.UID
159+
config.TTRPC.GID = config.GRPC.GID
160+
}
158161
log.G(ctx).WithFields(logrus.Fields{
159162
"version": version.Version,
160163
"revision": version.Revision,
@@ -193,7 +196,7 @@ func App() *cli.App {
193196
serve(ctx, l, server.ServeMetrics)
194197
}
195198
// setup the ttrpc endpoint
196-
tl, err := sys.GetLocalListener(ttrpcAddress, config.GRPC.UID, config.GRPC.GID)
199+
tl, err := sys.GetLocalListener(config.TTRPC.Address, config.TTRPC.UID, config.TTRPC.GID)
197200
if err != nil {
198201
return errors.Wrapf(err, "failed to get listener for main ttrpc endpoint")
199202
}
@@ -207,7 +210,7 @@ func App() *cli.App {
207210
serve(ctx, l, server.ServeTCP)
208211
}
209212
// setup the main grpc endpoint
210-
l, err := sys.GetLocalListener(address, config.GRPC.UID, config.GRPC.GID)
213+
l, err := sys.GetLocalListener(config.GRPC.Address, config.GRPC.UID, config.GRPC.GID)
211214
if err != nil {
212215
return errors.Wrapf(err, "failed to get listener for main endpoint")
213216
}

‎plugin/context.go‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,13 @@ import (
2828

2929
// InitContext is used for plugin inititalization
3030
type InitContext struct {
31-
Context context.Context
32-
Root string
33-
State string
34-
Config interface{}
35-
Address string
36-
Events *exchange.Exchange
31+
Context context.Context
32+
Root string
33+
State string
34+
Config interface{}
35+
Address string
36+
TTRPCAddress string
37+
Events *exchange.Exchange
3738

3839
Meta *Meta // plugins can fill in metadata at init.
3940

‎runtime/v2/binary.go‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -35,22 +35,24 @@ import (
3535
"github.com/sirupsen/logrus"
3636
)
3737

38-
func shimBinary(ctx context.Context, bundle *Bundle, runtime, containerdAddress string, events *exchange.Exchange, rt *runtime.TaskList) *binary {
38+
func shimBinary(ctx context.Context, bundle *Bundle, runtime, containerdAddress string, containerdTTRPCAddress string, events *exchange.Exchange, rt *runtime.TaskList) *binary {
3939
return &binary{
40-
bundle: bundle,
41-
runtime: runtime,
42-
containerdAddress: containerdAddress,
43-
events: events,
44-
rtTasks: rt,
40+
bundle: bundle,
41+
runtime: runtime,
42+
containerdAddress: containerdAddress,
43+
containerdTTRPCAddress: containerdTTRPCAddress,
44+
events: events,
45+
rtTasks: rt,
4546
}
4647
}
4748

4849
type binary struct {
49-
runtime string
50-
containerdAddress string
51-
bundle *Bundle
52-
events *exchange.Exchange
53-
rtTasks *runtime.TaskList
50+
runtime string
51+
containerdAddress string
52+
containerdTTRPCAddress string
53+
bundle *Bundle
54+
events *exchange.Exchange
55+
rtTasks *runtime.TaskList
5456
}
5557

5658
func (b *binary) Start(ctx context.Context, opts *types.Any, onClose func()) (_ *shim, err error) {
@@ -64,6 +66,7 @@ func (b *binary) Start(ctx context.Context, opts *types.Any, onClose func()) (_
6466
ctx,
6567
b.runtime,
6668
b.containerdAddress,
69+
b.containerdTTRPCAddress,
6770
b.bundle.Path,
6871
opts,
6972
args...,
@@ -124,6 +127,7 @@ func (b *binary) Delete(ctx context.Context) (*runtime.Exit, error) {
124127
cmd, err := client.Command(ctx,
125128
b.runtime,
126129
b.containerdAddress,
130+
b.containerdTTRPCAddress,
127131
bundlePath,
128132
nil,
129133
"-id", b.bundle.ID,

‎runtime/v2/example/example.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ type service struct {
4444
}
4545

4646
// StartShim is a binary call that executes a new shim returning the address
47-
func (s *service) StartShim(ctx context.Context, id, containerdBinary, containerdAddress string) (string, error) {
47+
func (s *service) StartShim(ctx context.Context, id, containerdBinary, containerdAddress, containerdTTRPCAddress string) (string, error) {
4848
return "", nil
4949
}
5050

‎runtime/v2/manager.go‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -69,25 +69,26 @@ func init() {
6969
if err != nil {
7070
return nil, err
7171
}
72-
return New(ic.Context, ic.Root, ic.State, ic.Address, ic.Events, m.(*metadata.DB))
72+
return New(ic.Context, ic.Root, ic.State, ic.Address, ic.TTRPCAddress, ic.Events, m.(*metadata.DB))
7373
},
7474
})
7575
}
7676

7777
// New task manager for v2 shims
78-
func New(ctx context.Context, root, state, containerdAddress string, events *exchange.Exchange, db *metadata.DB) (*TaskManager, error) {
78+
func New(ctx context.Context, root, state, containerdAddress, containerdTTRPCAddress string, events *exchange.Exchange, db *metadata.DB) (*TaskManager, error) {
7979
for _, d := range []string{root, state} {
8080
if err := os.MkdirAll(d, 0711); err != nil {
8181
return nil, err
8282
}
8383
}
8484
m := &TaskManager{
85-
root: root,
86-
state: state,
87-
containerdAddress: containerdAddress,
88-
tasks: runtime.NewTaskList(),
89-
events: events,
90-
db: db,
85+
root: root,
86+
state: state,
87+
containerdAddress: containerdAddress,
88+
containerdTTRPCAddress: containerdTTRPCAddress,
89+
tasks: runtime.NewTaskList(),
90+
events: events,
91+
db: db,
9192
}
9293
if err := m.loadExistingTasks(ctx); err != nil {
9394
return nil, err
@@ -97,9 +98,10 @@ func New(ctx context.Context, root, state, containerdAddress string, events *exc
9798

9899
// TaskManager manages v2 shim's and their tasks
99100
type TaskManager struct {
100-
root string
101-
state string
102-
containerdAddress string
101+
root string
102+
state string
103+
containerdAddress string
104+
containerdTTRPCAddress string
103105

104106
tasks *runtime.TaskList
105107
events *exchange.Exchange
@@ -131,7 +133,7 @@ func (m *TaskManager) Create(ctx context.Context, id string, opts runtime.Create
131133
topts = opts.RuntimeOptions
132134
}
133135

134-
b := shimBinary(ctx, bundle, opts.Runtime, m.containerdAddress, m.events, m.tasks)
136+
b := shimBinary(ctx, bundle, opts.Runtime, m.containerdAddress, m.containerdTTRPCAddress, m.events, m.tasks)
135137
shim, err := b.Start(ctx, topts, func() {
136138
log.G(ctx).WithField("id", id).Info("shim disconnected")
137139
_, err := m.tasks.Get(ctx, id)
@@ -254,7 +256,7 @@ func (m *TaskManager) loadTasks(ctx context.Context) error {
254256
bundle.Delete()
255257
continue
256258
}
257-
binaryCall := shimBinary(ctx, bundle, container.Runtime.Name, m.containerdAddress, m.events, m.tasks)
259+
binaryCall := shimBinary(ctx, bundle, container.Runtime.Name, m.containerdAddress, m.containerdTTRPCAddress, m.events, m.tasks)
258260
shim, err := loadShim(ctx, bundle, m.events, m.tasks, func() {
259261
log.G(ctx).WithField("id", id).Info("shim disconnected")
260262
_, err := m.tasks.Get(ctx, id)

‎runtime/v2/runc/v1/service.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ type service struct {
102102
cancel func()
103103
}
104104

105-
func newCommand(ctx context.Context, id, containerdBinary, containerdAddress string) (*exec.Cmd, error) {
105+
func newCommand(ctx context.Context, id, containerdBinary, containerdAddress, containerdTTRPCAddress string) (*exec.Cmd, error) {
106106
ns, err := namespaces.NamespaceRequired(ctx)
107107
if err != nil {
108108
return nil, err
@@ -119,6 +119,7 @@ func newCommand(ctx context.Context, id, containerdBinary, containerdAddress str
119119
"-namespace", ns,
120120
"-id", id,
121121
"-address", containerdAddress,
122+
"-ttrpc-address", containerdTTRPCAddress,
122123
}
123124
cmd := exec.Command(self, args...)
124125
cmd.Dir = cwd
@@ -129,8 +130,8 @@ func newCommand(ctx context.Context, id, containerdBinary, containerdAddress str
129130
return cmd, nil
130131
}
131132

132-
func (s *service) StartShim(ctx context.Context, id, containerdBinary, containerdAddress string) (string, error) {
133-
cmd, err := newCommand(ctx, id, containerdBinary, containerdAddress)
133+
func (s *service) StartShim(ctx context.Context, id, containerdBinary, containerdAddress, containerdTTRPCAddress string) (string, error) {
134+
cmd, err := newCommand(ctx, id, containerdBinary, containerdAddress, containerdTTRPCAddress)
134135
if err != nil {
135136
return "", err
136137
}

‎runtime/v2/runc/v2/service.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ type service struct {
118118
cancel func()
119119
}
120120

121-
func newCommand(ctx context.Context, id, containerdBinary, containerdAddress string) (*exec.Cmd, error) {
121+
func newCommand(ctx context.Context, id, containerdBinary, containerdAddress, containerdTTRPCAddress string) (*exec.Cmd, error) {
122122
ns, err := namespaces.NamespaceRequired(ctx)
123123
if err != nil {
124124
return nil, err
@@ -135,6 +135,7 @@ func newCommand(ctx context.Context, id, containerdBinary, containerdAddress str
135135
"-namespace", ns,
136136
"-id", id,
137137
"-address", containerdAddress,
138+
"-ttrpc-address", containerdTTRPCAddress,
138139
}
139140
cmd := exec.Command(self, args...)
140141
cmd.Dir = cwd
@@ -158,8 +159,8 @@ func readSpec() (*spec, error) {
158159
return &s, nil
159160
}
160161

161-
func (s *service) StartShim(ctx context.Context, id, containerdBinary, containerdAddress string) (string, error) {
162-
cmd, err := newCommand(ctx, id, containerdBinary, containerdAddress)
162+
func (s *service) StartShim(ctx context.Context, id, containerdBinary, containerdAddress, containerdTTRPCAddress string) (string, error) {
163+
cmd, err := newCommand(ctx, id, containerdBinary, containerdAddress, containerdTTRPCAddress)
163164
if err != nil {
164165
return "", err
165166
}

‎runtime/v2/shim/shim.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ type Init func(context.Context, string, Publisher, func()) (Shim, error)
5757
type Shim interface {
5858
shimapi.TaskService
5959
Cleanup(ctx context.Context) (*shimapi.DeleteResponse, error)
60-
StartShim(ctx context.Context, id, containerdBinary, containerdAddress string) (string, error)
60+
StartShim(ctx context.Context, id, containerdBinary, containerdAddress, containerdTTRPCAddress string) (string, error)
6161
}
6262

6363
// OptsKey is the context key for the Opts value.
@@ -89,6 +89,7 @@ var (
8989
socketFlag string
9090
bundlePath string
9191
addressFlag string
92+
ttrpcAddressFlag string
9293
containerdBinaryFlag string
9394
action string
9495
)
@@ -101,6 +102,7 @@ func parseFlags() {
101102
flag.StringVar(&bundlePath, "bundle", "", "path to the bundle if not workdir")
102103

103104
flag.StringVar(&addressFlag, "address", "", "grpc address back to main containerd")
105+
flag.StringVar(&ttrpcAddressFlag, "ttrpc-address", "", "ttrpc address back to main containerd")
104106
flag.StringVar(&containerdBinaryFlag, "publish-binary", "containerd", "path to publish binary (used for publishing events)")
105107

106108
flag.Parse()
@@ -163,8 +165,7 @@ func run(id string, initFunc Init, config Config) error {
163165
}
164166
}
165167

166-
address := fmt.Sprintf("%s.ttrpc", addressFlag)
167-
publisher, err := newPublisher(address)
168+
publisher, err := newPublisher(ttrpcAddressFlag)
168169
if err != nil {
169170
return err
170171
}
@@ -203,7 +204,7 @@ func run(id string, initFunc Init, config Config) error {
203204
}
204205
return nil
205206
case "start":
206-
address, err := service.StartShim(ctx, idFlag, containerdBinaryFlag, addressFlag)
207+
address, err := service.StartShim(ctx, idFlag, containerdBinaryFlag, addressFlag, ttrpcAddressFlag)
207208
if err != nil {
208209
return err
209210
}

‎runtime/v2/shim/util.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ import (
3838
var runtimePaths sync.Map
3939

4040
// Command returns the shim command with the provided args and configuration
41-
func Command(ctx context.Context, runtime, containerdAddress, path string, opts *types.Any, cmdArgs ...string) (*exec.Cmd, error) {
41+
func Command(ctx context.Context, runtime, containerdAddress, containerdTTRPCAddress, path string, opts *types.Any, cmdArgs ...string) (*exec.Cmd, error) {
4242
ns, err := namespaces.NamespaceRequired(ctx)
4343
if err != nil {
4444
return nil, err
@@ -50,6 +50,7 @@ func Command(ctx context.Context, runtime, containerdAddress, path string, opts
5050
args := []string{
5151
"-namespace", ns,
5252
"-address", containerdAddress,
53+
"-ttrpc-address", containerdTTRPCAddress,
5354
"-publish-binary", self,
5455
}
5556
args = append(args, cmdArgs...)

‎services/server/config/config.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ type Config struct {
3737
PluginDir string `toml:"plugin_dir"`
3838
// GRPC configuration settings
3939
GRPC GRPCConfig `toml:"grpc"`
40+
// TTRPC configuration settings
41+
TTRPC TTRPCConfig `toml:"ttrpc"`
4042
// Debug and profiling settings
4143
Debug Debug `toml:"debug"`
4244
// Metrics and monitoring settings
@@ -125,6 +127,13 @@ type GRPCConfig struct {
125127
MaxSendMsgSize int `toml:"max_send_message_size"`
126128
}
127129

130+
// TTRPCConfig provides TTRPC configuration for the socket
131+
type TTRPCConfig struct {
132+
Address string `toml:"address"`
133+
UID int `toml:"uid"`
134+
GID int `toml:"gid"`
135+
}
136+
128137
// Debug provides debug configuration
129138
type Debug struct {
130139
Address string `toml:"address"`

0 commit comments

Comments
 (0)
X Tutup