X Tutup
Skip to content

Commit bcc4a14

Browse files
dmcgowandmcgowan
committed
Support applying with parent directories
Signed-off-by: Derek McGowan <derek@mcgstyle.net>
1 parent 5a0ff41 commit bcc4a14

File tree

11 files changed

+483
-160
lines changed

11 files changed

+483
-160
lines changed

‎archive/tar.go‎

Lines changed: 113 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -110,11 +110,15 @@ func Apply(ctx context.Context, root string, r io.Reader, opts ...ApplyOpt) (int
110110
if options.Filter == nil {
111111
options.Filter = all
112112
}
113+
if options.applyFunc == nil {
114+
options.applyFunc = applyNaive
115+
}
113116

114-
return apply(ctx, root, tar.NewReader(r), options)
117+
return options.applyFunc(ctx, root, tar.NewReader(r), options)
115118
}
116119

117-
// applyNaive applies a tar stream of an OCI style diff tar.
120+
// applyNaive applies a tar stream of an OCI style diff tar to a directory
121+
// applying each file as either a whole file or whiteout.
118122
// See https://github.com/opencontainers/image-spec/blob/master/layer.md#applying-changesets
119123
func applyNaive(ctx context.Context, root string, tr *tar.Reader, options ApplyOptions) (size int64, err error) {
120124
var (
@@ -123,8 +127,50 @@ func applyNaive(ctx context.Context, root string, tr *tar.Reader, options ApplyO
123127
// Used for handling opaque directory markers which
124128
// may occur out of order
125129
unpackedPaths = make(map[string]struct{})
130+
131+
convertWhiteout = options.ConvertWhiteout
126132
)
127133

134+
if convertWhiteout == nil {
135+
// handle whiteouts by removing the target files
136+
convertWhiteout = func(hdr *tar.Header, path string) (bool, error) {
137+
base := filepath.Base(path)
138+
dir := filepath.Dir(path)
139+
if base == whiteoutOpaqueDir {
140+
_, err := os.Lstat(dir)
141+
if err != nil {
142+
return false, err
143+
}
144+
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
145+
if err != nil {
146+
if os.IsNotExist(err) {
147+
err = nil // parent was deleted
148+
}
149+
return err
150+
}
151+
if path == dir {
152+
return nil
153+
}
154+
if _, exists := unpackedPaths[path]; !exists {
155+
err := os.RemoveAll(path)
156+
return err
157+
}
158+
return nil
159+
})
160+
return false, err
161+
}
162+
163+
if strings.HasPrefix(base, whiteoutPrefix) {
164+
originalBase := base[len(whiteoutPrefix):]
165+
originalPath := filepath.Join(dir, originalBase)
166+
167+
return false, os.RemoveAll(originalPath)
168+
}
169+
170+
return true, nil
171+
}
172+
}
173+
128174
// Iterate through the files in the archive.
129175
for {
130176
select {
@@ -182,55 +228,13 @@ func applyNaive(ctx context.Context, root string, tr *tar.Reader, options ApplyO
182228
if base == "" {
183229
parentPath = filepath.Dir(path)
184230
}
185-
if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) {
186-
err = mkdirAll(parentPath, 0755)
187-
if err != nil {
188-
return 0, err
189-
}
231+
if err := mkparent(ctx, parentPath, root, options.Parents); err != nil {
232+
return 0, err
190233
}
191234
}
192235

193236
// Naive whiteout convert function which handles whiteout files by
194237
// removing the target files.
195-
convertWhiteout := func(hdr *tar.Header, path string) (bool, error) {
196-
base := filepath.Base(path)
197-
dir := filepath.Dir(path)
198-
if base == whiteoutOpaqueDir {
199-
_, err := os.Lstat(dir)
200-
if err != nil {
201-
return false, err
202-
}
203-
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
204-
if err != nil {
205-
if os.IsNotExist(err) {
206-
err = nil // parent was deleted
207-
}
208-
return err
209-
}
210-
if path == dir {
211-
return nil
212-
}
213-
if _, exists := unpackedPaths[path]; !exists {
214-
err := os.RemoveAll(path)
215-
return err
216-
}
217-
return nil
218-
})
219-
return false, err
220-
}
221-
222-
if strings.HasPrefix(base, whiteoutPrefix) {
223-
originalBase := base[len(whiteoutPrefix):]
224-
originalPath := filepath.Join(dir, originalBase)
225-
226-
return false, os.RemoveAll(originalPath)
227-
}
228-
229-
return true, nil
230-
}
231-
if options.ConvertWhiteout != nil {
232-
convertWhiteout = options.ConvertWhiteout
233-
}
234238
if err := validateWhiteout(path); err != nil {
235239
return 0, err
236240
}
@@ -375,6 +379,66 @@ func createTarFile(ctx context.Context, path, extractDir string, hdr *tar.Header
375379
return chtimes(path, boundTime(latestTime(hdr.AccessTime, hdr.ModTime)), boundTime(hdr.ModTime))
376380
}
377381

382+
func mkparent(ctx context.Context, path, root string, parents []string) error {
383+
if dir, err := os.Lstat(path); err == nil {
384+
if dir.IsDir() {
385+
return nil
386+
}
387+
return &os.PathError{
388+
Op: "mkparent",
389+
Path: path,
390+
Err: syscall.ENOTDIR,
391+
}
392+
} else if !os.IsNotExist(err) {
393+
return err
394+
}
395+
396+
i := len(path)
397+
for i > len(root) && !os.IsPathSeparator(path[i-1]) {
398+
i--
399+
}
400+
401+
if i > len(root)+1 {
402+
if err := mkparent(ctx, path[:i-1], root, parents); err != nil {
403+
return err
404+
}
405+
}
406+
407+
if err := mkdir(path, 0755); err != nil {
408+
// Check that still doesn't exist
409+
dir, err1 := os.Lstat(path)
410+
if err1 == nil && dir.IsDir() {
411+
return nil
412+
}
413+
return err
414+
}
415+
416+
for _, p := range parents {
417+
ppath, err := fs.RootPath(p, path[len(root):])
418+
if err != nil {
419+
return err
420+
}
421+
422+
dir, err := os.Lstat(ppath)
423+
if err == nil {
424+
if !dir.IsDir() {
425+
// Replaced, do not copy attributes
426+
break
427+
}
428+
if err := copyDirInfo(dir, path); err != nil {
429+
return err
430+
}
431+
return copyUpXAttrs(path, ppath)
432+
} else if !os.IsNotExist(err) {
433+
return err
434+
}
435+
}
436+
437+
log.G(ctx).Debugf("parent directory %q not found: default permissions(0755) used", path)
438+
439+
return nil
440+
}
441+
378442
type changeWriter struct {
379443
tw *tar.Writer
380444
source string
@@ -545,6 +609,9 @@ func (cw *changeWriter) Close() error {
545609
}
546610

547611
func (cw *changeWriter) includeParents(hdr *tar.Header) error {
612+
if cw.addedDirs == nil {
613+
return nil
614+
}
548615
name := strings.TrimRight(hdr.Name, "/")
549616
fname := filepath.Join(cw.source, name)
550617
parent := filepath.Dir(name)

‎archive/tar_linux_test.go‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
// +build linux
2+
3+
/*
4+
Copyright The containerd Authors.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
package archive
20+
21+
import (
22+
"bytes"
23+
"context"
24+
"fmt"
25+
"io"
26+
"io/ioutil"
27+
"os"
28+
"strings"
29+
"testing"
30+
31+
"github.com/containerd/containerd/log/logtest"
32+
"github.com/containerd/containerd/mount"
33+
"github.com/containerd/containerd/pkg/testutil"
34+
"github.com/containerd/containerd/snapshots/overlay"
35+
"github.com/containerd/continuity/fs"
36+
"github.com/containerd/continuity/fs/fstest"
37+
"github.com/pkg/errors"
38+
)
39+
40+
func TestOverlayApply(t *testing.T) {
41+
testutil.RequiresRoot(t)
42+
43+
base, err := ioutil.TempDir("", "test-ovl-diff-apply-")
44+
if err != nil {
45+
t.Fatalf("unable to create temp dir: %+v", err)
46+
}
47+
defer os.RemoveAll(base)
48+
49+
if err := overlay.Supported(base); err != nil {
50+
t.Skipf("skipping because overlay is not supported %v", err)
51+
}
52+
fstest.FSSuite(t, overlayDiffApplier{
53+
tmp: base,
54+
diff: WriteDiff,
55+
t: t,
56+
})
57+
}
58+
59+
func TestOverlayApplyNoParents(t *testing.T) {
60+
testutil.RequiresRoot(t)
61+
62+
base, err := ioutil.TempDir("", "test-ovl-diff-apply-")
63+
if err != nil {
64+
t.Fatalf("unable to create temp dir: %+v", err)
65+
}
66+
defer os.RemoveAll(base)
67+
68+
if err := overlay.Supported(base); err != nil {
69+
t.Skipf("skipping because overlay is not supported %v", err)
70+
}
71+
fstest.FSSuite(t, overlayDiffApplier{
72+
tmp: base,
73+
diff: func(ctx context.Context, w io.Writer, a, b string) error {
74+
cw := newChangeWriter(w, b)
75+
cw.addedDirs = nil
76+
err := fs.Changes(ctx, a, b, cw.HandleChange)
77+
if err != nil {
78+
return errors.Wrap(err, "failed to create diff tar stream")
79+
}
80+
return cw.Close()
81+
},
82+
t: t,
83+
})
84+
}
85+
86+
type overlayDiffApplier struct {
87+
tmp string
88+
diff func(context.Context, io.Writer, string, string) error
89+
t *testing.T
90+
}
91+
92+
type overlayContext struct {
93+
merged string
94+
lowers []string
95+
mounted bool
96+
}
97+
98+
type contextKey struct{}
99+
100+
func (d overlayDiffApplier) TestContext(ctx context.Context) (context.Context, func(), error) {
101+
merged, err := ioutil.TempDir(d.tmp, "merged")
102+
if err != nil {
103+
return ctx, nil, errors.Wrap(err, "failed to make merged dir")
104+
}
105+
106+
oc := &overlayContext{
107+
merged: merged,
108+
}
109+
110+
ctx = logtest.WithT(ctx, d.t)
111+
112+
return context.WithValue(ctx, contextKey{}, oc), func() {
113+
if oc.mounted {
114+
mount.Unmount(oc.merged, 0)
115+
}
116+
}, nil
117+
}
118+
119+
func (d overlayDiffApplier) Apply(ctx context.Context, a fstest.Applier) (string, func(), error) {
120+
oc := ctx.Value(contextKey{}).(*overlayContext)
121+
122+
applyCopy, err := ioutil.TempDir(d.tmp, "apply-copy-")
123+
if err != nil {
124+
return "", nil, errors.Wrap(err, "failed to create temp dir")
125+
}
126+
defer os.RemoveAll(applyCopy)
127+
128+
base := oc.merged
129+
if len(oc.lowers) == 1 {
130+
base = oc.lowers[0]
131+
}
132+
133+
if err = fs.CopyDir(applyCopy, base); err != nil {
134+
return "", nil, errors.Wrap(err, "failed to copy base")
135+
}
136+
137+
if err := a.Apply(applyCopy); err != nil {
138+
return "", nil, errors.Wrap(err, "failed to apply changes to copy of base")
139+
}
140+
141+
buf := bytes.NewBuffer(nil)
142+
143+
if err := d.diff(ctx, buf, base, applyCopy); err != nil {
144+
return "", nil, errors.Wrap(err, "failed to create diff")
145+
}
146+
147+
if oc.mounted {
148+
if err := mount.Unmount(oc.merged, 0); err != nil {
149+
return "", nil, errors.Wrap(err, "failed to unmount")
150+
}
151+
oc.mounted = false
152+
}
153+
154+
next, err := ioutil.TempDir(d.tmp, "lower-")
155+
if err != nil {
156+
return "", nil, errors.Wrap(err, "failed to create temp dir")
157+
}
158+
159+
if _, err = Apply(ctx, next, buf, WithConvertWhiteout(OverlayConvertWhiteout), WithParents(oc.lowers)); err != nil {
160+
return "", nil, errors.Wrap(err, "failed to apply tar stream")
161+
}
162+
163+
oc.lowers = append([]string{next}, oc.lowers...)
164+
165+
if len(oc.lowers) == 1 {
166+
return oc.lowers[0], nil, nil
167+
}
168+
169+
m := mount.Mount{
170+
Type: "overlay",
171+
Source: "overlay",
172+
Options: []string{
173+
fmt.Sprintf("lowerdir=%s", strings.Join(oc.lowers, ":")),
174+
},
175+
}
176+
177+
if err := m.Mount(oc.merged); err != nil {
178+
return "", nil, errors.Wrapf(err, "failed to mount: %v", m)
179+
}
180+
oc.mounted = true
181+
182+
return oc.merged, nil, nil
183+
}

0 commit comments

Comments
 (0)
X Tutup