X Tutup
Skip to content

Commit f57c5cd

Browse files
dmcgowandmcgowan
committed
Refactor image importer
Allow customization of reference creation. Add option for digest references. Signed-off-by: Derek McGowan <derek@mcgstyle.net>
1 parent 05984a9 commit f57c5cd

File tree

6 files changed

+206
-212
lines changed

6 files changed

+206
-212
lines changed

‎cmd/ctr/commands/images/import.go‎

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,10 @@ import (
2020
"fmt"
2121
"io"
2222
"os"
23+
"time"
2324

25+
"github.com/containerd/containerd"
2426
"github.com/containerd/containerd/cmd/ctr/commands"
25-
"github.com/containerd/containerd/images"
2627
oci "github.com/containerd/containerd/images/oci"
2728
"github.com/containerd/containerd/log"
2829
"github.com/urfave/cli"
@@ -53,23 +54,34 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb
5354
Usage: "image format. See DESCRIPTION.",
5455
},
5556
cli.StringFlag{
56-
Name: "oci-name",
57-
Value: "unknown/unknown",
58-
Usage: "prefix added to either oci.v1 ref annotation or digest",
57+
Name: "prefix,oci-name",
58+
Value: "",
59+
Usage: "prefix image name for added images",
60+
},
61+
cli.BoolFlag{
62+
Name: "digests",
63+
Usage: "whether to create digest images",
5964
},
60-
// TODO(AkihiroSuda): support commands.LabelFlag (for all children objects)
6165
}, commands.SnapshotterFlags...),
6266

6367
Action: func(context *cli.Context) error {
6468
var (
65-
in = context.Args().First()
66-
imageImporter images.Importer
69+
in = context.Args().First()
70+
opts []containerd.ImportOpt
6771
)
6872

6973
switch format := context.String("format"); format {
7074
case "oci.v1":
71-
imageImporter = &oci.V1Importer{
72-
ImageName: context.String("oci-name"),
75+
opts = append(opts, containerd.WithImporter(&oci.V1Importer{}))
76+
77+
prefix := context.String("prefix")
78+
if prefix == "" {
79+
prefix = fmt.Sprintf("import-%s", time.Now().Format("2006-01-02"))
80+
}
81+
82+
opts = append(opts, containerd.WithImageRefTranslator(oci.RefTranslator(prefix)))
83+
if context.Bool("digests") {
84+
opts = append(opts, containerd.WithDigestRef(oci.DigestTranslator(prefix)))
7385
}
7486
default:
7587
return fmt.Errorf("unknown format %s", format)
@@ -90,20 +102,24 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb
90102
return err
91103
}
92104
}
93-
imgs, err := client.Import(ctx, imageImporter, r)
105+
imgs, err := client.Import(ctx, r, opts...)
106+
closeErr := r.Close()
94107
if err != nil {
95108
return err
96109
}
97-
if err = r.Close(); err != nil {
98-
return err
110+
if closeErr != nil {
111+
return closeErr
99112
}
100113

101114
log.G(ctx).Debugf("unpacking %d images", len(imgs))
102115

103116
for _, img := range imgs {
117+
// TODO: Allow configuration of the platform
118+
image := containerd.NewImage(client, img)
119+
104120
// TODO: Show unpack status
105-
fmt.Printf("unpacking %s (%s)...", img.Name(), img.Target().Digest)
106-
err = img.Unpack(ctx, context.String("snapshotter"))
121+
fmt.Printf("unpacking %s (%s)...", img.Name, img.Target.Digest)
122+
err = image.Unpack(ctx, context.String("snapshotter"))
107123
if err != nil {
108124
return err
109125
}

‎images/importexport.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import (
2727
// Importer is the interface for image importer.
2828
type Importer interface {
2929
// Import imports an image from a tar stream.
30-
Import(ctx context.Context, store content.Store, reader io.Reader) ([]Image, error)
30+
Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error)
3131
}
3232

3333
// Exporter is the interface for image exporter.

‎images/oci/importer.go‎

Lines changed: 48 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -19,114 +19,89 @@ package oci
1919

2020
import (
2121
"archive/tar"
22+
"bytes"
2223
"context"
23-
"encoding/json"
24-
"fmt"
2524
"io"
2625
"io/ioutil"
2726
"path"
2827
"strings"
2928

3029
"github.com/containerd/containerd/content"
31-
"github.com/containerd/containerd/errdefs"
3230
"github.com/containerd/containerd/images"
31+
"github.com/containerd/containerd/log"
3332
digest "github.com/opencontainers/go-digest"
3433
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
3534
"github.com/pkg/errors"
3635
)
3736

3837
// V1Importer implements OCI Image Spec v1.
39-
type V1Importer struct {
40-
// ImageName is preprended to either `:` + OCI ref name or `@` + digest (for anonymous refs).
41-
// This field is mandatory atm, but may change in the future. maybe ref map[string]string as in moby/moby#33355
42-
ImageName string
43-
}
38+
type V1Importer struct{}
4439

4540
var _ images.Importer = &V1Importer{}
4641

4742
// Import implements Importer.
48-
func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) ([]images.Image, error) {
49-
if oi.ImageName == "" {
50-
return nil, errors.New("ImageName not set")
51-
}
52-
tr := tar.NewReader(reader)
53-
var imgrecs []images.Image
54-
foundIndexJSON := false
43+
func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) (ocispec.Descriptor, error) {
44+
var (
45+
desc ocispec.Descriptor
46+
tr = tar.NewReader(reader)
47+
)
5548
for {
5649
hdr, err := tr.Next()
5750
if err == io.EOF {
5851
break
5952
}
6053
if err != nil {
61-
return nil, err
54+
return ocispec.Descriptor{}, err
6255
}
6356
if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
57+
log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored")
6458
continue
6559
}
6660
hdrName := path.Clean(hdr.Name)
6761
if hdrName == "index.json" {
68-
if foundIndexJSON {
69-
return nil, errors.New("duplicated index.json")
62+
if desc.Digest != "" {
63+
return ocispec.Descriptor{}, errors.New("duplicated index.json")
7064
}
71-
foundIndexJSON = true
72-
imgrecs, err = onUntarIndexJSON(tr, oi.ImageName)
65+
desc, err = onUntarIndexJSON(ctx, tr, store, hdr.Size)
7366
if err != nil {
74-
return nil, err
67+
return ocispec.Descriptor{}, err
7568
}
76-
continue
77-
}
78-
if strings.HasPrefix(hdrName, "blobs/") {
69+
} else if strings.HasPrefix(hdrName, "blobs/") {
7970
if err := onUntarBlob(ctx, tr, store, hdrName, hdr.Size); err != nil {
80-
return nil, err
71+
return ocispec.Descriptor{}, err
8172
}
73+
} else if hdrName == ocispec.ImageLayoutFile {
74+
// TODO Validate
75+
} else {
76+
log.G(ctx).WithField("file", hdr.Name).Debug("unknown file ignored")
8277
}
8378
}
84-
if !foundIndexJSON {
85-
return nil, errors.New("no index.json found")
86-
}
87-
for _, img := range imgrecs {
88-
err := setGCRefContentLabels(ctx, store, img.Target)
89-
if err != nil {
90-
return imgrecs, err
91-
}
79+
if desc.Digest == "" {
80+
return ocispec.Descriptor{}, errors.New("no index.json found")
9281
}
93-
// FIXME(AkihiroSuda): set GC labels for unreferrenced blobs (i.e. with unknown media types)?
94-
return imgrecs, nil
82+
83+
return desc, nil
9584
}
9685

97-
func onUntarIndexJSON(r io.Reader, imageName string) ([]images.Image, error) {
86+
func onUntarIndexJSON(ctx context.Context, r io.Reader, store content.Ingester, size int64) (ocispec.Descriptor, error) {
9887
b, err := ioutil.ReadAll(r)
9988
if err != nil {
100-
return nil, err
89+
return ocispec.Descriptor{}, err
10190
}
102-
var idx ocispec.Index
103-
if err := json.Unmarshal(b, &idx); err != nil {
104-
return nil, err
91+
desc := ocispec.Descriptor{
92+
MediaType: ocispec.MediaTypeImageIndex,
93+
Digest: digest.FromBytes(b),
94+
Size: size,
10595
}
106-
var imgrecs []images.Image
107-
for _, m := range idx.Manifests {
108-
ref, err := normalizeImageRef(imageName, m)
109-
if err != nil {
110-
return nil, err
111-
}
112-
imgrecs = append(imgrecs, images.Image{
113-
Name: ref,
114-
Target: m,
115-
})
96+
if int64(len(b)) != size {
97+
return ocispec.Descriptor{}, errors.Errorf("size mismatch %d v %d", len(b), size)
11698
}
117-
return imgrecs, nil
118-
}
11999

120-
func normalizeImageRef(imageName string, manifest ocispec.Descriptor) (string, error) {
121-
digest := manifest.Digest
122-
if digest == "" {
123-
return "", errors.Errorf("manifest with empty digest: %v", manifest)
124-
}
125-
ociRef := manifest.Annotations[ocispec.AnnotationRefName]
126-
if ociRef == "" {
127-
return imageName + "@" + digest.String(), nil
100+
if err := content.WriteBlob(ctx, store, "index-"+desc.Digest.String(), bytes.NewReader(b), desc); err != nil {
101+
return ocispec.Descriptor{}, err
128102
}
129-
return imageName + ":" + ociRef, nil
103+
104+
return desc, err
130105
}
131106

132107
func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name string, size int64) error {
@@ -140,65 +115,22 @@ func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name
140115
return errors.Errorf("unsupported algorithm: %s", algo)
141116
}
142117
dgst := digest.NewDigestFromHex(algo.String(), split[2])
143-
return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst})
118+
return content.WriteBlob(ctx, store, "blob-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst})
144119
}
145120

146-
// GetChildrenDescriptors returns children blob descriptors for the following supported types:
147-
// - images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest
148-
// - images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex
149-
func GetChildrenDescriptors(r io.Reader, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
150-
switch desc.MediaType {
151-
case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
152-
var manifest ocispec.Manifest
153-
if err := json.NewDecoder(r).Decode(&manifest); err != nil {
154-
return nil, err
155-
}
156-
return append([]ocispec.Descriptor{manifest.Config}, manifest.Layers...), nil
157-
case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
158-
var index ocispec.Index
159-
if err := json.NewDecoder(r).Decode(&index); err != nil {
160-
return nil, err
161-
}
162-
return index.Manifests, nil
121+
// RefTranslator creates a reference using an OCI ref annotation,
122+
// which is mentioned in the spec as only a tag compontent,
123+
// concatenated with an image name
124+
func RefTranslator(prefix string) func(string) string {
125+
return func(ref string) string {
126+
return prefix + ":" + ref
163127
}
164-
return nil, nil
165128
}
166129

167-
func setGCRefContentLabels(ctx context.Context, store content.Store, desc ocispec.Descriptor) error {
168-
info, err := store.Info(ctx, desc.Digest)
169-
if err != nil {
170-
if errdefs.IsNotFound(err) {
171-
// when the archive is created from multi-arch image,
172-
// it may contain only blobs for a certain platform.
173-
// So ErrNotFound (on manifest list) is expected here.
174-
return nil
175-
}
176-
return err
177-
}
178-
ra, err := store.ReaderAt(ctx, desc)
179-
if err != nil {
180-
return err
181-
}
182-
defer ra.Close()
183-
r := content.NewReader(ra)
184-
children, err := GetChildrenDescriptors(r, desc)
185-
if err != nil {
186-
return err
187-
}
188-
if info.Labels == nil {
189-
info.Labels = map[string]string{}
190-
}
191-
for i, child := range children {
192-
// Note: child blob is not guaranteed to be written to the content store. (multi-arch)
193-
info.Labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = child.Digest.String()
194-
}
195-
if _, err := store.Update(ctx, info, "labels"); err != nil {
196-
return err
197-
}
198-
for _, child := range children {
199-
if err := setGCRefContentLabels(ctx, store, child); err != nil {
200-
return err
201-
}
130+
// DigestTranslator creates a digest reference by adding the
131+
// digest to an image name
132+
func DigestTranslator(prefix string) func(digest.Digest) string {
133+
return func(dgst digest.Digest) string {
134+
return prefix + "@" + dgst.String()
202135
}
203-
return nil
204136
}

‎images/oci/importer_test.go‎

Lines changed: 0 additions & 53 deletions
This file was deleted.

0 commit comments

Comments
 (0)
X Tutup