alextab93/buildx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request #3430 from tonistiigi/imagetools-create-platforms

Browse Source 
imagetools: add platforms filter to imagetools create
This commit is contained in:
Tõnis Tiigi
2025-09-23 08:47:52 -07:00
committed by GitHub
parent e4ca56fe62 f448f3e22b
commit 568c693972
6 changed files with 189 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/build.go
+1 -1
View File
@@ -723,7 +723,7 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[
return err
}
dt, desc, err := itpull.Combine(ctx, srcs, indexAnnotations, false)
dt, desc, _, err := itpull.Combine(ctx, srcs, indexAnnotations, false)
if err != nil {
return err
}
commands/imagetools/create.go
+145 -9
View File
@@ -7,6 +7,8 @@ import (
"os"
"strings"
"github.com/containerd/containerd/v2/core/images"
"github.com/containerd/platforms"
"github.com/distribution/reference"
"github.com/docker/buildx/builder"
"github.com/docker/buildx/util/buildflags"
@@ -14,6 +16,7 @@ import (
"github.com/docker/buildx/util/imagetools"
"github.com/docker/buildx/util/progress"
"github.com/docker/cli/cli/command"
"github.com/moby/buildkit/util/attestation"
"github.com/moby/buildkit/util/progress/progressui"
"github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -31,6 +34,7 @@ type createOptions struct {
actionAppend bool
progress string
preferIndex bool
platforms []string
}
func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, args []string) error {
@@ -67,6 +71,11 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
return err
}
platforms, err := parsePlatforms(in.platforms)
if err != nil {
return err
}
repos := map[string]struct{}{}
for _, t := range tags {
@@ -160,7 +169,12 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
return errors.Wrapf(err, "failed to parse annotations")
}
dt, desc, err := r.Combine(ctx, srcs, annotations, in.preferIndex)
dt, desc, srcMap, err := r.Combine(ctx, srcs, annotations, in.preferIndex)
if err != nil {
return err
}
dt, desc, manifests, err := filterPlatforms(dt, desc, srcMap, platforms)
if err != nil {
return err
}
@@ -170,6 +184,11 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
return nil
}
// manifests can be nil only if pushing one single-platform desc directly
if manifests == nil {
manifests = []descWithSource{{Descriptor: desc, Source: srcs[0]}}
}
// new resolver cause need new auth
r = imagetools.New(imageopt)
@@ -187,17 +206,12 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
eg.Go(func() error {
return progress.Wrap(fmt.Sprintf("pushing %s", t.String()), pw.Write, func(sub progress.SubLogger) error {
eg2, _ := errgroup.WithContext(ctx)
for _, s := range srcs {
if reference.Domain(s.Ref) == reference.Domain(t) && reference.Path(s.Ref) == reference.Path(t) {
continue
}
s := s
for _, desc := range manifests {
eg2.Go(func() error {
sub.Log(1, fmt.Appendf(nil, "copying %s from %s to %s\n", s.Desc.Digest.String(), s.Ref.String(), t.String()))
return r.Copy(ctx, s, t)
sub.Log(1, fmt.Appendf(nil, "copying %s from %s to %s\n", desc.Digest.String(), desc.Source.Ref.String(), t.String()))
return r.Copy(ctx, desc.Source, t)
})
}
if err := eg2.Wait(); err != nil {
return err
}
@@ -216,6 +230,107 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg
return err
}
type descWithSource struct {
ocispecs.Descriptor
Source *imagetools.Source
}
func filterPlatforms(dt []byte, desc ocispecs.Descriptor, srcMap map[digest.Digest]*imagetools.Source, plats []ocispecs.Platform) ([]byte, ocispecs.Descriptor, []descWithSource, error) {
if len(plats) == 0 {
return dt, desc, nil, nil
}
matcher := platforms.Any(plats...)
if !images.IsIndexType(desc.MediaType) {
var mfst ocispecs.Manifest
if err := json.Unmarshal(dt, &mfst); err != nil {
return nil, ocispecs.Descriptor{}, nil, errors.Wrapf(err, "failed to parse manifest")
}
if desc.Platform == nil {
return nil, ocispecs.Descriptor{}, nil, errors.Errorf("cannot filter platforms from a manifest without platform information")
}
if !matcher.Match(*desc.Platform) {
return nil, ocispecs.Descriptor{}, nil, errors.Errorf("input platform %s does not match any of the provided platforms", platforms.Format(*desc.Platform))
}
return dt, desc, nil, nil
}
var idx ocispecs.Index
if err := json.Unmarshal(dt, &idx); err != nil {
return nil, ocispecs.Descriptor{}, nil, errors.Wrapf(err, "failed to parse index")
}
manifestMap := map[digest.Digest]ocispecs.Descriptor{}
for _, m := range idx.Manifests {
manifestMap[m.Digest] = m
}
references := map[digest.Digest]struct{}{}
for _, m := range idx.Manifests {
if refType, ok := m.Annotations[attestation.DockerAnnotationReferenceType]; ok && refType == attestation.DockerAnnotationReferenceTypeDefault {
dgstStr, ok := m.Annotations[attestation.DockerAnnotationReferenceDigest]
if !ok {
continue
}
dgst, err := digest.Parse(dgstStr)
if err != nil {
continue
}
subject, ok := manifestMap[dgst]
if !ok {
continue
}
if subject.Platform == nil || matcher.Match(*subject.Platform) {
references[m.Digest] = struct{}{}
}
}
}
var mfsts []ocispecs.Descriptor
var mfstsWithSource []descWithSource
for _, m := range idx.Manifests {
if _, isRef := references[m.Digest]; isRef || m.Platform == nil || matcher.Match(*m.Platform) {
src, ok := srcMap[m.Digest]
if !ok {
defaultSource, ok := srcMap[desc.Digest]
if !ok {
return nil, ocispecs.Descriptor{}, nil, errors.Errorf("internal error: no source found for %s", m.Digest)
}
src = defaultSource
}
mfsts = append(mfsts, m)
mfstsWithSource = append(mfstsWithSource, descWithSource{
Descriptor: m,
Source: src,
})
}
}
if len(mfsts) == len(idx.Manifests) {
// all platforms matched, no need to rewrite index
return dt, desc, mfstsWithSource, nil
}
if len(mfsts) == 0 {
return nil, ocispecs.Descriptor{}, nil, errors.Errorf("none of the manifests match the provided platforms")
}
idx.Manifests = mfsts
idxBytes, err := json.MarshalIndent(&idx, "", " ")
if err != nil {
return nil, ocispecs.Descriptor{}, nil, errors.Wrap(err, "failed to marshal index")
}
desc = ocispecs.Descriptor{
MediaType: desc.MediaType,
Size: int64(len(idxBytes)),
Digest: digest.FromBytes(idxBytes),
Annotations: desc.Annotations,
}
return idxBytes, desc, mfstsWithSource, nil
}
func parseSources(in []string) ([]*imagetools.Source, error) {
out := make([]*imagetools.Source, len(in))
for i, in := range in {
@@ -228,6 +343,26 @@ func parseSources(in []string) ([]*imagetools.Source, error) {
return out, nil
}
func parsePlatforms(in []string) ([]ocispecs.Platform, error) {
out := make([]ocispecs.Platform, 0, len(in))
for _, p := range in {
if arr := strings.Split(p, ","); len(arr) > 1 {
v, err := parsePlatforms(arr)
if err != nil {
return nil, err
}
out = append(out, v...)
continue
}
plat, err := platforms.Parse(p)
if err != nil {
return nil, errors.Wrapf(err, "invalid platform %q", p)
}
out = append(out, plat)
}
return out, nil
}
func parseRefs(in []string) ([]reference.Named, error) {
refs := make([]reference.Named, len(in))
for i, in := range in {
@@ -291,6 +426,7 @@ func createCmd(dockerCli command.Cli, opts RootOptions) *cobra.Command {
flags.StringVar(&options.progress, "progress", "auto", `Set type of progress output ("auto", "plain", "tty", "rawjson"). Use plain to show container output`)
flags.StringArrayVarP(&options.annotations, "annotation", "", []string{}, "Add annotation to the image")
flags.BoolVar(&options.preferIndex, "prefer-index", true, "When only a single source is specified, prefer outputting an image index or manifest list instead of performing a carbon copy")
flags.StringArrayVarP(&options.platforms, "platform", "p", []string{}, "Filter specified platforms of target image")
return cmd
}
docs/reference/buildx_imagetools_create.md
+1
View File
@@ -17,6 +17,7 @@ Create a new image based on source images
| `-D`, `--debug` | `bool` | | Enable debug logging |
| [`--dry-run`](#dry-run) | `bool` | | Show final image instead of pushing |
| [`-f`](#file), [`--file`](#file) | `stringArray` | | Read source descriptor from file |
| `-p`, `--platform` | `stringArray` | | Filter specified platforms of target image |
| `--prefer-index` | `bool` | `true` | When only a single source is specified, prefer outputting an image index or manifest list instead of performing a carbon copy |
| `--progress` | `string` | `auto` | Set type of progress output (`auto`, `plain`, `tty`, `rawjson`). Use plain to show container output |
| [`-t`](#tag), [`--tag`](#tag) | `stringArray` | | Set reference for new image |
util/imagetools/create.go
+32 -27
View File
@@ -28,7 +28,7 @@ type Source struct {
Ref reference.Named
}
func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes.AnnotationKey]string, preferIndex bool) ([]byte, ocispecs.Descriptor, error) {
func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes.AnnotationKey]string, preferIndex bool) ([]byte, ocispecs.Descriptor, map[digest.Digest]*Source, error) {
eg, ctx := errgroup.WithContext(ctx)
dts := make([][]byte, len(srcs))
@@ -73,7 +73,7 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
}
if err := eg.Wait(); err != nil {
return nil, ocispecs.Descriptor{}, err
return nil, ocispecs.Descriptor{}, nil, err
}
// on single source, return original bytes
@@ -83,21 +83,25 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
// of preferIndex since if set to true then the source is already in the preferred format, and if false
// it doesn't matter since we're not going to split it into separate manifests
case images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex:
return dts[0], srcs[0].Desc, nil
srcMap := map[digest.Digest]*Source{
srcs[0].Desc.Digest: srcs[0],
}
return dts[0], srcs[0].Desc, srcMap, nil
default:
if !preferIndex {
return dts[0], srcs[0].Desc, nil
return dts[0], srcs[0].Desc, nil, nil
}
}
}
m := map[digest.Digest]int{}
newDescs := make([]ocispecs.Descriptor, 0, len(srcs))
indexes := map[digest.Digest]int{}
sources := map[digest.Digest]*Source{}
descs := make([]ocispecs.Descriptor, 0, len(srcs))
addDesc := func(d ocispecs.Descriptor) {
idx, ok := m[d.Digest]
addDesc := func(d ocispecs.Descriptor, src *Source) {
idx, ok := indexes[d.Digest]
if ok {
old := newDescs[idx]
old := descs[idx]
if old.MediaType == "" {
old.MediaType = d.MediaType
}
@@ -108,11 +112,12 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
old.Annotations = map[string]string{}
}
maps.Copy(old.Annotations, d.Annotations)
newDescs[idx] = old
descs[idx] = old
} else {
m[d.Digest] = len(newDescs)
newDescs = append(newDescs, d)
indexes[d.Digest] = len(descs)
descs = append(descs, d)
}
sources[d.Digest] = src
}
for i, src := range srcs {
@@ -120,25 +125,25 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
case images.MediaTypeDockerSchema2ManifestList, ocispecs.MediaTypeImageIndex:
var mfst ocispecs.Index
if err := json.Unmarshal(dts[i], &mfst); err != nil {
return nil, ocispecs.Descriptor{}, errors.WithStack(err)
return nil, ocispecs.Descriptor{}, nil, errors.WithStack(err)
}
for _, d := range mfst.Manifests {
addDesc(d)
addDesc(d, src)
}
default:
addDesc(src.Desc)
addDesc(src.Desc, src)
}
}
dockerMfsts := 0
for _, desc := range newDescs {
for _, desc := range descs {
if strings.HasPrefix(desc.MediaType, "application/vnd.docker.") {
dockerMfsts++
}
}
var mt string
if dockerMfsts == len(newDescs) {
if dockerMfsts == len(descs) {
// all manifests are Docker types, use Docker manifest list
mt = images.MediaTypeDockerSchema2ManifestList
} else {
@@ -154,18 +159,18 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
case exptypes.AnnotationIndex:
indexAnnotation[k.Key] = v
case exptypes.AnnotationManifestDescriptor:
for i := range newDescs {
if newDescs[i].Annotations == nil {
newDescs[i].Annotations = map[string]string{}
for i := range descs {
if descs[i].Annotations == nil {
descs[i].Annotations = map[string]string{}
}
if k.Platform == nil || k.PlatformString() == platforms.Format(*newDescs[i].Platform) {
newDescs[i].Annotations[k.Key] = v
if k.Platform == nil || k.PlatformString() == platforms.Format(*descs[i].Platform) {
descs[i].Annotations[k.Key] = v
}
}
case exptypes.AnnotationManifest, "":
return nil, ocispecs.Descriptor{}, errors.Errorf("%q annotations are not supported yet", k.Type)
return nil, ocispecs.Descriptor{}, nil, errors.Errorf("%q annotations are not supported yet", k.Type)
case exptypes.AnnotationIndexDescriptor:
return nil, ocispecs.Descriptor{}, errors.Errorf("%q annotations are invalid while creating an image", k.Type)
return nil, ocispecs.Descriptor{}, nil, errors.Errorf("%q annotations are invalid while creating an image", k.Type)
}
}
}
@@ -175,18 +180,18 @@ func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes
Versioned: specs.Versioned{
SchemaVersion: 2,
},
Manifests: newDescs,
Manifests: descs,
Annotations: indexAnnotation,
}, "", " ")
if err != nil {
return nil, ocispecs.Descriptor{}, errors.Wrap(err, "failed to marshal index")
return nil, ocispecs.Descriptor{}, nil, errors.Wrap(err, "failed to marshal index")
}
return idxBytes, ocispecs.Descriptor{
MediaType: mt,
Size: int64(len(idxBytes)),
Digest: digest.FromBytes(idxBytes),
}, nil
}, sources, nil
}
func (r *Resolver) Push(ctx context.Context, ref reference.Named, desc ocispecs.Descriptor, dt []byte) error {
vendor/github.com/moby/buildkit/util/attestation/types.go
Generated Vendored
+9
View File
@@ -0,0 +1,9 @@
package attestation
const (
DockerAnnotationReferenceType = "vnd.docker.reference.type"
DockerAnnotationReferenceDigest = "vnd.docker.reference.digest"
DockerAnnotationReferenceDescription = "vnd.docker.reference.description"
DockerAnnotationReferenceTypeDefault = "attestation-manifest"
)
vendor/modules.txt
Vendored
+1
View File
@@ -504,6 +504,7 @@ github.com/moby/buildkit/util/apicaps
github.com/moby/buildkit/util/apicaps/pb
github.com/moby/buildkit/util/appcontext
github.com/moby/buildkit/util/appdefaults
github.com/moby/buildkit/util/attestation
github.com/moby/buildkit/util/bklog
github.com/moby/buildkit/util/contentutil
github.com/moby/buildkit/util/disk