diff --git a/build/build.go b/build/build.go index 7d7e3dcc4..ab430a028 100644 --- a/build/build.go +++ b/build/build.go @@ -723,7 +723,7 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[ return err } - dt, desc, _, err := itpull.Combine(ctx, srcs, indexAnnotations, false) + dt, desc, _, err := itpull.Combine(ctx, srcs, indexAnnotations, false, nil) if err != nil { return err } diff --git a/commands/imagetools/create.go b/commands/imagetools/create.go index 5e2c23174..e9da9489f 100644 --- a/commands/imagetools/create.go +++ b/commands/imagetools/create.go @@ -7,7 +7,7 @@ import ( "os" "strings" - "github.com/containerd/containerd/v2/core/images" + "github.com/containerd/containerd/v2/core/remotes" "github.com/containerd/platforms" "github.com/distribution/reference" "github.com/docker/buildx/builder" @@ -16,7 +16,6 @@ import ( "github.com/docker/buildx/util/imagetools" "github.com/docker/buildx/util/progress" "github.com/docker/cli/cli/command" - "github.com/moby/buildkit/util/attestation" "github.com/moby/buildkit/util/progress/progressui" "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" @@ -169,12 +168,11 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg return errors.Wrapf(err, "failed to parse annotations") } - dt, desc, srcMap, err := r.Combine(ctx, srcs, annotations, in.preferIndex) - if err != nil { - return err - } + ctx = remotes.WithMediaTypeKeyPrefix(ctx, "application/vnd.oci.empty.v1+json", "empty") + ctx = remotes.WithMediaTypeKeyPrefix(ctx, "application/vnd.dev.cosign.artifact.sig.v1+json", "cosign") + ctx = remotes.WithMediaTypeKeyPrefix(ctx, "application/vnd.dev.cosign.simplesigning.v1+json", "simplesigning") - dt, desc, manifests, err := filterPlatforms(dt, desc, srcMap, platforms) + dt, desc, manifests, err := r.Combine(ctx, srcs, annotations, in.preferIndex, platforms) if err != nil { return err } @@ -186,7 +184,7 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg // manifests can be nil only if pushing one single-platform desc directly if manifests == nil { - manifests = []descWithSource{{Descriptor: desc, Source: srcs[0]}} + manifests = []imagetools.DescWithSource{{Descriptor: desc, Source: srcs[0]}} } // new resolver cause need new auth @@ -234,124 +232,6 @@ func runCreate(ctx context.Context, dockerCli command.Cli, in createOptions, arg return err } -type descWithSource struct { - ocispecs.Descriptor - Source *imagetools.Source -} - -func filterPlatforms(dt []byte, desc ocispecs.Descriptor, srcMap map[digest.Digest]*imagetools.Source, plats []ocispecs.Platform) ([]byte, ocispecs.Descriptor, []descWithSource, error) { - matcher := platforms.Any(plats...) - - if !images.IsIndexType(desc.MediaType) { - if len(plats) == 0 { - return dt, desc, nil, nil - } - var mfst ocispecs.Manifest - if err := json.Unmarshal(dt, &mfst); err != nil { - return nil, ocispecs.Descriptor{}, nil, errors.Wrapf(err, "failed to parse manifest") - } - if desc.Platform == nil { - return nil, ocispecs.Descriptor{}, nil, errors.Errorf("cannot filter platforms from a manifest without platform information") - } - if !matcher.Match(*desc.Platform) { - return nil, ocispecs.Descriptor{}, nil, errors.Errorf("input platform %s does not match any of the provided platforms", platforms.Format(*desc.Platform)) - } - return dt, desc, nil, nil - } - - var idx ocispecs.Index - if err := json.Unmarshal(dt, &idx); err != nil { - return nil, ocispecs.Descriptor{}, nil, errors.Wrapf(err, "failed to parse index") - } - if len(plats) == 0 { - mfsts := make([]descWithSource, len(idx.Manifests)) - for i, m := range idx.Manifests { - src, ok := srcMap[m.Digest] - if !ok { - defaultSource, ok := srcMap[desc.Digest] - if !ok { - return nil, ocispecs.Descriptor{}, nil, errors.Errorf("internal error: no source found for %s", m.Digest) - } - src = defaultSource - } - mfsts[i] = descWithSource{ - Descriptor: m, - Source: src, - } - } - return dt, desc, mfsts, nil - } - - manifestMap := map[digest.Digest]ocispecs.Descriptor{} - for _, m := range idx.Manifests { - manifestMap[m.Digest] = m - } - references := map[digest.Digest]struct{}{} - for _, m := range idx.Manifests { - if refType, ok := m.Annotations[attestation.DockerAnnotationReferenceType]; ok && refType == attestation.DockerAnnotationReferenceTypeDefault { - dgstStr, ok := m.Annotations[attestation.DockerAnnotationReferenceDigest] - if !ok { - continue - } - dgst, err := digest.Parse(dgstStr) - if err != nil { - continue - } - subject, ok := manifestMap[dgst] - if !ok { - continue - } - if subject.Platform == nil || matcher.Match(*subject.Platform) { - references[m.Digest] = struct{}{} - } - } - } - - var mfsts []ocispecs.Descriptor - var mfstsWithSource []descWithSource - - for _, m := range idx.Manifests { - if _, isRef := references[m.Digest]; isRef || m.Platform == nil || matcher.Match(*m.Platform) { - src, ok := srcMap[m.Digest] - if !ok { - defaultSource, ok := srcMap[desc.Digest] - if !ok { - return nil, ocispecs.Descriptor{}, nil, errors.Errorf("internal error: no source found for %s", m.Digest) - } - src = defaultSource - } - mfsts = append(mfsts, m) - mfstsWithSource = append(mfstsWithSource, descWithSource{ - Descriptor: m, - Source: src, - }) - } - } - if len(mfsts) == len(idx.Manifests) { - // all platforms matched, no need to rewrite index - return dt, desc, mfstsWithSource, nil - } - - if len(mfsts) == 0 { - return nil, ocispecs.Descriptor{}, nil, errors.Errorf("none of the manifests match the provided platforms") - } - - idx.Manifests = mfsts - idxBytes, err := json.MarshalIndent(&idx, "", " ") - if err != nil { - return nil, ocispecs.Descriptor{}, nil, errors.Wrap(err, "failed to marshal index") - } - - desc = ocispecs.Descriptor{ - MediaType: desc.MediaType, - Size: int64(len(idxBytes)), - Digest: digest.FromBytes(idxBytes), - Annotations: desc.Annotations, - } - - return idxBytes, desc, mfstsWithSource, nil -} - func parseSources(in []string) ([]*imagetools.Source, error) { out := make([]*imagetools.Source, len(in)) for i, in := range in { diff --git a/util/imagetools/create.go b/util/imagetools/create.go index 89eb7ab0b..fdf9395c4 100644 --- a/util/imagetools/create.go +++ b/util/imagetools/create.go @@ -15,6 +15,7 @@ import ( "github.com/containerd/platforms" "github.com/distribution/reference" "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/util/attestation" "github.com/moby/buildkit/util/contentutil" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" @@ -23,12 +24,34 @@ import ( "golang.org/x/sync/errgroup" ) +const ( + artifactTypeAttestationManifest = "application/vnd.docker.attestation.manifest.v1+json" + artifactTypeCosignSignature = "application/vnd.dev.cosign.artifact.sig.v1+json" +) + +var supportedArtifactTypes = map[string]struct{}{ + artifactTypeAttestationManifest: {}, + artifactTypeCosignSignature: {}, +} + type Source struct { Desc ocispecs.Descriptor Ref reference.Named } -func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes.AnnotationKey]string, preferIndex bool) ([]byte, ocispecs.Descriptor, map[digest.Digest]*Source, error) { +func (r *Resolver) Combine(ctx context.Context, srcs []*Source, ann map[exptypes.AnnotationKey]string, preferIndex bool, platforms []ocispecs.Platform) ([]byte, ocispecs.Descriptor, []DescWithSource, error) { + dt, desc, srcMap, err := r.combine(ctx, srcs, ann, preferIndex) + if err != nil { + return nil, ocispecs.Descriptor{}, nil, err + } + dt, desc, mfstsWithSource, err := r.filterPlatforms(ctx, dt, desc, srcMap, platforms) + if err != nil { + return nil, ocispecs.Descriptor{}, nil, err + } + return dt, desc, mfstsWithSource, nil +} + +func (r *Resolver) combine(ctx context.Context, srcs []*Source, ann map[exptypes.AnnotationKey]string, preferIndex bool) ([]byte, ocispecs.Descriptor, map[digest.Digest]*Source, error) { eg, ctx := errgroup.WithContext(ctx) dts := make([][]byte, len(srcs)) @@ -251,7 +274,28 @@ func (r *Resolver) Copy(ctx context.Context, src *Source, dest reference.Named) source, repo := u.Hostname(), strings.TrimPrefix(u.Path, "/") desc.Annotations["containerd.io/distribution.source."+source] = repo - err = contentutil.CopyChain(ctx, contentutil.FromPusher(p), contentutil.FromFetcher(f), desc) + referrersFetcher, ok := f.(remotes.ReferrersFetcher) + if !ok { + return errors.Errorf("fetcher for %s does not support referrers", src.Ref.String()) + } + + opts := []contentutil.CopyOption{ + contentutil.WithReferrers(referrersFunc(func(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + descs, err := referrersFetcher.FetchReferrers(ctx, desc.Digest) + if err != nil { + return nil, err + } + var filtered []ocispecs.Descriptor + for _, d := range descs { + if _, ok := supportedArtifactTypes[d.ArtifactType]; ok { + filtered = append(filtered, d) + } + } + return filtered, nil + })), + } + + err = contentutil.CopyChain(ctx, contentutil.FromPusher(p), contentutil.FromFetcher(f), desc, opts...) if err != nil { return err } @@ -289,6 +333,173 @@ func (r *Resolver) loadPlatform(ctx context.Context, p2 *ocispecs.Platform, in s return nil } +type referrersFunc func(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) + +func (f referrersFunc) Referrers(ctx context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + return f(ctx, desc) +} + +type DescWithSource struct { + ocispecs.Descriptor + Source *Source +} + +func (r *Resolver) filterPlatforms(ctx context.Context, dt []byte, desc ocispecs.Descriptor, srcMap map[digest.Digest]*Source, plats []ocispecs.Platform) ([]byte, ocispecs.Descriptor, []DescWithSource, error) { + matcher := platforms.Any(plats...) + if len(plats) == 0 { + matcher = platforms.All + } + + if !images.IsIndexType(desc.MediaType) { + var mfst ocispecs.Manifest + if err := json.Unmarshal(dt, &mfst); err != nil { + return nil, ocispecs.Descriptor{}, nil, errors.Wrapf(err, "failed to parse manifest") + } + if desc.Platform == nil { + return nil, ocispecs.Descriptor{}, nil, errors.Errorf("cannot filter platforms from a manifest without platform information") + } + if !matcher.Match(*desc.Platform) { + return nil, ocispecs.Descriptor{}, nil, errors.Errorf("input platform %s does not match any of the provided platforms", platforms.Format(*desc.Platform)) + } + return dt, desc, nil, nil + } + + var idx ocispecs.Index + if err := json.Unmarshal(dt, &idx); err != nil { + return nil, ocispecs.Descriptor{}, nil, errors.Wrapf(err, "failed to parse index") + } + + var manifestMap = map[digest.Digest]ocispecs.Descriptor{} + for _, m := range idx.Manifests { + manifestMap[m.Digest] = m + } + var references = map[digest.Digest]ocispecs.Descriptor{} + var matchedManifests = map[digest.Digest]struct{}{} + for _, m := range idx.Manifests { + if m.Platform == nil || matcher.Match(*m.Platform) { + matchedManifests[m.Digest] = struct{}{} + } + if refType, ok := m.Annotations[attestation.DockerAnnotationReferenceType]; ok && refType == attestation.DockerAnnotationReferenceTypeDefault { + dgstStr, ok := m.Annotations[attestation.DockerAnnotationReferenceDigest] + if !ok { + continue + } + dgst, err := digest.Parse(dgstStr) + if err != nil { + continue + } + subject, ok := manifestMap[dgst] + if !ok { + continue + } + if subject.Platform == nil || matcher.Match(*subject.Platform) { + references[m.Digest] = subject + } + } + } + + var mfsts []ocispecs.Descriptor + var mfstsWithSource []DescWithSource + + for _, m := range idx.Manifests { + _, isRef := references[m.Digest] + if isRef || m.Platform == nil || matcher.Match(*m.Platform) { + src, ok := srcMap[m.Digest] + if !ok { + defaultSource, ok := srcMap[desc.Digest] + if !ok { + return nil, ocispecs.Descriptor{}, nil, errors.Errorf("internal error: no source found for %s", m.Digest) + } + src = defaultSource + } + mfsts = append(mfsts, m) + mfstsWithSource = append(mfstsWithSource, DescWithSource{ + Descriptor: m, + Source: src, + }) + } + } + + if len(mfsts) == 0 { + return nil, ocispecs.Descriptor{}, nil, errors.Errorf("none of the manifests match the provided platforms") + } + + // try to pull in attestation manifest via referrer if one exists + addedRef := false + for d := range matchedManifests { + hasRef := false + for _, subject := range references { + if subject.Digest == d { + hasRef = true + break + } + } + if hasRef { + continue + } + src, ok := srcMap[d] + if !ok { + defaultSource, ok := srcMap[desc.Digest] + if !ok { + return nil, ocispecs.Descriptor{}, nil, errors.Errorf("internal error: no source found for %s", d) + } + src = defaultSource + } + f, err := r.resolver().Fetcher(ctx, src.Ref.String()) + if err != nil { + return nil, ocispecs.Descriptor{}, nil, err + } + rf, ok := f.(remotes.ReferrersFetcher) + if !ok { + return nil, ocispecs.Descriptor{}, nil, errors.Errorf("fetcher for %s does not support referrers", srcMap[d].Ref.String()) + } + refs, err := rf.FetchReferrers(ctx, d, remotes.WithReferrerArtifactTypes(artifactTypeAttestationManifest)) + if err != nil { + if errors.Is(err, errdefs.ErrNotFound) { + continue + } + return nil, ocispecs.Descriptor{}, nil, err + } + for _, ref := range refs { + if _, ok := references[ref.Digest]; ok { + continue + } + ref.Platform = &ocispecs.Platform{ + OS: "unknown", Architecture: "unknown", + } + if ref.Annotations == nil { + ref.Annotations = map[string]string{} + } + ref.Annotations[attestation.DockerAnnotationReferenceType] = attestation.DockerAnnotationReferenceTypeDefault + ref.Annotations[attestation.DockerAnnotationReferenceDigest] = d.String() + ref.ArtifactType = "" + mfsts = append(mfsts, ref) + addedRef = true + break + } + } + + if len(mfsts) == len(idx.Manifests) && !addedRef { + // all platforms matched, no need to rewrite index + return dt, desc, mfstsWithSource, nil + } + + idx.Manifests = mfsts + idxBytes, err := json.MarshalIndent(&idx, "", " ") + if err != nil { + return nil, ocispecs.Descriptor{}, nil, errors.Wrap(err, "failed to marshal index") + } + + desc = ocispecs.Descriptor{ + MediaType: desc.MediaType, + Size: int64(len(idxBytes)), + Digest: digest.FromBytes(idxBytes), + Annotations: desc.Annotations, + } + + return idxBytes, desc, mfstsWithSource, nil +} + func detectMediaType(dt []byte) (string, error) { var mfst struct { MediaType string `json:"mediaType"`