alextab93/buildx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

policy: simplify recursive material resolution

Browse Source 
Unify root/material unknown resolution with recursive Input traversal.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi
2026-02-26 09:14:25 -08:00
parent f35e45c307
commit 4880756a0f
27 changed files with 1842 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.golangci.yml
+3
View File
@@ -109,6 +109,9 @@ linters:
- linters:
- revive
text: if-return
- linters:
- staticcheck
text: "SA1019: .* is deprecated: .*in-toto/attestation"
paths:
- .*\.pb\.go$
commands/policy/eval.go
+162 -113
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"io/fs"
"maps"
"os"
"path/filepath"
"slices"
@@ -17,7 +18,6 @@ import (
"github.com/docker/buildx/util/confutil"
"github.com/docker/buildx/util/sourcemeta"
"github.com/docker/cli/cli/command"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/frontend/dockerui"
gwpb "github.com/moby/buildkit/frontend/gateway/pb"
"github.com/moby/buildkit/solver/pb"
@@ -27,7 +27,6 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"google.golang.org/protobuf/types/known/timestamppb"
)
type evalOpts struct {
@@ -111,48 +110,50 @@ func runEval(ctx context.Context, dockerCli command.Cli, source string, opts eva
srcReq := &gwpb.ResolveSourceMetaResponse{
Source: src,
}
input, err := policy.SourceToInput(ctx, verifier, srcReq, &p, nil)
if err != nil {
return err
}
maxAttempts := 5
var unknowns []string
var lastUnknowns []string
var trimmedUnknowns []string
var input policy.Input
var doneInvalidCheck bool
var invalidFields []string
for {
maxAttempts--
if maxAttempts <= 0 {
return errors.New("maximum attempts reached for resolving source metadata")
}
input, unknowns, err = policy.SourceToInput(ctx, verifier, srcReq, &p)
if err != nil {
return err
unknowns := input.Unknowns()
trimmedUnknowns = make([]string, 0, len(unknowns))
for _, u := range unknowns {
trimmedUnknowns = append(trimmedUnknowns, strings.TrimPrefix(u, "input."))
}
trimmedUnknowns = trimInputPrefixSlice(unknowns)
if lastUnknowns != nil && slices.Equal(trimmedUnknowns, lastUnknowns) {
break
}
lastUnknowns = slices.Clone(trimmedUnknowns)
toReload := []string{}
for _, f := range opts.fields {
if slices.Contains(trimmedUnknowns, f) {
toReload = append(toReload, f)
} else if !doneInvalidCheck {
invalidFields = append(invalidFields, f)
}
}
doneInvalidCheck = true
toReload, invalid := selectReloadFields(opts.fields, trimmedUnknowns)
invalidFields = invalid
if len(toReload) > 0 {
req := &gwpb.ResolveSourceMetaRequest{}
if err := policy.AddUnknowns(req, toReload); err != nil {
return err
}
opt := sourceResolverOpt(req, &p)
resp, err := metaResolver.ResolveSourceMetadata(ctx, src, opt)
retry, next, err := policy.ResolveInputUnknowns(ctx, &input, srcReq.Source, toReload, platform, &p, metaResolver, verifier, nil)
if err != nil {
return err
}
srcReq = buildSourceMetaResponse(resp)
continue
if next != nil {
resp, err := metaResolver.ResolveSourceMetadata(ctx, next.Source, sourcemeta.ToResolverOpt(next, &p))
if err != nil {
return err
}
srcReq = sourcemeta.ToGatewayMetaResponse(resp)
input, err = policy.SourceToInput(ctx, verifier, srcReq, &p, nil)
if err != nil {
return err
}
continue
}
if retry {
continue
}
}
break
}
@@ -160,11 +161,15 @@ func runEval(ctx context.Context, dockerCli command.Cli, source string, opts eva
if len(invalidFields) > 0 {
logrus.Warnf("invalid fields: %v", strings.Join(invalidFields, ", "))
}
if len(trimmedUnknowns) > 0 {
logrus.Infof("unresolved fields: %v", strings.Join(trimmedUnknowns, ", "))
reportedUnknowns := summarizeEvalUnknowns(trimmedUnknowns, opts.fields)
if len(reportedUnknowns) > 0 {
logrus.Infof("unresolved fields: %v", strings.Join(reportedUnknowns, ", "))
}
dt, err := json.MarshalIndent(input, "", " ")
printInput := input
sanitizePrintInput(&printInput)
dt, err := json.MarshalIndent(printInput, "", " ")
if err != nil {
return errors.Wrap(err, "failed to marshal policy input")
}
@@ -198,6 +203,9 @@ func runEval(ctx context.Context, dockerCli command.Cli, source string, opts eva
env := policy.Env{
Filename: filepath.Base(policyName),
}
policyLog := func(_ logrus.Level, msg string) {
logrus.Debug(msg)
}
policyEval := policy.NewPolicy(policy.Opt{
Files: []policy.File{
@@ -207,9 +215,11 @@ func runEval(ctx context.Context, dockerCli command.Cli, source string, opts eva
},
},
Env: env,
Log: policyLog,
FS: fsProvider,
VerifierProvider: verifier,
DefaultPlatform: &p,
SourceResolver: metaResolver,
})
srcReq := &gwpb.ResolveSourceMetaResponse{
@@ -233,114 +243,153 @@ func runEval(ctx context.Context, dockerCli command.Cli, source string, opts eva
return evalDecisionError(decision)
}
opt := sourceResolverOpt(next, &p)
resp, err := metaResolver.ResolveSourceMetadata(ctx, src, opt)
opt := sourcemeta.ToResolverOpt(next, &p)
target := src
if next.Source != nil {
target = next.Source
}
resp, err := metaResolver.ResolveSourceMetadata(ctx, target, opt)
if err != nil {
return err
}
srcReq = buildSourceMetaResponse(resp)
srcReq = sourcemeta.ToGatewayMetaResponse(resp)
}
}
func toGatewayDescriptor(desc ocispecs.Descriptor) *gwpb.Descriptor {
return &gwpb.Descriptor{
MediaType: desc.MediaType,
Digest: desc.Digest.String(),
Size: desc.Size,
Annotations: desc.Annotations,
func selectReloadFields(fields []string, unknowns []string) ([]string, []string) {
if len(fields) == 0 {
return nil, nil
}
reload := map[string]struct{}{}
var invalid []string
for _, field := range fields {
if prereq, ok := materialFieldPrerequisites(field); ok {
added := false
for _, p := range prereq {
if slices.Contains(unknowns, p) {
reload[p] = struct{}{}
added = true
}
}
if slices.Contains(unknowns, field) {
reload[field] = struct{}{}
added = true
} else if ancestor := findUnknownAncestor(field, unknowns); ancestor != "" {
reload[ancestor] = struct{}{}
added = true
}
if !added {
invalid = append(invalid, field)
}
continue
}
if slices.Contains(unknowns, field) {
reload[field] = struct{}{}
continue
}
invalid = append(invalid, field)
}
return slices.Collect(maps.Keys(reload)), invalid
}
func toGatewayAttestationChain(chain *sourceresolver.AttestationChain) *gwpb.AttestationChain {
if chain == nil {
func findUnknownAncestor(field string, unknowns []string) string {
var best string
for _, unknown := range unknowns {
if field == unknown {
return unknown
}
if strings.HasPrefix(field, unknown+".") {
if len(unknown) > len(best) {
best = unknown
}
continue
}
if strings.HasPrefix(field, unknown+"[") {
if len(unknown) > len(best) {
best = unknown
}
}
}
return best
}
func materialFieldPrerequisites(field string) ([]string, bool) {
const seg = ".image.provenance.materials["
if !strings.HasPrefix(field, seg[1:]) {
return nil, false
}
provenancePath := strings.TrimSuffix(seg, ".materials[")
out := map[string]struct{}{strings.TrimPrefix(provenancePath, "."): {}}
collectMaterialPrerequisites(field, seg, provenancePath, 0, out)
keys := slices.Collect(maps.Keys(out))
slices.Sort(keys)
return keys, true
}
func collectMaterialPrerequisites(field, seg, provenancePath string, start int, out map[string]struct{}) {
i := strings.Index(field[start:], seg)
if i < 0 {
return
}
i += start
out[field[:i]+provenancePath] = struct{}{}
collectMaterialPrerequisites(field, seg, provenancePath, i+len(seg), out)
}
func summarizeEvalUnknowns(unknowns, requested []string) []string {
if len(unknowns) == 0 {
return nil
}
signatures := make([]string, 0, len(chain.SignatureManifests))
for _, dgst := range chain.SignatureManifests {
signatures = append(signatures, dgst.String())
}
blobs := make(map[string]*gwpb.Blob, len(chain.Blobs))
for dgst, blob := range chain.Blobs {
blobs[dgst.String()] = &gwpb.Blob{
Descriptor_: toGatewayDescriptor(blob.Descriptor),
Data: blob.Data,
if len(requested) > 0 {
out := map[string]struct{}{}
for _, field := range requested {
if slices.Contains(unknowns, field) {
out[field] = struct{}{}
continue
}
if ancestor := findUnknownAncestor(field, unknowns); ancestor != "" {
out[ancestor] = struct{}{}
}
}
keys := slices.Collect(maps.Keys(out))
slices.Sort(keys)
return keys
}
return &gwpb.AttestationChain{
Root: chain.Root.String(),
ImageManifest: chain.ImageManifest.String(),
AttestationManifest: chain.AttestationManifest.String(),
SignatureManifests: signatures,
Blobs: blobs,
out := map[string]struct{}{}
for _, u := range unknowns {
out[summarizeUnknownField(u)] = struct{}{}
}
keys := slices.Collect(maps.Keys(out))
slices.Sort(keys)
return keys
}
func sourceResolverOpt(req *gwpb.ResolveSourceMetaRequest, platform *ocispecs.Platform) sourceresolver.Opt {
opt := sourceresolver.Opt{
LogName: req.LogName,
SourcePolicies: req.SourcePolicies,
func summarizeUnknownField(field string) string {
if base, _, ok := strings.Cut(field, ".materials["); ok {
return base + ".materials"
}
if req.Image != nil {
opt.ImageOpt = &sourceresolver.ResolveImageOpt{
NoConfig: req.Image.NoConfig,
AttestationChain: req.Image.AttestationChain,
ResolveAttestations: slices.Clone(req.Image.ResolveAttestations),
Platform: platform,
ResolveMode: req.ResolveMode,
}
if strings.HasPrefix(field, "materials[") {
return "materials"
}
if req.Git != nil {
opt.GitOpt = &sourceresolver.ResolveGitOpt{
ReturnObject: req.Git.ReturnObject,
}
parts := strings.Split(field, ".")
if len(parts) > 1 {
return strings.Join(parts[:2], ".")
}
return opt
return field
}
func buildSourceMetaResponse(resp *sourceresolver.MetaResponse) *gwpb.ResolveSourceMetaResponse {
out := &gwpb.ResolveSourceMetaResponse{
Source: resp.Op,
func sanitizePrintInput(inp *policy.Input) {
if inp == nil {
return
}
if resp.Image != nil {
chain := toGatewayAttestationChain(resp.Image.AttestationChain)
out.Image = &gwpb.ResolveSourceImageResponse{
Digest: resp.Image.Digest.String(),
Config: resp.Image.Config,
AttestationChain: chain,
}
inp.Env.Depth = 0
if inp.Image == nil || inp.Image.Provenance == nil || len(inp.Image.Provenance.Materials) == 0 {
return
}
if resp.Git != nil {
out.Git = &gwpb.ResolveSourceGitResponse{
Checksum: resp.Git.Checksum,
Ref: resp.Git.Ref,
CommitChecksum: resp.Git.CommitChecksum,
CommitObject: resp.Git.CommitObject,
TagObject: resp.Git.TagObject,
}
for i := range inp.Image.Provenance.Materials {
sanitizePrintInput(&inp.Image.Provenance.Materials[i])
}
if resp.HTTP != nil {
var lastModified *timestamppb.Timestamp
if resp.HTTP.LastModified != nil {
lastModified = timestamppb.New(*resp.HTTP.LastModified)
}
out.HTTP = &gwpb.ResolveSourceHTTPResponse{
Checksum: resp.HTTP.Digest.String(),
Filename: resp.HTTP.Filename,
LastModified: lastModified,
}
}
return out
}
func trimInputPrefixSlice(fields []string) []string {
if len(fields) == 0 {
return fields
}
out := make([]string, 0, len(fields))
for _, field := range fields {
out = append(out, strings.TrimPrefix(field, "input."))
}
return out
}
func evalDecisionError(decision *policysession.DecisionResponse) error {
commands/policy/eval_test.go
+125 -1
View File
@@ -3,6 +3,8 @@ package policy
import (
"testing"
policytypes "github.com/docker/buildx/policy"
"github.com/docker/buildx/util/sourcemeta"
gwpb "github.com/moby/buildkit/frontend/gateway/pb"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/stretchr/testify/require"
@@ -18,10 +20,132 @@ func TestSourceResolverOptIncludesResolveAttestations(t *testing.T) {
}
platform := &ocispecs.Platform{OS: "linux", Architecture: "amd64"}
opt := sourceResolverOpt(req, platform)
opt := sourcemeta.ToResolverOpt(req, platform)
require.NotNil(t, opt.ImageOpt)
require.True(t, opt.ImageOpt.NoConfig)
require.Equal(t, []string{"https://slsa.dev/provenance/v0.2"}, opt.ImageOpt.ResolveAttestations)
require.Equal(t, "default", opt.ImageOpt.ResolveMode)
require.Equal(t, platform, opt.ImageOpt.Platform)
}
func TestSanitizePrintInputClearsDepthRecursively(t *testing.T) {
inp := policytypes.Input{
Env: policytypes.Env{Depth: 7, Filename: "Dockerfile"},
Image: &policytypes.Image{
Provenance: &policytypes.ImageProvenance{
Materials: []policytypes.Input{
{
Env: policytypes.Env{Depth: 3, Target: "app"},
Image: &policytypes.Image{
Provenance: &policytypes.ImageProvenance{
Materials: []policytypes.Input{
{Env: policytypes.Env{Depth: 2}},
},
},
},
},
},
},
},
}
sanitizePrintInput(&inp)
require.Zero(t, inp.Env.Depth)
require.Equal(t, "Dockerfile", inp.Env.Filename)
require.Zero(t, inp.Image.Provenance.Materials[0].Env.Depth)
require.Equal(t, "app", inp.Image.Provenance.Materials[0].Env.Target)
require.Zero(t, inp.Image.Provenance.Materials[0].Image.Provenance.Materials[0].Env.Depth)
}
func TestSelectReloadFields(t *testing.T) {
unknowns := []string{
"image.provenance",
"image.provenance.materials[0].image.hasProvenance",
"git.tag",
}
t.Run("exact match", func(t *testing.T) {
reload, invalid := selectReloadFields([]string{"git.tag"}, unknowns)
require.Equal(t, []string{"git.tag"}, reload)
require.Nil(t, invalid)
})
t.Run("ancestor mapping", func(t *testing.T) {
reload, invalid := selectReloadFields([]string{"image.provenance.materials[0].image.labels"}, unknowns)
require.ElementsMatch(t, []string{"image.provenance"}, reload)
require.Nil(t, invalid)
})
t.Run("dedupe mapped reloads", func(t *testing.T) {
reload, invalid := selectReloadFields([]string{
"image.provenance.materials[0].image.labels",
"image.provenance.materials[0].image.user",
}, unknowns)
require.ElementsMatch(t, []string{
"image.provenance",
}, reload)
require.Nil(t, invalid)
})
t.Run("invalid fields reported", func(t *testing.T) {
reload, invalid := selectReloadFields([]string{"image.labels", "foo.bar"}, unknowns)
require.Empty(t, reload)
require.Equal(t, []string{"image.labels", "foo.bar"}, invalid)
})
t.Run("mix exact mapped invalid", func(t *testing.T) {
reload, invalid := selectReloadFields([]string{
"git.tag",
"image.provenance.materials[0].image.env",
"no.such.field",
}, unknowns)
require.ElementsMatch(t, []string{"git.tag", "image.provenance"}, reload)
require.Equal(t, []string{"no.such.field"}, invalid)
})
t.Run("nested material prerequisites", func(t *testing.T) {
reload, invalid := selectReloadFields([]string{
"image.provenance.materials[0].image.provenance.materials[1].image.labels",
}, unknowns)
require.ElementsMatch(t, []string{
"image.provenance",
}, reload)
require.Nil(t, invalid)
})
t.Run("material field after provenance loaded", func(t *testing.T) {
reload, invalid := selectReloadFields([]string{
"image.provenance.materials[0].image.hasProvenance",
}, []string{
"image.provenance.materials[0].image.hasProvenance",
})
require.ElementsMatch(t, []string{
"image.provenance.materials[0].image.hasProvenance",
}, reload)
require.Nil(t, invalid)
})
}
func TestMaterialFieldPrerequisites(t *testing.T) {
t.Run("non material field", func(t *testing.T) {
prereq, ok := materialFieldPrerequisites("image.provenance")
require.False(t, ok)
require.Nil(t, prereq)
})
t.Run("single level material field", func(t *testing.T) {
prereq, ok := materialFieldPrerequisites("image.provenance.materials[0].image.labels")
require.True(t, ok)
require.Equal(t, []string{"image.provenance"}, prereq)
})
t.Run("nested material field", func(t *testing.T) {
prereq, ok := materialFieldPrerequisites("image.provenance.materials[0].image.provenance.materials[1].image.labels")
require.True(t, ok)
require.Equal(t, []string{
"image.provenance",
"image.provenance.materials[0].image.provenance",
}, prereq)
})
}
commands/policy/test.go
+2 -2
View File
@@ -158,12 +158,12 @@ func (r *policyTestOptionsProvider) Resolve(ctx context.Context, source *pb.Sour
if err := r.init(ctx); err != nil {
return nil, err
}
opt := sourceResolverOpt(req, r.platform)
opt := sourcemeta.ToResolverOpt(req, r.platform)
resp, err := r.metaResolver.ResolveSourceMetadata(ctx, source, opt)
if err != nil {
return nil, err
}
return buildSourceMetaResponse(resp), nil
return sourcemeta.ToGatewayMetaResponse(resp), nil
}
func (r *policyTestOptionsProvider) init(ctx context.Context) error {
go.mod
+1
View File
@@ -183,6 +183,7 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/package-url/packageurl-go v0.1.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
policy/input.go
+88
View File
@@ -0,0 +1,88 @@
package policy
import (
"context"
"github.com/containerd/platforms"
gwpb "github.com/moby/buildkit/frontend/gateway/pb"
"github.com/moby/buildkit/solver/pb"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
const maxMaterialDepth = 24
func SourceToInput(ctx context.Context, verifier PolicyVerifierProvider, src *gwpb.ResolveSourceMetaResponse, platform *ocispecs.Platform, logf func(logrus.Level, string)) (Input, error) {
seen := map[string]struct{}{}
return sourceToInputRecursive(ctx, verifier, src, platform, 0, seen, logf)
}
func sourceToInputRecursive(ctx context.Context, verifier PolicyVerifierProvider, src *gwpb.ResolveSourceMetaResponse, platform *ocispecs.Platform, depth int, seen map[string]struct{}, logf func(logrus.Level, string)) (Input, error) {
if depth > maxMaterialDepth {
return Input{}, errors.Errorf("provenance materials depth exceeds limit %d", maxMaterialDepth)
}
if src == nil || src.Source == nil {
return Input{}, errors.New("source metadata response is required")
}
inp, unknowns, err := sourceToInput(ctx, verifier, src, platform, logf)
if err != nil {
return Input{}, err
}
inp.setUnknowns(unknowns)
inp.Env.Depth = depth
if inp.Image == nil || inp.Image.Provenance == nil || len(inp.Image.Provenance.materialsRaw) == 0 {
return inp, nil
}
key := sourceUniqueIdentifier(src.Source, platform)
if _, ok := seen[key]; ok {
return Input{}, nil
}
seen[key] = struct{}{}
defer delete(seen, key)
materials := make([]Input, 0, len(inp.Image.Provenance.materialsRaw))
for _, m := range inp.Image.Provenance.materialsRaw {
matSrc, matPlatform, err := parseSLSAMaterial(m)
if err != nil {
materials = append(materials, Input{})
continue
}
if matSrc == nil {
materials = append(materials, Input{})
continue
}
matResp := &gwpb.ResolveSourceMetaResponse{Source: matSrc}
child, err := sourceToInputRecursive(ctx, verifier, matResp, firstNonNilPlatform(matPlatform, platform), depth+1, seen, logf)
if err != nil {
return Input{}, errors.Wrapf(err, "failed to build material input for %q", m.URI)
}
materials = append(materials, child)
}
inp.Image.Provenance.Materials = materials
return inp, nil
}
// sourceUniqueIdentifier is used only for recursive cycle detection safety.
func sourceUniqueIdentifier(src *pb.SourceOp, platform *ocispecs.Platform) string {
if src == nil {
return ""
}
key := src.Identifier
if platform != nil {
key += "|" + platforms.Format(*platform)
}
return key
}
func firstNonNilPlatform(values ...*ocispecs.Platform) *ocispecs.Platform {
for _, v := range values {
if v != nil {
return v
}
}
return nil
}
policy/input_unknowns.go
+47
View File
@@ -0,0 +1,47 @@
package policy
import (
"fmt"
"strings"
)
func (inp *Input) setUnknowns(unknowns []string) {
if inp == nil {
return
}
if len(unknowns) == 0 {
inp.unknowns = nil
return
}
out := make([]string, 0, len(unknowns))
for _, u := range unknowns {
v := strings.TrimPrefix(u, "input.")
if v == "" {
continue
}
out = append(out, v)
}
inp.unknowns = out
}
func (inp Input) Unknowns() []string {
var refs []string
collectInputUnknowns(inp, "input", &refs)
return refs
}
func collectInputUnknowns(inp Input, prefix string, refs *[]string) {
for _, u := range inp.unknowns {
if u == "" {
continue
}
*refs = append(*refs, prefix+"."+u)
}
if inp.Image == nil || inp.Image.Provenance == nil {
return
}
for i := range inp.Image.Provenance.Materials {
childPrefix := fmt.Sprintf("%s.image.provenance.materials[%d]", prefix, i)
collectInputUnknowns(inp.Image.Provenance.Materials[i], childPrefix, refs)
}
}
policy/input_unknowns_test.go
+34
View File
@@ -0,0 +1,34 @@
package policy
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestInputUnknownRefsNestedMaterials(t *testing.T) {
inp := Input{
Image: &Image{
Provenance: &ImageProvenance{
Materials: []Input{
{
Image: &Image{
Provenance: &ImageProvenance{
Materials: []Input{{}},
},
},
},
},
},
},
}
inp.setUnknowns([]string{"input.image.provenance"})
inp.Image.Provenance.Materials[0].setUnknowns([]string{"input.image.hasProvenance"})
inp.Image.Provenance.Materials[0].Image.Provenance.Materials[0].setUnknowns([]string{"input.git.commit"})
require.Equal(t, []string{
"input.image.provenance",
"input.image.provenance.materials[0].image.hasProvenance",
"input.image.provenance.materials[0].image.provenance.materials[0].git.commit",
}, inp.Unknowns())
}
policy/materials.go
+116
View File
@@ -0,0 +1,116 @@
package policy
import (
"path"
"strconv"
"strings"
"github.com/distribution/reference"
"github.com/docker/buildx/util/urlutil"
slsa1 "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v1"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/gitutil"
"github.com/moby/buildkit/util/purl"
"github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
func isMaterialKey(key string) (idx int, rest string, ok bool) {
const prefix = "image.provenance.materials["
if !strings.HasPrefix(key, prefix) {
return 0, "", false
}
rest = strings.TrimPrefix(key, prefix)
end := strings.IndexByte(rest, ']')
if end < 0 {
return 0, "", false
}
n, err := strconv.Atoi(rest[:end])
if err != nil {
return 0, "", false
}
rest = strings.TrimPrefix(rest[end+1:], ".")
return n, rest, true
}
func parseSLSAMaterial(m slsa1.ResourceDescriptor) (*pb.SourceOp, *ocispecs.Platform, error) {
uri := m.URI
dgst := m.Digest
if strings.HasPrefix(uri, "pkg:docker/") {
return dockerMaterialSource(uri, dgst)
}
if gu, err := gitutil.ParseURL(uri); err == nil {
if strings.HasSuffix(strings.ToLower(gu.Path), ".git") || gu.Scheme != gitutil.HTTPProtocol && gu.Scheme != gitutil.HTTPSProtocol {
return gitMaterialSource(uri)
}
}
if urlutil.IsHTTPURL(uri) {
return &pb.SourceOp{Identifier: uri}, nil, nil
}
return nil, nil, errors.Errorf("unsupported material URI %q", uri)
}
func dockerMaterialSource(uri string, dgst map[string]string) (*pb.SourceOp, *ocispecs.Platform, error) {
refStr, platform, err := purl.PURLToRef(uri)
if err != nil {
return nil, nil, err
}
named, err := reference.ParseNormalizedNamed(refStr)
if err != nil {
return nil, nil, errors.Wrapf(err, "invalid docker reference %q from %q", refStr, uri)
}
if checksum := strings.TrimSpace(dgst["sha256"]); checksum != "" {
dgstRef, err := digest.Parse(checksum)
if err != nil {
dgstRef, err = digest.Parse("sha256:" + checksum)
}
if err != nil {
return nil, nil, errors.Wrapf(err, "invalid material digest %q for %q", checksum, uri)
}
if canonical, ok := named.(reference.Canonical); ok {
if canonical.Digest() != dgstRef {
return nil, nil, errors.Errorf("material digest mismatch for %q: ref has %s but provenance has %s", uri, canonical.Digest(), dgstRef)
}
} else {
named, err = reference.WithDigest(named, dgstRef)
if err != nil {
return nil, nil, errors.Wrapf(err, "failed to add digest %q to %q", dgstRef, refStr)
}
}
}
return &pb.SourceOp{Identifier: "docker-image://" + named.String()}, platform, nil
}
func gitMaterialSource(uri string) (*pb.SourceOp, *ocispecs.Platform, error) {
gu, err := gitutil.ParseURL(uri)
if err != nil {
return nil, nil, err
}
switch gu.Scheme {
case gitutil.HTTPSProtocol, gitutil.HTTPProtocol, gitutil.SSHProtocol, gitutil.GitProtocol:
default:
return nil, nil, errors.Errorf("unsupported git material URI %q", uri)
}
id := gu.Host + path.Join("/", gu.Path)
if gu.Opts != nil && (gu.Opts.Ref != "" || gu.Opts.Subdir != "") {
id += "#" + gu.Opts.Ref
if gu.Opts.Subdir != "" {
id += ":" + gu.Opts.Subdir
}
}
return &pb.SourceOp{
Identifier: "git://" + id,
Attrs: map[string]string{
pb.AttrFullRemoteURL: uri,
},
}, nil, nil
}
policy/provenance.go
+63 -12
View File
@@ -2,6 +2,8 @@ package policy
import (
"encoding/json"
"fmt"
"maps"
"slices"
"strings"
"time"
@@ -10,6 +12,7 @@ import (
slsa1 "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v1"
gwpb "github.com/moby/buildkit/frontend/gateway/pb"
provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types"
"github.com/sirupsen/logrus"
)
const predicateTypeAnnotation = "in-toto.io/predicate-type"
@@ -24,12 +27,11 @@ type inTotoStatement struct {
Predicate json.RawMessage `json:"predicate"`
}
func parseProvenance(ac *gwpb.AttestationChain) (*ImageProvenance, error) {
func parseProvenance(ac *gwpb.AttestationChain, logf func(logrus.Level, string)) (*ImageProvenance, error) {
if ac == nil || len(ac.Blobs) == 0 {
return nil, nil
}
// Prefer blobs that explicitly declare a provenance predicate type.
for _, b := range ac.Blobs {
if b == nil || b.Descriptor_ == nil || len(b.Data) == 0 {
continue
@@ -41,7 +43,7 @@ func parseProvenance(ac *gwpb.AttestationChain) (*ImageProvenance, error) {
if !slices.Contains(resolveProvenanceAttestations, pt) {
continue
}
prv, err := parseProvenanceBlob(b.Data, pt)
prv, err := parseProvenanceBlob(b.Data, pt, logf)
if err != nil {
return nil, err
}
@@ -53,25 +55,30 @@ func parseProvenance(ac *gwpb.AttestationChain) (*ImageProvenance, error) {
return nil, nil
}
func parseProvenanceBlob(dt []byte, pt string) (*ImageProvenance, error) {
func parseProvenanceBlob(dt []byte, pt string, logf func(logrus.Level, string)) (*ImageProvenance, error) {
var stmt inTotoStatement
if err := json.Unmarshal(dt, &stmt); err != nil || len(stmt.Predicate) == 0 {
if err := json.Unmarshal(dt, &stmt); err != nil {
return nil, nil
}
if stmt.PredicateType != "" && stmt.PredicateType != pt {
if len(stmt.Predicate) == 0 {
return nil, nil
}
switch pt {
predicateType := stmt.PredicateType
if predicateType == "" {
predicateType = pt
}
switch predicateType {
case slsa1.PredicateSLSAProvenance:
return parseSLSA1Provenance(stmt.Predicate)
return parseSLSA1Provenance(stmt.Predicate, logf)
case slsa02.PredicateSLSAProvenance:
return parseSLSA02Provenance(stmt.Predicate)
return parseSLSA02Provenance(stmt.Predicate, logf)
default:
return nil, nil
}
return nil, nil
}
func parseSLSA1Provenance(dt []byte) (*ImageProvenance, error) {
func parseSLSA1Provenance(dt []byte, logf func(logrus.Level, string)) (*ImageProvenance, error) {
var pred provenancetypes.ProvenancePredicateSLSA1
if err := json.Unmarshal(dt, &pred); err != nil {
return nil, nil
@@ -92,6 +99,7 @@ func parseSLSA1Provenance(dt []byte) (*ImageProvenance, error) {
BuildArgs: extractBuildArgs(pred.BuildDefinition.ExternalParameters.Request.Args),
RawArgs: pred.BuildDefinition.ExternalParameters.Request.Args,
}
prv.materialsRaw = rawMaterialsFromSLSA1(pred.BuildDefinition.ResolvedDependencies, logf)
if md := pred.RunDetails.Metadata; md != nil {
prv.InvocationID = md.InvocationID
@@ -108,7 +116,7 @@ func parseSLSA1Provenance(dt []byte) (*ImageProvenance, error) {
return prv, nil
}
func parseSLSA02Provenance(dt []byte) (*ImageProvenance, error) {
func parseSLSA02Provenance(dt []byte, logf func(logrus.Level, string)) (*ImageProvenance, error) {
var pred provenancetypes.ProvenancePredicateSLSA02
if err := json.Unmarshal(dt, &pred); err != nil {
return nil, nil
@@ -129,6 +137,7 @@ func parseSLSA02Provenance(dt []byte) (*ImageProvenance, error) {
BuildArgs: extractBuildArgs(pred.Invocation.Parameters.Args),
RawArgs: pred.Invocation.Parameters.Args,
}
prv.materialsRaw = rawMaterialsFromSLSA02(pred.Materials, logf)
if md := pred.Metadata; md != nil {
prv.InvocationID = md.BuildInvocationID
@@ -146,6 +155,48 @@ func parseSLSA02Provenance(dt []byte) (*ImageProvenance, error) {
return prv, nil
}
func rawMaterialsFromSLSA1(materials []slsa1.ResourceDescriptor, logf func(logrus.Level, string)) []slsa1.ResourceDescriptor {
if len(materials) == 0 {
return nil
}
out := make([]slsa1.ResourceDescriptor, 0, len(materials))
for _, m := range materials {
rd := slsa1.ResourceDescriptor{
URI: m.URI,
Digest: maps.Clone(m.Digest),
}
if _, _, err := parseSLSAMaterial(rd); err != nil {
if logf != nil {
logf(logrus.WarnLevel, fmt.Sprintf("skipping unsupported provenance material %q: %v", m.URI, err))
}
continue
}
out = append(out, rd)
}
return out
}
func rawMaterialsFromSLSA02(materials []slsa02.ProvenanceMaterial, logf func(logrus.Level, string)) []slsa1.ResourceDescriptor {
if len(materials) == 0 {
return nil
}
out := make([]slsa1.ResourceDescriptor, 0, len(materials))
for _, m := range materials {
rd := slsa1.ResourceDescriptor{
URI: m.URI,
Digest: maps.Clone(m.Digest),
}
if _, _, err := parseSLSAMaterial(rd); err != nil {
if logf != nil {
logf(logrus.WarnLevel, fmt.Sprintf("skipping unsupported provenance material %q: %v", m.URI, err))
}
continue
}
out = append(out, rd)
}
return out
}
func boolPtr(v bool) *bool {
return &v
}
policy/resolve.go
+152
View File
@@ -0,0 +1,152 @@
package policy
import (
"context"
"slices"
"strings"
"github.com/docker/buildx/util/sourcemeta"
"github.com/moby/buildkit/client/llb/sourceresolver"
gwpb "github.com/moby/buildkit/frontend/gateway/pb"
"github.com/moby/buildkit/solver/pb"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
type SourceMetadataResolver interface {
ResolveSourceMetadata(context.Context, *pb.SourceOp, sourceresolver.Opt) (*sourceresolver.MetaResponse, error)
}
func ResolveInputUnknowns(ctx context.Context, input *Input, rootSource *pb.SourceOp, unknowns []string, rootPlatform *pb.Platform, defaultPlatform *ocispecs.Platform, resolver SourceMetadataResolver, verifier PolicyVerifierProvider, logf func(logrus.Level, string)) (bool, *gwpb.ResolveSourceMetaRequest, error) {
if input == nil || len(unknowns) == 0 {
return false, nil, nil
}
return resolveNodeUnknowns(ctx, input, rootSource, defaultPlatform, normalizeNodeUnknowns(unknowns), rootPlatform, defaultPlatform, resolver, verifier, logf, nil)
}
func resolveNodeUnknowns(ctx context.Context, node *Input, source *pb.SourceOp, nodePlatform *ocispecs.Platform, unknowns []string, rootPlatform *pb.Platform, defaultPlatform *ocispecs.Platform, resolver SourceMetadataResolver, verifier PolicyVerifierProvider, logf func(logrus.Level, string), setNode func(Input) error) (bool, *gwpb.ResolveSourceMetaRequest, error) {
directUnknowns, childUnknowns := splitNodeUnknowns(unknowns)
if len(directUnknowns) > 0 {
req, err := sourceResolveRequest(source, nodePlatform, setNode == nil, directUnknowns, rootPlatform, logf)
if err != nil {
return false, nil, err
}
if req != nil {
if setNode == nil {
return false, req, nil
}
if resolver == nil {
return false, nil, errors.Errorf("material metadata resolution requires source resolver")
}
resp, err := resolveSourceMetaWithResolver(ctx, resolver, req, defaultPlatform)
if err != nil {
return false, nil, errors.Wrap(err, "failed to resolve source metadata for material")
}
nextInput, err := SourceToInput(ctx, verifier, resp, nodePlatform, logf)
if err != nil {
return false, nil, errors.Wrap(err, "failed to rebuild material input")
}
if err := setNode(nextInput); err != nil {
return false, nil, err
}
return true, nil, nil
}
}
for idx, childNodeUnknowns := range childUnknowns {
if node == nil || node.Image == nil || node.Image.Provenance == nil {
continue
}
if idx < 0 || idx >= len(node.Image.Provenance.Materials) || idx >= len(node.Image.Provenance.materialsRaw) {
continue
}
raw := node.Image.Provenance.materialsRaw[idx]
childSource, childNodePlatform, err := parseSLSAMaterial(raw)
if err != nil {
continue
}
childPlatform := firstNonNilPlatform(childNodePlatform, nodePlatform)
retry, next, err := resolveNodeUnknowns(ctx, &node.Image.Provenance.Materials[idx], childSource, childPlatform, childNodeUnknowns, rootPlatform, defaultPlatform, resolver, verifier, logf, func(next Input) error {
node.Image.Provenance.Materials[idx] = next
return nil
})
if err != nil {
return false, nil, err
}
if retry || next != nil {
return retry, next, nil
}
}
return false, nil, nil
}
func splitNodeUnknowns(unknowns []string) ([]string, map[int][]string) {
child := map[int][]string{}
var direct []string
for _, u := range unknowns {
if u == "" {
continue
}
if idx, rest, ok := isMaterialKey(u); ok {
if rest == "" {
continue
}
if !slices.Contains(child[idx], rest) {
child[idx] = append(child[idx], rest)
}
continue
}
if !slices.Contains(direct, u) {
direct = append(direct, u)
}
}
return direct, child
}
func normalizeNodeUnknowns(unknowns []string) []string {
out := make([]string, 0, len(unknowns))
for _, u := range unknowns {
v := strings.TrimPrefix(u, "input.")
if v == "" {
continue
}
if !slices.Contains(out, v) {
out = append(out, v)
}
}
return out
}
func sourceResolveRequest(source *pb.SourceOp, nodePlatform *ocispecs.Platform, rootNode bool, fields []string, rootPlatform *pb.Platform, logf func(logrus.Level, string)) (*gwpb.ResolveSourceMetaRequest, error) {
if source == nil {
return nil, nil
}
req := &gwpb.ResolveSourceMetaRequest{Source: source}
if nodePlatform != nil {
req.Platform = &pb.Platform{OS: nodePlatform.OS, Architecture: nodePlatform.Architecture, Variant: nodePlatform.Variant}
} else if rootNode {
req.Platform = rootPlatform
}
if err := AddUnknownsWithLogger(logf, req, fields); err != nil {
return nil, err
}
if req.Image == nil && req.Git == nil && !hasHTTPUnknowns(fields) {
return nil, nil
}
return req, nil
}
func resolveSourceMetaWithResolver(ctx context.Context, resolver SourceMetadataResolver, req *gwpb.ResolveSourceMetaRequest, defaultPlatform *ocispecs.Platform) (*gwpb.ResolveSourceMetaResponse, error) {
if resolver == nil {
return nil, errors.New("source resolver is not configured")
}
if req == nil || req.Source == nil {
return nil, errors.New("source metadata request is missing source")
}
resp, err := resolver.ResolveSourceMetadata(ctx, req.Source, sourcemeta.ToResolverOpt(req, defaultPlatform))
if err != nil {
return nil, err
}
return sourcemeta.ToGatewayMetaResponse(resp), nil
}
policy/resolve_test.go
+68
View File
@@ -0,0 +1,68 @@
package policy
import (
"context"
"testing"
slsa1 "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v1"
"github.com/moby/buildkit/client/llb/sourceresolver"
"github.com/moby/buildkit/solver/pb"
"github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/stretchr/testify/require"
)
type stubSourceResolver struct {
resolveFn func(context.Context, *pb.SourceOp, sourceresolver.Opt) (*sourceresolver.MetaResponse, error)
}
func (s stubSourceResolver) ResolveSourceMetadata(ctx context.Context, op *pb.SourceOp, opt sourceresolver.Opt) (*sourceresolver.MetaResponse, error) {
return s.resolveFn(ctx, op, opt)
}
func TestResolveInputUnknownsResolvesMaterialField(t *testing.T) {
inp := Input{
Image: &Image{
Provenance: &ImageProvenance{
Materials: []Input{
{Image: &Image{Ref: "docker.io/library/alpine:3.20"}},
},
materialsRaw: []slsa1.ResourceDescriptor{
{URI: "pkg:docker/library/alpine@3.20?platform=linux/amd64"},
},
},
},
}
inp.Image.Provenance.Materials[0].setUnknowns([]string{"input.image.hasProvenance"})
resolver := stubSourceResolver{
resolveFn: func(_ context.Context, op *pb.SourceOp, _ sourceresolver.Opt) (*sourceresolver.MetaResponse, error) {
require.Equal(t, "docker-image://docker.io/library/alpine:3.20", op.Identifier)
return &sourceresolver.MetaResponse{
Op: op,
Image: &sourceresolver.ResolveImageResponse{
Digest: digest.Digest("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
AttestationChain: &sourceresolver.AttestationChain{
AttestationManifest: digest.Digest("sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"),
},
},
}, nil
},
}
retry, next, err := ResolveInputUnknowns(
context.Background(),
&inp,
&pb.SourceOp{Identifier: "docker-image://docker.io/library/busybox:latest"},
[]string{"image.provenance.materials[0].image.hasProvenance"},
&pb.Platform{OS: "linux", Architecture: "amd64"},
&ocispecs.Platform{OS: "linux", Architecture: "amd64"},
resolver,
nil,
nil,
)
require.NoError(t, err)
require.True(t, retry)
require.Nil(t, next)
require.True(t, inp.Image.Provenance.Materials[0].Image.HasProvenance)
}
policy/tester.go
+1 -1
View File
@@ -454,7 +454,7 @@ func resolveTestInput(ctx context.Context, files []File, resolver *TestOptionsPr
return nil, false, err
}
if next == nil {
inp, _, err := SourceToInputWithLogger(ctx, resolver.VerifierProvider, srcReq, platform, nil)
inp, _, err := sourceToInput(ctx, resolver.VerifierProvider, srcReq, platform, nil)
if err != nil {
return nil, false, err
}
policy/types.go
+7
View File
@@ -3,6 +3,7 @@ package policy
import (
"time"
slsa1 "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v1"
"github.com/moby/buildkit/util/gitutil/gitobject"
policytypes "github.com/moby/policy-helpers/types"
)
@@ -13,6 +14,8 @@ type Input struct {
Image *Image `json:"image,omitempty"`
HTTP *HTTP `json:"http,omitempty"`
Git *Git `json:"git,omitempty"`
unknowns []string `json:"-"`
}
type Decision struct {
@@ -25,6 +28,7 @@ type Env struct {
Labels map[string]string `json:"labels,omitempty"`
Filename string `json:"filename,omitempty"`
Target string `json:"target,omitempty"`
Depth int `json:"depth"`
}
type HTTP struct {
@@ -149,6 +153,9 @@ type ImageProvenance struct {
Hermetic *bool `json:"hermetic,omitempty"`
Completeness *ImageProvenanceCompleteness `json:"completeness,omitempty"`
Materials []Input `json:"materials,omitempty"`
materialsRaw []slsa1.ResourceDescriptor `json:"-"`
}
type ImageProvenanceConfigSource struct {
policy/utils_test.go
+6 -3
View File
@@ -29,6 +29,8 @@ func TestTrimKey(t *testing.T) {
{"git.tag[0]", "git.tag"},
{"input.git.tag.author", "git.tag"},
{"input.git.tag[0]", "git.tag"},
{"input.image.provenance.materials[0].image.hasProvenance", "image.provenance.materials[0].image.hasProvenance"},
{"image.provenance.materials[0].image.labels", "image.provenance.materials[0].image.labels"},
{"a.b.c", "a.b"},
}
@@ -46,16 +48,17 @@ func TestCollectUnknowns(t *testing.T) {
p if {
input.git.tag[0].author == "a"
input.image.signatures[_].signer.certificateIssuer != ""
input.image.provenance.materials[0].image.hasProvenance
data.foo.bar == 1
}
`)
require.NoError(t, err)
all := collectUnknowns([]*ast.Module{mod}, nil)
require.ElementsMatch(t, []string{"git.tag", "image.signatures"}, all)
require.ElementsMatch(t, []string{"git.tag", "image.signatures", "image.provenance.materials[0].image.hasProvenance"}, all)
filtered := collectUnknowns([]*ast.Module{mod}, []string{"input.image.signatures"})
require.Equal(t, []string{"image.signatures"}, filtered)
filtered := collectUnknowns([]*ast.Module{mod}, []string{"input.image.signatures", "input.image.provenance.materials[0].image.hasProvenance"})
require.ElementsMatch(t, []string{"image.signatures", "image.provenance.materials[0].image.hasProvenance"}, filtered)
}
func TestRuntimeUnknownInputRefs(t *testing.T) {
policy/validate.go
+157 -127
View File
@@ -39,6 +39,8 @@ type Policy struct {
denyIdentifiers map[string]struct{}
}
const maxResolveIterations = 10
type state struct {
Input Input
Unknowns map[string]struct{}
@@ -131,7 +133,7 @@ func (p *Policy) CheckPolicy(ctx context.Context, req *policysession.CheckPolicy
if req.Source == nil || req.Source.Source == nil {
return nil, nil, errors.Errorf("no source info in request")
}
src := req.Source
var platform *ocispecs.Platform
if req.Platform != nil {
pl, err := platformFromReq(req)
@@ -143,17 +145,15 @@ func (p *Policy) CheckPolicy(ctx context.Context, req *policysession.CheckPolicy
platform = p.opt.DefaultPlatform
}
inp, unknowns, err := SourceToInputWithLogger(ctx, p.opt.VerifierProvider, src, platform, p.opt.Log)
inp, err := SourceToInput(ctx, p.opt.VerifierProvider, req.Source, platform, p.opt.Log)
if err != nil {
return nil, nil, errors.Wrapf(err, "failed to convert source to policy input")
return nil, nil, errors.Wrap(err, "failed to build policy input")
}
inp.Env = p.opt.Env
caps := &ast.Capabilities{
Builtins: builtins(),
Features: slices.Clone(ast.Features),
}
comp := ast.NewCompiler().WithCapabilities(caps).WithKeepModules(true)
if p.opt.Log != nil {
comp = comp.WithEnablePrintStatements(true)
@@ -170,7 +170,6 @@ func (p *Policy) CheckPolicy(ctx context.Context, req *policysession.CheckPolicy
var root fs.StatFS
var closeFS func() error
defer func() {
if closeFS != nil {
closeFS()
@@ -210,7 +209,6 @@ func (p *Policy) CheckPolicy(ctx context.Context, req *policysession.CheckPolicy
if err != nil {
return nil, errors.Wrapf(err, "failed to parse imported policy file %s for module %s", fn, k)
}
// rewrite package to be less strict
pkgParts := strings.Split(pkgPath, ".")
ref := ast.Ref{mod.Package.Path[0]}
for _, p := range pkgParts {
@@ -224,155 +222,171 @@ func (p *Policy) CheckPolicy(ctx context.Context, req *policysession.CheckPolicy
return out, nil
})
opts := []func(*rego.Rego){
baseOpts := []func(*rego.Rego){
rego.SetRegoVersion(ast.RegoV1),
rego.Query("data.docker.decision"),
rego.Input(inp),
rego.SkipPartialNamespace(true),
rego.Compiler(comp),
rego.Module(builtinPolicyModuleFilename, builtinPolicyModule),
}
if p.opt.Log != nil {
opts = append(opts,
baseOpts = append(baseOpts,
rego.EnablePrintStatements(true),
rego.PrintHook(p),
)
}
st := &state{
Input: inp,
}
for _, f := range p.funcs {
opts = append(opts, f.impl(st))
}
opts = append(opts, rego.Module(builtinPolicyModuleFilename, builtinPolicyModule))
for _, file := range p.opt.Files {
opts = append(opts, rego.Module(file.Filename, string(file.Data)))
}
dt, err := json.MarshalIndent(inp, "", " ")
if err != nil {
return nil, nil, errors.Wrapf(err, "failed to marshal policy input")
baseOpts = append(baseOpts, rego.Module(file.Filename, string(file.Data)))
}
p.log(logrus.InfoLevel, "checking policy for source %s", sourceName(req))
p.log(logrus.DebugLevel, "policy input: %s", dt)
if len(unknowns) > 0 {
p.log(logrus.DebugLevel, "unknowns for policy evaluation: %+v", unknowns)
opts = append(opts, rego.Unknowns(unknowns))
}
r := rego.New(opts...)
for range maxResolveIterations {
runInput := inp
applyEnvWithDepth(&runInput, p.opt.Env, 0)
if len(unknowns) > 0 {
pq, err := r.Partial(ctx)
runOpts := append([]func(*rego.Rego){}, baseOpts...)
runOpts = append(runOpts, rego.Input(runInput))
st := &state{Input: runInput}
for _, f := range p.funcs {
runOpts = append(runOpts, f.impl(st))
}
dt, err := json.MarshalIndent(runInput, "", " ")
if err != nil {
return nil, nil, errors.Wrapf(err, "failed to marshal policy input")
}
p.log(logrus.DebugLevel, "policy input: %s", dt)
unknowns := inp.Unknowns()
if len(unknowns) > 0 {
p.log(logrus.DebugLevel, "unknowns for policy evaluation: %+v", summarizeUnknownsForLog(unknowns))
runOpts = append(runOpts, rego.Unknowns(unknowns))
}
r := rego.New(runOpts...)
if len(unknowns) > 0 {
pq, err := r.Partial(ctx)
if err != nil {
return nil, nil, err
}
unk := collectUnknowns(pq.Support, unknowns)
unk = append(unk, runtimeUnknownInputRefs(st)...)
retry, next, err := p.resolveUnknowns(ctx, &inp, req, platform, unk)
if err != nil {
return nil, nil, err
}
if next != nil {
return nil, next, nil
}
if retry {
continue
}
}
st.ImagePins = nil
rs, err := r.Eval(ctx)
if err != nil {
return nil, nil, err
}
unk := collectUnknowns(pq.Support, unknowns)
unk = append(unk, runtimeUnknownInputRefs(st)...)
if len(unk) > 0 {
next := &gwpb.ResolveSourceMetaRequest{
Source: req.Source.Source,
Platform: req.Platform,
}
if err := AddUnknownsWithLogger(p.opt.Log, next, unk); err != nil {
return nil, nil, err
}
if next.Image != nil || next.Git != nil || hasHTTPUnknowns(unk) {
p.log(logrus.InfoLevel, "policy decision for source %s: resolve missing fields %+v", sourceName(req), summarizeUnknownsForLog(unk))
return nil, next, nil
}
}
}
st.ImagePins = nil
rs, err := r.Eval(ctx)
if err != nil {
return nil, nil, err
}
rtUnk := runtimeUnknownInputRefs(st)
if len(rtUnk) > 0 {
next := &gwpb.ResolveSourceMetaRequest{
Source: req.Source.Source,
Platform: req.Platform,
}
if err := AddUnknownsWithLogger(p.opt.Log, next, rtUnk); err != nil {
retry, next, err := p.resolveUnknowns(ctx, &inp, req, platform, runtimeUnknownInputRefs(st))
if err != nil {
return nil, nil, err
}
if next.Image != nil || next.Git != nil || hasHTTPUnknowns(rtUnk) {
p.log(logrus.InfoLevel, "policy decision for source %s: resolve missing fields %+v", sourceName(req), summarizeUnknownsForLog(rtUnk))
if next != nil {
return nil, next, nil
}
}
if len(rs) == 0 {
return nil, nil, errors.Errorf("policy returned zero result")
}
rsz := rs[0]
if len(rsz.Expressions) == 0 {
return nil, nil, errors.Errorf("policy returned zero expressions")
}
v := rsz.Expressions[0].Value
vt, ok := v.(map[string]any)
if !ok {
return nil, nil, errors.Errorf("unexpected policy return type: %T %s", vt, rsz.Expressions[0].Text)
}
resp := &policysession.DecisionResponse{
Action: moby_buildkit_v1_sourcepolicy.PolicyAction_DENY,
}
p.log(logrus.DebugLevel, "policy response: %+v", vt)
if v, ok := vt["allow"]; ok {
if vv, ok := v.(bool); !ok {
return nil, nil, errors.Errorf("invalid allowed property type %T, expecting bool", v)
} else if vv {
resp.Action = moby_buildkit_v1_sourcepolicy.PolicyAction_ALLOW
if retry {
continue
}
}
if v, ok := vt["deny_msg"]; ok {
if vv, ok := v.([]any); ok {
for _, m := range vv {
if m, ok := m.(string); ok {
resp.DenyMessages = append(resp.DenyMessages, &policysession.DenyMessage{
Message: m,
})
if len(rs) == 0 {
return nil, nil, errors.Errorf("policy returned zero result")
}
rsz := rs[0]
if len(rsz.Expressions) == 0 {
return nil, nil, errors.Errorf("policy returned zero expressions")
}
v := rsz.Expressions[0].Value
vt, ok := v.(map[string]any)
if !ok {
return nil, nil, errors.Errorf("unexpected policy return type: %T %s", vt, rsz.Expressions[0].Text)
}
resp := &policysession.DecisionResponse{
Action: moby_buildkit_v1_sourcepolicy.PolicyAction_DENY,
}
p.log(logrus.DebugLevel, "policy response: %+v", vt)
if v, ok := vt["allow"]; ok {
if vv, ok := v.(bool); !ok {
return nil, nil, errors.Errorf("invalid allowed property type %T, expecting bool", v)
} else if vv {
resp.Action = moby_buildkit_v1_sourcepolicy.PolicyAction_ALLOW
}
}
if v, ok := vt["deny_msg"]; ok {
if vv, ok := v.([]any); ok {
for _, m := range vv {
if m, ok := m.(string); ok {
resp.DenyMessages = append(resp.DenyMessages, &policysession.DenyMessage{
Message: m,
})
}
}
}
}
}
if resp.Action == moby_buildkit_v1_sourcepolicy.PolicyAction_ALLOW {
if len(st.ImagePins) > 1 {
return nil, nil, errors.Errorf("multiple image pins set to %s: %v", sourceName(req), st.ImagePins)
}
if len(st.ImagePins) == 1 {
newSrc, err := addPinToImage(src.Source, slices.Collect(maps.Keys(st.ImagePins))[0])
if err != nil {
return nil, nil, errors.Wrapf(err, "failed to add image pin to source")
if resp.Action == moby_buildkit_v1_sourcepolicy.PolicyAction_ALLOW {
if len(st.ImagePins) > 1 {
return nil, nil, errors.Errorf("multiple image pins set to %s: %v", sourceName(req), st.ImagePins)
}
p.log(logrus.InfoLevel, "policy decision for source %s: convert to %s", sourceName(req), newSrc.Identifier)
if len(st.ImagePins) == 1 {
newSrc, err := addPinToImage(req.Source.Source, slices.Collect(maps.Keys(st.ImagePins))[0])
if err != nil {
return nil, nil, errors.Wrapf(err, "failed to add image pin to source")
}
p.log(logrus.InfoLevel, "policy decision for source %s: convert to %s", sourceName(req), newSrc.Identifier)
return &policysession.DecisionResponse{
Action: moby_buildkit_v1_sourcepolicy.PolicyAction_CONVERT,
Update: newSrc,
}, nil, nil
return &policysession.DecisionResponse{
Action: moby_buildkit_v1_sourcepolicy.PolicyAction_CONVERT,
Update: newSrc,
}, nil, nil
}
}
p.log(logrus.InfoLevel, "policy decision for source %s: %s", sourceName(req), resp.Action)
for _, dm := range resp.DenyMessages {
p.log(logrus.InfoLevel, " - %s", dm.Message)
}
if resp.Action == moby_buildkit_v1_sourcepolicy.PolicyAction_DENY {
p.recordDenyIdentifier(req)
}
return resp, nil, nil
}
p.log(logrus.InfoLevel, "policy decision for source %s: %s", sourceName(req), resp.Action)
for _, dm := range resp.DenyMessages {
p.log(logrus.InfoLevel, " - %s", dm.Message)
}
if resp.Action == moby_buildkit_v1_sourcepolicy.PolicyAction_DENY {
p.recordDenyIdentifier(req)
}
return resp, nil, nil
return nil, nil, errors.Errorf("maximum attempts reached for resolving policy metadata")
}
func (p *Policy) resolveUnknowns(ctx context.Context, input *Input, req *policysession.CheckPolicyRequest, defaultPlatform *ocispecs.Platform, unk []string) (bool, *gwpb.ResolveSourceMetaRequest, error) {
var resolver SourceMetadataResolver
if p.opt.SourceResolver != nil {
resolver = p.opt.SourceResolver
}
retry, next, err := ResolveInputUnknowns(ctx, input, req.Source.Source, unk, req.Platform, defaultPlatform, resolver, p.opt.VerifierProvider, p.opt.Log)
if err != nil {
return false, nil, err
}
if next != nil {
p.log(logrus.InfoLevel, "policy decision for source %s: resolve missing fields %+v", sourceName(req), summarizeUnknownsForLog(unk))
return false, next, nil
}
return retry, nil, nil
}
func platformFromReq(req *policysession.CheckPolicyRequest) (*ocispecs.Platform, error) {
if req.Platform != nil {
platformStr := req.Platform.OS + "/" + req.Platform.Architecture
@@ -404,11 +418,7 @@ func (p *Policy) Print(ctx print.Context, msg string) error {
return nil
}
func SourceToInput(ctx context.Context, getVerifier PolicyVerifierProvider, src *gwpb.ResolveSourceMetaResponse, platform *ocispecs.Platform) (Input, []string, error) {
return SourceToInputWithLogger(ctx, getVerifier, src, platform, nil)
}
func SourceToInputWithLogger(ctx context.Context, getVerifier PolicyVerifierProvider, src *gwpb.ResolveSourceMetaResponse, platform *ocispecs.Platform, logf func(logrus.Level, string)) (Input, []string, error) {
func sourceToInput(ctx context.Context, getVerifier PolicyVerifierProvider, src *gwpb.ResolveSourceMetaResponse, platform *ocispecs.Platform, logf func(logrus.Level, string)) (Input, []string, error) {
var inp Input
var unknowns []string
@@ -625,7 +635,9 @@ func SourceToInputWithLogger(ctx context.Context, getVerifier PolicyVerifierProv
if err := json.Unmarshal(cfg, &img); err != nil {
return inp, nil, errors.Wrapf(err, "failed to unmarshal image config")
}
inp.Image.CreatedTime = img.Created.Format(time.RFC3339)
if img.Created != nil {
inp.Image.CreatedTime = img.Created.Format(time.RFC3339)
}
inp.Image.Labels = img.Config.Labels
inp.Image.Env = img.Config.Env
inp.Image.User = img.Config.User
@@ -639,7 +651,7 @@ func SourceToInputWithLogger(ctx context.Context, getVerifier PolicyVerifierProv
}
if ac := src.Image.AttestationChain; ac != nil {
if prv, err := parseProvenance(ac); err != nil {
if prv, err := parseProvenance(ac, logf); err != nil {
if logf != nil {
logf(logrus.DebugLevel, fmt.Sprintf("failed to parse image provenance: %v", err))
}
@@ -681,6 +693,20 @@ func withPrefix(arr []string, prefix string) []string {
return out
}
func applyEnvWithDepth(inp *Input, env Env, depth int) {
if inp == nil {
return
}
inp.Env = env
inp.Env.Depth = depth
if inp.Image == nil || inp.Image.Provenance == nil || len(inp.Image.Provenance.Materials) == 0 {
return
}
for i := range inp.Image.Provenance.Materials {
applyEnvWithDepth(&inp.Image.Provenance.Materials[i], env, depth+1)
}
}
func AddUnknowns(req *gwpb.ResolveSourceMetaRequest, unk []string) error {
return AddUnknownsWithLogger(nil, req, unk)
}
@@ -811,6 +837,7 @@ func summarizeUnknownsForLog(unk []string) []string {
out := make([]string, 0, len(unk))
seen := map[string]struct{}{}
for _, u := range unk {
u = strings.TrimPrefix(u, "input.")
if strings.HasPrefix(u, "image.signatures") {
u = "image.signatures"
}
@@ -849,6 +876,9 @@ func hasHTTPUnknowns(unk []string) bool {
func trimKey(s string) string {
s = strings.TrimPrefix(s, "input.")
if strings.HasPrefix(s, "image.provenance.materials[") {
return s
}
const (
dot = '.'
policy/validate_test.go
+2 -2
View File
@@ -23,7 +23,7 @@ import (
"github.com/stretchr/testify/require"
)
func TestSourceToInputWithLogger(t *testing.T) {
func TestSourceToInputSingleSource(t *testing.T) {
tm := time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC)
tests := []struct {
@@ -866,7 +866,7 @@ func TestSourceToInputWithLogger(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
inp, unknowns, err := SourceToInputWithLogger(t.Context(), tc.verifier, tc.src, tc.platform, nil)
inp, unknowns, err := sourceToInput(t.Context(), tc.verifier, tc.src, tc.platform, nil)
if tc.assert != nil {
tc.assert(t, inp, unknowns, err)
return
util/sourcemeta/convert.go
+102
View File
@@ -0,0 +1,102 @@
package sourcemeta
import (
"github.com/moby/buildkit/client/llb/sourceresolver"
gwpb "github.com/moby/buildkit/frontend/gateway/pb"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"google.golang.org/protobuf/types/known/timestamppb"
)
func ToResolverOpt(req *gwpb.ResolveSourceMetaRequest, defaultPlatform *ocispecs.Platform) sourceresolver.Opt {
platform := defaultPlatform
if req != nil && req.Platform != nil {
platform = &ocispecs.Platform{
Architecture: req.Platform.Architecture,
OS: req.Platform.OS,
Variant: req.Platform.Variant,
}
}
opt := sourceresolver.Opt{}
if req != nil {
opt.LogName = req.LogName
opt.SourcePolicies = req.SourcePolicies
}
if req != nil && req.Image != nil {
opt.ImageOpt = &sourceresolver.ResolveImageOpt{
NoConfig: req.Image.NoConfig,
AttestationChain: req.Image.AttestationChain,
ResolveAttestations: append([]string(nil), req.Image.ResolveAttestations...),
ResolveMode: req.ResolveMode,
Platform: platform,
}
}
if req != nil && req.Git != nil {
opt.GitOpt = &sourceresolver.ResolveGitOpt{ReturnObject: req.Git.ReturnObject}
}
return opt
}
func ToGatewayMetaResponse(resp *sourceresolver.MetaResponse) *gwpb.ResolveSourceMetaResponse {
out := &gwpb.ResolveSourceMetaResponse{Source: resp.Op}
if resp.Image != nil {
out.Image = &gwpb.ResolveSourceImageResponse{
Digest: resp.Image.Digest.String(),
Config: resp.Image.Config,
AttestationChain: toGatewayAttestationChain(resp.Image.AttestationChain),
}
}
if resp.Git != nil {
out.Git = &gwpb.ResolveSourceGitResponse{
Checksum: resp.Git.Checksum,
Ref: resp.Git.Ref,
CommitChecksum: resp.Git.CommitChecksum,
CommitObject: resp.Git.CommitObject,
TagObject: resp.Git.TagObject,
}
}
if resp.HTTP != nil {
var lastModified *timestamppb.Timestamp
if resp.HTTP.LastModified != nil {
lastModified = timestamppb.New(*resp.HTTP.LastModified)
}
out.HTTP = &gwpb.ResolveSourceHTTPResponse{
Checksum: resp.HTTP.Digest.String(),
Filename: resp.HTTP.Filename,
LastModified: lastModified,
}
}
return out
}
func toGatewayDescriptor(desc ocispecs.Descriptor) *gwpb.Descriptor {
return &gwpb.Descriptor{
MediaType: desc.MediaType,
Digest: desc.Digest.String(),
Size: desc.Size,
Annotations: desc.Annotations,
}
}
func toGatewayAttestationChain(chain *sourceresolver.AttestationChain) *gwpb.AttestationChain {
if chain == nil {
return nil
}
signatures := make([]string, 0, len(chain.SignatureManifests))
for _, dgst := range chain.SignatureManifests {
signatures = append(signatures, dgst.String())
}
blobs := make(map[string]*gwpb.Blob, len(chain.Blobs))
for dgst, blob := range chain.Blobs {
blobs[dgst.String()] = &gwpb.Blob{
Descriptor_: toGatewayDescriptor(blob.Descriptor),
Data: blob.Data,
}
}
return &gwpb.AttestationChain{
Root: chain.Root.String(),
ImageManifest: chain.ImageManifest.String(),
AttestationManifest: chain.AttestationManifest.String(),
SignatureManifests: signatures,
Blobs: blobs,
}
}
vendor/github.com/moby/buildkit/util/purl/image.go
Generated Vendored
+132
View File
@@ -0,0 +1,132 @@
package purl
import (
"strings"
"github.com/containerd/platforms"
"github.com/distribution/reference"
digest "github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
packageurl "github.com/package-url/packageurl-go"
"github.com/pkg/errors"
)
// RefToPURL converts an image reference with optional platform constraint to a package URL.
// Image references are defined in https://github.com/distribution/distribution/blob/v2.8.1/reference/reference.go#L1
// Package URLs are defined in https://github.com/package-url/purl-spec
func RefToPURL(purlType string, ref string, platform *ocispecs.Platform) (string, error) {
named, err := reference.ParseNormalizedNamed(ref)
if err != nil {
return "", errors.Wrapf(err, "failed to parse ref %q", ref)
}
var qualifiers []packageurl.Qualifier
if canonical, ok := named.(reference.Canonical); ok {
qualifiers = append(qualifiers, packageurl.Qualifier{
Key: "digest",
Value: canonical.Digest().String(),
})
} else {
named = reference.TagNameOnly(named)
}
version := ""
if tagged, ok := named.(reference.Tagged); ok {
version = tagged.Tag()
}
name := reference.FamiliarName(named)
ns := ""
parts := strings.Split(name, "/")
if len(parts) > 1 {
ns = strings.Join(parts[:len(parts)-1], "/")
}
name = parts[len(parts)-1]
if platform != nil {
p := platforms.Normalize(*platform)
qualifiers = append(qualifiers, packageurl.Qualifier{
Key: "platform",
Value: platforms.Format(p),
})
}
p := packageurl.NewPackageURL(purlType, ns, name, version, qualifiers, "")
return p.ToString(), nil
}
// PURLToRef converts a package URL to an image reference and platform.
func PURLToRef(purl string) (string, *ocispecs.Platform, error) {
p, err := packageurl.FromString(purl)
if err != nil {
return "", nil, err
}
if p.Type != "docker" {
return "", nil, errors.Errorf("invalid package type %q, expecting docker", p.Type)
}
ref := p.Name
if p.Namespace != "" {
ref = p.Namespace + "/" + ref
}
dgstVersion := ""
if p.Version != "" {
dgst, err := digest.Parse(p.Version)
if err == nil {
ref = ref + "@" + dgst.String()
dgstVersion = dgst.String()
} else {
ref += ":" + p.Version
}
}
var platform *ocispecs.Platform
for _, q := range p.Qualifiers {
if q.Key == "platform" {
p, err := platforms.Parse(q.Value)
if err != nil {
return "", nil, err
}
// OS-version and OS-features are not included when serializing a
// platform as a string, however, containerd platforms.Parse appends
// missing information (including os-version) based on the host's
// platform.
//
// Given that this information is not obtained from the package-URL,
// we're resetting this information. Ideally, we'd do the same for
// "OS" and "architecture" (when not included in the URL).
//
// See:
// - https://github.com/containerd/containerd/commit/cfb30a31a8507e4417d42d38c9a99b04fc8af8a9 (https://github.com/containerd/containerd/pull/8778)
// - https://github.com/moby/buildkit/pull/4315#discussion_r1355141241
p.OSVersion = ""
p.OSFeatures = nil
platform = &p
}
if q.Key == "digest" {
if dgstVersion != "" {
if dgstVersion != q.Value {
return "", nil, errors.Errorf("digest %q does not match version %q", q.Value, dgstVersion)
}
continue
}
dgst, err := digest.Parse(q.Value)
if err != nil {
return "", nil, err
}
ref = ref + "@" + dgst.String()
dgstVersion = dgst.String()
}
}
if dgstVersion == "" && p.Version == "" {
ref += ":latest"
}
named, err := reference.ParseNormalizedNamed(ref)
if err != nil {
return "", nil, errors.Wrapf(err, "invalid image url %q", purl)
}
return named.String(), platform, nil
}
vendor/github.com/package-url/packageurl-go/.gitignore
Generated
+14
View File
@@ -0,0 +1,14 @@
# Binaries for programs and plugins
*.exe
*.dll
*.so
*.dylib
# Test binary, build with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
.glide/
vendor/github.com/package-url/packageurl-go/.golangci.yaml
Generated Vendored
+17
View File
@@ -0,0 +1,17 @@
# individual linter configs go here
linters-settings:
# default linters are enabled `golangci-lint help linters`
linters:
disable-all: true
enable:
- deadcode
- errcheck
- gosimple
- govet
- ineffassign
- staticcheck
- structcheck
- typecheck
- unused
- varcheck
vendor/github.com/package-url/packageurl-go/LICENSE
Generated Vendored
+18
View File
@@ -0,0 +1,18 @@
Copyright (c) the purl authors
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
vendor/github.com/package-url/packageurl-go/Makefile
Generated Vendored
+8
View File
@@ -0,0 +1,8 @@
.PHONY: test clean lint
test:
go test -v -cover ./...
lint:
go get -u golang.org/x/lint/golint
golint -set_exit_status
vendor/github.com/package-url/packageurl-go/README.md
Generated Vendored
+74
View File
@@ -0,0 +1,74 @@
# packageurl-go
[![build](https://github.com/package-url/packageurl-go/workflows/test/badge.svg)](https://github.com/package-url/packageurl-go/actions?query=workflow%3Atest) [![Coverage Status](https://coveralls.io/repos/github/package-url/packageurl-go/badge.svg)](https://coveralls.io/github/package-url/packageurl-go) [![PkgGoDev](https://pkg.go.dev/badge/github.com/package-url/packageurl-go)](https://pkg.go.dev/github.com/package-url/packageurl-go) [![Go Report Card](https://goreportcard.com/badge/github.com/package-url/packageurl-go)](https://goreportcard.com/report/github.com/package-url/packageurl-go)
Go implementation of the package url spec.
## Install
```
go get -u github.com/package-url/packageurl-go
```
## Versioning
The versions will follow the spec. So if the spec is released at ``1.0``. Then all versions in the ``1.x.y`` will follow the ``1.x`` spec.
## Usage
### Create from parts
```go
package main
import (
"fmt"
"github.com/package-url/packageurl-go"
)
func main() {
instance := packageurl.NewPackageURL("test", "ok", "name", "version", nil, "")
fmt.Printf("%s", instance.ToString())
}
```
### Parse from string
```go
package main
import (
"fmt"
"github.com/package-url/packageurl-go"
)
func main() {
instance, err := packageurl.FromString("test:ok/name@version")
if err != nil {
panic(err)
}
fmt.Printf("%#v", instance)
}
```
## Test
Testing using the normal ``go test`` command. Using ``make test`` will pull the test fixtures shared between all package-url projects and then execute the tests.
```
$ make test
go test -v -cover ./...
=== RUN TestFromStringExamples
--- PASS: TestFromStringExamples (0.00s)
=== RUN TestToStringExamples
--- PASS: TestToStringExamples (0.00s)
=== RUN TestStringer
--- PASS: TestStringer (0.00s)
=== RUN TestQualifiersMapConversion
--- PASS: TestQualifiersMapConversion (0.00s)
PASS
coverage: 90.7% of statements
ok github.com/package-url/packageurl-go 0.004s coverage: 90.7% of statements
```
vendor/github.com/package-url/packageurl-go/VERSION
Generated Vendored
+1
View File
@@ -0,0 +1 @@
0.0.0
vendor/github.com/package-url/packageurl-go/packageurl.go
Generated Vendored
+438
View File
@@ -0,0 +1,438 @@
/*
Copyright (c) the purl authors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
// Package packageurl implements the package-url spec
package packageurl
import (
"errors"
"fmt"
"net/url"
"regexp"
"sort"
"strings"
)
var (
// QualifierKeyPattern describes a valid qualifier key:
//
// - The key must be composed only of ASCII letters and numbers, '.',
// '-' and '_' (period, dash and underscore).
// - A key cannot start with a number.
QualifierKeyPattern = regexp.MustCompile(`^[A-Za-z\.\-_][0-9A-Za-z\.\-_]*$`)
)
// These are the known purl types as defined in the spec. Some of these require
// special treatment during parsing.
// https://github.com/package-url/purl-spec#known-purl-types
var (
// TypeBitbucket is a pkg:bitbucket purl.
TypeBitbucket = "bitbucket"
// TypeCocoapods is a pkg:cocoapods purl.
TypeCocoapods = "cocoapods"
// TypeCargo is a pkg:cargo purl.
TypeCargo = "cargo"
// TypeComposer is a pkg:composer purl.
TypeComposer = "composer"
// TypeConan is a pkg:conan purl.
TypeConan = "conan"
// TypeConda is a pkg:conda purl.
TypeConda = "conda"
// TypeCran is a pkg:cran purl.
TypeCran = "cran"
// TypeDebian is a pkg:deb purl.
TypeDebian = "deb"
// TypeDocker is a pkg:docker purl.
TypeDocker = "docker"
// TypeGem is a pkg:gem purl.
TypeGem = "gem"
// TypeGeneric is a pkg:generic purl.
TypeGeneric = "generic"
// TypeGithub is a pkg:github purl.
TypeGithub = "github"
// TypeGolang is a pkg:golang purl.
TypeGolang = "golang"
// TypeHackage is a pkg:hackage purl.
TypeHackage = "hackage"
// TypeHex is a pkg:hex purl.
TypeHex = "hex"
// TypeMaven is a pkg:maven purl.
TypeMaven = "maven"
// TypeNPM is a pkg:npm purl.
TypeNPM = "npm"
// TypeNuget is a pkg:nuget purl.
TypeNuget = "nuget"
// TypeOCI is a pkg:oci purl
TypeOCI = "oci"
// TypePyPi is a pkg:pypi purl.
TypePyPi = "pypi"
// TypeRPM is a pkg:rpm purl.
TypeRPM = "rpm"
// TypeSwift is pkg:swift purl
TypeSwift = "swift"
// TypeHuggingface is pkg:huggingface purl.
TypeHuggingface = "huggingface"
// TypeMLflow is pkg:mlflow purl.
TypeMLFlow = "mlflow"
)
// Qualifier represents a single key=value qualifier in the package url
type Qualifier struct {
Key string
Value string
}
func (q Qualifier) String() string {
// A value must be a percent-encoded string
return fmt.Sprintf("%s=%s", q.Key, url.PathEscape(q.Value))
}
// Qualifiers is a slice of key=value pairs, with order preserved as it appears
// in the package URL.
type Qualifiers []Qualifier
// QualifiersFromMap constructs a Qualifiers slice from a string map. To get a
// deterministic qualifier order (despite maps not providing any iteration order
// guarantees) the returned Qualifiers are sorted in increasing order of key.
func QualifiersFromMap(mm map[string]string) Qualifiers {
q := Qualifiers{}
for k, v := range mm {
q = append(q, Qualifier{Key: k, Value: v})
}
// sort for deterministic qualifier order
sort.Slice(q, func(i int, j int) bool { return q[i].Key < q[j].Key })
return q
}
// Map converts a Qualifiers struct to a string map.
func (qq Qualifiers) Map() map[string]string {
m := make(map[string]string)
for i := 0; i < len(qq); i++ {
k := qq[i].Key
v := qq[i].Value
m[k] = v
}
return m
}
func (qq Qualifiers) String() string {
var kvPairs []string
for _, q := range qq {
kvPairs = append(kvPairs, q.String())
}
return strings.Join(kvPairs, "&")
}
// PackageURL is the struct representation of the parts that make a package url
type PackageURL struct {
Type string
Namespace string
Name string
Version string
Qualifiers Qualifiers
Subpath string
}
// NewPackageURL creates a new PackageURL struct instance based on input
func NewPackageURL(purlType, namespace, name, version string,
qualifiers Qualifiers, subpath string) *PackageURL {
return &PackageURL{
Type: purlType,
Namespace: namespace,
Name: name,
Version: version,
Qualifiers: qualifiers,
Subpath: subpath,
}
}
// ToString returns the human-readable instance of the PackageURL structure.
// This is the literal purl as defined by the spec.
func (p *PackageURL) ToString() string {
// Start with the type and a colon
purl := fmt.Sprintf("pkg:%s/", p.Type)
// Add namespaces if provided
if p.Namespace != "" {
var ns []string
for _, item := range strings.Split(p.Namespace, "/") {
ns = append(ns, url.QueryEscape(item))
}
purl = purl + strings.Join(ns, "/") + "/"
}
// The name is always required and must be a percent-encoded string
// Use url.QueryEscape instead of PathEscape, as it handles @ signs
purl = purl + url.QueryEscape(p.Name)
// If a version is provided, add it after the at symbol
if p.Version != "" {
// A name must be a percent-encoded string
purl = purl + "@" + url.PathEscape(p.Version)
}
// Iterate over qualifiers and make groups of key=value
var qualifiers []string
for _, q := range p.Qualifiers {
qualifiers = append(qualifiers, q.String())
}
// If there are one or more key=value pairs, append on the package url
if len(qualifiers) != 0 {
purl = purl + "?" + strings.Join(qualifiers, "&")
}
// Add a subpath if available
if p.Subpath != "" {
purl = purl + "#" + p.Subpath
}
return purl
}
func (p PackageURL) String() string {
return p.ToString()
}
// FromString parses a valid package url string into a PackageURL structure
func FromString(purl string) (PackageURL, error) {
initialIndex := strings.Index(purl, "#")
// Start with purl being stored in the remainder
remainder := purl
substring := ""
if initialIndex != -1 {
initialSplit := strings.SplitN(purl, "#", 2)
remainder = initialSplit[0]
rightSide := initialSplit[1]
rightSide = strings.TrimLeft(rightSide, "/")
rightSide = strings.TrimRight(rightSide, "/")
var rightSides []string
for _, item := range strings.Split(rightSide, "/") {
item = strings.Replace(item, ".", "", -1)
item = strings.Replace(item, "..", "", -1)
if item != "" {
i, err := url.PathUnescape(item)
if err != nil {
return PackageURL{}, fmt.Errorf("failed to unescape path: %s", err)
}
rightSides = append(rightSides, i)
}
}
substring = strings.Join(rightSides, "/")
}
qualifiers := Qualifiers{}
index := strings.LastIndex(remainder, "?")
// If we don't have anything to split then return an empty result
if index != -1 {
qualifier := remainder[index+1:]
for _, item := range strings.Split(qualifier, "&") {
kv := strings.Split(item, "=")
key := strings.ToLower(kv[0])
key, err := url.PathUnescape(key)
if err != nil {
return PackageURL{}, fmt.Errorf("failed to unescape qualifier key: %s", err)
}
if !validQualifierKey(key) {
return PackageURL{}, fmt.Errorf("invalid qualifier key: '%s'", key)
}
// TODO
// - If the `key` is `checksums`, split the `value` on ',' to create
// a list of `checksums`
if kv[1] == "" {
continue
}
value, err := url.PathUnescape(kv[1])
if err != nil {
return PackageURL{}, fmt.Errorf("failed to unescape qualifier value: %s", err)
}
qualifiers = append(qualifiers, Qualifier{key, value})
}
remainder = remainder[:index]
}
nextSplit := strings.SplitN(remainder, ":", 2)
if len(nextSplit) != 2 || nextSplit[0] != "pkg" {
return PackageURL{}, errors.New("scheme is missing")
}
// leading slashes after pkg: are to be ignored (pkg://maven is
// equivalent to pkg:maven)
remainder = strings.TrimLeft(nextSplit[1], "/")
nextSplit = strings.SplitN(remainder, "/", 2)
if len(nextSplit) != 2 {
return PackageURL{}, errors.New("type is missing")
}
// purl type is case-insensitive, canonical form is lower-case
purlType := strings.ToLower(nextSplit[0])
remainder = nextSplit[1]
index = strings.LastIndex(remainder, "/")
name := typeAdjustName(purlType, remainder[index+1:], qualifiers)
version := ""
atIndex := strings.Index(name, "@")
if atIndex != -1 {
v, err := url.PathUnescape(name[atIndex+1:])
if err != nil {
return PackageURL{}, fmt.Errorf("failed to unescape purl version: %s", err)
}
version = typeAdjustVersion(purlType, v)
unecapeName, err := url.PathUnescape(name[:atIndex])
if err != nil {
return PackageURL{}, fmt.Errorf("failed to unescape purl name: %s", err)
}
name = unecapeName
}
var namespaces []string
if index != -1 {
remainder = remainder[:index]
for _, item := range strings.Split(remainder, "/") {
if item != "" {
unescaped, err := url.PathUnescape(item)
if err != nil {
return PackageURL{}, fmt.Errorf("failed to unescape path: %s", err)
}
namespaces = append(namespaces, unescaped)
}
}
}
namespace := strings.Join(namespaces, "/")
namespace = typeAdjustNamespace(purlType, namespace)
// Fail if name is empty at this point
if name == "" {
return PackageURL{}, errors.New("name is required")
}
err := validCustomRules(purlType, name, namespace, version, qualifiers)
if err != nil {
return PackageURL{}, err
}
return PackageURL{
Type: purlType,
Namespace: namespace,
Name: name,
Version: version,
Qualifiers: qualifiers,
Subpath: substring,
}, nil
}
// Make any purl type-specific adjustments to the parsed namespace.
// See https://github.com/package-url/purl-spec#known-purl-types
func typeAdjustNamespace(purlType, ns string) string {
switch purlType {
case TypeBitbucket, TypeDebian, TypeGithub, TypeGolang, TypeNPM, TypeRPM, TypeComposer:
return strings.ToLower(ns)
}
return ns
}
// Make any purl type-specific adjustments to the parsed name.
// See https://github.com/package-url/purl-spec#known-purl-types
func typeAdjustName(purlType, name string, qualifiers Qualifiers) string {
quals := qualifiers.Map()
switch purlType {
case TypeBitbucket, TypeDebian, TypeGithub, TypeGolang, TypeNPM, TypeComposer:
return strings.ToLower(name)
case TypePyPi:
return strings.ToLower(strings.ReplaceAll(name, "_", "-"))
case TypeMLFlow:
return adjustMlflowName(name, quals)
}
return name
}
// Make any purl type-specific adjustments to the parsed version.
// See https://github.com/package-url/purl-spec#known-purl-types
func typeAdjustVersion(purlType, version string) string {
switch purlType {
case TypeHuggingface:
return strings.ToLower(version)
}
return version
}
// https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst#mlflow
func adjustMlflowName(name string, qualifiers map[string]string) string {
if repo, ok := qualifiers["repository_url"]; ok {
if strings.Contains(repo, "azureml") {
// Azure ML is case-sensitive and must be kept as-is
return name
} else if strings.Contains(repo, "databricks") {
// Databricks is case-insensitive and must be lowercased
return strings.ToLower(name)
} else {
// Unknown repository type, keep as-is
return name
}
} else {
// No repository qualifier given, keep as-is
return name
}
}
// validQualifierKey validates a qualifierKey against our QualifierKeyPattern.
func validQualifierKey(key string) bool {
return QualifierKeyPattern.MatchString(key)
}
// validCustomRules evaluates additional rules for each package url type, as specified in the package-url specification.
// On success, it returns nil. On failure, a descriptive error will be returned.
func validCustomRules(purlType, name, ns, version string, qualifiers Qualifiers) error {
q := qualifiers.Map()
switch purlType {
case TypeConan:
if ns != "" {
if val, ok := q["channel"]; ok {
if val == "" {
return errors.New("the qualifier channel must be not empty if namespace is present")
}
} else {
return errors.New("channel qualifier does not exist")
}
} else {
if val, ok := q["channel"]; ok {
if val != "" {
return errors.New("namespace is required if channel is non empty")
}
}
}
case TypeSwift:
if ns == "" {
return errors.New("namespace is required")
}
if version == "" {
return errors.New("version is required")
}
case TypeCran:
if version == "" {
return errors.New("version is required")
}
}
return nil
}
vendor/modules.txt
Vendored
+4
View File
@@ -730,6 +730,7 @@ github.com/moby/buildkit/util/pgpsign
github.com/moby/buildkit/util/progress
github.com/moby/buildkit/util/progress/progressui
github.com/moby/buildkit/util/progress/progresswriter
github.com/moby/buildkit/util/purl
github.com/moby/buildkit/util/resolver/config
github.com/moby/buildkit/util/resolver/limited
github.com/moby/buildkit/util/resolver/retryhandler
@@ -930,6 +931,9 @@ github.com/opencontainers/go-digest
## explicit; go 1.18
github.com/opencontainers/image-spec/specs-go
github.com/opencontainers/image-spec/specs-go/v1
# github.com/package-url/packageurl-go v0.1.1
## explicit; go 1.17
github.com/package-url/packageurl-go
# github.com/pelletier/go-toml/v2 v2.2.4
## explicit; go 1.21.0
github.com/pelletier/go-toml/v2