policy: simplify recursive material resolution

Unify root/material unknown resolution with recursive Input traversal.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

commands/policy/eval.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/fs"
+	"maps"
 	"os"
 	"path/filepath"
 	"slices"
@@ -17,7 +18,6 @@ import (
 	"github.com/docker/buildx/util/confutil"
 	"github.com/docker/buildx/util/sourcemeta"
 	"github.com/docker/cli/cli/command"
-	"github.com/moby/buildkit/client/llb/sourceresolver"
 	"github.com/moby/buildkit/frontend/dockerui"
 	gwpb "github.com/moby/buildkit/frontend/gateway/pb"
 	"github.com/moby/buildkit/solver/pb"
@@ -27,7 +27,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 type evalOpts struct {
@@ -111,48 +110,50 @@ func runEval(ctx context.Context, dockerCli command.Cli, source string, opts eva
 		srcReq := &gwpb.ResolveSourceMetaResponse{
 			Source: src,
 		}
+		input, err := policy.SourceToInput(ctx, verifier, srcReq, &p, nil)
+		if err != nil {
+			return err
+		}
 		maxAttempts := 5
-		var unknowns []string
 		var lastUnknowns []string
 		var trimmedUnknowns []string
-		var input policy.Input
-		var doneInvalidCheck bool
 		var invalidFields []string
 		for {
 			maxAttempts--
 			if maxAttempts <= 0 {
 				return errors.New("maximum attempts reached for resolving source metadata")
 			}
-			input, unknowns, err = policy.SourceToInput(ctx, verifier, srcReq, &p)
-			if err != nil {
-				return err
+			unknowns := input.Unknowns()
+			trimmedUnknowns = make([]string, 0, len(unknowns))
+			for _, u := range unknowns {
+				trimmedUnknowns = append(trimmedUnknowns, strings.TrimPrefix(u, "input."))
 			}
-			trimmedUnknowns = trimInputPrefixSlice(unknowns)
 			if lastUnknowns != nil && slices.Equal(trimmedUnknowns, lastUnknowns) {
 				break
 			}
 			lastUnknowns = slices.Clone(trimmedUnknowns)
-			toReload := []string{}
-			for _, f := range opts.fields {
-				if slices.Contains(trimmedUnknowns, f) {
-					toReload = append(toReload, f)
-				} else if !doneInvalidCheck {
-					invalidFields = append(invalidFields, f)
-				}
-			}
-			doneInvalidCheck = true
+			toReload, invalid := selectReloadFields(opts.fields, trimmedUnknowns)
+			invalidFields = invalid
 			if len(toReload) > 0 {
-				req := &gwpb.ResolveSourceMetaRequest{}
-				if err := policy.AddUnknowns(req, toReload); err != nil {
-					return err
-				}
-				opt := sourceResolverOpt(req, &p)
-				resp, err := metaResolver.ResolveSourceMetadata(ctx, src, opt)
+				retry, next, err := policy.ResolveInputUnknowns(ctx, &input, srcReq.Source, toReload, platform, &p, metaResolver, verifier, nil)
 				if err != nil {
 					return err
 				}
-				srcReq = buildSourceMetaResponse(resp)
-				continue
+				if next != nil {
+					resp, err := metaResolver.ResolveSourceMetadata(ctx, next.Source, sourcemeta.ToResolverOpt(next, &p))
+					if err != nil {
+						return err
+					}
+					srcReq = sourcemeta.ToGatewayMetaResponse(resp)
+					input, err = policy.SourceToInput(ctx, verifier, srcReq, &p, nil)
+					if err != nil {
+						return err
+					}
+					continue
+				}
+				if retry {
+					continue
+				}
 			}
 			break
 		}
@@ -160,11 +161,15 @@ func runEval(ctx context.Context, dockerCli command.Cli, source string, opts eva
 		if len(invalidFields) > 0 {
 			logrus.Warnf("invalid fields: %v", strings.Join(invalidFields, ", "))
 		}
-		if len(trimmedUnknowns) > 0 {
-			logrus.Infof("unresolved fields: %v", strings.Join(trimmedUnknowns, ", "))
+		reportedUnknowns := summarizeEvalUnknowns(trimmedUnknowns, opts.fields)
+		if len(reportedUnknowns) > 0 {
+			logrus.Infof("unresolved fields: %v", strings.Join(reportedUnknowns, ", "))
 		}
 
-		dt, err := json.MarshalIndent(input, "", "  ")
+		printInput := input
+		sanitizePrintInput(&printInput)
+
+		dt, err := json.MarshalIndent(printInput, "", "  ")
 		if err != nil {
 			return errors.Wrap(err, "failed to marshal policy input")
 		}
@@ -198,6 +203,9 @@ func runEval(ctx context.Context, dockerCli command.Cli, source string, opts eva
 	env := policy.Env{
 		Filename: filepath.Base(policyName),
 	}
+	policyLog := func(_ logrus.Level, msg string) {
+		logrus.Debug(msg)
+	}
 
 	policyEval := policy.NewPolicy(policy.Opt{
 		Files: []policy.File{
@@ -207,9 +215,11 @@ func runEval(ctx context.Context, dockerCli command.Cli, source string, opts eva
 			},
 		},
 		Env:              env,
+		Log:              policyLog,
 		FS:               fsProvider,
 		VerifierProvider: verifier,
 		DefaultPlatform:  &p,
+		SourceResolver:   metaResolver,
 	})
 
 	srcReq := &gwpb.ResolveSourceMetaResponse{
@@ -233,114 +243,153 @@ func runEval(ctx context.Context, dockerCli command.Cli, source string, opts eva
 			return evalDecisionError(decision)
 		}
 
-		opt := sourceResolverOpt(next, &p)
-		resp, err := metaResolver.ResolveSourceMetadata(ctx, src, opt)
+		opt := sourcemeta.ToResolverOpt(next, &p)
+		target := src
+		if next.Source != nil {
+			target = next.Source
+		}
+		resp, err := metaResolver.ResolveSourceMetadata(ctx, target, opt)
 		if err != nil {
 			return err
 		}
-		srcReq = buildSourceMetaResponse(resp)
+		srcReq = sourcemeta.ToGatewayMetaResponse(resp)
 	}
 }
 
-func toGatewayDescriptor(desc ocispecs.Descriptor) *gwpb.Descriptor {
-	return &gwpb.Descriptor{
-		MediaType:   desc.MediaType,
-		Digest:      desc.Digest.String(),
-		Size:        desc.Size,
-		Annotations: desc.Annotations,
+func selectReloadFields(fields []string, unknowns []string) ([]string, []string) {
+	if len(fields) == 0 {
+		return nil, nil
 	}
+	reload := map[string]struct{}{}
+	var invalid []string
+	for _, field := range fields {
+		if prereq, ok := materialFieldPrerequisites(field); ok {
+			added := false
+			for _, p := range prereq {
+				if slices.Contains(unknowns, p) {
+					reload[p] = struct{}{}
+					added = true
+				}
+			}
+			if slices.Contains(unknowns, field) {
+				reload[field] = struct{}{}
+				added = true
+			} else if ancestor := findUnknownAncestor(field, unknowns); ancestor != "" {
+				reload[ancestor] = struct{}{}
+				added = true
+			}
+			if !added {
+				invalid = append(invalid, field)
+			}
+			continue
+		}
+		if slices.Contains(unknowns, field) {
+			reload[field] = struct{}{}
+			continue
+		}
+		invalid = append(invalid, field)
+	}
+	return slices.Collect(maps.Keys(reload)), invalid
 }
 
-func toGatewayAttestationChain(chain *sourceresolver.AttestationChain) *gwpb.AttestationChain {
-	if chain == nil {
+func findUnknownAncestor(field string, unknowns []string) string {
+	var best string
+	for _, unknown := range unknowns {
+		if field == unknown {
+			return unknown
+		}
+		if strings.HasPrefix(field, unknown+".") {
+			if len(unknown) > len(best) {
+				best = unknown
+			}
+			continue
+		}
+		if strings.HasPrefix(field, unknown+"[") {
+			if len(unknown) > len(best) {
+				best = unknown
+			}
+		}
+	}
+	return best
+}
+
+func materialFieldPrerequisites(field string) ([]string, bool) {
+	const seg = ".image.provenance.materials["
+	if !strings.HasPrefix(field, seg[1:]) {
+		return nil, false
+	}
+	provenancePath := strings.TrimSuffix(seg, ".materials[")
+	out := map[string]struct{}{strings.TrimPrefix(provenancePath, "."): {}}
+	collectMaterialPrerequisites(field, seg, provenancePath, 0, out)
+	keys := slices.Collect(maps.Keys(out))
+	slices.Sort(keys)
+	return keys, true
+}
+
+func collectMaterialPrerequisites(field, seg, provenancePath string, start int, out map[string]struct{}) {
+	i := strings.Index(field[start:], seg)
+	if i < 0 {
+		return
+	}
+	i += start
+	out[field[:i]+provenancePath] = struct{}{}
+	collectMaterialPrerequisites(field, seg, provenancePath, i+len(seg), out)
+}
+
+func summarizeEvalUnknowns(unknowns, requested []string) []string {
+	if len(unknowns) == 0 {
 		return nil
 	}
-	signatures := make([]string, 0, len(chain.SignatureManifests))
-	for _, dgst := range chain.SignatureManifests {
-		signatures = append(signatures, dgst.String())
-	}
-	blobs := make(map[string]*gwpb.Blob, len(chain.Blobs))
-	for dgst, blob := range chain.Blobs {
-		blobs[dgst.String()] = &gwpb.Blob{
-			Descriptor_: toGatewayDescriptor(blob.Descriptor),
-			Data:        blob.Data,
+	if len(requested) > 0 {
+		out := map[string]struct{}{}
+		for _, field := range requested {
+			if slices.Contains(unknowns, field) {
+				out[field] = struct{}{}
+				continue
+			}
+			if ancestor := findUnknownAncestor(field, unknowns); ancestor != "" {
+				out[ancestor] = struct{}{}
+			}
 		}
+		keys := slices.Collect(maps.Keys(out))
+		slices.Sort(keys)
+		return keys
 	}
-	return &gwpb.AttestationChain{
-		Root:                chain.Root.String(),
-		ImageManifest:       chain.ImageManifest.String(),
-		AttestationManifest: chain.AttestationManifest.String(),
-		SignatureManifests:  signatures,
-		Blobs:               blobs,
+
+	out := map[string]struct{}{}
+	for _, u := range unknowns {
+		out[summarizeUnknownField(u)] = struct{}{}
 	}
+	keys := slices.Collect(maps.Keys(out))
+	slices.Sort(keys)
+	return keys
 }
 
-func sourceResolverOpt(req *gwpb.ResolveSourceMetaRequest, platform *ocispecs.Platform) sourceresolver.Opt {
-	opt := sourceresolver.Opt{
-		LogName:        req.LogName,
-		SourcePolicies: req.SourcePolicies,
+func summarizeUnknownField(field string) string {
+	if base, _, ok := strings.Cut(field, ".materials["); ok {
+		return base + ".materials"
 	}
-	if req.Image != nil {
-		opt.ImageOpt = &sourceresolver.ResolveImageOpt{
-			NoConfig:            req.Image.NoConfig,
-			AttestationChain:    req.Image.AttestationChain,
-			ResolveAttestations: slices.Clone(req.Image.ResolveAttestations),
-			Platform:            platform,
-			ResolveMode:         req.ResolveMode,
-		}
+	if strings.HasPrefix(field, "materials[") {
+		return "materials"
 	}
-	if req.Git != nil {
-		opt.GitOpt = &sourceresolver.ResolveGitOpt{
-			ReturnObject: req.Git.ReturnObject,
-		}
+	parts := strings.Split(field, ".")
+	if len(parts) > 1 {
+		return strings.Join(parts[:2], ".")
 	}
-	return opt
+	return field
 }
 
-func buildSourceMetaResponse(resp *sourceresolver.MetaResponse) *gwpb.ResolveSourceMetaResponse {
-	out := &gwpb.ResolveSourceMetaResponse{
-		Source: resp.Op,
+func sanitizePrintInput(inp *policy.Input) {
+	if inp == nil {
+		return
 	}
-	if resp.Image != nil {
-		chain := toGatewayAttestationChain(resp.Image.AttestationChain)
-		out.Image = &gwpb.ResolveSourceImageResponse{
-			Digest:           resp.Image.Digest.String(),
-			Config:           resp.Image.Config,
-			AttestationChain: chain,
-		}
+	inp.Env.Depth = 0
+	if inp.Image == nil || inp.Image.Provenance == nil || len(inp.Image.Provenance.Materials) == 0 {
+		return
 	}
-	if resp.Git != nil {
-		out.Git = &gwpb.ResolveSourceGitResponse{
-			Checksum:       resp.Git.Checksum,
-			Ref:            resp.Git.Ref,
-			CommitChecksum: resp.Git.CommitChecksum,
-			CommitObject:   resp.Git.CommitObject,
-			TagObject:      resp.Git.TagObject,
-		}
+	for i := range inp.Image.Provenance.Materials {
+		sanitizePrintInput(&inp.Image.Provenance.Materials[i])
 	}
-	if resp.HTTP != nil {
-		var lastModified *timestamppb.Timestamp
-		if resp.HTTP.LastModified != nil {
-			lastModified = timestamppb.New(*resp.HTTP.LastModified)
-		}
-		out.HTTP = &gwpb.ResolveSourceHTTPResponse{
-			Checksum:     resp.HTTP.Digest.String(),
-			Filename:     resp.HTTP.Filename,
-			LastModified: lastModified,
-		}
-	}
-	return out
-}
-
-func trimInputPrefixSlice(fields []string) []string {
-	if len(fields) == 0 {
-		return fields
-	}
-	out := make([]string, 0, len(fields))
-	for _, field := range fields {
-		out = append(out, strings.TrimPrefix(field, "input."))
-	}
-	return out
 }
 
 func evalDecisionError(decision *policysession.DecisionResponse) error {

commands/policy/eval_test.go

@@ -3,6 +3,8 @@ package policy
 import (
 	"testing"
 
+	policytypes "github.com/docker/buildx/policy"
+	"github.com/docker/buildx/util/sourcemeta"
 	gwpb "github.com/moby/buildkit/frontend/gateway/pb"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/stretchr/testify/require"
@@ -18,10 +20,132 @@ func TestSourceResolverOptIncludesResolveAttestations(t *testing.T) {
 	}
 	platform := &ocispecs.Platform{OS: "linux", Architecture: "amd64"}
 
-	opt := sourceResolverOpt(req, platform)
+	opt := sourcemeta.ToResolverOpt(req, platform)
 	require.NotNil(t, opt.ImageOpt)
 	require.True(t, opt.ImageOpt.NoConfig)
 	require.Equal(t, []string{"https://slsa.dev/provenance/v0.2"}, opt.ImageOpt.ResolveAttestations)
 	require.Equal(t, "default", opt.ImageOpt.ResolveMode)
 	require.Equal(t, platform, opt.ImageOpt.Platform)
 }
+
+func TestSanitizePrintInputClearsDepthRecursively(t *testing.T) {
+	inp := policytypes.Input{
+		Env: policytypes.Env{Depth: 7, Filename: "Dockerfile"},
+		Image: &policytypes.Image{
+			Provenance: &policytypes.ImageProvenance{
+				Materials: []policytypes.Input{
+					{
+						Env: policytypes.Env{Depth: 3, Target: "app"},
+						Image: &policytypes.Image{
+							Provenance: &policytypes.ImageProvenance{
+								Materials: []policytypes.Input{
+									{Env: policytypes.Env{Depth: 2}},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	sanitizePrintInput(&inp)
+
+	require.Zero(t, inp.Env.Depth)
+	require.Equal(t, "Dockerfile", inp.Env.Filename)
+	require.Zero(t, inp.Image.Provenance.Materials[0].Env.Depth)
+	require.Equal(t, "app", inp.Image.Provenance.Materials[0].Env.Target)
+	require.Zero(t, inp.Image.Provenance.Materials[0].Image.Provenance.Materials[0].Env.Depth)
+}
+
+func TestSelectReloadFields(t *testing.T) {
+	unknowns := []string{
+		"image.provenance",
+		"image.provenance.materials[0].image.hasProvenance",
+		"git.tag",
+	}
+
+	t.Run("exact match", func(t *testing.T) {
+		reload, invalid := selectReloadFields([]string{"git.tag"}, unknowns)
+		require.Equal(t, []string{"git.tag"}, reload)
+		require.Nil(t, invalid)
+	})
+
+	t.Run("ancestor mapping", func(t *testing.T) {
+		reload, invalid := selectReloadFields([]string{"image.provenance.materials[0].image.labels"}, unknowns)
+		require.ElementsMatch(t, []string{"image.provenance"}, reload)
+		require.Nil(t, invalid)
+	})
+
+	t.Run("dedupe mapped reloads", func(t *testing.T) {
+		reload, invalid := selectReloadFields([]string{
+			"image.provenance.materials[0].image.labels",
+			"image.provenance.materials[0].image.user",
+		}, unknowns)
+		require.ElementsMatch(t, []string{
+			"image.provenance",
+		}, reload)
+		require.Nil(t, invalid)
+	})
+
+	t.Run("invalid fields reported", func(t *testing.T) {
+		reload, invalid := selectReloadFields([]string{"image.labels", "foo.bar"}, unknowns)
+		require.Empty(t, reload)
+		require.Equal(t, []string{"image.labels", "foo.bar"}, invalid)
+	})
+
+	t.Run("mix exact mapped invalid", func(t *testing.T) {
+		reload, invalid := selectReloadFields([]string{
+			"git.tag",
+			"image.provenance.materials[0].image.env",
+			"no.such.field",
+		}, unknowns)
+		require.ElementsMatch(t, []string{"git.tag", "image.provenance"}, reload)
+		require.Equal(t, []string{"no.such.field"}, invalid)
+	})
+
+	t.Run("nested material prerequisites", func(t *testing.T) {
+		reload, invalid := selectReloadFields([]string{
+			"image.provenance.materials[0].image.provenance.materials[1].image.labels",
+		}, unknowns)
+		require.ElementsMatch(t, []string{
+			"image.provenance",
+		}, reload)
+		require.Nil(t, invalid)
+	})
+
+	t.Run("material field after provenance loaded", func(t *testing.T) {
+		reload, invalid := selectReloadFields([]string{
+			"image.provenance.materials[0].image.hasProvenance",
+		}, []string{
+			"image.provenance.materials[0].image.hasProvenance",
+		})
+		require.ElementsMatch(t, []string{
+			"image.provenance.materials[0].image.hasProvenance",
+		}, reload)
+		require.Nil(t, invalid)
+	})
+}
+
+func TestMaterialFieldPrerequisites(t *testing.T) {
+	t.Run("non material field", func(t *testing.T) {
+		prereq, ok := materialFieldPrerequisites("image.provenance")
+		require.False(t, ok)
+		require.Nil(t, prereq)
+	})
+
+	t.Run("single level material field", func(t *testing.T) {
+		prereq, ok := materialFieldPrerequisites("image.provenance.materials[0].image.labels")
+		require.True(t, ok)
+		require.Equal(t, []string{"image.provenance"}, prereq)
+	})
+
+	t.Run("nested material field", func(t *testing.T) {
+		prereq, ok := materialFieldPrerequisites("image.provenance.materials[0].image.provenance.materials[1].image.labels")
+		require.True(t, ok)
+		require.Equal(t, []string{
+			"image.provenance",
+			"image.provenance.materials[0].image.provenance",
+		}, prereq)
+	})
+}

commands/policy/test.go

@@ -158,12 +158,12 @@ func (r *policyTestOptionsProvider) Resolve(ctx context.Context, source *pb.Sour
 	if err := r.init(ctx); err != nil {
 		return nil, err
 	}
-	opt := sourceResolverOpt(req, r.platform)
+	opt := sourcemeta.ToResolverOpt(req, r.platform)
 	resp, err := r.metaResolver.ResolveSourceMetadata(ctx, source, opt)
 	if err != nil {
 		return nil, err
 	}
-	return buildSourceMetaResponse(resp), nil
+	return sourcemeta.ToGatewayMetaResponse(resp), nil
 }
 
 func (r *policyTestOptionsProvider) init(ctx context.Context) error {