policy: add progress vertex error integration test

Refactor policy error unit tests to table-driven subtests with slug names.
Add rawjson integration coverage to verify policy vertex captures DENY build
errors in progress output.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

policy/policy_error_test.go

@@ -10,47 +10,42 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestPolicyIsPolicyErrorMatchesRecordedSource(t *testing.T) {
-	p := NewPolicy(Opt{})
-	req := &policysession.CheckPolicyRequest{
-		Source: &gwpb.ResolveSourceMetaResponse{
-			Source: &solverpb.SourceOp{
-				Identifier: "docker-image://busybox:latest",
-			},
+func TestPolicyIsPolicyError(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{
+			name: "matches-recorded-source",
+			err:  errors.New("failed to solve: error evaluating the source policy: source \"docker-image://busybox:latest\" not allowed by policy: action DENY"),
+			want: true,
+		},
+		{
+			name: "does-not-match-without-buildkit-pattern",
+			err:  errors.New("failed to parse dockerfile for docker-image://busybox:latest"),
+			want: false,
+		},
+		{
+			name: "does-not-match-unrelated-error",
+			err:  errors.New("failed to solve: error evaluating the source policy: source \"docker-image://alpine:latest\" not allowed by policy: action DENY"),
+			want: false,
 		},
 	}
-	p.recordDenyIdentifier(req)
 
-	err := errors.New("failed to solve: error evaluating the source policy: source \"docker-image://busybox:latest\" not allowed by policy: action DENY")
-	require.True(t, p.IsPolicyError(err))
-}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p := NewPolicy(Opt{})
+			req := &policysession.CheckPolicyRequest{
+				Source: &gwpb.ResolveSourceMetaResponse{
+					Source: &solverpb.SourceOp{
+						Identifier: "docker-image://busybox:latest",
+					},
+				},
+			}
+			p.recordDenyIdentifier(req)
 
-func TestPolicyIsPolicyErrorDoesNotMatchWithoutBuildkitPattern(t *testing.T) {
-	p := NewPolicy(Opt{})
-	req := &policysession.CheckPolicyRequest{
-		Source: &gwpb.ResolveSourceMetaResponse{
-			Source: &solverpb.SourceOp{
-				Identifier: "docker-image://busybox:latest",
-			},
-		},
+			require.Equal(t, tt.want, p.IsPolicyError(tt.err))
+		})
 	}
-	p.recordDenyIdentifier(req)
-
-	err := errors.New("failed to parse dockerfile for docker-image://busybox:latest")
-	require.False(t, p.IsPolicyError(err))
-}
-
-func TestPolicyIsPolicyErrorDoesNotMatchUnrelatedError(t *testing.T) {
-	p := NewPolicy(Opt{})
-	req := &policysession.CheckPolicyRequest{
-		Source: &gwpb.ResolveSourceMetaResponse{
-			Source: &solverpb.SourceOp{
-				Identifier: "docker-image://busybox:latest",
-			},
-		},
-	}
-	p.recordDenyIdentifier(req)
-
-	err := errors.New("failed to solve: error evaluating the source policy: source \"docker-image://alpine:latest\" not allowed by policy: action DENY")
-	require.False(t, p.IsPolicyError(err))
 }

tests/policy_build.go

@@ -1,11 +1,13 @@
 package tests
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/containerd/continuity/fs/fstest"
@@ -13,6 +15,7 @@ import (
 	"github.com/distribution/reference"
 	"github.com/docker/buildx/util/gitutil"
 	"github.com/docker/buildx/util/gitutil/gittestutil"
+	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/util/contentutil"
 	"github.com/moby/buildkit/util/testutil"
@@ -25,6 +28,7 @@ import (
 var policyBuildTests = []func(t *testing.T, sb integration.Sandbox){
 	testBuildPolicyAllow,
 	testBuildPolicyDeny,
+	testBuildPolicyDenyProgressStream,
 	testBuildPolicyImageName,
 	testBuildPolicyEnv,
 	testBuildPolicyHTTP,
@@ -103,6 +107,80 @@ decision := {"allow": allow, "deny_msg": deny_msg}
 	require.Contains(t, string(out), "DENY")
 }
 
+func testBuildPolicyDenyProgressStream(t *testing.T, sb integration.Sandbox) {
+	skipNoCompatBuildKit(t, sb, ">= 0.26.0-0", "policy input requires BuildKit v0.26.0+")
+	dockerfile := []byte(`
+FROM busybox:latest
+RUN echo policy-nope
+`)
+	policyFile := []byte(`
+package docker
+
+default allow = false
+
+allow if not input.image
+
+decision := {"allow": allow}
+`)
+	dir := tmpdir(
+		t,
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+		fstest.CreateFile("policy.rego", policyFile, 0600),
+	)
+	policyPath := filepath.Join(dir, "policy.rego")
+
+	cmd := buildxCmd(sb, withDir(dir), withArgs(
+		"build",
+		"--progress=rawjson",
+		"--policy", "filename="+policyPath,
+		"--output=type=cacheonly",
+		dir,
+	))
+	out, err := cmd.CombinedOutput()
+	outStr := string(out)
+	require.Error(t, err, outStr)
+
+	policyVertexName := "loading policies " + policyPath
+	sawPolicyVertex := false
+	sawPolicyCompleted := false
+	policyVertexDigest := ""
+	policyVertexError := ""
+
+	for line := range strings.SplitSeq(outStr, "\n") {
+		line = strings.TrimSpace(line)
+		if !strings.HasPrefix(line, "{") {
+			continue
+		}
+		var st client.SolveStatus
+		require.NoError(t, json.Unmarshal([]byte(line), &st), line)
+		for _, v := range st.Vertexes {
+			if v == nil {
+				continue
+			}
+			if v.Name == policyVertexName {
+				sawPolicyVertex = true
+				if v.Digest != "" {
+					policyVertexDigest = string(v.Digest)
+				}
+			}
+			if policyVertexDigest == "" || string(v.Digest) != policyVertexDigest {
+				continue
+			}
+			if v.Completed != nil {
+				sawPolicyCompleted = true
+			}
+			if v.Error != "" {
+				policyVertexError = v.Error
+			}
+		}
+	}
+
+	require.True(t, sawPolicyVertex, outStr)
+	require.True(t, sawPolicyCompleted, outStr)
+	require.Contains(t, policyVertexError, "not allowed by policy", outStr)
+	require.Contains(t, outStr, "not allowed by policy")
+}
+
 func testBuildPolicyImageName(t *testing.T, sb integration.Sandbox) {
 	skipNoCompatBuildKit(t, sb, ">= 0.26.0-0", "policy input requires BuildKit v0.26.0+")
 	registry, err := sb.NewRegistry()