tests: add remote multi-node worker

Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>

.github/workflows/build.yml

@@ -66,13 +66,18 @@ jobs:
           - ""
           - experimental
         include:
-          - worker: docker
+          - worker: remote+multinode
             pkg: ./tests
-          - worker: docker+containerd  # same as docker, but with containerd snapshotter
+          - worker: remote+multinode
+            pkg: ./tests
+            mode: experimental
+          - worker: docker
             pkg: ./tests
           - worker: docker
             pkg: ./tests
             mode: experimental
+          - worker: docker+containerd  # same as docker, but with containerd snapshotter
+            pkg: ./tests
           - worker: docker+containerd  # same as docker, but with containerd snapshotter
             pkg: ./tests
             mode: experimental
@@ -106,14 +111,14 @@ jobs:
           fi
           testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$"
           case "${{ matrix.worker }}" in
-            docker | docker+containerd | docker@* | docker+containerd@*)
+            docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode)
               echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV
               ;;
             *)
               echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV
               ;;
           esac
-          if [[ "${{ matrix.worker }}" == "docker"* ]]; then
+          if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" ]]; then
             echo "TEST_DOCKERD=1" >> $GITHUB_ENV
           fi
           if [ "${{ matrix.mode }}" = "experimental" ]; then

tests/integration.go

@@ -141,6 +141,15 @@ func isDockerContainerWorker(sb integration.Sandbox) bool {
 	return name == "docker-container"
 }
 
+func isRemoteWorker(sb integration.Sandbox) bool {
+	name, _, _ := driverName(sb.Name())
+	return name == "remote"
+}
+
+func isRemoteMultiNodeWorker(sb integration.Sandbox) bool {
+	return sb.Name() == "remote+multinode"
+}
+
 func driverName(sbName string) (string, bool, bool) {
 	name := sbName
 	var hasVersion, hasFeature bool

tests/integration_test.go

@@ -14,6 +14,7 @@ func init() {
 	if bkworkers.IsTestDockerd() {
 		workers.InitDockerWorker()
 		workers.InitDockerContainerWorker()
+		workers.InitRemoteMultiNodeWorker()
 	} else {
 		workers.InitRemoteWorker()
 	}

tests/policy_build.go

@@ -779,8 +779,7 @@ decision := {"allow": allow}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			if tc.requiresHTTPChecksum {
-				sbDriver, _, _ := driverName(sb.Name())
+				if !isRemoteWorker(sb) {
-				if sbDriver != "remote" {
 					t.Skip("http checksum policy input requires remote driver")
 				}
 				skipNoCompatBuildKit(t, sb, ">= 0.26.3-0", "http checksum policy input")
@@ -1084,8 +1083,7 @@ decision := {"allow": allow}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			if tc.requiresGitResolve {
-				sbDriver, _, _ := driverName(sb.Name())
+				if !isRemoteWorker(sb) {
-				if sbDriver != "remote" {
 					t.Skip("git policy metadata requires remote driver")
 				}
 			}

tests/policy_eval.go

@@ -492,8 +492,7 @@ decision := {"allow": allow}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			if tc.needsChecksum {
-				sbDriver, _, _ := driverName(sb.Name())
+				if !isRemoteWorker(sb) {
-				if sbDriver != "remote" {
 					t.Skip("http checksum policy eval requires remote driver")
 				}
 				skipNoCompatBuildKit(t, sb, ">= 0.26.3-0", "http checksum policy input")

tests/workers/remote-multinode.go

@@ -0,0 +1,155 @@
+package workers
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+
+	"github.com/docker/buildx/driver"
+	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/util/testutil/integration"
+	"github.com/pkg/errors"
+)
+
+func InitRemoteMultiNodeWorker() {
+	integration.Register(&remoteMultiNodeWorker{
+		id: "remote+multinode",
+	})
+}
+
+type remoteMultiNodeWorker struct {
+	id string
+
+	unsupported []string
+
+	docker      integration.Backend
+	dockerClose func() error
+	dockerErr   error
+	dockerOnce  sync.Once
+}
+
+func (w *remoteMultiNodeWorker) Name() string {
+	return w.id
+}
+
+func (w *remoteMultiNodeWorker) Rootless() bool {
+	return false
+}
+
+func (w *remoteMultiNodeWorker) NetNSDetached() bool {
+	return false
+}
+
+func (w *remoteMultiNodeWorker) New(ctx context.Context, cfg *integration.BackendConfig) (integration.Backend, func() error, error) {
+	w.dockerOnce.Do(func() {
+		w.docker, w.dockerClose, w.dockerErr = dockerWorker{id: w.id}.New(ctx, cfg)
+	})
+	if w.dockerErr != nil {
+		return w.docker, w.dockerClose, w.dockerErr
+	}
+
+	cfgfile, release, err := integration.WriteConfig(cfg.DaemonConfig)
+	if err != nil {
+		return nil, nil, err
+	}
+	if release != nil {
+		defer release()
+	}
+	defer os.RemoveAll(filepath.Dir(cfgfile))
+
+	name := "integration-remote-multinode-" + identity.NewID()
+	ctnBuilder0 := name + "-amd64"
+	ctnBuilder1 := name + "-arm64"
+
+	run := func(ctx context.Context, args ...string) ([]byte, error) {
+		cmd := exec.CommandContext(ctx, "buildx", args...)
+		cmd.Env = append(
+			os.Environ(),
+			"BUILDX_CONFIG=/tmp/buildx-"+name,
+			"DOCKER_CONTEXT="+w.docker.DockerAddress(),
+		)
+		return cmd.CombinedOutput()
+	}
+
+	if out, err := run(ctx, "create",
+		"--name="+ctnBuilder0,
+		"--driver=docker-container",
+		"--buildkitd-config="+cfgfile,
+		"--driver-opt=network=host",
+		"--platform=linux/amd64",
+	); err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to create builder %s: %s", ctnBuilder0, string(out))
+	}
+	if out, err := run(ctx, "inspect", "--bootstrap", ctnBuilder0); err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to bootstrap builder %s: %s", ctnBuilder0, string(out))
+	}
+
+	if out, err := run(ctx, "create",
+		"--name="+ctnBuilder1,
+		"--driver=docker-container",
+		"--buildkitd-config="+cfgfile,
+		"--driver-opt=network=host",
+		"--platform=linux/arm64",
+	); err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to create builder %s: %s", ctnBuilder1, string(out))
+	}
+	if out, err := run(ctx, "inspect", "--bootstrap", ctnBuilder1); err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to bootstrap builder %s: %s", ctnBuilder1, string(out))
+	}
+
+	endpoint0 := fmt.Sprintf("docker-container://%s0", driver.BuilderName(ctnBuilder0))
+	endpoint1 := fmt.Sprintf("docker-container://%s0", driver.BuilderName(ctnBuilder1))
+	if out, err := run(ctx, "create", "--name="+name, "--driver=remote", endpoint0); err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to create builder %s: %s", name, string(out))
+	}
+	if out, err := run(ctx, "create", "--append", "--name="+name, endpoint1); err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to append builder %s: %s", name, string(out))
+	}
+	if out, err := run(ctx, "inspect", "--bootstrap", name); err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to bootstrap builder %s: %s", name, string(out))
+	}
+
+	cl := func() error {
+		runCleanup := func(args ...string) error {
+			out, err := run(context.Background(), args...)
+			if err != nil {
+				return errors.Wrapf(err, "%s: %s", args[1], string(out))
+			}
+			return nil
+		}
+
+		setErr := func(dst *error, err error) {
+			if err != nil && *dst == nil {
+				*dst = err
+			}
+		}
+
+		var err error
+		setErr(&err, runCleanup("rm", "-f", name))
+		setErr(&err, runCleanup("rm", "-f", ctnBuilder0))
+		setErr(&err, runCleanup("rm", "-f", ctnBuilder1))
+		return err
+	}
+
+	return &backend{
+		builder:             name,
+		context:             w.docker.DockerAddress(),
+		unsupportedFeatures: w.unsupported,
+	}, cl, nil
+}
+
+func (w *remoteMultiNodeWorker) Close() error {
+	if c := w.dockerClose; c != nil {
+		return c()
+	}
+
+	w.docker = nil
+	w.dockerClose = nil
+	w.dockerErr = nil
+	w.dockerOnce = sync.Once{}
+
+	return nil
+}