From 766592b2d184632ddc671bb9e6ec1f21c7760731 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Thu, 12 Mar 2026 10:52:20 +0100 Subject: [PATCH] tests: add remote multi-node worker Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/build.yml | 13 ++- tests/integration.go | 9 ++ tests/integration_test.go | 1 + tests/policy_build.go | 6 +- tests/policy_eval.go | 3 +- tests/workers/remote-multinode.go | 155 ++++++++++++++++++++++++++++++ 6 files changed, 177 insertions(+), 10 deletions(-) create mode 100644 tests/workers/remote-multinode.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 436b5198d..8e2a32aa5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,13 +66,18 @@ jobs: - "" - experimental include: - - worker: docker + - worker: remote+multinode pkg: ./tests - - worker: docker+containerd # same as docker, but with containerd snapshotter + - worker: remote+multinode + pkg: ./tests + mode: experimental + - worker: docker pkg: ./tests - worker: docker pkg: ./tests mode: experimental + - worker: docker+containerd # same as docker, but with containerd snapshotter + pkg: ./tests - worker: docker+containerd # same as docker, but with containerd snapshotter pkg: ./tests mode: experimental @@ -106,14 +111,14 @@ jobs: fi testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$" case "${{ matrix.worker }}" in - docker | docker+containerd | docker@* | docker+containerd@*) + docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode) echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV ;; *) echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV ;; esac - if [[ "${{ matrix.worker }}" == "docker"* ]]; then + if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" ]]; then echo "TEST_DOCKERD=1" >> $GITHUB_ENV fi if [ "${{ matrix.mode }}" = "experimental" ]; then diff --git a/tests/integration.go b/tests/integration.go index 91edb178a..ea2551fae 100644 --- a/tests/integration.go +++ b/tests/integration.go @@ -141,6 +141,15 @@ func isDockerContainerWorker(sb integration.Sandbox) bool { return name == "docker-container" } +func isRemoteWorker(sb integration.Sandbox) bool { + name, _, _ := driverName(sb.Name()) + return name == "remote" +} + +func isRemoteMultiNodeWorker(sb integration.Sandbox) bool { + return sb.Name() == "remote+multinode" +} + func driverName(sbName string) (string, bool, bool) { name := sbName var hasVersion, hasFeature bool diff --git a/tests/integration_test.go b/tests/integration_test.go index 813983aae..574251dc5 100644 --- a/tests/integration_test.go +++ b/tests/integration_test.go @@ -14,6 +14,7 @@ func init() { if bkworkers.IsTestDockerd() { workers.InitDockerWorker() workers.InitDockerContainerWorker() + workers.InitRemoteMultiNodeWorker() } else { workers.InitRemoteWorker() } diff --git a/tests/policy_build.go b/tests/policy_build.go index 244b8e00d..16be87ae5 100644 --- a/tests/policy_build.go +++ b/tests/policy_build.go @@ -779,8 +779,7 @@ decision := {"allow": allow} for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { if tc.requiresHTTPChecksum { - sbDriver, _, _ := driverName(sb.Name()) - if sbDriver != "remote" { + if !isRemoteWorker(sb) { t.Skip("http checksum policy input requires remote driver") } skipNoCompatBuildKit(t, sb, ">= 0.26.3-0", "http checksum policy input") @@ -1084,8 +1083,7 @@ decision := {"allow": allow} for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { if tc.requiresGitResolve { - sbDriver, _, _ := driverName(sb.Name()) - if sbDriver != "remote" { + if !isRemoteWorker(sb) { t.Skip("git policy metadata requires remote driver") } } diff --git a/tests/policy_eval.go b/tests/policy_eval.go index cfd97348a..19018648d 100644 --- a/tests/policy_eval.go +++ b/tests/policy_eval.go @@ -492,8 +492,7 @@ decision := {"allow": allow} for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { if tc.needsChecksum { - sbDriver, _, _ := driverName(sb.Name()) - if sbDriver != "remote" { + if !isRemoteWorker(sb) { t.Skip("http checksum policy eval requires remote driver") } skipNoCompatBuildKit(t, sb, ">= 0.26.3-0", "http checksum policy input") diff --git a/tests/workers/remote-multinode.go b/tests/workers/remote-multinode.go new file mode 100644 index 000000000..659e0c0f3 --- /dev/null +++ b/tests/workers/remote-multinode.go @@ -0,0 +1,155 @@ +package workers + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "sync" + + "github.com/docker/buildx/driver" + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/util/testutil/integration" + "github.com/pkg/errors" +) + +func InitRemoteMultiNodeWorker() { + integration.Register(&remoteMultiNodeWorker{ + id: "remote+multinode", + }) +} + +type remoteMultiNodeWorker struct { + id string + + unsupported []string + + docker integration.Backend + dockerClose func() error + dockerErr error + dockerOnce sync.Once +} + +func (w *remoteMultiNodeWorker) Name() string { + return w.id +} + +func (w *remoteMultiNodeWorker) Rootless() bool { + return false +} + +func (w *remoteMultiNodeWorker) NetNSDetached() bool { + return false +} + +func (w *remoteMultiNodeWorker) New(ctx context.Context, cfg *integration.BackendConfig) (integration.Backend, func() error, error) { + w.dockerOnce.Do(func() { + w.docker, w.dockerClose, w.dockerErr = dockerWorker{id: w.id}.New(ctx, cfg) + }) + if w.dockerErr != nil { + return w.docker, w.dockerClose, w.dockerErr + } + + cfgfile, release, err := integration.WriteConfig(cfg.DaemonConfig) + if err != nil { + return nil, nil, err + } + if release != nil { + defer release() + } + defer os.RemoveAll(filepath.Dir(cfgfile)) + + name := "integration-remote-multinode-" + identity.NewID() + ctnBuilder0 := name + "-amd64" + ctnBuilder1 := name + "-arm64" + + run := func(ctx context.Context, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "buildx", args...) + cmd.Env = append( + os.Environ(), + "BUILDX_CONFIG=/tmp/buildx-"+name, + "DOCKER_CONTEXT="+w.docker.DockerAddress(), + ) + return cmd.CombinedOutput() + } + + if out, err := run(ctx, "create", + "--name="+ctnBuilder0, + "--driver=docker-container", + "--buildkitd-config="+cfgfile, + "--driver-opt=network=host", + "--platform=linux/amd64", + ); err != nil { + return nil, nil, errors.Wrapf(err, "failed to create builder %s: %s", ctnBuilder0, string(out)) + } + if out, err := run(ctx, "inspect", "--bootstrap", ctnBuilder0); err != nil { + return nil, nil, errors.Wrapf(err, "failed to bootstrap builder %s: %s", ctnBuilder0, string(out)) + } + + if out, err := run(ctx, "create", + "--name="+ctnBuilder1, + "--driver=docker-container", + "--buildkitd-config="+cfgfile, + "--driver-opt=network=host", + "--platform=linux/arm64", + ); err != nil { + return nil, nil, errors.Wrapf(err, "failed to create builder %s: %s", ctnBuilder1, string(out)) + } + if out, err := run(ctx, "inspect", "--bootstrap", ctnBuilder1); err != nil { + return nil, nil, errors.Wrapf(err, "failed to bootstrap builder %s: %s", ctnBuilder1, string(out)) + } + + endpoint0 := fmt.Sprintf("docker-container://%s0", driver.BuilderName(ctnBuilder0)) + endpoint1 := fmt.Sprintf("docker-container://%s0", driver.BuilderName(ctnBuilder1)) + if out, err := run(ctx, "create", "--name="+name, "--driver=remote", endpoint0); err != nil { + return nil, nil, errors.Wrapf(err, "failed to create builder %s: %s", name, string(out)) + } + if out, err := run(ctx, "create", "--append", "--name="+name, endpoint1); err != nil { + return nil, nil, errors.Wrapf(err, "failed to append builder %s: %s", name, string(out)) + } + if out, err := run(ctx, "inspect", "--bootstrap", name); err != nil { + return nil, nil, errors.Wrapf(err, "failed to bootstrap builder %s: %s", name, string(out)) + } + + cl := func() error { + runCleanup := func(args ...string) error { + out, err := run(context.Background(), args...) + if err != nil { + return errors.Wrapf(err, "%s: %s", args[1], string(out)) + } + return nil + } + + setErr := func(dst *error, err error) { + if err != nil && *dst == nil { + *dst = err + } + } + + var err error + setErr(&err, runCleanup("rm", "-f", name)) + setErr(&err, runCleanup("rm", "-f", ctnBuilder0)) + setErr(&err, runCleanup("rm", "-f", ctnBuilder1)) + return err + } + + return &backend{ + builder: name, + context: w.docker.DockerAddress(), + unsupportedFeatures: w.unsupported, + }, cl, nil +} + +func (w *remoteMultiNodeWorker) Close() error { + if c := w.dockerClose; c != nil { + return c() + } + + w.docker = nil + w.dockerClose = nil + w.dockerErr = nil + w.dockerOnce = sync.Once{} + + return nil +}