alextab93/buildx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request #3724 from crazy-max/tests-multi-node

Browse Source 
ci: run integration tests with the remote multi-node worker
This commit is contained in:
Tõnis Tiigi
2026-03-12 11:37:37 -07:00
committed by GitHub
parent 08152f55be 176e497922
commit 18c3dc273c
7 changed files with 187 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build.yml
+9 -4
View File
@@ -66,13 +66,18 @@ jobs:
- ""
- experimental
include:
- worker: docker
- worker: remote+multinode
pkg: ./tests
- worker: docker+containerd # same as docker, but with containerd snapshotter
- worker: remote+multinode
pkg: ./tests
mode: experimental
- worker: docker
pkg: ./tests
- worker: docker
pkg: ./tests
mode: experimental
- worker: docker+containerd # same as docker, but with containerd snapshotter
pkg: ./tests
- worker: docker+containerd # same as docker, but with containerd snapshotter
pkg: ./tests
mode: experimental
@@ -106,14 +111,14 @@ jobs:
fi
testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$"
case "${{ matrix.worker }}" in
docker | docker+containerd | docker@* | docker+containerd@*)
docker | docker+containerd | docker@* | docker+containerd@* | remote+multinode)
echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV
;;
*)
echo "TESTFLAGS=${{ env.TESTFLAGS }} $testFlags" >> $GITHUB_ENV
;;
esac
if [[ "${{ matrix.worker }}" == "docker"* ]]; then
if [[ "${{ matrix.worker }}" == "docker"* || "${{ matrix.worker }}" == "remote+multinode" ]]; then
echo "TEST_DOCKERD=1" >> $GITHUB_ENV
fi
if [ "${{ matrix.mode }}" = "experimental" ]; then
tests/history.go
+10
View File
@@ -77,6 +77,11 @@ func testHistoryInspect(t *testing.T, sb integration.Sandbox) {
}
func testHistoryLs(t *testing.T, sb integration.Sandbox) {
if isRemoteMultiNodeWorker(sb) {
// FIXME: "history ls" fails on multi nodes
t.Skip("fails with multi nodes")
}
ref := buildTestProject(t, sb)
require.NotEmpty(t, ref.Ref)
@@ -141,6 +146,11 @@ func testHistoryLsStoppedBuilder(t *testing.T, sb integration.Sandbox) {
}
func testHistoryBuildName(t *testing.T, sb integration.Sandbox) {
if isRemoteMultiNodeWorker(sb) {
// FIXME: "history ls" fails on multi nodes
t.Skip("fails with multi nodes")
}
t.Run("override", func(t *testing.T) {
dir := createTestProject(t)
out, err := buildCmd(sb, withArgs("--build-arg=BUILDKIT_BUILD_NAME=foobar", "--metadata-file", filepath.Join(dir, "md.json"), dir))
tests/integration.go
+9
View File
@@ -141,6 +141,15 @@ func isDockerContainerWorker(sb integration.Sandbox) bool {
return name == "docker-container"
}
func isRemoteWorker(sb integration.Sandbox) bool {
name, _, _ := driverName(sb.Name())
return name == "remote"
}
func isRemoteMultiNodeWorker(sb integration.Sandbox) bool {
return sb.Name() == "remote+multinode"
}
func driverName(sbName string) (string, bool, bool) {
name := sbName
var hasVersion, hasFeature bool
tests/integration_test.go
+1
View File
@@ -14,6 +14,7 @@ func init() {
if bkworkers.IsTestDockerd() {
workers.InitDockerWorker()
workers.InitDockerContainerWorker()
workers.InitRemoteMultiNodeWorker()
} else {
workers.InitRemoteWorker()
}
tests/policy_build.go
+2 -4
View File
@@ -779,8 +779,7 @@ decision := {"allow": allow}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
if tc.requiresHTTPChecksum {
sbDriver, _, _ := driverName(sb.Name())
if sbDriver != "remote" {
if !isRemoteWorker(sb) {
t.Skip("http checksum policy input requires remote driver")
}
skipNoCompatBuildKit(t, sb, ">= 0.26.3-0", "http checksum policy input")
@@ -1084,8 +1083,7 @@ decision := {"allow": allow}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
if tc.requiresGitResolve {
sbDriver, _, _ := driverName(sb.Name())
if sbDriver != "remote" {
if !isRemoteWorker(sb) {
t.Skip("git policy metadata requires remote driver")
}
}
tests/policy_eval.go
+1 -2
View File
@@ -492,8 +492,7 @@ decision := {"allow": allow}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
if tc.needsChecksum {
sbDriver, _, _ := driverName(sb.Name())
if sbDriver != "remote" {
if !isRemoteWorker(sb) {
t.Skip("http checksum policy eval requires remote driver")
}
skipNoCompatBuildKit(t, sb, ">= 0.26.3-0", "http checksum policy input")
tests/workers/remote-multinode.go
+155
View File
@@ -0,0 +1,155 @@
package workers
import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"sync"
"github.com/docker/buildx/driver"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/util/testutil/integration"
"github.com/pkg/errors"
)
func InitRemoteMultiNodeWorker() {
integration.Register(&remoteMultiNodeWorker{
id: "remote+multinode",
})
}
type remoteMultiNodeWorker struct {
id string
unsupported []string
docker integration.Backend
dockerClose func() error
dockerErr error
dockerOnce sync.Once
}
func (w *remoteMultiNodeWorker) Name() string {
return w.id
}
func (w *remoteMultiNodeWorker) Rootless() bool {
return false
}
func (w *remoteMultiNodeWorker) NetNSDetached() bool {
return false
}
func (w *remoteMultiNodeWorker) New(ctx context.Context, cfg *integration.BackendConfig) (integration.Backend, func() error, error) {
w.dockerOnce.Do(func() {
w.docker, w.dockerClose, w.dockerErr = dockerWorker{id: w.id}.New(ctx, cfg)
})
if w.dockerErr != nil {
return w.docker, w.dockerClose, w.dockerErr
}
cfgfile, release, err := integration.WriteConfig(cfg.DaemonConfig)
if err != nil {
return nil, nil, err
}
if release != nil {
defer release()
}
defer os.RemoveAll(filepath.Dir(cfgfile))
name := "integration-remote-multinode-" + identity.NewID()
ctnBuilder0 := name + "-amd64"
ctnBuilder1 := name + "-arm64"
run := func(ctx context.Context, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, "buildx", args...)
cmd.Env = append(
os.Environ(),
"BUILDX_CONFIG=/tmp/buildx-"+name,
"DOCKER_CONTEXT="+w.docker.DockerAddress(),
)
return cmd.CombinedOutput()
}
if out, err := run(ctx, "create",
"--name="+ctnBuilder0,
"--driver=docker-container",
"--buildkitd-config="+cfgfile,
"--driver-opt=network=host",
"--platform=linux/amd64",
); err != nil {
return nil, nil, errors.Wrapf(err, "failed to create builder %s: %s", ctnBuilder0, string(out))
}
if out, err := run(ctx, "inspect", "--bootstrap", ctnBuilder0); err != nil {
return nil, nil, errors.Wrapf(err, "failed to bootstrap builder %s: %s", ctnBuilder0, string(out))
}
if out, err := run(ctx, "create",
"--name="+ctnBuilder1,
"--driver=docker-container",
"--buildkitd-config="+cfgfile,
"--driver-opt=network=host",
"--platform=linux/arm64",
); err != nil {
return nil, nil, errors.Wrapf(err, "failed to create builder %s: %s", ctnBuilder1, string(out))
}
if out, err := run(ctx, "inspect", "--bootstrap", ctnBuilder1); err != nil {
return nil, nil, errors.Wrapf(err, "failed to bootstrap builder %s: %s", ctnBuilder1, string(out))
}
endpoint0 := fmt.Sprintf("docker-container://%s0", driver.BuilderName(ctnBuilder0))
endpoint1 := fmt.Sprintf("docker-container://%s0", driver.BuilderName(ctnBuilder1))
if out, err := run(ctx, "create", "--name="+name, "--driver=remote", endpoint0); err != nil {
return nil, nil, errors.Wrapf(err, "failed to create builder %s: %s", name, string(out))
}
if out, err := run(ctx, "create", "--append", "--name="+name, endpoint1); err != nil {
return nil, nil, errors.Wrapf(err, "failed to append builder %s: %s", name, string(out))
}
if out, err := run(ctx, "inspect", "--bootstrap", name); err != nil {
return nil, nil, errors.Wrapf(err, "failed to bootstrap builder %s: %s", name, string(out))
}
cl := func() error {
runCleanup := func(args ...string) error {
out, err := run(context.Background(), args...)
if err != nil {
return errors.Wrapf(err, "%s: %s", args[1], string(out))
}
return nil
}
setErr := func(dst *error, err error) {
if err != nil && *dst == nil {
*dst = err
}
}
var err error
setErr(&err, runCleanup("rm", "-f", name))
setErr(&err, runCleanup("rm", "-f", ctnBuilder0))
setErr(&err, runCleanup("rm", "-f", ctnBuilder1))
return err
}
return &backend{
builder: name,
context: w.docker.DockerAddress(),
unsupportedFeatures: w.unsupported,
}, cl, nil
}
func (w *remoteMultiNodeWorker) Close() error {
if c := w.dockerClose; c != nil {
return c()
}
w.docker = nil
w.dockerClose = nil
w.dockerErr = nil
w.dockerOnce = sync.Once{}
return nil
}