diff --git a/driver/docker-container/driver.go b/driver/docker-container/driver.go
index 2e287721b..e6781f773 100644
--- a/driver/docker-container/driver.go
+++ b/driver/docker-container/driver.go
@@ -229,16 +229,10 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 		if len(d.gpus) > 0 && d.hasGPUCapability(ctx, cfg.Image, d.gpus) {
 			hc.DeviceRequests = d.gpus
 		}
+		if d.cgroupParent != "" {
+			hc.CgroupParent = d.cgroupParent
+		}
 		if resp, err := d.DockerAPI.Info(ctx, dockerclient.InfoOptions{}); err == nil {
-			if resp.Info.CgroupDriver == "cgroupfs" {
-				// Place all buildkit containers inside this cgroup by default so limits can be attached
-				// to all build activity on the host.
-				hc.CgroupParent = "/docker/buildx"
-				if d.cgroupParent != "" {
-					hc.CgroupParent = d.cgroupParent
-				}
-			}
-
 			for _, f := range security.DecodeOptions(resp.Info.SecurityOptions) {
 				if f.Name == "userns" {
 					hc.UsernsMode = "host"
diff --git a/driver/docker-container/driver_test.go b/driver/docker-container/driver_test.go
new file mode 100644
index 000000000..a240ec2d5
--- /dev/null
+++ b/driver/docker-container/driver_test.go
@@ -0,0 +1,78 @@
+package docker
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/docker/buildx/driver"
+	"github.com/docker/buildx/util/progress"
+	"github.com/moby/buildkit/client"
+	dockerclient "github.com/moby/moby/client"
+	"github.com/stretchr/testify/require"
+)
+
+var errContainerCreateCaptured = errors.New("container create captured")
+
+type containerCreateCapturingAPI struct {
+	dockerclient.APIClient
+	containerCreateOptions dockerclient.ContainerCreateOptions
+}
+
+func (c *containerCreateCapturingAPI) ImagePull(context.Context, string, dockerclient.ImagePullOptions) (dockerclient.ImagePullResponse, error) {
+	return nil, errors.New("skip image pull in test")
+}
+
+func (c *containerCreateCapturingAPI) ImageInspect(context.Context, string, ...dockerclient.ImageInspectOption) (dockerclient.ImageInspectResult, error) {
+	return dockerclient.ImageInspectResult{}, nil
+}
+
+func (c *containerCreateCapturingAPI) Info(context.Context, dockerclient.InfoOptions) (dockerclient.SystemInfoResult, error) {
+	result := dockerclient.SystemInfoResult{}
+	result.Info.CgroupDriver = "cgroupfs"
+	return result, nil
+}
+
+func (c *containerCreateCapturingAPI) ContainerCreate(_ context.Context, options dockerclient.ContainerCreateOptions) (dockerclient.ContainerCreateResult, error) {
+	c.containerCreateOptions = options
+	return dockerclient.ContainerCreateResult{}, errContainerCreateCaptured
+}
+
+func TestCreateCgroupParent(t *testing.T) {
+	tests := []struct {
+		name                 string
+		driverOpts           map[string]string
+		expectedCgroupParent string
+	}{
+		{
+			name:                 "cgroupfs default",
+			expectedCgroupParent: "",
+		},
+		{
+			name:                 "explicit cgroup parent",
+			driverOpts:           map[string]string{"cgroup-parent": "/custom/buildx"},
+			expectedCgroupParent: "/custom/buildx",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			api := &containerCreateCapturingAPI{}
+			drv, err := (&factory{}).New(context.Background(), driver.InitConfig{
+				Name:       "buildx-test",
+				DockerAPI:  api,
+				DriverOpts: tt.driverOpts,
+			})
+			require.NoError(t, err)
+
+			d := drv.(*Driver)
+			d.gpus = nil
+			err = progress.Wrap("test", func(*client.SolveStatus) {}, func(l progress.SubLogger) error {
+				return d.create(context.Background(), l)
+			})
+			require.ErrorIs(t, err, errContainerCreateCaptured)
+			require.NotNil(t, api.containerCreateOptions.HostConfig)
+			require.Equal(t, tt.expectedCgroupParent, api.containerCreateOptions.HostConfig.CgroupParent)
+		})
+	}
+}