alextab93/buildx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

controller: remove remaining parts of the controller

Browse Source 
Removes all references to the controller and moves the remaining
sections of code to other packages.

Processes has been moved to monitor where it is used and the data
structs have been removed so buildflags is used directly. The controller
build function has been moved to the commands package.

Signed-off-by: Jonathan A. Sternberg <jonathan.sternberg@docker.com>
This commit is contained in:
Jonathan A. Sternberg
2025-06-05 11:57:03 -05:00
parent 60b1eda2df
commit 1d7cda1232
36 changed files with 659 additions and 1502 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/labeler.yml
-5
View File
@@ -48,11 +48,6 @@ area/cli:
- cmd/**
- commands/**
# Add 'area/controller' label to changes in the controller
area/controller:
- changed-files:
- any-glob-to-any-file: 'controller/**'
# Add 'area/docs' label to markdown files in the docs folder
area/docs:
- changed-files:
PROJECT.md
-1
View File
@@ -79,7 +79,6 @@ Area or component of the project affected. Please note that the table below may
| `area/checks` | Any | `checks` |
| `area/ci` | Any | Project CI |
| `area/cli` | Any | `cli` |
| `area/controller` | Any | `controller` |
| `area/debug` | Any | `debug` |
| `area/dependencies` | Any | Project dependencies |
| `area/dockerfile` | Any | `dockerfile` |
bake/bake.go
+9 -11
View File
@@ -19,7 +19,6 @@ import (
composecli "github.com/compose-spec/compose-go/v2/cli"
"github.com/docker/buildx/bake/hclparser"
"github.com/docker/buildx/build"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/platformutil"
"github.com/docker/buildx/util/progress"
@@ -1439,20 +1438,19 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
})
}
}
secrets = secrets.Normalize()
bo.SecretSpecs = secrets.ToPB()
secretAttachment, err := controllerapi.CreateSecrets(bo.SecretSpecs)
bo.SecretSpecs = secrets.Normalize()
secretAttachment, err := build.CreateSecrets(bo.SecretSpecs)
if err != nil {
return nil, err
}
bo.Session = append(bo.Session, secretAttachment)
bo.SSHSpecs = t.SSH.ToPB()
bo.SSHSpecs = t.SSH
if len(bo.SSHSpecs) == 0 && buildflags.IsGitSSH(bi.ContextPath) || (inp != nil && buildflags.IsGitSSH(inp.URL)) {
bo.SSHSpecs = []*controllerapi.SSH{{ID: "default"}}
bo.SSHSpecs = []*buildflags.SSH{{ID: "default"}}
}
sshAttachment, err := controllerapi.CreateSSH(bo.SSHSpecs)
sshAttachment, err := build.CreateSSH(bo.SSHSpecs)
if err != nil {
return nil, err
}
@@ -1469,13 +1467,13 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
}
if t.CacheFrom != nil {
bo.CacheFrom = controllerapi.CreateCaches(t.CacheFrom.ToPB())
bo.CacheFrom = build.CreateCaches(t.CacheFrom)
}
if t.CacheTo != nil {
bo.CacheTo = controllerapi.CreateCaches(t.CacheTo.ToPB())
bo.CacheTo = build.CreateCaches(t.CacheTo)
}
bo.Exports, bo.ExportsLocalPathsTemporary, err = controllerapi.CreateExports(t.Outputs.ToPB())
bo.Exports, bo.ExportsLocalPathsTemporary, err = build.CreateExports(t.Outputs)
if err != nil {
return nil, err
}
@@ -1490,7 +1488,7 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
}
}
bo.Attests = controllerapi.CreateAttestations(t.Attest.ToPB())
bo.Attests = t.Attest.ToMap()
bo.SourcePolicy, err = build.ReadSourcePolicy()
if err != nil {
bake/entitlements_test.go
+6 -6
View File
@@ -8,7 +8,7 @@ import (
"testing"
"github.com/docker/buildx/build"
"github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/osutil"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/util/entitlements"
@@ -264,7 +264,7 @@ func TestValidateEntitlements(t *testing.T) {
{
name: "SSHMissing",
opt: build.Options{
SSHSpecs: []*pb.SSH{
SSHSpecs: []*buildflags.SSH{
{
ID: "test",
},
@@ -296,7 +296,7 @@ func TestValidateEntitlements(t *testing.T) {
{
name: "SecretFromSubFile",
opt: build.Options{
SecretSpecs: []*pb.Secret{
SecretSpecs: []*buildflags.Secret{
{
FilePath: filepath.Join(dir1, "subfile"),
},
@@ -309,7 +309,7 @@ func TestValidateEntitlements(t *testing.T) {
{
name: "SecretFromEscapeLink",
opt: build.Options{
SecretSpecs: []*pb.Secret{
SecretSpecs: []*buildflags.Secret{
{
FilePath: escapeLink,
},
@@ -325,7 +325,7 @@ func TestValidateEntitlements(t *testing.T) {
{
name: "SecretFromEscapeLinkAllowRoot",
opt: build.Options{
SecretSpecs: []*pb.Secret{
SecretSpecs: []*buildflags.Secret{
{
FilePath: escapeLink,
},
@@ -352,7 +352,7 @@ func TestValidateEntitlements(t *testing.T) {
{
name: "SecretFromEscapeLinkAllowAny",
opt: build.Options{
SecretSpecs: []*pb.Secret{
SecretSpecs: []*buildflags.Secret{
{
FilePath: escapeLink,
},
bake/remote.go
+7 -6
View File
@@ -7,9 +7,10 @@ import (
"os"
"strings"
"github.com/docker/buildx/build"
"github.com/docker/buildx/builder"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/progress"
"github.com/docker/go-units"
"github.com/moby/buildkit/client"
@@ -33,27 +34,27 @@ func ReadRemoteFiles(ctx context.Context, nodes []builder.Node, url string, name
st, ok := dockerui.DetectGitContext(url, false)
if ok {
if ssh, err := controllerapi.CreateSSH([]*controllerapi.SSH{{
if ssh, err := build.CreateSSH([]*buildflags.SSH{{
ID: "default",
Paths: strings.Split(os.Getenv("BUILDX_BAKE_GIT_SSH"), ","),
}}); err == nil {
sessions = append(sessions, ssh)
}
var gitAuthSecrets []*controllerapi.Secret
var gitAuthSecrets []*buildflags.Secret
if _, ok := os.LookupEnv("BUILDX_BAKE_GIT_AUTH_TOKEN"); ok {
gitAuthSecrets = append(gitAuthSecrets, &controllerapi.Secret{
gitAuthSecrets = append(gitAuthSecrets, &buildflags.Secret{
ID: llb.GitAuthTokenKey,
Env: "BUILDX_BAKE_GIT_AUTH_TOKEN",
})
}
if _, ok := os.LookupEnv("BUILDX_BAKE_GIT_AUTH_HEADER"); ok {
gitAuthSecrets = append(gitAuthSecrets, &controllerapi.Secret{
gitAuthSecrets = append(gitAuthSecrets, &buildflags.Secret{
ID: llb.GitAuthHeaderKey,
Env: "BUILDX_BAKE_GIT_AUTH_HEADER",
})
}
if len(gitAuthSecrets) > 0 {
if secrets, err := controllerapi.CreateSecrets(gitAuthSecrets); err == nil {
if secrets, err := build.CreateSecrets(gitAuthSecrets); err == nil {
sessions = append(sessions, secrets)
}
}
build/build.go
+3 -3
View File
@@ -19,8 +19,8 @@ import (
"github.com/containerd/containerd/v2/core/images"
"github.com/distribution/reference"
"github.com/docker/buildx/builder"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/confutil"
"github.com/docker/buildx/util/desktop"
"github.com/docker/buildx/util/dockerutil"
@@ -78,8 +78,8 @@ type Options struct {
NoCacheFilter []string
Platforms []ocispecs.Platform
Pull bool
SecretSpecs []*controllerapi.Secret
SSHSpecs []*controllerapi.SSH
SecretSpecs buildflags.Secrets
SSHSpecs []*buildflags.SSH
ShmSize opts.MemBytes
Tags []string
Target string
build/invoke.go
+33 -4
View File
@@ -8,12 +8,41 @@ import (
"sync/atomic"
"syscall"
controllerapi "github.com/docker/buildx/controller/pb"
gateway "github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
type InvokeConfig struct {
Entrypoint []string
Cmd []string
NoCmd bool
Env []string
User string
NoUser bool
Cwd string
NoCwd bool
Tty bool
Rollback bool
Initial bool
SuspendOn SuspendOn
}
func (cfg *InvokeConfig) NeedsDebug(err error) bool {
return cfg.SuspendOn.DebugEnabled(err)
}
type SuspendOn int
const (
SuspendError SuspendOn = iota
SuspendAlways
)
func (s SuspendOn) DebugEnabled(err error) bool {
return err != nil || s == SuspendAlways
}
type Container struct {
cancelOnce sync.Once
containerCancel func(error)
@@ -24,7 +53,7 @@ type Container struct {
resultCtx *ResultHandle
}
func NewContainer(ctx context.Context, resultCtx *ResultHandle, cfg *controllerapi.InvokeConfig) (*Container, error) {
func NewContainer(ctx context.Context, resultCtx *ResultHandle, cfg *InvokeConfig) (*Container, error) {
mainCtx := ctx
ctrCh := make(chan *Container)
@@ -97,7 +126,7 @@ func (c *Container) markUnavailable() {
c.isUnavailable.Store(true)
}
func (c *Container) Exec(ctx context.Context, cfg *controllerapi.InvokeConfig, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
func (c *Container) Exec(ctx context.Context, cfg *InvokeConfig, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
if isInit := c.initStarted.CompareAndSwap(false, true); isInit {
defer func() {
// container can't be used after init exits
@@ -112,7 +141,7 @@ func (c *Container) Exec(ctx context.Context, cfg *controllerapi.InvokeConfig, s
return err
}
func exec(ctx context.Context, resultCtx *ResultHandle, cfg *controllerapi.InvokeConfig, ctr gateway.Container, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
func exec(ctx context.Context, resultCtx *ResultHandle, cfg *InvokeConfig, ctr gateway.Container, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
processCfg, err := resultCtx.getProcessConfig(cfg, stdin, stdout, stderr)
if err != nil {
return err
build/opt.go
+225
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"io"
"maps"
"os"
"path/filepath"
"slices"
@@ -11,12 +12,15 @@ import (
"strings"
"syscall"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/containerd/console"
"github.com/containerd/containerd/v2/core/content"
"github.com/containerd/containerd/v2/plugins/content/local"
"github.com/containerd/platforms"
"github.com/distribution/reference"
"github.com/docker/buildx/builder"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/confutil"
"github.com/docker/buildx/util/dockerutil"
"github.com/docker/buildx/util/osutil"
@@ -26,6 +30,9 @@ import (
"github.com/moby/buildkit/client/ociindex"
gateway "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/secrets/secretsprovider"
"github.com/moby/buildkit/session/sshforward/sshprovider"
"github.com/moby/buildkit/session/upload/uploadprovider"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/apicaps"
@@ -659,3 +666,221 @@ type fs struct {
}
var _ fsutil.FS = &fs{}
func CreateSSH(ssh []*buildflags.SSH) (session.Attachable, error) {
configs := make([]sshprovider.AgentConfig, 0, len(ssh))
for _, ssh := range ssh {
cfg := sshprovider.AgentConfig{
ID: ssh.ID,
Paths: slices.Clone(ssh.Paths),
}
configs = append(configs, cfg)
}
return sshprovider.NewSSHAgentProvider(configs)
}
func CreateSecrets(secrets []*buildflags.Secret) (session.Attachable, error) {
fs := make([]secretsprovider.Source, 0, len(secrets))
for _, secret := range secrets {
fs = append(fs, secretsprovider.Source{
ID: secret.ID,
FilePath: secret.FilePath,
Env: secret.Env,
})
}
store, err := secretsprovider.NewStore(fs)
if err != nil {
return nil, err
}
return secretsprovider.NewSecretProvider(store), nil
}
func CreateExports(entries []*buildflags.ExportEntry) ([]client.ExportEntry, []string, error) {
var outs []client.ExportEntry
var localPaths []string
if len(entries) == 0 {
return nil, nil, nil
}
var stdoutUsed bool
for _, entry := range entries {
if entry.Type == "" {
return nil, nil, errors.Errorf("type is required for output")
}
out := client.ExportEntry{
Type: entry.Type,
Attrs: map[string]string{},
}
maps.Copy(out.Attrs, entry.Attrs)
supportFile := false
supportDir := false
switch out.Type {
case client.ExporterLocal:
supportDir = true
case client.ExporterTar:
supportFile = true
case client.ExporterOCI, client.ExporterDocker:
tar, err := strconv.ParseBool(out.Attrs["tar"])
if err != nil {
tar = true
}
supportFile = tar
supportDir = !tar
case "registry":
out.Type = client.ExporterImage
out.Attrs["push"] = "true"
}
if supportDir {
if entry.Destination == "" {
return nil, nil, errors.Errorf("dest is required for %s exporter", out.Type)
}
if entry.Destination == "-" {
return nil, nil, errors.Errorf("dest cannot be stdout for %s exporter", out.Type)
}
fi, err := os.Stat(entry.Destination)
if err != nil && !os.IsNotExist(err) {
return nil, nil, errors.Wrapf(err, "invalid destination directory: %s", entry.Destination)
}
if err == nil && !fi.IsDir() {
return nil, nil, errors.Errorf("destination directory %s is a file", entry.Destination)
}
out.OutputDir = entry.Destination
localPaths = append(localPaths, entry.Destination)
}
if supportFile {
if entry.Destination == "" && out.Type != client.ExporterDocker {
entry.Destination = "-"
}
if entry.Destination == "-" {
if stdoutUsed {
return nil, nil, errors.Errorf("multiple outputs configured to write to stdout")
}
if _, err := console.ConsoleFromFile(os.Stdout); err == nil {
return nil, nil, errors.Errorf("dest file is required for %s exporter. refusing to write to console", out.Type)
}
out.Output = wrapWriteCloser(os.Stdout)
stdoutUsed = true
} else if entry.Destination != "" {
fi, err := os.Stat(entry.Destination)
if err != nil && !os.IsNotExist(err) {
return nil, nil, errors.Wrapf(err, "invalid destination file: %s", entry.Destination)
}
if err == nil && fi.IsDir() {
return nil, nil, errors.Errorf("destination file %s is a directory", entry.Destination)
}
f, err := os.Create(entry.Destination)
if err != nil {
return nil, nil, errors.Errorf("failed to open %s", err)
}
out.Output = wrapWriteCloser(f)
localPaths = append(localPaths, entry.Destination)
}
}
outs = append(outs, out)
}
return outs, localPaths, nil
}
func wrapWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
return func(map[string]string) (io.WriteCloser, error) {
return wc, nil
}
}
func CreateCaches(entries []*buildflags.CacheOptionsEntry) []client.CacheOptionsEntry {
var outs []client.CacheOptionsEntry
if len(entries) == 0 {
return nil
}
for _, entry := range entries {
out := client.CacheOptionsEntry{
Type: entry.Type,
Attrs: map[string]string{},
}
maps.Copy(out.Attrs, entry.Attrs)
addGithubToken(&out)
addAwsCredentials(&out)
if !isActive(&out) {
continue
}
outs = append(outs, out)
}
return outs
}
func addGithubToken(ci *client.CacheOptionsEntry) {
if ci.Type != "gha" {
return
}
version, ok := ci.Attrs["version"]
if !ok {
// https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L19
if v, ok := os.LookupEnv("ACTIONS_CACHE_SERVICE_V2"); ok {
if b, err := strconv.ParseBool(v); err == nil && b {
version = "2"
}
}
}
if _, ok := ci.Attrs["token"]; !ok {
if v, ok := os.LookupEnv("ACTIONS_RUNTIME_TOKEN"); ok {
ci.Attrs["token"] = v
}
}
if _, ok := ci.Attrs["url_v2"]; !ok && version == "2" {
// https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L34-L35
if v, ok := os.LookupEnv("ACTIONS_RESULTS_URL"); ok {
ci.Attrs["url_v2"] = v
}
}
if _, ok := ci.Attrs["url"]; !ok {
// https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L28-L33
if v, ok := os.LookupEnv("ACTIONS_CACHE_URL"); ok {
ci.Attrs["url"] = v
} else if v, ok := os.LookupEnv("ACTIONS_RESULTS_URL"); ok {
ci.Attrs["url"] = v
}
}
}
func addAwsCredentials(ci *client.CacheOptionsEntry) {
if ci.Type != "s3" {
return
}
_, okAccessKeyID := ci.Attrs["access_key_id"]
_, okSecretAccessKey := ci.Attrs["secret_access_key"]
// If the user provides access_key_id, secret_access_key, do not override the session token.
if okAccessKeyID && okSecretAccessKey {
return
}
ctx := context.TODO()
awsConfig, err := awsconfig.LoadDefaultConfig(ctx)
if err != nil {
return
}
credentials, err := awsConfig.Credentials.Retrieve(ctx)
if err != nil {
return
}
if !okAccessKeyID && credentials.AccessKeyID != "" {
ci.Attrs["access_key_id"] = credentials.AccessKeyID
}
if !okSecretAccessKey && credentials.SecretAccessKey != "" {
ci.Attrs["secret_access_key"] = credentials.SecretAccessKey
}
if _, ok := ci.Attrs["session_token"]; !ok && credentials.SessionToken != "" {
ci.Attrs["session_token"] = credentials.SessionToken
}
}
func isActive(ce *client.CacheOptionsEntry) bool {
// Always active if not gha.
if ce.Type != "gha" {
return true
}
return ce.Attrs["token"] != "" && (ce.Attrs["url"] != "" || ce.Attrs["url_v2"] != "")
}
build/opt_test.go
+40
View File
@@ -0,0 +1,40 @@
package build
import (
"testing"
"github.com/docker/buildx/util/buildflags"
"github.com/moby/buildkit/client"
"github.com/stretchr/testify/require"
)
func TestCacheOptions_DerivedVars(t *testing.T) {
t.Setenv("ACTIONS_RUNTIME_TOKEN", "sensitive_token")
t.Setenv("ACTIONS_CACHE_URL", "https://cache.github.com")
t.Setenv("AWS_ACCESS_KEY_ID", "definitely_dont_look_here")
t.Setenv("AWS_SECRET_ACCESS_KEY", "hackers_please_dont_steal")
t.Setenv("AWS_SESSION_TOKEN", "not_a_mitm_attack")
cacheFrom, err := buildflags.ParseCacheEntry([]string{"type=gha", "type=s3,region=us-west-2,bucket=my_bucket,name=my_image"})
require.NoError(t, err)
require.Equal(t, []client.CacheOptionsEntry{
{
Type: "gha",
Attrs: map[string]string{
"token": "sensitive_token",
"url": "https://cache.github.com",
},
},
{
Type: "s3",
Attrs: map[string]string{
"region": "us-west-2",
"bucket": "my_bucket",
"name": "my_image",
"access_key_id": "definitely_dont_look_here",
"secret_access_key": "hackers_please_dont_steal",
"session_token": "not_a_mitm_attack",
},
},
}, CreateCaches(cacheFrom))
}
build/result.go
+6 -7
View File
@@ -7,7 +7,6 @@ import (
"io"
"sync"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
gateway "github.com/moby/buildkit/frontend/gateway/client"
@@ -292,7 +291,7 @@ func (r *ResultHandle) build(buildFunc gateway.BuildFunc) (err error) {
return err
}
func (r *ResultHandle) getContainerConfig(cfg *controllerapi.InvokeConfig) (containerCfg gateway.NewContainerRequest, _ error) {
func (r *ResultHandle) getContainerConfig(cfg *InvokeConfig) (containerCfg gateway.NewContainerRequest, _ error) {
if r.res != nil && r.solveErr == nil {
logrus.Debugf("creating container from successful build")
ccfg, err := containerConfigFromResult(r.res, cfg)
@@ -311,7 +310,7 @@ func (r *ResultHandle) getContainerConfig(cfg *controllerapi.InvokeConfig) (cont
return containerCfg, nil
}
func (r *ResultHandle) getProcessConfig(cfg *controllerapi.InvokeConfig, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) (_ gateway.StartRequest, err error) {
func (r *ResultHandle) getProcessConfig(cfg *InvokeConfig, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) (_ gateway.StartRequest, err error) {
processCfg := newStartRequest(stdin, stdout, stderr)
if r.res != nil && r.solveErr == nil {
logrus.Debugf("creating container from successful build")
@@ -327,7 +326,7 @@ func (r *ResultHandle) getProcessConfig(cfg *controllerapi.InvokeConfig, stdin i
return processCfg, nil
}
func containerConfigFromResult(res *gateway.Result, cfg *controllerapi.InvokeConfig) (*gateway.NewContainerRequest, error) {
func containerConfigFromResult(res *gateway.Result, cfg *InvokeConfig) (*gateway.NewContainerRequest, error) {
if cfg.Initial {
return nil, errors.Errorf("starting from the container from the initial state of the step is supported only on the failed steps")
}
@@ -352,7 +351,7 @@ func containerConfigFromResult(res *gateway.Result, cfg *controllerapi.InvokeCon
}, nil
}
func populateProcessConfigFromResult(req *gateway.StartRequest, res *gateway.Result, cfg *controllerapi.InvokeConfig) error {
func populateProcessConfigFromResult(req *gateway.StartRequest, res *gateway.Result, cfg *InvokeConfig) error {
imgData := res.Metadata[exptypes.ExporterImageConfigKey]
var img *ocispecs.Image
if len(imgData) > 0 {
@@ -403,7 +402,7 @@ func populateProcessConfigFromResult(req *gateway.StartRequest, res *gateway.Res
return nil
}
func containerConfigFromError(solveErr *errdefs.SolveError, cfg *controllerapi.InvokeConfig) (*gateway.NewContainerRequest, error) {
func containerConfigFromError(solveErr *errdefs.SolveError, cfg *InvokeConfig) (*gateway.NewContainerRequest, error) {
exec, err := execOpFromError(solveErr)
if err != nil {
return nil, err
@@ -431,7 +430,7 @@ func containerConfigFromError(solveErr *errdefs.SolveError, cfg *controllerapi.I
}, nil
}
func populateProcessConfigFromError(req *gateway.StartRequest, solveErr *errdefs.SolveError, cfg *controllerapi.InvokeConfig) error {
func populateProcessConfigFromError(req *gateway.StartRequest, solveErr *errdefs.SolveError, cfg *InvokeConfig) error {
exec, err := execOpFromError(solveErr)
if err != nil {
return err
commands/bake.go
+1 -2
View File
@@ -22,7 +22,6 @@ import (
"github.com/docker/buildx/bake/hclparser"
"github.com/docker/buildx/build"
"github.com/docker/buildx/builder"
"github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/localstate"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/cobrautil/completion"
@@ -348,7 +347,7 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba
continue
}
pf := &pb.CallFunc{
pf := &buildflags.CallFunc{
Name: req.CallFunc.Name,
Format: req.CallFunc.Format,
IgnoreStatus: req.CallFunc.IgnoreStatus,
commands/build.go
+244 -46
View File
@@ -21,9 +21,6 @@ import (
"github.com/docker/buildx/build"
"github.com/docker/buildx/builder"
"github.com/docker/buildx/commands/debug"
cbuild "github.com/docker/buildx/controller/build"
controllererrors "github.com/docker/buildx/controller/errdefs"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/monitor"
"github.com/docker/buildx/store"
"github.com/docker/buildx/store/storeutil"
@@ -31,8 +28,10 @@ import (
"github.com/docker/buildx/util/cobrautil"
"github.com/docker/buildx/util/confutil"
"github.com/docker/buildx/util/desktop"
"github.com/docker/buildx/util/dockerutil"
"github.com/docker/buildx/util/metricutil"
"github.com/docker/buildx/util/osutil"
"github.com/docker/buildx/util/platformutil"
"github.com/docker/buildx/util/progress"
"github.com/docker/buildx/util/tracing"
"github.com/docker/cli/cli"
@@ -45,8 +44,10 @@ import (
"github.com/moby/buildkit/frontend/subrequests/lint"
"github.com/moby/buildkit/frontend/subrequests/outline"
"github.com/moby/buildkit/frontend/subrequests/targets"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/moby/buildkit/solver/errdefs"
solverpb "github.com/moby/buildkit/solver/pb"
sourcepolicy "github.com/moby/buildkit/sourcepolicy/pb"
"github.com/moby/buildkit/util/grpcerrors"
"github.com/moby/buildkit/util/progress/progressui"
"github.com/moby/sys/atomicwriter"
@@ -104,7 +105,7 @@ type buildOptions struct {
invokeConfig *invokeConfig
}
func (o *buildOptions) toControllerOptions() (*cbuild.Options, error) {
func (o *buildOptions) toOptions() (*BuildOptions, error) {
var err error
buildArgs, err := listToMap(o.buildArgs, true)
@@ -117,7 +118,7 @@ func (o *buildOptions) toControllerOptions() (*cbuild.Options, error) {
return nil, err
}
opts := cbuild.Options{
opts := BuildOptions{
Allow: o.allow,
Annotations: o.annotations,
BuildArgs: buildArgs,
@@ -132,7 +133,7 @@ func (o *buildOptions) toControllerOptions() (*cbuild.Options, error) {
ShmSize: int64(o.shmSize),
Tags: o.tags,
Target: o.target,
Ulimits: dockerUlimitToControllerUlimit(o.ulimits),
Ulimits: o.ulimits,
Builder: o.builder,
NoCache: o.noCache,
Pull: o.pull,
@@ -179,17 +180,15 @@ func (o *buildOptions) toControllerOptions() (*cbuild.Options, error) {
}
}
cacheFrom, err := buildflags.ParseCacheEntry(o.cacheFrom)
opts.CacheFrom, err = buildflags.ParseCacheEntry(o.cacheFrom)
if err != nil {
return nil, err
}
opts.CacheFrom = cacheFrom.ToPB()
cacheTo, err := buildflags.ParseCacheEntry(o.cacheTo)
opts.CacheTo, err = buildflags.ParseCacheEntry(o.cacheTo)
if err != nil {
return nil, err
}
opts.CacheTo = cacheTo.ToPB()
opts.Secrets, err = buildflags.ParseSecretSpecs(o.secrets)
if err != nil {
@@ -293,7 +292,7 @@ func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions)
end(err)
}()
opts, err := options.toControllerOptions()
opts, err := options.toOptions()
if err != nil {
return err
}
@@ -350,14 +349,7 @@ func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions)
}
done := timeBuildCommand(mp, attributes)
var resp *client.SolveResponse
var inputs *build.Inputs
var retErr error
if confutil.IsExperimental() {
resp, inputs, retErr = runControllerBuild(ctx, dockerCli, opts, options, printer)
} else {
resp, inputs, retErr = runBasicBuild(ctx, dockerCli, opts, printer)
}
resp, inputs, retErr := runBuildWithOptions(ctx, dockerCli, opts, options, printer)
if err := printer.Wait(); retErr == nil {
retErr = err
@@ -415,11 +407,7 @@ func getImageID(resp map[string]string) string {
return dgst
}
func runBasicBuild(ctx context.Context, dockerCli command.Cli, opts *cbuild.Options, printer *progress.Printer) (*client.SolveResponse, *build.Inputs, error) {
return cbuild.RunBuild(ctx, dockerCli, opts, dockerCli.In(), printer, nil)
}
func runControllerBuild(ctx context.Context, dockerCli command.Cli, opts *cbuild.Options, options buildOptions, printer *progress.Printer) (_ *client.SolveResponse, _ *build.Inputs, retErr error) {
func runBuildWithOptions(ctx context.Context, dockerCli command.Cli, opts *BuildOptions, options buildOptions, printer *progress.Printer) (_ *client.SolveResponse, _ *build.Inputs, retErr error) {
if options.invokeConfig != nil && (options.dockerfileName == "-" || options.contextPath == "-") {
// stdin must be usable for monitor
return nil, nil, errors.Errorf("Dockerfile or context from stdin is not supported with invoke")
@@ -440,9 +428,9 @@ func runControllerBuild(ctx context.Context, dockerCli command.Cli, opts *cbuild
}
for {
resp, inputs, err := cbuild.RunBuild(ctx, dockerCli, opts, in, printer, &bh)
resp, inputs, err := RunBuild(ctx, dockerCli, opts, in, printer, &bh)
if err != nil {
var be *controllererrors.BuildError
var be *BuildError
if errors.As(err, &be) {
retErr = err
// We can proceed to monitor
@@ -765,21 +753,6 @@ func listToMap(values []string, defaultEnv bool) (map[string]string, error) {
return result, nil
}
func dockerUlimitToControllerUlimit(u *dockeropts.UlimitOpt) *controllerapi.UlimitOpt {
if u == nil {
return nil
}
values := make(map[string]*controllerapi.Ulimit)
for _, u := range u.GetList() {
values[u.Name] = &controllerapi.Ulimit{
Name: u.Name,
Hard: u.Hard,
Soft: u.Soft,
}
}
return &controllerapi.UlimitOpt{Values: values}
}
func printWarnings(w io.Writer, warnings []client.VertexWarning, mode progressui.DisplayMode) {
if len(warnings) == 0 || mode == progressui.QuietMode || mode == progressui.RawJSONMode {
return
@@ -819,7 +792,7 @@ func printWarnings(w io.Writer, warnings []client.VertexWarning, mode progressui
}
}
func printResult(w io.Writer, f *controllerapi.CallFunc, res map[string]string, target string, inp *build.Inputs) (int, error) {
func printResult(w io.Writer, f *buildflags.CallFunc, res map[string]string, target string, inp *build.Inputs) (int, error) {
switch f.Name {
case "outline":
return 0, printValue(w, outline.PrintOutline, outline.SubrequestsOutlineDefinition.Version, f.Format, res)
@@ -920,19 +893,19 @@ func printValue(w io.Writer, printer callFunc, version string, format string, re
}
type invokeConfig struct {
controllerapi.InvokeConfig
build.InvokeConfig
invokeFlag string
}
func (cfg *invokeConfig) parseInvokeConfig(invoke, on string) error {
switch on {
case "always":
cfg.SuspendOn = controllerapi.SuspendAlways
cfg.SuspendOn = build.SuspendAlways
case "error":
cfg.SuspendOn = controllerapi.SuspendError
cfg.SuspendOn = build.SuspendError
default:
if invoke != "" {
cfg.SuspendOn = controllerapi.SuspendAlways
cfg.SuspendOn = build.SuspendAlways
}
}
@@ -1057,3 +1030,228 @@ func otelErrorType(err error) string {
}
return name
}
const defaultTargetName = "default"
type BuildOptions struct {
ContextPath string
DockerfileName string
CallFunc *buildflags.CallFunc
NamedContexts map[string]string
Allow []string
Attests buildflags.Attests
BuildArgs map[string]string
CacheFrom []*buildflags.CacheOptionsEntry
CacheTo []*buildflags.CacheOptionsEntry
CgroupParent string
Exports []*buildflags.ExportEntry
ExtraHosts []string
Labels map[string]string
NetworkMode string
NoCacheFilter []string
Platforms []string
Secrets buildflags.Secrets
ShmSize int64
SSH []*buildflags.SSH
Tags []string
Target string
Ulimits *dockeropts.UlimitOpt
Builder string
NoCache bool
Pull bool
ExportPush bool
ExportLoad bool
SourcePolicy *sourcepolicy.Policy
Ref string
GroupRef string
Annotations []string
ProvenanceResponseMode string
}
// RunBuild runs the specified build and returns the result.
func RunBuild(ctx context.Context, dockerCli command.Cli, in *BuildOptions, inStream io.Reader, progress progress.Writer, bh *build.Handler) (*client.SolveResponse, *build.Inputs, error) {
if in.NoCache && len(in.NoCacheFilter) > 0 {
return nil, nil, errors.Errorf("--no-cache and --no-cache-filter cannot currently be used together")
}
contexts := map[string]build.NamedContext{}
for name, path := range in.NamedContexts {
contexts[name] = build.NamedContext{Path: path}
}
opts := build.Options{
Inputs: build.Inputs{
ContextPath: in.ContextPath,
DockerfilePath: in.DockerfileName,
InStream: build.NewSyncMultiReader(inStream),
NamedContexts: contexts,
},
Ref: in.Ref,
BuildArgs: in.BuildArgs,
CgroupParent: in.CgroupParent,
ExtraHosts: in.ExtraHosts,
Labels: in.Labels,
NetworkMode: in.NetworkMode,
NoCache: in.NoCache,
NoCacheFilter: in.NoCacheFilter,
Pull: in.Pull,
ShmSize: dockeropts.MemBytes(in.ShmSize),
Tags: in.Tags,
Target: in.Target,
Ulimits: in.Ulimits,
GroupRef: in.GroupRef,
ProvenanceResponseMode: confutil.ParseMetadataProvenance(in.ProvenanceResponseMode),
}
platforms, err := platformutil.Parse(in.Platforms)
if err != nil {
return nil, nil, err
}
opts.Platforms = platforms
dockerConfig := dockerCli.ConfigFile()
opts.Session = append(opts.Session, authprovider.NewDockerAuthProvider(authprovider.DockerAuthProviderConfig{
ConfigFile: dockerConfig,
}))
secrets, err := build.CreateSecrets(in.Secrets)
if err != nil {
return nil, nil, err
}
opts.Session = append(opts.Session, secrets)
sshSpecs := in.SSH
if len(sshSpecs) == 0 && buildflags.IsGitSSH(in.ContextPath) {
sshSpecs = append(sshSpecs, &buildflags.SSH{ID: "default"})
}
ssh, err := build.CreateSSH(sshSpecs)
if err != nil {
return nil, nil, err
}
opts.Session = append(opts.Session, ssh)
outputs, _, err := build.CreateExports(in.Exports)
if err != nil {
return nil, nil, err
}
if in.ExportPush {
var pushUsed bool
for i := range outputs {
if outputs[i].Type == client.ExporterImage {
outputs[i].Attrs["push"] = "true"
pushUsed = true
}
}
if !pushUsed {
outputs = append(outputs, client.ExportEntry{
Type: client.ExporterImage,
Attrs: map[string]string{
"push": "true",
},
})
}
}
if in.ExportLoad {
var loadUsed bool
for i := range outputs {
if outputs[i].Type == client.ExporterDocker {
if _, ok := outputs[i].Attrs["dest"]; !ok {
loadUsed = true
break
}
}
}
if !loadUsed {
outputs = append(outputs, client.ExportEntry{
Type: client.ExporterDocker,
Attrs: map[string]string{},
})
}
}
annotations, err := buildflags.ParseAnnotations(in.Annotations)
if err != nil {
return nil, nil, errors.Wrap(err, "parse annotations")
}
for _, o := range outputs {
for k, v := range annotations {
o.Attrs[k.String()] = v
}
}
opts.Exports = outputs
opts.CacheFrom = build.CreateCaches(in.CacheFrom)
opts.CacheTo = build.CreateCaches(in.CacheTo)
opts.Attests = in.Attests.ToMap()
opts.SourcePolicy = in.SourcePolicy
allow, err := buildflags.ParseEntitlements(in.Allow)
if err != nil {
return nil, nil, err
}
opts.Allow = allow
if in.CallFunc != nil {
opts.CallFunc = &build.CallFunc{
Name: in.CallFunc.Name,
Format: in.CallFunc.Format,
IgnoreStatus: in.CallFunc.IgnoreStatus,
}
}
// key string used for kubernetes "sticky" mode
contextPathHash, err := filepath.Abs(in.ContextPath)
if err != nil {
contextPathHash = in.ContextPath
}
b, err := builder.New(dockerCli,
builder.WithName(in.Builder),
builder.WithContextPathHash(contextPathHash),
)
if err != nil {
return nil, nil, err
}
if err = updateLastActivity(dockerCli, b.NodeGroup); err != nil {
return nil, nil, errors.Wrapf(err, "failed to update builder last activity time")
}
nodes, err := b.LoadNodes(ctx)
if err != nil {
return nil, nil, err
}
var inputs *build.Inputs
buildOptions := map[string]build.Options{defaultTargetName: opts}
resp, err := build.BuildWithResultHandler(ctx, nodes, buildOptions, dockerutil.NewClient(dockerCli), confutil.NewConfig(dockerCli), progress, bh)
err = wrapBuildError(err, false)
if err != nil {
return nil, nil, WrapBuild(err)
}
if i, ok := buildOptions[defaultTargetName]; ok {
inputs = &i.Inputs
}
return resp[defaultTargetName], inputs, nil
}
type BuildError struct {
err error
}
func (e *BuildError) Unwrap() error {
return e.err
}
func (e *BuildError) Error() string {
return e.err.Error()
}
func WrapBuild(err error) error {
if err == nil {
return nil
}
return &BuildError{err: err}
}
controller/build/build.go
-272
View File
@@ -1,272 +0,0 @@
package build
import (
"context"
"io"
"path/filepath"
"strings"
"github.com/docker/buildx/build"
"github.com/docker/buildx/builder"
"github.com/docker/buildx/controller/errdefs"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/store"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/confutil"
"github.com/docker/buildx/util/dockerutil"
"github.com/docker/buildx/util/platformutil"
"github.com/docker/buildx/util/progress"
"github.com/docker/cli/cli/command"
dockeropts "github.com/docker/cli/opts"
"github.com/docker/docker/api/types/container"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/moby/buildkit/util/grpcerrors"
"github.com/pkg/errors"
"google.golang.org/grpc/codes"
)
const defaultTargetName = "default"
// RunBuild runs the specified build and returns the result.
//
// NOTE: When an error happens during the build and this function acquires the debuggable *build.ResultHandle,
// this function returns it in addition to the error (i.e. it does "return nil, res, err"). The caller can
// inspect the result and debug the cause of that error.
func RunBuild(ctx context.Context, dockerCli command.Cli, in *Options, inStream io.Reader, progress progress.Writer, bh *build.Handler) (*client.SolveResponse, *build.Inputs, error) {
if in.NoCache && len(in.NoCacheFilter) > 0 {
return nil, nil, errors.Errorf("--no-cache and --no-cache-filter cannot currently be used together")
}
contexts := map[string]build.NamedContext{}
for name, path := range in.NamedContexts {
contexts[name] = build.NamedContext{Path: path}
}
opts := build.Options{
Inputs: build.Inputs{
ContextPath: in.ContextPath,
DockerfilePath: in.DockerfileName,
InStream: build.NewSyncMultiReader(inStream),
NamedContexts: contexts,
},
Ref: in.Ref,
BuildArgs: in.BuildArgs,
CgroupParent: in.CgroupParent,
ExtraHosts: in.ExtraHosts,
Labels: in.Labels,
NetworkMode: in.NetworkMode,
NoCache: in.NoCache,
NoCacheFilter: in.NoCacheFilter,
Pull: in.Pull,
ShmSize: dockeropts.MemBytes(in.ShmSize),
Tags: in.Tags,
Target: in.Target,
Ulimits: controllerUlimitOpt2DockerUlimit(in.Ulimits),
GroupRef: in.GroupRef,
ProvenanceResponseMode: confutil.ParseMetadataProvenance(in.ProvenanceResponseMode),
}
platforms, err := platformutil.Parse(in.Platforms)
if err != nil {
return nil, nil, err
}
opts.Platforms = platforms
dockerConfig := dockerCli.ConfigFile()
opts.Session = append(opts.Session, authprovider.NewDockerAuthProvider(authprovider.DockerAuthProviderConfig{
ConfigFile: dockerConfig,
}))
secrets, err := controllerapi.CreateSecrets(in.Secrets)
if err != nil {
return nil, nil, err
}
opts.Session = append(opts.Session, secrets)
sshSpecs := in.SSH
if len(sshSpecs) == 0 && buildflags.IsGitSSH(in.ContextPath) {
sshSpecs = append(sshSpecs, &controllerapi.SSH{ID: "default"})
}
ssh, err := controllerapi.CreateSSH(sshSpecs)
if err != nil {
return nil, nil, err
}
opts.Session = append(opts.Session, ssh)
outputs, _, err := controllerapi.CreateExports(in.Exports)
if err != nil {
return nil, nil, err
}
if in.ExportPush {
var pushUsed bool
for i := range outputs {
if outputs[i].Type == client.ExporterImage {
outputs[i].Attrs["push"] = "true"
pushUsed = true
}
}
if !pushUsed {
outputs = append(outputs, client.ExportEntry{
Type: client.ExporterImage,
Attrs: map[string]string{
"push": "true",
},
})
}
}
if in.ExportLoad {
var loadUsed bool
for i := range outputs {
if outputs[i].Type == client.ExporterDocker {
if _, ok := outputs[i].Attrs["dest"]; !ok {
loadUsed = true
break
}
}
}
if !loadUsed {
outputs = append(outputs, client.ExportEntry{
Type: client.ExporterDocker,
Attrs: map[string]string{},
})
}
}
annotations, err := buildflags.ParseAnnotations(in.Annotations)
if err != nil {
return nil, nil, errors.Wrap(err, "parse annotations")
}
for _, o := range outputs {
for k, v := range annotations {
o.Attrs[k.String()] = v
}
}
opts.Exports = outputs
opts.CacheFrom = controllerapi.CreateCaches(in.CacheFrom)
opts.CacheTo = controllerapi.CreateCaches(in.CacheTo)
opts.Attests = controllerapi.CreateAttestations(in.Attests)
opts.SourcePolicy = in.SourcePolicy
allow, err := buildflags.ParseEntitlements(in.Allow)
if err != nil {
return nil, nil, err
}
opts.Allow = allow
if in.CallFunc != nil {
opts.CallFunc = &build.CallFunc{
Name: in.CallFunc.Name,
Format: in.CallFunc.Format,
IgnoreStatus: in.CallFunc.IgnoreStatus,
}
}
// key string used for kubernetes "sticky" mode
contextPathHash, err := filepath.Abs(in.ContextPath)
if err != nil {
contextPathHash = in.ContextPath
}
// TODO: this should not be loaded this side of the controller api
b, err := builder.New(dockerCli,
builder.WithName(in.Builder),
builder.WithContextPathHash(contextPathHash),
)
if err != nil {
return nil, nil, err
}
if err = updateLastActivity(dockerCli, b.NodeGroup); err != nil {
return nil, nil, errors.Wrapf(err, "failed to update builder last activity time")
}
nodes, err := b.LoadNodes(ctx)
if err != nil {
return nil, nil, err
}
var inputs *build.Inputs
buildOptions := map[string]build.Options{defaultTargetName: opts}
resp, err := buildTargets(ctx, dockerCli, nodes, buildOptions, progress, bh)
err = wrapBuildError(err, false)
if err != nil {
return nil, nil, errdefs.WrapBuild(err)
}
if i, ok := buildOptions[defaultTargetName]; ok {
inputs = &i.Inputs
}
return resp, inputs, nil
}
// buildTargets runs the specified build and returns the result.
//
// NOTE: When an error happens during the build and this function acquires the debuggable *build.ResultHandle,
// this function returns it in addition to the error (i.e. it does "return nil, res, err"). The caller can
// inspect the result and debug the cause of that error.
func buildTargets(ctx context.Context, dockerCli command.Cli, nodes []builder.Node, opts map[string]build.Options, progress progress.Writer, bh *build.Handler) (*client.SolveResponse, error) {
resp, err := build.BuildWithResultHandler(ctx, nodes, opts, dockerutil.NewClient(dockerCli), confutil.NewConfig(dockerCli), progress, bh)
if err != nil {
return nil, err
}
return resp[defaultTargetName], err
}
func wrapBuildError(err error, bake bool) error {
if err == nil {
return nil
}
st, ok := grpcerrors.AsGRPCStatus(err)
if ok {
if st.Code() == codes.Unimplemented && strings.Contains(st.Message(), "unsupported frontend capability moby.buildkit.frontend.contexts") {
msg := "current frontend does not support --build-context."
if bake {
msg = "current frontend does not support defining additional contexts for targets."
}
msg += " Named contexts are supported since Dockerfile v1.4. Use #syntax directive in Dockerfile or update to latest BuildKit."
return &wrapped{err, msg}
}
}
return err
}
type wrapped struct {
err error
msg string
}
func (w *wrapped) Error() string {
return w.msg
}
func (w *wrapped) Unwrap() error {
return w.err
}
func updateLastActivity(dockerCli command.Cli, ng *store.NodeGroup) error {
txn, release, err := storeutil.GetStore(dockerCli)
if err != nil {
return err
}
defer release()
return txn.UpdateLastActivity(ng)
}
func controllerUlimitOpt2DockerUlimit(u *controllerapi.UlimitOpt) *dockeropts.UlimitOpt {
if u == nil {
return nil
}
values := make(map[string]*container.Ulimit)
for k, v := range u.Values {
values[k] = &container.Ulimit{
Name: v.Name,
Hard: v.Hard,
Soft: v.Soft,
}
}
return dockeropts.NewUlimitOpt(&values)
}
controller/build/options.go
-216
View File
@@ -1,216 +0,0 @@
package build
import (
"path/filepath"
"strings"
"github.com/docker/buildx/controller/pb"
sourcepolicy "github.com/moby/buildkit/sourcepolicy/pb"
"github.com/moby/buildkit/util/gitutil"
)
type Options struct {
ContextPath string
DockerfileName string
CallFunc *pb.CallFunc
NamedContexts map[string]string
Allow []string
Attests []*pb.Attest
BuildArgs map[string]string
CacheFrom []*pb.CacheOptionsEntry
CacheTo []*pb.CacheOptionsEntry
CgroupParent string
Exports []*pb.ExportEntry
ExtraHosts []string
Labels map[string]string
NetworkMode string
NoCacheFilter []string
Platforms []string
Secrets []*pb.Secret
ShmSize int64
SSH []*pb.SSH
Tags []string
Target string
Ulimits *pb.UlimitOpt
Builder string
NoCache bool
Pull bool
ExportPush bool
ExportLoad bool
SourcePolicy *sourcepolicy.Policy
Ref string
GroupRef string
Annotations []string
ProvenanceResponseMode string
}
// ResolveOptionPaths resolves all paths contained in BuildOptions
// and replaces them to absolute paths.
func ResolveOptionPaths(options *Options) (_ *Options, err error) {
localContext := false
if options.ContextPath != "" && options.ContextPath != "-" {
if !isRemoteURL(options.ContextPath) {
localContext = true
options.ContextPath, err = filepath.Abs(options.ContextPath)
if err != nil {
return nil, err
}
}
}
if options.DockerfileName != "" && options.DockerfileName != "-" {
if localContext && !isHTTPURL(options.DockerfileName) {
options.DockerfileName, err = filepath.Abs(options.DockerfileName)
if err != nil {
return nil, err
}
}
}
var contexts map[string]string
for k, v := range options.NamedContexts {
if isRemoteURL(v) || strings.HasPrefix(v, "docker-image://") {
// url prefix, this is a remote path
} else if p, ok := strings.CutPrefix(v, "oci-layout://"); ok {
// oci layout prefix, this is a local path
p, err = filepath.Abs(p)
if err != nil {
return nil, err
}
v = "oci-layout://" + p
} else {
// no prefix, assume local path
v, err = filepath.Abs(v)
if err != nil {
return nil, err
}
}
if contexts == nil {
contexts = make(map[string]string)
}
contexts[k] = v
}
options.NamedContexts = contexts
var cacheFrom []*pb.CacheOptionsEntry
for _, co := range options.CacheFrom {
switch co.Type {
case "local":
var attrs map[string]string
for k, v := range co.Attrs {
if attrs == nil {
attrs = make(map[string]string)
}
switch k {
case "src":
p := v
if p != "" {
p, err = filepath.Abs(p)
if err != nil {
return nil, err
}
}
attrs[k] = p
default:
attrs[k] = v
}
}
co.Attrs = attrs
cacheFrom = append(cacheFrom, co)
default:
cacheFrom = append(cacheFrom, co)
}
}
options.CacheFrom = cacheFrom
var cacheTo []*pb.CacheOptionsEntry
for _, co := range options.CacheTo {
switch co.Type {
case "local":
var attrs map[string]string
for k, v := range co.Attrs {
if attrs == nil {
attrs = make(map[string]string)
}
switch k {
case "dest":
p := v
if p != "" {
p, err = filepath.Abs(p)
if err != nil {
return nil, err
}
}
attrs[k] = p
default:
attrs[k] = v
}
}
co.Attrs = attrs
cacheTo = append(cacheTo, co)
default:
cacheTo = append(cacheTo, co)
}
}
options.CacheTo = cacheTo
var exports []*pb.ExportEntry
for _, e := range options.Exports {
if e.Destination != "" && e.Destination != "-" {
e.Destination, err = filepath.Abs(e.Destination)
if err != nil {
return nil, err
}
}
exports = append(exports, e)
}
options.Exports = exports
var secrets []*pb.Secret
for _, s := range options.Secrets {
if s.FilePath != "" {
s.FilePath, err = filepath.Abs(s.FilePath)
if err != nil {
return nil, err
}
}
secrets = append(secrets, s)
}
options.Secrets = secrets
var ssh []*pb.SSH
for _, s := range options.SSH {
var ps []string
for _, pt := range s.Paths {
p := pt
if p != "" {
p, err = filepath.Abs(p)
if err != nil {
return nil, err
}
}
ps = append(ps, p)
}
s.Paths = ps
ssh = append(ssh, s)
}
options.SSH = ssh
return options, nil
}
// isHTTPURL returns true if the provided str is an HTTP(S) URL by checking if it
// has a http:// or https:// scheme. No validation is performed to verify if the
// URL is well-formed.
func isHTTPURL(str string) bool {
return strings.HasPrefix(str, "https://") || strings.HasPrefix(str, "http://")
}
func isRemoteURL(c string) bool {
if isHTTPURL(c) {
return true
}
if _, err := gitutil.ParseGitRef(c); err == nil {
return true
}
return false
}
controller/build/options_test.go
-249
View File
@@ -1,249 +0,0 @@
package build
import (
"os"
"path/filepath"
"testing"
"github.com/docker/buildx/controller/pb"
"github.com/stretchr/testify/require"
)
func TestResolvePaths(t *testing.T) {
tmpwd, err := os.MkdirTemp("", "testresolvepaths")
require.NoError(t, err)
defer os.Remove(tmpwd)
require.NoError(t, os.Chdir(tmpwd))
tests := []struct {
name string
options *Options
want *Options
}{
{
name: "contextpath",
options: &Options{ContextPath: "test"},
want: &Options{ContextPath: filepath.Join(tmpwd, "test")},
},
{
name: "contextpath-cwd",
options: &Options{ContextPath: "."},
want: &Options{ContextPath: tmpwd},
},
{
name: "contextpath-dash",
options: &Options{ContextPath: "-"},
want: &Options{ContextPath: "-"},
},
{
name: "contextpath-ssh",
options: &Options{ContextPath: "git@github.com:docker/buildx.git"},
want: &Options{ContextPath: "git@github.com:docker/buildx.git"},
},
{
name: "dockerfilename",
options: &Options{DockerfileName: "test", ContextPath: "."},
want: &Options{DockerfileName: filepath.Join(tmpwd, "test"), ContextPath: tmpwd},
},
{
name: "dockerfilename-dash",
options: &Options{DockerfileName: "-", ContextPath: "."},
want: &Options{DockerfileName: "-", ContextPath: tmpwd},
},
{
name: "dockerfilename-remote",
options: &Options{DockerfileName: "test", ContextPath: "git@github.com:docker/buildx.git"},
want: &Options{DockerfileName: "test", ContextPath: "git@github.com:docker/buildx.git"},
},
{
name: "contexts",
options: &Options{NamedContexts: map[string]string{
"a": "test1", "b": "test2",
"alpine": "docker-image://alpine@sha256:0123456789", "project": "https://github.com/myuser/project.git",
}},
want: &Options{NamedContexts: map[string]string{
"a": filepath.Join(tmpwd, "test1"), "b": filepath.Join(tmpwd, "test2"),
"alpine": "docker-image://alpine@sha256:0123456789", "project": "https://github.com/myuser/project.git",
}},
},
{
name: "cache-from",
options: &Options{
CacheFrom: []*pb.CacheOptionsEntry{
{
Type: "local",
Attrs: map[string]string{"src": "test"},
},
{
Type: "registry",
Attrs: map[string]string{"ref": "user/app"},
},
},
},
want: &Options{
CacheFrom: []*pb.CacheOptionsEntry{
{
Type: "local",
Attrs: map[string]string{"src": filepath.Join(tmpwd, "test")},
},
{
Type: "registry",
Attrs: map[string]string{"ref": "user/app"},
},
},
},
},
{
name: "cache-to",
options: &Options{
CacheTo: []*pb.CacheOptionsEntry{
{
Type: "local",
Attrs: map[string]string{"dest": "test"},
},
{
Type: "registry",
Attrs: map[string]string{"ref": "user/app"},
},
},
},
want: &Options{
CacheTo: []*pb.CacheOptionsEntry{
{
Type: "local",
Attrs: map[string]string{"dest": filepath.Join(tmpwd, "test")},
},
{
Type: "registry",
Attrs: map[string]string{"ref": "user/app"},
},
},
},
},
{
name: "exports",
options: &Options{
Exports: []*pb.ExportEntry{
{
Type: "local",
Destination: "-",
},
{
Type: "local",
Destination: "test1",
},
{
Type: "tar",
Destination: "test3",
},
{
Type: "oci",
Destination: "-",
},
{
Type: "docker",
Destination: "test4",
},
{
Type: "image",
Attrs: map[string]string{"push": "true"},
},
},
},
want: &Options{
Exports: []*pb.ExportEntry{
{
Type: "local",
Destination: "-",
},
{
Type: "local",
Destination: filepath.Join(tmpwd, "test1"),
},
{
Type: "tar",
Destination: filepath.Join(tmpwd, "test3"),
},
{
Type: "oci",
Destination: "-",
},
{
Type: "docker",
Destination: filepath.Join(tmpwd, "test4"),
},
{
Type: "image",
Attrs: map[string]string{"push": "true"},
},
},
},
},
{
name: "secrets",
options: &Options{
Secrets: []*pb.Secret{
{
FilePath: "test1",
},
{
ID: "val",
Env: "a",
},
{
ID: "test",
FilePath: "test3",
},
},
},
want: &Options{
Secrets: []*pb.Secret{
{
FilePath: filepath.Join(tmpwd, "test1"),
},
{
ID: "val",
Env: "a",
},
{
ID: "test",
FilePath: filepath.Join(tmpwd, "test3"),
},
},
},
},
{
name: "ssh",
options: &Options{
SSH: []*pb.SSH{
{
ID: "default",
Paths: []string{"test1", "test2"},
},
{
ID: "a",
Paths: []string{"test3"},
},
},
},
want: &Options{
SSH: []*pb.SSH{
{
ID: "default",
Paths: []string{filepath.Join(tmpwd, "test1"), filepath.Join(tmpwd, "test2")},
},
{
ID: "a",
Paths: []string{filepath.Join(tmpwd, "test3")},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ResolveOptionPaths(tt.options)
require.NoError(t, err)
require.Equal(t, tt.want, got)
})
}
}
controller/errdefs/build.go
-20
View File
@@ -1,20 +0,0 @@
package errdefs
type BuildError struct {
err error
}
func (e *BuildError) Unwrap() error {
return e.err
}
func (e *BuildError) Error() string {
return e.err.Error()
}
func WrapBuild(err error) error {
if err == nil {
return nil
}
return &BuildError{err: err}
}
controller/pb/attest.go
-26
View File
@@ -1,26 +0,0 @@
package pb
type Attest struct {
Type string
Disabled bool
Attrs string
}
func CreateAttestations(attests []*Attest) map[string]*string {
result := map[string]*string{}
for _, attest := range attests {
// ignore duplicates
if _, ok := result[attest.Type]; ok {
continue
}
if attest.Disabled {
result[attest.Type] = nil
continue
}
attrs := attest.Attrs
result[attest.Type] = &attrs
}
return result
}
controller/pb/cache.go
-28
View File
@@ -1,28 +0,0 @@
package pb
import (
"maps"
"github.com/moby/buildkit/client"
)
type CacheOptionsEntry struct {
Type string
Attrs map[string]string
}
func CreateCaches(entries []*CacheOptionsEntry) []client.CacheOptionsEntry {
var outs []client.CacheOptionsEntry
if len(entries) == 0 {
return nil
}
for _, entry := range entries {
out := client.CacheOptionsEntry{
Type: entry.Type,
Attrs: map[string]string{},
}
maps.Copy(out.Attrs, entry.Attrs)
outs = append(outs, out)
}
return outs
}
controller/pb/export.go
-114
View File
@@ -1,114 +0,0 @@
package pb
import (
"io"
"maps"
"os"
"strconv"
"github.com/containerd/console"
"github.com/moby/buildkit/client"
"github.com/pkg/errors"
)
type ExportEntry struct {
Type string
Attrs map[string]string
Destination string
}
func CreateExports(entries []*ExportEntry) ([]client.ExportEntry, []string, error) {
var outs []client.ExportEntry
var localPaths []string
if len(entries) == 0 {
return nil, nil, nil
}
var stdoutUsed bool
for _, entry := range entries {
if entry.Type == "" {
return nil, nil, errors.Errorf("type is required for output")
}
out := client.ExportEntry{
Type: entry.Type,
Attrs: map[string]string{},
}
maps.Copy(out.Attrs, entry.Attrs)
supportFile := false
supportDir := false
switch out.Type {
case client.ExporterLocal:
supportDir = true
case client.ExporterTar:
supportFile = true
case client.ExporterOCI, client.ExporterDocker:
tar, err := strconv.ParseBool(out.Attrs["tar"])
if err != nil {
tar = true
}
supportFile = tar
supportDir = !tar
case "registry":
out.Type = client.ExporterImage
out.Attrs["push"] = "true"
}
if supportDir {
if entry.Destination == "" {
return nil, nil, errors.Errorf("dest is required for %s exporter", out.Type)
}
if entry.Destination == "-" {
return nil, nil, errors.Errorf("dest cannot be stdout for %s exporter", out.Type)
}
fi, err := os.Stat(entry.Destination)
if err != nil && !os.IsNotExist(err) {
return nil, nil, errors.Wrapf(err, "invalid destination directory: %s", entry.Destination)
}
if err == nil && !fi.IsDir() {
return nil, nil, errors.Errorf("destination directory %s is a file", entry.Destination)
}
out.OutputDir = entry.Destination
localPaths = append(localPaths, entry.Destination)
}
if supportFile {
if entry.Destination == "" && out.Type != client.ExporterDocker {
entry.Destination = "-"
}
if entry.Destination == "-" {
if stdoutUsed {
return nil, nil, errors.Errorf("multiple outputs configured to write to stdout")
}
if _, err := console.ConsoleFromFile(os.Stdout); err == nil {
return nil, nil, errors.Errorf("dest file is required for %s exporter. refusing to write to console", out.Type)
}
out.Output = wrapWriteCloser(os.Stdout)
stdoutUsed = true
} else if entry.Destination != "" {
fi, err := os.Stat(entry.Destination)
if err != nil && !os.IsNotExist(err) {
return nil, nil, errors.Wrapf(err, "invalid destination file: %s", entry.Destination)
}
if err == nil && fi.IsDir() {
return nil, nil, errors.Errorf("destination file %s is a directory", entry.Destination)
}
f, err := os.Create(entry.Destination)
if err != nil {
return nil, nil, errors.Errorf("failed to open %s", err)
}
out.Output = wrapWriteCloser(f)
localPaths = append(localPaths, entry.Destination)
}
}
outs = append(outs, out)
}
return outs, localPaths, nil
}
func wrapWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
return func(map[string]string) (io.WriteCloser, error) {
return wc, nil
}
}
controller/pb/invoke.go
-56
View File
@@ -1,56 +0,0 @@
package pb
import (
"fmt"
"strings"
)
type CallFunc struct {
Name string
Format string
IgnoreStatus bool
}
func (x *CallFunc) String() string {
var elems []string
if x.Name != "" {
elems = append(elems, fmt.Sprintf("Name:%q", x.Name))
}
if x.Format != "" {
elems = append(elems, fmt.Sprintf("Format:%q", x.Format))
}
if x.IgnoreStatus {
elems = append(elems, fmt.Sprintf("IgnoreStatus:%v", x.IgnoreStatus))
}
return strings.Join(elems, " ")
}
type InvokeConfig struct {
Entrypoint []string
Cmd []string
NoCmd bool
Env []string
User string
NoUser bool
Cwd string
NoCwd bool
Tty bool
Rollback bool
Initial bool
SuspendOn SuspendOn
}
func (cfg *InvokeConfig) NeedsDebug(err error) bool {
return cfg.SuspendOn.DebugEnabled(err)
}
type SuspendOn int
const (
SuspendError SuspendOn = iota
SuspendAlways
)
func (s SuspendOn) DebugEnabled(err error) bool {
return err != nil || s == SuspendAlways
}
controller/pb/secrets.go
-28
View File
@@ -1,28 +0,0 @@
package pb
import (
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/secrets/secretsprovider"
)
type Secret struct {
ID string
FilePath string
Env string
}
func CreateSecrets(secrets []*Secret) (session.Attachable, error) {
fs := make([]secretsprovider.Source, 0, len(secrets))
for _, secret := range secrets {
fs = append(fs, secretsprovider.Source{
ID: secret.ID,
FilePath: secret.FilePath,
Env: secret.Env,
})
}
store, err := secretsprovider.NewStore(fs)
if err != nil {
return nil, err
}
return secretsprovider.NewSecretProvider(store), nil
}
controller/pb/ssh.go
-25
View File
@@ -1,25 +0,0 @@
package pb
import (
"slices"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/sshforward/sshprovider"
)
type SSH struct {
ID string
Paths []string
}
func CreateSSH(ssh []*SSH) (session.Attachable, error) {
configs := make([]sshprovider.AgentConfig, 0, len(ssh))
for _, ssh := range ssh {
cfg := sshprovider.AgentConfig{
ID: ssh.ID,
Paths: slices.Clone(ssh.Paths),
}
configs = append(configs, cfg)
}
return sshprovider.NewSSHAgentProvider(configs)
}
controller/pb/ulimit.go
-11
View File
@@ -1,11 +0,0 @@
package pb
type UlimitOpt struct {
Values map[string]*Ulimit
}
type Ulimit struct {
Name string
Hard int64
Soft int64
}
docs/debugging.md
-41
View File
@@ -26,7 +26,6 @@ Arguments available after `buildx debug build` are the same as the normal `build
```console
$ docker buildx debug --invoke /bin/sh build .
[+] Building 4.2s (19/19) FINISHED
=> [internal] connecting to local controller 0.0s
=> [internal] load build definition from Dockerfile 0.0s
=> => transferring dockerfile: 32B 0.0s
=> [internal] load .dockerignore 0.0s
@@ -68,7 +67,6 @@ If you want to start a debug session when a build fails, you can use
```console
$ docker buildx debug --on=error build .
[+] Building 4.2s (19/19) FINISHED
=> [internal] connecting to local controller 0.0s
=> [internal] load build definition from Dockerfile 0.0s
=> => transferring dockerfile: 32B 0.0s
=> [internal] load .dockerignore 0.0s
@@ -94,7 +92,6 @@ can use `buildx debug` command to start a debug session.
```
$ docker buildx debug
[+] Building 4.2s (19/19) FINISHED
=> [internal] connecting to local controller 0.0s
(buildx)
```
@@ -125,41 +122,3 @@ Available commands are:
rollback re-runs the interactive container with the step's rootfs contents
```
## Build controllers
Debugging is performed using a buildx "controller", which provides a high-level
abstraction to perform builds. By default, the local controller is used for a
more stable experience which runs all builds in-process. However, you can also
use the remote controller to detach the build process from the CLI.
To detach the build process from the CLI, you can use the `--detach=true` flag with
the build command.
```console
$ docker buildx debug --invoke /bin/sh build --detach=true .
```
If you start a debugging session using the `--invoke` flag with a detached
build, then you can attach to it using the `buildx debug` command to
immediately enter the monitor mode.
```console
$ docker buildx debug
[+] Building 0.0s (1/1) FINISHED
=> [internal] connecting to remote controller
(buildx) list
ID CURRENT_SESSION
xfe1162ovd9def8yapb4ys66t false
(buildx) attach xfe1162ovd9def8yapb4ys66t
Attached to process "". Press Ctrl-a-c to switch to the new container
(buildx) ps
PID CURRENT_SESSION COMMAND
3ug8iqaufiwwnukimhqqt06jz false [sh]
(buildx) attach 3ug8iqaufiwwnukimhqqt06jz
Attached to process "3ug8iqaufiwwnukimhqqt06jz". Press Ctrl-a-c to switch to the new container
(buildx) Switched IO
/ # ls
bin etc lib mnt proc run srv tmp var
dev home media opt root sbin sys usr work
/ #
```
monitor/commands/exec.go
+4 -4
View File
@@ -5,7 +5,7 @@ import (
"fmt"
"io"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/build"
"github.com/docker/buildx/monitor/types"
"github.com/pkg/errors"
)
@@ -13,11 +13,11 @@ import (
type ExecCmd struct {
m types.Monitor
invokeConfig *controllerapi.InvokeConfig
invokeConfig *build.InvokeConfig
stdout io.WriteCloser
}
func NewExecCmd(m types.Monitor, invokeConfig *controllerapi.InvokeConfig, stdout io.WriteCloser) types.Command {
func NewExecCmd(m types.Monitor, invokeConfig *build.InvokeConfig, stdout io.WriteCloser) types.Command {
return &ExecCmd{m, invokeConfig, stdout}
}
@@ -38,7 +38,7 @@ func (cm *ExecCmd) Exec(ctx context.Context, args []string) error {
if len(args) < 2 {
return errors.Errorf("command must be passed")
}
cfg := &controllerapi.InvokeConfig{
cfg := &build.InvokeConfig{
Entrypoint: []string{args[1]},
Cmd: args[2:],
NoCmd: false,
monitor/commands/rollback.go
+3 -3
View File
@@ -5,18 +5,18 @@ import (
"fmt"
"io"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/build"
"github.com/docker/buildx/monitor/types"
)
type RollbackCmd struct {
m types.Monitor
invokeConfig *controllerapi.InvokeConfig
invokeConfig *build.InvokeConfig
stdout io.WriteCloser
}
func NewRollbackCmd(m types.Monitor, invokeConfig *controllerapi.InvokeConfig, stdout io.WriteCloser) types.Command {
func NewRollbackCmd(m types.Monitor, invokeConfig *build.InvokeConfig, stdout io.WriteCloser) types.Command {
return &RollbackCmd{m, invokeConfig, stdout}
}
monitor/monitor.go
+10 -11
View File
@@ -12,9 +12,8 @@ import (
"github.com/containerd/console"
"github.com/docker/buildx/build"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/controller/processes"
"github.com/docker/buildx/monitor/commands"
"github.com/docker/buildx/monitor/processes"
"github.com/docker/buildx/monitor/types"
"github.com/docker/buildx/util/ioset"
"github.com/docker/buildx/util/progress"
@@ -29,7 +28,7 @@ import (
var ErrReload = errors.New("monitor: reload")
type Monitor struct {
invokeConfig *controllerapi.InvokeConfig
invokeConfig *build.InvokeConfig
printer *progress.Printer
stdin *ioset.SingleForwarder
@@ -41,7 +40,7 @@ type Monitor struct {
mu sync.Mutex
}
func New(cfg *controllerapi.InvokeConfig, stdin io.ReadCloser, stdout, stderr io.WriteCloser, printer *progress.Printer) *Monitor {
func New(cfg *build.InvokeConfig, stdin io.ReadCloser, stdout, stderr io.WriteCloser, printer *progress.Printer) *Monitor {
m := &Monitor{
invokeConfig: cfg,
printer: printer,
@@ -113,7 +112,7 @@ func (m *Monitor) Close() error {
}
// RunMonitor provides an interactive session for running and managing containers via specified IO.
func RunMonitor(ctx context.Context, invokeConfig *controllerapi.InvokeConfig, rCtx *build.ResultHandle, stdin io.ReadCloser, stdout, stderr io.WriteCloser, progress *progress.Printer) error {
func RunMonitor(ctx context.Context, invokeConfig *build.InvokeConfig, rCtx *build.ResultHandle, stdin io.ReadCloser, stdout, stderr io.WriteCloser, progress *progress.Printer) error {
if err := progress.Pause(); err != nil {
return err
}
@@ -333,7 +332,7 @@ type monitor struct {
processes *processes.Manager
}
func (m *monitor) Invoke(ctx context.Context, pid string, cfg *controllerapi.InvokeConfig, ioIn io.ReadCloser, ioOut io.WriteCloser, ioErr io.WriteCloser) error {
func (m *monitor) Invoke(ctx context.Context, pid string, cfg *build.InvokeConfig, ioIn io.ReadCloser, ioOut io.WriteCloser, ioErr io.WriteCloser) error {
proc, ok := m.processes.Get(pid)
if !ok {
// Start a new process.
@@ -361,19 +360,19 @@ func (m *monitor) Invoke(ctx context.Context, pid string, cfg *controllerapi.Inv
}
}
func (m *monitor) Rollback(ctx context.Context, cfg *controllerapi.InvokeConfig) string {
func (m *monitor) Rollback(ctx context.Context, cfg *build.InvokeConfig) string {
pid := identity.NewID()
cfg1 := cfg
cfg1.Rollback = true
return m.startInvoke(ctx, pid, cfg1)
}
func (m *monitor) Exec(ctx context.Context, cfg *controllerapi.InvokeConfig) string {
func (m *monitor) Exec(ctx context.Context, cfg *build.InvokeConfig) string {
return m.startInvoke(ctx, identity.NewID(), cfg)
}
func (m *monitor) Attach(ctx context.Context, pid string) {
m.startInvoke(ctx, pid, &controllerapi.InvokeConfig{})
m.startInvoke(ctx, pid, &build.InvokeConfig{})
}
func (m *monitor) Detach() {
@@ -394,7 +393,7 @@ func (m *monitor) close() {
m.Detach()
}
func (m *monitor) startInvoke(ctx context.Context, pid string, cfg *controllerapi.InvokeConfig) string {
func (m *monitor) startInvoke(ctx context.Context, pid string, cfg *build.InvokeConfig) string {
if m.invokeCancel != nil {
m.invokeCancel() // Finish existing attach
}
@@ -420,7 +419,7 @@ func (m *monitor) startInvoke(ctx context.Context, pid string, cfg *controllerap
return pid
}
func (m *monitor) invoke(ctx context.Context, pid string, cfg *controllerapi.InvokeConfig) error {
func (m *monitor) invoke(ctx context.Context, pid string, cfg *build.InvokeConfig) error {
m.muxIO.Enable(1)
defer m.muxIO.Disable(1)
if err := m.muxIO.SwitchTo(1); err != nil {
controller/processes/processes.go → monitor/processes/processes.go
+3 -4
View File
@@ -6,7 +6,6 @@ import (
"sync/atomic"
"github.com/docker/buildx/build"
"github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/util/ioset"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ import (
// Process provides methods to control a process.
type Process struct {
inEnd *ioset.Forwarder
invokeConfig *pb.InvokeConfig
invokeConfig *build.InvokeConfig
errCh chan error
processCancel func()
serveIOCancel func(error)
@@ -98,7 +97,7 @@ func (m *Manager) DeleteProcess(id string) error {
// When a container isn't available (i.e. first time invoking or the container has exited) or cfg.Rollback is set,
// this method will start a new container and run the process in it. Otherwise, this method starts a new process in the
// existing container.
func (m *Manager) StartProcess(pid string, resultCtx *build.ResultHandle, cfg *pb.InvokeConfig) (*Process, error) {
func (m *Manager) StartProcess(pid string, resultCtx *build.ResultHandle, cfg *build.InvokeConfig) (*Process, error) {
// Get the target result to invoke a container from
var ctr *build.Container
if a := m.container.Load(); a != nil {
@@ -157,5 +156,5 @@ func (m *Manager) StartProcess(pid string, resultCtx *build.ResultHandle, cfg *p
type ProcessInfo struct {
ProcessID string
InvokeConfig *pb.InvokeConfig
InvokeConfig *build.InvokeConfig
}
monitor/types/types.go
+5 -5
View File
@@ -4,8 +4,8 @@ import (
"context"
"io"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/controller/processes"
"github.com/docker/buildx/build"
"github.com/docker/buildx/monitor/processes"
)
// Monitor provides APIs for attaching and controlling the buildx server.
@@ -14,17 +14,17 @@ type Monitor interface {
// If pid doesn't match to any running processes, it starts a new process with the specified config.
// If there is no container running or InvokeConfig.Rollback is specified, the process will start in a newly created container.
// NOTE: If needed, in the future, we can split this API into three APIs (NewContainer, NewProcess and Attach).
Invoke(ctx context.Context, pid string, options *controllerapi.InvokeConfig, ioIn io.ReadCloser, ioOut io.WriteCloser, ioErr io.WriteCloser) error
Invoke(ctx context.Context, pid string, options *build.InvokeConfig, ioIn io.ReadCloser, ioOut io.WriteCloser, ioErr io.WriteCloser) error
ListProcesses(ctx context.Context) (infos []*processes.ProcessInfo, retErr error)
DisconnectProcess(ctx context.Context, pid string) error
// Rollback re-runs the interactive container with initial rootfs contents.
Rollback(ctx context.Context, cfg *controllerapi.InvokeConfig) string
Rollback(ctx context.Context, cfg *build.InvokeConfig) string
// Rollback executes a process in the interactive container.
Exec(ctx context.Context, cfg *controllerapi.InvokeConfig) string
Exec(ctx context.Context, cfg *build.InvokeConfig) string
// Attach attaches IO to a process in the container.
Attach(ctx context.Context, pid string)
util/buildflags/attests.go
+26 -87
View File
@@ -7,7 +7,6 @@ import (
"strconv"
"strings"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/pkg/errors"
"github.com/tonistiigi/go-csvvalue"
)
@@ -33,16 +32,32 @@ func (a Attests) Normalize() Attests {
return removeAttestDupes(a)
}
func (a Attests) ToPB() []*controllerapi.Attest {
if len(a) == 0 {
return nil
}
func (a Attests) ToMap() map[string]*string {
result := map[string]*string{}
for _, attest := range a {
// ignore duplicates
if _, ok := result[attest.Type]; ok {
continue
}
entries := make([]*controllerapi.Attest, len(a))
for i, entry := range a {
entries[i] = entry.ToPB()
if attest.Disabled {
result[attest.Type] = nil
continue
}
var b csvBuilder
if attest.Type != "" {
b.Write("type", attest.Type)
}
if attest.Disabled {
b.Write("disabled", "true")
}
b.WriteAttributes(attest.Attrs)
s := b.String()
result[attest.Type] = &s
}
return entries
return result
}
type Attest struct {
@@ -72,23 +87,6 @@ func (a *Attest) String() string {
return b.String()
}
func (a *Attest) ToPB() *controllerapi.Attest {
var b csvBuilder
if a.Type != "" {
b.Write("type", a.Type)
}
if a.Disabled {
b.Write("disabled", "true")
}
b.WriteAttributes(a.Attrs)
return &controllerapi.Attest{
Type: a.Type,
Disabled: a.Disabled,
Attrs: b.String(),
}
}
func (a *Attest) MarshalJSON() ([]byte, error) {
m := make(map[string]any, len(a.Attrs)+2)
for k, v := range a.Attrs {
@@ -182,7 +180,7 @@ func CanonicalizeAttest(attestType string, in string) string {
return fmt.Sprintf("type=%s,%s", attestType, in)
}
func ParseAttests(in []string) ([]*controllerapi.Attest, error) {
func ParseAttests(in []string) (Attests, error) {
var outs []*Attest
for _, s := range in {
var out Attest
@@ -191,66 +189,7 @@ func ParseAttests(in []string) ([]*controllerapi.Attest, error) {
}
outs = append(outs, &out)
}
return ConvertAttests(outs)
}
// ConvertAttests converts Attestations for the controller API from
// the ones in this package.
//
// Attestations of the same type will cause an error. Some tools,
// like bake, remove the duplicates before calling this function.
func ConvertAttests(in []*Attest) ([]*controllerapi.Attest, error) {
out := make([]*controllerapi.Attest, 0, len(in))
// Check for duplicate attestations while we convert them
// to the controller API.
found := map[string]struct{}{}
for _, attest := range in {
if _, ok := found[attest.Type]; ok {
return nil, errors.Errorf("duplicate attestation field %s", attest.Type)
}
found[attest.Type] = struct{}{}
out = append(out, attest.ToPB())
}
return out, nil
}
func ParseAttest(in string) (*controllerapi.Attest, error) {
if in == "" {
return nil, nil
}
fields, err := csvvalue.Fields(in, nil)
if err != nil {
return nil, err
}
attest := controllerapi.Attest{
Attrs: in,
}
for _, field := range fields {
key, value, ok := strings.Cut(field, "=")
if !ok {
return nil, errors.Errorf("invalid value %s", field)
}
key = strings.TrimSpace(strings.ToLower(key))
switch key {
case "type":
attest.Type = value
case "disabled":
disabled, err := strconv.ParseBool(value)
if err != nil {
return nil, errors.Wrapf(err, "invalid value %s", field)
}
attest.Disabled = disabled
}
}
if attest.Type == "" {
return nil, errors.Errorf("attestation type not specified")
}
return &attest, nil
return outs, nil
}
func removeAttestDupes(s []*Attest) []*Attest {
util/buildflags/cache.go
-103
View File
@@ -1,15 +1,10 @@
package buildflags
import (
"context"
"encoding/json"
"maps"
"os"
"strconv"
"strings"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/pkg/errors"
"github.com/tonistiigi/go-csvvalue"
"github.com/zclconf/go-cty/cty"
@@ -35,22 +30,6 @@ func (o CacheOptions) Normalize() CacheOptions {
return removeDupes(o)
}
func (o CacheOptions) ToPB() []*controllerapi.CacheOptionsEntry {
if len(o) == 0 {
return nil
}
var outs []*controllerapi.CacheOptionsEntry
for _, entry := range o {
pb := entry.ToPB()
if !isActive(pb) {
continue
}
outs = append(outs, pb)
}
return outs
}
type CacheOptionsEntry struct {
Type string `json:"type"`
Attrs map[string]string `json:"attrs,omitempty"`
@@ -81,16 +60,6 @@ func (e *CacheOptionsEntry) String() string {
return b.String()
}
func (e *CacheOptionsEntry) ToPB() *controllerapi.CacheOptionsEntry {
ci := &controllerapi.CacheOptionsEntry{
Type: e.Type,
Attrs: maps.Clone(e.Attrs),
}
addGithubToken(ci)
addAwsCredentials(ci)
return ci
}
func (e *CacheOptionsEntry) MarshalJSON() ([]byte, error) {
m := maps.Clone(e.Attrs)
if m == nil {
@@ -204,75 +173,3 @@ func ParseCacheEntry(in []string) (CacheOptions, error) {
}
return opts, nil
}
func addGithubToken(ci *controllerapi.CacheOptionsEntry) {
if ci.Type != "gha" {
return
}
version, ok := ci.Attrs["version"]
if !ok {
// https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L19
if v, ok := os.LookupEnv("ACTIONS_CACHE_SERVICE_V2"); ok {
if b, err := strconv.ParseBool(v); err == nil && b {
version = "2"
}
}
}
if _, ok := ci.Attrs["token"]; !ok {
if v, ok := os.LookupEnv("ACTIONS_RUNTIME_TOKEN"); ok {
ci.Attrs["token"] = v
}
}
if _, ok := ci.Attrs["url_v2"]; !ok && version == "2" {
// https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L34-L35
if v, ok := os.LookupEnv("ACTIONS_RESULTS_URL"); ok {
ci.Attrs["url_v2"] = v
}
}
if _, ok := ci.Attrs["url"]; !ok {
// https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L28-L33
if v, ok := os.LookupEnv("ACTIONS_CACHE_URL"); ok {
ci.Attrs["url"] = v
} else if v, ok := os.LookupEnv("ACTIONS_RESULTS_URL"); ok {
ci.Attrs["url"] = v
}
}
}
func addAwsCredentials(ci *controllerapi.CacheOptionsEntry) {
if ci.Type != "s3" {
return
}
_, okAccessKeyID := ci.Attrs["access_key_id"]
_, okSecretAccessKey := ci.Attrs["secret_access_key"]
// If the user provides access_key_id, secret_access_key, do not override the session token.
if okAccessKeyID && okSecretAccessKey {
return
}
ctx := context.TODO()
awsConfig, err := awsconfig.LoadDefaultConfig(ctx)
if err != nil {
return
}
credentials, err := awsConfig.Credentials.Retrieve(ctx)
if err != nil {
return
}
if !okAccessKeyID && credentials.AccessKeyID != "" {
ci.Attrs["access_key_id"] = credentials.AccessKeyID
}
if !okSecretAccessKey && credentials.SecretAccessKey != "" {
ci.Attrs["secret_access_key"] = credentials.SecretAccessKey
}
if _, ok := ci.Attrs["session_token"]; !ok && credentials.SessionToken != "" {
ci.Attrs["session_token"] = credentials.SessionToken
}
}
func isActive(pb *controllerapi.CacheOptionsEntry) bool {
// Always active if not gha.
if pb.Type != "gha" {
return true
}
return pb.Attrs["token"] != "" && (pb.Attrs["url"] != "" || pb.Attrs["url_v2"] != "")
}
util/buildflags/cache_test.go
-32
View File
@@ -4,42 +4,10 @@ import (
"encoding/json"
"testing"
"github.com/docker/buildx/controller/pb"
"github.com/stretchr/testify/require"
"github.com/zclconf/go-cty/cty"
)
func TestCacheOptions_DerivedVars(t *testing.T) {
t.Setenv("ACTIONS_RUNTIME_TOKEN", "sensitive_token")
t.Setenv("ACTIONS_CACHE_URL", "https://cache.github.com")
t.Setenv("AWS_ACCESS_KEY_ID", "definitely_dont_look_here")
t.Setenv("AWS_SECRET_ACCESS_KEY", "hackers_please_dont_steal")
t.Setenv("AWS_SESSION_TOKEN", "not_a_mitm_attack")
cacheFrom, err := ParseCacheEntry([]string{"type=gha", "type=s3,region=us-west-2,bucket=my_bucket,name=my_image"})
require.NoError(t, err)
require.Equal(t, []*pb.CacheOptionsEntry{
{
Type: "gha",
Attrs: map[string]string{
"token": "sensitive_token",
"url": "https://cache.github.com",
},
},
{
Type: "s3",
Attrs: map[string]string{
"region": "us-west-2",
"bucket": "my_bucket",
"name": "my_image",
"access_key_id": "definitely_dont_look_here",
"secret_access_key": "hackers_please_dont_steal",
"session_token": "not_a_mitm_attack",
},
},
}, cacheFrom.ToPB())
}
func TestCacheOptions(t *testing.T) {
t.Run("MarshalJSON", func(t *testing.T) {
cache := CacheOptions{
util/buildflags/callfunc.go
+23 -3
View File
@@ -1,17 +1,37 @@
package buildflags
import (
"fmt"
"strconv"
"strings"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/pkg/errors"
"github.com/tonistiigi/go-csvvalue"
)
const defaultCallFunc = "build"
func ParseCallFunc(str string) (*controllerapi.CallFunc, error) {
type CallFunc struct {
Name string
Format string
IgnoreStatus bool
}
func (x *CallFunc) String() string {
var elems []string
if x.Name != "" {
elems = append(elems, fmt.Sprintf("Name:%q", x.Name))
}
if x.Format != "" {
elems = append(elems, fmt.Sprintf("Format:%q", x.Format))
}
if x.IgnoreStatus {
elems = append(elems, fmt.Sprintf("IgnoreStatus:%v", x.IgnoreStatus))
}
return strings.Join(elems, " ")
}
func ParseCallFunc(str string) (*CallFunc, error) {
if str == "" {
return nil, nil
}
@@ -20,7 +40,7 @@ func ParseCallFunc(str string) (*controllerapi.CallFunc, error) {
if err != nil {
return nil, err
}
f := &controllerapi.CallFunc{}
f := &CallFunc{}
for _, field := range fields {
parts := strings.SplitN(field, "=", 2)
if len(parts) == 2 {
util/buildflags/export.go
+4 -25
View File
@@ -9,7 +9,6 @@ import (
"strings"
"github.com/containerd/platforms"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -38,18 +37,6 @@ func (e Exports) Normalize() Exports {
return removeDupes(e)
}
func (e Exports) ToPB() []*controllerapi.ExportEntry {
if len(e) == 0 {
return nil
}
entries := make([]*controllerapi.ExportEntry, len(e))
for i, entry := range e {
entries[i] = entry.ToPB()
}
return entries
}
type ExportEntry struct {
Type string `json:"type"`
Attrs map[string]string `json:"attrs,omitempty"`
@@ -77,14 +64,6 @@ func (e *ExportEntry) String() string {
return b.String()
}
func (e *ExportEntry) ToPB() *controllerapi.ExportEntry {
return &controllerapi.ExportEntry{
Type: e.Type,
Attrs: maps.Clone(e.Attrs),
Destination: e.Destination,
}
}
func (e *ExportEntry) MarshalJSON() ([]byte, error) {
m := maps.Clone(e.Attrs)
if m == nil {
@@ -164,12 +143,12 @@ func (e *ExportEntry) validate() error {
return nil
}
func ParseExports(inp []string) ([]*controllerapi.ExportEntry, error) {
func ParseExports(inp []string) ([]*ExportEntry, error) {
if len(inp) == 0 {
return nil, nil
}
export := make(Exports, 0, len(inp))
exports := make(Exports, 0, len(inp))
for _, s := range inp {
if s == "" {
continue
@@ -179,9 +158,9 @@ func ParseExports(inp []string) ([]*controllerapi.ExportEntry, error) {
if err := out.UnmarshalText([]byte(s)); err != nil {
return nil, err
}
export = append(export, &out)
exports = append(exports, &out)
}
return export.ToPB(), nil
return exports, nil
}
func ParseAnnotations(inp []string) (map[exptypes.AnnotationKey]string, error) {
util/buildflags/secrets.go
+4 -25
View File
@@ -4,7 +4,6 @@ import (
"encoding/json"
"strings"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/pkg/errors"
"github.com/tonistiigi/go-csvvalue"
)
@@ -30,18 +29,6 @@ func (s Secrets) Normalize() Secrets {
return removeSecretDupes(s)
}
func (s Secrets) ToPB() []*controllerapi.Secret {
if len(s) == 0 {
return nil
}
entries := make([]*controllerapi.Secret, len(s))
for i, entry := range s {
entries[i] = entry.ToPB()
}
return entries
}
type Secret struct {
ID string `json:"id,omitempty"`
FilePath string `json:"src,omitempty"`
@@ -66,14 +53,6 @@ func (s *Secret) String() string {
return b.String()
}
func (s *Secret) ToPB() *controllerapi.Secret {
return &controllerapi.Secret{
ID: s.ID,
FilePath: s.FilePath,
Env: s.Env,
}
}
func (s *Secret) UnmarshalJSON(data []byte) error {
var v struct {
ID string `json:"id,omitempty"`
@@ -132,8 +111,8 @@ func (s *Secret) UnmarshalText(text []byte) error {
return nil
}
func ParseSecretSpecs(sl []string) ([]*controllerapi.Secret, error) {
fs := make([]*controllerapi.Secret, 0, len(sl))
func ParseSecretSpecs(sl []string) (Secrets, error) {
fs := make([]*Secret, 0, len(sl))
for _, v := range sl {
if v == "" {
continue
@@ -148,12 +127,12 @@ func ParseSecretSpecs(sl []string) ([]*controllerapi.Secret, error) {
return fs, nil
}
func parseSecret(value string) (*controllerapi.Secret, error) {
func parseSecret(value string) (*Secret, error) {
var s Secret
if err := s.UnmarshalText([]byte(value)); err != nil {
return nil, err
}
return s.ToPB(), nil
return &s, nil
}
func removeSecretDupes(s []*Secret) []*Secret {
util/buildflags/ssh.go
+3 -23
View File
@@ -6,7 +6,6 @@ import (
"slices"
"strings"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/moby/buildkit/util/gitutil"
)
@@ -31,18 +30,6 @@ func (s SSHKeys) Normalize() SSHKeys {
return removeSSHDupes(s)
}
func (s SSHKeys) ToPB() []*controllerapi.SSH {
if len(s) == 0 {
return nil
}
entries := make([]*controllerapi.SSH, len(s))
for i, entry := range s {
entries[i] = entry.ToPB()
}
return entries
}
type SSH struct {
ID string `json:"id,omitempty" cty:"id"`
Paths []string `json:"paths,omitempty" cty:"paths"`
@@ -70,13 +57,6 @@ func (s *SSH) String() string {
return b.String()
}
func (s *SSH) ToPB() *controllerapi.SSH {
return &controllerapi.SSH{
ID: s.ID,
Paths: s.Paths,
}
}
func (s *SSH) UnmarshalJSON(data []byte) error {
var v struct {
ID string `json:"id,omitempty"`
@@ -103,8 +83,8 @@ func (s *SSH) UnmarshalText(text []byte) error {
return nil
}
func ParseSSHSpecs(sl []string) ([]*controllerapi.SSH, error) {
var outs []*controllerapi.SSH
func ParseSSHSpecs(sl []string) ([]*SSH, error) {
var outs []*SSH
if len(sl) == 0 {
return nil, nil
}
@@ -118,7 +98,7 @@ func ParseSSHSpecs(sl []string) ([]*controllerapi.SSH, error) {
if err := out.UnmarshalText([]byte(s)); err != nil {
return nil, err
}
outs = append(outs, out.ToPB())
outs = append(outs, &out)
}
return outs, nil
}