alextab93/buildx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request #3222 from jsternberg/controller-remove-final

Browse Source 
controller: remove remaining parts of the controller
This commit is contained in:
Tõnis Tiigi
2025-06-05 10:18:40 -07:00
committed by GitHub
parent fb916a960c 1d7cda1232
commit c45185fde0
36 changed files with 659 additions and 1502 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/labeler.yml
-5
View File
@@ -48,11 +48,6 @@ area/cli:
- cmd/**
- commands/**
# Add 'area/controller' label to changes in the controller
area/controller:
- changed-files:
- any-glob-to-any-file: 'controller/**'
# Add 'area/docs' label to markdown files in the docs folder
area/docs:
- changed-files:
PROJECT.md
-1
View File
@@ -79,7 +79,6 @@ Area or component of the project affected. Please note that the table below may
| `area/checks` | Any | `checks` |
| `area/ci` | Any | Project CI |
| `area/cli` | Any | `cli` |
| `area/controller` | Any | `controller` |
| `area/debug` | Any | `debug` |
| `area/dependencies` | Any | Project dependencies |
| `area/dockerfile` | Any | `dockerfile` |
bake/bake.go
+9 -11
View File
@@ -19,7 +19,6 @@ import (
composecli "github.com/compose-spec/compose-go/v2/cli"
"github.com/docker/buildx/bake/hclparser"
"github.com/docker/buildx/build"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/platformutil"
"github.com/docker/buildx/util/progress"
@@ -1439,20 +1438,19 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
})
}
}
secrets = secrets.Normalize()
bo.SecretSpecs = secrets.ToPB()
secretAttachment, err := controllerapi.CreateSecrets(bo.SecretSpecs)
bo.SecretSpecs = secrets.Normalize()
secretAttachment, err := build.CreateSecrets(bo.SecretSpecs)
if err != nil {
return nil, err
}
bo.Session = append(bo.Session, secretAttachment)
bo.SSHSpecs = t.SSH.ToPB()
bo.SSHSpecs = t.SSH
if len(bo.SSHSpecs) == 0 && buildflags.IsGitSSH(bi.ContextPath) || (inp != nil && buildflags.IsGitSSH(inp.URL)) {
bo.SSHSpecs = []*controllerapi.SSH{{ID: "default"}}
bo.SSHSpecs = []*buildflags.SSH{{ID: "default"}}
}
sshAttachment, err := controllerapi.CreateSSH(bo.SSHSpecs)
sshAttachment, err := build.CreateSSH(bo.SSHSpecs)
if err != nil {
return nil, err
}
@@ -1469,13 +1467,13 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
}
if t.CacheFrom != nil {
bo.CacheFrom = controllerapi.CreateCaches(t.CacheFrom.ToPB())
bo.CacheFrom = build.CreateCaches(t.CacheFrom)
}
if t.CacheTo != nil {
bo.CacheTo = controllerapi.CreateCaches(t.CacheTo.ToPB())
bo.CacheTo = build.CreateCaches(t.CacheTo)
}
bo.Exports, bo.ExportsLocalPathsTemporary, err = controllerapi.CreateExports(t.Outputs.ToPB())
bo.Exports, bo.ExportsLocalPathsTemporary, err = build.CreateExports(t.Outputs)
if err != nil {
return nil, err
}
@@ -1490,7 +1488,7 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
}
}
bo.Attests = controllerapi.CreateAttestations(t.Attest.ToPB())
bo.Attests = t.Attest.ToMap()
bo.SourcePolicy, err = build.ReadSourcePolicy()
if err != nil {
bake/entitlements_test.go
+6 -6
View File
@@ -8,7 +8,7 @@ import (
"testing"
"github.com/docker/buildx/build"
"github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/osutil"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/util/entitlements"
@@ -264,7 +264,7 @@ func TestValidateEntitlements(t *testing.T) {
{
name: "SSHMissing",
opt: build.Options{
SSHSpecs: []*pb.SSH{
SSHSpecs: []*buildflags.SSH{
{
ID: "test",
},
@@ -296,7 +296,7 @@ func TestValidateEntitlements(t *testing.T) {
{
name: "SecretFromSubFile",
opt: build.Options{
SecretSpecs: []*pb.Secret{
SecretSpecs: []*buildflags.Secret{
{
FilePath: filepath.Join(dir1, "subfile"),
},
@@ -309,7 +309,7 @@ func TestValidateEntitlements(t *testing.T) {
{
name: "SecretFromEscapeLink",
opt: build.Options{
SecretSpecs: []*pb.Secret{
SecretSpecs: []*buildflags.Secret{
{
FilePath: escapeLink,
},
@@ -325,7 +325,7 @@ func TestValidateEntitlements(t *testing.T) {
{
name: "SecretFromEscapeLinkAllowRoot",
opt: build.Options{
SecretSpecs: []*pb.Secret{
SecretSpecs: []*buildflags.Secret{
{
FilePath: escapeLink,
},
@@ -352,7 +352,7 @@ func TestValidateEntitlements(t *testing.T) {
{
name: "SecretFromEscapeLinkAllowAny",
opt: build.Options{
SecretSpecs: []*pb.Secret{
SecretSpecs: []*buildflags.Secret{
{
FilePath: escapeLink,
},
bake/remote.go
+7 -6
View File
@@ -7,9 +7,10 @@ import (
"os"
"strings"
"github.com/docker/buildx/build"
"github.com/docker/buildx/builder"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/progress"
"github.com/docker/go-units"
"github.com/moby/buildkit/client"
@@ -33,27 +34,27 @@ func ReadRemoteFiles(ctx context.Context, nodes []builder.Node, url string, name
st, ok := dockerui.DetectGitContext(url, false)
if ok {
if ssh, err := controllerapi.CreateSSH([]*controllerapi.SSH{{
if ssh, err := build.CreateSSH([]*buildflags.SSH{{
ID: "default",
Paths: strings.Split(os.Getenv("BUILDX_BAKE_GIT_SSH"), ","),
}}); err == nil {
sessions = append(sessions, ssh)
}
var gitAuthSecrets []*controllerapi.Secret
var gitAuthSecrets []*buildflags.Secret
if _, ok := os.LookupEnv("BUILDX_BAKE_GIT_AUTH_TOKEN"); ok {
gitAuthSecrets = append(gitAuthSecrets, &controllerapi.Secret{
gitAuthSecrets = append(gitAuthSecrets, &buildflags.Secret{
ID: llb.GitAuthTokenKey,
Env: "BUILDX_BAKE_GIT_AUTH_TOKEN",
})
}
if _, ok := os.LookupEnv("BUILDX_BAKE_GIT_AUTH_HEADER"); ok {
gitAuthSecrets = append(gitAuthSecrets, &controllerapi.Secret{
gitAuthSecrets = append(gitAuthSecrets, &buildflags.Secret{
ID: llb.GitAuthHeaderKey,
Env: "BUILDX_BAKE_GIT_AUTH_HEADER",
})
}
if len(gitAuthSecrets) > 0 {
if secrets, err := controllerapi.CreateSecrets(gitAuthSecrets); err == nil {
if secrets, err := build.CreateSecrets(gitAuthSecrets); err == nil {
sessions = append(sessions, secrets)
}
}
build/build.go
+3 -3
View File
@@ -19,8 +19,8 @@ import (
"github.com/containerd/containerd/v2/core/images"
"github.com/distribution/reference"
"github.com/docker/buildx/builder"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/confutil"
"github.com/docker/buildx/util/desktop"
"github.com/docker/buildx/util/dockerutil"
@@ -78,8 +78,8 @@ type Options struct {
NoCacheFilter []string
Platforms []ocispecs.Platform
Pull bool
SecretSpecs []*controllerapi.Secret
SSHSpecs []*controllerapi.SSH
SecretSpecs buildflags.Secrets
SSHSpecs []*buildflags.SSH
ShmSize opts.MemBytes
Tags []string
Target string
build/invoke.go
+33 -4
View File
@@ -8,12 +8,41 @@ import (
"sync/atomic"
"syscall"
controllerapi "github.com/docker/buildx/controller/pb"
gateway "github.com/moby/buildkit/frontend/gateway/client"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
type InvokeConfig struct {
Entrypoint []string
Cmd []string
NoCmd bool
Env []string
User string
NoUser bool
Cwd string
NoCwd bool
Tty bool
Rollback bool
Initial bool
SuspendOn SuspendOn
}
func (cfg *InvokeConfig) NeedsDebug(err error) bool {
return cfg.SuspendOn.DebugEnabled(err)
}
type SuspendOn int
const (
SuspendError SuspendOn = iota
SuspendAlways
)
func (s SuspendOn) DebugEnabled(err error) bool {
return err != nil || s == SuspendAlways
}
type Container struct {
cancelOnce sync.Once
containerCancel func(error)
@@ -24,7 +53,7 @@ type Container struct {
resultCtx *ResultHandle
}
func NewContainer(ctx context.Context, resultCtx *ResultHandle, cfg *controllerapi.InvokeConfig) (*Container, error) {
func NewContainer(ctx context.Context, resultCtx *ResultHandle, cfg *InvokeConfig) (*Container, error) {
mainCtx := ctx
ctrCh := make(chan *Container)
@@ -97,7 +126,7 @@ func (c *Container) markUnavailable() {
c.isUnavailable.Store(true)
}
func (c *Container) Exec(ctx context.Context, cfg *controllerapi.InvokeConfig, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
func (c *Container) Exec(ctx context.Context, cfg *InvokeConfig, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
if isInit := c.initStarted.CompareAndSwap(false, true); isInit {
defer func() {
// container can't be used after init exits
@@ -112,7 +141,7 @@ func (c *Container) Exec(ctx context.Context, cfg *controllerapi.InvokeConfig, s
return err
}
func exec(ctx context.Context, resultCtx *ResultHandle, cfg *controllerapi.InvokeConfig, ctr gateway.Container, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
func exec(ctx context.Context, resultCtx *ResultHandle, cfg *InvokeConfig, ctr gateway.Container, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error {
processCfg, err := resultCtx.getProcessConfig(cfg, stdin, stdout, stderr)
if err != nil {
return err
build/opt.go
+225
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"io"
"maps"
"os"
"path/filepath"
"slices"
@@ -11,12 +12,15 @@ import (
"strings"
"syscall"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/containerd/console"
"github.com/containerd/containerd/v2/core/content"
"github.com/containerd/containerd/v2/plugins/content/local"
"github.com/containerd/platforms"
"github.com/distribution/reference"
"github.com/docker/buildx/builder"
"github.com/docker/buildx/driver"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/confutil"
"github.com/docker/buildx/util/dockerutil"
"github.com/docker/buildx/util/osutil"
@@ -26,6 +30,9 @@ import (
"github.com/moby/buildkit/client/ociindex"
gateway "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/secrets/secretsprovider"
"github.com/moby/buildkit/session/sshforward/sshprovider"
"github.com/moby/buildkit/session/upload/uploadprovider"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/apicaps"
@@ -659,3 +666,221 @@ type fs struct {
}
var _ fsutil.FS = &fs{}
func CreateSSH(ssh []*buildflags.SSH) (session.Attachable, error) {
configs := make([]sshprovider.AgentConfig, 0, len(ssh))
for _, ssh := range ssh {
cfg := sshprovider.AgentConfig{
ID: ssh.ID,
Paths: slices.Clone(ssh.Paths),
}
configs = append(configs, cfg)
}
return sshprovider.NewSSHAgentProvider(configs)
}
func CreateSecrets(secrets []*buildflags.Secret) (session.Attachable, error) {
fs := make([]secretsprovider.Source, 0, len(secrets))
for _, secret := range secrets {
fs = append(fs, secretsprovider.Source{
ID: secret.ID,
FilePath: secret.FilePath,
Env: secret.Env,
})
}
store, err := secretsprovider.NewStore(fs)
if err != nil {
return nil, err
}
return secretsprovider.NewSecretProvider(store), nil
}
func CreateExports(entries []*buildflags.ExportEntry) ([]client.ExportEntry, []string, error) {
var outs []client.ExportEntry
var localPaths []string
if len(entries) == 0 {
return nil, nil, nil
}
var stdoutUsed bool
for _, entry := range entries {
if entry.Type == "" {
return nil, nil, errors.Errorf("type is required for output")
}
out := client.ExportEntry{
Type: entry.Type,
Attrs: map[string]string{},
}
maps.Copy(out.Attrs, entry.Attrs)
supportFile := false
supportDir := false
switch out.Type {
case client.ExporterLocal:
supportDir = true
case client.ExporterTar:
supportFile = true
case client.ExporterOCI, client.ExporterDocker:
tar, err := strconv.ParseBool(out.Attrs["tar"])
if err != nil {
tar = true
}
supportFile = tar
supportDir = !tar
case "registry":
out.Type = client.ExporterImage
out.Attrs["push"] = "true"
}
if supportDir {
if entry.Destination == "" {
return nil, nil, errors.Errorf("dest is required for %s exporter", out.Type)
}
if entry.Destination == "-" {
return nil, nil, errors.Errorf("dest cannot be stdout for %s exporter", out.Type)
}
fi, err := os.Stat(entry.Destination)
if err != nil && !os.IsNotExist(err) {
return nil, nil, errors.Wrapf(err, "invalid destination directory: %s", entry.Destination)
}
if err == nil && !fi.IsDir() {
return nil, nil, errors.Errorf("destination directory %s is a file", entry.Destination)
}
out.OutputDir = entry.Destination
localPaths = append(localPaths, entry.Destination)
}
if supportFile {
if entry.Destination == "" && out.Type != client.ExporterDocker {
entry.Destination = "-"
}
if entry.Destination == "-" {
if stdoutUsed {
return nil, nil, errors.Errorf("multiple outputs configured to write to stdout")
}
if _, err := console.ConsoleFromFile(os.Stdout); err == nil {
return nil, nil, errors.Errorf("dest file is required for %s exporter. refusing to write to console", out.Type)
}
out.Output = wrapWriteCloser(os.Stdout)
stdoutUsed = true
} else if entry.Destination != "" {
fi, err := os.Stat(entry.Destination)
if err != nil && !os.IsNotExist(err) {
return nil, nil, errors.Wrapf(err, "invalid destination file: %s", entry.Destination)
}
if err == nil && fi.IsDir() {
return nil, nil, errors.Errorf("destination file %s is a directory", entry.Destination)
}
f, err := os.Create(entry.Destination)
if err != nil {
return nil, nil, errors.Errorf("failed to open %s", err)
}
out.Output = wrapWriteCloser(f)
localPaths = append(localPaths, entry.Destination)
}
}
outs = append(outs, out)
}
return outs, localPaths, nil
}
func wrapWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
return func(map[string]string) (io.WriteCloser, error) {
return wc, nil
}
}
func CreateCaches(entries []*buildflags.CacheOptionsEntry) []client.CacheOptionsEntry {
var outs []client.CacheOptionsEntry
if len(entries) == 0 {
return nil
}
for _, entry := range entries {
out := client.CacheOptionsEntry{
Type: entry.Type,
Attrs: map[string]string{},
}
maps.Copy(out.Attrs, entry.Attrs)
addGithubToken(&out)
addAwsCredentials(&out)
if !isActive(&out) {
continue
}
outs = append(outs, out)
}
return outs
}
func addGithubToken(ci *client.CacheOptionsEntry) {
if ci.Type != "gha" {
return
}
version, ok := ci.Attrs["version"]
if !ok {
// https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L19
if v, ok := os.LookupEnv("ACTIONS_CACHE_SERVICE_V2"); ok {
if b, err := strconv.ParseBool(v); err == nil && b {
version = "2"
}
}
}
if _, ok := ci.Attrs["token"]; !ok {
if v, ok := os.LookupEnv("ACTIONS_RUNTIME_TOKEN"); ok {
ci.Attrs["token"] = v
}
}
if _, ok := ci.Attrs["url_v2"]; !ok && version == "2" {
// https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L34-L35
if v, ok := os.LookupEnv("ACTIONS_RESULTS_URL"); ok {
ci.Attrs["url_v2"] = v
}
}
if _, ok := ci.Attrs["url"]; !ok {
// https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L28-L33
if v, ok := os.LookupEnv("ACTIONS_CACHE_URL"); ok {
ci.Attrs["url"] = v
} else if v, ok := os.LookupEnv("ACTIONS_RESULTS_URL"); ok {
ci.Attrs["url"] = v
}
}
}
func addAwsCredentials(ci *client.CacheOptionsEntry) {
if ci.Type != "s3" {
return
}
_, okAccessKeyID := ci.Attrs["access_key_id"]
_, okSecretAccessKey := ci.Attrs["secret_access_key"]
// If the user provides access_key_id, secret_access_key, do not override the session token.
if okAccessKeyID && okSecretAccessKey {
return
}
ctx := context.TODO()
awsConfig, err := awsconfig.LoadDefaultConfig(ctx)
if err != nil {
return
}
credentials, err := awsConfig.Credentials.Retrieve(ctx)
if err != nil {
return
}
if !okAccessKeyID && credentials.AccessKeyID != "" {
ci.Attrs["access_key_id"] = credentials.AccessKeyID
}
if !okSecretAccessKey && credentials.SecretAccessKey != "" {
ci.Attrs["secret_access_key"] = credentials.SecretAccessKey
}
if _, ok := ci.Attrs["session_token"]; !ok && credentials.SessionToken != "" {
ci.Attrs["session_token"] = credentials.SessionToken
}
}
func isActive(ce *client.CacheOptionsEntry) bool {
// Always active if not gha.
if ce.Type != "gha" {
return true
}
return ce.Attrs["token"] != "" && (ce.Attrs["url"] != "" || ce.Attrs["url_v2"] != "")
}
build/opt_test.go
+40
View File
@@ -0,0 +1,40 @@
package build
import (
"testing"
"github.com/docker/buildx/util/buildflags"
"github.com/moby/buildkit/client"
"github.com/stretchr/testify/require"
)
func TestCacheOptions_DerivedVars(t *testing.T) {
t.Setenv("ACTIONS_RUNTIME_TOKEN", "sensitive_token")
t.Setenv("ACTIONS_CACHE_URL", "https://cache.github.com")
t.Setenv("AWS_ACCESS_KEY_ID", "definitely_dont_look_here")
t.Setenv("AWS_SECRET_ACCESS_KEY", "hackers_please_dont_steal")
t.Setenv("AWS_SESSION_TOKEN", "not_a_mitm_attack")
cacheFrom, err := buildflags.ParseCacheEntry([]string{"type=gha", "type=s3,region=us-west-2,bucket=my_bucket,name=my_image"})
require.NoError(t, err)
require.Equal(t, []client.CacheOptionsEntry{
{
Type: "gha",
Attrs: map[string]string{
"token": "sensitive_token",
"url": "https://cache.github.com",
},
},
{
Type: "s3",
Attrs: map[string]string{
"region": "us-west-2",
"bucket": "my_bucket",
"name": "my_image",
"access_key_id": "definitely_dont_look_here",
"secret_access_key": "hackers_please_dont_steal",
"session_token": "not_a_mitm_attack",
},
},
}, CreateCaches(cacheFrom))
}
build/result.go
+6 -7
View File
@@ -7,7 +7,6 @@ import (
"io"
"sync"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
gateway "github.com/moby/buildkit/frontend/gateway/client"
@@ -292,7 +291,7 @@ func (r *ResultHandle) build(buildFunc gateway.BuildFunc) (err error) {
return err
}
func (r *ResultHandle) getContainerConfig(cfg *controllerapi.InvokeConfig) (containerCfg gateway.NewContainerRequest, _ error) {
func (r *ResultHandle) getContainerConfig(cfg *InvokeConfig) (containerCfg gateway.NewContainerRequest, _ error) {
if r.res != nil && r.solveErr == nil {
logrus.Debugf("creating container from successful build")
ccfg, err := containerConfigFromResult(r.res, cfg)
@@ -311,7 +310,7 @@ func (r *ResultHandle) getContainerConfig(cfg *controllerapi.InvokeConfig) (cont
return containerCfg, nil
}
func (r *ResultHandle) getProcessConfig(cfg *controllerapi.InvokeConfig, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) (_ gateway.StartRequest, err error) {
func (r *ResultHandle) getProcessConfig(cfg *InvokeConfig, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) (_ gateway.StartRequest, err error) {
processCfg := newStartRequest(stdin, stdout, stderr)
if r.res != nil && r.solveErr == nil {
logrus.Debugf("creating container from successful build")
@@ -327,7 +326,7 @@ func (r *ResultHandle) getProcessConfig(cfg *controllerapi.InvokeConfig, stdin i
return processCfg, nil
}
func containerConfigFromResult(res *gateway.Result, cfg *controllerapi.InvokeConfig) (*gateway.NewContainerRequest, error) {
func containerConfigFromResult(res *gateway.Result, cfg *InvokeConfig) (*gateway.NewContainerRequest, error) {
if cfg.Initial {
return nil, errors.Errorf("starting from the container from the initial state of the step is supported only on the failed steps")
}
@@ -352,7 +351,7 @@ func containerConfigFromResult(res *gateway.Result, cfg *controllerapi.InvokeCon
}, nil
}
func populateProcessConfigFromResult(req *gateway.StartRequest, res *gateway.Result, cfg *controllerapi.InvokeConfig) error {
func populateProcessConfigFromResult(req *gateway.StartRequest, res *gateway.Result, cfg *InvokeConfig) error {
imgData := res.Metadata[exptypes.ExporterImageConfigKey]
var img *ocispecs.Image
if len(imgData) > 0 {
@@ -403,7 +402,7 @@ func populateProcessConfigFromResult(req *gateway.StartRequest, res *gateway.Res
return nil
}
func containerConfigFromError(solveErr *errdefs.SolveError, cfg *controllerapi.InvokeConfig) (*gateway.NewContainerRequest, error) {
func containerConfigFromError(solveErr *errdefs.SolveError, cfg *InvokeConfig) (*gateway.NewContainerRequest, error) {
exec, err := execOpFromError(solveErr)
if err != nil {
return nil, err
@@ -431,7 +430,7 @@ func containerConfigFromError(solveErr *errdefs.SolveError, cfg *controllerapi.I
}, nil
}
func populateProcessConfigFromError(req *gateway.StartRequest, solveErr *errdefs.SolveError, cfg *controllerapi.InvokeConfig) error {
func populateProcessConfigFromError(req *gateway.StartRequest, solveErr *errdefs.SolveError, cfg *InvokeConfig) error {
exec, err := execOpFromError(solveErr)
if err != nil {
return err
commands/bake.go
+1 -2
View File
@@ -22,7 +22,6 @@ import (
"github.com/docker/buildx/bake/hclparser"
"github.com/docker/buildx/build"
"github.com/docker/buildx/builder"
"github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/localstate"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/cobrautil/completion"
@@ -348,7 +347,7 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba
continue
}
pf := &pb.CallFunc{
pf := &buildflags.CallFunc{
Name: req.CallFunc.Name,
Format: req.CallFunc.Format,
IgnoreStatus: req.CallFunc.IgnoreStatus,
commands/build.go
+244 -46
View File
@@ -21,9 +21,6 @@ import (
"github.com/docker/buildx/build"
"github.com/docker/buildx/builder"
"github.com/docker/buildx/commands/debug"
cbuild "github.com/docker/buildx/controller/build"
controllererrors "github.com/docker/buildx/controller/errdefs"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/monitor"
"github.com/docker/buildx/store"
"github.com/docker/buildx/store/storeutil"
@@ -31,8 +28,10 @@ import (
"github.com/docker/buildx/util/cobrautil"
"github.com/docker/buildx/util/confutil"
"github.com/docker/buildx/util/desktop"
"github.com/docker/buildx/util/dockerutil"
"github.com/docker/buildx/util/metricutil"
"github.com/docker/buildx/util/osutil"
"github.com/docker/buildx/util/platformutil"
"github.com/docker/buildx/util/progress"
"github.com/docker/buildx/util/tracing"
"github.com/docker/cli/cli"
@@ -45,8 +44,10 @@ import (
"github.com/moby/buildkit/frontend/subrequests/lint"
"github.com/moby/buildkit/frontend/subrequests/outline"
"github.com/moby/buildkit/frontend/subrequests/targets"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/moby/buildkit/solver/errdefs"
solverpb "github.com/moby/buildkit/solver/pb"
sourcepolicy "github.com/moby/buildkit/sourcepolicy/pb"
"github.com/moby/buildkit/util/grpcerrors"
"github.com/moby/buildkit/util/progress/progressui"
"github.com/moby/sys/atomicwriter"
@@ -104,7 +105,7 @@ type buildOptions struct {
invokeConfig *invokeConfig
}
func (o *buildOptions) toControllerOptions() (*cbuild.Options, error) {
func (o *buildOptions) toOptions() (*BuildOptions, error) {
var err error
buildArgs, err := listToMap(o.buildArgs, true)
@@ -117,7 +118,7 @@ func (o *buildOptions) toControllerOptions() (*cbuild.Options, error) {
return nil, err
}
opts := cbuild.Options{
opts := BuildOptions{
Allow: o.allow,
Annotations: o.annotations,
BuildArgs: buildArgs,
@@ -132,7 +133,7 @@ func (o *buildOptions) toControllerOptions() (*cbuild.Options, error) {
ShmSize: int64(o.shmSize),
Tags: o.tags,
Target: o.target,
Ulimits: dockerUlimitToControllerUlimit(o.ulimits),
Ulimits: o.ulimits,
Builder: o.builder,
NoCache: o.noCache,
Pull: o.pull,
@@ -179,17 +180,15 @@ func (o *buildOptions) toControllerOptions() (*cbuild.Options, error) {
}
}
cacheFrom, err := buildflags.ParseCacheEntry(o.cacheFrom)
opts.CacheFrom, err = buildflags.ParseCacheEntry(o.cacheFrom)
if err != nil {
return nil, err
}
opts.CacheFrom = cacheFrom.ToPB()
cacheTo, err := buildflags.ParseCacheEntry(o.cacheTo)
opts.CacheTo, err = buildflags.ParseCacheEntry(o.cacheTo)
if err != nil {
return nil, err
}
opts.CacheTo = cacheTo.ToPB()
opts.Secrets, err = buildflags.ParseSecretSpecs(o.secrets)
if err != nil {
@@ -293,7 +292,7 @@ func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions)
end(err)
}()
opts, err := options.toControllerOptions()
opts, err := options.toOptions()
if err != nil {
return err
}
@@ -350,14 +349,7 @@ func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions)
}
done := timeBuildCommand(mp, attributes)
var resp *client.SolveResponse
var inputs *build.Inputs
var retErr error
if confutil.IsExperimental() {
resp, inputs, retErr = runControllerBuild(ctx, dockerCli, opts, options, printer)
} else {
resp, inputs, retErr = runBasicBuild(ctx, dockerCli, opts, printer)
}
resp, inputs, retErr := runBuildWithOptions(ctx, dockerCli, opts, options, printer)
if err := printer.Wait(); retErr == nil {
retErr = err
@@ -415,11 +407,7 @@ func getImageID(resp map[string]string) string {
return dgst
}
func runBasicBuild(ctx context.Context, dockerCli command.Cli, opts *cbuild.Options, printer *progress.Printer) (*client.SolveResponse, *build.Inputs, error) {
return cbuild.RunBuild(ctx, dockerCli, opts, dockerCli.In(), printer, nil)
}
func runControllerBuild(ctx context.Context, dockerCli command.Cli, opts *cbuild.Options, options buildOptions, printer *progress.Printer) (_ *client.SolveResponse, _ *build.Inputs, retErr error) {
func runBuildWithOptions(ctx context.Context, dockerCli command.Cli, opts *BuildOptions, options buildOptions, printer *progress.Printer) (_ *client.SolveResponse, _ *build.Inputs, retErr error) {
if options.invokeConfig != nil && (options.dockerfileName == "-" || options.contextPath == "-") {
// stdin must be usable for monitor
return nil, nil, errors.Errorf("Dockerfile or context from stdin is not supported with invoke")
@@ -440,9 +428,9 @@ func runControllerBuild(ctx context.Context, dockerCli command.Cli, opts *cbuild
}
for {
resp, inputs, err := cbuild.RunBuild(ctx, dockerCli, opts, in, printer, &bh)
resp, inputs, err := RunBuild(ctx, dockerCli, opts, in, printer, &bh)
if err != nil {
var be *controllererrors.BuildError
var be *BuildError
if errors.As(err, &be) {
retErr = err
// We can proceed to monitor
@@ -765,21 +753,6 @@ func listToMap(values []string, defaultEnv bool) (map[string]string, error) {
return result, nil
}
func dockerUlimitToControllerUlimit(u *dockeropts.UlimitOpt) *controllerapi.UlimitOpt {
if u == nil {
return nil
}
values := make(map[string]*controllerapi.Ulimit)
for _, u := range u.GetList() {
values[u.Name] = &controllerapi.Ulimit{
Name: u.Name,
Hard: u.Hard,
Soft: u.Soft,
}
}
return &controllerapi.UlimitOpt{Values: values}
}
func printWarnings(w io.Writer, warnings []client.VertexWarning, mode progressui.DisplayMode) {
if len(warnings) == 0 || mode == progressui.QuietMode || mode == progressui.RawJSONMode {
return
@@ -819,7 +792,7 @@ func printWarnings(w io.Writer, warnings []client.VertexWarning, mode progressui
}
}
func printResult(w io.Writer, f *controllerapi.CallFunc, res map[string]string, target string, inp *build.Inputs) (int, error) {
func printResult(w io.Writer, f *buildflags.CallFunc, res map[string]string, target string, inp *build.Inputs) (int, error) {
switch f.Name {
case "outline":
return 0, printValue(w, outline.PrintOutline, outline.SubrequestsOutlineDefinition.Version, f.Format, res)
@@ -920,19 +893,19 @@ func printValue(w io.Writer, printer callFunc, version string, format string, re
}
type invokeConfig struct {
controllerapi.InvokeConfig
build.InvokeConfig
invokeFlag string
}
func (cfg *invokeConfig) parseInvokeConfig(invoke, on string) error {
switch on {
case "always":
cfg.SuspendOn = controllerapi.SuspendAlways
cfg.SuspendOn = build.SuspendAlways
case "error":
cfg.SuspendOn = controllerapi.SuspendError
cfg.SuspendOn = build.SuspendError
default:
if invoke != "" {
cfg.SuspendOn = controllerapi.SuspendAlways
cfg.SuspendOn = build.SuspendAlways
}
}
@@ -1057,3 +1030,228 @@ func otelErrorType(err error) string {
}
return name
}
const defaultTargetName = "default"
type BuildOptions struct {
ContextPath string
DockerfileName string
CallFunc *buildflags.CallFunc
NamedContexts map[string]string
Allow []string
Attests buildflags.Attests
BuildArgs map[string]string
CacheFrom []*buildflags.CacheOptionsEntry
CacheTo []*buildflags.CacheOptionsEntry
CgroupParent string
Exports []*buildflags.ExportEntry
ExtraHosts []string
Labels map[string]string
NetworkMode string
NoCacheFilter []string
Platforms []string
Secrets buildflags.Secrets
ShmSize int64
SSH []*buildflags.SSH
Tags []string
Target string
Ulimits *dockeropts.UlimitOpt
Builder string
NoCache bool
Pull bool
ExportPush bool
ExportLoad bool
SourcePolicy *sourcepolicy.Policy
Ref string
GroupRef string
Annotations []string
ProvenanceResponseMode string
}
// RunBuild runs the specified build and returns the result.
func RunBuild(ctx context.Context, dockerCli command.Cli, in *BuildOptions, inStream io.Reader, progress progress.Writer, bh *build.Handler) (*client.SolveResponse, *build.Inputs, error) {
if in.NoCache && len(in.NoCacheFilter) > 0 {
return nil, nil, errors.Errorf("--no-cache and --no-cache-filter cannot currently be used together")
}
contexts := map[string]build.NamedContext{}
for name, path := range in.NamedContexts {
contexts[name] = build.NamedContext{Path: path}
}
opts := build.Options{
Inputs: build.Inputs{
ContextPath: in.ContextPath,
DockerfilePath: in.DockerfileName,
InStream: build.NewSyncMultiReader(inStream),
NamedContexts: contexts,
},
Ref: in.Ref,
BuildArgs: in.BuildArgs,
CgroupParent: in.CgroupParent,
ExtraHosts: in.ExtraHosts,
Labels: in.Labels,
NetworkMode: in.NetworkMode,
NoCache: in.NoCache,
NoCacheFilter: in.NoCacheFilter,
Pull: in.Pull,
ShmSize: dockeropts.MemBytes(in.ShmSize),
Tags: in.Tags,
Target: in.Target,
Ulimits: in.Ulimits,
GroupRef: in.GroupRef,
ProvenanceResponseMode: confutil.ParseMetadataProvenance(in.ProvenanceResponseMode),
}
platforms, err := platformutil.Parse(in.Platforms)
if err != nil {
return nil, nil, err
}
opts.Platforms = platforms
dockerConfig := dockerCli.ConfigFile()
opts.Session = append(opts.Session, authprovider.NewDockerAuthProvider(authprovider.DockerAuthProviderConfig{
ConfigFile: dockerConfig,
}))
secrets, err := build.CreateSecrets(in.Secrets)
if err != nil {
return nil, nil, err
}
opts.Session = append(opts.Session, secrets)
sshSpecs := in.SSH
if len(sshSpecs) == 0 && buildflags.IsGitSSH(in.ContextPath) {
sshSpecs = append(sshSpecs, &buildflags.SSH{ID: "default"})
}
ssh, err := build.CreateSSH(sshSpecs)
if err != nil {
return nil, nil, err
}
opts.Session = append(opts.Session, ssh)
outputs, _, err := build.CreateExports(in.Exports)
if err != nil {
return nil, nil, err
}
if in.ExportPush {
var pushUsed bool
for i := range outputs {
if outputs[i].Type == client.ExporterImage {
outputs[i].Attrs["push"] = "true"
pushUsed = true
}
}
if !pushUsed {
outputs = append(outputs, client.ExportEntry{
Type: client.ExporterImage,
Attrs: map[string]string{
"push": "true",
},
})
}
}
if in.ExportLoad {
var loadUsed bool
for i := range outputs {
if outputs[i].Type == client.ExporterDocker {
if _, ok := outputs[i].Attrs["dest"]; !ok {
loadUsed = true
break
}
}
}
if !loadUsed {
outputs = append(outputs, client.ExportEntry{
Type: client.ExporterDocker,
Attrs: map[string]string{},
})
}
}
annotations, err := buildflags.ParseAnnotations(in.Annotations)
if err != nil {
return nil, nil, errors.Wrap(err, "parse annotations")
}
for _, o := range outputs {
for k, v := range annotations {
o.Attrs[k.String()] = v
}
}
opts.Exports = outputs
opts.CacheFrom = build.CreateCaches(in.CacheFrom)
opts.CacheTo = build.CreateCaches(in.CacheTo)
opts.Attests = in.Attests.ToMap()
opts.SourcePolicy = in.SourcePolicy
allow, err := buildflags.ParseEntitlements(in.Allow)
if err != nil {
return nil, nil, err
}
opts.Allow = allow
if in.CallFunc != nil {
opts.CallFunc = &build.CallFunc{
Name: in.CallFunc.Name,
Format: in.CallFunc.Format,
IgnoreStatus: in.CallFunc.IgnoreStatus,
}
}
// key string used for kubernetes "sticky" mode
contextPathHash, err := filepath.Abs(in.ContextPath)
if err != nil {
contextPathHash = in.ContextPath
}
b, err := builder.New(dockerCli,
builder.WithName(in.Builder),
builder.WithContextPathHash(contextPathHash),
)
if err != nil {
return nil, nil, err
}
if err = updateLastActivity(dockerCli, b.NodeGroup); err != nil {
return nil, nil, errors.Wrapf(err, "failed to update builder last activity time")
}
nodes, err := b.LoadNodes(ctx)
if err != nil {
return nil, nil, err
}
var inputs *build.Inputs
buildOptions := map[string]build.Options{defaultTargetName: opts}
resp, err := build.BuildWithResultHandler(ctx, nodes, buildOptions, dockerutil.NewClient(dockerCli), confutil.NewConfig(dockerCli), progress, bh)
err = wrapBuildError(err, false)
if err != nil {
return nil, nil, WrapBuild(err)
}
if i, ok := buildOptions[defaultTargetName]; ok {
inputs = &i.Inputs
}
return resp[defaultTargetName], inputs, nil
}
type BuildError struct {
err error
}
func (e *BuildError) Unwrap() error {
return e.err
}
func (e *BuildError) Error() string {
return e.err.Error()
}
func WrapBuild(err error) error {
if err == nil {
return nil
}
return &BuildError{err: err}
}
controller/build/build.go
-272
View File
@@ -1,272 +0,0 @@
package build
import (
"context"
"io"
"path/filepath"
"strings"
"github.com/docker/buildx/build"
"github.com/docker/buildx/builder"
"github.com/docker/buildx/controller/errdefs"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/store"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/buildx/util/buildflags"
"github.com/docker/buildx/util/confutil"
"github.com/docker/buildx/util/dockerutil"
"github.com/docker/buildx/util/platformutil"
"github.com/docker/buildx/util/progress"
"github.com/docker/cli/cli/command"
dockeropts "github.com/docker/cli/opts"
"github.com/docker/docker/api/types/container"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/moby/buildkit/util/grpcerrors"
"github.com/pkg/errors"
"google.golang.org/grpc/codes"
)
const defaultTargetName = "default"
// RunBuild runs the specified build and returns the result.
//
// NOTE: When an error happens during the build and this function acquires the debuggable *build.ResultHandle,
// this function returns it in addition to the error (i.e. it does "return nil, res, err"). The caller can
// inspect the result and debug the cause of that error.
func RunBuild(ctx context.Context, dockerCli command.Cli, in *Options, inStream io.Reader, progress progress.Writer, bh *build.Handler) (*client.SolveResponse, *build.Inputs, error) {
if in.NoCache && len(in.NoCacheFilter) > 0 {
return nil, nil, errors.Errorf("--no-cache and --no-cache-filter cannot currently be used together")
}
contexts := map[string]build.NamedContext{}
for name, path := range in.NamedContexts {
contexts[name] = build.NamedContext{Path: path}
}
opts := build.Options{
Inputs: build.Inputs{
ContextPath: in.ContextPath,
DockerfilePath: in.DockerfileName,
InStream: build.NewSyncMultiReader(inStream),
NamedContexts: contexts,
},
Ref: in.Ref,
BuildArgs: in.BuildArgs,
CgroupParent: in.CgroupParent,
ExtraHosts: in.ExtraHosts,
Labels: in.Labels,
NetworkMode: in.NetworkMode,
NoCache: in.NoCache,
NoCacheFilter: in.NoCacheFilter,
Pull: in.Pull,
ShmSize: dockeropts.MemBytes(in.ShmSize),
Tags: in.Tags,
Target: in.Target,
Ulimits: controllerUlimitOpt2DockerUlimit(in.Ulimits),
GroupRef: in.GroupRef,
ProvenanceResponseMode: confutil.ParseMetadataProvenance(in.ProvenanceResponseMode),
}
platforms, err := platformutil.Parse(in.Platforms)
if err != nil {
return nil, nil, err
}
opts.Platforms = platforms
dockerConfig := dockerCli.ConfigFile()
opts.Session = append(opts.Session, authprovider.NewDockerAuthProvider(authprovider.DockerAuthProviderConfig{
ConfigFile: dockerConfig,
}))
secrets, err := controllerapi.CreateSecrets(in.Secrets)
if err != nil {
return nil, nil, err
}
opts.Session = append(opts.Session, secrets)
sshSpecs := in.SSH
if len(sshSpecs) == 0 && buildflags.IsGitSSH(in.ContextPath) {
sshSpecs = append(sshSpecs, &controllerapi.SSH{ID: "default"})
}
ssh, err := controllerapi.CreateSSH(sshSpecs)
if err != nil {
return nil, nil, err
}
opts.Session = append(opts.Session, ssh)
outputs, _, err := controllerapi.CreateExports(in.Exports)
if err != nil {
return nil, nil, err
}
if in.ExportPush {
var pushUsed bool
for i := range outputs {
if outputs[i].Type == client.ExporterImage {
outputs[i].Attrs["push"] = "true"
pushUsed = true
}
}
if !pushUsed {
outputs = append(outputs, client.ExportEntry{
Type: client.ExporterImage,
Attrs: map[string]string{
"push": "true",
},
})
}
}
if in.ExportLoad {
var loadUsed bool
for i := range outputs {
if outputs[i].Type == client.ExporterDocker {
if _, ok := outputs[i].Attrs["dest"]; !ok {
loadUsed = true
break
}
}
}
if !loadUsed {
outputs = append(outputs, client.ExportEntry{
Type: client.ExporterDocker,
Attrs: map[string]string{},
})
}
}
annotations, err := buildflags.ParseAnnotations(in.Annotations)
if err != nil {
return nil, nil, errors.Wrap(err, "parse annotations")
}
for _, o := range outputs {
for k, v := range annotations {
o.Attrs[k.String()] = v
}
}
opts.Exports = outputs
opts.CacheFrom = controllerapi.CreateCaches(in.CacheFrom)
opts.CacheTo = controllerapi.CreateCaches(in.CacheTo)
opts.Attests = controllerapi.CreateAttestations(in.Attests)
opts.SourcePolicy = in.SourcePolicy
allow, err := buildflags.ParseEntitlements(in.Allow)
if err != nil {
return nil, nil, err
}
opts.Allow = allow
if in.CallFunc != nil {
opts.CallFunc = &build.CallFunc{
Name: in.CallFunc.Name,
Format: in.CallFunc.Format,
IgnoreStatus: in.CallFunc.IgnoreStatus,
}
}
// key string used for kubernetes "sticky" mode
contextPathHash, err := filepath.Abs(in.ContextPath)
if err != nil {
contextPathHash = in.ContextPath
}
// TODO: this should not be loaded this side of the controller api
b, err := builder.New(dockerCli,
builder.WithName(in.Builder),
builder.WithContextPathHash(contextPathHash),
)
if err != nil {
return nil, nil, err
}
if err = updateLastActivity(dockerCli, b.NodeGroup); err != nil {
return nil, nil, errors.Wrapf(err, "failed to update builder last activity time")
}
nodes, err := b.LoadNodes(ctx)
if err != nil {
return nil, nil, err
}
var inputs *build.Inputs
buildOptions := map[string]build.Options{defaultTargetName: opts}
resp, err := buildTargets(ctx, dockerCli, nodes, buildOptions, progress, bh)
err = wrapBuildError(err, false)
if err != nil {
return nil, nil, errdefs.WrapBuild(err)
}
if i, ok := buildOptions[defaultTargetName]; ok {
inputs = &i.Inputs
}
return resp, inputs, nil
}
// buildTargets runs the specified build and returns the result.
//
// NOTE: When an error happens during the build and this function acquires the debuggable *build.ResultHandle,
// this function returns it in addition to the error (i.e. it does "return nil, res, err"). The caller can
// inspect the result and debug the cause of that error.
func buildTargets(ctx context.Context, dockerCli command.Cli, nodes []builder.Node, opts map[string]build.Options, progress progress.Writer, bh *build.Handler) (*client.SolveResponse, error) {
resp, err := build.BuildWithResultHandler(ctx, nodes, opts, dockerutil.NewClient(dockerCli), confutil.NewConfig(dockerCli), progress, bh)
if err != nil {
return nil, err
}
return resp[defaultTargetName], err
}
func wrapBuildError(err error, bake bool) error {
if err == nil {
return nil
}
st, ok := grpcerrors.AsGRPCStatus(err)
if ok {
if st.Code() == codes.Unimplemented && strings.Contains(st.Message(), "unsupported frontend capability moby.buildkit.frontend.contexts") {
msg := "current frontend does not support --build-context."
if bake {
msg = "current frontend does not support defining additional contexts for targets."
}
msg += " Named contexts are supported since Dockerfile v1.4. Use #syntax directive in Dockerfile or update to latest BuildKit."
return &wrapped{err, msg}
}
}
return err
}
type wrapped struct {
err error
msg string
}
func (w *wrapped) Error() string {
return w.msg
}
func (w *wrapped) Unwrap() error {
return w.err
}
func updateLastActivity(dockerCli command.Cli, ng *store.NodeGroup) error {
txn, release, err := storeutil.GetStore(dockerCli)
if err != nil {
return err
}
defer release()
return txn.UpdateLastActivity(ng)
}
func controllerUlimitOpt2DockerUlimit(u *controllerapi.UlimitOpt) *dockeropts.UlimitOpt {
if u == nil {
return nil
}
values := make(map[string]*container.Ulimit)
for k, v := range u.Values {
values[k] = &container.Ulimit{
Name: v.Name,
Hard: v.Hard,
Soft: v.Soft,
}
}
return dockeropts.NewUlimitOpt(&values)
}
controller/build/options.go
-216
View File
@@ -1,216 +0,0 @@
package build
import (
"path/filepath"
"strings"
"github.com/docker/buildx/controller/pb"
sourcepolicy "github.com/moby/buildkit/sourcepolicy/pb"
"github.com/moby/buildkit/util/gitutil"
)
type Options struct {
ContextPath string
DockerfileName string
CallFunc *pb.CallFunc
NamedContexts map[string]string
Allow []string
Attests []*pb.Attest
BuildArgs map[string]string
CacheFrom []*pb.CacheOptionsEntry
CacheTo []*pb.CacheOptionsEntry
CgroupParent string
Exports []*pb.ExportEntry
ExtraHosts []string
Labels map[string]string
NetworkMode string
NoCacheFilter []string
Platforms []string
Secrets []*pb.Secret
ShmSize int64
SSH []*pb.SSH
Tags []string
Target string
Ulimits *pb.UlimitOpt
Builder string
NoCache bool
Pull bool
ExportPush bool
ExportLoad bool
SourcePolicy *sourcepolicy.Policy
Ref string
GroupRef string
Annotations []string
ProvenanceResponseMode string
}
// ResolveOptionPaths resolves all paths contained in BuildOptions
// and replaces them to absolute paths.
func ResolveOptionPaths(options *Options) (_ *Options, err error) {
localContext := false
if options.ContextPath != "" && options.ContextPath != "-" {
if !isRemoteURL(options.ContextPath) {
localContext = true
options.ContextPath, err = filepath.Abs(options.ContextPath)
if err != nil {
return nil, err
}
}
}
if options.DockerfileName != "" && options.DockerfileName != "-" {
if localContext && !isHTTPURL(options.DockerfileName) {
options.DockerfileName, err = filepath.Abs(options.DockerfileName)
if err != nil {
return nil, err
}
}
}
var contexts map[string]string
for k, v := range options.NamedContexts {
if isRemoteURL(v) || strings.HasPrefix(v, "docker-image://") {
// url prefix, this is a remote path
} else if p, ok := strings.CutPrefix(v, "oci-layout://"); ok {
// oci layout prefix, this is a local path
p, err = filepath.Abs(p)
if err != nil {
return nil, err
}
v = "oci-layout://" + p
} else {
// no prefix, assume local path
v, err = filepath.Abs(v)
if err != nil {
return nil, err
}
}
if contexts == nil {
contexts = make(map[string]string)
}
contexts[k] = v
}
options.NamedContexts = contexts
var cacheFrom []*pb.CacheOptionsEntry
for _, co := range options.CacheFrom {
switch co.Type {
case "local":
var attrs map[string]string
for k, v := range co.Attrs {
if attrs == nil {
attrs = make(map[string]string)
}
switch k {
case "src":
p := v
if p != "" {
p, err = filepath.Abs(p)
if err != nil {
return nil, err
}
}
attrs[k] = p
default:
attrs[k] = v
}
}
co.Attrs = attrs
cacheFrom = append(cacheFrom, co)
default:
cacheFrom = append(cacheFrom, co)
}
}
options.CacheFrom = cacheFrom
var cacheTo []*pb.CacheOptionsEntry
for _, co := range options.CacheTo {
switch co.Type {
case "local":
var attrs map[string]string
for k, v := range co.Attrs {
if attrs == nil {
attrs = make(map[string]string)
}
switch k {
case "dest":
p := v
if p != "" {
p, err = filepath.Abs(p)
if err != nil {
return nil, err
}
}
attrs[k] = p
default:
attrs[k] = v
}
}
co.Attrs = attrs
cacheTo = append(cacheTo, co)
default:
cacheTo = append(cacheTo, co)
}
}
options.CacheTo = cacheTo
var exports []*pb.ExportEntry
for _, e := range options.Exports {
if e.Destination != "" && e.Destination != "-" {
e.Destination, err = filepath.Abs(e.Destination)
if err != nil {
return nil, err
}
}
exports = append(exports, e)
}
options.Exports = exports
var secrets []*pb.Secret
for _, s := range options.Secrets {
if s.FilePath != "" {
s.FilePath, err = filepath.Abs(s.FilePath)
if err != nil {
return nil, err
}
}
secrets = append(secrets, s)
}
options.Secrets = secrets
var ssh []*pb.SSH
for _, s := range options.SSH {
var ps []string
for _, pt := range s.Paths {
p := pt
if p != "" {
p, err = filepath.Abs(p)
if err != nil {
return nil, err
}
}
ps = append(ps, p)
}
s.Paths = ps
ssh = append(ssh, s)
}
options.SSH = ssh
return options, nil
}
// isHTTPURL returns true if the provided str is an HTTP(S) URL by checking if it
// has a http:// or https:// scheme. No validation is performed to verify if the
// URL is well-formed.
func isHTTPURL(str string) bool {
return strings.HasPrefix(str, "https://") || strings.HasPrefix(str, "http://")
}
func isRemoteURL(c string) bool {
if isHTTPURL(c) {
return true
}
if _, err := gitutil.ParseGitRef(c); err == nil {
return true
}
return false
}
controller/build/options_test.go
-249
View File
@@ -1,249 +0,0 @@
package build
import (
"os"
"path/filepath"
"testing"
"github.com/docker/buildx/controller/pb"
"github.com/stretchr/testify/require"
)
func TestResolvePaths(t *testing.T) {
tmpwd, err := os.MkdirTemp("", "testresolvepaths")
require.NoError(t, err)
defer os.Remove(tmpwd)
require.NoError(t, os.Chdir(tmpwd))
tests := []struct {
name string
options *Options
want *Options
}{
{
name: "contextpath",
options: &Options{ContextPath: "test"},
want: &Options{ContextPath: filepath.Join(tmpwd, "test")},
},
{
name: "contextpath-cwd",
options: &Options{ContextPath: "."},
want: &Options{ContextPath: tmpwd},
},
{
name: "contextpath-dash",
options: &Options{ContextPath: "-"},
want: &Options{ContextPath: "-"},
},
{
name: "contextpath-ssh",
options: &Options{ContextPath: "git@github.com:docker/buildx.git"},
want: &Options{ContextPath: "git@github.com:docker/buildx.git"},
},
{
name: "dockerfilename",
options: &Options{DockerfileName: "test", ContextPath: "."},
want: &Options{DockerfileName: filepath.Join(tmpwd, "test"), ContextPath: tmpwd},
},
{
name: "dockerfilename-dash",
options: &Options{DockerfileName: "-", ContextPath: "."},
want: &Options{DockerfileName: "-", ContextPath: tmpwd},
},
{
name: "dockerfilename-remote",
options: &Options{DockerfileName: "test", ContextPath: "git@github.com:docker/buildx.git"},
want: &Options{DockerfileName: "test", ContextPath: "git@github.com:docker/buildx.git"},
},
{
name: "contexts",
options: &Options{NamedContexts: map[string]string{
"a": "test1", "b": "test2",
"alpine": "docker-image://alpine@sha256:0123456789", "project": "https://github.com/myuser/project.git",
}},
want: &Options{NamedContexts: map[string]string{
"a": filepath.Join(tmpwd, "test1"), "b": filepath.Join(tmpwd, "test2"),
"alpine": "docker-image://alpine@sha256:0123456789", "project": "https://github.com/myuser/project.git",
}},
},
{
name: "cache-from",
options: &Options{
CacheFrom: []*pb.CacheOptionsEntry{
{
Type: "local",
Attrs: map[string]string{"src": "test"},
},
{
Type: "registry",
Attrs: map[string]string{"ref": "user/app"},
},
},
},
want: &Options{
CacheFrom: []*pb.CacheOptionsEntry{
{
Type: "local",
Attrs: map[string]string{"src": filepath.Join(tmpwd, "test")},
},
{
Type: "registry",
Attrs: map[string]string{"ref": "user/app"},
},
},
},
},
{
name: "cache-to",
options: &Options{
CacheTo: []*pb.CacheOptionsEntry{
{
Type: "local",
Attrs: map[string]string{"dest": "test"},
},
{
Type: "registry",
Attrs: map[string]string{"ref": "user/app"},
},
},
},
want: &Options{
CacheTo: []*pb.CacheOptionsEntry{
{
Type: "local",
Attrs: map[string]string{"dest": filepath.Join(tmpwd, "test")},
},
{
Type: "registry",
Attrs: map[string]string{"ref": "user/app"},
},
},
},
},
{
name: "exports",
options: &Options{
Exports: []*pb.ExportEntry{
{
Type: "local",
Destination: "-",
},
{
Type: "local",
Destination: "test1",
},
{
Type: "tar",
Destination: "test3",
},
{
Type: "oci",
Destination: "-",
},
{
Type: "docker",
Destination: "test4",
},
{
Type: "image",
Attrs: map[string]string{"push": "true"},
},
},
},
want: &Options{
Exports: []*pb.ExportEntry{
{
Type: "local",
Destination: "-",
},
{
Type: "local",
Destination: filepath.Join(tmpwd, "test1"),
},
{
Type: "tar",
Destination: filepath.Join(tmpwd, "test3"),
},
{
Type: "oci",
Destination: "-",
},
{
Type: "docker",
Destination: filepath.Join(tmpwd, "test4"),
},
{
Type: "image",
Attrs: map[string]string{"push": "true"},
},
},
},
},
{
name: "secrets",
options: &Options{
Secrets: []*pb.Secret{
{
FilePath: "test1",
},
{
ID: "val",
Env: "a",
},
{
ID: "test",
FilePath: "test3",
},
},
},
want: &Options{
Secrets: []*pb.Secret{
{
FilePath: filepath.Join(tmpwd, "test1"),
},
{
ID: "val",
Env: "a",
},
{
ID: "test",
FilePath: filepath.Join(tmpwd, "test3"),
},
},
},
},
{
name: "ssh",
options: &Options{
SSH: []*pb.SSH{
{
ID: "default",
Paths: []string{"test1", "test2"},
},
{
ID: "a",
Paths: []string{"test3"},
},
},
},
want: &Options{
SSH: []*pb.SSH{
{
ID: "default",
Paths: []string{filepath.Join(tmpwd, "test1"), filepath.Join(tmpwd, "test2")},
},
{
ID: "a",
Paths: []string{filepath.Join(tmpwd, "test3")},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ResolveOptionPaths(tt.options)
require.NoError(t, err)
require.Equal(t, tt.want, got)
})
}
}
controller/errdefs/build.go
-20
View File
@@ -1,20 +0,0 @@
package errdefs
type BuildError struct {
err error
}
func (e *BuildError) Unwrap() error {
return e.err
}
func (e *BuildError) Error() string {
return e.err.Error()
}
func WrapBuild(err error) error {
if err == nil {
return nil
}
return &BuildError{err: err}
}
controller/pb/attest.go
-26
View File
@@ -1,26 +0,0 @@
package pb
type Attest struct {
Type string
Disabled bool
Attrs string
}
func CreateAttestations(attests []*Attest) map[string]*string {
result := map[string]*string{}
for _, attest := range attests {
// ignore duplicates
if _, ok := result[attest.Type]; ok {
continue
}
if attest.Disabled {
result[attest.Type] = nil
continue
}
attrs := attest.Attrs
result[attest.Type] = &attrs
}
return result
}
controller/pb/cache.go
-28
View File
@@ -1,28 +0,0 @@
package pb
import (
"maps"
"github.com/moby/buildkit/client"
)
type CacheOptionsEntry struct {
Type string
Attrs map[string]string
}
func CreateCaches(entries []*CacheOptionsEntry) []client.CacheOptionsEntry {
var outs []client.CacheOptionsEntry
if len(entries) == 0 {
return nil
}
for _, entry := range entries {
out := client.CacheOptionsEntry{
Type: entry.Type,
Attrs: map[string]string{},
}
maps.Copy(out.Attrs, entry.Attrs)
outs = append(outs, out)
}
return outs
}
controller/pb/export.go
-114
View File
@@ -1,114 +0,0 @@
package pb
import (
"io"
"maps"
"os"
"strconv"
"github.com/containerd/console"
"github.com/moby/buildkit/client"
"github.com/pkg/errors"
)
type ExportEntry struct {
Type string
Attrs map[string]string
Destination string
}
func CreateExports(entries []*ExportEntry) ([]client.ExportEntry, []string, error) {
var outs []client.ExportEntry
var localPaths []string
if len(entries) == 0 {
return nil, nil, nil
}
var stdoutUsed bool
for _, entry := range entries {
if entry.Type == "" {
return nil, nil, errors.Errorf("type is required for output")
}
out := client.ExportEntry{
Type: entry.Type,
Attrs: map[string]string{},
}
maps.Copy(out.Attrs, entry.Attrs)
supportFile := false
supportDir := false
switch out.Type {
case client.ExporterLocal:
supportDir = true
case client.ExporterTar:
supportFile = true
case client.ExporterOCI, client.ExporterDocker:
tar, err := strconv.ParseBool(out.Attrs["tar"])
if err != nil {
tar = true
}
supportFile = tar
supportDir = !tar
case "registry":
out.Type = client.ExporterImage
out.Attrs["push"] = "true"
}
if supportDir {
if entry.Destination == "" {
return nil, nil, errors.Errorf("dest is required for %s exporter", out.Type)
}
if entry.Destination == "-" {
return nil, nil, errors.Errorf("dest cannot be stdout for %s exporter", out.Type)
}
fi, err := os.Stat(entry.Destination)
if err != nil && !os.IsNotExist(err) {
return nil, nil, errors.Wrapf(err, "invalid destination directory: %s", entry.Destination)
}
if err == nil && !fi.IsDir() {
return nil, nil, errors.Errorf("destination directory %s is a file", entry.Destination)
}
out.OutputDir = entry.Destination
localPaths = append(localPaths, entry.Destination)
}
if supportFile {
if entry.Destination == "" && out.Type != client.ExporterDocker {
entry.Destination = "-"
}
if entry.Destination == "-" {
if stdoutUsed {
return nil, nil, errors.Errorf("multiple outputs configured to write to stdout")
}
if _, err := console.ConsoleFromFile(os.Stdout); err == nil {
return nil, nil, errors.Errorf("dest file is required for %s exporter. refusing to write to console", out.Type)
}
out.Output = wrapWriteCloser(os.Stdout)
stdoutUsed = true
} else if entry.Destination != "" {
fi, err := os.Stat(entry.Destination)
if err != nil && !os.IsNotExist(err) {
return nil, nil, errors.Wrapf(err, "invalid destination file: %s", entry.Destination)
}
if err == nil && fi.IsDir() {
return nil, nil, errors.Errorf("destination file %s is a directory", entry.Destination)
}
f, err := os.Create(entry.Destination)
if err != nil {
return nil, nil, errors.Errorf("failed to open %s", err)
}
out.Output = wrapWriteCloser(f)
localPaths = append(localPaths, entry.Destination)
}
}
outs = append(outs, out)
}
return outs, localPaths, nil
}
func wrapWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) {
return func(map[string]string) (io.WriteCloser, error) {
return wc, nil
}
}
controller/pb/invoke.go
-56
View File
@@ -1,56 +0,0 @@
package pb
import (
"fmt"
"strings"
)
type CallFunc struct {
Name string
Format string
IgnoreStatus bool
}
func (x *CallFunc) String() string {
var elems []string
if x.Name != "" {
elems = append(elems, fmt.Sprintf("Name:%q", x.Name))
}
if x.Format != "" {
elems = append(elems, fmt.Sprintf("Format:%q", x.Format))
}
if x.IgnoreStatus {
elems = append(elems, fmt.Sprintf("IgnoreStatus:%v", x.IgnoreStatus))
}
return strings.Join(elems, " ")
}
type InvokeConfig struct {
Entrypoint []string
Cmd []string
NoCmd bool
Env []string
User string
NoUser bool
Cwd string
NoCwd bool
Tty bool
Rollback bool
Initial bool
SuspendOn SuspendOn
}
func (cfg *InvokeConfig) NeedsDebug(err error) bool {
return cfg.SuspendOn.DebugEnabled(err)
}
type SuspendOn int
const (
SuspendError SuspendOn = iota
SuspendAlways
)
func (s SuspendOn) DebugEnabled(err error) bool {
return err != nil || s == SuspendAlways
}
controller/pb/secrets.go
-28
View File
@@ -1,28 +0,0 @@
package pb
import (
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/secrets/secretsprovider"
)
type Secret struct {
ID string
FilePath string
Env string
}
func CreateSecrets(secrets []*Secret) (session.Attachable, error) {
fs := make([]secretsprovider.Source, 0, len(secrets))
for _, secret := range secrets {
fs = append(fs, secretsprovider.Source{
ID: secret.ID,
FilePath: secret.FilePath,
Env: secret.Env,
})
}
store, err := secretsprovider.NewStore(fs)
if err != nil {
return nil, err
}
return secretsprovider.NewSecretProvider(store), nil
}
controller/pb/ssh.go
-25
View File
@@ -1,25 +0,0 @@
package pb
import (
"slices"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/sshforward/sshprovider"
)
type SSH struct {
ID string
Paths []string
}
func CreateSSH(ssh []*SSH) (session.Attachable, error) {
configs := make([]sshprovider.AgentConfig, 0, len(ssh))
for _, ssh := range ssh {
cfg := sshprovider.AgentConfig{
ID: ssh.ID,
Paths: slices.Clone(ssh.Paths),
}
configs = append(configs, cfg)
}
return sshprovider.NewSSHAgentProvider(configs)
}
controller/pb/ulimit.go
-11
View File
@@ -1,11 +0,0 @@
package pb
type UlimitOpt struct {
Values map[string]*Ulimit
}
type Ulimit struct {
Name string
Hard int64
Soft int64
}
docs/debugging.md
-41
View File
@@ -26,7 +26,6 @@ Arguments available after `buildx debug build` are the same as the normal `build
```console
$ docker buildx debug --invoke /bin/sh build .
[+] Building 4.2s (19/19) FINISHED
=> [internal] connecting to local controller 0.0s
=> [internal] load build definition from Dockerfile 0.0s
=> => transferring dockerfile: 32B 0.0s
=> [internal] load .dockerignore 0.0s
@@ -68,7 +67,6 @@ If you want to start a debug session when a build fails, you can use
```console
$ docker buildx debug --on=error build .
[+] Building 4.2s (19/19) FINISHED
=> [internal] connecting to local controller 0.0s
=> [internal] load build definition from Dockerfile 0.0s
=> => transferring dockerfile: 32B 0.0s
=> [internal] load .dockerignore 0.0s
@@ -94,7 +92,6 @@ can use `buildx debug` command to start a debug session.
```
$ docker buildx debug
[+] Building 4.2s (19/19) FINISHED
=> [internal] connecting to local controller 0.0s
(buildx)
```
@@ -125,41 +122,3 @@ Available commands are:
rollback re-runs the interactive container with the step's rootfs contents
```
## Build controllers
Debugging is performed using a buildx "controller", which provides a high-level
abstraction to perform builds. By default, the local controller is used for a
more stable experience which runs all builds in-process. However, you can also
use the remote controller to detach the build process from the CLI.
To detach the build process from the CLI, you can use the `--detach=true` flag with
the build command.
```console
$ docker buildx debug --invoke /bin/sh build --detach=true .
```
If you start a debugging session using the `--invoke` flag with a detached
build, then you can attach to it using the `buildx debug` command to
immediately enter the monitor mode.
```console
$ docker buildx debug
[+] Building 0.0s (1/1) FINISHED
=> [internal] connecting to remote controller
(buildx) list
ID CURRENT_SESSION
xfe1162ovd9def8yapb4ys66t false
(buildx) attach xfe1162ovd9def8yapb4ys66t
Attached to process "". Press Ctrl-a-c to switch to the new container
(buildx) ps
PID CURRENT_SESSION COMMAND
3ug8iqaufiwwnukimhqqt06jz false [sh]
(buildx) attach 3ug8iqaufiwwnukimhqqt06jz
Attached to process "3ug8iqaufiwwnukimhqqt06jz". Press Ctrl-a-c to switch to the new container
(buildx) Switched IO
/ # ls
bin etc lib mnt proc run srv tmp var
dev home media opt root sbin sys usr work
/ #
```
monitor/commands/exec.go
+4 -4
View File
@@ -5,7 +5,7 @@ import (
"fmt"
"io"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/build"
"github.com/docker/buildx/monitor/types"
"github.com/pkg/errors"
)
@@ -13,11 +13,11 @@ import (
type ExecCmd struct {
m types.Monitor
invokeConfig *controllerapi.InvokeConfig
invokeConfig *build.InvokeConfig
stdout io.WriteCloser
}
func NewExecCmd(m types.Monitor, invokeConfig *controllerapi.InvokeConfig, stdout io.WriteCloser) types.Command {
func NewExecCmd(m types.Monitor, invokeConfig *build.InvokeConfig, stdout io.WriteCloser) types.Command {
return &ExecCmd{m, invokeConfig, stdout}
}
@@ -38,7 +38,7 @@ func (cm *ExecCmd) Exec(ctx context.Context, args []string) error {
if len(args) < 2 {
return errors.Errorf("command must be passed")
}
cfg := &controllerapi.InvokeConfig{
cfg := &build.InvokeConfig{
Entrypoint: []string{args[1]},
Cmd: args[2:],
NoCmd: false,
monitor/commands/rollback.go
+3 -3
View File
@@ -5,18 +5,18 @@ import (
"fmt"
"io"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/build"
"github.com/docker/buildx/monitor/types"
)
type RollbackCmd struct {
m types.Monitor
invokeConfig *controllerapi.InvokeConfig
invokeConfig *build.InvokeConfig
stdout io.WriteCloser
}
func NewRollbackCmd(m types.Monitor, invokeConfig *controllerapi.InvokeConfig, stdout io.WriteCloser) types.Command {
func NewRollbackCmd(m types.Monitor, invokeConfig *build.InvokeConfig, stdout io.WriteCloser) types.Command {
return &RollbackCmd{m, invokeConfig, stdout}
}
monitor/monitor.go
+10 -11
View File
@@ -12,9 +12,8 @@ import (
"github.com/containerd/console"
"github.com/docker/buildx/build"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/controller/processes"
"github.com/docker/buildx/monitor/commands"
"github.com/docker/buildx/monitor/processes"
"github.com/docker/buildx/monitor/types"
"github.com/docker/buildx/util/ioset"
"github.com/docker/buildx/util/progress"
@@ -29,7 +28,7 @@ import (
var ErrReload = errors.New("monitor: reload")
type Monitor struct {
invokeConfig *controllerapi.InvokeConfig
invokeConfig *build.InvokeConfig
printer *progress.Printer
stdin *ioset.SingleForwarder
@@ -41,7 +40,7 @@ type Monitor struct {
mu sync.Mutex
}
func New(cfg *controllerapi.InvokeConfig, stdin io.ReadCloser, stdout, stderr io.WriteCloser, printer *progress.Printer) *Monitor {
func New(cfg *build.InvokeConfig, stdin io.ReadCloser, stdout, stderr io.WriteCloser, printer *progress.Printer) *Monitor {
m := &Monitor{
invokeConfig: cfg,
printer: printer,
@@ -113,7 +112,7 @@ func (m *Monitor) Close() error {
}
// RunMonitor provides an interactive session for running and managing containers via specified IO.
func RunMonitor(ctx context.Context, invokeConfig *controllerapi.InvokeConfig, rCtx *build.ResultHandle, stdin io.ReadCloser, stdout, stderr io.WriteCloser, progress *progress.Printer) error {
func RunMonitor(ctx context.Context, invokeConfig *build.InvokeConfig, rCtx *build.ResultHandle, stdin io.ReadCloser, stdout, stderr io.WriteCloser, progress *progress.Printer) error {
if err := progress.Pause(); err != nil {
return err
}
@@ -333,7 +332,7 @@ type monitor struct {
processes *processes.Manager
}
func (m *monitor) Invoke(ctx context.Context, pid string, cfg *controllerapi.InvokeConfig, ioIn io.ReadCloser, ioOut io.WriteCloser, ioErr io.WriteCloser) error {
func (m *monitor) Invoke(ctx context.Context, pid string, cfg *build.InvokeConfig, ioIn io.ReadCloser, ioOut io.WriteCloser, ioErr io.WriteCloser) error {
proc, ok := m.processes.Get(pid)
if !ok {
// Start a new process.
@@ -361,19 +360,19 @@ func (m *monitor) Invoke(ctx context.Context, pid string, cfg *controllerapi.Inv
}
}
func (m *monitor) Rollback(ctx context.Context, cfg *controllerapi.InvokeConfig) string {
func (m *monitor) Rollback(ctx context.Context, cfg *build.InvokeConfig) string {
pid := identity.NewID()
cfg1 := cfg
cfg1.Rollback = true
return m.startInvoke(ctx, pid, cfg1)
}
func (m *monitor) Exec(ctx context.Context, cfg *controllerapi.InvokeConfig) string {
func (m *monitor) Exec(ctx context.Context, cfg *build.InvokeConfig) string {
return m.startInvoke(ctx, identity.NewID(), cfg)
}
func (m *monitor) Attach(ctx context.Context, pid string) {
m.startInvoke(ctx, pid, &controllerapi.InvokeConfig{})
m.startInvoke(ctx, pid, &build.InvokeConfig{})
}
func (m *monitor) Detach() {
@@ -394,7 +393,7 @@ func (m *monitor) close() {
m.Detach()
}
func (m *monitor) startInvoke(ctx context.Context, pid string, cfg *controllerapi.InvokeConfig) string {
func (m *monitor) startInvoke(ctx context.Context, pid string, cfg *build.InvokeConfig) string {
if m.invokeCancel != nil {
m.invokeCancel() // Finish existing attach
}
@@ -420,7 +419,7 @@ func (m *monitor) startInvoke(ctx context.Context, pid string, cfg *controllerap
return pid
}
func (m *monitor) invoke(ctx context.Context, pid string, cfg *controllerapi.InvokeConfig) error {
func (m *monitor) invoke(ctx context.Context, pid string, cfg *build.InvokeConfig) error {
m.muxIO.Enable(1)
defer m.muxIO.Disable(1)
if err := m.muxIO.SwitchTo(1); err != nil {
controller/processes/processes.go → monitor/processes/processes.go
+3 -4
View File
@@ -6,7 +6,6 @@ import (
"sync/atomic"
"github.com/docker/buildx/build"
"github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/util/ioset"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ import (
// Process provides methods to control a process.
type Process struct {
inEnd *ioset.Forwarder
invokeConfig *pb.InvokeConfig
invokeConfig *build.InvokeConfig
errCh chan error
processCancel func()
serveIOCancel func(error)
@@ -98,7 +97,7 @@ func (m *Manager) DeleteProcess(id string) error {
// When a container isn't available (i.e. first time invoking or the container has exited) or cfg.Rollback is set,
// this method will start a new container and run the process in it. Otherwise, this method starts a new process in the
// existing container.
func (m *Manager) StartProcess(pid string, resultCtx *build.ResultHandle, cfg *pb.InvokeConfig) (*Process, error) {
func (m *Manager) StartProcess(pid string, resultCtx *build.ResultHandle, cfg *build.InvokeConfig) (*Process, error) {
// Get the target result to invoke a container from
var ctr *build.Container
if a := m.container.Load(); a != nil {
@@ -157,5 +156,5 @@ func (m *Manager) StartProcess(pid string, resultCtx *build.ResultHandle, cfg *p
type ProcessInfo struct {
ProcessID string
InvokeConfig *pb.InvokeConfig
InvokeConfig *build.InvokeConfig
}
monitor/types/types.go
+5 -5
View File
@@ -4,8 +4,8 @@ import (
"context"
"io"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/docker/buildx/controller/processes"
"github.com/docker/buildx/build"
"github.com/docker/buildx/monitor/processes"
)
// Monitor provides APIs for attaching and controlling the buildx server.
@@ -14,17 +14,17 @@ type Monitor interface {
// If pid doesn't match to any running processes, it starts a new process with the specified config.
// If there is no container running or InvokeConfig.Rollback is specified, the process will start in a newly created container.
// NOTE: If needed, in the future, we can split this API into three APIs (NewContainer, NewProcess and Attach).
Invoke(ctx context.Context, pid string, options *controllerapi.InvokeConfig, ioIn io.ReadCloser, ioOut io.WriteCloser, ioErr io.WriteCloser) error
Invoke(ctx context.Context, pid string, options *build.InvokeConfig, ioIn io.ReadCloser, ioOut io.WriteCloser, ioErr io.WriteCloser) error
ListProcesses(ctx context.Context) (infos []*processes.ProcessInfo, retErr error)
DisconnectProcess(ctx context.Context, pid string) error
// Rollback re-runs the interactive container with initial rootfs contents.
Rollback(ctx context.Context, cfg *controllerapi.InvokeConfig) string
Rollback(ctx context.Context, cfg *build.InvokeConfig) string
// Rollback executes a process in the interactive container.
Exec(ctx context.Context, cfg *controllerapi.InvokeConfig) string
Exec(ctx context.Context, cfg *build.InvokeConfig) string
// Attach attaches IO to a process in the container.
Attach(ctx context.Context, pid string)
util/buildflags/attests.go
+26 -87
View File
@@ -7,7 +7,6 @@ import (
"strconv"
"strings"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/pkg/errors"
"github.com/tonistiigi/go-csvvalue"
)
@@ -33,16 +32,32 @@ func (a Attests) Normalize() Attests {
return removeAttestDupes(a)
}
func (a Attests) ToPB() []*controllerapi.Attest {
if len(a) == 0 {
return nil
}
func (a Attests) ToMap() map[string]*string {
result := map[string]*string{}
for _, attest := range a {
// ignore duplicates
if _, ok := result[attest.Type]; ok {
continue
}
entries := make([]*controllerapi.Attest, len(a))
for i, entry := range a {
entries[i] = entry.ToPB()
if attest.Disabled {
result[attest.Type] = nil
continue
}
var b csvBuilder
if attest.Type != "" {
b.Write("type", attest.Type)
}
if attest.Disabled {
b.Write("disabled", "true")
}
b.WriteAttributes(attest.Attrs)
s := b.String()
result[attest.Type] = &s
}
return entries
return result
}
type Attest struct {
@@ -72,23 +87,6 @@ func (a *Attest) String() string {
return b.String()
}
func (a *Attest) ToPB() *controllerapi.Attest {
var b csvBuilder
if a.Type != "" {
b.Write("type", a.Type)
}
if a.Disabled {
b.Write("disabled", "true")
}
b.WriteAttributes(a.Attrs)
return &controllerapi.Attest{
Type: a.Type,
Disabled: a.Disabled,
Attrs: b.String(),
}
}
func (a *Attest) MarshalJSON() ([]byte, error) {
m := make(map[string]any, len(a.Attrs)+2)
for k, v := range a.Attrs {
@@ -182,7 +180,7 @@ func CanonicalizeAttest(attestType string, in string) string {
return fmt.Sprintf("type=%s,%s", attestType, in)
}
func ParseAttests(in []string) ([]*controllerapi.Attest, error) {
func ParseAttests(in []string) (Attests, error) {
var outs []*Attest
for _, s := range in {
var out Attest
@@ -191,66 +189,7 @@ func ParseAttests(in []string) ([]*controllerapi.Attest, error) {
}
outs = append(outs, &out)
}
return ConvertAttests(outs)
}
// ConvertAttests converts Attestations for the controller API from
// the ones in this package.
//
// Attestations of the same type will cause an error. Some tools,
// like bake, remove the duplicates before calling this function.
func ConvertAttests(in []*Attest) ([]*controllerapi.Attest, error) {
out := make([]*controllerapi.Attest, 0, len(in))
// Check for duplicate attestations while we convert them
// to the controller API.
found := map[string]struct{}{}
for _, attest := range in {
if _, ok := found[attest.Type]; ok {
return nil, errors.Errorf("duplicate attestation field %s", attest.Type)
}
found[attest.Type] = struct{}{}
out = append(out, attest.ToPB())
}
return out, nil
}
func ParseAttest(in string) (*controllerapi.Attest, error) {
if in == "" {
return nil, nil
}
fields, err := csvvalue.Fields(in, nil)
if err != nil {
return nil, err
}
attest := controllerapi.Attest{
Attrs: in,
}
for _, field := range fields {
key, value, ok := strings.Cut(field, "=")
if !ok {
return nil, errors.Errorf("invalid value %s", field)
}
key = strings.TrimSpace(strings.ToLower(key))
switch key {
case "type":
attest.Type = value
case "disabled":
disabled, err := strconv.ParseBool(value)
if err != nil {
return nil, errors.Wrapf(err, "invalid value %s", field)
}
attest.Disabled = disabled
}
}
if attest.Type == "" {
return nil, errors.Errorf("attestation type not specified")
}
return &attest, nil
return outs, nil
}
func removeAttestDupes(s []*Attest) []*Attest {
util/buildflags/cache.go
-103
View File
@@ -1,15 +1,10 @@
package buildflags
import (
"context"
"encoding/json"
"maps"
"os"
"strconv"
"strings"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/pkg/errors"
"github.com/tonistiigi/go-csvvalue"
"github.com/zclconf/go-cty/cty"
@@ -35,22 +30,6 @@ func (o CacheOptions) Normalize() CacheOptions {
return removeDupes(o)
}
func (o CacheOptions) ToPB() []*controllerapi.CacheOptionsEntry {
if len(o) == 0 {
return nil
}
var outs []*controllerapi.CacheOptionsEntry
for _, entry := range o {
pb := entry.ToPB()
if !isActive(pb) {
continue
}
outs = append(outs, pb)
}
return outs
}
type CacheOptionsEntry struct {
Type string `json:"type"`
Attrs map[string]string `json:"attrs,omitempty"`
@@ -81,16 +60,6 @@ func (e *CacheOptionsEntry) String() string {
return b.String()
}
func (e *CacheOptionsEntry) ToPB() *controllerapi.CacheOptionsEntry {
ci := &controllerapi.CacheOptionsEntry{
Type: e.Type,
Attrs: maps.Clone(e.Attrs),
}
addGithubToken(ci)
addAwsCredentials(ci)
return ci
}
func (e *CacheOptionsEntry) MarshalJSON() ([]byte, error) {
m := maps.Clone(e.Attrs)
if m == nil {
@@ -204,75 +173,3 @@ func ParseCacheEntry(in []string) (CacheOptions, error) {
}
return opts, nil
}
func addGithubToken(ci *controllerapi.CacheOptionsEntry) {
if ci.Type != "gha" {
return
}
version, ok := ci.Attrs["version"]
if !ok {
// https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L19
if v, ok := os.LookupEnv("ACTIONS_CACHE_SERVICE_V2"); ok {
if b, err := strconv.ParseBool(v); err == nil && b {
version = "2"
}
}
}
if _, ok := ci.Attrs["token"]; !ok {
if v, ok := os.LookupEnv("ACTIONS_RUNTIME_TOKEN"); ok {
ci.Attrs["token"] = v
}
}
if _, ok := ci.Attrs["url_v2"]; !ok && version == "2" {
// https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L34-L35
if v, ok := os.LookupEnv("ACTIONS_RESULTS_URL"); ok {
ci.Attrs["url_v2"] = v
}
}
if _, ok := ci.Attrs["url"]; !ok {
// https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L28-L33
if v, ok := os.LookupEnv("ACTIONS_CACHE_URL"); ok {
ci.Attrs["url"] = v
} else if v, ok := os.LookupEnv("ACTIONS_RESULTS_URL"); ok {
ci.Attrs["url"] = v
}
}
}
func addAwsCredentials(ci *controllerapi.CacheOptionsEntry) {
if ci.Type != "s3" {
return
}
_, okAccessKeyID := ci.Attrs["access_key_id"]
_, okSecretAccessKey := ci.Attrs["secret_access_key"]
// If the user provides access_key_id, secret_access_key, do not override the session token.
if okAccessKeyID && okSecretAccessKey {
return
}
ctx := context.TODO()
awsConfig, err := awsconfig.LoadDefaultConfig(ctx)
if err != nil {
return
}
credentials, err := awsConfig.Credentials.Retrieve(ctx)
if err != nil {
return
}
if !okAccessKeyID && credentials.AccessKeyID != "" {
ci.Attrs["access_key_id"] = credentials.AccessKeyID
}
if !okSecretAccessKey && credentials.SecretAccessKey != "" {
ci.Attrs["secret_access_key"] = credentials.SecretAccessKey
}
if _, ok := ci.Attrs["session_token"]; !ok && credentials.SessionToken != "" {
ci.Attrs["session_token"] = credentials.SessionToken
}
}
func isActive(pb *controllerapi.CacheOptionsEntry) bool {
// Always active if not gha.
if pb.Type != "gha" {
return true
}
return pb.Attrs["token"] != "" && (pb.Attrs["url"] != "" || pb.Attrs["url_v2"] != "")
}
util/buildflags/cache_test.go
-32
View File
@@ -4,42 +4,10 @@ import (
"encoding/json"
"testing"
"github.com/docker/buildx/controller/pb"
"github.com/stretchr/testify/require"
"github.com/zclconf/go-cty/cty"
)
func TestCacheOptions_DerivedVars(t *testing.T) {
t.Setenv("ACTIONS_RUNTIME_TOKEN", "sensitive_token")
t.Setenv("ACTIONS_CACHE_URL", "https://cache.github.com")
t.Setenv("AWS_ACCESS_KEY_ID", "definitely_dont_look_here")
t.Setenv("AWS_SECRET_ACCESS_KEY", "hackers_please_dont_steal")
t.Setenv("AWS_SESSION_TOKEN", "not_a_mitm_attack")
cacheFrom, err := ParseCacheEntry([]string{"type=gha", "type=s3,region=us-west-2,bucket=my_bucket,name=my_image"})
require.NoError(t, err)
require.Equal(t, []*pb.CacheOptionsEntry{
{
Type: "gha",
Attrs: map[string]string{
"token": "sensitive_token",
"url": "https://cache.github.com",
},
},
{
Type: "s3",
Attrs: map[string]string{
"region": "us-west-2",
"bucket": "my_bucket",
"name": "my_image",
"access_key_id": "definitely_dont_look_here",
"secret_access_key": "hackers_please_dont_steal",
"session_token": "not_a_mitm_attack",
},
},
}, cacheFrom.ToPB())
}
func TestCacheOptions(t *testing.T) {
t.Run("MarshalJSON", func(t *testing.T) {
cache := CacheOptions{
util/buildflags/callfunc.go
+23 -3
View File
@@ -1,17 +1,37 @@
package buildflags
import (
"fmt"
"strconv"
"strings"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/pkg/errors"
"github.com/tonistiigi/go-csvvalue"
)
const defaultCallFunc = "build"
func ParseCallFunc(str string) (*controllerapi.CallFunc, error) {
type CallFunc struct {
Name string
Format string
IgnoreStatus bool
}
func (x *CallFunc) String() string {
var elems []string
if x.Name != "" {
elems = append(elems, fmt.Sprintf("Name:%q", x.Name))
}
if x.Format != "" {
elems = append(elems, fmt.Sprintf("Format:%q", x.Format))
}
if x.IgnoreStatus {
elems = append(elems, fmt.Sprintf("IgnoreStatus:%v", x.IgnoreStatus))
}
return strings.Join(elems, " ")
}
func ParseCallFunc(str string) (*CallFunc, error) {
if str == "" {
return nil, nil
}
@@ -20,7 +40,7 @@ func ParseCallFunc(str string) (*controllerapi.CallFunc, error) {
if err != nil {
return nil, err
}
f := &controllerapi.CallFunc{}
f := &CallFunc{}
for _, field := range fields {
parts := strings.SplitN(field, "=", 2)
if len(parts) == 2 {
util/buildflags/export.go
+4 -25
View File
@@ -9,7 +9,6 @@ import (
"strings"
"github.com/containerd/platforms"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/exporter/containerimage/exptypes"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
@@ -38,18 +37,6 @@ func (e Exports) Normalize() Exports {
return removeDupes(e)
}
func (e Exports) ToPB() []*controllerapi.ExportEntry {
if len(e) == 0 {
return nil
}
entries := make([]*controllerapi.ExportEntry, len(e))
for i, entry := range e {
entries[i] = entry.ToPB()
}
return entries
}
type ExportEntry struct {
Type string `json:"type"`
Attrs map[string]string `json:"attrs,omitempty"`
@@ -77,14 +64,6 @@ func (e *ExportEntry) String() string {
return b.String()
}
func (e *ExportEntry) ToPB() *controllerapi.ExportEntry {
return &controllerapi.ExportEntry{
Type: e.Type,
Attrs: maps.Clone(e.Attrs),
Destination: e.Destination,
}
}
func (e *ExportEntry) MarshalJSON() ([]byte, error) {
m := maps.Clone(e.Attrs)
if m == nil {
@@ -164,12 +143,12 @@ func (e *ExportEntry) validate() error {
return nil
}
func ParseExports(inp []string) ([]*controllerapi.ExportEntry, error) {
func ParseExports(inp []string) ([]*ExportEntry, error) {
if len(inp) == 0 {
return nil, nil
}
export := make(Exports, 0, len(inp))
exports := make(Exports, 0, len(inp))
for _, s := range inp {
if s == "" {
continue
@@ -179,9 +158,9 @@ func ParseExports(inp []string) ([]*controllerapi.ExportEntry, error) {
if err := out.UnmarshalText([]byte(s)); err != nil {
return nil, err
}
export = append(export, &out)
exports = append(exports, &out)
}
return export.ToPB(), nil
return exports, nil
}
func ParseAnnotations(inp []string) (map[exptypes.AnnotationKey]string, error) {
util/buildflags/secrets.go
+4 -25
View File
@@ -4,7 +4,6 @@ import (
"encoding/json"
"strings"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/pkg/errors"
"github.com/tonistiigi/go-csvvalue"
)
@@ -30,18 +29,6 @@ func (s Secrets) Normalize() Secrets {
return removeSecretDupes(s)
}
func (s Secrets) ToPB() []*controllerapi.Secret {
if len(s) == 0 {
return nil
}
entries := make([]*controllerapi.Secret, len(s))
for i, entry := range s {
entries[i] = entry.ToPB()
}
return entries
}
type Secret struct {
ID string `json:"id,omitempty"`
FilePath string `json:"src,omitempty"`
@@ -66,14 +53,6 @@ func (s *Secret) String() string {
return b.String()
}
func (s *Secret) ToPB() *controllerapi.Secret {
return &controllerapi.Secret{
ID: s.ID,
FilePath: s.FilePath,
Env: s.Env,
}
}
func (s *Secret) UnmarshalJSON(data []byte) error {
var v struct {
ID string `json:"id,omitempty"`
@@ -132,8 +111,8 @@ func (s *Secret) UnmarshalText(text []byte) error {
return nil
}
func ParseSecretSpecs(sl []string) ([]*controllerapi.Secret, error) {
fs := make([]*controllerapi.Secret, 0, len(sl))
func ParseSecretSpecs(sl []string) (Secrets, error) {
fs := make([]*Secret, 0, len(sl))
for _, v := range sl {
if v == "" {
continue
@@ -148,12 +127,12 @@ func ParseSecretSpecs(sl []string) ([]*controllerapi.Secret, error) {
return fs, nil
}
func parseSecret(value string) (*controllerapi.Secret, error) {
func parseSecret(value string) (*Secret, error) {
var s Secret
if err := s.UnmarshalText([]byte(value)); err != nil {
return nil, err
}
return s.ToPB(), nil
return &s, nil
}
func removeSecretDupes(s []*Secret) []*Secret {
util/buildflags/ssh.go
+3 -23
View File
@@ -6,7 +6,6 @@ import (
"slices"
"strings"
controllerapi "github.com/docker/buildx/controller/pb"
"github.com/moby/buildkit/util/gitutil"
)
@@ -31,18 +30,6 @@ func (s SSHKeys) Normalize() SSHKeys {
return removeSSHDupes(s)
}
func (s SSHKeys) ToPB() []*controllerapi.SSH {
if len(s) == 0 {
return nil
}
entries := make([]*controllerapi.SSH, len(s))
for i, entry := range s {
entries[i] = entry.ToPB()
}
return entries
}
type SSH struct {
ID string `json:"id,omitempty" cty:"id"`
Paths []string `json:"paths,omitempty" cty:"paths"`
@@ -70,13 +57,6 @@ func (s *SSH) String() string {
return b.String()
}
func (s *SSH) ToPB() *controllerapi.SSH {
return &controllerapi.SSH{
ID: s.ID,
Paths: s.Paths,
}
}
func (s *SSH) UnmarshalJSON(data []byte) error {
var v struct {
ID string `json:"id,omitempty"`
@@ -103,8 +83,8 @@ func (s *SSH) UnmarshalText(text []byte) error {
return nil
}
func ParseSSHSpecs(sl []string) ([]*controllerapi.SSH, error) {
var outs []*controllerapi.SSH
func ParseSSHSpecs(sl []string) ([]*SSH, error) {
var outs []*SSH
if len(sl) == 0 {
return nil, nil
}
@@ -118,7 +98,7 @@ func ParseSSHSpecs(sl []string) ([]*controllerapi.SSH, error) {
if err := out.UnmarshalText([]byte(s)); err != nil {
return nil, err
}
outs = append(outs, out.ToPB())
outs = append(outs, &out)
}
return outs, nil
}