diff --git a/.github/labeler.yml b/.github/labeler.yml index 45bef67ac..ed39bc80a 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -48,11 +48,6 @@ area/cli: - cmd/** - commands/** -# Add 'area/controller' label to changes in the controller -area/controller: - - changed-files: - - any-glob-to-any-file: 'controller/**' - # Add 'area/docs' label to markdown files in the docs folder area/docs: - changed-files: diff --git a/PROJECT.md b/PROJECT.md index ee29867dd..2c318d67c 100644 --- a/PROJECT.md +++ b/PROJECT.md @@ -79,7 +79,6 @@ Area or component of the project affected. Please note that the table below may | `area/checks` | Any | `checks` | | `area/ci` | Any | Project CI | | `area/cli` | Any | `cli` | -| `area/controller` | Any | `controller` | | `area/debug` | Any | `debug` | | `area/dependencies` | Any | Project dependencies | | `area/dockerfile` | Any | `dockerfile` | diff --git a/bake/bake.go b/bake/bake.go index ad626ee6b..ca4cb277d 100644 --- a/bake/bake.go +++ b/bake/bake.go @@ -19,7 +19,6 @@ import ( composecli "github.com/compose-spec/compose-go/v2/cli" "github.com/docker/buildx/bake/hclparser" "github.com/docker/buildx/build" - controllerapi "github.com/docker/buildx/controller/pb" "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/platformutil" "github.com/docker/buildx/util/progress" @@ -1439,20 +1438,19 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) { }) } } - secrets = secrets.Normalize() - bo.SecretSpecs = secrets.ToPB() - secretAttachment, err := controllerapi.CreateSecrets(bo.SecretSpecs) + bo.SecretSpecs = secrets.Normalize() + secretAttachment, err := build.CreateSecrets(bo.SecretSpecs) if err != nil { return nil, err } bo.Session = append(bo.Session, secretAttachment) - bo.SSHSpecs = t.SSH.ToPB() + bo.SSHSpecs = t.SSH if len(bo.SSHSpecs) == 0 && buildflags.IsGitSSH(bi.ContextPath) || (inp != nil && buildflags.IsGitSSH(inp.URL)) { - bo.SSHSpecs = []*controllerapi.SSH{{ID: "default"}} + bo.SSHSpecs = []*buildflags.SSH{{ID: "default"}} } - sshAttachment, err := controllerapi.CreateSSH(bo.SSHSpecs) + sshAttachment, err := build.CreateSSH(bo.SSHSpecs) if err != nil { return nil, err } @@ -1469,13 +1467,13 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) { } if t.CacheFrom != nil { - bo.CacheFrom = controllerapi.CreateCaches(t.CacheFrom.ToPB()) + bo.CacheFrom = build.CreateCaches(t.CacheFrom) } if t.CacheTo != nil { - bo.CacheTo = controllerapi.CreateCaches(t.CacheTo.ToPB()) + bo.CacheTo = build.CreateCaches(t.CacheTo) } - bo.Exports, bo.ExportsLocalPathsTemporary, err = controllerapi.CreateExports(t.Outputs.ToPB()) + bo.Exports, bo.ExportsLocalPathsTemporary, err = build.CreateExports(t.Outputs) if err != nil { return nil, err } @@ -1490,7 +1488,7 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) { } } - bo.Attests = controllerapi.CreateAttestations(t.Attest.ToPB()) + bo.Attests = t.Attest.ToMap() bo.SourcePolicy, err = build.ReadSourcePolicy() if err != nil { diff --git a/bake/entitlements_test.go b/bake/entitlements_test.go index 2bd2a5ba8..b00cfe239 100644 --- a/bake/entitlements_test.go +++ b/bake/entitlements_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/docker/buildx/build" - "github.com/docker/buildx/controller/pb" + "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/osutil" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/util/entitlements" @@ -264,7 +264,7 @@ func TestValidateEntitlements(t *testing.T) { { name: "SSHMissing", opt: build.Options{ - SSHSpecs: []*pb.SSH{ + SSHSpecs: []*buildflags.SSH{ { ID: "test", }, @@ -296,7 +296,7 @@ func TestValidateEntitlements(t *testing.T) { { name: "SecretFromSubFile", opt: build.Options{ - SecretSpecs: []*pb.Secret{ + SecretSpecs: []*buildflags.Secret{ { FilePath: filepath.Join(dir1, "subfile"), }, @@ -309,7 +309,7 @@ func TestValidateEntitlements(t *testing.T) { { name: "SecretFromEscapeLink", opt: build.Options{ - SecretSpecs: []*pb.Secret{ + SecretSpecs: []*buildflags.Secret{ { FilePath: escapeLink, }, @@ -325,7 +325,7 @@ func TestValidateEntitlements(t *testing.T) { { name: "SecretFromEscapeLinkAllowRoot", opt: build.Options{ - SecretSpecs: []*pb.Secret{ + SecretSpecs: []*buildflags.Secret{ { FilePath: escapeLink, }, @@ -352,7 +352,7 @@ func TestValidateEntitlements(t *testing.T) { { name: "SecretFromEscapeLinkAllowAny", opt: build.Options{ - SecretSpecs: []*pb.Secret{ + SecretSpecs: []*buildflags.Secret{ { FilePath: escapeLink, }, diff --git a/bake/remote.go b/bake/remote.go index 9b7cd8c54..ab8f330b7 100644 --- a/bake/remote.go +++ b/bake/remote.go @@ -7,9 +7,10 @@ import ( "os" "strings" + "github.com/docker/buildx/build" "github.com/docker/buildx/builder" - controllerapi "github.com/docker/buildx/controller/pb" "github.com/docker/buildx/driver" + "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/progress" "github.com/docker/go-units" "github.com/moby/buildkit/client" @@ -33,27 +34,27 @@ func ReadRemoteFiles(ctx context.Context, nodes []builder.Node, url string, name st, ok := dockerui.DetectGitContext(url, false) if ok { - if ssh, err := controllerapi.CreateSSH([]*controllerapi.SSH{{ + if ssh, err := build.CreateSSH([]*buildflags.SSH{{ ID: "default", Paths: strings.Split(os.Getenv("BUILDX_BAKE_GIT_SSH"), ","), }}); err == nil { sessions = append(sessions, ssh) } - var gitAuthSecrets []*controllerapi.Secret + var gitAuthSecrets []*buildflags.Secret if _, ok := os.LookupEnv("BUILDX_BAKE_GIT_AUTH_TOKEN"); ok { - gitAuthSecrets = append(gitAuthSecrets, &controllerapi.Secret{ + gitAuthSecrets = append(gitAuthSecrets, &buildflags.Secret{ ID: llb.GitAuthTokenKey, Env: "BUILDX_BAKE_GIT_AUTH_TOKEN", }) } if _, ok := os.LookupEnv("BUILDX_BAKE_GIT_AUTH_HEADER"); ok { - gitAuthSecrets = append(gitAuthSecrets, &controllerapi.Secret{ + gitAuthSecrets = append(gitAuthSecrets, &buildflags.Secret{ ID: llb.GitAuthHeaderKey, Env: "BUILDX_BAKE_GIT_AUTH_HEADER", }) } if len(gitAuthSecrets) > 0 { - if secrets, err := controllerapi.CreateSecrets(gitAuthSecrets); err == nil { + if secrets, err := build.CreateSecrets(gitAuthSecrets); err == nil { sessions = append(sessions, secrets) } } diff --git a/build/build.go b/build/build.go index a8f691286..6c20d0eb3 100644 --- a/build/build.go +++ b/build/build.go @@ -19,8 +19,8 @@ import ( "github.com/containerd/containerd/v2/core/images" "github.com/distribution/reference" "github.com/docker/buildx/builder" - controllerapi "github.com/docker/buildx/controller/pb" "github.com/docker/buildx/driver" + "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/confutil" "github.com/docker/buildx/util/desktop" "github.com/docker/buildx/util/dockerutil" @@ -78,8 +78,8 @@ type Options struct { NoCacheFilter []string Platforms []ocispecs.Platform Pull bool - SecretSpecs []*controllerapi.Secret - SSHSpecs []*controllerapi.SSH + SecretSpecs buildflags.Secrets + SSHSpecs []*buildflags.SSH ShmSize opts.MemBytes Tags []string Target string diff --git a/build/invoke.go b/build/invoke.go index 5c84188cf..5075530b3 100644 --- a/build/invoke.go +++ b/build/invoke.go @@ -8,12 +8,41 @@ import ( "sync/atomic" "syscall" - controllerapi "github.com/docker/buildx/controller/pb" gateway "github.com/moby/buildkit/frontend/gateway/client" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) +type InvokeConfig struct { + Entrypoint []string + Cmd []string + NoCmd bool + Env []string + User string + NoUser bool + Cwd string + NoCwd bool + Tty bool + Rollback bool + Initial bool + SuspendOn SuspendOn +} + +func (cfg *InvokeConfig) NeedsDebug(err error) bool { + return cfg.SuspendOn.DebugEnabled(err) +} + +type SuspendOn int + +const ( + SuspendError SuspendOn = iota + SuspendAlways +) + +func (s SuspendOn) DebugEnabled(err error) bool { + return err != nil || s == SuspendAlways +} + type Container struct { cancelOnce sync.Once containerCancel func(error) @@ -24,7 +53,7 @@ type Container struct { resultCtx *ResultHandle } -func NewContainer(ctx context.Context, resultCtx *ResultHandle, cfg *controllerapi.InvokeConfig) (*Container, error) { +func NewContainer(ctx context.Context, resultCtx *ResultHandle, cfg *InvokeConfig) (*Container, error) { mainCtx := ctx ctrCh := make(chan *Container) @@ -97,7 +126,7 @@ func (c *Container) markUnavailable() { c.isUnavailable.Store(true) } -func (c *Container) Exec(ctx context.Context, cfg *controllerapi.InvokeConfig, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error { +func (c *Container) Exec(ctx context.Context, cfg *InvokeConfig, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error { if isInit := c.initStarted.CompareAndSwap(false, true); isInit { defer func() { // container can't be used after init exits @@ -112,7 +141,7 @@ func (c *Container) Exec(ctx context.Context, cfg *controllerapi.InvokeConfig, s return err } -func exec(ctx context.Context, resultCtx *ResultHandle, cfg *controllerapi.InvokeConfig, ctr gateway.Container, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error { +func exec(ctx context.Context, resultCtx *ResultHandle, cfg *InvokeConfig, ctr gateway.Container, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error { processCfg, err := resultCtx.getProcessConfig(cfg, stdin, stdout, stderr) if err != nil { return err diff --git a/build/opt.go b/build/opt.go index a1014022a..7770aad36 100644 --- a/build/opt.go +++ b/build/opt.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "io" + "maps" "os" "path/filepath" "slices" @@ -11,12 +12,15 @@ import ( "strings" "syscall" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/containerd/console" "github.com/containerd/containerd/v2/core/content" "github.com/containerd/containerd/v2/plugins/content/local" "github.com/containerd/platforms" "github.com/distribution/reference" "github.com/docker/buildx/builder" "github.com/docker/buildx/driver" + "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/confutil" "github.com/docker/buildx/util/dockerutil" "github.com/docker/buildx/util/osutil" @@ -26,6 +30,9 @@ import ( "github.com/moby/buildkit/client/ociindex" gateway "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/secrets/secretsprovider" + "github.com/moby/buildkit/session/sshforward/sshprovider" "github.com/moby/buildkit/session/upload/uploadprovider" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/apicaps" @@ -659,3 +666,221 @@ type fs struct { } var _ fsutil.FS = &fs{} + +func CreateSSH(ssh []*buildflags.SSH) (session.Attachable, error) { + configs := make([]sshprovider.AgentConfig, 0, len(ssh)) + for _, ssh := range ssh { + cfg := sshprovider.AgentConfig{ + ID: ssh.ID, + Paths: slices.Clone(ssh.Paths), + } + configs = append(configs, cfg) + } + return sshprovider.NewSSHAgentProvider(configs) +} + +func CreateSecrets(secrets []*buildflags.Secret) (session.Attachable, error) { + fs := make([]secretsprovider.Source, 0, len(secrets)) + for _, secret := range secrets { + fs = append(fs, secretsprovider.Source{ + ID: secret.ID, + FilePath: secret.FilePath, + Env: secret.Env, + }) + } + store, err := secretsprovider.NewStore(fs) + if err != nil { + return nil, err + } + return secretsprovider.NewSecretProvider(store), nil +} + +func CreateExports(entries []*buildflags.ExportEntry) ([]client.ExportEntry, []string, error) { + var outs []client.ExportEntry + var localPaths []string + if len(entries) == 0 { + return nil, nil, nil + } + var stdoutUsed bool + for _, entry := range entries { + if entry.Type == "" { + return nil, nil, errors.Errorf("type is required for output") + } + + out := client.ExportEntry{ + Type: entry.Type, + Attrs: map[string]string{}, + } + maps.Copy(out.Attrs, entry.Attrs) + + supportFile := false + supportDir := false + switch out.Type { + case client.ExporterLocal: + supportDir = true + case client.ExporterTar: + supportFile = true + case client.ExporterOCI, client.ExporterDocker: + tar, err := strconv.ParseBool(out.Attrs["tar"]) + if err != nil { + tar = true + } + supportFile = tar + supportDir = !tar + case "registry": + out.Type = client.ExporterImage + out.Attrs["push"] = "true" + } + + if supportDir { + if entry.Destination == "" { + return nil, nil, errors.Errorf("dest is required for %s exporter", out.Type) + } + if entry.Destination == "-" { + return nil, nil, errors.Errorf("dest cannot be stdout for %s exporter", out.Type) + } + + fi, err := os.Stat(entry.Destination) + if err != nil && !os.IsNotExist(err) { + return nil, nil, errors.Wrapf(err, "invalid destination directory: %s", entry.Destination) + } + if err == nil && !fi.IsDir() { + return nil, nil, errors.Errorf("destination directory %s is a file", entry.Destination) + } + out.OutputDir = entry.Destination + localPaths = append(localPaths, entry.Destination) + } + if supportFile { + if entry.Destination == "" && out.Type != client.ExporterDocker { + entry.Destination = "-" + } + if entry.Destination == "-" { + if stdoutUsed { + return nil, nil, errors.Errorf("multiple outputs configured to write to stdout") + } + if _, err := console.ConsoleFromFile(os.Stdout); err == nil { + return nil, nil, errors.Errorf("dest file is required for %s exporter. refusing to write to console", out.Type) + } + out.Output = wrapWriteCloser(os.Stdout) + stdoutUsed = true + } else if entry.Destination != "" { + fi, err := os.Stat(entry.Destination) + if err != nil && !os.IsNotExist(err) { + return nil, nil, errors.Wrapf(err, "invalid destination file: %s", entry.Destination) + } + if err == nil && fi.IsDir() { + return nil, nil, errors.Errorf("destination file %s is a directory", entry.Destination) + } + f, err := os.Create(entry.Destination) + if err != nil { + return nil, nil, errors.Errorf("failed to open %s", err) + } + out.Output = wrapWriteCloser(f) + localPaths = append(localPaths, entry.Destination) + } + } + + outs = append(outs, out) + } + return outs, localPaths, nil +} + +func wrapWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) { + return func(map[string]string) (io.WriteCloser, error) { + return wc, nil + } +} + +func CreateCaches(entries []*buildflags.CacheOptionsEntry) []client.CacheOptionsEntry { + var outs []client.CacheOptionsEntry + if len(entries) == 0 { + return nil + } + + for _, entry := range entries { + out := client.CacheOptionsEntry{ + Type: entry.Type, + Attrs: map[string]string{}, + } + maps.Copy(out.Attrs, entry.Attrs) + addGithubToken(&out) + addAwsCredentials(&out) + if !isActive(&out) { + continue + } + outs = append(outs, out) + } + return outs +} + +func addGithubToken(ci *client.CacheOptionsEntry) { + if ci.Type != "gha" { + return + } + version, ok := ci.Attrs["version"] + if !ok { + // https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L19 + if v, ok := os.LookupEnv("ACTIONS_CACHE_SERVICE_V2"); ok { + if b, err := strconv.ParseBool(v); err == nil && b { + version = "2" + } + } + } + if _, ok := ci.Attrs["token"]; !ok { + if v, ok := os.LookupEnv("ACTIONS_RUNTIME_TOKEN"); ok { + ci.Attrs["token"] = v + } + } + if _, ok := ci.Attrs["url_v2"]; !ok && version == "2" { + // https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L34-L35 + if v, ok := os.LookupEnv("ACTIONS_RESULTS_URL"); ok { + ci.Attrs["url_v2"] = v + } + } + if _, ok := ci.Attrs["url"]; !ok { + // https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L28-L33 + if v, ok := os.LookupEnv("ACTIONS_CACHE_URL"); ok { + ci.Attrs["url"] = v + } else if v, ok := os.LookupEnv("ACTIONS_RESULTS_URL"); ok { + ci.Attrs["url"] = v + } + } +} + +func addAwsCredentials(ci *client.CacheOptionsEntry) { + if ci.Type != "s3" { + return + } + _, okAccessKeyID := ci.Attrs["access_key_id"] + _, okSecretAccessKey := ci.Attrs["secret_access_key"] + // If the user provides access_key_id, secret_access_key, do not override the session token. + if okAccessKeyID && okSecretAccessKey { + return + } + ctx := context.TODO() + awsConfig, err := awsconfig.LoadDefaultConfig(ctx) + if err != nil { + return + } + credentials, err := awsConfig.Credentials.Retrieve(ctx) + if err != nil { + return + } + if !okAccessKeyID && credentials.AccessKeyID != "" { + ci.Attrs["access_key_id"] = credentials.AccessKeyID + } + if !okSecretAccessKey && credentials.SecretAccessKey != "" { + ci.Attrs["secret_access_key"] = credentials.SecretAccessKey + } + if _, ok := ci.Attrs["session_token"]; !ok && credentials.SessionToken != "" { + ci.Attrs["session_token"] = credentials.SessionToken + } +} + +func isActive(ce *client.CacheOptionsEntry) bool { + // Always active if not gha. + if ce.Type != "gha" { + return true + } + return ce.Attrs["token"] != "" && (ce.Attrs["url"] != "" || ce.Attrs["url_v2"] != "") +} diff --git a/build/opt_test.go b/build/opt_test.go new file mode 100644 index 000000000..c57660139 --- /dev/null +++ b/build/opt_test.go @@ -0,0 +1,40 @@ +package build + +import ( + "testing" + + "github.com/docker/buildx/util/buildflags" + "github.com/moby/buildkit/client" + "github.com/stretchr/testify/require" +) + +func TestCacheOptions_DerivedVars(t *testing.T) { + t.Setenv("ACTIONS_RUNTIME_TOKEN", "sensitive_token") + t.Setenv("ACTIONS_CACHE_URL", "https://cache.github.com") + t.Setenv("AWS_ACCESS_KEY_ID", "definitely_dont_look_here") + t.Setenv("AWS_SECRET_ACCESS_KEY", "hackers_please_dont_steal") + t.Setenv("AWS_SESSION_TOKEN", "not_a_mitm_attack") + + cacheFrom, err := buildflags.ParseCacheEntry([]string{"type=gha", "type=s3,region=us-west-2,bucket=my_bucket,name=my_image"}) + require.NoError(t, err) + require.Equal(t, []client.CacheOptionsEntry{ + { + Type: "gha", + Attrs: map[string]string{ + "token": "sensitive_token", + "url": "https://cache.github.com", + }, + }, + { + Type: "s3", + Attrs: map[string]string{ + "region": "us-west-2", + "bucket": "my_bucket", + "name": "my_image", + "access_key_id": "definitely_dont_look_here", + "secret_access_key": "hackers_please_dont_steal", + "session_token": "not_a_mitm_attack", + }, + }, + }, CreateCaches(cacheFrom)) +} diff --git a/build/result.go b/build/result.go index 904907cea..d18f4a0b3 100644 --- a/build/result.go +++ b/build/result.go @@ -7,7 +7,6 @@ import ( "io" "sync" - controllerapi "github.com/docker/buildx/controller/pb" "github.com/moby/buildkit/client" "github.com/moby/buildkit/exporter/containerimage/exptypes" gateway "github.com/moby/buildkit/frontend/gateway/client" @@ -292,7 +291,7 @@ func (r *ResultHandle) build(buildFunc gateway.BuildFunc) (err error) { return err } -func (r *ResultHandle) getContainerConfig(cfg *controllerapi.InvokeConfig) (containerCfg gateway.NewContainerRequest, _ error) { +func (r *ResultHandle) getContainerConfig(cfg *InvokeConfig) (containerCfg gateway.NewContainerRequest, _ error) { if r.res != nil && r.solveErr == nil { logrus.Debugf("creating container from successful build") ccfg, err := containerConfigFromResult(r.res, cfg) @@ -311,7 +310,7 @@ func (r *ResultHandle) getContainerConfig(cfg *controllerapi.InvokeConfig) (cont return containerCfg, nil } -func (r *ResultHandle) getProcessConfig(cfg *controllerapi.InvokeConfig, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) (_ gateway.StartRequest, err error) { +func (r *ResultHandle) getProcessConfig(cfg *InvokeConfig, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) (_ gateway.StartRequest, err error) { processCfg := newStartRequest(stdin, stdout, stderr) if r.res != nil && r.solveErr == nil { logrus.Debugf("creating container from successful build") @@ -327,7 +326,7 @@ func (r *ResultHandle) getProcessConfig(cfg *controllerapi.InvokeConfig, stdin i return processCfg, nil } -func containerConfigFromResult(res *gateway.Result, cfg *controllerapi.InvokeConfig) (*gateway.NewContainerRequest, error) { +func containerConfigFromResult(res *gateway.Result, cfg *InvokeConfig) (*gateway.NewContainerRequest, error) { if cfg.Initial { return nil, errors.Errorf("starting from the container from the initial state of the step is supported only on the failed steps") } @@ -352,7 +351,7 @@ func containerConfigFromResult(res *gateway.Result, cfg *controllerapi.InvokeCon }, nil } -func populateProcessConfigFromResult(req *gateway.StartRequest, res *gateway.Result, cfg *controllerapi.InvokeConfig) error { +func populateProcessConfigFromResult(req *gateway.StartRequest, res *gateway.Result, cfg *InvokeConfig) error { imgData := res.Metadata[exptypes.ExporterImageConfigKey] var img *ocispecs.Image if len(imgData) > 0 { @@ -403,7 +402,7 @@ func populateProcessConfigFromResult(req *gateway.StartRequest, res *gateway.Res return nil } -func containerConfigFromError(solveErr *errdefs.SolveError, cfg *controllerapi.InvokeConfig) (*gateway.NewContainerRequest, error) { +func containerConfigFromError(solveErr *errdefs.SolveError, cfg *InvokeConfig) (*gateway.NewContainerRequest, error) { exec, err := execOpFromError(solveErr) if err != nil { return nil, err @@ -431,7 +430,7 @@ func containerConfigFromError(solveErr *errdefs.SolveError, cfg *controllerapi.I }, nil } -func populateProcessConfigFromError(req *gateway.StartRequest, solveErr *errdefs.SolveError, cfg *controllerapi.InvokeConfig) error { +func populateProcessConfigFromError(req *gateway.StartRequest, solveErr *errdefs.SolveError, cfg *InvokeConfig) error { exec, err := execOpFromError(solveErr) if err != nil { return err diff --git a/commands/bake.go b/commands/bake.go index 24e777612..571385058 100644 --- a/commands/bake.go +++ b/commands/bake.go @@ -22,7 +22,6 @@ import ( "github.com/docker/buildx/bake/hclparser" "github.com/docker/buildx/build" "github.com/docker/buildx/builder" - "github.com/docker/buildx/controller/pb" "github.com/docker/buildx/localstate" "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/cobrautil/completion" @@ -348,7 +347,7 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba continue } - pf := &pb.CallFunc{ + pf := &buildflags.CallFunc{ Name: req.CallFunc.Name, Format: req.CallFunc.Format, IgnoreStatus: req.CallFunc.IgnoreStatus, diff --git a/commands/build.go b/commands/build.go index 60062a9af..a4c4e2588 100644 --- a/commands/build.go +++ b/commands/build.go @@ -21,9 +21,6 @@ import ( "github.com/docker/buildx/build" "github.com/docker/buildx/builder" "github.com/docker/buildx/commands/debug" - cbuild "github.com/docker/buildx/controller/build" - controllererrors "github.com/docker/buildx/controller/errdefs" - controllerapi "github.com/docker/buildx/controller/pb" "github.com/docker/buildx/monitor" "github.com/docker/buildx/store" "github.com/docker/buildx/store/storeutil" @@ -31,8 +28,10 @@ import ( "github.com/docker/buildx/util/cobrautil" "github.com/docker/buildx/util/confutil" "github.com/docker/buildx/util/desktop" + "github.com/docker/buildx/util/dockerutil" "github.com/docker/buildx/util/metricutil" "github.com/docker/buildx/util/osutil" + "github.com/docker/buildx/util/platformutil" "github.com/docker/buildx/util/progress" "github.com/docker/buildx/util/tracing" "github.com/docker/cli/cli" @@ -45,8 +44,10 @@ import ( "github.com/moby/buildkit/frontend/subrequests/lint" "github.com/moby/buildkit/frontend/subrequests/outline" "github.com/moby/buildkit/frontend/subrequests/targets" + "github.com/moby/buildkit/session/auth/authprovider" "github.com/moby/buildkit/solver/errdefs" solverpb "github.com/moby/buildkit/solver/pb" + sourcepolicy "github.com/moby/buildkit/sourcepolicy/pb" "github.com/moby/buildkit/util/grpcerrors" "github.com/moby/buildkit/util/progress/progressui" "github.com/moby/sys/atomicwriter" @@ -104,7 +105,7 @@ type buildOptions struct { invokeConfig *invokeConfig } -func (o *buildOptions) toControllerOptions() (*cbuild.Options, error) { +func (o *buildOptions) toOptions() (*BuildOptions, error) { var err error buildArgs, err := listToMap(o.buildArgs, true) @@ -117,7 +118,7 @@ func (o *buildOptions) toControllerOptions() (*cbuild.Options, error) { return nil, err } - opts := cbuild.Options{ + opts := BuildOptions{ Allow: o.allow, Annotations: o.annotations, BuildArgs: buildArgs, @@ -132,7 +133,7 @@ func (o *buildOptions) toControllerOptions() (*cbuild.Options, error) { ShmSize: int64(o.shmSize), Tags: o.tags, Target: o.target, - Ulimits: dockerUlimitToControllerUlimit(o.ulimits), + Ulimits: o.ulimits, Builder: o.builder, NoCache: o.noCache, Pull: o.pull, @@ -179,17 +180,15 @@ func (o *buildOptions) toControllerOptions() (*cbuild.Options, error) { } } - cacheFrom, err := buildflags.ParseCacheEntry(o.cacheFrom) + opts.CacheFrom, err = buildflags.ParseCacheEntry(o.cacheFrom) if err != nil { return nil, err } - opts.CacheFrom = cacheFrom.ToPB() - cacheTo, err := buildflags.ParseCacheEntry(o.cacheTo) + opts.CacheTo, err = buildflags.ParseCacheEntry(o.cacheTo) if err != nil { return nil, err } - opts.CacheTo = cacheTo.ToPB() opts.Secrets, err = buildflags.ParseSecretSpecs(o.secrets) if err != nil { @@ -293,7 +292,7 @@ func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions) end(err) }() - opts, err := options.toControllerOptions() + opts, err := options.toOptions() if err != nil { return err } @@ -350,14 +349,7 @@ func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions) } done := timeBuildCommand(mp, attributes) - var resp *client.SolveResponse - var inputs *build.Inputs - var retErr error - if confutil.IsExperimental() { - resp, inputs, retErr = runControllerBuild(ctx, dockerCli, opts, options, printer) - } else { - resp, inputs, retErr = runBasicBuild(ctx, dockerCli, opts, printer) - } + resp, inputs, retErr := runBuildWithOptions(ctx, dockerCli, opts, options, printer) if err := printer.Wait(); retErr == nil { retErr = err @@ -415,11 +407,7 @@ func getImageID(resp map[string]string) string { return dgst } -func runBasicBuild(ctx context.Context, dockerCli command.Cli, opts *cbuild.Options, printer *progress.Printer) (*client.SolveResponse, *build.Inputs, error) { - return cbuild.RunBuild(ctx, dockerCli, opts, dockerCli.In(), printer, nil) -} - -func runControllerBuild(ctx context.Context, dockerCli command.Cli, opts *cbuild.Options, options buildOptions, printer *progress.Printer) (_ *client.SolveResponse, _ *build.Inputs, retErr error) { +func runBuildWithOptions(ctx context.Context, dockerCli command.Cli, opts *BuildOptions, options buildOptions, printer *progress.Printer) (_ *client.SolveResponse, _ *build.Inputs, retErr error) { if options.invokeConfig != nil && (options.dockerfileName == "-" || options.contextPath == "-") { // stdin must be usable for monitor return nil, nil, errors.Errorf("Dockerfile or context from stdin is not supported with invoke") @@ -440,9 +428,9 @@ func runControllerBuild(ctx context.Context, dockerCli command.Cli, opts *cbuild } for { - resp, inputs, err := cbuild.RunBuild(ctx, dockerCli, opts, in, printer, &bh) + resp, inputs, err := RunBuild(ctx, dockerCli, opts, in, printer, &bh) if err != nil { - var be *controllererrors.BuildError + var be *BuildError if errors.As(err, &be) { retErr = err // We can proceed to monitor @@ -765,21 +753,6 @@ func listToMap(values []string, defaultEnv bool) (map[string]string, error) { return result, nil } -func dockerUlimitToControllerUlimit(u *dockeropts.UlimitOpt) *controllerapi.UlimitOpt { - if u == nil { - return nil - } - values := make(map[string]*controllerapi.Ulimit) - for _, u := range u.GetList() { - values[u.Name] = &controllerapi.Ulimit{ - Name: u.Name, - Hard: u.Hard, - Soft: u.Soft, - } - } - return &controllerapi.UlimitOpt{Values: values} -} - func printWarnings(w io.Writer, warnings []client.VertexWarning, mode progressui.DisplayMode) { if len(warnings) == 0 || mode == progressui.QuietMode || mode == progressui.RawJSONMode { return @@ -819,7 +792,7 @@ func printWarnings(w io.Writer, warnings []client.VertexWarning, mode progressui } } -func printResult(w io.Writer, f *controllerapi.CallFunc, res map[string]string, target string, inp *build.Inputs) (int, error) { +func printResult(w io.Writer, f *buildflags.CallFunc, res map[string]string, target string, inp *build.Inputs) (int, error) { switch f.Name { case "outline": return 0, printValue(w, outline.PrintOutline, outline.SubrequestsOutlineDefinition.Version, f.Format, res) @@ -920,19 +893,19 @@ func printValue(w io.Writer, printer callFunc, version string, format string, re } type invokeConfig struct { - controllerapi.InvokeConfig + build.InvokeConfig invokeFlag string } func (cfg *invokeConfig) parseInvokeConfig(invoke, on string) error { switch on { case "always": - cfg.SuspendOn = controllerapi.SuspendAlways + cfg.SuspendOn = build.SuspendAlways case "error": - cfg.SuspendOn = controllerapi.SuspendError + cfg.SuspendOn = build.SuspendError default: if invoke != "" { - cfg.SuspendOn = controllerapi.SuspendAlways + cfg.SuspendOn = build.SuspendAlways } } @@ -1057,3 +1030,228 @@ func otelErrorType(err error) string { } return name } + +const defaultTargetName = "default" + +type BuildOptions struct { + ContextPath string + DockerfileName string + CallFunc *buildflags.CallFunc + NamedContexts map[string]string + Allow []string + Attests buildflags.Attests + BuildArgs map[string]string + CacheFrom []*buildflags.CacheOptionsEntry + CacheTo []*buildflags.CacheOptionsEntry + CgroupParent string + Exports []*buildflags.ExportEntry + ExtraHosts []string + Labels map[string]string + NetworkMode string + NoCacheFilter []string + Platforms []string + Secrets buildflags.Secrets + ShmSize int64 + SSH []*buildflags.SSH + Tags []string + Target string + Ulimits *dockeropts.UlimitOpt + Builder string + NoCache bool + Pull bool + ExportPush bool + ExportLoad bool + SourcePolicy *sourcepolicy.Policy + Ref string + GroupRef string + Annotations []string + ProvenanceResponseMode string +} + +// RunBuild runs the specified build and returns the result. +func RunBuild(ctx context.Context, dockerCli command.Cli, in *BuildOptions, inStream io.Reader, progress progress.Writer, bh *build.Handler) (*client.SolveResponse, *build.Inputs, error) { + if in.NoCache && len(in.NoCacheFilter) > 0 { + return nil, nil, errors.Errorf("--no-cache and --no-cache-filter cannot currently be used together") + } + + contexts := map[string]build.NamedContext{} + for name, path := range in.NamedContexts { + contexts[name] = build.NamedContext{Path: path} + } + + opts := build.Options{ + Inputs: build.Inputs{ + ContextPath: in.ContextPath, + DockerfilePath: in.DockerfileName, + InStream: build.NewSyncMultiReader(inStream), + NamedContexts: contexts, + }, + Ref: in.Ref, + BuildArgs: in.BuildArgs, + CgroupParent: in.CgroupParent, + ExtraHosts: in.ExtraHosts, + Labels: in.Labels, + NetworkMode: in.NetworkMode, + NoCache: in.NoCache, + NoCacheFilter: in.NoCacheFilter, + Pull: in.Pull, + ShmSize: dockeropts.MemBytes(in.ShmSize), + Tags: in.Tags, + Target: in.Target, + Ulimits: in.Ulimits, + GroupRef: in.GroupRef, + ProvenanceResponseMode: confutil.ParseMetadataProvenance(in.ProvenanceResponseMode), + } + + platforms, err := platformutil.Parse(in.Platforms) + if err != nil { + return nil, nil, err + } + opts.Platforms = platforms + + dockerConfig := dockerCli.ConfigFile() + opts.Session = append(opts.Session, authprovider.NewDockerAuthProvider(authprovider.DockerAuthProviderConfig{ + ConfigFile: dockerConfig, + })) + + secrets, err := build.CreateSecrets(in.Secrets) + if err != nil { + return nil, nil, err + } + opts.Session = append(opts.Session, secrets) + + sshSpecs := in.SSH + if len(sshSpecs) == 0 && buildflags.IsGitSSH(in.ContextPath) { + sshSpecs = append(sshSpecs, &buildflags.SSH{ID: "default"}) + } + ssh, err := build.CreateSSH(sshSpecs) + if err != nil { + return nil, nil, err + } + opts.Session = append(opts.Session, ssh) + + outputs, _, err := build.CreateExports(in.Exports) + if err != nil { + return nil, nil, err + } + if in.ExportPush { + var pushUsed bool + for i := range outputs { + if outputs[i].Type == client.ExporterImage { + outputs[i].Attrs["push"] = "true" + pushUsed = true + } + } + if !pushUsed { + outputs = append(outputs, client.ExportEntry{ + Type: client.ExporterImage, + Attrs: map[string]string{ + "push": "true", + }, + }) + } + } + if in.ExportLoad { + var loadUsed bool + for i := range outputs { + if outputs[i].Type == client.ExporterDocker { + if _, ok := outputs[i].Attrs["dest"]; !ok { + loadUsed = true + break + } + } + } + if !loadUsed { + outputs = append(outputs, client.ExportEntry{ + Type: client.ExporterDocker, + Attrs: map[string]string{}, + }) + } + } + + annotations, err := buildflags.ParseAnnotations(in.Annotations) + if err != nil { + return nil, nil, errors.Wrap(err, "parse annotations") + } + + for _, o := range outputs { + for k, v := range annotations { + o.Attrs[k.String()] = v + } + } + + opts.Exports = outputs + + opts.CacheFrom = build.CreateCaches(in.CacheFrom) + opts.CacheTo = build.CreateCaches(in.CacheTo) + + opts.Attests = in.Attests.ToMap() + + opts.SourcePolicy = in.SourcePolicy + + allow, err := buildflags.ParseEntitlements(in.Allow) + if err != nil { + return nil, nil, err + } + opts.Allow = allow + + if in.CallFunc != nil { + opts.CallFunc = &build.CallFunc{ + Name: in.CallFunc.Name, + Format: in.CallFunc.Format, + IgnoreStatus: in.CallFunc.IgnoreStatus, + } + } + + // key string used for kubernetes "sticky" mode + contextPathHash, err := filepath.Abs(in.ContextPath) + if err != nil { + contextPathHash = in.ContextPath + } + + b, err := builder.New(dockerCli, + builder.WithName(in.Builder), + builder.WithContextPathHash(contextPathHash), + ) + if err != nil { + return nil, nil, err + } + if err = updateLastActivity(dockerCli, b.NodeGroup); err != nil { + return nil, nil, errors.Wrapf(err, "failed to update builder last activity time") + } + nodes, err := b.LoadNodes(ctx) + if err != nil { + return nil, nil, err + } + + var inputs *build.Inputs + buildOptions := map[string]build.Options{defaultTargetName: opts} + resp, err := build.BuildWithResultHandler(ctx, nodes, buildOptions, dockerutil.NewClient(dockerCli), confutil.NewConfig(dockerCli), progress, bh) + err = wrapBuildError(err, false) + if err != nil { + return nil, nil, WrapBuild(err) + } + if i, ok := buildOptions[defaultTargetName]; ok { + inputs = &i.Inputs + } + return resp[defaultTargetName], inputs, nil +} + +type BuildError struct { + err error +} + +func (e *BuildError) Unwrap() error { + return e.err +} + +func (e *BuildError) Error() string { + return e.err.Error() +} + +func WrapBuild(err error) error { + if err == nil { + return nil + } + return &BuildError{err: err} +} diff --git a/controller/build/build.go b/controller/build/build.go deleted file mode 100644 index 207936097..000000000 --- a/controller/build/build.go +++ /dev/null @@ -1,272 +0,0 @@ -package build - -import ( - "context" - "io" - "path/filepath" - "strings" - - "github.com/docker/buildx/build" - "github.com/docker/buildx/builder" - "github.com/docker/buildx/controller/errdefs" - controllerapi "github.com/docker/buildx/controller/pb" - "github.com/docker/buildx/store" - "github.com/docker/buildx/store/storeutil" - "github.com/docker/buildx/util/buildflags" - "github.com/docker/buildx/util/confutil" - "github.com/docker/buildx/util/dockerutil" - "github.com/docker/buildx/util/platformutil" - "github.com/docker/buildx/util/progress" - "github.com/docker/cli/cli/command" - dockeropts "github.com/docker/cli/opts" - "github.com/docker/docker/api/types/container" - "github.com/moby/buildkit/client" - "github.com/moby/buildkit/session/auth/authprovider" - "github.com/moby/buildkit/util/grpcerrors" - "github.com/pkg/errors" - "google.golang.org/grpc/codes" -) - -const defaultTargetName = "default" - -// RunBuild runs the specified build and returns the result. -// -// NOTE: When an error happens during the build and this function acquires the debuggable *build.ResultHandle, -// this function returns it in addition to the error (i.e. it does "return nil, res, err"). The caller can -// inspect the result and debug the cause of that error. -func RunBuild(ctx context.Context, dockerCli command.Cli, in *Options, inStream io.Reader, progress progress.Writer, bh *build.Handler) (*client.SolveResponse, *build.Inputs, error) { - if in.NoCache && len(in.NoCacheFilter) > 0 { - return nil, nil, errors.Errorf("--no-cache and --no-cache-filter cannot currently be used together") - } - - contexts := map[string]build.NamedContext{} - for name, path := range in.NamedContexts { - contexts[name] = build.NamedContext{Path: path} - } - - opts := build.Options{ - Inputs: build.Inputs{ - ContextPath: in.ContextPath, - DockerfilePath: in.DockerfileName, - InStream: build.NewSyncMultiReader(inStream), - NamedContexts: contexts, - }, - Ref: in.Ref, - BuildArgs: in.BuildArgs, - CgroupParent: in.CgroupParent, - ExtraHosts: in.ExtraHosts, - Labels: in.Labels, - NetworkMode: in.NetworkMode, - NoCache: in.NoCache, - NoCacheFilter: in.NoCacheFilter, - Pull: in.Pull, - ShmSize: dockeropts.MemBytes(in.ShmSize), - Tags: in.Tags, - Target: in.Target, - Ulimits: controllerUlimitOpt2DockerUlimit(in.Ulimits), - GroupRef: in.GroupRef, - ProvenanceResponseMode: confutil.ParseMetadataProvenance(in.ProvenanceResponseMode), - } - - platforms, err := platformutil.Parse(in.Platforms) - if err != nil { - return nil, nil, err - } - opts.Platforms = platforms - - dockerConfig := dockerCli.ConfigFile() - opts.Session = append(opts.Session, authprovider.NewDockerAuthProvider(authprovider.DockerAuthProviderConfig{ - ConfigFile: dockerConfig, - })) - - secrets, err := controllerapi.CreateSecrets(in.Secrets) - if err != nil { - return nil, nil, err - } - opts.Session = append(opts.Session, secrets) - - sshSpecs := in.SSH - if len(sshSpecs) == 0 && buildflags.IsGitSSH(in.ContextPath) { - sshSpecs = append(sshSpecs, &controllerapi.SSH{ID: "default"}) - } - ssh, err := controllerapi.CreateSSH(sshSpecs) - if err != nil { - return nil, nil, err - } - opts.Session = append(opts.Session, ssh) - - outputs, _, err := controllerapi.CreateExports(in.Exports) - if err != nil { - return nil, nil, err - } - if in.ExportPush { - var pushUsed bool - for i := range outputs { - if outputs[i].Type == client.ExporterImage { - outputs[i].Attrs["push"] = "true" - pushUsed = true - } - } - if !pushUsed { - outputs = append(outputs, client.ExportEntry{ - Type: client.ExporterImage, - Attrs: map[string]string{ - "push": "true", - }, - }) - } - } - if in.ExportLoad { - var loadUsed bool - for i := range outputs { - if outputs[i].Type == client.ExporterDocker { - if _, ok := outputs[i].Attrs["dest"]; !ok { - loadUsed = true - break - } - } - } - if !loadUsed { - outputs = append(outputs, client.ExportEntry{ - Type: client.ExporterDocker, - Attrs: map[string]string{}, - }) - } - } - - annotations, err := buildflags.ParseAnnotations(in.Annotations) - if err != nil { - return nil, nil, errors.Wrap(err, "parse annotations") - } - - for _, o := range outputs { - for k, v := range annotations { - o.Attrs[k.String()] = v - } - } - - opts.Exports = outputs - - opts.CacheFrom = controllerapi.CreateCaches(in.CacheFrom) - opts.CacheTo = controllerapi.CreateCaches(in.CacheTo) - - opts.Attests = controllerapi.CreateAttestations(in.Attests) - - opts.SourcePolicy = in.SourcePolicy - - allow, err := buildflags.ParseEntitlements(in.Allow) - if err != nil { - return nil, nil, err - } - opts.Allow = allow - - if in.CallFunc != nil { - opts.CallFunc = &build.CallFunc{ - Name: in.CallFunc.Name, - Format: in.CallFunc.Format, - IgnoreStatus: in.CallFunc.IgnoreStatus, - } - } - - // key string used for kubernetes "sticky" mode - contextPathHash, err := filepath.Abs(in.ContextPath) - if err != nil { - contextPathHash = in.ContextPath - } - - // TODO: this should not be loaded this side of the controller api - b, err := builder.New(dockerCli, - builder.WithName(in.Builder), - builder.WithContextPathHash(contextPathHash), - ) - if err != nil { - return nil, nil, err - } - if err = updateLastActivity(dockerCli, b.NodeGroup); err != nil { - return nil, nil, errors.Wrapf(err, "failed to update builder last activity time") - } - nodes, err := b.LoadNodes(ctx) - if err != nil { - return nil, nil, err - } - - var inputs *build.Inputs - buildOptions := map[string]build.Options{defaultTargetName: opts} - resp, err := buildTargets(ctx, dockerCli, nodes, buildOptions, progress, bh) - err = wrapBuildError(err, false) - if err != nil { - return nil, nil, errdefs.WrapBuild(err) - } - if i, ok := buildOptions[defaultTargetName]; ok { - inputs = &i.Inputs - } - return resp, inputs, nil -} - -// buildTargets runs the specified build and returns the result. -// -// NOTE: When an error happens during the build and this function acquires the debuggable *build.ResultHandle, -// this function returns it in addition to the error (i.e. it does "return nil, res, err"). The caller can -// inspect the result and debug the cause of that error. -func buildTargets(ctx context.Context, dockerCli command.Cli, nodes []builder.Node, opts map[string]build.Options, progress progress.Writer, bh *build.Handler) (*client.SolveResponse, error) { - resp, err := build.BuildWithResultHandler(ctx, nodes, opts, dockerutil.NewClient(dockerCli), confutil.NewConfig(dockerCli), progress, bh) - if err != nil { - return nil, err - } - return resp[defaultTargetName], err -} - -func wrapBuildError(err error, bake bool) error { - if err == nil { - return nil - } - st, ok := grpcerrors.AsGRPCStatus(err) - if ok { - if st.Code() == codes.Unimplemented && strings.Contains(st.Message(), "unsupported frontend capability moby.buildkit.frontend.contexts") { - msg := "current frontend does not support --build-context." - if bake { - msg = "current frontend does not support defining additional contexts for targets." - } - msg += " Named contexts are supported since Dockerfile v1.4. Use #syntax directive in Dockerfile or update to latest BuildKit." - return &wrapped{err, msg} - } - } - return err -} - -type wrapped struct { - err error - msg string -} - -func (w *wrapped) Error() string { - return w.msg -} - -func (w *wrapped) Unwrap() error { - return w.err -} - -func updateLastActivity(dockerCli command.Cli, ng *store.NodeGroup) error { - txn, release, err := storeutil.GetStore(dockerCli) - if err != nil { - return err - } - defer release() - return txn.UpdateLastActivity(ng) -} - -func controllerUlimitOpt2DockerUlimit(u *controllerapi.UlimitOpt) *dockeropts.UlimitOpt { - if u == nil { - return nil - } - values := make(map[string]*container.Ulimit) - for k, v := range u.Values { - values[k] = &container.Ulimit{ - Name: v.Name, - Hard: v.Hard, - Soft: v.Soft, - } - } - return dockeropts.NewUlimitOpt(&values) -} diff --git a/controller/build/options.go b/controller/build/options.go deleted file mode 100644 index 817f48368..000000000 --- a/controller/build/options.go +++ /dev/null @@ -1,216 +0,0 @@ -package build - -import ( - "path/filepath" - "strings" - - "github.com/docker/buildx/controller/pb" - sourcepolicy "github.com/moby/buildkit/sourcepolicy/pb" - "github.com/moby/buildkit/util/gitutil" -) - -type Options struct { - ContextPath string - DockerfileName string - CallFunc *pb.CallFunc - NamedContexts map[string]string - Allow []string - Attests []*pb.Attest - BuildArgs map[string]string - CacheFrom []*pb.CacheOptionsEntry - CacheTo []*pb.CacheOptionsEntry - CgroupParent string - Exports []*pb.ExportEntry - ExtraHosts []string - Labels map[string]string - NetworkMode string - NoCacheFilter []string - Platforms []string - Secrets []*pb.Secret - ShmSize int64 - SSH []*pb.SSH - Tags []string - Target string - Ulimits *pb.UlimitOpt - Builder string - NoCache bool - Pull bool - ExportPush bool - ExportLoad bool - SourcePolicy *sourcepolicy.Policy - Ref string - GroupRef string - Annotations []string - ProvenanceResponseMode string -} - -// ResolveOptionPaths resolves all paths contained in BuildOptions -// and replaces them to absolute paths. -func ResolveOptionPaths(options *Options) (_ *Options, err error) { - localContext := false - if options.ContextPath != "" && options.ContextPath != "-" { - if !isRemoteURL(options.ContextPath) { - localContext = true - options.ContextPath, err = filepath.Abs(options.ContextPath) - if err != nil { - return nil, err - } - } - } - if options.DockerfileName != "" && options.DockerfileName != "-" { - if localContext && !isHTTPURL(options.DockerfileName) { - options.DockerfileName, err = filepath.Abs(options.DockerfileName) - if err != nil { - return nil, err - } - } - } - - var contexts map[string]string - for k, v := range options.NamedContexts { - if isRemoteURL(v) || strings.HasPrefix(v, "docker-image://") { - // url prefix, this is a remote path - } else if p, ok := strings.CutPrefix(v, "oci-layout://"); ok { - // oci layout prefix, this is a local path - p, err = filepath.Abs(p) - if err != nil { - return nil, err - } - v = "oci-layout://" + p - } else { - // no prefix, assume local path - v, err = filepath.Abs(v) - if err != nil { - return nil, err - } - } - - if contexts == nil { - contexts = make(map[string]string) - } - contexts[k] = v - } - options.NamedContexts = contexts - - var cacheFrom []*pb.CacheOptionsEntry - for _, co := range options.CacheFrom { - switch co.Type { - case "local": - var attrs map[string]string - for k, v := range co.Attrs { - if attrs == nil { - attrs = make(map[string]string) - } - switch k { - case "src": - p := v - if p != "" { - p, err = filepath.Abs(p) - if err != nil { - return nil, err - } - } - attrs[k] = p - default: - attrs[k] = v - } - } - co.Attrs = attrs - cacheFrom = append(cacheFrom, co) - default: - cacheFrom = append(cacheFrom, co) - } - } - options.CacheFrom = cacheFrom - - var cacheTo []*pb.CacheOptionsEntry - for _, co := range options.CacheTo { - switch co.Type { - case "local": - var attrs map[string]string - for k, v := range co.Attrs { - if attrs == nil { - attrs = make(map[string]string) - } - switch k { - case "dest": - p := v - if p != "" { - p, err = filepath.Abs(p) - if err != nil { - return nil, err - } - } - attrs[k] = p - default: - attrs[k] = v - } - } - co.Attrs = attrs - cacheTo = append(cacheTo, co) - default: - cacheTo = append(cacheTo, co) - } - } - options.CacheTo = cacheTo - var exports []*pb.ExportEntry - for _, e := range options.Exports { - if e.Destination != "" && e.Destination != "-" { - e.Destination, err = filepath.Abs(e.Destination) - if err != nil { - return nil, err - } - } - exports = append(exports, e) - } - options.Exports = exports - - var secrets []*pb.Secret - for _, s := range options.Secrets { - if s.FilePath != "" { - s.FilePath, err = filepath.Abs(s.FilePath) - if err != nil { - return nil, err - } - } - secrets = append(secrets, s) - } - options.Secrets = secrets - - var ssh []*pb.SSH - for _, s := range options.SSH { - var ps []string - for _, pt := range s.Paths { - p := pt - if p != "" { - p, err = filepath.Abs(p) - if err != nil { - return nil, err - } - } - ps = append(ps, p) - } - s.Paths = ps - ssh = append(ssh, s) - } - options.SSH = ssh - - return options, nil -} - -// isHTTPURL returns true if the provided str is an HTTP(S) URL by checking if it -// has a http:// or https:// scheme. No validation is performed to verify if the -// URL is well-formed. -func isHTTPURL(str string) bool { - return strings.HasPrefix(str, "https://") || strings.HasPrefix(str, "http://") -} - -func isRemoteURL(c string) bool { - if isHTTPURL(c) { - return true - } - if _, err := gitutil.ParseGitRef(c); err == nil { - return true - } - return false -} diff --git a/controller/build/options_test.go b/controller/build/options_test.go deleted file mode 100644 index aa814e780..000000000 --- a/controller/build/options_test.go +++ /dev/null @@ -1,249 +0,0 @@ -package build - -import ( - "os" - "path/filepath" - "testing" - - "github.com/docker/buildx/controller/pb" - "github.com/stretchr/testify/require" -) - -func TestResolvePaths(t *testing.T) { - tmpwd, err := os.MkdirTemp("", "testresolvepaths") - require.NoError(t, err) - defer os.Remove(tmpwd) - require.NoError(t, os.Chdir(tmpwd)) - tests := []struct { - name string - options *Options - want *Options - }{ - { - name: "contextpath", - options: &Options{ContextPath: "test"}, - want: &Options{ContextPath: filepath.Join(tmpwd, "test")}, - }, - { - name: "contextpath-cwd", - options: &Options{ContextPath: "."}, - want: &Options{ContextPath: tmpwd}, - }, - { - name: "contextpath-dash", - options: &Options{ContextPath: "-"}, - want: &Options{ContextPath: "-"}, - }, - { - name: "contextpath-ssh", - options: &Options{ContextPath: "git@github.com:docker/buildx.git"}, - want: &Options{ContextPath: "git@github.com:docker/buildx.git"}, - }, - { - name: "dockerfilename", - options: &Options{DockerfileName: "test", ContextPath: "."}, - want: &Options{DockerfileName: filepath.Join(tmpwd, "test"), ContextPath: tmpwd}, - }, - { - name: "dockerfilename-dash", - options: &Options{DockerfileName: "-", ContextPath: "."}, - want: &Options{DockerfileName: "-", ContextPath: tmpwd}, - }, - { - name: "dockerfilename-remote", - options: &Options{DockerfileName: "test", ContextPath: "git@github.com:docker/buildx.git"}, - want: &Options{DockerfileName: "test", ContextPath: "git@github.com:docker/buildx.git"}, - }, - { - name: "contexts", - options: &Options{NamedContexts: map[string]string{ - "a": "test1", "b": "test2", - "alpine": "docker-image://alpine@sha256:0123456789", "project": "https://github.com/myuser/project.git", - }}, - want: &Options{NamedContexts: map[string]string{ - "a": filepath.Join(tmpwd, "test1"), "b": filepath.Join(tmpwd, "test2"), - "alpine": "docker-image://alpine@sha256:0123456789", "project": "https://github.com/myuser/project.git", - }}, - }, - { - name: "cache-from", - options: &Options{ - CacheFrom: []*pb.CacheOptionsEntry{ - { - Type: "local", - Attrs: map[string]string{"src": "test"}, - }, - { - Type: "registry", - Attrs: map[string]string{"ref": "user/app"}, - }, - }, - }, - want: &Options{ - CacheFrom: []*pb.CacheOptionsEntry{ - { - Type: "local", - Attrs: map[string]string{"src": filepath.Join(tmpwd, "test")}, - }, - { - Type: "registry", - Attrs: map[string]string{"ref": "user/app"}, - }, - }, - }, - }, - { - name: "cache-to", - options: &Options{ - CacheTo: []*pb.CacheOptionsEntry{ - { - Type: "local", - Attrs: map[string]string{"dest": "test"}, - }, - { - Type: "registry", - Attrs: map[string]string{"ref": "user/app"}, - }, - }, - }, - want: &Options{ - CacheTo: []*pb.CacheOptionsEntry{ - { - Type: "local", - Attrs: map[string]string{"dest": filepath.Join(tmpwd, "test")}, - }, - { - Type: "registry", - Attrs: map[string]string{"ref": "user/app"}, - }, - }, - }, - }, - { - name: "exports", - options: &Options{ - Exports: []*pb.ExportEntry{ - { - Type: "local", - Destination: "-", - }, - { - Type: "local", - Destination: "test1", - }, - { - Type: "tar", - Destination: "test3", - }, - { - Type: "oci", - Destination: "-", - }, - { - Type: "docker", - Destination: "test4", - }, - { - Type: "image", - Attrs: map[string]string{"push": "true"}, - }, - }, - }, - want: &Options{ - Exports: []*pb.ExportEntry{ - { - Type: "local", - Destination: "-", - }, - { - Type: "local", - Destination: filepath.Join(tmpwd, "test1"), - }, - { - Type: "tar", - Destination: filepath.Join(tmpwd, "test3"), - }, - { - Type: "oci", - Destination: "-", - }, - { - Type: "docker", - Destination: filepath.Join(tmpwd, "test4"), - }, - { - Type: "image", - Attrs: map[string]string{"push": "true"}, - }, - }, - }, - }, - { - name: "secrets", - options: &Options{ - Secrets: []*pb.Secret{ - { - FilePath: "test1", - }, - { - ID: "val", - Env: "a", - }, - { - ID: "test", - FilePath: "test3", - }, - }, - }, - want: &Options{ - Secrets: []*pb.Secret{ - { - FilePath: filepath.Join(tmpwd, "test1"), - }, - { - ID: "val", - Env: "a", - }, - { - ID: "test", - FilePath: filepath.Join(tmpwd, "test3"), - }, - }, - }, - }, - { - name: "ssh", - options: &Options{ - SSH: []*pb.SSH{ - { - ID: "default", - Paths: []string{"test1", "test2"}, - }, - { - ID: "a", - Paths: []string{"test3"}, - }, - }, - }, - want: &Options{ - SSH: []*pb.SSH{ - { - ID: "default", - Paths: []string{filepath.Join(tmpwd, "test1"), filepath.Join(tmpwd, "test2")}, - }, - { - ID: "a", - Paths: []string{filepath.Join(tmpwd, "test3")}, - }, - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := ResolveOptionPaths(tt.options) - require.NoError(t, err) - require.Equal(t, tt.want, got) - }) - } -} diff --git a/controller/errdefs/build.go b/controller/errdefs/build.go deleted file mode 100644 index acc17c115..000000000 --- a/controller/errdefs/build.go +++ /dev/null @@ -1,20 +0,0 @@ -package errdefs - -type BuildError struct { - err error -} - -func (e *BuildError) Unwrap() error { - return e.err -} - -func (e *BuildError) Error() string { - return e.err.Error() -} - -func WrapBuild(err error) error { - if err == nil { - return nil - } - return &BuildError{err: err} -} diff --git a/controller/pb/attest.go b/controller/pb/attest.go deleted file mode 100644 index 2c6028842..000000000 --- a/controller/pb/attest.go +++ /dev/null @@ -1,26 +0,0 @@ -package pb - -type Attest struct { - Type string - Disabled bool - Attrs string -} - -func CreateAttestations(attests []*Attest) map[string]*string { - result := map[string]*string{} - for _, attest := range attests { - // ignore duplicates - if _, ok := result[attest.Type]; ok { - continue - } - - if attest.Disabled { - result[attest.Type] = nil - continue - } - - attrs := attest.Attrs - result[attest.Type] = &attrs - } - return result -} diff --git a/controller/pb/cache.go b/controller/pb/cache.go deleted file mode 100644 index 87adf4fee..000000000 --- a/controller/pb/cache.go +++ /dev/null @@ -1,28 +0,0 @@ -package pb - -import ( - "maps" - - "github.com/moby/buildkit/client" -) - -type CacheOptionsEntry struct { - Type string - Attrs map[string]string -} - -func CreateCaches(entries []*CacheOptionsEntry) []client.CacheOptionsEntry { - var outs []client.CacheOptionsEntry - if len(entries) == 0 { - return nil - } - for _, entry := range entries { - out := client.CacheOptionsEntry{ - Type: entry.Type, - Attrs: map[string]string{}, - } - maps.Copy(out.Attrs, entry.Attrs) - outs = append(outs, out) - } - return outs -} diff --git a/controller/pb/export.go b/controller/pb/export.go deleted file mode 100644 index 9842fed7a..000000000 --- a/controller/pb/export.go +++ /dev/null @@ -1,114 +0,0 @@ -package pb - -import ( - "io" - "maps" - "os" - "strconv" - - "github.com/containerd/console" - "github.com/moby/buildkit/client" - "github.com/pkg/errors" -) - -type ExportEntry struct { - Type string - Attrs map[string]string - Destination string -} - -func CreateExports(entries []*ExportEntry) ([]client.ExportEntry, []string, error) { - var outs []client.ExportEntry - var localPaths []string - if len(entries) == 0 { - return nil, nil, nil - } - var stdoutUsed bool - for _, entry := range entries { - if entry.Type == "" { - return nil, nil, errors.Errorf("type is required for output") - } - - out := client.ExportEntry{ - Type: entry.Type, - Attrs: map[string]string{}, - } - maps.Copy(out.Attrs, entry.Attrs) - - supportFile := false - supportDir := false - switch out.Type { - case client.ExporterLocal: - supportDir = true - case client.ExporterTar: - supportFile = true - case client.ExporterOCI, client.ExporterDocker: - tar, err := strconv.ParseBool(out.Attrs["tar"]) - if err != nil { - tar = true - } - supportFile = tar - supportDir = !tar - case "registry": - out.Type = client.ExporterImage - out.Attrs["push"] = "true" - } - - if supportDir { - if entry.Destination == "" { - return nil, nil, errors.Errorf("dest is required for %s exporter", out.Type) - } - if entry.Destination == "-" { - return nil, nil, errors.Errorf("dest cannot be stdout for %s exporter", out.Type) - } - - fi, err := os.Stat(entry.Destination) - if err != nil && !os.IsNotExist(err) { - return nil, nil, errors.Wrapf(err, "invalid destination directory: %s", entry.Destination) - } - if err == nil && !fi.IsDir() { - return nil, nil, errors.Errorf("destination directory %s is a file", entry.Destination) - } - out.OutputDir = entry.Destination - localPaths = append(localPaths, entry.Destination) - } - if supportFile { - if entry.Destination == "" && out.Type != client.ExporterDocker { - entry.Destination = "-" - } - if entry.Destination == "-" { - if stdoutUsed { - return nil, nil, errors.Errorf("multiple outputs configured to write to stdout") - } - if _, err := console.ConsoleFromFile(os.Stdout); err == nil { - return nil, nil, errors.Errorf("dest file is required for %s exporter. refusing to write to console", out.Type) - } - out.Output = wrapWriteCloser(os.Stdout) - stdoutUsed = true - } else if entry.Destination != "" { - fi, err := os.Stat(entry.Destination) - if err != nil && !os.IsNotExist(err) { - return nil, nil, errors.Wrapf(err, "invalid destination file: %s", entry.Destination) - } - if err == nil && fi.IsDir() { - return nil, nil, errors.Errorf("destination file %s is a directory", entry.Destination) - } - f, err := os.Create(entry.Destination) - if err != nil { - return nil, nil, errors.Errorf("failed to open %s", err) - } - out.Output = wrapWriteCloser(f) - localPaths = append(localPaths, entry.Destination) - } - } - - outs = append(outs, out) - } - return outs, localPaths, nil -} - -func wrapWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser, error) { - return func(map[string]string) (io.WriteCloser, error) { - return wc, nil - } -} diff --git a/controller/pb/invoke.go b/controller/pb/invoke.go deleted file mode 100644 index 505d2c561..000000000 --- a/controller/pb/invoke.go +++ /dev/null @@ -1,56 +0,0 @@ -package pb - -import ( - "fmt" - "strings" -) - -type CallFunc struct { - Name string - Format string - IgnoreStatus bool -} - -func (x *CallFunc) String() string { - var elems []string - if x.Name != "" { - elems = append(elems, fmt.Sprintf("Name:%q", x.Name)) - } - if x.Format != "" { - elems = append(elems, fmt.Sprintf("Format:%q", x.Format)) - } - if x.IgnoreStatus { - elems = append(elems, fmt.Sprintf("IgnoreStatus:%v", x.IgnoreStatus)) - } - return strings.Join(elems, " ") -} - -type InvokeConfig struct { - Entrypoint []string - Cmd []string - NoCmd bool - Env []string - User string - NoUser bool - Cwd string - NoCwd bool - Tty bool - Rollback bool - Initial bool - SuspendOn SuspendOn -} - -func (cfg *InvokeConfig) NeedsDebug(err error) bool { - return cfg.SuspendOn.DebugEnabled(err) -} - -type SuspendOn int - -const ( - SuspendError SuspendOn = iota - SuspendAlways -) - -func (s SuspendOn) DebugEnabled(err error) bool { - return err != nil || s == SuspendAlways -} diff --git a/controller/pb/secrets.go b/controller/pb/secrets.go deleted file mode 100644 index f58583f69..000000000 --- a/controller/pb/secrets.go +++ /dev/null @@ -1,28 +0,0 @@ -package pb - -import ( - "github.com/moby/buildkit/session" - "github.com/moby/buildkit/session/secrets/secretsprovider" -) - -type Secret struct { - ID string - FilePath string - Env string -} - -func CreateSecrets(secrets []*Secret) (session.Attachable, error) { - fs := make([]secretsprovider.Source, 0, len(secrets)) - for _, secret := range secrets { - fs = append(fs, secretsprovider.Source{ - ID: secret.ID, - FilePath: secret.FilePath, - Env: secret.Env, - }) - } - store, err := secretsprovider.NewStore(fs) - if err != nil { - return nil, err - } - return secretsprovider.NewSecretProvider(store), nil -} diff --git a/controller/pb/ssh.go b/controller/pb/ssh.go deleted file mode 100644 index 5b91cf9e4..000000000 --- a/controller/pb/ssh.go +++ /dev/null @@ -1,25 +0,0 @@ -package pb - -import ( - "slices" - - "github.com/moby/buildkit/session" - "github.com/moby/buildkit/session/sshforward/sshprovider" -) - -type SSH struct { - ID string - Paths []string -} - -func CreateSSH(ssh []*SSH) (session.Attachable, error) { - configs := make([]sshprovider.AgentConfig, 0, len(ssh)) - for _, ssh := range ssh { - cfg := sshprovider.AgentConfig{ - ID: ssh.ID, - Paths: slices.Clone(ssh.Paths), - } - configs = append(configs, cfg) - } - return sshprovider.NewSSHAgentProvider(configs) -} diff --git a/controller/pb/ulimit.go b/controller/pb/ulimit.go deleted file mode 100644 index 35e068daa..000000000 --- a/controller/pb/ulimit.go +++ /dev/null @@ -1,11 +0,0 @@ -package pb - -type UlimitOpt struct { - Values map[string]*Ulimit -} - -type Ulimit struct { - Name string - Hard int64 - Soft int64 -} diff --git a/docs/debugging.md b/docs/debugging.md index a964294ca..8e3662f3f 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -26,7 +26,6 @@ Arguments available after `buildx debug build` are the same as the normal `build ```console $ docker buildx debug --invoke /bin/sh build . [+] Building 4.2s (19/19) FINISHED - => [internal] connecting to local controller 0.0s => [internal] load build definition from Dockerfile 0.0s => => transferring dockerfile: 32B 0.0s => [internal] load .dockerignore 0.0s @@ -68,7 +67,6 @@ If you want to start a debug session when a build fails, you can use ```console $ docker buildx debug --on=error build . [+] Building 4.2s (19/19) FINISHED - => [internal] connecting to local controller 0.0s => [internal] load build definition from Dockerfile 0.0s => => transferring dockerfile: 32B 0.0s => [internal] load .dockerignore 0.0s @@ -94,7 +92,6 @@ can use `buildx debug` command to start a debug session. ``` $ docker buildx debug [+] Building 4.2s (19/19) FINISHED - => [internal] connecting to local controller 0.0s (buildx) ``` @@ -125,41 +122,3 @@ Available commands are: rollback re-runs the interactive container with the step's rootfs contents ``` -## Build controllers - -Debugging is performed using a buildx "controller", which provides a high-level -abstraction to perform builds. By default, the local controller is used for a -more stable experience which runs all builds in-process. However, you can also -use the remote controller to detach the build process from the CLI. - -To detach the build process from the CLI, you can use the `--detach=true` flag with -the build command. - -```console -$ docker buildx debug --invoke /bin/sh build --detach=true . -``` - -If you start a debugging session using the `--invoke` flag with a detached -build, then you can attach to it using the `buildx debug` command to -immediately enter the monitor mode. - -```console -$ docker buildx debug -[+] Building 0.0s (1/1) FINISHED - => [internal] connecting to remote controller -(buildx) list -ID CURRENT_SESSION -xfe1162ovd9def8yapb4ys66t false -(buildx) attach xfe1162ovd9def8yapb4ys66t -Attached to process "". Press Ctrl-a-c to switch to the new container -(buildx) ps -PID CURRENT_SESSION COMMAND -3ug8iqaufiwwnukimhqqt06jz false [sh] -(buildx) attach 3ug8iqaufiwwnukimhqqt06jz -Attached to process "3ug8iqaufiwwnukimhqqt06jz". Press Ctrl-a-c to switch to the new container -(buildx) Switched IO -/ # ls -bin etc lib mnt proc run srv tmp var -dev home media opt root sbin sys usr work -/ # -``` diff --git a/monitor/commands/exec.go b/monitor/commands/exec.go index 5a5968eeb..00d861edf 100644 --- a/monitor/commands/exec.go +++ b/monitor/commands/exec.go @@ -5,7 +5,7 @@ import ( "fmt" "io" - controllerapi "github.com/docker/buildx/controller/pb" + "github.com/docker/buildx/build" "github.com/docker/buildx/monitor/types" "github.com/pkg/errors" ) @@ -13,11 +13,11 @@ import ( type ExecCmd struct { m types.Monitor - invokeConfig *controllerapi.InvokeConfig + invokeConfig *build.InvokeConfig stdout io.WriteCloser } -func NewExecCmd(m types.Monitor, invokeConfig *controllerapi.InvokeConfig, stdout io.WriteCloser) types.Command { +func NewExecCmd(m types.Monitor, invokeConfig *build.InvokeConfig, stdout io.WriteCloser) types.Command { return &ExecCmd{m, invokeConfig, stdout} } @@ -38,7 +38,7 @@ func (cm *ExecCmd) Exec(ctx context.Context, args []string) error { if len(args) < 2 { return errors.Errorf("command must be passed") } - cfg := &controllerapi.InvokeConfig{ + cfg := &build.InvokeConfig{ Entrypoint: []string{args[1]}, Cmd: args[2:], NoCmd: false, diff --git a/monitor/commands/rollback.go b/monitor/commands/rollback.go index b6ef4a7f8..c2d9457c1 100644 --- a/monitor/commands/rollback.go +++ b/monitor/commands/rollback.go @@ -5,18 +5,18 @@ import ( "fmt" "io" - controllerapi "github.com/docker/buildx/controller/pb" + "github.com/docker/buildx/build" "github.com/docker/buildx/monitor/types" ) type RollbackCmd struct { m types.Monitor - invokeConfig *controllerapi.InvokeConfig + invokeConfig *build.InvokeConfig stdout io.WriteCloser } -func NewRollbackCmd(m types.Monitor, invokeConfig *controllerapi.InvokeConfig, stdout io.WriteCloser) types.Command { +func NewRollbackCmd(m types.Monitor, invokeConfig *build.InvokeConfig, stdout io.WriteCloser) types.Command { return &RollbackCmd{m, invokeConfig, stdout} } diff --git a/monitor/monitor.go b/monitor/monitor.go index 35ef37bf5..09901291c 100644 --- a/monitor/monitor.go +++ b/monitor/monitor.go @@ -12,9 +12,8 @@ import ( "github.com/containerd/console" "github.com/docker/buildx/build" - controllerapi "github.com/docker/buildx/controller/pb" - "github.com/docker/buildx/controller/processes" "github.com/docker/buildx/monitor/commands" + "github.com/docker/buildx/monitor/processes" "github.com/docker/buildx/monitor/types" "github.com/docker/buildx/util/ioset" "github.com/docker/buildx/util/progress" @@ -29,7 +28,7 @@ import ( var ErrReload = errors.New("monitor: reload") type Monitor struct { - invokeConfig *controllerapi.InvokeConfig + invokeConfig *build.InvokeConfig printer *progress.Printer stdin *ioset.SingleForwarder @@ -41,7 +40,7 @@ type Monitor struct { mu sync.Mutex } -func New(cfg *controllerapi.InvokeConfig, stdin io.ReadCloser, stdout, stderr io.WriteCloser, printer *progress.Printer) *Monitor { +func New(cfg *build.InvokeConfig, stdin io.ReadCloser, stdout, stderr io.WriteCloser, printer *progress.Printer) *Monitor { m := &Monitor{ invokeConfig: cfg, printer: printer, @@ -113,7 +112,7 @@ func (m *Monitor) Close() error { } // RunMonitor provides an interactive session for running and managing containers via specified IO. -func RunMonitor(ctx context.Context, invokeConfig *controllerapi.InvokeConfig, rCtx *build.ResultHandle, stdin io.ReadCloser, stdout, stderr io.WriteCloser, progress *progress.Printer) error { +func RunMonitor(ctx context.Context, invokeConfig *build.InvokeConfig, rCtx *build.ResultHandle, stdin io.ReadCloser, stdout, stderr io.WriteCloser, progress *progress.Printer) error { if err := progress.Pause(); err != nil { return err } @@ -333,7 +332,7 @@ type monitor struct { processes *processes.Manager } -func (m *monitor) Invoke(ctx context.Context, pid string, cfg *controllerapi.InvokeConfig, ioIn io.ReadCloser, ioOut io.WriteCloser, ioErr io.WriteCloser) error { +func (m *monitor) Invoke(ctx context.Context, pid string, cfg *build.InvokeConfig, ioIn io.ReadCloser, ioOut io.WriteCloser, ioErr io.WriteCloser) error { proc, ok := m.processes.Get(pid) if !ok { // Start a new process. @@ -361,19 +360,19 @@ func (m *monitor) Invoke(ctx context.Context, pid string, cfg *controllerapi.Inv } } -func (m *monitor) Rollback(ctx context.Context, cfg *controllerapi.InvokeConfig) string { +func (m *monitor) Rollback(ctx context.Context, cfg *build.InvokeConfig) string { pid := identity.NewID() cfg1 := cfg cfg1.Rollback = true return m.startInvoke(ctx, pid, cfg1) } -func (m *monitor) Exec(ctx context.Context, cfg *controllerapi.InvokeConfig) string { +func (m *monitor) Exec(ctx context.Context, cfg *build.InvokeConfig) string { return m.startInvoke(ctx, identity.NewID(), cfg) } func (m *monitor) Attach(ctx context.Context, pid string) { - m.startInvoke(ctx, pid, &controllerapi.InvokeConfig{}) + m.startInvoke(ctx, pid, &build.InvokeConfig{}) } func (m *monitor) Detach() { @@ -394,7 +393,7 @@ func (m *monitor) close() { m.Detach() } -func (m *monitor) startInvoke(ctx context.Context, pid string, cfg *controllerapi.InvokeConfig) string { +func (m *monitor) startInvoke(ctx context.Context, pid string, cfg *build.InvokeConfig) string { if m.invokeCancel != nil { m.invokeCancel() // Finish existing attach } @@ -420,7 +419,7 @@ func (m *monitor) startInvoke(ctx context.Context, pid string, cfg *controllerap return pid } -func (m *monitor) invoke(ctx context.Context, pid string, cfg *controllerapi.InvokeConfig) error { +func (m *monitor) invoke(ctx context.Context, pid string, cfg *build.InvokeConfig) error { m.muxIO.Enable(1) defer m.muxIO.Disable(1) if err := m.muxIO.SwitchTo(1); err != nil { diff --git a/controller/processes/processes.go b/monitor/processes/processes.go similarity index 96% rename from controller/processes/processes.go rename to monitor/processes/processes.go index 28879be2d..6f523aa62 100644 --- a/controller/processes/processes.go +++ b/monitor/processes/processes.go @@ -6,7 +6,6 @@ import ( "sync/atomic" "github.com/docker/buildx/build" - "github.com/docker/buildx/controller/pb" "github.com/docker/buildx/util/ioset" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -15,7 +14,7 @@ import ( // Process provides methods to control a process. type Process struct { inEnd *ioset.Forwarder - invokeConfig *pb.InvokeConfig + invokeConfig *build.InvokeConfig errCh chan error processCancel func() serveIOCancel func(error) @@ -98,7 +97,7 @@ func (m *Manager) DeleteProcess(id string) error { // When a container isn't available (i.e. first time invoking or the container has exited) or cfg.Rollback is set, // this method will start a new container and run the process in it. Otherwise, this method starts a new process in the // existing container. -func (m *Manager) StartProcess(pid string, resultCtx *build.ResultHandle, cfg *pb.InvokeConfig) (*Process, error) { +func (m *Manager) StartProcess(pid string, resultCtx *build.ResultHandle, cfg *build.InvokeConfig) (*Process, error) { // Get the target result to invoke a container from var ctr *build.Container if a := m.container.Load(); a != nil { @@ -157,5 +156,5 @@ func (m *Manager) StartProcess(pid string, resultCtx *build.ResultHandle, cfg *p type ProcessInfo struct { ProcessID string - InvokeConfig *pb.InvokeConfig + InvokeConfig *build.InvokeConfig } diff --git a/monitor/types/types.go b/monitor/types/types.go index fb17b52f7..23de71d56 100644 --- a/monitor/types/types.go +++ b/monitor/types/types.go @@ -4,8 +4,8 @@ import ( "context" "io" - controllerapi "github.com/docker/buildx/controller/pb" - "github.com/docker/buildx/controller/processes" + "github.com/docker/buildx/build" + "github.com/docker/buildx/monitor/processes" ) // Monitor provides APIs for attaching and controlling the buildx server. @@ -14,17 +14,17 @@ type Monitor interface { // If pid doesn't match to any running processes, it starts a new process with the specified config. // If there is no container running or InvokeConfig.Rollback is specified, the process will start in a newly created container. // NOTE: If needed, in the future, we can split this API into three APIs (NewContainer, NewProcess and Attach). - Invoke(ctx context.Context, pid string, options *controllerapi.InvokeConfig, ioIn io.ReadCloser, ioOut io.WriteCloser, ioErr io.WriteCloser) error + Invoke(ctx context.Context, pid string, options *build.InvokeConfig, ioIn io.ReadCloser, ioOut io.WriteCloser, ioErr io.WriteCloser) error ListProcesses(ctx context.Context) (infos []*processes.ProcessInfo, retErr error) DisconnectProcess(ctx context.Context, pid string) error // Rollback re-runs the interactive container with initial rootfs contents. - Rollback(ctx context.Context, cfg *controllerapi.InvokeConfig) string + Rollback(ctx context.Context, cfg *build.InvokeConfig) string // Rollback executes a process in the interactive container. - Exec(ctx context.Context, cfg *controllerapi.InvokeConfig) string + Exec(ctx context.Context, cfg *build.InvokeConfig) string // Attach attaches IO to a process in the container. Attach(ctx context.Context, pid string) diff --git a/util/buildflags/attests.go b/util/buildflags/attests.go index 934a1dc50..23cb26c4e 100644 --- a/util/buildflags/attests.go +++ b/util/buildflags/attests.go @@ -7,7 +7,6 @@ import ( "strconv" "strings" - controllerapi "github.com/docker/buildx/controller/pb" "github.com/pkg/errors" "github.com/tonistiigi/go-csvvalue" ) @@ -33,16 +32,32 @@ func (a Attests) Normalize() Attests { return removeAttestDupes(a) } -func (a Attests) ToPB() []*controllerapi.Attest { - if len(a) == 0 { - return nil - } +func (a Attests) ToMap() map[string]*string { + result := map[string]*string{} + for _, attest := range a { + // ignore duplicates + if _, ok := result[attest.Type]; ok { + continue + } - entries := make([]*controllerapi.Attest, len(a)) - for i, entry := range a { - entries[i] = entry.ToPB() + if attest.Disabled { + result[attest.Type] = nil + continue + } + + var b csvBuilder + if attest.Type != "" { + b.Write("type", attest.Type) + } + if attest.Disabled { + b.Write("disabled", "true") + } + b.WriteAttributes(attest.Attrs) + + s := b.String() + result[attest.Type] = &s } - return entries + return result } type Attest struct { @@ -72,23 +87,6 @@ func (a *Attest) String() string { return b.String() } -func (a *Attest) ToPB() *controllerapi.Attest { - var b csvBuilder - if a.Type != "" { - b.Write("type", a.Type) - } - if a.Disabled { - b.Write("disabled", "true") - } - b.WriteAttributes(a.Attrs) - - return &controllerapi.Attest{ - Type: a.Type, - Disabled: a.Disabled, - Attrs: b.String(), - } -} - func (a *Attest) MarshalJSON() ([]byte, error) { m := make(map[string]any, len(a.Attrs)+2) for k, v := range a.Attrs { @@ -182,7 +180,7 @@ func CanonicalizeAttest(attestType string, in string) string { return fmt.Sprintf("type=%s,%s", attestType, in) } -func ParseAttests(in []string) ([]*controllerapi.Attest, error) { +func ParseAttests(in []string) (Attests, error) { var outs []*Attest for _, s := range in { var out Attest @@ -191,66 +189,7 @@ func ParseAttests(in []string) ([]*controllerapi.Attest, error) { } outs = append(outs, &out) } - return ConvertAttests(outs) -} - -// ConvertAttests converts Attestations for the controller API from -// the ones in this package. -// -// Attestations of the same type will cause an error. Some tools, -// like bake, remove the duplicates before calling this function. -func ConvertAttests(in []*Attest) ([]*controllerapi.Attest, error) { - out := make([]*controllerapi.Attest, 0, len(in)) - - // Check for duplicate attestations while we convert them - // to the controller API. - found := map[string]struct{}{} - for _, attest := range in { - if _, ok := found[attest.Type]; ok { - return nil, errors.Errorf("duplicate attestation field %s", attest.Type) - } - found[attest.Type] = struct{}{} - out = append(out, attest.ToPB()) - } - return out, nil -} - -func ParseAttest(in string) (*controllerapi.Attest, error) { - if in == "" { - return nil, nil - } - - fields, err := csvvalue.Fields(in, nil) - if err != nil { - return nil, err - } - - attest := controllerapi.Attest{ - Attrs: in, - } - for _, field := range fields { - key, value, ok := strings.Cut(field, "=") - if !ok { - return nil, errors.Errorf("invalid value %s", field) - } - key = strings.TrimSpace(strings.ToLower(key)) - - switch key { - case "type": - attest.Type = value - case "disabled": - disabled, err := strconv.ParseBool(value) - if err != nil { - return nil, errors.Wrapf(err, "invalid value %s", field) - } - attest.Disabled = disabled - } - } - if attest.Type == "" { - return nil, errors.Errorf("attestation type not specified") - } - - return &attest, nil + return outs, nil } func removeAttestDupes(s []*Attest) []*Attest { diff --git a/util/buildflags/cache.go b/util/buildflags/cache.go index 50a30ab23..a7bc70e27 100644 --- a/util/buildflags/cache.go +++ b/util/buildflags/cache.go @@ -1,15 +1,10 @@ package buildflags import ( - "context" "encoding/json" "maps" - "os" - "strconv" "strings" - awsconfig "github.com/aws/aws-sdk-go-v2/config" - controllerapi "github.com/docker/buildx/controller/pb" "github.com/pkg/errors" "github.com/tonistiigi/go-csvvalue" "github.com/zclconf/go-cty/cty" @@ -35,22 +30,6 @@ func (o CacheOptions) Normalize() CacheOptions { return removeDupes(o) } -func (o CacheOptions) ToPB() []*controllerapi.CacheOptionsEntry { - if len(o) == 0 { - return nil - } - - var outs []*controllerapi.CacheOptionsEntry - for _, entry := range o { - pb := entry.ToPB() - if !isActive(pb) { - continue - } - outs = append(outs, pb) - } - return outs -} - type CacheOptionsEntry struct { Type string `json:"type"` Attrs map[string]string `json:"attrs,omitempty"` @@ -81,16 +60,6 @@ func (e *CacheOptionsEntry) String() string { return b.String() } -func (e *CacheOptionsEntry) ToPB() *controllerapi.CacheOptionsEntry { - ci := &controllerapi.CacheOptionsEntry{ - Type: e.Type, - Attrs: maps.Clone(e.Attrs), - } - addGithubToken(ci) - addAwsCredentials(ci) - return ci -} - func (e *CacheOptionsEntry) MarshalJSON() ([]byte, error) { m := maps.Clone(e.Attrs) if m == nil { @@ -204,75 +173,3 @@ func ParseCacheEntry(in []string) (CacheOptions, error) { } return opts, nil } - -func addGithubToken(ci *controllerapi.CacheOptionsEntry) { - if ci.Type != "gha" { - return - } - version, ok := ci.Attrs["version"] - if !ok { - // https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L19 - if v, ok := os.LookupEnv("ACTIONS_CACHE_SERVICE_V2"); ok { - if b, err := strconv.ParseBool(v); err == nil && b { - version = "2" - } - } - } - if _, ok := ci.Attrs["token"]; !ok { - if v, ok := os.LookupEnv("ACTIONS_RUNTIME_TOKEN"); ok { - ci.Attrs["token"] = v - } - } - if _, ok := ci.Attrs["url_v2"]; !ok && version == "2" { - // https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L34-L35 - if v, ok := os.LookupEnv("ACTIONS_RESULTS_URL"); ok { - ci.Attrs["url_v2"] = v - } - } - if _, ok := ci.Attrs["url"]; !ok { - // https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L28-L33 - if v, ok := os.LookupEnv("ACTIONS_CACHE_URL"); ok { - ci.Attrs["url"] = v - } else if v, ok := os.LookupEnv("ACTIONS_RESULTS_URL"); ok { - ci.Attrs["url"] = v - } - } -} - -func addAwsCredentials(ci *controllerapi.CacheOptionsEntry) { - if ci.Type != "s3" { - return - } - _, okAccessKeyID := ci.Attrs["access_key_id"] - _, okSecretAccessKey := ci.Attrs["secret_access_key"] - // If the user provides access_key_id, secret_access_key, do not override the session token. - if okAccessKeyID && okSecretAccessKey { - return - } - ctx := context.TODO() - awsConfig, err := awsconfig.LoadDefaultConfig(ctx) - if err != nil { - return - } - credentials, err := awsConfig.Credentials.Retrieve(ctx) - if err != nil { - return - } - if !okAccessKeyID && credentials.AccessKeyID != "" { - ci.Attrs["access_key_id"] = credentials.AccessKeyID - } - if !okSecretAccessKey && credentials.SecretAccessKey != "" { - ci.Attrs["secret_access_key"] = credentials.SecretAccessKey - } - if _, ok := ci.Attrs["session_token"]; !ok && credentials.SessionToken != "" { - ci.Attrs["session_token"] = credentials.SessionToken - } -} - -func isActive(pb *controllerapi.CacheOptionsEntry) bool { - // Always active if not gha. - if pb.Type != "gha" { - return true - } - return pb.Attrs["token"] != "" && (pb.Attrs["url"] != "" || pb.Attrs["url_v2"] != "") -} diff --git a/util/buildflags/cache_test.go b/util/buildflags/cache_test.go index 02d2f8ba4..6558034a7 100644 --- a/util/buildflags/cache_test.go +++ b/util/buildflags/cache_test.go @@ -4,42 +4,10 @@ import ( "encoding/json" "testing" - "github.com/docker/buildx/controller/pb" "github.com/stretchr/testify/require" "github.com/zclconf/go-cty/cty" ) -func TestCacheOptions_DerivedVars(t *testing.T) { - t.Setenv("ACTIONS_RUNTIME_TOKEN", "sensitive_token") - t.Setenv("ACTIONS_CACHE_URL", "https://cache.github.com") - t.Setenv("AWS_ACCESS_KEY_ID", "definitely_dont_look_here") - t.Setenv("AWS_SECRET_ACCESS_KEY", "hackers_please_dont_steal") - t.Setenv("AWS_SESSION_TOKEN", "not_a_mitm_attack") - - cacheFrom, err := ParseCacheEntry([]string{"type=gha", "type=s3,region=us-west-2,bucket=my_bucket,name=my_image"}) - require.NoError(t, err) - require.Equal(t, []*pb.CacheOptionsEntry{ - { - Type: "gha", - Attrs: map[string]string{ - "token": "sensitive_token", - "url": "https://cache.github.com", - }, - }, - { - Type: "s3", - Attrs: map[string]string{ - "region": "us-west-2", - "bucket": "my_bucket", - "name": "my_image", - "access_key_id": "definitely_dont_look_here", - "secret_access_key": "hackers_please_dont_steal", - "session_token": "not_a_mitm_attack", - }, - }, - }, cacheFrom.ToPB()) -} - func TestCacheOptions(t *testing.T) { t.Run("MarshalJSON", func(t *testing.T) { cache := CacheOptions{ diff --git a/util/buildflags/callfunc.go b/util/buildflags/callfunc.go index 7ee1a85f6..7dfe63f00 100644 --- a/util/buildflags/callfunc.go +++ b/util/buildflags/callfunc.go @@ -1,17 +1,37 @@ package buildflags import ( + "fmt" "strconv" "strings" - controllerapi "github.com/docker/buildx/controller/pb" "github.com/pkg/errors" "github.com/tonistiigi/go-csvvalue" ) const defaultCallFunc = "build" -func ParseCallFunc(str string) (*controllerapi.CallFunc, error) { +type CallFunc struct { + Name string + Format string + IgnoreStatus bool +} + +func (x *CallFunc) String() string { + var elems []string + if x.Name != "" { + elems = append(elems, fmt.Sprintf("Name:%q", x.Name)) + } + if x.Format != "" { + elems = append(elems, fmt.Sprintf("Format:%q", x.Format)) + } + if x.IgnoreStatus { + elems = append(elems, fmt.Sprintf("IgnoreStatus:%v", x.IgnoreStatus)) + } + return strings.Join(elems, " ") +} + +func ParseCallFunc(str string) (*CallFunc, error) { if str == "" { return nil, nil } @@ -20,7 +40,7 @@ func ParseCallFunc(str string) (*controllerapi.CallFunc, error) { if err != nil { return nil, err } - f := &controllerapi.CallFunc{} + f := &CallFunc{} for _, field := range fields { parts := strings.SplitN(field, "=", 2) if len(parts) == 2 { diff --git a/util/buildflags/export.go b/util/buildflags/export.go index 8460d3e29..ad96c76a5 100644 --- a/util/buildflags/export.go +++ b/util/buildflags/export.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/containerd/platforms" - controllerapi "github.com/docker/buildx/controller/pb" "github.com/moby/buildkit/client" "github.com/moby/buildkit/exporter/containerimage/exptypes" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" @@ -38,18 +37,6 @@ func (e Exports) Normalize() Exports { return removeDupes(e) } -func (e Exports) ToPB() []*controllerapi.ExportEntry { - if len(e) == 0 { - return nil - } - - entries := make([]*controllerapi.ExportEntry, len(e)) - for i, entry := range e { - entries[i] = entry.ToPB() - } - return entries -} - type ExportEntry struct { Type string `json:"type"` Attrs map[string]string `json:"attrs,omitempty"` @@ -77,14 +64,6 @@ func (e *ExportEntry) String() string { return b.String() } -func (e *ExportEntry) ToPB() *controllerapi.ExportEntry { - return &controllerapi.ExportEntry{ - Type: e.Type, - Attrs: maps.Clone(e.Attrs), - Destination: e.Destination, - } -} - func (e *ExportEntry) MarshalJSON() ([]byte, error) { m := maps.Clone(e.Attrs) if m == nil { @@ -164,12 +143,12 @@ func (e *ExportEntry) validate() error { return nil } -func ParseExports(inp []string) ([]*controllerapi.ExportEntry, error) { +func ParseExports(inp []string) ([]*ExportEntry, error) { if len(inp) == 0 { return nil, nil } - export := make(Exports, 0, len(inp)) + exports := make(Exports, 0, len(inp)) for _, s := range inp { if s == "" { continue @@ -179,9 +158,9 @@ func ParseExports(inp []string) ([]*controllerapi.ExportEntry, error) { if err := out.UnmarshalText([]byte(s)); err != nil { return nil, err } - export = append(export, &out) + exports = append(exports, &out) } - return export.ToPB(), nil + return exports, nil } func ParseAnnotations(inp []string) (map[exptypes.AnnotationKey]string, error) { diff --git a/util/buildflags/secrets.go b/util/buildflags/secrets.go index 498a588b0..994fc416a 100644 --- a/util/buildflags/secrets.go +++ b/util/buildflags/secrets.go @@ -4,7 +4,6 @@ import ( "encoding/json" "strings" - controllerapi "github.com/docker/buildx/controller/pb" "github.com/pkg/errors" "github.com/tonistiigi/go-csvvalue" ) @@ -30,18 +29,6 @@ func (s Secrets) Normalize() Secrets { return removeSecretDupes(s) } -func (s Secrets) ToPB() []*controllerapi.Secret { - if len(s) == 0 { - return nil - } - - entries := make([]*controllerapi.Secret, len(s)) - for i, entry := range s { - entries[i] = entry.ToPB() - } - return entries -} - type Secret struct { ID string `json:"id,omitempty"` FilePath string `json:"src,omitempty"` @@ -66,14 +53,6 @@ func (s *Secret) String() string { return b.String() } -func (s *Secret) ToPB() *controllerapi.Secret { - return &controllerapi.Secret{ - ID: s.ID, - FilePath: s.FilePath, - Env: s.Env, - } -} - func (s *Secret) UnmarshalJSON(data []byte) error { var v struct { ID string `json:"id,omitempty"` @@ -132,8 +111,8 @@ func (s *Secret) UnmarshalText(text []byte) error { return nil } -func ParseSecretSpecs(sl []string) ([]*controllerapi.Secret, error) { - fs := make([]*controllerapi.Secret, 0, len(sl)) +func ParseSecretSpecs(sl []string) (Secrets, error) { + fs := make([]*Secret, 0, len(sl)) for _, v := range sl { if v == "" { continue @@ -148,12 +127,12 @@ func ParseSecretSpecs(sl []string) ([]*controllerapi.Secret, error) { return fs, nil } -func parseSecret(value string) (*controllerapi.Secret, error) { +func parseSecret(value string) (*Secret, error) { var s Secret if err := s.UnmarshalText([]byte(value)); err != nil { return nil, err } - return s.ToPB(), nil + return &s, nil } func removeSecretDupes(s []*Secret) []*Secret { diff --git a/util/buildflags/ssh.go b/util/buildflags/ssh.go index 80367552c..5af892d17 100644 --- a/util/buildflags/ssh.go +++ b/util/buildflags/ssh.go @@ -6,7 +6,6 @@ import ( "slices" "strings" - controllerapi "github.com/docker/buildx/controller/pb" "github.com/moby/buildkit/util/gitutil" ) @@ -31,18 +30,6 @@ func (s SSHKeys) Normalize() SSHKeys { return removeSSHDupes(s) } -func (s SSHKeys) ToPB() []*controllerapi.SSH { - if len(s) == 0 { - return nil - } - - entries := make([]*controllerapi.SSH, len(s)) - for i, entry := range s { - entries[i] = entry.ToPB() - } - return entries -} - type SSH struct { ID string `json:"id,omitempty" cty:"id"` Paths []string `json:"paths,omitempty" cty:"paths"` @@ -70,13 +57,6 @@ func (s *SSH) String() string { return b.String() } -func (s *SSH) ToPB() *controllerapi.SSH { - return &controllerapi.SSH{ - ID: s.ID, - Paths: s.Paths, - } -} - func (s *SSH) UnmarshalJSON(data []byte) error { var v struct { ID string `json:"id,omitempty"` @@ -103,8 +83,8 @@ func (s *SSH) UnmarshalText(text []byte) error { return nil } -func ParseSSHSpecs(sl []string) ([]*controllerapi.SSH, error) { - var outs []*controllerapi.SSH +func ParseSSHSpecs(sl []string) ([]*SSH, error) { + var outs []*SSH if len(sl) == 0 { return nil, nil } @@ -118,7 +98,7 @@ func ParseSSHSpecs(sl []string) ([]*controllerapi.SSH, error) { if err := out.UnmarshalText([]byte(s)); err != nil { return nil, err } - outs = append(outs, out.ToPB()) + outs = append(outs, &out) } return outs, nil }