From cb54ddb9fe014a87ad4445378634cd541bab098f Mon Sep 17 00:00:00 2001 From: Roberto Villarreal Date: Sun, 15 Jun 2025 23:53:11 -0600 Subject: [PATCH 1/2] Allow bake files to be specified via environment variable The environment variable `BUILDX_BAKE_FILE` (and optional variable `BUILDX_BAKE_FILE_SEPARATOR`) can be used to specify one or more bake files (similar to `compose`). This is mutually exclusive with`--file` (which takes precedence). This is done very early to ensure the values are treated just like `--file`, e.g., participate in telemetry. This includes leaving relative paths as-is, which deviates from `compose` (which makes them absolute). Signed-off-by: Roberto Villarreal --- commands/bake.go | 43 +++++++++ docs/reference/buildx_bake.md | 5 ++ tests/bake.go | 160 ++++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+) diff --git a/commands/bake.go b/commands/bake.go index 571385058..a62e7a8b5 100644 --- a/commands/bake.go +++ b/commands/bake.go @@ -40,6 +40,11 @@ import ( "go.opentelemetry.io/otel/attribute" ) +const ( + bakeEnvFileSeparator = "BUILDX_BAKE_PATH_SEPARATOR" + bakeEnvFilePath = "BUILDX_BAKE_FILE" +) + type bakeOptions struct { files []string overrides []string @@ -452,6 +457,13 @@ func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command { Aliases: []string{"f"}, Short: "Build from a file", RunE: func(cmd *cobra.Command, args []string) error { + if len(options.files) == 0 { + envFiles, err := bakeEnvFiles(os.LookupEnv) + if err != nil { + return err + } + options.files = envFiles + } // reset to nil to avoid override is unset if !cmd.Flags().Lookup("no-cache").Changed { cFlags.noCache = nil @@ -504,6 +516,37 @@ func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command { return cmd } +func bakeEnvFiles(lookup func(string string) (string, bool)) ([]string, error) { + sep, _ := lookup(bakeEnvFileSeparator) + if sep == "" { + sep = string(os.PathListSeparator) + } + f, ok := lookup(bakeEnvFilePath) + if ok { + return cleanPaths(strings.Split(f, sep)) + } + return []string{}, nil +} + +func cleanPaths(p []string) ([]string, error) { + var paths []string + for _, f := range p { + f = strings.TrimSpace(f) + if f == "" { + continue + } + if f == "-" { + paths = append(paths, f) + continue + } + if _, err := os.Stat(f); err != nil { + return nil, err + } + paths = append(paths, f) + } + return paths, nil +} + func saveLocalStateGroup(dockerCli command.Cli, in bakeOptions, targets []string, bo map[string]build.Options) error { l, err := localstate.New(confutil.NewConfig(dockerCli)) if err != nil { diff --git a/docs/reference/buildx_bake.md b/docs/reference/buildx_bake.md index ab896703a..49e5e3036 100644 --- a/docs/reference/buildx_bake.md +++ b/docs/reference/buildx_bake.md @@ -143,6 +143,11 @@ Use the `-f` / `--file` option to specify the build definition file to use. The file can be an HCL, JSON or Compose file. If multiple files are specified, all are read and the build configurations are combined. +Alternatively, the environment variable `BUILDX_BAKE_FILE` can be used to specify the build definition to use. +This is mutually exclusive with `-f` / `--file`; if both are specified, the environment variable is ignored. +Multiple definitions can be specified by separating them with the system's path separator +(typically `;` on Windows and `:` elsewhere), but can be changed with `BUILDX_BAKE_PATH_SEPARATOR`. + You can pass the names of the targets to build, to build only specific target(s). The following example builds the `db` and `webapp-release` targets that are defined in the `docker-bake.dev.hcl` file: diff --git a/tests/bake.go b/tests/bake.go index fa6e2f4dd..f141f314f 100644 --- a/tests/bake.go +++ b/tests/bake.go @@ -81,6 +81,7 @@ var bakeTests = []func(t *testing.T, sb integration.Sandbox){ testBakeMultiPlatform, testBakeCheckCallOutput, testBakeExtraHosts, + testBakeFileFromEnvironment, } func testBakePrint(t *testing.T, sb integration.Sandbox) { @@ -2188,6 +2189,165 @@ target "default" { require.NoError(t, err, out) } +func testBakeFileFromEnvironment(t *testing.T, sb integration.Sandbox) { + bakeFileFirst := []byte(` +target "first" { + dockerfile-inline = "FROM scratch\nCOPY first /" +} +`) + bakeFileSecond := []byte(` +target "second" { + dockerfile-inline = "FROM scratch\nCOPY second /" +} +`) + + t.Run("single file", func(t *testing.T) { + dir := tmpdir(t, + fstest.CreateFile("first.hcl", bakeFileFirst, 0600), + fstest.CreateFile("first", []byte("first"), 0600), + ) + cmd := buildxCmd(sb, + withDir(dir), + withArgs("bake", "--progress=plain", "first"), + withEnv("BUILDX_BAKE_FILE=first.hcl")) + + dt, err := cmd.CombinedOutput() + require.NoError(t, err, string(dt)) + require.Contains(t, string(dt), `#1 [internal] load local bake definitions`) + require.Contains(t, string(dt), `#1 reading first.hcl`) + }) + + t.Run("single file, default ignored if present", func(t *testing.T) { + dir := tmpdir(t, + fstest.CreateFile("first.hcl", bakeFileFirst, 0600), + fstest.CreateFile("first", []byte("first"), 0600), + fstest.CreateFile("docker-bake.hcl", []byte("invalid bake file"), 0600), + ) + cmd := buildxCmd(sb, + withDir(dir), + withArgs("bake", "--progress=plain", "first"), + withEnv("BUILDX_BAKE_FILE=first.hcl")) + + dt, err := cmd.CombinedOutput() + require.NoError(t, err, string(dt)) + require.Contains(t, string(dt), `#1 [internal] load local bake definitions`) + require.Contains(t, string(dt), `#1 reading first.hcl`) + require.NotContains(t, string(dt), "docker-bake.hcl") + }) + + t.Run("multiple files", func(t *testing.T) { + dir := tmpdir(t, + fstest.CreateFile("first.hcl", bakeFileFirst, 0600), + fstest.CreateFile("first", []byte("first"), 0600), + fstest.CreateFile("second.hcl", bakeFileSecond, 0600), + fstest.CreateFile("second", []byte("second"), 0600), + ) + + cmd := buildxCmd(sb, + withDir(dir), + withArgs("bake", "--progress=plain", "second", "first"), + withEnv("BUILDX_BAKE_FILE=first.hcl"+string(os.PathListSeparator)+"second.hcl")) + dt, err := cmd.CombinedOutput() + require.NoError(t, err, string(dt)) + require.Contains(t, string(dt), `#1 [internal] load local bake definitions`) + require.Contains(t, string(dt), `#1 reading first.hcl`) + require.Contains(t, string(dt), `#1 reading second.hcl`) + }) + + t.Run("multiple files, custom separator", func(t *testing.T) { + dir := tmpdir(t, + fstest.CreateFile("first.hcl", bakeFileFirst, 0600), + fstest.CreateFile("first", []byte("first"), 0600), + fstest.CreateFile("second.hcl", bakeFileSecond, 0600), + fstest.CreateFile("second", []byte("second"), 0600), + ) + + cmd := buildxCmd(sb, + withDir(dir), + withArgs("bake", "--progress=plain", "second", "first"), + withEnv("BUILDX_BAKE_PATH_SEPARATOR=@", "BUILDX_BAKE_FILE=first.hcl@second.hcl")) + + dt, err := cmd.CombinedOutput() + require.NoError(t, err, string(dt)) + require.Contains(t, string(dt), `#1 [internal] load local bake definitions`) + require.Contains(t, string(dt), `#1 reading first.hcl`) + require.Contains(t, string(dt), `#1 reading second.hcl`) + }) + + t.Run("multiple files, one STDIN", func(t *testing.T) { + dir := tmpdir(t, + fstest.CreateFile("first.hcl", bakeFileFirst, 0600), + fstest.CreateFile("first", []byte("first"), 0600), + fstest.CreateFile("second", []byte("second"), 0600), + ) + + cmd := buildxCmd(sb, + withDir(dir), + withArgs("bake", "--progress=plain", "second", "first"), + withEnv("BUILDX_BAKE_FILE=first.hcl"+string(os.PathListSeparator)+"-")) + w, err := cmd.StdinPipe() + require.NoError(t, err) + go func() { + defer w.Close() + w.Write(bakeFileSecond) + }() + + dt, err := cmd.CombinedOutput() + require.NoError(t, err, string(dt)) + require.Contains(t, string(dt), `#1 [internal] load local bake definitions`) + require.Contains(t, string(dt), `#1 reading first.hcl`) + require.Contains(t, string(dt), `#1 reading from stdin`) + }) + + t.Run("env ignored if file arg passed", func(t *testing.T) { + dir := tmpdir(t, + fstest.CreateFile("first.hcl", bakeFileFirst, 0600), + fstest.CreateFile("first", []byte("first"), 0600), + fstest.CreateFile("second.hcl", bakeFileSecond, 0600), + ) + cmd := buildxCmd(sb, + withDir(dir), + withArgs("bake", "--progress=plain", "-f", "first.hcl", "first", "second"), + withEnv("BUILDX_BAKE_FILE=second.hcl")) + + dt, err := cmd.CombinedOutput() + require.Error(t, err, string(dt)) + require.Contains(t, string(dt), "failed to find target second") + }) + + t.Run("file does not exist", func(t *testing.T) { + dir := tmpdir(t, + fstest.CreateFile("first.hcl", bakeFileFirst, 0600), + fstest.CreateFile("first", []byte("first"), 0600), + ) + cmd := buildxCmd(sb, + withDir(dir), + withArgs("bake", "--progress=plain", "first"), + withEnv("BUILDX_BAKE_FILE=wrong.hcl")) + + dt, err := cmd.CombinedOutput() + require.Error(t, err, string(dt)) + require.Contains(t, string(dt), "wrong.hcl: no such file or directory") + }) + + for kind, val := range map[string]string{"missing": "", "whitespace": " "} { + t.Run(kind+" value ignored", func(t *testing.T) { + dir := tmpdir(t, + fstest.CreateFile("first.hcl", bakeFileFirst, 0600), + fstest.CreateFile("first", []byte("first"), 0600), + ) + cmd := buildxCmd(sb, + withDir(dir), + withArgs("bake", "--progress=plain", "first"), + withEnv(fmt.Sprintf("BUILDX_BAKE_FILE=%s", val))) + + dt, err := cmd.CombinedOutput() + require.Error(t, err, string(dt)) + require.Contains(t, string(dt), "couldn't find a bake definition") + }) + } +} + func writeTempPrivateKey(fp string) error { privateKey, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { From d44ffb4bd49fc8de7eacf93bd8b0d3c380833b74 Mon Sep 17 00:00:00 2001 From: Roberto Villarreal Date: Fri, 20 Jun 2025 19:24:30 -0600 Subject: [PATCH 2/2] Display source of bake definitions when read from environment While it would make sense to add "from file" to complement "from env," (in the common case of `--file` or using the default), it wouldn't provide any real value. A simpler solution would have been looking for the existence of the variable at the point where printing happens. It felt wrong duplicating the logic. Executing the same logic (if it was extracted) wouldn't be as bad, but still not ideal. A 'correct' solution would be to explicitly track the source of each definition, which would be clearer and more future-proof. It didn't seem like this feature warranted that amount of engineering (with no known features that might make use of it). This implementation seemed like a fair compromise; none of the functions are exported, and all have only one caller. I also considered converting prefixing environment values with `env://` so they could be thought of (and processed like) `cmd://` values. I didn't think it would be viewed as a good solution. Co-authored-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> Signed-off-by: Roberto Villarreal --- commands/bake.go | 16 +++++++++++----- tests/bake.go | 10 +++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/commands/bake.go b/commands/bake.go index a62e7a8b5..9e24aa78b 100644 --- a/commands/bake.go +++ b/commands/bake.go @@ -67,7 +67,7 @@ type bakeOptions struct { listVars bool } -func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in bakeOptions, cFlags commonFlags) (err error) { +func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in bakeOptions, cFlags commonFlags, filesFromEnv bool) (err error) { mp := dockerCli.MeterProvider() ctx, end, err := tracing.TraceCurrentCommand(ctx, append([]string{"bake"}, targets...), @@ -185,7 +185,7 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba return err } - files, inp, err := readBakeFiles(ctx, nodes, url, in.files, dockerCli.In(), printer) + files, inp, err := readBakeFiles(ctx, nodes, url, in.files, dockerCli.In(), printer, filesFromEnv) if err != nil { return err } @@ -457,12 +457,14 @@ func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command { Aliases: []string{"f"}, Short: "Build from a file", RunE: func(cmd *cobra.Command, args []string) error { + filesFromEnv := false if len(options.files) == 0 { envFiles, err := bakeEnvFiles(os.LookupEnv) if err != nil { return err } options.files = envFiles + filesFromEnv = true } // reset to nil to avoid override is unset if !cmd.Flags().Lookup("no-cache").Changed { @@ -481,7 +483,7 @@ func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command { options.builder = rootOpts.builder options.metadataFile = cFlags.metadataFile // Other common flags (noCache, pull and progress) are processed in runBake function. - return runBake(cmd.Context(), dockerCli, args, options, cFlags) + return runBake(cmd.Context(), dockerCli, args, options, cFlags, filesFromEnv) }, ValidArgsFunction: completion.BakeTargets(options.files), } @@ -596,7 +598,7 @@ func bakeArgs(args []string) (url, cmdContext string, targets []string) { return url, cmdContext, targets } -func readBakeFiles(ctx context.Context, nodes []builder.Node, url string, names []string, stdin io.Reader, pw progress.Writer) (files []bake.File, inp *bake.Input, err error) { +func readBakeFiles(ctx context.Context, nodes []builder.Node, url string, names []string, stdin io.Reader, pw progress.Writer, filesFromEnv bool) (files []bake.File, inp *bake.Input, err error) { var lnames []string // local var rnames []string // remote var anames []string // both @@ -621,7 +623,11 @@ func readBakeFiles(ctx context.Context, nodes []builder.Node, url string, names if len(lnames) > 0 || url == "" { var lfiles []bake.File - progress.Wrap("[internal] load local bake definitions", pw.Write, func(sub progress.SubLogger) error { + where := "" + if filesFromEnv { + where = " from " + bakeEnvFilePath + " env" + } + progress.Wrap("[internal] load local bake definitions"+where, pw.Write, func(sub progress.SubLogger) error { if url != "" { lfiles, err = bake.ReadLocalFiles(lnames, stdin, sub) } else { diff --git a/tests/bake.go b/tests/bake.go index f141f314f..03354ec8f 100644 --- a/tests/bake.go +++ b/tests/bake.go @@ -2213,7 +2213,7 @@ target "second" { dt, err := cmd.CombinedOutput() require.NoError(t, err, string(dt)) - require.Contains(t, string(dt), `#1 [internal] load local bake definitions`) + require.Contains(t, string(dt), `#1 [internal] load local bake definitions from BUILDX_BAKE_FILE env`) require.Contains(t, string(dt), `#1 reading first.hcl`) }) @@ -2230,7 +2230,7 @@ target "second" { dt, err := cmd.CombinedOutput() require.NoError(t, err, string(dt)) - require.Contains(t, string(dt), `#1 [internal] load local bake definitions`) + require.Contains(t, string(dt), `#1 [internal] load local bake definitions from BUILDX_BAKE_FILE env`) require.Contains(t, string(dt), `#1 reading first.hcl`) require.NotContains(t, string(dt), "docker-bake.hcl") }) @@ -2249,7 +2249,7 @@ target "second" { withEnv("BUILDX_BAKE_FILE=first.hcl"+string(os.PathListSeparator)+"second.hcl")) dt, err := cmd.CombinedOutput() require.NoError(t, err, string(dt)) - require.Contains(t, string(dt), `#1 [internal] load local bake definitions`) + require.Contains(t, string(dt), `#1 [internal] load local bake definitions from BUILDX_BAKE_FILE env`) require.Contains(t, string(dt), `#1 reading first.hcl`) require.Contains(t, string(dt), `#1 reading second.hcl`) }) @@ -2269,7 +2269,7 @@ target "second" { dt, err := cmd.CombinedOutput() require.NoError(t, err, string(dt)) - require.Contains(t, string(dt), `#1 [internal] load local bake definitions`) + require.Contains(t, string(dt), `#1 [internal] load local bake definitions from BUILDX_BAKE_FILE env`) require.Contains(t, string(dt), `#1 reading first.hcl`) require.Contains(t, string(dt), `#1 reading second.hcl`) }) @@ -2294,7 +2294,7 @@ target "second" { dt, err := cmd.CombinedOutput() require.NoError(t, err, string(dt)) - require.Contains(t, string(dt), `#1 [internal] load local bake definitions`) + require.Contains(t, string(dt), `#1 [internal] load local bake definitions from BUILDX_BAKE_FILE env`) require.Contains(t, string(dt), `#1 reading first.hcl`) require.Contains(t, string(dt), `#1 reading from stdin`) })