alextab93/buildx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request #3152 from crazy-max/history-export-finalize

Browse Source 
history: make sure build record is finalized before exporting
This commit is contained in:
Tõnis Tiigi
2025-05-13 14:20:04 -07:00
committed by GitHub
parent 3f0aec1b3e 68ce10c4d9
commit 277548e91b
6 changed files with 209 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commands/history/export.go
+27 -5
View File
@@ -20,10 +20,11 @@ import (
)
type exportOptions struct {
builder string
refs []string
output string
all bool
builder string
refs []string
output string
all bool
finalize bool
}
func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) error {
@@ -62,6 +63,26 @@ func runExport(ctx context.Context, dockerCli command.Cli, opts exportOptions) e
return errors.Errorf("no record found for ref %q", ref)
}
if opts.finalize {
var finalized bool
for _, rec := range recs {
if rec.Trace == nil {
finalized = true
if err := finalizeRecord(ctx, rec.Ref, nodes); err != nil {
return err
}
}
}
if finalized {
recs, err = queryRecords(ctx, ref, nodes, &queryOptions{
CompletedOnly: true,
})
if err != nil {
return err
}
}
}
if ref == "" {
slices.SortFunc(recs, func(a, b historyRecord) int {
return b.CreatedAt.AsTime().Compare(a.CreatedAt.AsTime())
@@ -154,7 +175,8 @@ func exportCmd(dockerCli command.Cli, rootOpts RootOptions) *cobra.Command {
flags := cmd.Flags()
flags.StringVarP(&options.output, "output", "o", "", "Output file path")
flags.BoolVar(&options.all, "all", false, "Export all records for the builder")
flags.BoolVar(&options.all, "all", false, "Export all build records for the builder")
flags.BoolVar(&options.finalize, "finalize", false, "Ensure build records are finalized before exporting")
return cmd
}
commands/history/trace.go
+1 -9
View File
@@ -17,7 +17,6 @@ import (
"github.com/docker/buildx/util/otelutil"
"github.com/docker/buildx/util/otelutil/jaeger"
"github.com/docker/cli/cli/command"
controlapi "github.com/moby/buildkit/api/services/control"
"github.com/opencontainers/go-digest"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/browser"
@@ -57,14 +56,7 @@ func loadTrace(ctx context.Context, ref string, nodes []builder.Node) (string, [
// build is complete but no trace yet. try to finalize the trace
time.Sleep(1 * time.Second) // give some extra time for last parts of trace to be written
c, err := rec.node.Driver.Client(ctx)
if err != nil {
return "", nil, err
}
_, err = c.ControlClient().UpdateBuildHistory(ctx, &controlapi.UpdateBuildHistoryRequest{
Ref: rec.Ref,
Finalize: true,
})
err := finalizeRecord(ctx, rec.Ref, []builder.Node{*rec.node})
if err != nil {
return "", nil, err
}
commands/history/utils.go
+22
View File
@@ -247,6 +247,28 @@ func queryRecords(ctx context.Context, ref string, nodes []builder.Node, opts *q
return out, nil
}
func finalizeRecord(ctx context.Context, ref string, nodes []builder.Node) error {
eg, ctx := errgroup.WithContext(ctx)
for _, node := range nodes {
node := node
eg.Go(func() error {
if node.Driver == nil {
return nil
}
c, err := node.Driver.Client(ctx)
if err != nil {
return err
}
_, err = c.ControlClient().UpdateBuildHistory(ctx, &controlapi.UpdateBuildHistoryRequest{
Ref: ref,
Finalize: true,
})
return err
})
}
return eg.Wait()
}
func formatDuration(d time.Duration) string {
if d < time.Minute {
return fmt.Sprintf("%.1fs", d.Seconds())
docs/reference/buildx_history_export.md
+17 -6
View File
@@ -5,12 +5,13 @@ Export build records into Docker Desktop bundle
### Options
| Name | Type | Default | Description |
|:---------------------------------------|:---------|:--------|:-----------------------------------------|
| [`--all`](#all) | `bool` | | Export all records for the builder |
| [`--builder`](#builder) | `string` | | Override the configured builder instance |
| [`-D`](#debug), [`--debug`](#debug) | `bool` | | Enable debug logging |
| [`-o`](#output), [`--output`](#output) | `string` | | Output file path |
| Name | Type | Default | Description |
|:---------------------------------------|:---------|:--------|:----------------------------------------------------|
| [`--all`](#all) | `bool` | | Export all build records for the builder |
| [`--builder`](#builder) | `string` | | Override the configured builder instance |
| [`-D`](#debug), [`--debug`](#debug) | `bool` | | Enable debug logging |
| [`--finalize`](#finalize) | `bool` | | Ensure build records are finalized before exporting |
| [`-o`](#output), [`--output`](#output) | `string` | | Output file path |
<!---MARKER_GEN_END-->
@@ -49,6 +50,16 @@ docker buildx history export --builder builder0 ^1 -o builder0-build.dockerbuild
docker buildx history export --debug qu2gsuo8ejqrwdfii23xkkckt -o debug-build.dockerbuild
```
### <a name="finalize"></a> Ensure build records are finalized before exporting (--finalize)
Clients can report their own traces concurrently, and not all traces may be
saved yet by the time of the export. Use the `--finalize` flag to ensure all
traces are finalized before exporting.
```console
docker buildx history export --finalize qu2gsuo8ejqrwdfii23xkkckt -o finalized-build.dockerbuild
```
### <a name="output"></a> Export a single build to a custom file (--output)
```console
tests/history.go
+141
View File
@@ -0,0 +1,141 @@
package tests
import (
"encoding/json"
"os"
"path"
"path/filepath"
"strings"
"testing"
"time"
"github.com/moby/buildkit/util/testutil/integration"
"github.com/stretchr/testify/require"
)
var historyTests = []func(t *testing.T, sb integration.Sandbox){
testHistoryExport,
testHistoryExportFinalize,
testHistoryInspect,
testHistoryLs,
testHistoryRm,
}
func testHistoryExport(t *testing.T, sb integration.Sandbox) {
ref := buildTestProject(t, sb)
require.NotEmpty(t, ref.Ref)
outFile := path.Join(t.TempDir(), "export.dockerbuild")
cmd := buildxCmd(sb, withArgs("history", "export", ref.Ref, "--output", outFile))
out, err := cmd.Output()
require.NoError(t, err, string(out))
require.FileExists(t, outFile)
}
func testHistoryExportFinalize(t *testing.T, sb integration.Sandbox) {
ref := buildTestProject(t, sb)
require.NotEmpty(t, ref.Ref)
outFile := path.Join(t.TempDir(), "export.dockerbuild")
cmd := buildxCmd(sb, withArgs("history", "export", ref.Ref, "--finalize", "--output", outFile))
out, err := cmd.Output()
require.NoError(t, err, string(out))
require.FileExists(t, outFile)
}
func testHistoryInspect(t *testing.T, sb integration.Sandbox) {
ref := buildTestProject(t, sb)
require.NotEmpty(t, ref.Ref)
cmd := buildxCmd(sb, withArgs("history", "inspect", ref.Ref, "--format=json"))
out, err := cmd.Output()
require.NoError(t, err, string(out))
type recT struct {
Name string
Ref string
Context string
Dockerfile string
StartedAt *time.Time
CompletedAt *time.Time
Duration time.Duration
Status string
NumCompletedSteps int32
NumTotalSteps int32
NumCachedSteps int32
}
var rec recT
err = json.Unmarshal(out, &rec)
require.NoError(t, err)
require.Equal(t, ref.Ref, rec.Ref)
require.NotEmpty(t, rec.Name)
}
func testHistoryLs(t *testing.T, sb integration.Sandbox) {
ref := buildTestProject(t, sb)
require.NotEmpty(t, ref.Ref)
cmd := buildxCmd(sb, withArgs("history", "ls", "--filter=ref="+ref.Ref, "--format=json"))
out, err := cmd.Output()
require.NoError(t, err, string(out))
type recT struct {
Ref string `json:"ref"`
Name string `json:"name"`
Status string `json:"status"`
CreatedAt *time.Time `json:"created_at"`
CompletedAt *time.Time `json:"completed_at"`
TotalSteps int32 `json:"total_steps"`
CompletedSteps int32 `json:"completed_steps"`
CachedSteps int32 `json:"cached_steps"`
}
var rec recT
err = json.Unmarshal(out, &rec)
require.NoError(t, err)
require.Equal(t, ref.String(), rec.Ref)
require.NotEmpty(t, rec.Name)
}
func testHistoryRm(t *testing.T, sb integration.Sandbox) {
ref := buildTestProject(t, sb)
require.NotEmpty(t, ref.Ref)
cmd := buildxCmd(sb, withArgs("history", "rm", ref.Ref))
out, err := cmd.Output()
require.NoError(t, err, string(out))
}
type buildRef struct {
Builder string
Node string
Ref string
}
func (b buildRef) String() string {
return b.Builder + "/" + b.Node + "/" + b.Ref
}
func buildTestProject(t *testing.T, sb integration.Sandbox) buildRef {
dir := createTestProject(t)
out, err := buildCmd(sb, withArgs("--metadata-file", filepath.Join(dir, "md.json"), dir))
require.NoError(t, err, string(out))
dt, err := os.ReadFile(filepath.Join(dir, "md.json"))
require.NoError(t, err)
type mdT struct {
BuildRef string `json:"buildx.build.ref"`
}
var md mdT
err = json.Unmarshal(dt, &md)
require.NoError(t, err)
refParts := strings.Split(md.BuildRef, "/")
require.Len(t, refParts, 3)
return buildRef{
Builder: refParts[0],
Node: refParts[1],
Ref: refParts[2],
}
}
tests/integration_test.go
+1
View File
@@ -24,6 +24,7 @@ func TestIntegration(t *testing.T) {
tests = append(tests, commonTests...)
tests = append(tests, buildTests...)
tests = append(tests, bakeTests...)
tests = append(tests, historyTests...)
tests = append(tests, inspectTests...)
tests = append(tests, lsTests...)
tests = append(tests, imagetoolsTests...)