diff --git a/bake/bake.go b/bake/bake.go
index 3c8d90b7e..7a89e0d1e 100644
--- a/bake/bake.go
+++ b/bake/bake.go
@@ -23,12 +23,10 @@ import (
 	"github.com/docker/buildx/util/buildflags"
 	"github.com/docker/buildx/util/platformutil"
 	"github.com/docker/buildx/util/progress"
-	"github.com/docker/cli/cli/config"
 	dockeropts "github.com/docker/cli/opts"
 	hcl "github.com/hashicorp/hcl/v2"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/client/llb"
-	"github.com/moby/buildkit/session/auth/authprovider"
 	"github.com/pkg/errors"
 	"github.com/zclconf/go-cty/cty"
 	"github.com/zclconf/go-cty/cty/convert"
@@ -1256,18 +1254,12 @@ func (t *Target) GetName(ectx *hcl.EvalContext, block *hcl.Block, loadDeps func(
 }
 
 func TargetsToBuildOpt(m map[string]*Target, inp *Input) (map[string]build.Options, error) {
-	// make sure local credentials are loaded multiple times for different targets
-	authProvider := authprovider.NewDockerAuthProvider(authprovider.DockerAuthProviderConfig{
-		ConfigFile: config.LoadDefaultConfigFile(os.Stderr),
-	})
-
 	m2 := make(map[string]build.Options, len(m))
 	for k, v := range m {
 		bo, err := toBuildOpt(v, inp)
 		if err != nil {
 			return nil, err
 		}
-		bo.Session = append(bo.Session, authProvider)
 		m2[k] = *bo
 	}
 	return m2, nil
diff --git a/commands/bake.go b/commands/bake.go
index ce261087e..754b43024 100644
--- a/commands/bake.go
+++ b/commands/bake.go
@@ -29,11 +29,13 @@ import (
 	"github.com/docker/buildx/util/confutil"
 	"github.com/docker/buildx/util/desktop"
 	"github.com/docker/buildx/util/dockerutil"
+	"github.com/docker/buildx/util/dockerutil/dockerconfig"
 	"github.com/docker/buildx/util/osutil"
 	"github.com/docker/buildx/util/progress"
 	"github.com/docker/buildx/util/tracing"
 	"github.com/docker/cli/cli/command"
 	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/session/auth/authprovider"
 	"github.com/moby/buildkit/util/progress/progressui"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -252,6 +254,16 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba
 		return err
 	}
 
+	// make sure local credentials aren't loaded multiple times for different targets
+	authProvider := authprovider.NewDockerAuthProvider(authprovider.DockerAuthProviderConfig{
+		AuthConfigProvider: dockerconfig.LoadAuthConfig(dockerCli),
+	})
+
+	for k, opt := range bo {
+		opt.Session = append(opt.Session, authProvider)
+		bo[k] = opt
+	}
+
 	def := struct {
 		Group  map[string]*bake.Group  `json:"group,omitempty"`
 		Target map[string]*bake.Target `json:"target"`
diff --git a/commands/build.go b/commands/build.go
index 726df0885..f213ae275 100644
--- a/commands/build.go
+++ b/commands/build.go
@@ -27,6 +27,7 @@ import (
 	"github.com/docker/buildx/util/confutil"
 	"github.com/docker/buildx/util/desktop"
 	"github.com/docker/buildx/util/dockerutil"
+	"github.com/docker/buildx/util/dockerutil/dockerconfig"
 	"github.com/docker/buildx/util/ioset"
 	"github.com/docker/buildx/util/metricutil"
 	"github.com/docker/buildx/util/osutil"
@@ -1019,7 +1020,7 @@ func RunBuild(ctx context.Context, dockerCli command.Cli, in *BuildOptions, inSt
 	opts.Platforms = platforms
 
 	opts.Session = append(opts.Session, authprovider.NewDockerAuthProvider(authprovider.DockerAuthProviderConfig{
-		ConfigFile: dockerCli.ConfigFile(),
+		AuthConfigProvider: dockerconfig.LoadAuthConfig(dockerCli),
 	}))
 
 	secrets, err := build.CreateSecrets(in.Secrets)
diff --git a/util/dockerutil/dockerconfig/configprovider.go b/util/dockerutil/dockerconfig/configprovider.go
new file mode 100644
index 000000000..609081555
--- /dev/null
+++ b/util/dockerutil/dockerconfig/configprovider.go
@@ -0,0 +1,254 @@
+package dockerconfig
+
+import (
+	"cmp"
+	"context"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/docker/buildx/util/confutil"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/cli/cli/config"
+	"github.com/docker/cli/cli/config/configfile"
+	"github.com/docker/cli/cli/config/types"
+	"github.com/moby/buildkit/session/auth/authprovider"
+)
+
+func LoadAuthConfig(cli command.Cli) authprovider.AuthConfigProvider {
+	acp := &authConfigProvider{
+		buildxConfig:    confutil.NewConfig(cli),
+		defaultConfig:   cli.ConfigFile(),
+		authConfigCache: map[string]authConfigCacheEntry{},
+	}
+	return acp.load
+}
+
+type authConfigProvider struct {
+	initOnce           sync.Once
+	defaultConfig      *configfile.ConfigFile
+	buildxConfig       *confutil.Config
+	authConfigCache    map[string]authConfigCacheEntry
+	mu                 sync.Mutex // mutex for authConfigCache
+	alternativeConfigs []*alternativeConfig
+}
+
+func (ap *authConfigProvider) load(ctx context.Context, host string, scopes []string, cacheExpireCheck authprovider.ExpireCachedAuthCheck) (types.AuthConfig, error) {
+	ap.initOnce.Do(func() {
+		ap.init()
+	})
+
+	ap.mu.Lock()
+	defer ap.mu.Unlock()
+
+	candidates := []*alternativeConfig{}
+	parsedScopes := parseScopes(scopes)
+
+	if len(parsedScopes) == 1 {
+		for _, cfg := range ap.alternativeConfigs {
+			if cfg.host != host {
+				continue
+			}
+			if cfg.matchesScopes(parsedScopes) {
+				candidates = append(candidates, cfg)
+			}
+		}
+	}
+	key := host
+	cfg := ap.defaultConfig
+	if len(candidates) > 0 {
+		// matches with repo before those without repo
+		// matches with scope set sorted before those without scope
+		slices.SortFunc(candidates, func(a, b *alternativeConfig) int {
+			return cmp.Or(
+				strings.Compare(b.repo, a.repo),
+				cmp.Compare(len(b.scope), len(a.scope)),
+			)
+		})
+		candidates = candidates[:1]
+		key += "|" + candidates[0].dir
+		if candidates[0].configFile == nil {
+			if cfgDir, err := config.Load(candidates[0].dir); err == nil {
+				cfg = cfgDir
+				candidates[0].configFile = cfg
+			}
+		} else {
+			cfg = candidates[0].configFile
+		}
+	}
+
+	entry, exists := ap.authConfigCache[key]
+	if exists && (cacheExpireCheck == nil || !cacheExpireCheck(entry.Created, key)) {
+		return *entry.Auth, nil
+	}
+
+	hostKey := host
+	if host == authprovider.DockerHubRegistryHost {
+		hostKey = authprovider.DockerHubConfigfileKey
+	}
+
+	ac, err := cfg.GetAuthConfig(hostKey)
+	if err != nil {
+		return types.AuthConfig{}, err
+	}
+
+	entry = authConfigCacheEntry{
+		Created: time.Now(),
+		Auth:    &ac,
+	}
+
+	ap.authConfigCache[key] = entry
+
+	return ac, nil
+}
+
+func (ap *authConfigProvider) init() error {
+	base := filepath.Join(ap.buildxConfig.Dir(), "config")
+	return filepath.WalkDir(base, func(path string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+		if d.Name() != config.ConfigFileName {
+			return nil
+		}
+		dir := filepath.Dir(path)
+		rdir, err := filepath.Rel(base, dir)
+		if err != nil {
+			return err
+		}
+		cfg := parseConfigKey(rdir)
+		cfg.dir = dir
+		ap.alternativeConfigs = append(ap.alternativeConfigs, &cfg)
+		return nil
+	})
+}
+
+func parseConfigKey(key string) alternativeConfig {
+	var out alternativeConfig
+
+	var mainPart, scopePart string
+	if i := strings.IndexByte(key, '@'); i >= 0 {
+		mainPart = key[:i]
+		scopePart = key[i+1:]
+	} else {
+		mainPart = key
+	}
+
+	if scopePart != "" {
+		out.scope = make(map[string]struct{})
+		for s := range strings.SplitSeq(scopePart, ",") {
+			if s = strings.TrimSpace(s); s != "" {
+				out.scope[s] = struct{}{}
+			}
+		}
+	}
+
+	if mainPart == "" {
+		return out
+	}
+
+	slash := strings.IndexByte(mainPart, '/')
+	if slash < 0 {
+		out.host = mainPart
+		return out
+	}
+
+	out.host = mainPart[:slash]
+	out.repo = mainPart[slash+1:]
+
+	return out
+}
+
+type alternativeConfig struct {
+	dir string
+
+	host  string
+	repo  string
+	scope map[string]struct{}
+
+	configFile *configfile.ConfigFile
+}
+
+func (a *alternativeConfig) matchesScopes(q scopes) bool {
+	if a.repo != "" {
+		if _, ok := q["repository:"+a.repo]; !ok {
+			return false
+		}
+	}
+
+	if len(a.scope) > 0 {
+		if a.repo == "" {
+			// no repo means one query must match all scopes
+			for _, scopeActions := range q {
+				ok := true
+				for s := range a.scope {
+					if _, exists := scopeActions[s]; !exists {
+						ok = false
+						break
+					}
+				}
+				if ok {
+					return true
+				}
+			}
+			return false
+		}
+		for s := range a.scope {
+			for k, scopeActions := range q {
+				if k == "repository:"+a.repo {
+					if _, ok := scopeActions[s]; !ok {
+						return false
+					}
+				}
+			}
+		}
+	}
+
+	return true
+}
+
+type authConfigCacheEntry struct {
+	Created time.Time
+	Auth    *types.AuthConfig
+}
+
+type scopes map[string]map[string]struct{}
+
+func parseScopes(s []string) scopes {
+	// https://distribution.github.io/distribution/spec/auth/scope/
+	m := map[string]map[string]struct{}{}
+	for _, scopeStr := range s {
+		if scopeStr == "" {
+			return nil
+		}
+		// The scopeStr may have strings that contain multiple scopes separated by a space.
+		for scope := range strings.SplitSeq(scopeStr, " ") {
+			parts := strings.SplitN(scope, ":", 3)
+			names := []string{parts[0]}
+			if len(parts) > 1 {
+				names = append(names, parts[1])
+			}
+			var actions []string
+			if len(parts) == 3 {
+				actions = append(actions, strings.Split(parts[2], ",")...)
+			}
+			name := strings.Join(names, ":")
+			ma, ok := m[name]
+			if !ok {
+				ma = map[string]struct{}{}
+				m[name] = ma
+			}
+
+			for _, a := range actions {
+				ma[a] = struct{}{}
+			}
+		}
+	}
+	return m
+}
diff --git a/util/dockerutil/dockerconfig/configprovider_test.go b/util/dockerutil/dockerconfig/configprovider_test.go
new file mode 100644
index 000000000..d4fce2e83
--- /dev/null
+++ b/util/dockerutil/dockerconfig/configprovider_test.go
@@ -0,0 +1,200 @@
+package dockerconfig
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseConfigKey(t *testing.T) {
+	tests := map[string]struct {
+		in    string
+		host  string
+		repo  string
+		scope map[string]struct{}
+	}{
+		"domain-only": {
+			in:   "example.com",
+			host: "example.com",
+		},
+		"domain-with-path": {
+			in:   "example.com/a/b",
+			host: "example.com",
+			repo: "a/b",
+		},
+		"with-scopes": {
+			in:   "example.com/x@y,z",
+			host: "example.com",
+			repo: "x",
+			scope: map[string]struct{}{
+				"y": {},
+				"z": {},
+			},
+		},
+		"empty-scope-list": {
+			in:   "example.com/a/b@",
+			host: "example.com",
+			repo: "a/b",
+		},
+		"path-empty-before-at": {
+			in:   "example.com/@foo",
+			host: "example.com",
+			repo: "",
+			scope: map[string]struct{}{
+				"foo": {},
+			},
+		},
+		"scope-no-slash": {
+			in:   "registry-1.docker.io@pull,push",
+			host: "registry-1.docker.io",
+			repo: "",
+			scope: map[string]struct{}{
+				"pull": {},
+				"push": {},
+			},
+		},
+		"with-port-and-scopes": {
+			in:   "example.com:5000/x@y,z",
+			host: "example.com:5000",
+			repo: "x",
+			scope: map[string]struct{}{
+				"y": {},
+				"z": {},
+			},
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			out := parseConfigKey(tc.in)
+
+			require.Equal(t, tc.host, out.host)
+			require.Equal(t, tc.repo, out.repo)
+
+			if tc.scope == nil {
+				require.Nil(t, out.scope)
+			} else {
+				require.Equal(t, tc.scope, out.scope)
+			}
+		})
+	}
+}
+func TestMatchesScopes(t *testing.T) {
+	tests := map[string]struct {
+		cfgKey  string
+		query   []string
+		matches bool
+	}{
+		"invalid-scope": {
+			cfgKey:  "example.com/foo/bar@a,b",
+			query:   []string{"repository:foo/bar:pull"},
+			matches: false,
+		},
+		"sub-scope-match": {
+			cfgKey:  "example.com/foo/bar@push",
+			query:   []string{"repository:foo/bar:pull,push"},
+			matches: true,
+		},
+		"invalid-scope-single": {
+			cfgKey:  "example.com/foo/bar@push",
+			query:   []string{"repository:foo/bar:pull"},
+			matches: false,
+		},
+		"wrong-repo": {
+			cfgKey:  "example.com/foo/bar@push",
+			query:   []string{"repository:foo/bar2:pull,push"},
+			matches: false,
+		},
+		"multi-scope-match": {
+			cfgKey:  "example.com/foo/bar@push,pull",
+			query:   []string{"repository:foo/bar:pull,push"},
+			matches: true,
+		},
+		"empty-action-list-in-query": {
+			cfgKey:  "example.com/foo/bar@pull",
+			query:   []string{"repository:foo/bar:"}, // no actions
+			matches: false,
+		},
+
+		"repo-no-scopes-required": {
+			cfgKey:  "example.com/foo/bar",
+			query:   []string{"repository:foo/bar:pull"},
+			matches: true,
+		},
+
+		"repo-empty-scope-in-config": {
+			cfgKey:  "example.com/foo/bar@", // explicit empty scope list
+			query:   []string{"repository:foo/bar:pull"},
+			matches: true,
+		},
+
+		"multiple-repos-config-targets-only-one": {
+			cfgKey: "example.com/foo/bar@pull",
+			query: []string{
+				"repository:foo/bar:pull",
+				"repository:other:push",
+			},
+			matches: true,
+		},
+
+		"multiple-repos-but-target-lacks-scope": {
+			cfgKey: "example.com/foo/bar@push",
+			query: []string{
+				"repository:foo/bar:pull", // missing push
+				"repository:other:push",   // irrelevant
+			},
+			matches: false,
+		},
+
+		"no-repo-in-config-match-any": {
+			cfgKey: "example.com@pull",
+			query: []string{
+				"repository:a:pull",
+				"repository:b:pull,push",
+			},
+			matches: true,
+		},
+
+		"no-repo-in-config-but-no-repo-has-scope": {
+			cfgKey: "example.com@push",
+			query: []string{
+				"repository:a:pull",
+				"repository:b:pull",
+			},
+			matches: false,
+		},
+
+		"single-action-in-query-with-spaces": {
+			cfgKey:  "example.com/foo/bar@pull",
+			query:   []string{"repository:foo/bar:pull repository:foo/bar2:pull"},
+			matches: true,
+		},
+
+		"single-action-in-query-with-spaces-one-match": {
+			cfgKey:  "example.com@push",
+			query:   []string{"repository:foo/bar:push repository:foo/bar2:pull"},
+			matches: true,
+		},
+
+		"single-action-in-query-with-spaces-invalid": {
+			cfgKey:  "example.com@push",
+			query:   []string{"repository:foo/bar:pull repository:foo/bar2:pull"},
+			matches: false,
+		},
+
+		"multi-scope-config-order-does-not-matter": {
+			cfgKey:  "example.com/foo/bar@push,pull",
+			query:   []string{"repository:foo/bar:pull,push"},
+			matches: true,
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			cfg := parseConfigKey(tc.cfgKey)
+			q := parseScopes(tc.query)
+			got := cfg.matchesScopes(q)
+			require.Equal(t, tc.matches, got)
+		})
+	}
+}