diff --git a/dap/thread.go b/dap/thread.go
index 8eb26f653..37acbfb3b 100644
--- a/dap/thread.go
+++ b/dap/thread.go
@@ -173,6 +173,12 @@ func (t *thread) createBranch(dgst digest.Digest, exitpoint *step) (entrypoint *
 		parent: -1,
 	}
 
+	// The entrypoint doesn't have a source entry. Just skip this
+	// branch.
+	if entrypoint.frame.Source == nil {
+		return nil
+	}
+
 	// Create a pseudo-frame and attach it to the return point.
 	// This is mostly used for getting the correct inputs utilized
 	// by this frame.
@@ -216,11 +222,18 @@ func (t *thread) createBranch(dgst digest.Digest, exitpoint *step) (entrypoint *
 			inp := op.Inputs[i]
 
 			head := *entrypoint
-			entrypoint.dgst = ""
 
 			// Create the routine associated with this input.
 			// Associate it with the entrypoint in step.
 			head.in = t.createBranch(digest.Digest(inp.Digest), entrypoint)
+
+			// If this branch is empty (signified by a nil return value) then
+			// skip it.
+			if head.in == nil {
+				continue
+			}
+
+			entrypoint.dgst = ""
 			entrypoint = &head
 		}
 
diff --git a/tests/dap_build.go b/tests/dap_build.go
index ce7f169b6..d3a8afea5 100644
--- a/tests/dap_build.go
+++ b/tests/dap_build.go
@@ -299,54 +299,6 @@ func testDapBuildStepIn(t *testing.T, sb integration.Sandbox) {
 				Name:       `^\[stage-1 .*\] COPY .* /etc/bar`,
 			},
 		},
-		// the following three steps are unintended and are the result
-		// of a bug in the debug adapter.
-		// see issue https://github.com/docker/buildx/issues/3565
-		{
-			{
-				Name: `^\[internal\] load build context`,
-			},
-			{
-				SourceName: "Dockerfile",
-				Line:       3,
-				Name:       `^\[base .*\] COPY foo`,
-			},
-			{
-				SourceName: "Dockerfile",
-				Line:       7,
-				Name:       `^\[stage-1 .*\] COPY .* /etc/bar`,
-			},
-		},
-		// todo: this shouldn't be a stop point.
-		{
-			{
-				Name: `^\[internal\] load build context`,
-			},
-			{
-				SourceName: "Dockerfile",
-				Line:       3,
-				Name:       `^\[base .*\] COPY foo`,
-			},
-			{
-				SourceName: "Dockerfile",
-				Line:       7,
-				Name:       `^\[stage-1 .*\] COPY .* /etc/bar`,
-			},
-		},
-		// duplicate of stop point 3 because of unintended branch
-		// associated with the build context copy.
-		{
-			{
-				SourceName: "Dockerfile",
-				Line:       3,
-				Name:       `^\[base .*\] COPY foo`,
-			},
-			{
-				SourceName: "Dockerfile",
-				Line:       7,
-				Name:       `^\[stage-1 .*\] COPY .* /etc/bar`,
-			},
-		},
 		// stop point 4
 		{
 			{