From ab183bbfb54e84dc0c70f36409f1d62de794aec5 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 1 Jun 2026 21:49:36 +0000 Subject: [PATCH 1/4] Add Go module collections Signed-off-by: Solomon Hykes --- .dagger/modules/e2e/main.dang | 33 +++++++++++-------- go.dang | 62 +++++++++++++++++++++++++++++++---- 2 files changed, 75 insertions(+), 20 deletions(-) diff --git a/.dagger/modules/e2e/main.dang b/.dagger/modules/e2e/main.dang index 6d1c9b9..a6a5757 100644 --- a/.dagger/modules/e2e/main.dang +++ b/.dagger/modules/e2e/main.dang @@ -46,10 +46,8 @@ type E2e { .length > 0 } - let containsModulePath(mods: [{{path: String!}}!]!, want: String!): Boolean! { - mods - .filter { mod => mod.path == want } - .length > 0 + let containsModulePath(paths: [String!]!, want: String!): Boolean! { + containsPath(paths, want) } """ @@ -112,7 +110,7 @@ type E2e { let testDirs = mod.testDirectories(ws) assert( - containsModulePath(testDirs.{path}, baseFixturePath), + containsModulePath(testDirs.{path}.map { dir => dir.path }, baseFixturePath), "custom base test directory discovery did not include expected path", ) @@ -157,7 +155,8 @@ type E2e { ) let tool = go(version: "1.26.1", skipLint: skipLint, skipTest: skipTest, skipGenerate: skipGenerate) - let allModulePaths = tool.modules(ws).{path} + let allModules = tool.modules(ws) + let allModulePaths = allModules.keys assert( containsModulePath(allModulePaths, "fixtures/go-module-cross-include-a"), "unfiltered modules did not include expected fixture module", @@ -170,10 +169,18 @@ type E2e { containsModulePath(allModulePaths, "testdata/go-module-excluded"), "unfiltered modules did not include skipped fixture module by default", ) + assert( + allModules.get(key: "testdata/go-module-with-testdata").path == "testdata/go-module-with-testdata", + "module collection get did not return the requested module", + ) + assert( + allModules.subset(keys: ["fixtures/go-module-cross-include-a"]).keys.length == 1, + "module collection subset did not narrow to the requested keys", + ) let includedModulePaths = tool .modules(ws, include: ["fixtures/go-module-cross-include-a"]) - .{path} + .keys assert( containsModulePath(includedModulePaths, "fixtures/go-module-cross-include-a"), "modules include filter did not include matching module root", @@ -185,7 +192,7 @@ type E2e { let excludedModulePaths = tool .modules(ws, exclude: ["fixtures/go-module-cross-include-a"]) - .{path} + .keys assert( containsModulePath(excludedModulePaths, "fixtures/go-module-cross-include-a") == false, "modules exclude filter included excluded module root", @@ -197,7 +204,7 @@ type E2e { include: ["fixtures/go-module-cross-include-*"], exclude: ["fixtures/go-module-cross-include-b"], ) - .{path} + .keys assert( containsModulePath(combinedModulePaths, "fixtures/go-module-cross-include-a"), "modules combined filters did not include matching module root", @@ -209,7 +216,7 @@ type E2e { let contentMatchedModulePaths = tool .modules(ws, include: ["**/module-a-only.data"]) - .{path} + .keys assert( containsModulePath(contentMatchedModulePaths, "fixtures/go-module-cross-include-a"), "modules include filter did not match module directory contents", @@ -221,7 +228,7 @@ type E2e { let lintSkippedModulePaths = tool .modules(ws, includeSkipLint: false) - .{path} + .keys assert( containsModulePath(lintSkippedModulePaths, "testdata/go-module-excluded") == false, "modules includeSkipLint false included lint-skipped module", @@ -241,7 +248,7 @@ type E2e { let testSkippedModulePaths = tool .modules(ws, includeSkipTest: false) - .{path} + .keys assert( containsModulePath(testSkippedModulePaths, "testdata/go-module-excluded") == false, "modules includeSkipTest false included test-skipped module", @@ -261,7 +268,7 @@ type E2e { let generateSkippedModulePaths = tool .modules(ws, includeSkipGenerate: false) - .{path} + .keys assert( containsModulePath(generateSkippedModulePaths, "testdata/go-module-excluded") == false, "modules includeSkipGenerate false included generate-skipped module", diff --git a/go.dang b/go.dang index 815af65..6416839 100644 --- a/go.dang +++ b/go.dang @@ -80,7 +80,8 @@ type Go { pub skipGenerate: [String!]! = [] """ - Return every Go module discovered from workspace go.mod files. + Return Go modules discovered from workspace go.mod files, as a collection + keyed by module root path. Optional include/exclude patterns filter discovered module root directories. """ pub modules( @@ -90,8 +91,8 @@ type Go { includeSkipLint: Boolean! = true, includeSkipTest: Boolean! = true, includeSkipGenerate: Boolean! = true, - ): [GoModule!]! { - ws + ): GoModules! { + let paths = ws .directory("/", include: ["**/go.mod"]) .glob("**/go.mod") .map { modPath => @@ -116,6 +117,17 @@ type Go { (includeSkipTest or mod.skipTest(ws) == false) and (includeSkipGenerate or mod.skipGenerate(ws) == false) } + .map { mod => mod.path } + + GoModules( + paths: paths, + version: version, + baseImage: base, + includeExtraFiles: includeExtraFiles, + skipLintPaths: skipLint, + skipTestPaths: skipTest, + skipGeneratePaths: skipGenerate, + ) } let modulePathIncluded( @@ -171,7 +183,9 @@ type Go { Modules configured to skip lint are skipped. """ pub lintAll(ws: Workspace!): Void @check { - let layers = modules(ws, includeSkipLint: false).reduce(directory) { layers, mod => + let mods = modules(ws, includeSkipLint: false) + let layers = mods.paths.reduce(directory) { layers, path => + let mod = mods.module(path) layers.withDirectory( mod.path, directory.withFile("go.mod", mod.lintExec(ws).file("go.mod")), @@ -190,8 +204,9 @@ type Go { Modules configured to skip tests are skipped. """ pub testAll(ws: Workspace!): Void @check { - let mods = modules(ws, includeSkipTest: false) - let layers = mods.reduce(directory) { layers, mod => + let collection = modules(ws, includeSkipTest: false) + let layers = collection.paths.reduce(directory) { layers, path => + let mod = collection.module(path) layers.withDirectory( mod.path, directory.withFile("go.mod", mod.testExec(ws).file("go.mod")), @@ -211,7 +226,8 @@ type Go { are skipped. """ pub generateAll(ws: Workspace!): Changeset! @generate { - let mods = modules(ws).filter { mod => + let collection = modules(ws) + let mods = collection.paths.map { path => collection.module(path) }.filter { mod => mod.skipGenerate(ws) == false and mod.hasGenerateDirectives(ws) } @@ -231,6 +247,38 @@ type Go { } +""" +Go modules discovered in a workspace, keyed by module root path. +""" +type GoModules { + """ + Workspace-relative module root paths in discovery order. + """ + pub paths: [String!]! @keys + + let version: String + let baseImage: Container! + let includeExtraFiles: [String!]! + let skipLintPaths: [String!]! + let skipTestPaths: [String!]! + let skipGeneratePaths: [String!]! + + """ + Return the discovered Go module with the given workspace-relative root path. + """ + pub module(path: String!): GoModule! @get { + GoModule( + path: path, + version: version, + baseImage: baseImage, + includeExtraFiles: includeExtraFiles, + skipLintPaths: skipLintPaths, + skipTestPaths: skipTestPaths, + skipGeneratePaths: skipGeneratePaths, + ) + } +} + """ A Go module rooted at a workspace-relative path. """ From eaf9fc01efbf5b0043da4c2f1da2363eea468b81 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 10 Jun 2026 10:30:46 -0700 Subject: [PATCH 2/4] Add test directory and test collections Layer two more collections under GoModules, giving Go workspaces three selection dimensions: go-module, go-directory, and go-test. GoModule.testDirs exposes the existing test directory discovery as a collection keyed by workspace-relative path. Each GoDirectory exposes its tests as a collection keyed by test name, enumerated with go test -list; skipped directories produce an empty collection. Test execution is batched: the collection-level run executes one go test -run '^(A|B)$' over the current subset, and shadows the per-test run so that selecting several tests by name (dagger check --go-test=A --go-test=B) compiles and runs them in a single invocation. Filters push down, so only selected directories enumerate their tests. Note: test directory discovery (pre-existing go-includes behavior) does not surface test files at a module root; only subdirectories containing _test.go files become test directories. Signed-off-by: Solomon Hykes --- go.dang | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 142 insertions(+), 10 deletions(-) diff --git a/go.dang b/go.dang index 6416839..f55d247 100644 --- a/go.dang +++ b/go.dang @@ -435,6 +435,19 @@ type GoModule { Directories in this module containing Go test files. """ pub testDirectories(ws: Workspace!): [GoDirectory!]! { + testDirectoryPaths(ws).map { testPath => + GoDirectory( + path: testPath, + ws: ws, + modulePath: path, + baseImage: baseImage, + includeExtraFiles: includeExtraFiles, + skipTestPaths: skipTestPaths, + ) + } + } + + let testDirectoryPaths(ws: Workspace!): [String!]! { goIncludesHelper(ws) .withExec( ["go-includes", "--output", "/output", "--test-dirs", workspacePath], @@ -444,16 +457,20 @@ type GoModule { .contents .split("\n") .filter { testPath => testPath != "" } - .map { testPath => - GoDirectory( - path: testPath, - ws: ws, - modulePath: path, - baseImage: baseImage, - includeExtraFiles: includeExtraFiles, - skipTestPaths: skipTestPaths, - ) - } + } + + """ + The test directories of this module, as a collection keyed by path. + """ + pub testDirs(ws: Workspace!): GoTestDirs! { + GoTestDirs( + paths: testDirectoryPaths(ws), + ws: ws, + modulePath: path, + baseImage: baseImage, + includeExtraFiles: includeExtraFiles, + skipTestPaths: skipTestPaths, + ) } """ @@ -649,6 +666,36 @@ type GoModule { """ A workspace directory containing Go test files. """ +""" +Test directories of a Go module, keyed by workspace-relative path. +""" +type GoTestDirs { + """ + Workspace-relative test directory paths in discovery order. + """ + pub paths: [String!]! @keys + + let ws: Workspace! + let modulePath: String! + let baseImage: Container! + let includeExtraFiles: [String!]! + let skipTestPaths: [String!]! + + """ + Return the test directory with the given workspace-relative path. + """ + pub directory(path: String!): GoDirectory! @get { + GoDirectory( + path: path, + ws: ws, + modulePath: modulePath, + baseImage: baseImage, + includeExtraFiles: includeExtraFiles, + skipTestPaths: skipTestPaths, + ) + } +} + type GoDirectory { """ Workspace-relative path of this directory. @@ -805,4 +852,89 @@ type GoDirectory { null } + """ + Go test container for this directory, before a test command is added. + """ + pub testContainer(): Container! { + base + .withDirectory(".", source) + .withDirectory(".", testData) + .withWorkdir(path) + } + + """ + The tests in this directory, as a collection keyed by test name. + Skipped directories produce an empty collection. + """ + pub tests(): GoTests! { + let names = if (skipTest) { + [] + } else { + testContainer() + .withExec(["sh", "-c", "go test -list '.*' . | grep '^Test' || true"]) + .stdout + .split("\n") + .filter { name => name != "" } + } + GoTests(names: names, dir: self) + } + +} + +""" +Tests in a Go directory, keyed by test name. +""" +type GoTests { + """ + Test names in listing order. + """ + pub names: [String!]! @keys + + let dir: GoDirectory! + + """ + Return the test with the given name. + """ + pub test(name: String!): GoTest! @get { + GoTest(name: name, dir: dir) + } + + """ + Run every test in the current subset in a single go test invocation. + """ + pub run(): Void @check { + if (names.length == 0) { + null + } else { + dir + .testContainer() + .withExec(["go", "test", "-run", "^(" + names.join("|") + ")$", "."]) + .sync + null + } + } +} + +""" +A single Go test. +""" +type GoTest { + """ + The test's name. + """ + pub name: String! + + let dir: GoDirectory! + + """ + Run this test. Batched through the collection when several tests are + selected together. + """ + pub run(): Void @check { + dir + .testContainer() + .withExec(["go", "test", "-run", "^" + name + "$", "."]) + .sync + null + } } From 6ea03faa593c2fb80c9333d4934547831e70ef79 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 11 Jun 2026 16:25:03 -0700 Subject: [PATCH 3/4] Discover Go tests by static regex instead of go test -list GoDirectory.tests used to spin up a Go container and parse 'go test -list' stdout to enumerate test names. Replace that with a single Workspace.directory(...).search(pattern: "^func Test\\w+\\(") over the directory's *_test.go files and parse names from the matched lines in Dang. This is much faster (no compile, no container exec) and matches the same set go test -list does for top-level test functions. Test methods on testify-style suites are intentionally not listed: they cannot be selected with go test -run independently of their parent function. Add moduleIntrospectionCheck assertions covering the new GoTestDirs and GoTests collection surface end-to-end: keys/get round-trip on both levels, multi-test discovery from a single *_test.go file (cross- include-b has two top-level tests), and that a skip-configured directory produces an empty tests collection. Signed-off-by: Solomon Hykes --- .dagger/modules/e2e/main.dang | 50 +++++++++++++++++++++++++++++++++++ go.dang | 19 ++++++++++--- 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/.dagger/modules/e2e/main.dang b/.dagger/modules/e2e/main.dang index a6a5757..1ea2c50 100644 --- a/.dagger/modules/e2e/main.dang +++ b/.dagger/modules/e2e/main.dang @@ -433,6 +433,56 @@ type E2e { ), "go:generate scoped include did not include generator support files", ) + + # Test directory collection surface and static test discovery. + # cross-include-b has two top-level Test functions in a single file — + # exercises ripgrep-based listing and multi-match handling. + let crossB = tool.module(ws, "fixtures/go-module-cross-include-b") + let crossBTestDirs = crossB.testDirs(ws) + assert( + containsPath(crossBTestDirs.keys, "fixtures/go-module-cross-include-b"), + "testDirs collection did not include expected test directory", + ) + assert( + crossBTestDirs + .get(key: "fixtures/go-module-cross-include-b") + .path == "fixtures/go-module-cross-include-b", + "testDirs collection get did not return the requested directory", + ) + + let crossBTests = crossBTestDirs + .get(key: "fixtures/go-module-cross-include-b") + .tests() + assert( + crossBTests.keys.length == 2, + "tests collection did not discover expected number of tests", + ) + assert( + containsPath(crossBTests.keys, "TestSiblingModuleDirectiveIncludeIsNotMounted"), + "tests collection did not include expected test name", + ) + assert( + containsPath(crossBTests.keys, "TestSiblingModuleTestdataIsNotMounted"), + "tests collection did not include second test from same file", + ) + assert( + crossBTests + .get(key: "TestSiblingModuleDirectiveIncludeIsNotMounted") + .name == "TestSiblingModuleDirectiveIncludeIsNotMounted", + "tests collection get did not return the requested test", + ) + + # Skip-configured directory should report no tests at all. + let skippedTests = tool + .module(ws, "testdata/go-module-excluded") + .testDirs(ws) + .get(key: "testdata/go-module-excluded") + .tests() + assert( + skippedTests.keys.length == 0, + "tests collection on a skipped directory was not empty", + ) + null } diff --git a/go.dang b/go.dang index f55d247..9aa5019 100644 --- a/go.dang +++ b/go.dang @@ -865,15 +865,26 @@ type GoDirectory { """ The tests in this directory, as a collection keyed by test name. Skipped directories produce an empty collection. + + Test names are discovered by statically matching top-level test function + declarations in *_test.go files, without compiling or running go test. """ pub tests(): GoTests! { let names = if (skipTest) { [] } else { - testContainer() - .withExec(["sh", "-c", "go test -list '.*' . | grep '^Test' || true"]) - .stdout - .split("\n") + let glob = if (path == ".") { + "*_test.go" + } else { + path.trimSuffix("/") + "/*_test.go" + } + ws + .directory("/", include: [glob]) + .search(pattern: "^func Test\\w+\\(") + .{matchedLines} + .map { result => + result.matchedLines.trimSpace.trimPrefix("func ").split("(")[0] ?? "" + } .filter { name => name != "" } } GoTests(names: names, dir: self) From 6ce779b40babcdb2de5c7a014978636006c58df2 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 11 Jun 2026 22:18:52 -0700 Subject: [PATCH 4/4] Materialize test files at the Workspace boundary, share downstream Every function with a Workspace receiver or argument is invalidated unconditionally. GoDirectory.tests() was crossing Workspace once per test directory, which on dagger/dagger meant ~hundreds of invalidated calls and ~12 minutes to enumerate go-test. Thread a single Directory snapshot of all **/*_test.go files from the Workspace boundary (Go.modules / Go.module) down through GoModules -> GoModule -> GoTestDirs -> GoDirectory. GoDirectory.tests() now searches that pre-materialized Directory and filters results by filePath. The search arguments are identical for every directory, so the search hits the dagql cache (Directory receiver -> content-addressed) after the first call; per-directory restriction is pure-Dang on the cached result. Net: one Workspace crossing and one ripgrep per session, no matter how many directories the workspace contains. Signed-off-by: Solomon Hykes --- go.dang | 53 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/go.dang b/go.dang index 9aa5019..ae991f8 100644 --- a/go.dang +++ b/go.dang @@ -92,6 +92,12 @@ type Go { includeSkipTest: Boolean! = true, includeSkipGenerate: Boolean! = true, ): GoModules! { + # Single workspace-wide snapshot of every *_test.go file, materialized + # once at this Workspace boundary and threaded down through the + # collection so GoDirectory.tests() can search/filter without ever + # touching Workspace again. Functions taking Workspace always invalidate; + # operations on a Directory receiver cache by content hash. + let testFilesDir = ws.directory("/", include: ["**/*_test.go"]) let paths = ws .directory("/", include: ["**/go.mod"]) .glob("**/go.mod") @@ -110,6 +116,7 @@ type Go { skipLintPaths: skipLint, skipTestPaths: skipTest, skipGeneratePaths: skipGenerate, + testFilesDir: testFilesDir, ) } .filter { mod => @@ -127,6 +134,7 @@ type Go { skipLintPaths: skipLint, skipTestPaths: skipTest, skipGeneratePaths: skipGenerate, + testFilesDir: testFilesDir, ) } @@ -175,6 +183,7 @@ type Go { skipLintPaths: skipLint, skipTestPaths: skipTest, skipGeneratePaths: skipGenerate, + testFilesDir: ws.directory("/", include: ["**/*_test.go"]), ) } @@ -262,6 +271,7 @@ type GoModules { let skipLintPaths: [String!]! let skipTestPaths: [String!]! let skipGeneratePaths: [String!]! + let testFilesDir: Directory! """ Return the discovered Go module with the given workspace-relative root path. @@ -275,6 +285,7 @@ type GoModules { skipLintPaths: skipLintPaths, skipTestPaths: skipTestPaths, skipGeneratePaths: skipGeneratePaths, + testFilesDir: testFilesDir, ) } } @@ -318,6 +329,13 @@ type GoModule { """ let skipGeneratePaths: [String!]! + """ + Workspace-wide snapshot of every *_test.go file, threaded down from the + Go.modules / Go.module constructor. Used by GoDirectory.tests so that + test discovery never crosses Workspace again. + """ + let testFilesDir: Directory! + """ Return a workspace-root path for a module-relative subpath. """ @@ -443,6 +461,7 @@ type GoModule { baseImage: baseImage, includeExtraFiles: includeExtraFiles, skipTestPaths: skipTestPaths, + testFilesDir: testFilesDir, ) } } @@ -470,6 +489,7 @@ type GoModule { baseImage: baseImage, includeExtraFiles: includeExtraFiles, skipTestPaths: skipTestPaths, + testFilesDir: testFilesDir, ) } @@ -680,6 +700,7 @@ type GoTestDirs { let baseImage: Container! let includeExtraFiles: [String!]! let skipTestPaths: [String!]! + let testFilesDir: Directory! """ Return the test directory with the given workspace-relative path. @@ -692,6 +713,7 @@ type GoTestDirs { baseImage: baseImage, includeExtraFiles: includeExtraFiles, skipTestPaths: skipTestPaths, + testFilesDir: testFilesDir, ) } } @@ -727,6 +749,13 @@ type GoDirectory { """ let skipTestPaths: [String!]! + """ + Workspace-wide snapshot of every *_test.go file, threaded down from the + parent collection. tests() searches/filters this directory so the cost + is paid once per session (Directory receiver = content-addressed cache). + """ + let testFilesDir: Directory! + """ Return a workspace-root path for a module-relative subpath. """ @@ -868,20 +897,28 @@ type GoDirectory { Test names are discovered by statically matching top-level test function declarations in *_test.go files, without compiling or running go test. + + Search runs against testFilesDir, a Directory threaded from the parent + collection. The search arguments are identical for every GoDirectory, + so the result is cached once per session and reused; per-directory + restriction is a pure-Dang filter on the returned filePath. """ pub tests(): GoTests! { let names = if (skipTest) { [] } else { - let glob = if (path == ".") { - "*_test.go" - } else { - path.trimSuffix("/") + "/*_test.go" - } - ws - .directory("/", include: [glob]) + let prefix = if (path == ".") { "" } else { path.trimSuffix("/") + "/" } + testFilesDir .search(pattern: "^func Test\\w+\\(") - .{matchedLines} + .{filePath, matchedLines} + .filter { result => + if (path == ".") { + result.filePath.contains("/") == false + } else { + result.filePath.hasPrefix(prefix) and + result.filePath.trimPrefix(prefix).contains("/") == false + } + } .map { result => result.matchedLines.trimSpace.trimPrefix("func ").split("(")[0] ?? "" }