Skip to content

Commit 0ce3785

Browse files
fredbiclaude
andcommitted
fix(packages): read back a dependency's annotated source
A cheaper load that quietly drops a `swagger:strfmt` format is the failure this work has to be able to see, and cost measurements cannot see it. So the options that change how dependency types arrive — ToolchainFreeLoader, CompiledDependencies, and ExportData when a blob is supplied — are now compared whole-document against the plain scan over the fixture corpus, with every legitimate difference asserted AS a difference carrying its reason, so closing one fails the test instead of passing quietly. What that scoreboard showed is that the two routes drew the line in different places. The toolchain-free strategy resolves imports itself, so it decides per dependency while the load runs: source when its files carry a `swagger:` marker, export data otherwise. go/packages owns resolution and the only lever is a LoadMode — one value for the whole load, with no hook to say "except this one" — so it took every dependency from `go list -export` wholesale and lost strfmt's marks. It now repairs after the fact instead. The cheap load still reports where the source IS, so the graph is walked once, each unread dependency's files are scanned for the marker, and the few that carry it are parsed and joined to the export-data objects by name. Nothing is type-checked twice. The marker scan reads in fixed chunks rather than holding a file, so the largest source file in a graph costs what the smallest one does. The dependencies that are genuinely left with types and no source are recorded with the reason they were skipped, and the reason is replayed as a diagnostic at the point some builder asks for a declaration that is not there — not at load time, where it would be noise. The compiled-dependencies hint now asks the strategy that actually ran, so a scan that fell back to source no longer announces a trade it did not make. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Signed-off-by: Frederic BIDON <fredbi@yahoo.com>
1 parent 60730c0 commit 0ce3785

15 files changed

Lines changed: 1249 additions & 89 deletions
Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
// SPDX-FileCopyrightText: Copyright 2015-2025 go-swagger maintainers
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//go:build !wasm
5+
6+
package integration_test
7+
8+
import (
9+
"os"
10+
"testing"
11+
12+
"github.com/go-openapi/codescan"
13+
"github.com/go-openapi/codescan/internal/scantest"
14+
"github.com/go-openapi/testify/v2/assert"
15+
"github.com/go-openapi/testify/v2/require"
16+
)
17+
18+
// Only one strategy can take dependency types from the compiler, and the caller does not always get
19+
// the strategy they asked for — a virtual filesystem forces the toolchain-free one whatever else was
20+
// requested. So what the scan announces has to come from the strategy that ran, not from the field
21+
// that was set.
22+
//
23+
// This used to announce the request. A scan combining CompiledDependencies with the toolchain-free
24+
// loader was told its dependency types came from export data while it read every dependency from
25+
// source: a diagnostic contradicting the load it described, on the one channel this scanner asks
26+
// callers to trust.
27+
func TestCompiledDependencies_AnnouncedFromWhatRan(t *testing.T) {
28+
t.Parallel()
29+
30+
collect := func(t *testing.T, apply func(*codescan.Options)) []codescan.Diagnostic {
31+
t.Helper()
32+
33+
var seen []codescan.Diagnostic
34+
opts := &codescan.Options{
35+
Packages: []string{"./petstore/..."},
36+
WorkDir: scantest.FixturesDir() + "/goparsing",
37+
ScanModels: true,
38+
OnDiagnostic: func(d codescan.Diagnostic) {
39+
if d.Code == "scan.compiled-dependencies" {
40+
seen = append(seen, d)
41+
}
42+
},
43+
}
44+
apply(opts)
45+
_, err := codescan.Run(opts)
46+
require.NoError(t, err)
47+
48+
return seen
49+
}
50+
51+
t.Run("honoured: a hint describing what the load did", func(t *testing.T) {
52+
t.Parallel()
53+
54+
seen := collect(t, func(o *codescan.Options) { o.CompiledDependencies = true })
55+
56+
require.NotEmpty(t, seen)
57+
assert.Equal(t, codescan.SeverityHint, seen[0].Severity)
58+
assert.Contains(t, seen[0].Message, "come from compiled export data")
59+
})
60+
61+
t.Run("ignored: a warning saying so", func(t *testing.T) {
62+
t.Parallel()
63+
64+
seen := collect(t, func(o *codescan.Options) {
65+
o.ToolchainFreeLoader = true
66+
o.CompiledDependencies = true
67+
})
68+
69+
require.Len(t, seen, 1, "the option had no effect, and that is the whole of what there is to say")
70+
assert.Equal(t, codescan.SeverityWarning, seen[0].Severity,
71+
"a request that did nothing is more than informational")
72+
assert.Contains(t, seen[0].Message, "is ignored under")
73+
assert.NotContains(t, seen[0].Message, "come from compiled export data",
74+
"the load read every dependency from source; saying otherwise is what this test exists for")
75+
})
76+
77+
// The override that needs no loader mentioned at all.
78+
//
79+
// The tree here is deliberately not a working one and must not be read as a model for using FS: an
80+
// io/fs is the WHOLE world a scan may read, and a module rooted on its own reaches neither GOROOT
81+
// nor the module cache — absolute paths are resolved inside the FS, so every dependency including
82+
// the standard library synthesizes. That is beside the point being pinned, which is that FS decides
83+
// the strategy and the announcement follows the strategy.
84+
t.Run("ignored because a virtual filesystem forced the loader", func(t *testing.T) {
85+
t.Parallel()
86+
87+
seen := collect(t, func(o *codescan.Options) {
88+
o.FS = os.DirFS(scantest.FixturesDir())
89+
o.WorkDir = "."
90+
o.Packages = []string{"./goparsing/petstore/..."}
91+
o.CompiledDependencies = true
92+
})
93+
94+
require.Len(t, seen, 1, "FS overrides the strategy without the caller mentioning a loader at all")
95+
assert.Equal(t, codescan.SeverityWarning, seen[0].Severity)
96+
})
97+
98+
t.Run("not asked for, not mentioned", func(t *testing.T) {
99+
t.Parallel()
100+
101+
assert.Empty(t, collect(t, func(*codescan.Options) {}))
102+
})
103+
}

‎internal/integration/compiled_deps_test.go‎

Lines changed: 61 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
package integration_test
55

66
import (
7+
"strings"
78
"testing"
89

910
"github.com/go-openapi/codescan"
@@ -12,24 +13,27 @@ import (
1213
"github.com/go-openapi/testify/v2/require"
1314
)
1415

15-
// CompiledDependencies buys a large amount of time and costs dependency source. This pins what that
16-
// costs in the case that matters most here, so nobody rediscovers it in a published spec.
16+
// CompiledDependencies buys a large amount of time by taking dependency types from the compiler.
17+
//
18+
// It used to cost dependency source along with it, and this test pinned the loss. It no longer does:
19+
// a dependency whose source carries annotations is parsed for them after the load, so the case that
20+
// matters most here is the case that must NOT change.
1721
//
1822
// strfmt marks its own types — `// swagger:strfmt date-time` sits above strfmt.DateTime — and those
19-
// marks are the only reason a field of that type acquires a format. Export data carries the type and
20-
// not the comment, so the format goes and the field stays.
21-
func TestCompiledDependencies_LosesDependencyAnnotations(t *testing.T) {
23+
// marks are the only reason a field of that type acquires a format. Losing them was invisible in the
24+
// output: a valid spec that quietly said less.
25+
func TestCompiledDependencies_KeepsDependencyAnnotations(t *testing.T) {
2226
t.Parallel()
2327

24-
scan := func(compiled bool) (format string, hints int) {
28+
scan := func(compiled bool) (format string, sourceless []string) {
2529
doc, err := codescan.Run(&codescan.Options{
2630
Packages: []string{"./petstore/..."},
2731
WorkDir: scantest.FixturesDir() + "/goparsing",
2832
ScanModels: true,
2933
CompiledDependencies: compiled,
3034
OnDiagnostic: func(d codescan.Diagnostic) {
31-
if d.Code == "scan.compiled-dependencies" {
32-
hints++
35+
if strings.Contains(d.Message, "could not be read") {
36+
sourceless = append(sourceless, d.Message)
3337
}
3438
},
3539
})
@@ -40,14 +44,57 @@ func TestCompiledDependencies_LosesDependencyAnnotations(t *testing.T) {
4044
orderedAt, ok := order.Properties["orderedAt"]
4145
require.True(t, ok)
4246

43-
return orderedAt.Format, hints
47+
return orderedAt.Format, sourceless
4448
}
4549

46-
format, hints := scan(false)
50+
format, sourceless := scan(false)
4751
assert.Equal(t, "date-time", format, "reading strfmt's source is what supplies the format")
48-
assert.Zero(t, hints)
52+
assert.Empty(t, sourceless, "every declaration a builder wanted was there to read")
53+
54+
format, sourceless = scan(true)
55+
assert.Equal(t, "date-time", format,
56+
"strfmt carries the marker, so its source is read back and the format survives the compiled load")
57+
assert.NotContains(t, strings.Join(sourceless, "\n"), "strfmt",
58+
"and no builder went looking for a strfmt declaration and missed")
59+
60+
// What DOES appear here is a lookup for time.Time, and it costs the spec nothing: the type is
61+
// recognized from its identity, so the declaration was never needed. It is announced because the
62+
// lookup happened, not because anything was lost — the recognizers that make it redundant run
63+
// after it at that call site rather than before. Asserted loosely for that reason: the notice is
64+
// honest about what the scanner did, and tightening it here would pin an ordering that is worth
65+
// changing.
66+
assert.NotEmpty(t, format)
67+
}
68+
69+
// The complement: a dependency the policy declines to parse is announced, but only to whoever asks
70+
// for one of its declarations.
71+
//
72+
// makeplans carries no annotation of its own, so the marker scan passes over it and its source is
73+
// never read — which is the policy working, not a fault. The fixture then uses one of its types as a
74+
// model, so the definition collapses and a builder goes looking for a declaration that is not there.
75+
// That lookup is what earns the notice, and it names the type rather than the load.
76+
func TestCompiledDependencies_ReportsWhatTheSpecActuallyMissed(t *testing.T) {
77+
t.Parallel()
78+
79+
var sourceless []string
80+
doc, err := codescan.Run(&codescan.Options{
81+
Packages: []string{"./bookings/..."},
82+
WorkDir: scantest.FixturesDir() + "/goparsing",
83+
ScanModels: true,
84+
CompiledDependencies: true,
85+
OnDiagnostic: func(d codescan.Diagnostic) {
86+
if strings.Contains(d.Message, "could not be read") {
87+
sourceless = append(sourceless, d.Message)
88+
}
89+
},
90+
})
91+
require.NoError(t, err)
92+
93+
assert.NotContains(t, doc.Definitions, "Booking",
94+
"the model is declared in a dependency that says nothing about itself, so it is not read")
4995

50-
format, hints = scan(true)
51-
assert.Empty(t, format, "export data carries strfmt's types but not its annotations")
52-
assert.Equal(t, 1, hints, "and the loss is announced, since nothing in the output would show it")
96+
require.NotEmpty(t, sourceless, "the collapse shows up in the output as an absence; the notice is what names it")
97+
assert.Contains(t, strings.Join(sourceless, "\n"), "makeplans.Booking")
98+
assert.Contains(t, strings.Join(sourceless, "\n"), "nothing in its source is annotated",
99+
"the reason distinguishes the policy declining to read from source that could not be read")
53100
}
Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
// SPDX-FileCopyrightText: Copyright 2015-2025 go-swagger maintainers
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//go:build !wasm
5+
6+
package integration
7+
8+
import (
9+
"bytes"
10+
"go/ast"
11+
"go/parser"
12+
"go/token"
13+
"go/types"
14+
"strings"
15+
"testing"
16+
"testing/fstest"
17+
18+
"golang.org/x/tools/go/gcexportdata"
19+
20+
"github.com/go-openapi/codescan"
21+
"github.com/go-openapi/testify/v2/assert"
22+
"github.com/go-openapi/testify/v2/require"
23+
)
24+
25+
// A dependency whose types arrive without its source is a fact about the LOAD. Whether it matters is
26+
// a fact about the SPEC, and the two are not the same question.
27+
//
28+
// Announcing it at load time announces it for every package the roots reach — which includes
29+
// everything imported from inside a function body. Scanning a generated client under a WebAssembly
30+
// guest, where the whole standard library arrives this way, that is hundreds of notices about
31+
// packages like `strconv` that no part of the API surface will ever touch, and the handful that
32+
// matter are lost among them.
33+
//
34+
// So the notice is raised from the lookup that wanted a declaration and did not get one. These cases
35+
// pin the four outcomes that distinction has to produce.
36+
const (
37+
sourcelessLib = "package lib\n\ntype Stamp struct{ Seconds int64 }\n"
38+
sourcelessTime = "package time\n\ntype Time struct{ wall uint64 }\ntype Duration int64\n"
39+
)
40+
41+
// exportOnlyFor writes export data for the two packages these cases serve without source.
42+
//
43+
// Neither is on the virtual filesystem, so the loader can only know their types — the state a scan is
44+
// in under a WebAssembly guest, where the standard library ships as export data and there is no
45+
// GOROOT to read.
46+
func exportOnlyFor(t *testing.T) fstest.MapFS {
47+
t.Helper()
48+
49+
write := func(pkgPath, src string) []byte {
50+
fset := token.NewFileSet()
51+
f, err := parser.ParseFile(fset, "src.go", src, parser.ParseComments)
52+
require.NoError(t, err)
53+
54+
pkg, err := (&types.Config{}).Check(pkgPath, fset, []*ast.File{f}, nil)
55+
require.NoError(t, err)
56+
57+
var blob bytes.Buffer
58+
require.NoError(t, gcexportdata.Write(&blob, fset, pkg))
59+
60+
return blob.Bytes()
61+
}
62+
63+
return fstest.MapFS{
64+
"example.com/lib.export": &fstest.MapFile{Data: write("example.com/lib", sourcelessLib)},
65+
"time.export": &fstest.MapFile{Data: write("time", sourcelessTime)},
66+
}
67+
}
68+
69+
func TestExportOnly_ReportedWhereItCosts(t *testing.T) {
70+
t.Parallel()
71+
72+
for _, tc := range []struct {
73+
name string
74+
app string
75+
want string // the type the notice must name, or "" for silence
76+
}{
77+
{
78+
// The `strconv` case. The package is in the closure because a function body reaches it, and
79+
// nothing inside a body can reach the emitted document. Announcing this is the noise.
80+
name: "reached only from a function body",
81+
app: "package app\n\nimport \"example.com/lib\"\n\n// swagger:model Model\n" +
82+
"type Model struct {\n\tName string `json:\"name\"`\n}\n\nfunc use() { _ = lib.Stamp{} }\n",
83+
want: "",
84+
},
85+
{
86+
// A field the document carries. Whatever the declaration said about itself — a
87+
// swagger:strfmt, its godoc — is absent, and nothing else in the output shows the gap.
88+
name: "a field the API surface carries",
89+
app: "package app\n\nimport \"example.com/lib\"\n\n// swagger:model Model\n" +
90+
"type Model struct {\n\tAt lib.Stamp `json:\"at\"`\n}\n",
91+
want: "example.com/lib.Stamp",
92+
},
93+
{
94+
// Recognised from its identity alone, ahead of any declaration lookup — so nothing was
95+
// wanted from the source and nothing was lost. Silence here is the point of the change.
96+
name: "a recognised type needs no source",
97+
app: "package app\n\nimport \"time\"\n\n// swagger:model Model\n" +
98+
"type Model struct {\n\tAt time.Time `json:\"at\"`\n}\n",
99+
want: "",
100+
},
101+
{
102+
// The complement, and the case worth a notice: consumed, not recognised. What a
103+
// time.Duration should look like on the wire is the author's decision — which is why
104+
// go-openapi offers strfmt.Duration — so codescan says it cannot tell rather than guessing.
105+
name: "a consumed type with no recogniser",
106+
app: "package app\n\nimport \"time\"\n\n// swagger:model Model\n" +
107+
"type Model struct {\n\tD time.Duration `json:\"d\"`\n}\n",
108+
want: "time.Duration",
109+
},
110+
} {
111+
t.Run(tc.name, func(t *testing.T) {
112+
t.Parallel()
113+
114+
tree := fstest.MapFS{
115+
"go.mod": &fstest.MapFile{Data: []byte(
116+
"module example.com/app\n\ngo 1.25.0\n\nrequire example.com/lib v1.0.0\n")},
117+
"app/app.go": &fstest.MapFile{Data: []byte(tc.app)},
118+
}
119+
120+
var sourceless []string
121+
_, err := codescan.Run(&codescan.Options{
122+
Packages: []string{"./app"},
123+
WorkDir: ".",
124+
ScanModels: true,
125+
ToolchainFreeLoader: true,
126+
FS: tree,
127+
ExportData: exportOnlyFor(t),
128+
GOOS: "linux",
129+
GOARCH: "amd64",
130+
OnDiagnostic: func(d codescan.Diagnostic) {
131+
if strings.Contains(d.Message, "could not be read") {
132+
sourceless = append(sourceless, d.Message)
133+
}
134+
},
135+
})
136+
require.NoError(t, err)
137+
138+
if tc.want == "" {
139+
assert.Empty(t, sourceless,
140+
"nothing in the document reached this package's source, so there is nothing to report")
141+
142+
return
143+
}
144+
145+
require.Len(t, sourceless, 1, "the notice names the declaration that was wanted, once")
146+
assert.Contains(t, sourceless[0], tc.want)
147+
})
148+
}
149+
}

0 commit comments

Comments
 (0)
⌂ 换网址