Skip to content

Commit ee01904

Browse files
fredbiclaude
andcommitted
fix(expander): rebase a $ref found under an unmapped keyword
expandSchema walks the fields of Schema and stops there. A keyword the Swagger 2.0 model predates - propertyNames, contains, if/then/else, $defs - lands in Schema.ExtraProps as raw JSON, and inlining that subtree from another document copied its $ref into the root verbatim: "#/definitions/leaf" then named a definition of the root document rather than the one it came from, and resolving it raised "has no key". rebaseExtraRefs walks the raw JSON and rewrites every string under a "$ref" key with normalizeURI then denormalizeRef, the pair the SkipSchemas branch already uses for a mapped $ref, so "#/definitions/leaf" becomes "other.json#/definitions/leaf". The pointer is correct; it is not expanded, and the output is no more self-contained than before. Empty and unparseable strings are left alone: an unmapped keyword may hold any JSON, and a string under a "$ref" key is not necessarily a reference. Two alternatives were rejected: refusing to expand a $ref whose target the model cannot represent breaks documents that work today, and expanding the ExtraProps subtree properly needs a raw-JSON walker with its own cycle detection over untyped values. FuzzExpandSpec gains the property that found this - every local $ref surviving expansion must still name something - guarded by the same check on the input, so a document that already dangles proves nothing. Its stub loader used to answer every path, including the root document's own URL, so a $ref spelled "." inlined a foreign document; it now returns the document's own bytes for that path. No cycle is involved: the cut point of shared-node-cycles.json is unchanged over 40 expansions, with the same three outcomes as before. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Signed-off-by: Frederic BIDON <fredbi@yahoo.com>
1 parent af1d328 commit ee01904

5 files changed

Lines changed: 239 additions & 5 deletions

File tree

‎expander.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,8 @@ func expandSchema(target Schema, parentRefs []string, resolver *schemaLoader, ba
312312
return &target, nil
313313
}
314314

315+
rebaseExtraRefs(target.ExtraProps, resolver, basePath)
316+
315317
for k := range target.Definitions {
316318
tt, err := expandSchema(target.Definitions[k], parentRefs, resolver, basePath)
317319
if resolver.shouldStopOnError(err) {
@@ -424,6 +426,67 @@ func expandSchema(target Schema, parentRefs []string, resolver *schemaLoader, ba
424426
return &target, nil
425427
}
426428

429+
// rebaseExtraRefs rewrites the $ref held by keywords this model does not map, so that
430+
// they still point at their target once the schema is inlined into another document.
431+
//
432+
// expandSchema walks the fields of [Schema] and stops there. A keyword the Swagger 2.0
433+
// model predates - propertyNames, contains, if/then/else, $defs - lands in ExtraProps as
434+
// raw JSON, and a $ref inside it is copied into the root verbatim: "#/definitions/leaf"
435+
// then names a definition of the root document instead of the one it came from.
436+
//
437+
// Rebasing makes the pointer correct. It does not expand it, and it does not make the
438+
// expanded document self-contained: a $ref that came from another document keeps pointing
439+
// there.
440+
func rebaseExtraRefs(extra map[string]any, resolver *schemaLoader, basePath string) {
441+
for key := range extra {
442+
rebaseRawRefs(extra[key], resolver, basePath)
443+
}
444+
}
445+
446+
// rebaseRawRefs walks raw JSON and rebases every "$ref" string value it finds.
447+
func rebaseRawRefs(node any, resolver *schemaLoader, basePath string) {
448+
switch value := node.(type) {
449+
case map[string]any:
450+
for key := range value {
451+
if key == jsonRef {
452+
if ref, ok := value[key].(string); ok {
453+
if rebased, ok := rebaseRawRef(ref, resolver, basePath); ok {
454+
value[key] = rebased
455+
}
456+
457+
continue
458+
}
459+
}
460+
461+
rebaseRawRefs(value[key], resolver, basePath)
462+
}
463+
case []any:
464+
for i := range value {
465+
rebaseRawRefs(value[i], resolver, basePath)
466+
}
467+
}
468+
}
469+
470+
// rebaseRawRef resolves a $ref against basePath, then spells it relative to the document
471+
// being expanded, like the SkipSchemas branch of [expandSchema] does for a mapped $ref.
472+
//
473+
// It reports false when the $ref is empty or does not parse: an unmapped keyword may hold
474+
// any JSON, and a string under a "$ref" key is not necessarily a reference.
475+
func rebaseRawRef(ref string, resolver *schemaLoader, basePath string) (string, bool) {
476+
if ref == "" {
477+
return "", false
478+
}
479+
480+
rebased, err := NewRef(normalizeURI(ref, basePath))
481+
if err != nil {
482+
return "", false
483+
}
484+
485+
denormalized := denormalizeRef(&rebased, resolver.context.basePath, resolver.context.rootID)
486+
487+
return denormalized.String(), true
488+
}
489+
427490
func expandSchemaRef(target Schema, parentRefs []string, resolver *schemaLoader, basePath string) (*Schema, error) {
428491
// if a Ref is found, all sibling fields are skipped
429492
// Ref also changes the resolution scope of children expandSchema

‎expander_fuzz_test.go‎

Lines changed: 71 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ package spec
55

66
import (
77
"encoding/json"
8+
"sort"
9+
"strings"
810
"testing"
911

1012
"github.com/go-openapi/testify/v2/require"
@@ -15,10 +17,17 @@ import (
1517
// It answers any path, so the loader never touches the filesystem or the network, and it holds a
1618
// $ref back out to another document: since every path yields this same stub, that $ref chains
1719
// forever unless cycle detection and the expansion budget stop it.
20+
//
21+
// "unmapped" carries a $ref under propertyNames, a keyword this model does not map: inlining
22+
// that subtree must rebase the $ref on the stub, not copy it into the root.
23+
// fuzzBase is where the document being expanded is taken to live.
24+
const fuzzBase = "file:///fuzz/spec.json"
25+
1826
const remoteFuzzStub = `{"definitions":{` +
1927
`"remote":{"type":"string"},` +
2028
`"cycle":{"$ref":"other.json#/definitions/cycle"},` +
21-
`"deep":{"properties":{"a":{"$ref":"#/definitions/remote"},"b":{"$ref":"other.json#/definitions/deep"}}}` +
29+
`"deep":{"properties":{"a":{"$ref":"#/definitions/remote"},"b":{"$ref":"other.json#/definitions/deep"}}},` +
30+
`"unmapped":{"propertyNames":{"$ref":"#/definitions/remote"}}` +
2231
`}}`
2332

2433
// FuzzExpandSpec expands arbitrary documents.
@@ -27,9 +36,10 @@ const remoteFuzzStub = `{"definitions":{` +
2736
// $refs across documents and rewrites the tree as it goes. It is also where a hostile document
2837
// costs the most: see ErrExpandTooManyNodes and the confinement notes on ExpandOptions.
2938
//
30-
// The properties are that expansion terminates, and that what it produces is still a document -
39+
// The properties are that expansion terminates, that what it produces is still a document -
3140
// an expansion that succeeds but leaves behind something we can no longer read is a silent
32-
// corruption of the caller's spec.
41+
// corruption of the caller's spec - and that the local pointers it leaves behind still name
42+
// something. That last one is what found the ExtraProps defect fixed by rebaseExtraRefs.
3343
//
3444
// The document loader is stubbed, so nothing here reads a file or opens a socket.
3545
func FuzzExpandSpec(f *testing.F) {
@@ -43,9 +53,23 @@ func FuzzExpandSpec(f *testing.F) {
4353
return // not a document we claim to accept
4454
}
4555

56+
// what the input claims before anything is expanded: an input that already
57+
// dangles cannot say whether expansion broke a pointer
58+
original, err := json.Marshal(doc)
59+
if err != nil {
60+
return
61+
}
62+
soundBefore := len(unresolvedLocalRefs(&doc, string(original))) == 0
63+
4664
opts := &ExpandOptions{
47-
RelativeBase: "file:///fuzz/spec.json",
48-
PathLoader: func(string) (json.RawMessage, error) {
65+
RelativeBase: fuzzBase,
66+
PathLoader: func(path string) (json.RawMessage, error) {
67+
if normalizeBase(path) == normalizeBase(fuzzBase) {
68+
// a $ref spelled "." names the document being expanded: a real loader
69+
// hands back the bytes it was read from, not another document
70+
return original, nil
71+
}
72+
4973
return json.RawMessage(remoteFuzzStub), nil
5074
},
5175
MaxExpansionNodes: 500,
@@ -61,9 +85,50 @@ func FuzzExpandSpec(f *testing.F) {
6185
var reread Swagger
6286
require.NoErrorf(t, json.Unmarshal(expanded, &reread),
6387
"an expanded document no longer parses: %s", expanded)
88+
89+
if !soundBefore {
90+
return // a document whose own pointers were already dangling proves nothing
91+
}
92+
93+
require.Emptyf(t, unresolvedLocalRefs(&reread, string(expanded)),
94+
"expansion left a local $ref pointing at nothing.\ninput: %s\nexpanded: %s", data, expanded)
6495
})
6596
}
6697

98+
// unresolvedLocalRefs returns the local $ref of jazon that name nothing in doc.
99+
//
100+
// Only "#/..." pointers are looked at: a $ref into another document is the loader's business,
101+
// and the loader is a stub here. The document is walked with the same jsonpointer call the
102+
// resolver makes, so "has no key" here is "has no key" there.
103+
func unresolvedLocalRefs(doc any, jazon string) []string {
104+
var unresolved []string
105+
seen := make(map[string]struct{})
106+
107+
for _, matched := range rex.FindAllStringSubmatch(jazon, -1) {
108+
refString := matched[1]
109+
if !strings.HasPrefix(refString, "#/") {
110+
continue
111+
}
112+
if _, already := seen[refString]; already {
113+
continue
114+
}
115+
seen[refString] = struct{}{}
116+
117+
ref, err := NewRef(refString)
118+
if err != nil {
119+
continue // not a reference this package claims to understand
120+
}
121+
122+
if _, _, err := ref.GetPointer().Get(doc); err != nil {
123+
unresolved = append(unresolved, refString)
124+
}
125+
}
126+
127+
sort.Strings(unresolved)
128+
129+
return unresolved
130+
}
131+
67132
// expanderSeeds are the documents the fuzzer starts from.
68133
//
69134
// They aim at the shapes expansion has to walk: $refs in every position that accepts one,
@@ -78,6 +143,7 @@ func expanderSeeds() []string {
78143
`{"definitions":{"A":{"$ref":"other.json#/definitions/cycle"}}}`,
79144
`{"definitions":{"A":{"properties":{"a":{"$ref":"#/definitions/A"}},"items":{"$ref":"#/definitions/A"}}}}`,
80145
`{"definitions":{"A":{"allOf":[{"$ref":"#/definitions/A"}],"additionalProperties":{"$ref":"#/definitions/A"}}}}`,
146+
`{"definitions":{"A":{"$ref":"other.json#/definitions/unmapped"}}}`,
81147
`{"paths":{"/x":{"$ref":"other.json#/definitions/deep"}}}`,
82148
`{"paths":{"/x":{"get":{"parameters":[{"$ref":"#/parameters/p"}],` +
83149
`"responses":{"200":{"$ref":"#/responses/r"}}}}},` +

‎expander_test.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1082,3 +1082,57 @@ func TestExpand_Issue145(t *testing.T) {
10821082
})
10831083
})
10841084
}
1085+
1086+
func TestExpand_UnmappedSubtreeRef(t *testing.T) {
1087+
// A $ref under a keyword the Swagger 2.0 model does not map (propertyNames, if)
1088+
// lands in Schema.ExtraProps as raw JSON. Inlining the subtree from other.json used
1089+
// to carry those $ref into the root verbatim, where "#/definitions/leaf" names
1090+
// nothing. They are rebased on the document they came from instead.
1091+
fixturePath := filepath.Join("testdata", "expansion", "unmapped", "root.json")
1092+
jazon, doc := expandThisOrDieTrying(t, fixturePath)
1093+
require.NotEmpty(t, jazon)
1094+
1095+
holder := doc.Definitions["holder"]
1096+
1097+
t.Run("the mapped $ref is expanded", func(t *testing.T) {
1098+
mapped := holder.Properties["mapped"]
1099+
assert.EqualT(t, "", mapped.Ref.String())
1100+
assert.TrueT(t, mapped.Type.Contains("string"))
1101+
})
1102+
1103+
t.Run("the unmapped $ref are rebased, not expanded", func(t *testing.T) {
1104+
assert.EqualT(t, "other.json#/definitions/leaf", rawRef(t, holder.ExtraProps["propertyNames"]))
1105+
1106+
cond, ok := holder.ExtraProps["if"].(map[string]any)
1107+
require.TrueT(t, ok)
1108+
anyOf, ok := cond["anyOf"].([]any)
1109+
require.TrueT(t, ok)
1110+
require.EqualT(t, 1, len(anyOf))
1111+
assert.EqualT(t, "other.json#/definitions/leaf", rawRef(t, anyOf[0]))
1112+
})
1113+
1114+
t.Run("every surviving $ref resolves against the expanded document", func(t *testing.T) {
1115+
opts := &ExpandOptions{RelativeBase: fixturePath}
1116+
assertRefResolve(t, jazon, "", doc, opts)
1117+
})
1118+
1119+
t.Run("and against a document read back from the expanded bytes", func(t *testing.T) {
1120+
reloaded := new(Swagger)
1121+
require.NoError(t, json.Unmarshal([]byte(jazon), reloaded))
1122+
1123+
opts := &ExpandOptions{RelativeBase: fixturePath}
1124+
assertRefResolve(t, jazon, "", reloaded, opts)
1125+
})
1126+
}
1127+
1128+
// rawRef returns the $ref held by a raw JSON node.
1129+
func rawRef(t testing.TB, node any) string {
1130+
t.Helper()
1131+
1132+
asMap, ok := node.(map[string]any)
1133+
require.TrueT(t, ok)
1134+
ref, ok := asMap["$ref"].(string)
1135+
require.TrueT(t, ok)
1136+
1137+
return ref
1138+
}
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"definitions": {
3+
"deep": {
4+
"type": "object",
5+
"properties": {
6+
"mapped": {
7+
"$ref": "#/definitions/leaf"
8+
}
9+
},
10+
"propertyNames": {
11+
"$ref": "#/definitions/leaf"
12+
},
13+
"if": {
14+
"anyOf": [
15+
{
16+
"$ref": "#/definitions/leaf"
17+
}
18+
]
19+
}
20+
},
21+
"leaf": {
22+
"type": "string"
23+
}
24+
}
25+
}
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"swagger": "2.0",
3+
"info": {
4+
"title": "a $ref under a keyword this model does not map",
5+
"version": "1.0.0"
6+
},
7+
"paths": {
8+
"/things": {
9+
"get": {
10+
"responses": {
11+
"200": {
12+
"description": "a thing",
13+
"schema": {
14+
"$ref": "#/definitions/holder"
15+
}
16+
}
17+
}
18+
}
19+
}
20+
},
21+
"definitions": {
22+
"holder": {
23+
"$ref": "other.json#/definitions/deep"
24+
}
25+
}
26+
}

0 commit comments

Comments
 (0)
⌂ 换网址