chore: keys of keys walks up the tree (#15)

* chore: more interaction with keys

* chore: fix comment

* chore: enforce

Author: ThomasRooney

cmd/wasm/functions.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"github.com/speakeasy-api/jsonpath/pkg/jsonpath"
 	"github.com/speakeasy-api/jsonpath/pkg/jsonpath/config"
+	"github.com/speakeasy-api/jsonpath/pkg/jsonpath/token"
 	"github.com/speakeasy-api/jsonpath/pkg/overlay"
 	"gopkg.in/yaml.v3"
 	"reflect"
@@ -32,6 +33,7 @@ func CalculateOverlay(originalYAML, targetYAML, existingOverlay string) (string,
 	if err != nil {
 		return "", fmt.Errorf("failed to parse overlay schema in CalculateOverlay: %w", err)
 	}
+	existingOverlayDocument.JSONPathVersion = "rfc9535" // force this in the playground.
 	// now modify the original using the existing overlay
 	err = existingOverlayDocument.ApplyTo(&orig)
 	if err != nil {
@@ -108,10 +110,22 @@ func ApplyOverlay(originalYAML, overlayYAML string) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("failed to parse overlay schema in ApplyOverlay: %w", err)
 	}
-
+	err = overlay.Validate()
+	if err != nil {
+		return "", fmt.Errorf("failed to validate overlay schema in ApplyOverlay: %w", err)
+	}
+	hasFilterExpression := false
 	// check to see if we have an overlay with an error, or a partial overlay: i.e. any overlay actions are missing an update or remove
 	for i, action := range overlay.Actions {
+		tokenized := token.NewTokenizer(action.Target, config.WithPropertyNameExtension()).Tokenize()
+		for _, tok := range tokenized {
+			if tok.Token == token.FILTER {
+				hasFilterExpression = true
+				break
+			}
+		}
 		parsed, pathErr := jsonpath.NewPath(action.Target, config.WithPropertyNameExtension())
+
 		var node *yaml.Node
 		if pathErr != nil {
 			node, err = lookupOverlayActionTargetNode(overlayYAML, i)
@@ -132,6 +146,9 @@ func ApplyOverlay(originalYAML, overlayYAML string) (string, error) {
 			return applyOverlayJSONPathIncomplete(result, node)
 		}
 	}
+	if hasFilterExpression && overlay.JSONPathVersion != "rfc9535" {
+		return "", fmt.Errorf("invalid overlay schema: must have `x-speakeasy-jsonpath: rfc9535`")
+	}
 
 	err = overlay.ApplyTo(&orig)
 	if err != nil {

pkg/jsonpath/yaml_query.go

@@ -103,6 +103,7 @@ func (s innerSegment) Query(idx index, value *yaml.Node, root *yaml.Node) []*yam
 			// we just want to return the values
 			for i, child := range value.Content {
 				if i%2 == 1 {
+					idx.setPropertyKey(value.Content[i-1], value)
 					idx.setPropertyKey(child, value.Content[i-1])
 					result = append(result, child)
 				}
@@ -123,6 +124,7 @@ func (s innerSegment) Query(idx index, value *yaml.Node, root *yaml.Node) []*yam
 				val := value.Content[i+1]
 
 				if key.Value == s.dotName {
+					idx.setPropertyKey(key, value)
 					idx.setPropertyKey(val, key)
 					result = append(result, val)
 					break
@@ -156,8 +158,9 @@ func (s selector) Query(idx index, value *yaml.Node, root *yaml.Node) []*yaml.No
 				key = child.Value
 				continue
 			}
-			if key == s.name {
-				idx.setPropertyKey(child, value.Content[i])
+			if key == s.name && i%2 == 1 {
+				idx.setPropertyKey(value.Content[i], value.Content[i-1])
+				idx.setPropertyKey(value.Content[i-1], value)
 				return []*yaml.Node{child}
 			}
 		}
@@ -181,6 +184,7 @@ func (s selector) Query(idx index, value *yaml.Node, root *yaml.Node) []*yaml.No
 			var result []*yaml.Node
 			for i, child := range value.Content {
 				if i%2 == 1 {
+					idx.setPropertyKey(value.Content[i-1], value)
 					idx.setPropertyKey(child, value.Content[i-1])
 					result = append(result, child)
 				}
@@ -223,6 +227,7 @@ func (s selector) Query(idx index, value *yaml.Node, root *yaml.Node) []*yaml.No
 		switch value.Kind {
 		case yaml.MappingNode:
 			for i := 1; i < len(value.Content); i += 2 {
+				idx.setPropertyKey(value.Content[i-1], value)
 				idx.setPropertyKey(value.Content[i], value.Content[i-1])
 				if s.filter.Matches(idx, value.Content[i], root) {
 					result = append(result, value.Content[i])

pkg/jsonpath/yaml_query_test.go

@@ -107,7 +107,7 @@ store:
 			parser := newParserPrivate(tokenizer, tokenizer.Tokenize())
 			err = parser.parse()
 			if err != nil {
-				t.Errorf("Error parsing JSON ast: %v", err)
+				t.Errorf("Error parsing JSON Path: %v", err)
 				return
 			}
 
@@ -233,6 +233,28 @@ deeply:
 `,
 			expected: []string{"key1", "key2", "key3", "key4"},
 		},
+		{
+			name:  "Custom x-my-ignore extension filter",
+			input: "$.paths[?@[\"x-my-ignore\"][?@ == \"match\"]].found",
+			yaml: `
+openapi: 3.1.0
+info:
+  title: Test
+  version: 0.1.0
+  summary: Test Summary
+  description: |-
+    Some test description.
+    About our test document.
+paths:
+  /anything/ignored:
+    x-my-ignore: [match, not_matched]
+    found: true
+  /anything/not-ignored:
+    x-my-ignore: [not_matched]
+    found: false
+`,
+			expected: []string{"true"},
+		},
 	}
 
 	for _, test := range tests {
@@ -248,7 +270,7 @@ deeply:
 			parser := newParserPrivate(tokenizer, tokenizer.Tokenize(), config.WithPropertyNameExtension())
 			err = parser.parse()
 			if err != nil {
-				t.Errorf("Error parsing JSON ast: %v", err)
+				t.Errorf("Error parsing JSON Path: %v", err)
 				return
 			}
 

pkg/overlay/apply.go

@@ -59,8 +59,13 @@ func removeNode(idx parentIndex, node *yaml.Node) {
 		if child == node {
 			switch parent.Kind {
 			case yaml.MappingNode:
-				// we have to delete the key too
-				parent.Content = append(parent.Content[:i-1], parent.Content[i+1:]...)
+				if i%2 == 1 {
+					// if we select a value, we should delete the key too
+					parent.Content = append(parent.Content[:i-1], parent.Content[i+1:]...)
+				} else {
+					// if we select a key, we should delete the value
+					parent.Content = append(parent.Content[:i], parent.Content[i+1:]...)
+				}
 				return
 			case yaml.SequenceNode:
 				parent.Content = append(parent.Content[:i], parent.Content[i+1:]...)

pkg/overlay/schema.go

@@ -11,12 +11,14 @@ type Extensions map[string]any
 
 // Overlay is the top-level configuration for an OpenAPI overlay.
 type Overlay struct {
-	Extensions `yaml:"-,inline"`
+	Extensions Extensions `yaml:",inline"`
 
-	// Version is the version of the overlay configuration. As the RFC was never
-	// really ratifies, this value does not mean much.
+	// Version is the version of the overlay configuration.
 	Version string `yaml:"overlay"`
 
+	// JSONPathVersion should be set to rfc9535, and is used for backwards compatability purposes
+	JSONPathVersion string `yaml:"x-speakeasy-jsonpath,omitempty"`
+
 	// Info describes the metadata for the overlay.
 	Info Info `yaml:"info"`
 

pkg/overlay/testdata/openapi-overlayed.yaml

@@ -106,38 +106,7 @@ paths:
   /drinks:
     x-speakeasy-note:
       "$ref": "./removeNote.yaml"
-  /drink/{name}: #TODO: this should be by product code and we should have search by name
-    get:
-      operationId: getDrink
-      summary: Get a drink.
-      description: Get a drink by name, if authenticated this will include stock levels and product codes otherwise it will only include public information.
-      tags:
-        - drinks
-      parameters:
-        - name: name
-          in: path
-          required: true
-          schema:
-            type: string
-        - x-parameter-extension: foo
-          name: test
-          description: Test parameter
-          in: query
-          schema:
-            type: string
-      responses:
-        "200":
-          description: Test response
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Drink"
-                type: string
-          x-response-extension: foo
-        "5XX":
-          $ref: "#/components/responses/APIError"
-        default:
-          $ref: "#/components/responses/UnknownError"
+  /drink/{name}: {}
   /ingredients:
     get:
       operationId: listIngredients

pkg/overlay/testdata/openapi.yaml

@@ -131,7 +131,7 @@ paths:
         default:
           $ref: "#/components/responses/UnknownError"
 
-  /drink/{name}: #TODO: this should be by product code and we should have search by name
+  /drink/{name}:
     get:
       operationId: getDrink
       summary: Get a drink.

pkg/overlay/testdata/overlay-generated.yaml

@@ -18,19 +18,5 @@ actions:
         "$ref": "./removeNote.yaml"
   - target: $["paths"]["/drinks"]["get"]
     remove: true
-  - target: $["paths"]["/drink/{name}"]["get"]["parameters"]
-    update:
-      - x-parameter-extension: foo
-        name: test
-        description: Test parameter
-        in: query
-        schema:
-          type: string
-  - target: $["paths"]["/drink/{name}"]["get"]["responses"]["200"]["description"]
-    update: Test response
-  - target: $["paths"]["/drink/{name}"]["get"]["responses"]["200"]["content"]["application/json"]["schema"]
-    update:
-      type: string
-  - target: $["paths"]["/drink/{name}"]["get"]["responses"]["200"]
-    update:
-      x-response-extension: foo
+  - target: $["paths"]["/drink/{name}"]["get"]
+    remove: true

pkg/overlay/testdata/overlay.yaml

@@ -26,7 +26,9 @@ actions:
   - target: $.paths["/drinks"].get
     description: Test remove
     remove: true
-    x-action-extension: bar
+  - target: $.paths["/drink/{name}"].get~
+    description: Test removing a key -- should delete the node too
+    remove: true
   - target: $.paths["/drinks"]
     update:
       x-speakeasy-note:

web/src/Playground.tsx

@@ -85,6 +85,54 @@ function Playground() {
     [],
   );
 
+  const onChangeOverlay = useCallback(
+    async (value: string | undefined, _: editor.IModelContentChangedEvent) => {
+      try {
+        setChangedLoading(true);
+        result.current = value || "";
+        const response = await ApplyOverlay(
+          original.current,
+          result.current,
+          true,
+        );
+        if (response.type == "success") {
+          setApplyOverlayMode("original+overlay");
+          changed.current = response.result || "";
+          setError("");
+          setOverlayMarkers([]);
+          const info = await GetInfo(changed.current, false);
+          tryHandlePageTitle(JSON.parse(info));
+        } else if (response.type == "incomplete") {
+          setApplyOverlayMode("jsonpathexplorer");
+          changed.current = response.result || "";
+          setError("");
+          setOverlayMarkers([]);
+        } else if (response.type == "error") {
+          setApplyOverlayMode("jsonpathexplorer");
+          setOverlayMarkers([
+            {
+              startLineNumber: response.line,
+              endLineNumber: response.line,
+              startColumn: response.col,
+              endColumn: response.col + 1000, // end of line
+              message: response.error,
+              severity: MarkerSeverity.Error, // Use MarkerSeverity from Monaco
+            },
+          ]);
+        }
+      } catch (e: unknown) {
+        if (e instanceof Error) {
+          setError(e.message);
+        }
+      } finally {
+        setChangedLoading(false);
+      }
+    },
+    [],
+  );
+
+  const onChangeOverlayDebounced = useDebounceCallback(onChangeOverlay, 500);
+
   const getShareUrl = useCallback(async () => {
     try {
       setShareUrlLoading(true);
@@ -146,20 +194,7 @@ function Playground() {
           original.current = decompressed.original;
           result.current = decompressed.result;
 
-          const changedNew = await ApplyOverlay(
-            original.current,
-            result.current,
-            false,
-          );
-          if (changedNew.type == "success") {
-            const info = await GetInfo(original.current, false);
-            const parsedInfo = JSON.parse(info);
-            tryHandlePageTitle(parsedInfo);
-            posthog.capture("overlay.speakeasy.com:load-shared", {
-              openapi: parsedInfo,
-            });
-            changed.current = changedNew.result;
-          }
+          await onChangeOverlay(result.current, {} as any);
         } catch (error: any) {
           console.error("invalid share url:", error.message);
         }
@@ -241,54 +276,6 @@ function Playground() {
 
   const onChangeBDebounced = useDebounceCallback(onChangeB, 500);
 
-  const onChangeC = useCallback(
-    async (value: string | undefined, _: editor.IModelContentChangedEvent) => {
-      try {
-        setChangedLoading(true);
-        result.current = value || "";
-        const response = await ApplyOverlay(
-          original.current,
-          result.current,
-          true,
-        );
-        if (response.type == "success") {
-          setApplyOverlayMode("original+overlay");
-          changed.current = response.result || "";
-          setError("");
-          setOverlayMarkers([]);
-          const info = await GetInfo(changed.current, false);
-          tryHandlePageTitle(JSON.parse(info));
-        } else if (response.type == "incomplete") {
-          setApplyOverlayMode("jsonpathexplorer");
-          changed.current = response.result || "";
-          setError("");
-          setOverlayMarkers([]);
-        } else if (response.type == "error") {
-          setApplyOverlayMode("jsonpathexplorer");
-          setOverlayMarkers([
-            {
-              startLineNumber: response.line,
-              endLineNumber: response.line,
-              startColumn: response.col,
-              endColumn: response.col + 1000, // end of line
-              message: response.error,
-              severity: MarkerSeverity.Error, // Use MarkerSeverity from Monaco
-            },
-          ]);
-        }
-      } catch (e: unknown) {
-        if (e instanceof Error) {
-          setError(e.message);
-        }
-      } finally {
-        setChangedLoading(false);
-      }
-    },
-    [],
-  );
-
-  const onChangeCDebounced = useDebounceCallback(onChangeC, 500);
-
   const ref = useRef<ImperativePanelGroupHandle>(null);
 
   const maxLayout = useCallback((index: number) => {
@@ -460,7 +447,7 @@ function Playground() {
               <Editor
                 readonly={false}
                 value={result.current}
-                onChange={onChangeCDebounced}
+                onChange={onChangeOverlayDebounced}
                 loading={resultLoading}
                 markers={overlayMarkers}
                 title={"Overlay"}