From e50bb3bd5181a25a6d85b03aaa7641449c82c8b3 Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Mon, 18 Mar 2024 19:28:46 +0800
Subject: [PATCH 01/20] WIP: New SCIP-based GraphQL API draft

---
 .../graphqlbackend/codeintel.codenav.graphql  | 103 +++++++++++++++++-
 cmd/frontend/graphqlbackend/schema.graphql    |  40 ++++++-
 2 files changed, 140 insertions(+), 3 deletions(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index 1de7bbdd5976..6b102ed5fe42 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -105,6 +105,21 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
         """
         character: Int!
 
+        """
+        When specified, indicates that this request should be paginated and
+        to fetch results starting at this cursor.
+        A future request can be made for more results by passing in the
+        'LocationConnection.pageInfo.endCursor' that is returned.
+        """
+        after: String
+
+        """
+        When specified, indicates that this request should be paginated and
+        the first N results (relative to the cursor) should be returned. i.e.
+        how many results to return per page.
+        """
+        first: Int
+
         """
         When specified, it filters references by filename.
         """
@@ -118,12 +133,12 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
         """
         The line on which the symbol occurs (zero-based, inclusive).
         """
-        line: Int!
+        line: Int
 
         """
         The character (not byte) of the start line on which the symbol occurs (zero-based, inclusive).
         """
-        character: Int!
+        character: Int
 
         """
         When specified, indicates that this request should be paginated and
@@ -231,6 +246,19 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
         character: Int!
     ): Hover
 
+    """
+    Either one of 'symbol' or 'range' must be provided, but this isn't enforced at the GraphQL
+    layer due to the lack of support for input unions.
+    https://github.com/graphql/graphql-wg/blob/main/rfcs/InputUnion.md
+    """
+    usages(
+        symbol: LookupSCIPSymbol,
+        range: LookupRange,
+        filter: UsagesFilter,
+        first: Int,
+        after: String,
+    ): UsageConnection!
+
     """
     Code diagnostics provided through LSIF.
     """
@@ -247,6 +275,77 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
     snapshot(indexID: ID!): [SnapshotData!]
 }
 
+"""
+In the future, we may want to extend this to allow passing in a suffix or a "symbol pattern".
+"""
+input LookupSCIPSymbol {
+    name: String!
+    provenance: Provenance!
+}
+
+type SCIPSymbol {
+    name: String!
+    provenance: Provenance!
+}
+
+input LookupRange {
+    start: LookupPosition!
+    end: LookupPosition!
+}
+
+input LookupPosition {
+    line: Int!
+    character: Int!
+}
+
+input UsagesFilter {
+    and: [UsagesFilter!]
+    or: [UsagesFilter!]
+    not: UsagesFilter
+    # TODO: Can we be flexible here and allow patterns while still maintaining fast queries.
+    repository: String
+    path: String
+    kind: SymbolUsageKind
+    # TODO: Provide a way of controlling fallback behavior for indexes?
+    # Not super sure how that would work, given that for different repositories,
+    # the hashes will all be different?
+}
+
+enum SymbolUsageKind {
+    Definition,
+    Reference,
+    Implementation,
+    Super,
+}
+
+type UsageConnection {
+    nodes: [Usage!]!
+    pageInfo: PageInfo!
+}
+
+type Usage {
+    symbol: SCIPSymbol!
+    """
+    NOTE: Do not pass
+    TODO: For this blob, it is generally more useful to get +N/-N lines
+    rather than having to pass a range internally for the contents since
+    the offsets from the start will not be known a-priori.
+    """
+    blob: GitBlob!
+    """
+    Instead of blob { content }, allows accessing a sub-span of the content
+    using relative coordinates from the range of this usage. If linesBefore
+    or linesAfter is negative or exceeds the number of available lines,
+    the value is interpreted as until the start/end of the file.
+    """
+    surroundingBlobContent(surroundingLines: SurroundingLines = {linesBefore: 0, linesAfter: 0}): String!
+}
+
+input SurroundingLines {
+    linesBefore: Int
+    linesAfter: Int
+}
+
 """
 The SCIP snapshot decoration for a single SCIP Occurrence.
 """
diff --git a/cmd/frontend/graphqlbackend/schema.graphql b/cmd/frontend/graphqlbackend/schema.graphql
index f66a0f758c9b..3723ad8c32ff 100755
--- a/cmd/frontend/graphqlbackend/schema.graphql
+++ b/cmd/frontend/graphqlbackend/schema.graphql
@@ -6172,10 +6172,13 @@ type HighlightedFile {
     """
     html: String!
     """
-    Base64 encoded JSON payload of LSIF Typed with syntax highlighting data.
+    DEPRECATED: Base64 encoded JSON payload of SCIP with syntax highlighting data.
     """
     lsif: String!
     """
+    """
+    document: AssociatedSCIPDocument!
+    """
     A list of the desired line ranges. Each list is a list of lines, where each element is an HTML
     table row '<tr>...</tr>' string. This is useful if you only need to display specific subsets of
     the file.
@@ -6183,6 +6186,41 @@ type HighlightedFile {
     lineRanges(ranges: [HighlightLineRange!]!): [[String!]!]!
 }
 
+enum Provenance {
+    Precise,
+    Syntactic,
+    SearchBased,
+}
+
+interface WithProvenance {
+    """
+    Coarse-grained information about the data source.
+    """
+    provenance: Provenance!
+    """
+    Opaque fine-grained information describing the data source.
+
+    Provided only for debugging.
+
+    This field should be ignored when checking structural equality.
+    """
+    dataSource: String!
+}
+
+type AssociatedSCIPDocument implements WithProvenance {
+    """
+    Base64 encoded SCIP payload.
+
+    After decoding, the resulting byte buffer can be parsed with
+    pre-generated bindings in the sourcegraph/scip repository or
+    manually generating bindings using
+    https://github.com/sourcegraph/scip/blob/main/scip.proto
+    """
+    data: String!
+    provenance: Provenance!
+    dataSource: String!
+}
+
 """
 A file match.
 """

From e5b9e388659f1072465e6109b98c5cc7877ce2c6 Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Mon, 22 Apr 2024 21:35:51 +0800
Subject: [PATCH 02/20] Make repository and path filters objects instead of
 primitives

---
 .../graphqlbackend/codeintel.codenav.graphql     | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index 6b102ed5fe42..9d679d8d2145 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -303,14 +303,26 @@ input UsagesFilter {
     or: [UsagesFilter!]
     not: UsagesFilter
     # TODO: Can we be flexible here and allow patterns while still maintaining fast queries.
-    repository: String
-    path: String
+    repository: RepositoryFilter
+    path: PathFilter
     kind: SymbolUsageKind
     # TODO: Provide a way of controlling fallback behavior for indexes?
     # Not super sure how that would work, given that for different repositories,
     # the hashes will all be different?
 }
 
+input RepositoryFilter {
+    name: StringComparator!
+}
+
+input PathFilter {
+    name: StringComparator!
+}
+
+input StringComparator {
+    eq: String
+}
+
 enum SymbolUsageKind {
     Definition,
     Reference,

From df4fb888ae5570ca6f52221e7ca37338e60f80ea Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Mon, 22 Apr 2024 21:41:27 +0800
Subject: [PATCH 03/20] Make symbol name and provenance comparison objects
 instead of primitives

---
 .../graphqlbackend/codeintel.codenav.graphql       | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index 9d679d8d2145..de7268ef4c73 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -275,12 +275,20 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
     snapshot(indexID: ID!): [SnapshotData!]
 }
 
+input LookupSCIPSymbol {
+    name: SymbolNameComparator!
+    provenance: ProvenanceComparator!
+}
+
 """
 In the future, we may want to extend this to allow passing in a suffix or a "symbol pattern".
 """
-input LookupSCIPSymbol {
-    name: String!
-    provenance: Provenance!
+input SymbolNameComparator {
+    eq: String
+}
+
+input ProvenanceComparator {
+    eq: Provenance
 }
 
 type SCIPSymbol {

From 203e505ce26ca200b65d84faeebe068d305ced8a Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Tue, 23 Apr 2024 00:34:04 +0800
Subject: [PATCH 04/20] Move usages API to top-level

---
 .../graphqlbackend/codeintel.codenav.graphql  | 47 ++++++++++++-------
 1 file changed, 31 insertions(+), 16 deletions(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index de7268ef4c73..b4b2e21967ac 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -246,19 +246,6 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
         character: Int!
     ): Hover
 
-    """
-    Either one of 'symbol' or 'range' must be provided, but this isn't enforced at the GraphQL
-    layer due to the lack of support for input unions.
-    https://github.com/graphql/graphql-wg/blob/main/rfcs/InputUnion.md
-    """
-    usages(
-        symbol: LookupSCIPSymbol,
-        range: LookupRange,
-        filter: UsagesFilter,
-        first: Int,
-        after: String,
-    ): UsageConnection!
-
     """
     Code diagnostics provided through LSIF.
     """
@@ -275,6 +262,19 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
     snapshot(indexID: ID!): [SnapshotData!]
 }
 
+extend type Query {
+    usagesForSymbol(
+        symbol: LookupSCIPSymbol,
+        """
+        Non-optional due to implementation limitations
+        """
+        range: LookupRange!,
+        filter: UsagesFilter,
+        first: Int,
+        after: String,
+    ): UsageConnection!
+}
+
 input LookupSCIPSymbol {
     name: SymbolNameComparator!
     provenance: ProvenanceComparator!
@@ -297,8 +297,24 @@ type SCIPSymbol {
 }
 
 input LookupRange {
-    start: LookupPosition!
-    end: LookupPosition!
+    """
+    Defaults to instance-wide search if the repo is not specified.
+
+    This field is non-optional due to implementation limitations.
+    """
+    repo: String!
+    """
+    Defaults to HEAD of the default branch if not specified.
+    """
+    revision: String
+    """
+    Defaults to repo-wide search if the path is not specified.
+
+    This field is non-optional due to implementation limitations.
+    """
+    path: String!
+    start: LookupPosition
+    end: LookupPosition
 }
 
 input LookupPosition {
@@ -310,7 +326,6 @@ input UsagesFilter {
     and: [UsagesFilter!]
     or: [UsagesFilter!]
     not: UsagesFilter
-    # TODO: Can we be flexible here and allow patterns while still maintaining fast queries.
     repository: RepositoryFilter
     path: PathFilter
     kind: SymbolUsageKind

From 1949594a82eecd97458df34abe3106b9ddbc3420 Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Tue, 23 Apr 2024 14:35:08 +0800
Subject: [PATCH 05/20] Address feedback abotu optionality, add comments,
 introduce doc API, more tweaks

---
 .../graphqlbackend/codeintel.codenav.graphql  | 247 +++++++++++++++---
 cmd/frontend/graphqlbackend/schema.graphql    |  32 ++-
 2 files changed, 244 insertions(+), 35 deletions(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index b4b2e21967ac..446a583b2b10 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -47,6 +47,103 @@ extend type GitBlob {
     Experimental: This API is likely to change in the future.
     """
     symbolInfo(line: Int!, character: Int!): SymbolInfo
+
+    """
+    If filter.provenance? is not provided, then the API will go through
+    each provenance one by one in the order Precise -> Syntactic -> SearchBased
+    and stop when some data is available.
+    """
+    codeGraphData(filter: CodeGraphDataFilter): CodeGraphData
+}
+
+input CodeGraphDataFilter {
+    provenance: ProvenanceComparator
+}
+
+"""
+Invariants:
+1. forall sym in symbols.
+     (sym.provenance, sym.dataSource) == (scipDocument.provenance, scipDocument.dataSource)
+"""
+# Implementation notes:
+# 1. For precise code nav, we already have a code for inferring
+#    'visible uploads' using the commit graph machinery -> infer best upload
+#    -> get document for that using the given path.
+# 2. Syntactic will be similar to precise
+# 3. Search-based -> return empty document/lists.
+#
+# Ref panel logic:
+# 1. Request code graph data, specifying 'filter' if needed.
+#    Either the scipDocument or occurrences is fine.
+# 2. When a hover is triggered, attempt to look up occurrence in document or list
+#    or some other lookup structure constructed from those.
+# 3a. If one matching occurrence is found, use symbolName + range to call
+#     usagesForSymbol
+# 3b. If zero matching occurrences are found, use only range to call usagesForSymbol.
+# 3c. If more than one matching occurrence is found, use some more complex logic
+#     to offer a symbol picker.
+#     - Problem: For a symbol picker, ideally you'd use SymbolInformation.display_name,
+#       which is stored on SymbolInformation. However, display_name is not supported
+#       today by most SCIP indexers. And we don't have any API for getting the detailed
+#       SCIP SymbolInformation for a bunch of occurrences.
+type CodeGraphData {
+    scipDocument: AssociatedSCIPDocument
+
+    """
+    List of symbols with definitions in this document.
+
+    Invariant:
+    forall sym in symbols.
+        exists occ in occurrences such that
+            sym.name == occ.symbolName && occ.roles.contains(Definition)
+    """
+    # TODO: Should this be paginated to gracefully handle large generated files?
+    symbols: [SymbolInformation!]
+
+    """
+    Occurrences are guaranteed to be sorted by range.
+    """
+    # TODO: Should this be paginated to gracefully handle large generated files?
+    occurrences(filter: OccurrencesFilter): [Occurrence!]
+}
+
+type Occurrence {
+    symbolName: String
+    range: Range!
+    roles: [SymbolRole!]
+    # We can add diagnostics etc. here in the future if needed.
+}
+
+input OccurrencesFilter {
+    rangeFilter: RangeFilter
+    rolesFilter: RolesFilter
+    # We can add a DiagnosticsFilter etc. here in the future if needed.
+}
+
+input RangeFilter {
+    """
+    Zero-based line number, inclusive.
+    """
+    startLine: Int
+    """
+    Zero-based line number, inclusive.
+    """
+    endLine: Int
+}
+
+input RolesFilter {
+    or: [RolesFilter!]
+    equals: SymbolRole
+}
+
+enum SymbolRole {
+    Definition,
+    Reference,
+    """
+    Applicable for forward declarations in languages with header files (C, C++ etc.)
+    as well as standalone signatures in languages with separate interface files (OCaml etc.).
+    """
+    ForwardDefinition,
 }
 
 """
@@ -93,6 +190,8 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
 
     """
     A list of definitions of the symbol under the given document position.
+
+    DEPRECATED: Use usagesForSymbol with a Definition filter instead.
     """
     definitions(
         """
@@ -105,21 +204,6 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
         """
         character: Int!
 
-        """
-        When specified, indicates that this request should be paginated and
-        to fetch results starting at this cursor.
-        A future request can be made for more results by passing in the
-        'LocationConnection.pageInfo.endCursor' that is returned.
-        """
-        after: String
-
-        """
-        When specified, indicates that this request should be paginated and
-        the first N results (relative to the cursor) should be returned. i.e.
-        how many results to return per page.
-        """
-        first: Int
-
         """
         When specified, it filters references by filename.
         """
@@ -128,6 +212,8 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
 
     """
     A list of references of the symbol under the given document position.
+
+    DEPRECATED: Use usagesForSymbol with a Reference filter instead.
     """
     references(
         """
@@ -163,6 +249,8 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
 
     """
     A list of implementations of the symbol under the given document position.
+
+    DEPRECATED: Use usagesForSymbol with Implementations filter instead.
     """
     implementations(
         """
@@ -198,6 +286,8 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
 
     """
     A list of prototypes of the symbol under the given document position.
+
+    DEPRECATED: Use usagesForSymbol with a Supers filter instead.
     """
     prototypes(
         """
@@ -263,44 +353,114 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
 }
 
 extend type Query {
+    """
+    Identify usages for either a semantic symbol, or the symbol(s) implied
+    by a source range.
+
+    Ordering and uniqueness guarantees:
+    1. The usages returned will already be de-duplicated.
+    2. Results for a single file are contiguous.
+    3. Results for a single repository are contiguous.
+
+    Related: See `codeGraphData` on GitBlob.
+    """
     usagesForSymbol(
-        symbol: LookupSCIPSymbol,
         """
-        Non-optional due to implementation limitations
+        Symbol to perform the lookup for.
+
+        If provided, then the start and end position in 'range' are optional.
+
+        If not provided, then the start and end position in 'range' must be set.
+        In that case, the results will be merged across all semantic symbols
+        in the source range specified by [start, end]
+        """
+        symbol: SymbolComparator,
+        """
+        Information about an existing source range for a usage of the symbol.
+
+        TODO: Specify behavior for various cases of overlap between LookupRange
+        and actual occurrence range.
         """
         range: LookupRange!,
+
         filter: UsagesFilter,
-        first: Int,
-        after: String,
+
+        """
+        When specified, indicates that this request should be paginated and
+        the first N results (relative to the cursor) should be returned. i.e.
+        how many results to return per page.
+        """
+        first: Int
+
+        """
+        When specified, indicates that this request should be paginated and
+        to fetch results starting at this cursor.
+        A future request can be made for more results by passing in the
+        'UsageConnection.pageInfo.endCursor' that is returned.
+        """
+        after: String
     ): UsageConnection!
 }
 
-input LookupSCIPSymbol {
+input SymbolComparator {
     name: SymbolNameComparator!
     provenance: ProvenanceComparator!
 }
 
-"""
-In the future, we may want to extend this to allow passing in a suffix or a "symbol pattern".
-"""
+# Implementation note: In the future, we may want to extend this
+# to allow passing in a suffix or a "symbol pattern".
+# See https://github.com/sourcegraph/sourcegraph/issues/59957
 input SymbolNameComparator {
-    eq: String
+    equals: String
 }
 
 input ProvenanceComparator {
-    eq: Provenance
+    equals: Provenance
 }
 
-type SCIPSymbol {
+type SymbolInformation implements WithProvenance {
+    """
+    Symbol name using syntax specified by the SCIP schema.
+    https://github.com/sourcegraph/scip/blob/main/scip.proto#L147-L188
+
+    This value will generally be used in conjunction with
+    the `usagesBySymbol` API.
+    """
     name: String!
+    """
+    Hover documentation for the symbol, in Markdown format.
+
+    The caller must take care to escape any particular strings,
+    such as raw HTML, as necessary.
+
+    The value is null when the hover documentation is not found
+    by 'dataSource'.
+
+    The value is empty when `dataSource` is confident that there
+    is no appropriate hover documentation to display.
+    """
+    documentation: [String!]
+    """
+    Coarse-grained information about the data source.
+    """
     provenance: Provenance!
+    """
+    Opaque fine-grained information describing the data source.
+
+    Provided only for debugging.
+
+    This field should be ignored when checking structural equality.
+    """
+    dataSource: String
 }
 
 input LookupRange {
     """
     Defaults to instance-wide search if the repo is not specified.
 
-    This field is non-optional due to implementation limitations.
+    This field is necessary if LookupSymbol is a repository-local or
+    file-local symbol. It is mandatory for cross-repo symbols
+    due to implementation limitations.
     """
     repo: String!
     """
@@ -310,7 +470,9 @@ input LookupRange {
     """
     Defaults to repo-wide search if the path is not specified.
 
-    This field is non-optional due to implementation limitations.
+    This field is necessary if LookupSymbol is a file-local symbol.
+    It is mandatory for repository-local and cross-repo symbols
+    due to implementation limitations.
     """
     path: String!
     start: LookupPosition
@@ -318,10 +480,23 @@ input LookupRange {
 }
 
 input LookupPosition {
+    """
+    Zero-based count of newline (\n or \r\n) characters before this position.
+    """
     line: Int!
+    """
+    Zero-based UTF-16 code unit offset from preceding newline (\n or \r\n) character.
+    """
     character: Int!
 }
 
+"""
+An empty filter allows all kinds of usages for all paths in all repositories.
+
+However, if the symbol used for lookup is a file-local symbol or a
+repository-local symbol, then usages will automatically be limited to the
+same file or same repository respectively.
+"""
 input UsagesFilter {
     and: [UsagesFilter!]
     or: [UsagesFilter!]
@@ -329,9 +504,6 @@ input UsagesFilter {
     repository: RepositoryFilter
     path: PathFilter
     kind: SymbolUsageKind
-    # TODO: Provide a way of controlling fallback behavior for indexes?
-    # Not super sure how that would work, given that for different repositories,
-    # the hashes will all be different?
 }
 
 input RepositoryFilter {
@@ -343,7 +515,7 @@ input PathFilter {
 }
 
 input StringComparator {
-    eq: String
+    equals: String
 }
 
 enum SymbolUsageKind {
@@ -358,8 +530,17 @@ type UsageConnection {
     pageInfo: PageInfo!
 }
 
+type UsageRange {
+    repo: String!
+    revision: String!
+    range: Range!
+}
+
 type Usage {
-    symbol: SCIPSymbol!
+    symbol: SymbolInformation!
+
+    range: UsageRange
+
     """
     NOTE: Do not pass
     TODO: For this blob, it is generally more useful to get +N/-N lines
diff --git a/cmd/frontend/graphqlbackend/schema.graphql b/cmd/frontend/graphqlbackend/schema.graphql
index 3723ad8c32ff..296ef99d1e46 100755
--- a/cmd/frontend/graphqlbackend/schema.graphql
+++ b/cmd/frontend/graphqlbackend/schema.graphql
@@ -6173,9 +6173,11 @@ type HighlightedFile {
     html: String!
     """
     DEPRECATED: Base64 encoded JSON payload of SCIP with syntax highlighting data.
+    Use document instead.
     """
     lsif: String!
     """
+    The highlighted file with ranges represented using occurrences.
     """
     document: AssociatedSCIPDocument!
     """
@@ -6187,8 +6189,24 @@ type HighlightedFile {
 }
 
 enum Provenance {
+    """
+    Based on a compiler, a type-checker or a similar data source
+    which doesn't have false positives.
+
+    The results are specific to a specific build configuration,
+    such as for a specific OS or CPU, which can matter for
+    codebases having a large amount of platform-specific code.
+    """
     Precise,
+    """
+    Based on a data source that uses an abstract or concrete syntax
+    tree, but without access to reliable type information.
+    """
     Syntactic,
+    """
+    Based on a data source that only does textual analysis, say
+    using regular expressions.
+    """
     SearchBased,
 }
 
@@ -6204,7 +6222,7 @@ interface WithProvenance {
 
     This field should be ignored when checking structural equality.
     """
-    dataSource: String!
+    dataSource: String
 }
 
 type AssociatedSCIPDocument implements WithProvenance {
@@ -6217,8 +6235,18 @@ type AssociatedSCIPDocument implements WithProvenance {
     https://github.com/sourcegraph/scip/blob/main/scip.proto
     """
     data: String!
+    """
+    Coarse-grained information about the data source.
+    """
     provenance: Provenance!
-    dataSource: String!
+    """
+    Opaque fine-grained information describing the data source.
+
+    Provided only for debugging.
+
+    This field should be ignored when checking structural equality.
+    """
+    dataSource: String
 }
 
 """

From eebbd2984d496f26ab0025862766b83986698441 Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 25 Apr 2024 17:36:55 +0800
Subject: [PATCH 06/20] Add more comments, consistent naming, add spaces

---
 .../graphqlbackend/codeintel.codenav.graphql  | 58 +++++++++++++++----
 1 file changed, 46 insertions(+), 12 deletions(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index 446a583b2b10..4c6a8f326a52 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -368,13 +368,14 @@ extend type Query {
         """
         Symbol to perform the lookup for.
 
-        If provided, then the start and end position in 'range' are optional.
+        If provided, then the start and end position in `range` are optional.
 
-        If not provided, then the start and end position in 'range' must be set.
-        In that case, the results will be merged across all semantic symbols
-        in the source range specified by [start, end]
+        If not provided, and if multiple symbols are detected at the same range,
+        the combined results for all symbols will be returned.
+        In that case, `Usage.symbol.name` can be used to group results.
         """
         symbol: SymbolComparator,
+
         """
         Information about an existing source range for a usage of the symbol.
 
@@ -418,6 +419,10 @@ input ProvenanceComparator {
     equals: Provenance
 }
 
+"""
+Type representing useful information about a symbol in code,
+based on 'SymbolInformation' in SCIP.
+"""
 type SymbolInformation implements WithProvenance {
     """
     Symbol name using syntax specified by the SCIP schema.
@@ -427,6 +432,7 @@ type SymbolInformation implements WithProvenance {
     the `usagesBySymbol` API.
     """
     name: String!
+
     """
     Hover documentation for the symbol, in Markdown format.
 
@@ -440,10 +446,12 @@ type SymbolInformation implements WithProvenance {
     is no appropriate hover documentation to display.
     """
     documentation: [String!]
+
     """
     Coarse-grained information about the data source.
     """
     provenance: Provenance!
+
     """
     Opaque fine-grained information describing the data source.
 
@@ -459,14 +467,16 @@ input LookupRange {
     Defaults to instance-wide search if the repo is not specified.
 
     This field is necessary if LookupSymbol is a repository-local or
-    file-local symbol. It is mandatory for cross-repo symbols
+    file-local symbol. It is mandatory for cross-repository symbols
     due to implementation limitations.
     """
-    repo: String!
+    repository: String!
+
     """
     Defaults to HEAD of the default branch if not specified.
     """
     revision: String
+
     """
     Defaults to repo-wide search if the path is not specified.
 
@@ -475,7 +485,9 @@ input LookupRange {
     due to implementation limitations.
     """
     path: String!
+
     start: LookupPosition
+
     end: LookupPosition
 }
 
@@ -484,6 +496,7 @@ input LookupPosition {
     Zero-based count of newline (\n or \r\n) characters before this position.
     """
     line: Int!
+
     """
     Zero-based UTF-16 code unit offset from preceding newline (\n or \r\n) character.
     """
@@ -503,7 +516,7 @@ input UsagesFilter {
     not: UsagesFilter
     repository: RepositoryFilter
     path: PathFilter
-    kind: SymbolUsageKind
+    kind: SymbolUsageKindComparator
 }
 
 input RepositoryFilter {
@@ -518,10 +531,27 @@ input StringComparator {
     equals: String
 }
 
+input SymbolUsageKindComparator {
+    equals: SymbolUsageKind
+}
+
 enum SymbolUsageKind {
     Definition,
+
     Reference,
+
+    """
+    If a type T implements an interface X or inherits from another type X,
+    then searching for Implementations of X will return T as a result
+    if the data source supports it.
+    """
     Implementation,
+
+    """
+    If a type T implements an interface X or inherits from another type X,
+    then searching for Supers of T will return X as a result
+    if the data source supports it.
+    """
     Super,
 }
 
@@ -531,23 +561,27 @@ type UsageConnection {
 }
 
 type UsageRange {
-    repo: String!
+    repository: String!
     revision: String!
+    path: String!
     range: Range!
 }
 
 type Usage {
     symbol: SymbolInformation!
 
+    """
+    Invariant: `range.path` == `blob.path`. The path is made available
+    as part of this type for convenience.
+    """
     range: UsageRange
 
     """
-    NOTE: Do not pass
-    TODO: For this blob, it is generally more useful to get +N/-N lines
-    rather than having to pass a range internally for the contents since
-    the offsets from the start will not be known a-priori.
+    Instead of requesting `blob.content`, it may be more useful
+    to ask for `surroundingBlobContent`.
     """
     blob: GitBlob!
+
     """
     Instead of blob { content }, allows accessing a sub-span of the content
     using relative coordinates from the range of this usage. If linesBefore

From 072bbc056f3015b2e0da6a7d8ec9982114121234 Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 25 Apr 2024 17:38:21 +0800
Subject: [PATCH 07/20] Add filter for provenance

---
 cmd/frontend/graphqlbackend/codeintel.codenav.graphql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index 4c6a8f326a52..83f13b8cb9a8 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -517,6 +517,7 @@ input UsagesFilter {
     repository: RepositoryFilter
     path: PathFilter
     kind: SymbolUsageKindComparator
+    provenance: ProvenanceComparator
 }
 
 input RepositoryFilter {

From 8e947b7c7cedf535b7eb8b112b006598f70b2f5e Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 25 Apr 2024 17:42:35 +0800
Subject: [PATCH 08/20] Rename symbolName -> symbol in Occurrence

---
 cmd/frontend/graphqlbackend/codeintel.codenav.graphql | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index 83f13b8cb9a8..6b5dc1dc4a2b 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -108,7 +108,14 @@ type CodeGraphData {
 }
 
 type Occurrence {
-    symbolName: String
+    """
+    Symbol name using syntax specified by the SCIP schema.
+    https://github.com/sourcegraph/scip/blob/main/scip.proto#L147-L188
+
+    This value will generally be used in conjunction with
+    the `usagesBySymbol` API.
+    """
+    symbol: String
     range: Range!
     roles: [SymbolRole!]
     # We can add diagnostics etc. here in the future if needed.

From 4032028dbefea42fda0001d69d1a53c6dd77bbdb Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 25 Apr 2024 17:52:28 +0800
Subject: [PATCH 09/20] Replace DEPRECATED: comments with @deprecated

---
 .../graphqlbackend/codeintel.codenav.graphql     | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index 6b5dc1dc4a2b..e624fb1a1acf 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -197,8 +197,6 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
 
     """
     A list of definitions of the symbol under the given document position.
-
-    DEPRECATED: Use usagesForSymbol with a Definition filter instead.
     """
     definitions(
         """
@@ -215,12 +213,10 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
         When specified, it filters references by filename.
         """
         filter: String
-    ): LocationConnection!
+    ): LocationConnection! @deprecated(reason: "Superseded by usagesForSymbol; use the Definition filter")
 
     """
     A list of references of the symbol under the given document position.
-
-    DEPRECATED: Use usagesForSymbol with a Reference filter instead.
     """
     references(
         """
@@ -252,12 +248,10 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
         When specified, it filters references by filename.
         """
         filter: String
-    ): LocationConnection!
+    ): LocationConnection! @deprecated(reason: "Superseded by usagesForSymbol; use the Reference filter")
 
     """
     A list of implementations of the symbol under the given document position.
-
-    DEPRECATED: Use usagesForSymbol with Implementations filter instead.
     """
     implementations(
         """
@@ -289,12 +283,10 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
         When specified, it filters implementation by filename.
         """
         filter: String
-    ): LocationConnection!
+    ): LocationConnection! @deprecated(reason: "Superseded by usagesForSymbol; use the Implemention filter")
 
     """
     A list of prototypes of the symbol under the given document position.
-
-    DEPRECATED: Use usagesForSymbol with a Supers filter instead.
     """
     prototypes(
         """
@@ -326,7 +318,7 @@ type GitBlobLSIFData implements TreeEntryLSIFData {
         When specified, it filters prototypes by filename.
         """
         filter: String
-    ): LocationConnection!
+    ): LocationConnection! @deprecated(reason: "Superseded by usagesForSymbol; use the Super filter")
 
     """
     The hover result of the symbol under the given document position.

From 2146fed0c93fad4374217720747276a3d37010cf Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 25 Apr 2024 18:01:38 +0800
Subject: [PATCH 10/20] Tweak comments/replace symbolName->symbol

---
 .../graphqlbackend/codeintel.codenav.graphql  | 22 ++++++++++++-------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index e624fb1a1acf..335f7b836979 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -76,16 +76,17 @@ Invariants:
 # 1. Request code graph data, specifying 'filter' if needed.
 #    Either the scipDocument or occurrences is fine.
 # 2. When a hover is triggered, attempt to look up occurrence in document or list
-#    or some other lookup structure constructed from those.
-# 3a. If one matching occurrence is found, use symbolName + range to call
-#     usagesForSymbol
+#    of occurrences (or some other lookup structure constructed from those)
+#    based on the source range.
+# 3a. If one matching occurrence is found, use Occurrence.symbol and range to call
+#     usagesForSymbol.
 # 3b. If zero matching occurrences are found, use only range to call usagesForSymbol.
 # 3c. If more than one matching occurrence is found, use some more complex logic
 #     to offer a symbol picker.
-#     - Problem: For a symbol picker, ideally you'd use SymbolInformation.display_name,
-#       which is stored on SymbolInformation. However, display_name is not supported
-#       today by most SCIP indexers. And we don't have any API for getting the detailed
-#       SCIP SymbolInformation for a bunch of occurrences.
+#     - Problem: For a symbol picker, ideally you'd use SymbolInformation.display_name
+#       in SCIP. However, display_name is not supported today by most SCIP indexers.
+#       And we don't have any API for getting the detailed SCIP SymbolInformation
+#       for a bunch of occurrences.
 type CodeGraphData {
     scipDocument: AssociatedSCIPDocument
 
@@ -95,7 +96,7 @@ type CodeGraphData {
     Invariant:
     forall sym in symbols.
         exists occ in occurrences such that
-            sym.name == occ.symbolName && occ.roles.contains(Definition)
+            sym.name == occ.symbol && occ.roles.contains(Definition)
     """
     # TODO: Should this be paginated to gracefully handle large generated files?
     symbols: [SymbolInformation!]
@@ -502,6 +503,11 @@ input LookupPosition {
     character: Int!
 }
 
+# NOTE(id: filter-vs-comparator-terminology)
+#
+# 'Filter' is used for complex objects whereas 'Comparator' is used
+# for primitive objects.
+
 """
 An empty filter allows all kinds of usages for all paths in all repositories.
 

From 570798abc97e94e292d36d444923e0ae9fda9474 Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 2 May 2024 17:21:54 +0800
Subject: [PATCH 11/20] Return multiple CodeGraphData due to multiple tool
 possibility

---
 cmd/frontend/graphqlbackend/codeintel.codenav.graphql | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index 335f7b836979..66dff99583c6 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -49,11 +49,18 @@ extend type GitBlob {
     symbolInfo(line: Int!, character: Int!): SymbolInfo
 
     """
+    Return the code graph data associated with this blob.
+
+    If there are multiple tools (i.e. name and version pairs) which
+    have uploaded precise indexes for this blob, then this API will
+    return multiple results even if
+    filter == { provenance: { equals: Precise } }.
+
     If filter.provenance? is not provided, then the API will go through
     each provenance one by one in the order Precise -> Syntactic -> SearchBased
     and stop when some data is available.
     """
-    codeGraphData(filter: CodeGraphDataFilter): CodeGraphData
+    codeGraphData(filter: CodeGraphDataFilter): [CodeGraphData!]
 }
 
 input CodeGraphDataFilter {

From 92692299f84f9e0295efb7d147e562475d3facc1 Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 2 May 2024 17:23:11 +0800
Subject: [PATCH 12/20] Mention toolInfo and build identifiers.

---
 .../graphqlbackend/codeintel.codenav.graphql  | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index 66dff99583c6..45bfc9e4fb72 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -65,6 +65,8 @@ extend type GitBlob {
 
 input CodeGraphDataFilter {
     provenance: ProvenanceComparator
+    # EXTENSION: We can add filters for tool name (if there are overlay-style
+    # indexers) and build identifier (e.g. Linux vs Windows) here.
 }
 
 """
@@ -97,6 +99,20 @@ Invariants:
 type CodeGraphData {
     scipDocument: AssociatedSCIPDocument
 
+    """
+    The commit associated with this code graph data.
+
+    In general, this will be an ancestor of the commit at which code
+    graph data was requested, as code graph data may not be available
+    at the exact commit for the blob.
+    """
+    commit: String!
+
+    """
+    Information about the tool which generated this code graph data
+    """
+    toolInfo: CodeGraphToolInfo
+
     """
     List of symbols with definitions in this document.
 
@@ -113,6 +129,14 @@ type CodeGraphData {
     """
     # TODO: Should this be paginated to gracefully handle large generated files?
     occurrences(filter: OccurrencesFilter): [Occurrence!]
+
+    # EXTENSION: We can add a build identifier here to return different
+    # Documents for (say) Linux vs Windows.
+
+type CodeGraphToolInfo {
+    name: String
+    version: String
+}
 }
 
 type Occurrence {

From 7c4273982fdad5e774af0c87a0e5191b1ed62f81 Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 2 May 2024 17:24:02 +0800
Subject: [PATCH 13/20] Mention file navigation and use common-ish syntax for
 extension ideas

---
 cmd/frontend/graphqlbackend/codeintel.codenav.graphql | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index 45bfc9e4fb72..bdabc9d04deb 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -132,6 +132,8 @@ type CodeGraphData {
 
     # EXTENSION: We can add a build identifier here to return different
     # Documents for (say) Linux vs Windows.
+    # EXTENSION: We can add a fileSymbolInfo field here to allow 'Find usages' for the document.
+}
 
 type CodeGraphToolInfo {
     name: String
@@ -150,13 +152,13 @@ type Occurrence {
     symbol: String
     range: Range!
     roles: [SymbolRole!]
-    # We can add diagnostics etc. here in the future if needed.
+    # EXTENSION: We can add diagnostics etc. here in the future if needed.
 }
 
 input OccurrencesFilter {
     rangeFilter: RangeFilter
     rolesFilter: RolesFilter
-    # We can add a DiagnosticsFilter etc. here in the future if needed.
+    # EXTENSION: We can add a DiagnosticsFilter etc. here in the future if needed.
 }
 
 input RangeFilter {

From e90ab20452587f4d87d4b1acb3b729fc7737230f Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 2 May 2024 17:25:54 +0800
Subject: [PATCH 14/20] Fix naming convention + use pagination for symbols &
 occurrences

---
 .../graphqlbackend/codeintel.codenav.graphql  | 42 ++++++++++++-------
 1 file changed, 28 insertions(+), 14 deletions(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index bdabc9d04deb..cbb95c0f212a 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -87,7 +87,7 @@ Invariants:
 # 2. When a hover is triggered, attempt to look up occurrence in document or list
 #    of occurrences (or some other lookup structure constructed from those)
 #    based on the source range.
-# 3a. If one matching occurrence is found, use Occurrence.symbol and range to call
+# 3a. If one matching occurrence is found, use SCIPOccurrence.symbol and range to call
 #     usagesForSymbol.
 # 3b. If zero matching occurrences are found, use only range to call usagesForSymbol.
 # 3c. If more than one matching occurrence is found, use some more complex logic
@@ -121,14 +121,12 @@ type CodeGraphData {
         exists occ in occurrences such that
             sym.name == occ.symbol && occ.roles.contains(Definition)
     """
-    # TODO: Should this be paginated to gracefully handle large generated files?
-    symbols: [SymbolInformation!]
+    symbols: SymbolInformationConnection
 
     """
     Occurrences are guaranteed to be sorted by range.
     """
-    # TODO: Should this be paginated to gracefully handle large generated files?
-    occurrences(filter: OccurrencesFilter): [Occurrence!]
+    occurrences(filter: SCIPOccurrencesFilter): SCIPOccurrenceConnection
 
     # EXTENSION: We can add a build identifier here to return different
     # Documents for (say) Linux vs Windows.
@@ -139,9 +137,25 @@ type CodeGraphToolInfo {
     name: String
     version: String
 }
+
+type SymbolInformationConnection {
+    nodes: [SymbolInformation!]
+    pageInfo: PageInfo!
+}
+
+type SCIPOccurrenceConnection {
+    """
+    A list of occurrences within a Document
+    """
+    nodes: [SCIPOccurrence!]
+
+    """
+    Pagination information.
+    """
+    pageInfo: PageInfo!
 }
 
-type Occurrence {
+type SCIPOccurrence {
     """
     Symbol name using syntax specified by the SCIP schema.
     https://github.com/sourcegraph/scip/blob/main/scip.proto#L147-L188
@@ -155,7 +169,7 @@ type Occurrence {
     # EXTENSION: We can add diagnostics etc. here in the future if needed.
 }
 
-input OccurrencesFilter {
+input SCIPOccurrencesFilter {
     rangeFilter: RangeFilter
     rolesFilter: RolesFilter
     # EXTENSION: We can add a DiagnosticsFilter etc. here in the future if needed.
@@ -415,7 +429,7 @@ extend type Query {
         TODO: Specify behavior for various cases of overlap between LookupRange
         and actual occurrence range.
         """
-        range: LookupRange!,
+        range: RangeInput!,
 
         filter: UsagesFilter,
 
@@ -495,11 +509,11 @@ type SymbolInformation implements WithProvenance {
     dataSource: String
 }
 
-input LookupRange {
+input RangeInput {
     """
     Defaults to instance-wide search if the repo is not specified.
 
-    This field is necessary if LookupSymbol is a repository-local or
+    This field is necessary if SymbolComparator is a repository-local or
     file-local symbol. It is mandatory for cross-repository symbols
     due to implementation limitations.
     """
@@ -513,18 +527,18 @@ input LookupRange {
     """
     Defaults to repo-wide search if the path is not specified.
 
-    This field is necessary if LookupSymbol is a file-local symbol.
+    This field is necessary if SymbolComparator is a file-local symbol.
     It is mandatory for repository-local and cross-repo symbols
     due to implementation limitations.
     """
     path: String!
 
-    start: LookupPosition
+    start: PositionInput
 
-    end: LookupPosition
+    end: PositionInput
 }
 
-input LookupPosition {
+input PositionInput {
     """
     Zero-based count of newline (\n or \r\n) characters before this position.
     """

From 5f0c62a11edb519e4d5c0055fe39c98e4fbc57fb Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 2 May 2024 17:37:36 +0800
Subject: [PATCH 15/20] Use Connection for SCIP Document data

---
 cmd/frontend/graphqlbackend/schema.graphql | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/cmd/frontend/graphqlbackend/schema.graphql b/cmd/frontend/graphqlbackend/schema.graphql
index 296ef99d1e46..9424a32f8d4e 100755
--- a/cmd/frontend/graphqlbackend/schema.graphql
+++ b/cmd/frontend/graphqlbackend/schema.graphql
@@ -6233,8 +6233,19 @@ type AssociatedSCIPDocument implements WithProvenance {
     pre-generated bindings in the sourcegraph/scip repository or
     manually generating bindings using
     https://github.com/sourcegraph/scip/blob/main/scip.proto
+
+    'first' limits the number of bytes to be fetched. As a consequence,
+    a partial result may not be a well-formed Index. So users of this API
+    must either:
+    1. Use a resilient, streaming parser, that parses as many tag-length-value
+       triplets as possible before requesting more data.
+       An example of streaming parsing can be found here:
+       https://sourcegraph.com/github.com/sourcegraph/scip/-/blob/bindings/go/scip/parse.go
+    OR
+    2. Buffer all the strings, concatenate them, and parse the final result
+       with the provided generated bindings.
     """
-    data: String!
+    data(first: Int, after: String): BytesConnection!
     """
     Coarse-grained information about the data source.
     """
@@ -6249,6 +6260,11 @@ type AssociatedSCIPDocument implements WithProvenance {
     dataSource: String
 }
 
+type BytesConnection {
+    nodes: [String!]
+    hasNextPage: PageInfo!
+}
+
 """
 A file match.
 """

From dbd406886ad2246887c2cbb60a78a1aaad0cd0c3 Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 2 May 2024 17:49:20 +0800
Subject: [PATCH 16/20] Rename Provenance & remove WithProvenance interface

---
 .../graphqlbackend/codeintel.codenav.graphql  | 14 ++++++-------
 cmd/frontend/graphqlbackend/schema.graphql    | 21 +++----------------
 2 files changed, 10 insertions(+), 25 deletions(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index cbb95c0f212a..17a1e80a8e33 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -64,7 +64,7 @@ extend type GitBlob {
 }
 
 input CodeGraphDataFilter {
-    provenance: ProvenanceComparator
+    provenance: CodeGraphDataProvenanceComparator
     # EXTENSION: We can add filters for tool name (if there are overlay-style
     # indexers) and build identifier (e.g. Linux vs Windows) here.
 }
@@ -452,7 +452,7 @@ extend type Query {
 
 input SymbolComparator {
     name: SymbolNameComparator!
-    provenance: ProvenanceComparator!
+    provenance: CodeGraphDataProvenanceComparator!
 }
 
 # Implementation note: In the future, we may want to extend this
@@ -462,15 +462,15 @@ input SymbolNameComparator {
     equals: String
 }
 
-input ProvenanceComparator {
-    equals: Provenance
+input CodeGraphDataProvenanceComparator {
+    equals: CodeGraphDataProvenance
 }
 
 """
 Type representing useful information about a symbol in code,
 based on 'SymbolInformation' in SCIP.
 """
-type SymbolInformation implements WithProvenance {
+type SymbolInformation {
     """
     Symbol name using syntax specified by the SCIP schema.
     https://github.com/sourcegraph/scip/blob/main/scip.proto#L147-L188
@@ -497,7 +497,7 @@ type SymbolInformation implements WithProvenance {
     """
     Coarse-grained information about the data source.
     """
-    provenance: Provenance!
+    provenance: CodeGraphDataProvenance!
 
     """
     Opaque fine-grained information describing the data source.
@@ -569,7 +569,7 @@ input UsagesFilter {
     repository: RepositoryFilter
     path: PathFilter
     kind: SymbolUsageKindComparator
-    provenance: ProvenanceComparator
+    provenance: CodeGraphDataProvenanceComparator
 }
 
 input RepositoryFilter {
diff --git a/cmd/frontend/graphqlbackend/schema.graphql b/cmd/frontend/graphqlbackend/schema.graphql
index 9424a32f8d4e..865a51787426 100755
--- a/cmd/frontend/graphqlbackend/schema.graphql
+++ b/cmd/frontend/graphqlbackend/schema.graphql
@@ -6188,7 +6188,7 @@ type HighlightedFile {
     lineRanges(ranges: [HighlightLineRange!]!): [[String!]!]!
 }
 
-enum Provenance {
+enum CodeGraphDataProvenance {
     """
     Based on a compiler, a type-checker or a similar data source
     which doesn't have false positives.
@@ -6210,22 +6210,7 @@ enum Provenance {
     SearchBased,
 }
 
-interface WithProvenance {
-    """
-    Coarse-grained information about the data source.
-    """
-    provenance: Provenance!
-    """
-    Opaque fine-grained information describing the data source.
-
-    Provided only for debugging.
-
-    This field should be ignored when checking structural equality.
-    """
-    dataSource: String
-}
-
-type AssociatedSCIPDocument implements WithProvenance {
+type AssociatedSCIPDocument {
     """
     Base64 encoded SCIP payload.
 
@@ -6249,7 +6234,7 @@ type AssociatedSCIPDocument implements WithProvenance {
     """
     Coarse-grained information about the data source.
     """
-    provenance: Provenance!
+    provenance: CodeGraphDataProvenance!
     """
     Opaque fine-grained information describing the data source.
 

From e0af435feb99290ad1a819f65ca7ff1dae15e93f Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 2 May 2024 17:56:02 +0800
Subject: [PATCH 17/20] Add comment for symbols vs scipDocument.data.symbols

---
 cmd/frontend/graphqlbackend/codeintel.codenav.graphql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index 17a1e80a8e33..f427292e81a0 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -120,6 +120,8 @@ type CodeGraphData {
     forall sym in symbols.
         exists occ in occurrences such that
             sym.name == occ.symbol && occ.roles.contains(Definition)
+
+    These map 1-1 with the symbols in scipDocument.data
     """
     symbols: SymbolInformationConnection
 

From 653a19727629f3e42b0d98b67d91873e7f932fab Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 2 May 2024 19:59:00 +0800
Subject: [PATCH 18/20] Add filter equivalence and design comment

---
 .../graphqlbackend/codeintel.codenav.graphql  | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index f427292e81a0..cee8c034347b 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -563,7 +563,31 @@ An empty filter allows all kinds of usages for all paths in all repositories.
 However, if the symbol used for lookup is a file-local symbol or a
 repository-local symbol, then usages will automatically be limited to the
 same file or same repository respectively.
+
+Filters are combined monoidally, i.e. all filters are applied. For example,
+
+```
+{and: [u1, u2], or: [u3, u4], not: u5}
+```
+
+would be equivalent to:
+
+```
+{and: [{and: [u1, u2]}, {or: [u3, u4]}, {not: u5}]}
+```
 """
+# Design Rationale:
+#
+# 1. The ref panel did separate queries for 'this repo' vs 'all other repos'
+#    https://sourcegraph.com/github.com/sourcegraph/sourcegraph/-/blob/client/web/src/enterprise/codeintel/searchBased.ts?L105:13-105:23#tab=references
+#    That requires `NOT` (at least for a `repo`).
+# 2. The ref panel has a way to specify paths for filtering. This requires AND.
+#    AND will not be part of the first implementation pass. However,
+#    it would allow fetching less data instead of post-filtering client-side.
+# 3. Currently, the ref panel has a setting for only showing precise results.
+#    Once syntactic is added, if we want to allow the user to specify either
+#    syntactic or precise (but not search-based), we'd need OR.
+#    OR will also not be needed for the first implementation pass.
 input UsagesFilter {
     and: [UsagesFilter!]
     or: [UsagesFilter!]

From e20c0a5fb6dffaceb5c6fa6d1ec2cda0a20a518f Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 2 May 2024 20:05:17 +0800
Subject: [PATCH 19/20] Rename field & make blob: GitBlob optional

---
 cmd/frontend/graphqlbackend/codeintel.codenav.graphql | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index cee8c034347b..4801a195ba25 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -659,7 +659,9 @@ type Usage {
     Instead of requesting `blob.content`, it may be more useful
     to ask for `surroundingBlobContent`.
     """
-    blob: GitBlob!
+    # Left as optional in case we support generated files in the future,
+    # which may not be represented by GitBlob.
+    blob: GitBlob
 
     """
     Instead of blob { content }, allows accessing a sub-span of the content
@@ -667,7 +669,7 @@ type Usage {
     or linesAfter is negative or exceeds the number of available lines,
     the value is interpreted as until the start/end of the file.
     """
-    surroundingBlobContent(surroundingLines: SurroundingLines = {linesBefore: 0, linesAfter: 0}): String!
+    surroundingContent(surroundingLines: SurroundingLines = {linesBefore: 0, linesAfter: 0}): String
 }
 
 input SurroundingLines {

From 38fc9514df4645b316d9ab6266852a5821a1a117 Mon Sep 17 00:00:00 2001
From: Varun Gandhi <varun.gandhi@sourcegraph.com>
Date: Thu, 2 May 2024 20:06:25 +0800
Subject: [PATCH 20/20] Move comment to different field

---
 cmd/frontend/graphqlbackend/codeintel.codenav.graphql | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
index 4801a195ba25..38440807d4b3 100644
--- a/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
+++ b/cmd/frontend/graphqlbackend/codeintel.codenav.graphql
@@ -55,15 +55,16 @@ extend type GitBlob {
     have uploaded precise indexes for this blob, then this API will
     return multiple results even if
     filter == { provenance: { equals: Precise } }.
-
-    If filter.provenance? is not provided, then the API will go through
-    each provenance one by one in the order Precise -> Syntactic -> SearchBased
-    and stop when some data is available.
     """
     codeGraphData(filter: CodeGraphDataFilter): [CodeGraphData!]
 }
 
 input CodeGraphDataFilter {
+    """
+    If provenance is not provided, then the API will go through
+    each provenance one by one in the order Precise -> Syntactic -> SearchBased
+    and stop when some data is available.
+    """
     provenance: CodeGraphDataProvenanceComparator
     # EXTENSION: We can add filters for tool name (if there are overlay-style
     # indexers) and build identifier (e.g. Linux vs Windows) here.