Initial add

dist/index.d.cts

@@ -0,0 +1,3 @@
+import {LRParser} from "@lezer/lr"
+
+export const parser: LRParser

dist/index.d.ts

@@ -0,0 +1,3 @@
+import {LRParser} from "@lezer/lr"
+
+export const parser: LRParser

package.json

@@ -8,7 +8,7 @@
     "import": "./dist/index.es.js",
     "require": "./dist/index.cjs"
   },
-  "module": "dist/index.es.js",
+  "module": "dist/index.js",
   "types": "dist/index.d.ts",
   "author": "Marijn Haverbeke <[email protected]>",
   "license": "MIT",
@@ -30,6 +30,6 @@
     "build": "lezer-generator src/yaml.grammar -o src/parser && rollup -c",
     "build-debug": "lezer-generator src/yaml.grammar --names -o src/parser && rollup -c",
     "prepare": "npm run build",
-    "test": "npm run build && mocha test/test-yaml.js"
+    "test": "mocha test/test-yaml.js"
   }
 }

rollup.config.js

@@ -4,7 +4,7 @@ export default {
   input: "./src/parser.js",
   output: [
     { format: "cjs", file: "./dist/index.cjs" },
-    { format: "es", file: "./dist/index.es.js" }
+    { format: "es", file: "./dist/index.js" }
   ],
   external(id) {
     return !/^[\.\/]/.test(id)

src/tokens.js

@@ -0,0 +1,72 @@
+import {ExternalTokenizer, ContextTracker} from "@lezer/lr"
+import {
+  DirectiveEnd, DocEnd,
+  sequenceStartMark, sequenceContinueMark, blockEnd,
+  BracketL, BraceL, FlowSequence, FlowMapping
+} from "./parser.terms.js"
+
+const type_Top = 0, type_Seq = 1, type_Map = 2, type_Flow = 3
+
+class Context {
+  constructor(parent, depth, type) {
+    this.parent = parent
+    this.depth = depth
+    this.type = type
+    this.hash = (parent ? parent.hash + parent.hash << 8 : 0) + depth + (depth << 4) + type
+  }
+
+  static top = new Context(null, 0, type_Top)
+}
+
+function findColumn(input, pos) {
+  for (let col = 0, p = pos - input.pos - 1;; p--, col++) {
+    let ch = input.peek(p)
+    if (ch == 10 || ch == 13 || ch == -1) return col
+  }
+}
+
+function isSpace(ch) {
+  return ch == 32 || ch == 9 || ch == 10 || ch == 13 || ch < 0
+}
+
+export const indentation = new ContextTracker({
+  start: Context.top,
+  reduce(context, term) {
+    return context.type == type_Flow && (term == FlowSequence || term == FlowMapping) ? context.parent : context
+  },
+  shift(context, term, stack, input) {
+    if (term == sequenceStartMark) return new Context(context, findColumn(input, input.pos), type_Seq)
+    // FIXME mapStartMark
+    if (term == blockEnd) return context.parent
+    if (term == BracketL || term == BraceL) return new Context(context, 0, type_Flow)
+    return context
+  },
+  hash(context) { return context.hash }
+})
+
+export const newlines = new ExternalTokenizer((input, stack) => {
+  if (input.next == -1 && stack.canShift(blockEnd))
+    return input.acceptToken(blockEnd)
+  let prev = input.peek(-1)
+  if ((prev == 10 /* '\n' */ || prev == 13 /* '\r' */ || prev < 0) && stack.context.type != type_Flow) {
+    if (input.next == 45 /* '-' */ && input.peek(1) == 45 && input.peek(2) == 45 && isSpace(input.peek(3)))
+      return input.acceptToken(DirectiveEnd, 3)
+    if (input.next == 46 /* '.' */ && input.peek(1) == 46 && input.peek(2) == 46 && isSpace(input.peek(3)))
+      return input.acceptToken(DocEnd, 3)
+    let depth = 0
+    while (input.next == 32 /* ' ' */) { depth++; input.advance() }
+    if (depth < stack.context.depth &&
+        // Not blank
+        input.next != -1 && input.next != 10 && input.next != 13 && input.next != 35 /* '#' */)
+      input.acceptToken(blockEnd, -depth)
+  }
+}, {contextual: true})
+
+export const sequence = new ExternalTokenizer((input, stack) => {
+  if (input.next == 45 /* '-' */) {
+    input.advance()
+    if (input.next == 10 || input.next == 32 || input.next == 9)
+      input.acceptToken(stack.context.type == type_Seq && stack.context.depth == findColumn(input, input.pos - 1)
+                        ? sequenceContinueMark : sequenceStartMark)
+  }
+}, {contextual: true})

src/yaml.grammar

@@ -0,0 +1,75 @@
+@top Stream { Document? (DocEnd Document?)* }
+
+Document {
+  Directive* DirectiveEnd? element+ |
+  Directive+ DirectiveEnd? element*
+}
+
+element {
+  elementAtom |
+  BlockSequence |
+  BlockMapping |
+  Reference |
+  TypedElement { TypeTag element } |
+  NamedElement { NameTag element }
+}
+
+elementAtom {
+  QuotedLiteral |
+  Literal |
+  FlowSequence |
+  FlowMapping
+}
+
+FlowSequence { "[" (elementAtom ",")* elementAtom? "]" }
+FlowMapping { "{" (elementAtom ":" elementAtom ",")* elementAtom? "}" }
+
+BlockSequence {
+  (sequenceStartMark element?)
+  (sequenceContinueMark element?)*
+  blockEnd
+}
+
+BlockMapping {
+  "@FIXME" //  (("@KEY" | "?" elementAtom?) ":" element?)+ dedent
+}
+
+@skip { whitespace | linebreak | Comment }
+
+@context indentation from "./tokens.js"
+
+@external tokens newlines from "./tokens.js" {
+  blockEnd,
+  DirectiveEnd,
+  DocEnd
+}
+
+@external tokens sequence from "./tokens.js" {
+  sequenceStartMark[@name="-"]
+  sequenceContinueMark[@name="-"]
+}
+
+@tokens {
+  whitespace { $[ \t]+ }
+  linebreak { $[\n\r] }
+  QuotedLiteral {
+    '"' (!["\\] | "\\" _)* '"'? |
+    "'" (!['] | "''")* "'"?
+  }
+  Directive { "%" ![\n]* }
+  Comment { "#" ![\n]* }
+
+  anchorChar { $[!-+\--Z\\^-z|~\u00a0-\ud7ff\ue000-\ufffd\u{10000}-\u{10ffff}] }
+  Reference { "*" anchorChar+ }
+  NameTag { "&" anchorChar+ }
+  TypeTag { "!" (anchorChar* "!")? anchorChar+ }
+
+  safe { ![-?:,\[\]{}#&*!|>'"%@` \t\r\n] }
+
+  Literal { safe ($[ \t]* safe)* } // FIXME
+
+  "["[@export=BracketL] "]"
+  "{"[@export=BraceL] "}"
+}
+
+@detectDelim

test/block-sequence.txt

@@ -0,0 +1,19 @@
+# Simple sequence
+
+- one
+- two
+- three
+
+==>
+
+Stream(Document(BlockSequence(Literal,Literal,Literal)))
+
+# Nested sequence
+
+- - one
+  - two
+- three
+
+==>
+
+Stream(Document(BlockSequence(BlockSequence(Literal,Literal),Literal)))

test/test-yaml.js

@@ -0,0 +1,17 @@
+import {parser} from "../dist/index.js"
+import {fileTests} from "@lezer/generator/dist/test"
+
+import * as fs from "fs"
+import * as path from "path"
+import {fileURLToPath} from "url"
+let caseDir = path.dirname(fileURLToPath(import.meta.url))
+
+for (let file of fs.readdirSync(caseDir)) {
+  if (!/\.txt$/.test(file)) continue
+
+  let name = /^[^\.]*/.exec(file)[0]
+  describe(name, () => {
+    for (let {name, run} of fileTests(fs.readFileSync(path.join(caseDir, file), "utf8"), file))
+      it(name, () => run(parser))
+  })
+}