Everywhere: Rewrite the parser, again!

Cargo.toml

@@ -8,6 +8,7 @@ edition = "2024"
 
 [profile.dev.package]
 insta.opt-level = 3
+similar.opt-level = 3
 
 [workspace.metadata.crane]
 name = "pom"

pom-lexer/src/error.rs

@@ -1,15 +1,19 @@
 use pom_utils::span::Span;
 
+/// A lexer error with its [`ErrorKind`] and source [`Span`].
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub struct Error {
     pub kind: ErrorKind,
     pub span: Span,
 }
 
+/// The kind of lexer error.
 #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
 pub enum ErrorKind {
+    /// Character not recognized by any lexer rule.
     #[default]
     UnknownToken,
 }
 
+/// Alias for a collection of lexer errors.
 pub type Errors = Vec<Error>;

pom-lexer/src/lib.rs

@@ -7,6 +7,10 @@ mod lexer;
 #[cfg(test)]
 mod tests;
 
+/// Tokenizes `src` into a list of tokens and any lexer errors.
+///
+/// Unrecognized characters become [`TokenKind::Invalid`](token::TokenKind::Invalid)
+/// tokens rather than halting, so the output always covers the full input.
 pub fn lex(src: &str) -> (Tokens, Errors) {
     Lexer::new(src).lex()
 }

pom-lexer/src/snapshots/errors.snap

@@ -1,6 +1,6 @@
 ---
 source: pom-lexer/src/tests.rs
-expression: "lex(\"55 ^ $ 47\")"
+expression: "lex(\"55 ^ 47\")"
 ---
 (
     [
@@ -20,20 +20,11 @@ expression: "lex(\"55 ^ $ 47\")"
                 end: 4,
             },
         },
-        Token {
-            kind: Invalid(
-                UnknownToken,
-            ),
-            span: Span {
-                start: 5,
-                end: 6,
-            },
-        },
         Token {
             kind: Int,
             span: Span {
-                start: 7,
-                end: 9,
+                start: 5,
+                end: 7,
             },
         },
         Token {
@@ -52,12 +43,5 @@ expression: "lex(\"55 ^ $ 47\")"
                 end: 4,
             },
         },
-        Error {
-            kind: UnknownToken,
-            span: Span {
-                start: 5,
-                end: 6,
-            },
-        },
     ],
 )

pom-lexer/src/snapshots/sigils.snap

@@ -0,0 +1,37 @@
+---
+source: pom-lexer/src/tests.rs
+expression: "lex(\"$ @ ->\")"
+---
+(
+    [
+        Token {
+            kind: Dollar,
+            span: Span {
+                start: 0,
+                end: 1,
+            },
+        },
+        Token {
+            kind: At,
+            span: Span {
+                start: 2,
+                end: 3,
+            },
+        },
+        Token {
+            kind: Arrow,
+            span: Span {
+                start: 4,
+                end: 6,
+            },
+        },
+        Token {
+            kind: Eof,
+            span: Span {
+                start: 4294967295,
+                end: 4294967295,
+            },
+        },
+    ],
+    [],
+)

pom-lexer/src/tests.rs

@@ -33,5 +33,10 @@ fn operators() {
 
 #[test]
 fn errors() {
-    snap!(lex("55 ^ $ 47"));
+    snap!(lex("55 ^ 47"));
+}
+
+#[test]
+fn sigils() {
+    snap!(lex("$ @ ->"));
 }

pom-lexer/src/token.rs

@@ -21,25 +21,31 @@ impl Token {
 #[derive(Logos, Debug, Clone, Copy, PartialEq, Eq)]
 #[logos(skip r"[ \t\r\n\f]+", error = ErrorKind)]
 pub enum TokenKind {
+    #[token(r"->")]
+    Arrow,
+
     #[token(r"true")]
     #[token(r"false")]
     Bool,
 
+    #[token(r"@")]
+    At,
+
     #[token(r":")]
     Colon,
 
     #[token(r",")]
     Comma,
 
+    #[token(r"$")]
+    Dollar,
+
     #[token(r"=")]
     Equal,
 
     #[regex(r"[-+]?[0-9]+\.[0-9]+")]
     Float,
 
-    #[token(r"fn")]
-    Fn,
-
     #[regex(r"[a-zA-Z_][a-zA-Z0-9_]*")]
     Ident,
 
@@ -78,11 +84,14 @@ pub enum TokenKind {
 
     Invalid(ErrorKind),
 
+    /// Synthetic end-of-input marker (not produced by logos).
     Eof,
 }
 
+/// Alias for a sequence of tokens produced by the lexer.
 pub type Tokens = Vec<Token>;
 
+/// Maps a `Result<Token, E>` or `Option<Token>` to its `.kind`.
 pub trait TokenExt {
     type Output;
 
@@ -92,6 +101,7 @@ pub trait TokenExt {
 impl<E> TokenExt for Result<Token, E> {
     type Output = Result<TokenKind, E>;
 
+    #[inline]
     fn kind(self) -> Self::Output {
         self.map(|token| token.kind)
     }
@@ -100,6 +110,7 @@ impl<E> TokenExt for Result<Token, E> {
 impl TokenExt for Option<Token> {
     type Output = Option<TokenKind>;
 
+    #[inline]
     fn kind(self) -> Self::Output {
         self.map(|token| token.kind)
     }

pom-parser/src/ast.rs

@@ -5,6 +5,10 @@ use crate::ast::{expr::Expr, stmt::Stmt};
 pub mod expr;
 pub mod stmt;
 
+/// Root AST container.
+///
+/// `items` holds the top-level statements (by id). All statement and expression
+/// nodes are owned by the `stmts` and `exprs` arenas respectively.
 #[derive(Debug, Default, PartialEq)]
 pub struct Ast {
     pub items: Vec<Id<Stmt>>,

pom-parser/src/ast/expr.rs

@@ -1,50 +1,136 @@
 use pom_utils::{arena::Id, span::Span};
 
-use crate::{ast::stmt::Stmt, error::Error};
+use crate::{
+    ast::stmt::{Bind, Stmt},
+    error::Error,
+};
 
+/// An expression node: [`ExprKind`] paired with its source [`Span`].
 #[derive(Debug, PartialEq)]
 pub struct Expr {
     pub kind: ExprKind,
     pub span: Span,
 }
 
+/// The kind of an expression node.
 #[derive(Debug, PartialEq)]
 pub enum ExprKind {
+    /// Function type, right-associative.
+    ///
+    /// Examples:
+    /// - `(i32) -> i32`
+    FnType { lhs: Id<Expr>, rhs: Id<Expr> },
+
+    /// Binary arithmetic operation.
+    ///
+    /// Examples:
+    /// - `1 + 2`
+    /// - `a * b`
     Binary {
         lhs: Id<Expr>,
         op: BinaryOp,
         rhs: Id<Expr>,
     },
 
+    /// Brace-delimited block of statements.
+    ///
+    /// The last expression in a block does not require a trailing semicolon,
+    /// which is what allows blocks to double as precedence grouping
+    /// (`{1 + 2} * 3`). An empty block `{}` represents the unit value; this
+    /// is disambiguated in sema.
+    ///
+    /// Examples:
+    /// - `{ a := 5; a + 1 }`
+    /// - `{1 + 2} * 3`
+    /// - `{}`
     Block(Vec<Id<Stmt>>),
 
+    /// Boolean literal.
+    ///
+    /// Examples:
+    /// - `true`
+    /// - `false`
+    Bool(bool),
+
+    /// Function call, supports chaining.
+    ///
+    /// Examples:
+    /// - `f(1, 2)`
+    /// - `f(1)(2)`
     Call {
         callable: Id<Expr>,
         args: Vec<Id<Expr>>,
     },
 
-    Ident,
+    /// Floating-point literal.
+    ///
+    /// Examples:
+    /// - `3.14`
+    /// - `-0.5`
+    Float(f64),
 
-    Literal(Literal),
+    /// Function expression. The body extends as far right as possible.
+    ///
+    /// Examples:
+    /// - `@(x: i32) -> i32 x`
+    /// - `$(x) x`
+    Fn {
+        kind: FnKind,
+        params: Vec<Bind>,
+        ret: Option<Id<Expr>>,
+        body: Id<Expr>,
+    },
+
+    /// Named reference, resolved later in sema.
+    ///
+    /// Carries no data; the name is recovered via `span.text(src)`.
+    ///
+    /// Examples:
+    /// - `foo`
+    /// - `i32`
+    Ident,
 
-    Paren(Id<Expr>),
+    /// Integer literal.
+    ///
+    /// Examples:
+    /// - `42`
+    /// - `-1`
+    Int(i64),
 
-    Tuple(Vec<Id<Expr>>),
+    /// Parenthesized structure of possibly-named fields.
+    ///
+    /// Uses [`Bind`] because parenthesized expressions are
+    /// syntactically ambiguous between types and values — a `Bind` with only
+    /// `lhs` set covers both cases.
+    ///
+    /// Examples:
+    /// - `(a, b)`
+    /// - `(x: i32, y: f32)`
+    /// - `(expr)`
+    Tuple(Vec<Bind>),
 
+    /// Placeholder for a malformed expression during error recovery.
     Invalid(Id<Error>),
 }
 
+/// Distinguishes captureless (`@`) from capturing (`$`) function expressions.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FnKind {
+    /// Captures its runtime environment (`$`).
+    Closure,
+    /// Can only capture compile-time data (`@`).
+    Static,
+}
+
+/// Binary arithmetic operator.
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum BinaryOp {
+    /// `+`
     Add,
+    /// `-`
     Sub,
+    /// `*`
     Mul,
+    /// `/`
     Div,
 }
-
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum Literal {
-    Bool(bool),
-    Int(i64),
-    Float(f64),
-}