replace peg parser with iterative approach

skillls/parsing/iterative.py

@@ -0,0 +1,133 @@
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from logging import getLogger
+from typing import NamedTuple, Self
+
+from lsprotocol.types import Diagnostic, DiagnosticSeverity, Position, Range
+
+logger = getLogger(__name__)
+
+
+class Pair(NamedTuple):
+    start: str
+    end: str
+
+
+class SyntaxPair(Enum):
+    Paren = Pair("(", ")")
+    Square = Pair("[", "]")
+
+    @classmethod
+    def by_start_elem(cls, start: str) -> Self:
+        for option in cls:
+            if option.value[0] == start:
+                return option
+
+        raise ValueError(f"`{start}` not a valid start character")
+
+    @classmethod
+    def by_end_elem(cls, end: str) -> Self:
+        for option in cls:
+            if option.value[1] == end:
+                return option
+
+        raise ValueError(f"`{end}` not a valid end character")
+
+
+def char_range(line: int, char: int) -> Range:
+    return Range(Position(line, char), Position(line, char + 1))
+
+
+def pair_mismatch(line: int, char: int, msg: str) -> Diagnostic:
+    return Diagnostic(
+        char_range(line, char),
+        msg,
+        severity=DiagnosticSeverity.Error,
+    )
+
+
+class StackElement(NamedTuple):
+    range: Range
+    elem: SyntaxPair
+
+
+@dataclass()
+class IterativeParser:
+    _stack: list[StackElement] = field(default_factory=list)
+
+    def peek(self) -> StackElement:
+        return self._stack[-1]
+
+    def pop(self) -> StackElement:
+        return self._stack.pop()
+
+    def push(self, pair: StackElement) -> None:
+        return self._stack.append(pair)
+
+    def __call__(self, raw: list[str]) -> list[Diagnostic]:
+        in_string = False
+        errs = []
+        for line, raw_line in enumerate(raw):
+            for char, raw_char in enumerate(raw_line):
+                match raw_char:
+                    case ";":
+                        if not in_string:
+                            break
+                    case '"':
+                        in_string = not in_string
+                    case "(" | "[":
+                        if not in_string:
+                            self.push(
+                                StackElement(
+                                    char_range(line, char),
+                                    SyntaxPair.by_start_elem(raw_char),
+                                )
+                            )
+                    case "]" | ")":
+                        if not in_string:
+                            if not self._stack:
+                                errs.append(
+                                    pair_mismatch(
+                                        line, char, f"one {raw_char} too much"
+                                    )
+                                )
+                                continue
+
+                            expected = SyntaxPair.by_end_elem(raw_char)
+                            elem = self._stack.pop()
+                            if elem.elem == expected:
+                                continue
+
+                            if self._stack and self._stack[-1].elem == expected:
+                                errs.append(
+                                    pair_mismatch(
+                                        line, char, f"unclosed {elem.elem.value.start}"
+                                    )
+                                )
+                                self._stack.pop()
+                                self._stack.append(elem)
+                            else:
+                                errs.append(
+                                    pair_mismatch(
+                                        line, char, f"one {raw_char} too much"
+                                    )
+                                )
+                                self._stack.append(elem)
+
+        for rest in self._stack:
+            errs.append(
+                Diagnostic(
+                    rest.range,
+                    f"unclosed {rest.elem.value.start}",
+                    severity=DiagnosticSeverity.Error,
+                )
+            )
+
+        self._stack = []
+
+        return errs
+
+
+if __name__ == "__main__":
+    p = IterativeParser()
+    print(p(["((([]]))"]))

skillls/parsing/location.py

@@ -14,13 +14,14 @@ class Locator:
         counter = 0
         line = 0
         for ix, raw_line in enumerate(self.raw):
-            if counter + len(raw_line) > index:
+            if counter + len(raw_line) + 1 > index:
                 line = ix
                 break
             else:
-                counter += len(raw_line)
+                counter += len(raw_line) + 1
 
-        return Position(line, index - counter)
+        print(counter, line)
+        return Position(line + 1, index - counter + 1)
 
     @overload
     def locate(self, index: int) -> Position:
@@ -34,6 +35,7 @@ class Locator:
         if isinstance(index, int):
             return self._locate_pos(index)
 
+        print(index.start, index.end)
         start = self._locate_pos(index.start)
         end = self._locate_pos(index.end)
 

skillls/parsing/tokenize.py

@@ -1,5 +1,6 @@
+from collections.abc import Iterable, Iterator
 from typing import Any, Sequence
-from lsprotocol.types import Range
+from lsprotocol.types import DocumentSymbol, Range, SymbolKind
 from parsimonious import ParseError
 from dataclasses import dataclass
 
@@ -28,21 +29,25 @@ class List(BaseToken):
     value: list[BaseToken]
 
 
+def flatten(xs: Iterable) -> Iterator[Any]:
+    for x in xs:
+        if isinstance(x, Iterable) and not isinstance(x, (str, bytes)):
+            yield from flatten(x)
+        else:
+            yield x
+
+
 @dataclass
 class SkillVisitor(NodeVisitor):
     locator: Locator
 
     def visit_skill(self, _: Node, visited_children: Sequence[Any]) -> list[BaseToken]:
-        children = []
-        for childlist in visited_children:
-            for child in childlist:
-                if isinstance(child, BaseToken):
-                    children.append(child)
+        return list(flatten(visited_children))
 
-        return children
-
-    def visit_TOKEN(self, node: Node, _: Any) -> Token:
-        return Token(self.locator.locate(node), node.text)
+    def visit_TOKEN(self, node: Node, _: Any) -> DocumentSymbol:
+        r = self.locator.locate(node)
+        print(r)
+        return DocumentSymbol(node.text, SymbolKind.Property, r, r)
 
     def visit_LITERAL(self, node: Node, visited_children: list[None | Node]) -> Literal:
         value, *_ = visited_children
@@ -103,7 +108,7 @@ class SkillVisitor(NodeVisitor):
         return List(self.locator.locate(node), children)
 
     def visit_inline_assign(self, node: Node, visited_children: Sequence[Any]):
-        print(node)
+        return visited_children or node
 
     def generic_visit(
         self, node: Node, visited_children: Sequence[Any]