skill-ls/skillls/parsing/tokenize.py

117 lines
3.3 KiB
Python

from collections.abc import Iterable, Iterator
from typing import Any, Sequence
from lsprotocol.types import DocumentSymbol, Range, SymbolKind
from parsimonious import ParseError
from dataclasses import dataclass
from parsimonious.nodes import Node, NodeVisitor
from .location import Locator
@dataclass(frozen=True)
class BaseToken:
range: Range
@dataclass(frozen=True)
class Literal(BaseToken):
value: str | float | bool
@dataclass(frozen=True)
class Token(BaseToken):
value: str
@dataclass(frozen=True)
class List(BaseToken):
value: list[BaseToken]
def flatten(xs: Iterable) -> Iterator[Any]:
for x in xs:
if isinstance(x, Iterable) and not isinstance(x, (str, bytes)):
yield from flatten(x)
else:
yield x
@dataclass
class SkillVisitor(NodeVisitor):
locator: Locator
def visit_skill(self, _: Node, visited_children: Sequence[Any]) -> list[BaseToken]:
return list(flatten(visited_children))
def visit_TOKEN(self, node: Node, _: Any) -> DocumentSymbol:
r = self.locator.locate(node)
print(r)
return DocumentSymbol(node.text, SymbolKind.Property, r, r)
def visit_LITERAL(self, node: Node, visited_children: list[None | Node]) -> Literal:
value, *_ = visited_children
if value:
match value.expr_name:
case "L_t":
return Literal(self.locator.locate(node), True)
case "L_nil":
return Literal(self.locator.locate(node), False)
case "L_num":
return Literal(self.locator.locate(node), float(value.text))
case "L_string":
return Literal(self.locator.locate(node), value.text)
case _:
pass
raise ParseError("something went wrong during literal parsing")
def visit_listraw(
self, node: Node, visited_children: list[list[list[Any]]]
) -> List:
rest = visited_children[2]
children = []
for child in rest:
for part in child:
if isinstance(part, BaseToken):
children.append(part)
return List(self.locator.locate(node), children)
def visit_listc(self, node: Node, visited_children: list[list[list[Any]]]) -> List:
rest = ([[visited_children[0]]], visited_children[2])
children = []
for child_list in rest:
for child in child_list:
for part in child:
if isinstance(part, BaseToken):
children.append(part)
return List(self.locator.locate(node), children)
def visit_listskill(
self, node: Node, visited_children: list[list[list[Any]]]
) -> List:
rest = visited_children[1]
children = []
for child in rest:
for part in child:
if isinstance(part, BaseToken):
children.append(part)
return List(self.locator.locate(node), children)
def visit_inline_assign(self, node: Node, visited_children: Sequence[Any]):
return visited_children or node
def generic_visit(
self, node: Node, visited_children: Sequence[Any]
) -> Node | Sequence[None | Node]:
return visited_children or node