View Source Musing with CodeGen
Mix.install([:grammar])
Local setup
To use this livebook within a local clone the project, use the following setup, and launch the livebook from it's directory.
Mix.install(
[
{:grammar, path: Path.join(__DIR__, ".."), env: :dev}
],
lockfile: :grammar
)
Defining a grammar with CodeGen.Clause
Let's define target_module_name
as holding the name of the module we want to define as a specialized parser module.
This binding will be used all along this livebook when it comes to refer to the module name.
target_module_name = Pouet
Now define the clauses that will constitute the grammar.
alias Grammar.CodeGen
grammar_clauses = [
Grammar.CodeGen.Clause.new(:foo_bar, quote do [:foo] end, Code.string_to_quoted!(~S/[what] = params; "Found foo #{what} " <> <<0x1F92A::utf8>>/), false),
Grammar.CodeGen.Clause.new(:foo_bar, quote do [:bar] end, Code.string_to_quoted!(~S/[what] = params; "Found bar #{what} " <> <<0x1F92A::utf8>>/), false),
# Here we use Code.string_to_quoted!/1 to ensure the context in quotes is `nil` as it would be during compilation.
# Simply quoting would result in quoted expression in the context of `Elixir` as we are here evaluating the code (and not compiling it).
#
# Another option in our current context of evaluating things, is to explicitly state that params should be NOT hygienized
# e.g. Grammar.CodeGen.Clause.new(:foo, quote do ["foo"] end, quote do [foo] = var!(params); "#{foo} " <> <<0x1F600::utf8>> <> " !" end, false)
Grammar.CodeGen.Clause.new(:foo, quote do ["foo"] end, Code.string_to_quoted!(~S/[foo] = params; "#{foo} " <> <<0x1F601::utf8>> <> " !"/), false),
Grammar.CodeGen.Clause.new(:bar, quote do ["bar"] end, Code.string_to_quoted!(~S/[bar] = params; "#{bar} " <> <<0x1F601::utf8>> <> " !"/), false)
]
Generating and using a grammar
grammar_definition_ast = Grammar.CodeGen.build_grammar(target_module_name, grammar_clauses)
output the generated code in its Elixir form
grammar_definition_ast
|> Macro.to_string()
|> IO.puts()
Manually define the callback module that fits the grammar and make some tests
quote do
defmodule unquote(target_module_name) do
def clause_0_foo_bar([what]), do: "foo #{what}"
def clause_1_foo_bar([what]), do: "bar #{what}"
def clause_0_foo(["foo"]), do: "foo"
def clause_0_bar(["bar"]), do: "bar"
end
end
|> Code.eval_quoted()
{grammar, _bindings} = Code.eval_quoted(grammar_definition_ast)
Grammar.loop(grammar, Grammar.Tokenizer.new("foo")) |> IO.inspect()
Grammar.loop(grammar, Grammar.Tokenizer.new("bar")) |> IO.inspect()
The module we just define manually does NOT export the parse/1
function unlike the generated modules.
try do
target_module_name.parse("coucou")
rescue
e in UndefinedFunctionError -> e
end
Use CodeGen to produce the callback module
Generate the code of the callback functions refered to in the grammar clauses.
callback_functions_ast = Grammar.CodeGen.build_rule_body_functions(grammar_clauses)
output the generated code in its Elixir form
callback_functions_ast
|> Macro.to_string()
|> IO.puts()
(re)Define the callback module using generated function bodies
{{:module, module_name, code, _exported_funs}, _bindings} =
quote do
defmodule unquote(target_module_name) do
# Use var!/1 to avoid the compiler sanitize `code`
# as its value in passed through bindings
@grammar unquote(grammar_definition_ast)
unquote_splicing(callback_functions_ast)
def parse(input) do
tokenizer = Grammar.Tokenizer.new(input)
Grammar.loop(@grammar, tokenizer)
end
end
end
|> Code.eval_quoted()
module_name.parse("foo")
module_name.parse("bar")