defmodule Mix.Tasks.LocalizeMf2Treesitter.UpdateRuntime do @moduledoc """ Refresh the vendored tree-sitter C runtime from the canonical upstream at [`tree-sitter/tree-sitter`](https://github.com/tree-sitter/tree-sitter). This package embeds the tree-sitter runtime as a single-TU amalgamation under `c_src/runtime/`. That runtime is compiled alongside the generated `c_src/grammar/parser.c` into the NIF `.so`. The runtime's supported ABI version must be ≥ the version used by `parser.c`, or every parse fails at `ts_parser_set_language`. When the grammar sync (`mix localize_mf2_treesitter.sync`) pulls a `parser.c` generated by a newer tree-sitter CLI than the vendored runtime supports, the runtime needs to catch up. This task handles that: it downloads a specific tree-sitter release tarball, extracts the `lib/` tree, and overlays it onto `c_src/runtime/`. The pinned version lives at the top of this module as `@runtime_version`. Bump it and re-run the task to move to a newer runtime release. A good default is to match whatever tree-sitter CLI version was used to regenerate the grammar — check `src/parser.c`'s first-line comment. ## Usage # Download and overlay the pinned runtime version. mix localize_mf2_treesitter.update_runtime # Fail (exit 1) if anything has drifted from the pinned # version; do not modify files. Intended for CI. mix localize_mf2_treesitter.update_runtime --check ## What gets replaced (and what doesn't) Replaced: * `c_src/runtime/include/tree_sitter/*.h` * `c_src/runtime/src/*.c`, `c_src/runtime/src/*.h` * `c_src/runtime/src/portable/`, `unicode/`, `wasm/` subtrees * `c_src/runtime/LICENSE` (tree-sitter's MIT licence) Preserved (hand-maintained): * `c_src/runtime/src/lib.c` — our amalgamation wrapper. Adds `#define _POSIX_C_SOURCE 200112L` and `#include`s each of the runtime's `.c` files. If the upstream adds a new `.c` file, this task warns and prompts the maintainer to extend `lib.c`. ## Network dependency Downloads the release tarball from `https://github.com/tree-sitter/tree-sitter/archive/refs/tags/v.tar.gz` over verified HTTPS. No API keys, no authentication required. """ use Mix.Task @shortdoc "Refresh the vendored tree-sitter runtime from upstream." # Pinned tree-sitter runtime version. Bump and re-run the task to # pull a newer runtime. Match this to the tree-sitter CLI version # used to regenerate `c_src/grammar/parser.c` (check its first line). @runtime_version "0.26.8" @upstream_repo "tree-sitter/tree-sitter" @tarball_url "https://github.com/#{@upstream_repo}/archive/refs/tags/v#{@runtime_version}.tar.gz" # Files from the extracted tarball that we overlay onto c_src/runtime/. # Each entry is `{source_rel, dest_rel}` where source_rel is inside the # extracted `tree-sitter-/` directory and dest_rel is relative # to `c_src/runtime/`. Subtrees are walked recursively by the overlay # function, so we just list the top-level roots here. @subtrees [ {"lib/include/tree_sitter", "include/tree_sitter"}, {"lib/src", "src"} ] # Files we do NOT overwrite during the overlay. `lib.c` is our # hand-maintained amalgamation wrapper. @preserved_files MapSet.new([ "src/lib.c" ]) @top_level_files [ {"LICENSE", "LICENSE"} ] @switches [check: :boolean] @impl Mix.Task def run(argv) do {opts, _rest} = OptionParser.parse!(argv, switches: @switches) check? = opts[:check] == true runtime_dir = runtime_dir() Mix.shell().info("[runtime] source: tree-sitter v#{@runtime_version}") Mix.shell().info("[runtime] destination: #{runtime_dir}") {:ok, work_dir} = fetch_and_extract() try do drift = overlay(work_dir, runtime_dir, check?) missing = check_lib_c(work_dir, runtime_dir) cond do check? and drift != [] -> Mix.shell().error("[runtime] drift detected in #{length(drift)} file(s):") Enum.each(drift, fn path -> Mix.shell().error(" - #{path}") end) Mix.shell().error( "Run `mix localize_mf2_treesitter.update_runtime` to update." ) exit({:shutdown, 1}) check? -> Mix.shell().info( "[runtime] check: all vendored files match tree-sitter v#{@runtime_version}." ) drift == [] -> Mix.shell().info("[runtime] already in sync; no files changed.") true -> Mix.shell().info("[runtime] updated #{length(drift)} file(s).") Mix.shell().info( "[runtime] run `mix compile --force` to rebuild the NIF against the new runtime." ) end warn_about_missing_includes(missing) after File.rm_rf!(work_dir) end :ok end # ── Download + extract ──────────────────────────────────────────── defp fetch_and_extract do ensure_http_started() work_dir = Path.join(System.tmp_dir!(), "ts-runtime-#{System.unique_integer([:positive])}") File.mkdir_p!(work_dir) tarball_path = Path.join(work_dir, "archive.tar.gz") Mix.shell().info("[runtime] downloading #{@tarball_url}") download!(@tarball_url, tarball_path) # :erl_tar with :compressed handles gzip transparently. case :erl_tar.extract(String.to_charlist(tarball_path), [:compressed, {:cwd, String.to_charlist(work_dir)}]) do :ok -> extracted_root = Path.join(work_dir, "tree-sitter-#{@runtime_version}") unless File.dir?(extracted_root) do Mix.raise( "extracted tarball does not contain expected root dir: #{extracted_root}" ) end {:ok, extracted_root} {:error, reason} -> Mix.raise("tarball extraction failed: #{inspect(reason)}") end end defp download!(url, target) do url_charlist = String.to_charlist(url) request = {url_charlist, [{~c"accept", ~c"application/octet-stream"}]} case :httpc.request(:get, request, [ssl: ssl_options(), autoredirect: true], body_format: :binary) do {:ok, {{_http, status, _reason}, _headers, body}} when status in 200..299 -> File.write!(target, body) {:ok, {{_http, status, reason}, _headers, _body}} -> Mix.raise(""" HTTP #{status} #{reason} from #{url} Confirm that v#{@runtime_version} is a valid tree-sitter release tag (https://github.com/#{@upstream_repo}/releases). """) {:error, reason} -> Mix.raise("HTTP request to #{url} failed: #{inspect(reason)}") end end defp ensure_http_started do :ssl.start() :inets.start() end defp ssl_options do [ verify: :verify_peer, cacerts: :public_key.cacerts_get(), customize_hostname_check: [ match_fun: :public_key.pkix_verify_hostname_match_fun(:https) ] ] end # ── Overlay ─────────────────────────────────────────────────────── # Walk the configured subtrees + top-level files and copy every # file that differs, skipping the preserved set. Returns the list of # dest paths that would change (or did change, when not --check). defp overlay(work_dir, runtime_dir, check?) do subtree_drift = for {src_rel, dst_rel} <- @subtrees, file <- list_files_recursive(Path.join(work_dir, src_rel)), do: overlay_file( src_root: Path.join(work_dir, src_rel), dst_root: Path.join(runtime_dir, dst_rel), src_absolute: file, dst_prefix: dst_rel, check?: check? ) top_drift = for {src_rel, dst_rel} <- @top_level_files, do: overlay_single( src: Path.join(work_dir, src_rel), dst: Path.join(runtime_dir, dst_rel), dst_rel: dst_rel, check?: check? ) (subtree_drift ++ top_drift) |> Enum.reject(&is_nil/1) end defp overlay_file(src_root: src_root, dst_root: dst_root, src_absolute: src_absolute, dst_prefix: dst_prefix, check?: check?) do rel = Path.relative_to(src_absolute, src_root) dst_rel_full = Path.join(dst_prefix, rel) if MapSet.member?(@preserved_files, dst_rel_full) do nil else dst = Path.join(dst_root, rel) write_if_changed(src_absolute, dst, dst_rel_full, check?) end end defp overlay_single(src: src, dst: dst, dst_rel: dst_rel, check?: check?) do write_if_changed(src, dst, dst_rel, check?) end defp write_if_changed(src, dst, dst_rel, check?) do new_bytes = File.read!(src) cond do not File.exists?(dst) -> write_or_record(dst, new_bytes, dst_rel, check?) File.read!(dst) != new_bytes -> write_or_record(dst, new_bytes, dst_rel, check?) true -> nil end end defp write_or_record(dst, bytes, dst_rel, check?) do unless check? do File.mkdir_p!(Path.dirname(dst)) File.write!(dst, bytes) Mix.shell().info("[runtime] wrote #{dst_rel}") end dst_rel end defp list_files_recursive(path) do cond do File.regular?(path) -> [path] File.dir?(path) -> path |> File.ls!() |> Enum.flat_map(&list_files_recursive(Path.join(path, &1))) true -> [] end end # ── lib.c amalgamation sanity check ─────────────────────────────── # Verify our preserved lib.c `#include`s every .c file that upstream # ships under lib/src/. If upstream added a new .c, we need to extend # our lib.c or the runtime will link with symbols undefined. Returns # the list of missing includes; caller decides what to print. defp check_lib_c(work_dir, runtime_dir) do upstream_cs = Path.join(work_dir, "lib/src") |> File.ls!() |> Enum.filter(&String.ends_with?(&1, ".c")) |> Enum.reject(&(&1 == "lib.c")) our_lib_c = Path.join(runtime_dir, "src/lib.c") |> File.read!() Enum.reject(upstream_cs, fn c_file -> String.contains?(our_lib_c, "\"./#{c_file}\"") end) end defp warn_about_missing_includes([]), do: :ok defp warn_about_missing_includes(missing) do Mix.shell().info("") Mix.shell().error("[runtime] ⚠ upstream ships .c files not #include'd in src/lib.c:") Enum.each(missing, fn f -> Mix.shell().error(" - #{f}") end) Mix.shell().error(""" Edit c_src/runtime/src/lib.c and add `#include "./.c"` for each missing file, then rebuild. Without this, the amalgamation link step will produce undefined-symbol errors for whatever those new .c files define. """) end defp runtime_dir do Path.join([File.cwd!(), "c_src", "runtime"]) end end