defmodule Mix.Tasks.LocalizeMf2Treesitter.Sync do @moduledoc """ Sync the vendored grammar + queries from the published [`tree-sitter-mf2`](https://www.npmjs.com/package/tree-sitter-mf2) npm package. This package embeds the grammar source as a C build-time input: * `c_src/grammar/parser.c` + `c_src/grammar/tree_sitter/parser.h` — compiled alongside the tree-sitter runtime into the NIF `.so`. * `priv/queries/*.scm` — loaded at runtime by `Query.load/1`. The canonical source is the npm package, pinned to an exact version at the top of this module (`@tree_sitter_mf2_version`). Bump that string and re-run the task to move to a new grammar release. **Keep the pin in step with `mf2_wasm_editor`'s own sync task** — tree shape is the API boundary between server-side parse (this NIF) and browser-side parse (WASM editor); a version skew can produce different trees for the same input, breaking the canonicalisation round-trip. ## Usage # Fetch from npm at the pinned version and update local files. mix localize_mf2_treesitter.sync # CI check — exit non-zero if any vendored file has drifted # from the pinned version. Does not modify files. mix localize_mf2_treesitter.sync --check ## Offline / local-iteration override If you're iterating on the grammar locally and want this task to read from a sibling checkout rather than hit the network, set `MF2_TREESITTER_DIR`: MF2_TREESITTER_DIR=/path/to/mf2_treesitter mix localize_mf2_treesitter.sync The checkout's layout must match the npm package layout (it does, if you're pointing at a working tree of the [`mf2_treesitter`](https://github.com/elixir-localize/mf2_treesitter) repo). """ use Mix.Task @shortdoc "Sync vendored tree-sitter-mf2 grammar + queries from npm." # Pinned tree-sitter-mf2 version. Bump and re-run the task to pull # a newer grammar. Keep in sync with `mf2_wasm_editor`'s pin. @tree_sitter_mf2_version "0.1.4" # unpkg.com proxies the npm registry and serves individual files # from a package tarball over HTTPS at stable URLs. @cdn_base "https://unpkg.com/tree-sitter-mf2@#{@tree_sitter_mf2_version}" # {source_rel, dest_rel}: # source relative to the tree-sitter-mf2 package root # dest relative to this package's root @files [ {"src/parser.c", "c_src/grammar/parser.c"}, {"src/tree_sitter/parser.h", "c_src/grammar/tree_sitter/parser.h"}, {"queries/highlights.scm", "priv/queries/highlights.scm"} ] @switches [check: :boolean] @impl Mix.Task def run(argv) do {opts, _rest} = OptionParser.parse!(argv, switches: @switches) check? = opts[:check] == true source = resolve_source() package_root = File.cwd!() case source do {:cdn, base} -> Mix.shell().info("[sync] grammar source: #{base}") {:local, path} -> Mix.shell().info("[sync] grammar source: #{path} (local override)") end Mix.shell().info("[sync] package root: #{package_root}") drift = sync_files(source, package_root, @files, check?) cond do check? and drift != [] -> Mix.shell().error("[sync] drift detected in #{length(drift)} file(s):") Enum.each(drift, fn path -> Mix.shell().error(" - #{path}") end) Mix.shell().error("Run `mix localize_mf2_treesitter.sync` to update.") exit({:shutdown, 1}) check? -> Mix.shell().info( "[sync] check: all vendored files match tree-sitter-mf2@#{@tree_sitter_mf2_version}." ) drift == [] -> Mix.shell().info("[sync] already in sync; no files changed.") true -> Mix.shell().info("[sync] updated #{length(drift)} file(s).") Mix.shell().info("[sync] run `mix compile --force` to rebuild the NIF.") end :ok end defp sync_files(source, dst_root, pairs, check?) do Enum.reduce(pairs, [], fn {src_rel, dst_rel}, acc -> bytes = fetch_bytes!(source, src_rel) dst_path = Path.join(dst_root, dst_rel) if changed?(bytes, dst_path) do unless check? do File.mkdir_p!(Path.dirname(dst_path)) File.write!(dst_path, bytes) Mix.shell().info("[sync] wrote #{dst_rel}") end [dst_rel | acc] else acc end end) end defp changed?(new_bytes, dst) do cond do not File.exists?(dst) -> true File.read!(dst) != new_bytes -> true true -> false end end # Fetch raw bytes for one file under the grammar package root. # Dispatches on source kind — local checkout copies from disk, # CDN mode does a verified HTTPS GET. defp fetch_bytes!({:local, dir}, rel) do path = Path.join(dir, rel) unless File.exists?(path) do Mix.raise("missing source file at local override: #{path}") end File.read!(path) end defp fetch_bytes!({:cdn, base}, rel) do url = "#{base}/#{rel}" http_get!(url) end defp http_get!(url) do ensure_http_started() url_charlist = String.to_charlist(url) request = {url_charlist, [{~c"accept", ~c"*/*"}]} case :httpc.request(:get, request, [ssl: ssl_options()], body_format: :binary) do {:ok, {{_http, status, _reason}, _headers, body}} when status in 200..299 -> body {:ok, {{_http, status, reason}, _headers, _body}} -> Mix.raise(""" HTTP #{status} #{reason} from #{url} Check that tree-sitter-mf2@#{@tree_sitter_mf2_version} exists on npm (https://www.npmjs.com/package/tree-sitter-mf2), or set MF2_TREESITTER_DIR to a local checkout to bypass the CDN. """) {:error, reason} -> Mix.raise(""" HTTP request to #{url} failed: #{inspect(reason)} Check your network connection, or set MF2_TREESITTER_DIR to a local checkout to bypass the CDN. """) end end defp ensure_http_started do :ssl.start() :inets.start() end defp ssl_options do [ verify: :verify_peer, cacerts: :public_key.cacerts_get(), customize_hostname_check: [ match_fun: :public_key.pkix_verify_hostname_match_fun(:https) ] ] end # Returns the source to read files from: # {:local, path} when MF2_TREESITTER_DIR is set (local iteration) # {:cdn, base_url} otherwise (the default, reproducible path) defp resolve_source do case System.get_env("MF2_TREESITTER_DIR") do nil -> {:cdn, @cdn_base} "" -> {:cdn, @cdn_base} override -> path = Path.expand(override) unless File.dir?(path) do Mix.raise("MF2_TREESITTER_DIR set but not a directory: #{path}") end {:local, path} end end end