defmodule LlmsTxt do @moduledoc """ Reads a project's Hex dependencies from its lock file and turns each one's fetched hexdocs into an `llms.txt` file. This is the data layer behind the `mix docs.llms` task: it locates the packages, resolves the local hexdocs cache, writes one `llms.txt` per package and prunes documentation that no longer matches a current dependency. """ alias Mix.Dep.Lock alias LlmsTxt.Converter @doc """ Returns the `{app, version}` tuples for every Hex dependency in the lock file. Git and path dependencies are skipped because they have no Hex version and no documentation on HexDocs. """ def hex_packages do Lock.read() |> Stream.filter(fn {_app, lock} -> is_tuple(lock) and elem(lock, 0) == :hex end) |> Enum.map(fn {app, lock} -> {app, elem(lock, 2)} end) end @doc """ Returns the base directory of the local hexdocs cache. """ def hex_docs_base do hex_home = System.get_env("HEX_HOME") || Path.join(System.user_home!(), ".hex") Path.join([hex_home, "docs", "hexpm"]) end @doc """ Converts the fetched docs for a single package into a `llms.txt` file. Returns `{:ok, path}` with the written file or `{:error, :docs_not_found}` when no docs were fetched for the given package and version. """ def generate_package_llms(name, version, docs_base, output_dir) do package_dir = Path.join([docs_base, to_string(name), to_string(version)]) if File.dir?(package_dir) do package_output = Path.join([output_dir, to_string(name), to_string(version)]) File.mkdir_p!(package_output) output_file = Path.join(package_output, "llms.txt") File.write!(output_file, Converter.build_markdown(name, version, package_dir)) {:ok, output_file} else {:error, :docs_not_found} end end @doc """ Removes documentation that no longer matches a current dependency. Keeps only the `/` directory for each package in `packages`, deleting outdated version directories and packages that are no longer dependencies. """ def prune_stale_docs(output_dir, packages) do current_versions = packages |> Stream.map(fn {name, version} -> Path.join([output_dir, to_string(name), to_string(version)]) end) |> MapSet.new() output_dir |> Path.join("*/*") |> Path.wildcard() |> Stream.filter(&File.dir?/1) |> Stream.reject(&MapSet.member?(current_versions, &1)) |> Enum.each(&File.rm_rf!/1) remove_empty_package_dirs(output_dir) end defp remove_empty_package_dirs(output_dir) do output_dir |> Path.join("*") |> Path.wildcard() |> Stream.filter(&File.dir?/1) |> Stream.filter(&(File.ls!(&1) == [])) |> Enum.each(&File.rmdir!/1) end end