defmodule Noizu.Github do @moduledoc """ Noizu.Github is a library providing a simple wrapper around Github's API calls. It handles various API features such as completions, chat, edit, image generation, image editing, image variation, embeddings, audio transcription, audio translation, file management, and content moderation. ## Configuration To configure the library, you need to set the Github API key and optionally, the Github organization in your application's configuration: config :noizu_github, github_api_key: "your_api_key_here", """ #--------------------------------------- # Global Types #--------------------------------------- @type error_tuple :: {:error, details :: term} # option constraints @type stream_option() :: boolean # Common Type @type stream_options() :: %{ optional(:stream) => boolean() } | Keyword.t() | nil @github_base "https://api.github.com" require Logger def github_base(), do: @github_base def repo_name(options) do options[:repo] || Application.get_env(:noizu_github, NoizuLabs.Github.Config)[:repo] end def repo_owner(options) do options[:owner] || Application.get_env(:noizu_github, NoizuLabs.Github.Config)[:owner] end #------------------------------- # #------------------------------- def generic_stream_provider(callback) do fn event, payload -> case event do {:status, code} -> %{payload | status: code} {:headers, headers} -> %{payload | headers: headers} {:data, data} -> n = String.split(data, "\n\ndata:") |> Enum.map(fn data -> case Jason.decode(data, keys: :atoms) do {:ok, json} -> case json do %{:choices => [%{:delta => %{:content => c}, :finish_reason => _} | _]} -> c _ -> nil end _ -> nil end end) |> Enum.filter(&(&1)) |> Enum.join("") payload = %{payload | message: payload.message <> n} callback.(payload) # Call the provided callback function with the payload _ -> payload end end end #------------------------------- # #------------------------------- @doc """ A helper function to make API calls to the Github API. This function handles both non-stream and stream API calls. ## Parameters - type: The HTTP request method (e.g., :get, :post, :put, :patch, :delete) - url: The full URL for the API endpoint - body: The request body in map format - model: The model to be used for the response processing - options - stream: A boolean value to indicate whether the request should be processed as a stream or not (default: false) - raw: return raw response - response_log_callback: function(finch) callback for request log. - response_log_callback: function(finch, start_ms) callback for response log. ## Returns Returns a tuple {:ok, response} on successful API call, where response is the decoded JSON response in map format. Returns {:error, term} on failure, where term contains error details. ## Example url = "https://api.github.com/v1/completions" body = %{ prompt: "Once upon a time", model: "text-davinci-003", max_tokens: 50, temperature: 0.7 } {:ok, response} = Noizu.Github.api_call(:post, url, body, Noizu.Github.Completion, stream: false) """ def api_call(type, url, body, model, options \\ nil) do stream = options[:stream] || false raw = options[:raw] || false if stream do with {:ok, body} <- body && Jason.encode(body) || {:ok, nil}, {:ok, r = %{status: 200, message: _}} <- api_call_stream(type, url, body, options) do {:ok, r} #apply(model, :from_json, [json]) else error -> Logger.warning("STREAM API ERROR: \n #{inspect error}") error end else with {:ok, body} <- body && Jason.encode(body) || {:ok, nil}, {:ok, %Finch.Response{status: code, body: body, headers: headers} = response} <- api_call_fetch(type, url, body, options), true <- code in 200..299 || {:error, response}, {:ok, json} <- decode_response(body, raw) do unless raw do {:ok, apply(model, :from_json, [json, headers])} else {:ok, apply(model, :from_binary, [json, headers])} end else error -> Logger.warning("API ERROR: \n #{inspect error}") error end end end # Decode a (possibly empty) response body. `204`/`202` and other empty # success bodies decode to `nil`; raw requests pass the body through verbatim. defp decode_response("", _raw), do: {:ok, nil} defp decode_response(nil, _raw), do: {:ok, nil} defp decode_response(body, true), do: {:ok, body} defp decode_response(body, _raw), do: Jason.decode(body, keys: :atoms) #------------------------------- # #------------------------------- def headers(options) do token = options[:token] || Application.get_env(:noizu_github, NoizuLabs.Github.Config)[:api_key] [ {"Accept", "application/vnd.github+json"}, {"Content-Type", "application/json"}, {"Authorization", "Bearer #{token}"}, {"X-GitHub-Api-Version", "2022-11-28"}, ] end def extract_links(headers) do if links = headers[:link] do links |> String.split(",") |> Enum.map(fn link -> [url, rel] = link |> String.trim() |> String.split(";") url = String.replace(url, ~r//, "") rel = String.replace(rel, ~r/rel="/, "") rel = String.replace(rel, ~r/"/, "") rel = String.trim(rel) rel = %{"last" => :last, "next" => :next, "first" => :first, "prev" => :prev}[rel] {rel, String.trim(url)} end) |> Enum.into(%{}) else %{} end end #------------------------------- # #------------------------------- def put_field(body, field, options, default \\ nil) def put_field(body, :stream, options, default) do flag = options[:stream] && true || default Map.put(body, :stream, flag) end def put_field(body, {field_alias, field}, options, default) do if v = options[field_alias] || options[field] || default do Map.put(body, field, v) else body end end def put_field(body, field, options, default) do if v = options[field] || default do Map.put(body, field, v) else body end end #------------------------------- # #------------------------------- def get_field(field, options, default \\ nil) def get_field({field_alias, field}, options, default) do if v = (options[field_alias] || options[field] || default) do "#{field}=#{v}" end end def get_field(field, options, default) do if v = options[field] || default do "#{field}=#{v}" end end #------------------------------- # #------------------------------- defp api_call_fetch(type, url, body, options) do ts = :os.system_time(:millisecond) request = Finch.build(type, url, headers(options), body) |> tap( fn(finch) -> case options[:request_log_callback] do nil -> :nop v when is_function(v, 1) -> v.(finch) {m,f} -> apply(m, f, [finch]) _ -> :nop end end) # |> IO.inspect(label: "API_CALL_FETCH", limit: :infinity, printable_limit: :infinity, pretty: true) request |> Finch.request(Noizu.Github.Finch, [pool_timeout: 600_000, receive_timeout: 600_000]) |> tap(fn(finch) -> case options[:response_log_callback] do nil -> :nop v when is_function(v, 3) -> v.(finch, request, ts) {m,f} -> apply(m, f, [finch, request, ts]) _ -> :nop end end) end #------------------------------- # #------------------------------- defp api_call_stream(type, url, body, options) do callback = options[:stream] raw = options[:raw] ts = :os.system_time(:millisecond) request = Finch.build(type, url, headers(options), body) |> tap( fn(finch) -> case options[:request_log_callback] do nil -> :nop v when is_function(v, 1) -> v.(finch) {m,f} -> apply(m, f, [finch]) _ -> :nop end end) request |> Finch.stream(Noizu.Github.Finch, %{status: nil, raw: raw, message: ""}, callback, [timeout: 600_000, receive_timeout: 600_000]) |> tap(fn(finch) -> case options[:response_log_callback] do nil -> :nop v when is_function(v, 3) -> v.(finch, request, ts) {m,f} -> apply(m, f, [finch, request, ts]) _ -> :nop end end) end @doc """ Eagerly fetch all pages of a paginated endpoint, accumulating `:items`. Takes a function that accepts an options keyword list and returns `{:ok, result}` where `result` has `:items` and `:links` fields (any `Collection.*` or `Raw` result), plus the initial options. Follows `links[:next]` by incrementing the `:page` option until no next link is present. Returns `{:ok, all_items}` or the first `{:error, _}`. ## Example {:ok, all} = Noizu.Github.paginate( &Noizu.Github.Api.Issues.list_for_repo/1, state: "open", per_page: 100 ) """ @spec paginate((keyword -> {:ok, term} | {:error, term}), keyword) :: {:ok, list} | {:error, term} def paginate(fetcher, options \\ []) do do_paginate(fetcher, options, 1, []) end defp do_paginate(fetcher, options, page, acc) do opts = Keyword.put(options, :page, page) case fetcher.(opts) do {:ok, %{items: items, links: links}} -> acc = acc ++ items if links[:next] do do_paginate(fetcher, options, page + 1, acc) else {:ok, acc} end {:ok, %{data: _} = raw} -> {:ok, acc ++ [raw]} {:error, _} = error -> error end end @doc """ Return a lazy `Stream` that yields one `{:ok, result}` per page. Each element is the full page result (with `:items`, `:links`, etc.). The stream terminates when there is no `:next` link or when the fetcher returns an error (the error tuple is yielded as the final element). ## Example Noizu.Github.stream_pages( &Noizu.Github.Api.Issues.list_for_repo/1, state: "open", per_page: 100 ) |> Enum.flat_map(fn {:ok, %{items: items}} -> items {:error, _} -> [] end) """ @spec stream_pages((keyword -> {:ok, term} | {:error, term}), keyword) :: Enumerable.t() def stream_pages(fetcher, options \\ []) do Stream.resource( fn -> 1 end, fn :halt -> {:halt, :done} page -> opts = Keyword.put(options, :page, page) case fetcher.(opts) do {:ok, %{links: links} = result} -> next = if links[:next], do: page + 1, else: :halt {[{:ok, result}], next} {:error, _} = error -> {[error], :halt} end end, fn _ -> :ok end ) end end