%% Copyright 2026 Benoit Chesneau %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at %% %% http://www.apache.org/licenses/LICENSE-2.0 %% %% Unless required by applicable law or agreed to in writing, software %% distributed under the License is distributed on an "AS IS" BASIS, %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. %%% @doc Low-level NIF wrapper for Python integration. %%% %%% This module provides the direct NIF interface. Most users should use %%% the higher-level `py' module instead. %%% %%% @private -module(py_nif). -export([ init/0, init/1, finalize/0, worker_new/0, worker_new/1, worker_destroy/1, worker_call/5, worker_call/6, worker_eval/3, worker_eval/4, worker_exec/2, worker_next/2, import_module/2, get_attr/3, version/0, memory_stats/0, gc/0, gc/1, tracemalloc_start/0, tracemalloc_start/1, tracemalloc_stop/0, set_callback_handler/2, send_callback_response/2, resume_callback/2, %% Async workers async_worker_new/0, async_worker_destroy/1, async_call/6, async_gather/3, async_stream/6, %% Sub-interpreters (Python 3.12+) subinterp_supported/0, subinterp_worker_new/0, subinterp_worker_destroy/1, subinterp_call/5, parallel_execute/2, %% Execution mode info execution_mode/0, num_executors/0, %% Thread worker support (ThreadPoolExecutor) thread_worker_set_coordinator/1, thread_worker_write/2, thread_worker_signal_ready/1 ]). -on_load(load_nif/0). -define(NIF_STUB, erlang:nif_error(nif_not_loaded)). %%% ============================================================================ %%% NIF Loading %%% ============================================================================ load_nif() -> PrivDir = case code:priv_dir(erlang_python) of {error, bad_name} -> %% Fallback for development case code:which(?MODULE) of Filename when is_list(Filename) -> filename:join([filename:dirname(Filename), "..", "priv"]); _ -> "priv" end; Dir -> Dir end, NifPath = filename:join(PrivDir, "py_nif"), erlang:load_nif(NifPath, 0). %%% ============================================================================ %%% Initialization %%% ============================================================================ %% @doc Initialize the Python interpreter. %% Must be called before any other functions. %% Usually called automatically by the application. -spec init() -> ok | {error, term()}. init() -> init(#{}). %% @doc Initialize with options. %% Options: %% python_home => string() - Python installation directory %% python_path => [string()] - Additional module search paths %% isolated => boolean() - Run in isolated mode (default: false) -spec init(map()) -> ok | {error, term()}. init(_Opts) -> ?NIF_STUB. %% @doc Finalize the Python interpreter. %% Call this when shutting down. After this, no Python calls can be made. -spec finalize() -> ok. finalize() -> ?NIF_STUB. %%% ============================================================================ %%% Worker Management %%% ============================================================================ %% @doc Create a new Python worker context. %% Returns an opaque reference to be used with other worker functions. -spec worker_new() -> {ok, reference()} | {error, term()}. worker_new() -> worker_new(#{}). %% @doc Create a worker with options. %% Options: %% use_subinterpreter => boolean() - Use a separate sub-interpreter (Python 3.12+) -spec worker_new(map()) -> {ok, reference()} | {error, term()}. worker_new(_Opts) -> ?NIF_STUB. %% @doc Destroy a worker context. -spec worker_destroy(reference()) -> ok. worker_destroy(_WorkerRef) -> ?NIF_STUB. %% @doc Call a Python function from a worker. %% This is a dirty NIF that acquires the GIL. %% May return {suspended, ...} if Python calls erlang.call() (reentrant callback). -spec worker_call(reference(), binary(), binary(), list(), map()) -> {ok, term()} | {error, term()} | {suspended, term(), reference(), {binary(), term()}}. worker_call(_WorkerRef, _Module, _Func, _Args, _Kwargs) -> ?NIF_STUB. %% @doc Call a Python function from a worker with timeout. %% May return {suspended, ...} if Python calls erlang.call() (reentrant callback). -spec worker_call(reference(), binary(), binary(), list(), map(), non_neg_integer()) -> {ok, term()} | {error, term()} | {suspended, term(), reference(), {binary(), term()}}. worker_call(_WorkerRef, _Module, _Func, _Args, _Kwargs, _TimeoutMs) -> ?NIF_STUB. %% @doc Evaluate a Python expression in a worker. %% May return {suspended, ...} if Python calls erlang.call() (reentrant callback). -spec worker_eval(reference(), binary(), map()) -> {ok, term()} | {error, term()} | {suspended, term(), reference(), {binary(), term()}}. worker_eval(_WorkerRef, _Code, _Locals) -> ?NIF_STUB. %% @doc Evaluate a Python expression in a worker with timeout. %% May return {suspended, ...} if Python calls erlang.call() (reentrant callback). -spec worker_eval(reference(), binary(), map(), non_neg_integer()) -> {ok, term()} | {error, term()} | {suspended, term(), reference(), {binary(), term()}}. worker_eval(_WorkerRef, _Code, _Locals, _TimeoutMs) -> ?NIF_STUB. %% @doc Execute Python statements in a worker. -spec worker_exec(reference(), binary()) -> ok | {error, term()}. worker_exec(_WorkerRef, _Code) -> ?NIF_STUB. %% @doc Get next item from a generator/iterator. %% Returns {ok, Value} | {error, stop_iteration} | {error, Error} -spec worker_next(reference(), reference()) -> {ok, term()} | {error, term()}. worker_next(_WorkerRef, _GeneratorRef) -> ?NIF_STUB. %%% ============================================================================ %%% Module Operations %%% ============================================================================ %% @doc Import a Python module in a worker context. -spec import_module(reference(), binary()) -> {ok, reference()} | {error, term()}. import_module(_WorkerRef, _ModuleName) -> ?NIF_STUB. %% @doc Get an attribute from a Python object. -spec get_attr(reference(), reference(), binary()) -> {ok, term()} | {error, term()}. get_attr(_WorkerRef, _ObjRef, _AttrName) -> ?NIF_STUB. %%% ============================================================================ %%% Info %%% ============================================================================ %% @doc Get Python version info. -spec version() -> {ok, binary()} | {error, term()}. version() -> ?NIF_STUB. %%% ============================================================================ %%% Memory and GC %%% ============================================================================ %% @doc Get Python memory statistics. %% Returns a map with gc_stats, gc_count, gc_threshold, and optionally %% traced_memory_current and traced_memory_peak if tracemalloc is enabled. -spec memory_stats() -> {ok, map()} | {error, term()}. memory_stats() -> ?NIF_STUB. %% @doc Force Python garbage collection. %% Returns the number of unreachable objects collected. -spec gc() -> {ok, integer()} | {error, term()}. gc() -> ?NIF_STUB. %% @doc Force garbage collection of a specific generation. %% Generation: 0, 1, or 2 (default 2 = full collection). -spec gc(0..2) -> {ok, integer()} | {error, term()}. gc(_Generation) -> ?NIF_STUB. %% @doc Start memory tracing with tracemalloc. %% This allows tracking memory allocations. -spec tracemalloc_start() -> ok | {error, term()}. tracemalloc_start() -> ?NIF_STUB. %% @doc Start memory tracing with specified number of frames. -spec tracemalloc_start(pos_integer()) -> ok | {error, term()}. tracemalloc_start(_NFrame) -> ?NIF_STUB. %% @doc Stop memory tracing. -spec tracemalloc_stop() -> ok | {error, term()}. tracemalloc_stop() -> ?NIF_STUB. %%% ============================================================================ %%% Callback Support %%% ============================================================================ %% @doc Set callback handler process for a worker. %% Returns {ok, Fd} where Fd is the file descriptor for sending responses. -spec set_callback_handler(reference(), pid()) -> {ok, integer()} | {error, term()}. set_callback_handler(_WorkerRef, _HandlerPid) -> ?NIF_STUB. %% @doc Send a callback response to a worker via file descriptor. -spec send_callback_response(integer(), binary()) -> ok | {error, term()}. send_callback_response(_Fd, _Response) -> ?NIF_STUB. %% @doc Resume a suspended Python callback with the result. %% StateRef is the reference returned in the {suspended, ...} tuple. %% Result is the callback result as a binary (status byte + data). %% Returns {ok, FinalResult}, {error, Reason}, or another {suspended, ...} for nested callbacks. -spec resume_callback(reference(), binary()) -> {ok, term()} | {error, term()} | {suspended, term(), reference(), {binary(), term()}}. resume_callback(_StateRef, _Result) -> ?NIF_STUB. %%% ============================================================================ %%% Async Worker Support %%% ============================================================================ %% @doc Create a new async worker with background event loop. %% Returns an opaque reference to be used with async functions. -spec async_worker_new() -> {ok, reference()} | {error, term()}. async_worker_new() -> ?NIF_STUB. %% @doc Destroy an async worker. -spec async_worker_destroy(reference()) -> ok. async_worker_destroy(_WorkerRef) -> ?NIF_STUB. %% @doc Submit an async call to the event loop. %% Args: AsyncWorkerRef, Module, Func, Args, Kwargs, CallerPid %% Returns: {ok, AsyncId} | {ok, {immediate, Result}} | {error, term()} -spec async_call(reference(), binary(), binary(), list(), map(), pid()) -> {ok, non_neg_integer() | {immediate, term()}} | {error, term()}. async_call(_WorkerRef, _Module, _Func, _Args, _Kwargs, _CallerPid) -> ?NIF_STUB. %% @doc Execute multiple async calls concurrently using asyncio.gather. %% Args: AsyncWorkerRef, CallsList (list of {Module, Func, Args}), CallerPid %% Returns: {ok, AsyncId} | {ok, {immediate, Results}} | {error, term()} -spec async_gather(reference(), [{binary(), binary(), list()}], pid()) -> {ok, non_neg_integer() | {immediate, list()}} | {error, term()}. async_gather(_WorkerRef, _Calls, _CallerPid) -> ?NIF_STUB. %% @doc Stream from an async generator. %% Args: AsyncWorkerRef, Module, Func, Args, Kwargs, CallerPid %% Returns: {ok, AsyncId} | {error, term()} -spec async_stream(reference(), binary(), binary(), list(), map(), pid()) -> {ok, non_neg_integer()} | {error, term()}. async_stream(_WorkerRef, _Module, _Func, _Args, _Kwargs, _CallerPid) -> ?NIF_STUB. %%% ============================================================================ %%% Sub-interpreter Support (Python 3.12+) %%% ============================================================================ %% @doc Check if sub-interpreters with per-interpreter GIL are supported. %% Returns true on Python 3.12+, false otherwise. -spec subinterp_supported() -> boolean(). subinterp_supported() -> ?NIF_STUB. %% @doc Create a new sub-interpreter worker with its own GIL. %% Returns an opaque reference to be used with subinterp functions. -spec subinterp_worker_new() -> {ok, reference()} | {error, term()}. subinterp_worker_new() -> ?NIF_STUB. %% @doc Destroy a sub-interpreter worker. -spec subinterp_worker_destroy(reference()) -> ok | {error, term()}. subinterp_worker_destroy(_WorkerRef) -> ?NIF_STUB. %% @doc Call a Python function in a sub-interpreter. %% Args: WorkerRef, Module (binary), Func (binary), Args (list), Kwargs (map) -spec subinterp_call(reference(), binary(), binary(), list(), map()) -> {ok, term()} | {error, term()}. subinterp_call(_WorkerRef, _Module, _Func, _Args, _Kwargs) -> ?NIF_STUB. %% @doc Execute multiple calls in parallel across sub-interpreters. %% Args: WorkerRefs (list of refs), Calls (list of {Module, Func, Args}) %% Returns: List of results (one per call) -spec parallel_execute([reference()], [{binary(), binary(), list()}]) -> {ok, list()} | {error, term()}. parallel_execute(_WorkerRefs, _Calls) -> ?NIF_STUB. %%% ============================================================================ %%% Execution Mode Info %%% ============================================================================ %% @doc Get the current execution mode. %% Returns one of: free_threaded | subinterp | multi_executor %% - free_threaded: Python 3.13+ with no GIL (Py_GIL_DISABLED) %% - subinterp: Python 3.12+ with per-interpreter GIL %% - multi_executor: Traditional Python with N executor threads -spec execution_mode() -> free_threaded | subinterp | multi_executor. execution_mode() -> ?NIF_STUB. %% @doc Get the number of executor threads. %% For multi_executor mode, this is the number of executor threads. %% For other modes, returns 1. -spec num_executors() -> pos_integer(). num_executors() -> ?NIF_STUB. %%% ============================================================================ %%% Thread Worker Support (ThreadPoolExecutor) %%% ============================================================================ %% @doc Set the thread worker coordinator process. %% This process receives spawn and callback messages from Python threads %% spawned via ThreadPoolExecutor. -spec thread_worker_set_coordinator(pid()) -> ok | {error, term()}. thread_worker_set_coordinator(_Pid) -> ?NIF_STUB. %% @doc Write a callback response to a thread worker's pipe. %% Fd is the write end of the response pipe. %% Response is the result binary (status byte + python repr). -spec thread_worker_write(integer(), binary()) -> ok | {error, term()}. thread_worker_write(_Fd, _Response) -> ?NIF_STUB. %% @doc Signal that a thread worker handler is ready. %% Writes a zero-length response to the pipe to indicate readiness. -spec thread_worker_signal_ready(integer()) -> ok | {error, term()}. thread_worker_signal_ready(_Fd) -> ?NIF_STUB.