#!/usr/bin/env escript %%% @doc Erlang Concurrency example - Call Erlang from Python with async. %%% %%% This example demonstrates: %%% - Registering Erlang functions callable from Python %%% - Spawning Erlang processes from Python to achieve parallelism %%% - Comparing sequential vs parallel execution performance %%% - Leveraging Erlang's lightweight processes from Python %%% %%% Key insight: Parallelism happens on the Erlang side! %%% Python calls Erlang with a batch of work, and Erlang spawns %%% lightweight processes to handle each item concurrently. %%% %%% Prerequisites: %%% 1. Build the project: rebar3 compile %%% %%% Run from project root: %%% escript examples/erlang_concurrency.erl -mode(compile). main(_Args) -> setup_paths(), {ok, _} = application:ensure_all_started(erlang_python), io:format("~n=== Erlang Concurrency from Python Example ===~n~n"), %% Add examples directory to Python path ExamplesDir = examples_dir(), ok = add_to_python_path(ExamplesDir), %% Register Erlang functions that Python can call io:format("Registering Erlang functions...~n"), register_functions(), %% Run the demonstrations demo_basic_callbacks(), demo_parallel_execution(), demo_erlang_processes(), demo_distributed_work(), io:format("~n=== Done ===~n~n"), cleanup(). setup_paths() -> ScriptDir = filename:dirname(escript:script_name()), ProjectRoot = filename:dirname(ScriptDir), EbinDir = filename:join([ProjectRoot, "_build", "default", "lib", "erlang_python", "ebin"]), true = code:add_pathz(EbinDir). examples_dir() -> ScriptDir = filename:dirname(escript:script_name()), ProjectRoot = filename:dirname(ScriptDir), list_to_binary(filename:join(ProjectRoot, "examples")). add_to_python_path(Dir) -> {ok, _} = py:eval(<<"(__import__('sys').path.insert(0, path) if path not in __import__('sys').path else None, True)[1]">>, #{path => Dir}), ok. cleanup() -> %% Unregister all functions lists:foreach(fun(Name) -> py:unregister_function(Name) end, [slow_compute, fast_compute, spawn_workers, parallel_fetch, parallel_map, fib, prime_check, process_item, echo]), ok = application:stop(erlang_python). %%% ============================================================================ %%% Register Erlang Functions for Python %%% ============================================================================ register_functions() -> %% Simple echo function py:register_function(echo, fun([Msg]) -> Msg end), %% Simulated slow computation (sleeps to simulate I/O or heavy work) py:register_function(slow_compute, fun([N]) -> timer:sleep(100), % 100ms delay N * N end), %% Fast computation (no delay) py:register_function(fast_compute, fun([N]) -> N * N end), %% Fibonacci calculation (CPU-bound) py:register_function(fib, fun([N]) -> fib(N) end), %% Prime check (CPU-bound) py:register_function(prime_check, fun([N]) -> is_prime(N) end), %% CORE FUNCTION: Parallel map using Erlang processes %% This is the key function that enables Python to use Erlang's concurrency py:register_function(parallel_map, fun([FuncName, Items]) -> parallel_map_impl(FuncName, Items) end), %% Process an item with spawned Erlang process py:register_function(process_item, fun([Item]) -> Parent = self(), Ref = make_ref(), spawn(fun() -> %% Simulate processing timer:sleep(50), Result = process_data(Item), Parent ! {Ref, Result} end), receive {Ref, Result} -> Result after 5000 -> {error, timeout} end end), %% Spawn multiple Erlang workers and collect results py:register_function(spawn_workers, fun([Tasks]) -> Parent = self(), Refs = lists:map(fun(Task) -> Ref = make_ref(), spawn(fun() -> Result = execute_task(Task), Parent ! {Ref, Result} end), Ref end, Tasks), %% Collect all results lists:map(fun(Ref) -> receive {Ref, Result} -> Result after 5000 -> {error, timeout} end end, Refs) end), %% Parallel fetch simulation using Erlang processes py:register_function(parallel_fetch, fun([Urls]) -> Parent = self(), Refs = lists:map(fun(Url) -> Ref = make_ref(), spawn(fun() -> %% Simulate HTTP fetch with variable latency Latency = rand:uniform(100) + 50, timer:sleep(Latency), Result = #{url => Url, status => 200, latency_ms => Latency}, Parent ! {Ref, Result} end), Ref end, Urls), lists:map(fun(Ref) -> receive {Ref, Result} -> Result after 5000 -> {error, timeout} end end, Refs) end), io:format(" Registered: echo, slow_compute, fast_compute, fib, prime_check~n"), io:format(" Registered: parallel_map, process_item, spawn_workers, parallel_fetch~n"), ok. %% Implementation of parallel_map: spawns one process per item parallel_map_impl(FuncName, Items) when is_binary(FuncName) -> parallel_map_impl(binary_to_atom(FuncName, utf8), Items); parallel_map_impl(FuncName, Items) -> Parent = self(), %% Spawn a process for each item RefsAndItems = lists:map(fun(Item) -> Ref = make_ref(), spawn(fun() -> %% Execute the named function Result = execute_function(FuncName, Item), Parent ! {Ref, Result} end), {Ref, Item} end, Items), %% Collect results in order lists:map(fun({Ref, _Item}) -> receive {Ref, Result} -> Result after 5000 -> {error, timeout} end end, RefsAndItems). %% Execute a registered function by name execute_function(slow_compute, N) -> timer:sleep(100), N * N; execute_function(fast_compute, N) -> N * N; execute_function(fib, N) -> fib(N); execute_function(prime_check, N) -> is_prime(N); execute_function(process_item, Item) -> timer:sleep(50), process_data(Item); execute_function(Name, Arg) -> %% Fallback: try to call registered function case py_callback:lookup(atom_to_binary(Name, utf8)) of {ok, Fun} when is_function(Fun, 1) -> Fun([Arg]); {ok, {Module, Function}} -> apply(Module, Function, [[Arg]]); {error, not_found} -> {error, {unknown_function, Name}} end. %%% ============================================================================ %%% Demonstrations %%% ============================================================================ demo_basic_callbacks() -> io:format("~n--- Demo 1: Basic Erlang Callbacks ---~n"), %% Test echo {ok, Echo} = py:eval(<<"__import__('erlang_concurrency').call_erlang('echo', 'Hello from Python!')">>), io:format(" Echo result: ~s~n", [Echo]), %% Test computation {ok, Square} = py:eval(<<"__import__('erlang_concurrency').call_erlang('fast_compute', 7)">>), io:format(" fast_compute(7) = ~p~n", [Square]), %% Test Fibonacci {ok, Fib10} = py:eval(<<"__import__('erlang_concurrency').call_erlang('fib', 10)">>), io:format(" fib(10) = ~p~n", [Fib10]), ok. demo_parallel_execution() -> io:format("~n--- Demo 2: Sequential vs Parallel Execution ---~n"), %% Prepare test data {ok, NumbersPy} = py:eval(<<"list(range(1, 11))">>), io:format(" Processing 10 items with slow_compute (100ms each)...~n"), %% Sequential execution (from Python, calling Erlang one at a time) io:format("~n Sequential execution (10 x 100ms = ~~1 second expected):~n"), {ok, {SeqResults, SeqTime}} = py:eval( <<"__import__('erlang_concurrency').demo_sequential('slow_compute', items)">>, #{items => NumbersPy} ), io:format(" Results: ~p~n", [SeqResults]), io:format(" Time: ~.2f seconds~n", [SeqTime]), %% Parallel execution (Python calls Erlang once, Erlang spawns 10 processes) io:format("~n Parallel execution (10 processes, ~~0.1 second expected):~n"), {ok, {ParResults, ParTime}} = py:eval( <<"__import__('erlang_concurrency').demo_parallel('slow_compute', items)">>, #{items => NumbersPy} ), io:format(" Results: ~p~n", [ParResults]), io:format(" Time: ~.2f seconds~n", [ParTime]), Speedup = SeqTime / max(ParTime, 0.001), io:format("~n Speedup: ~.1fx faster with Erlang parallel processes!~n", [Speedup]), ok. demo_erlang_processes() -> io:format("~n--- Demo 3: Spawning Erlang Processes from Python ---~n"), %% Process items using Erlang processes spawned in parallel Items = [<<"task_a">>, <<"task_b">>, <<"task_c">>, <<"task_d">>, <<"task_e">>], io:format(" Processing ~p items via parallel Erlang processes...~n", [length(Items)]), StartTime = erlang:monotonic_time(millisecond), {ok, Results} = py:eval( <<"__import__('erlang_concurrency').parallel_map('process_item', items)">>, #{items => Items} ), EndTime = erlang:monotonic_time(millisecond), io:format(" Completed ~p items in ~p ms (each takes 50ms, parallel!)~n", [length(Items), EndTime - StartTime]), io:format(" Results:~n"), lists:foreach(fun(R) -> io:format(" ~p~n", [R]) end, Results), ok. demo_distributed_work() -> io:format("~n--- Demo 4: Distributed Work with Erlang Workers ---~n"), %% Use spawn_workers to demonstrate Erlang process parallelism Tasks = [ #{type => <<"compute">>, value => 100}, #{type => <<"compute">>, value => 200}, #{type => <<"io">>, value => <<"fetch_data">>}, #{type => <<"compute">>, value => 300}, #{type => <<"io">>, value => <<"process_file">>} ], io:format(" Submitting ~p tasks to Erlang worker pool...~n", [length(Tasks)]), StartTime = erlang:monotonic_time(millisecond), {ok, Results} = py:eval( <<"__import__('erlang_concurrency').call_erlang('spawn_workers', tasks)">>, #{tasks => Tasks} ), EndTime = erlang:monotonic_time(millisecond), io:format(" Completed in ~p ms~n", [EndTime - StartTime]), io:format(" Results:~n"), lists:foreach(fun(R) -> io:format(" ~p~n", [R]) end, Results), %% Demonstrate parallel_fetch io:format("~n Simulating parallel HTTP fetches via Erlang processes...~n"), Urls = [ <<"https://api.example.com/users">>, <<"https://api.example.com/posts">>, <<"https://api.example.com/comments">>, <<"https://api.example.com/albums">>, <<"https://api.example.com/photos">> ], StartTime2 = erlang:monotonic_time(millisecond), {ok, FetchResults} = py:eval( <<"__import__('erlang_concurrency').batch_fetch(urls)">>, #{urls => Urls} ), EndTime2 = erlang:monotonic_time(millisecond), %% Calculate total latency if sequential TotalLatency = lists:sum([maps:get(<<"latency_ms">>, R, 0) || R <- FetchResults]), io:format(" Fetched ~p URLs in ~p ms (would be ~p ms sequential):~n", [length(Urls), EndTime2 - StartTime2, TotalLatency]), lists:foreach(fun(R) -> io:format(" ~s: ~p ms~n", [maps:get(<<"url">>, R), maps:get(<<"latency_ms">>, R)]) end, FetchResults), ok. %%% ============================================================================ %%% Helper Functions %%% ============================================================================ fib(0) -> 0; fib(1) -> 1; fib(N) when N > 1 -> fib(N-1) + fib(N-2). is_prime(N) when N < 2 -> false; is_prime(2) -> true; is_prime(N) when N rem 2 =:= 0 -> false; is_prime(N) -> is_prime(N, 3). is_prime(N, I) when I * I > N -> true; is_prime(N, I) when N rem I =:= 0 -> false; is_prime(N, I) -> is_prime(N, I + 2). process_data(Item) when is_binary(Item) -> %% Simulate data processing #{ original => Item, processed => <>, timestamp => erlang:system_time(millisecond) }; process_data(Item) -> #{original => Item, processed => Item}. execute_task(#{<<"type">> := <<"compute">>, <<"value">> := V}) -> timer:sleep(50), % Simulate computation #{result => V * 2, type => compute}; execute_task(#{<<"type">> := <<"io">>, <<"value">> := V}) -> timer:sleep(rand:uniform(100)), % Simulate I/O with variable latency #{result => V, type => io}; execute_task(Task) -> #{result => Task, type => unknown}.