SpiderMan.CommonSpider (spider_man v0.4.0) View Source

A Common Spider what setting functions as callbacks instead of module defined

Usage

SpiderMan.CommonSpider.start(:spider_name, handle_response: fn response, context -> %{} end)

Link to this section Summary

Link to this section Types

Specs

callback() ::
  {:init, init()}
  | {:handle_response, handle_response()}
  | {:prepare_for_start, prepare_for_start()}
  | {:prepare_for_stop, prepare_for_stop()}
  | {:prepare_for_start_component, prepare_for_start_component()}
  | {:prepare_for_start_component, prepare_for_start_component()}

Specs

callbacks() :: [callback()]

Specs

handle_response() ::
  (SpiderMan.Response.t(), context :: map() ->
     %{
       optional(:requests) => [SpiderMan.Request.t()],
       optional(:items) => [SpiderMan.Item.t()]
     })

Specs

Specs

Link to this type

prepare_for_start_component()

View Source

Specs

prepare_for_start_component() ::
  (SpiderMan.component(), options :: keyword() | false -> options :: keyword())

Specs

prepare_for_stop() :: (SpiderMan.Engine.state() -> :ok)
Link to this type

prepare_for_stop_component()

View Source

Specs

prepare_for_stop_component() ::
  (SpiderMan.component(), options :: keyword() | false -> :ok)

Link to this section Functions

Link to this function

check_callbacks_and_merge_settings(callbacks, settings \\ [])

View Source

Specs

check_callbacks_and_merge_settings(callbacks(), SpiderMan.settings()) ::
  {:ok, SpiderMan.settings()} | {:error, String.t()}
Link to this function

start(spider, callbacks, settings \\ [])

View Source

Specs