View Source SpiderMan.CommonSpider (spider_man v0.5.1)

A Common Spider what setting functions as callbacks instead of module defined

Usage

SpiderMan.CommonSpider.start(:spider_name, handle_response: fn response, context -> %{} end)

Summary

Types

@type callback() ::
  {:init, init()}
  | {:handle_response, handle_response()}
  | {:prepare_for_start, prepare_for_start()}
  | {:prepare_for_stop, prepare_for_stop()}
  | {:prepare_for_start_component, prepare_for_start_component()}
  | {:prepare_for_start_component, prepare_for_start_component()}
@type callbacks() :: [callback()]
@type handle_response() ::
  (SpiderMan.Response.t(), context :: map() ->
     %{
       optional(:requests) => [SpiderMan.Request.t()],
       optional(:items) => [SpiderMan.Item.t()]
     })
Link to this type

prepare_for_start_component()

View Source
@type prepare_for_start_component() ::
  (SpiderMan.component(), options :: keyword() | false -> options :: keyword())
@type prepare_for_stop() :: (SpiderMan.Engine.state() -> :ok)
Link to this type

prepare_for_stop_component()

View Source
@type prepare_for_stop_component() ::
  (SpiderMan.component(), options :: keyword() | false -> :ok)

Functions

Link to this function

check_callbacks_and_merge_settings(callbacks, settings \\ [])

View Source
@spec check_callbacks_and_merge_settings(callbacks(), SpiderMan.settings()) ::
  {:ok, SpiderMan.settings()} | {:error, String.t()}
Link to this function

start(spider, callbacks, settings \\ [])

View Source