View Source SpiderMan.CommonSpider (spider_man v0.6.3)

A Common Spider what setting functions as callbacks instead of module defined

Usage

SpiderMan.CommonSpider.start(:spider_name, handle_response: fn response, context -> %{} end)

Summary

Types

callback()

@type callback() ::
  {:init, init()}
  | {:handle_response, handle_response()}
  | {:prepare_for_start, prepare_for_start()}
  | {:prepare_for_stop, prepare_for_stop()}
  | {:prepare_for_start_component, prepare_for_start_component()}
  | {:prepare_for_start_component, prepare_for_start_component()}

callbacks()

@type callbacks() :: [callback()]

handle_response()

@type handle_response() :: (SpiderMan.Response.t(), context :: map() ->
                        %{
                          optional(:requests) => [SpiderMan.Request.t()],
                          optional(:items) => [SpiderMan.Item.t()]
                        })

init()

prepare_for_start()

prepare_for_start_component()

@type prepare_for_start_component() :: (SpiderMan.component(),
                                  options :: keyword() | false ->
                                    options :: keyword())

prepare_for_stop()

@type prepare_for_stop() :: (SpiderMan.Engine.state() -> :ok)

prepare_for_stop_component()

@type prepare_for_stop_component() :: (SpiderMan.component(),
                                 options :: keyword() | false ->
                                   :ok)

Functions

check_callbacks_and_merge_settings(callbacks, settings \\ [])

@spec check_callbacks_and_merge_settings(callbacks(), SpiderMan.settings()) ::
  {:ok, SpiderMan.settings()} | {:error, String.t()}

start(spider, callbacks, settings \\ [])