Testing Guide - SnmpKit

This guide covers testing strategies, simulated devices, and best practices for testing SNMP applications with SnmpKit.

Table of Contents

Overview

Testing SNMP applications presents unique challenges:

  • External Dependencies - Real SNMP devices may not be available
  • Network Conditions - Timeouts, packet loss, and latency variations
  • Device State - SNMP values change over time
  • Scale Testing - Testing with many devices and large data sets
  • Error Conditions - Simulating various failure modes

SnmpKit addresses these challenges through:

  • Realistic Device Simulation - Simulated SNMP agents for testing
  • Comprehensive Test Helpers - Utilities for common testing patterns
  • Async Test Support - Efficient testing of concurrent operations
  • Performance Benchmarking - Built-in tools for performance testing

Test Setup

Basic Test Module Setup

defmodule MyApp.SNMPTest do
  use ExUnit.Case, async: true
  
  alias SnmpKit.{SNMP, MIB, Sim}
  
  # Test configuration
  @test_community "public"
  @test_timeout 5_000
  
  setup_all do
    # Start any required services
    {:ok, _} = SNMP.start_engine()
    :ok
  end
  
  setup do
    # Per-test setup
    %{
      target: "127.0.0.1",
      community: @test_community,
      timeout: @test_timeout
    }
  end
end

Application Test Helper

Create a test helper module for common operations:

defmodule MyApp.TestHelper do
  @moduledoc """
  Common test utilities for SNMP testing.
  """
  
  def start_test_device(profile \\ :generic_router, opts \\ []) do
    port = Keyword.get(opts, :port, get_free_port())
    
    {:ok, profile_data} = SnmpKit.SnmpSim.ProfileLoader.load_profile(profile)
    {:ok, device} = SnmpKit.Sim.start_device(profile_data, port: port)
    
    %{
      device: device,
      target: "127.0.0.1:#{port}",
      port: port
    }
  end
  
  def get_free_port do
    {:ok, socket} = :gen_tcp.listen(0, [])
    {:ok, port} = :inet.port(socket)
    :gen_tcp.close(socket)
    port
  end
  
  def wait_for_device(target, timeout \\ 5_000) do
    end_time = System.monotonic_time(:millisecond) + timeout
    wait_for_device_loop(target, end_time)
  end
  
  defp wait_for_device_loop(target, end_time) do
    if System.monotonic_time(:millisecond) < end_time do
      case SnmpKit.SNMP.get(target, "sysDescr.0", timeout: 1_000) do
        {:ok, _} -> :ok
        {:error, _} -> 
          :timer.sleep(100)
          wait_for_device_loop(target, end_time)
      end
    else
      {:error, :timeout}
    end
  end
end

Unit Testing

Testing MIB Operations

defmodule MyApp.MIBTest do
  use ExUnit.Case, async: true
  
  alias SnmpKit.MIB
  
  describe "OID resolution" do
    test "resolves standard system OIDs" do
      assert {:ok, [1, 3, 6, 1, 2, 1, 1, 1, 0]} = MIB.resolve("sysDescr.0")
      assert {:ok, [1, 3, 6, 1, 2, 1, 1, 3, 0]} = MIB.resolve("sysUpTime.0")
      assert {:ok, [1, 3, 6, 1, 2, 1, 1, 5, 0]} = MIB.resolve("sysName.0")
    end
    
    test "handles invalid OID names" do
      assert {:error, :not_found} = MIB.resolve("nonExistentOid.0")
      assert {:error, :invalid_format} = MIB.resolve("")
    end
    
    test "resolves bulk OIDs efficiently" do
      oids = ["sysDescr.0", "sysUpTime.0", "sysName.0"]
      
      {time, {:ok, results}} = :timer.tc(fn ->
        MIB.resolve_many(oids)
      end)
      
      assert length(results) == 3
      assert time < 10_000  # Should be fast (< 10ms)
    end
  end
  
  describe "reverse lookup" do
    test "converts OIDs back to names" do
      oid = [1, 3, 6, 1, 2, 1, 1, 1, 0]
      assert {:ok, "sysDescr.0"} = MIB.reverse_lookup(oid)
    end
    
    test "handles partial matches" do
      # OID that doesn't exactly match but has a parent
      oid = [1, 3, 6, 1, 2, 1, 1, 1, 999]
      assert {:ok, name} = MIB.reverse_lookup(oid)
      assert String.contains?(name, "sysDescr")
    end
  end
end

Testing SNMP Operations with Mocks

defmodule MyApp.SNMPMockTest do
  use ExUnit.Case, async: true
  
  import Mox
  
  # Define mock in test_helper.exs:
  # Mox.defmock(MyApp.SNMPMock, for: SnmpKit.SNMP.Behaviour)
  
  setup :verify_on_exit!
  
  test "handles SNMP timeouts gracefully" do
    MyApp.SNMPMock
    |> expect(:get, fn _target, _oid, _opts ->
      {:error, :timeout}
    end)
    
    result = MyApp.DeviceMonitor.get_device_status("192.168.1.1")
    assert {:error, :device_unreachable} = result
  end
  
  test "retries on temporary failures" do
    MyApp.SNMPMock
    |> expect(:get, 2, fn _target, _oid, _opts ->
      {:error, :timeout}
    end)
    |> expect(:get, fn _target, _oid, _opts ->
      {:ok, "Test Device"}
    end)
    
    result = MyApp.DeviceMonitor.get_device_status("192.168.1.1")
    assert {:ok, %{description: "Test Device"}} = result
  end
end

Integration Testing

Testing with Simulated Devices

defmodule MyApp.IntegrationTest do
  use ExUnit.Case, async: true
  
  alias MyApp.TestHelper
  
  describe "device communication" do
    setup do
      device_info = TestHelper.start_test_device(:cable_modem)
      :ok = TestHelper.wait_for_device(device_info.target)
      device_info
    end
    
    test "can query basic system information", %{target: target} do
      {:ok, description} = SnmpKit.SNMP.get(target, "sysDescr.0")
      assert is_binary(description)
      assert String.length(description) > 0
      
      {:ok, uptime} = SnmpKit.SNMP.get(target, "sysUpTime.0") 
      assert is_integer(uptime)
      assert uptime >= 0
    end
    
    test "can walk interface table", %{target: target} do
      {:ok, interfaces} = SnmpKit.SNMP.walk(target, "ifTable")
      assert is_list(interfaces)
      assert length(interfaces) > 0
      
      # Verify interface data structure
      for {oid, value} <- interfaces do
        assert is_list(oid)
        assert length(oid) > 0
        assert value != nil
      end
    end
    
    test "handles bulk operations", %{target: target} do
      {:ok, results} = SnmpKit.SNMP.bulk_walk(target, "system")
      assert is_list(results)
      assert length(results) > 0
    end
  end
  
  describe "error handling" do
    setup do
      TestHelper.start_test_device(:unreliable_device)
    end
    
    test "handles device unreachable", %{target: target} do
      # Stop the device to simulate unreachable condition
      GenServer.stop(device.pid)
      
      result = SnmpKit.SNMP.get(target, "sysDescr.0", timeout: 1_000)
      assert {:error, :timeout} = result
    end
    
    test "handles invalid OIDs gracefully", %{target: target} do
      result = SnmpKit.SNMP.get(target, "nonExistent.0")
      assert {:error, :no_such_name} = result
    end
  end
end

Testing with Real Devices

defmodule MyApp.RealDeviceTest do
  use ExUnit.Case
  
  # Only run these tests when real devices are available
  @moduletag :integration
  @moduletag :real_devices
  
  @real_device_ip System.get_env("TEST_DEVICE_IP", "192.168.1.1")
  @real_device_community System.get_env("TEST_DEVICE_COMMUNITY", "public")
  
  setup_all do
    # Skip if no real device configured
    if @real_device_ip == "192.168.1.1" do
      {:skip, "No real device configured"}
    else
      # Verify device is reachable
      case SnmpKit.SNMP.get(@real_device_ip, "sysDescr.0", 
                            community: @real_device_community, timeout: 5_000) do
        {:ok, _} -> :ok
        {:error, _} -> {:skip, "Real device not reachable"}
      end
    end
  end
  
  test "can communicate with real device" do
    {:ok, description} = SnmpKit.SNMP.get(@real_device_ip, "sysDescr.0",
                                          community: @real_device_community)
    assert is_binary(description)
    IO.puts("Real device description: #{description}")
  end
end

Simulated Devices

Creating Custom Device Profiles

defmodule MyApp.CustomDeviceTest do
  use ExUnit.Case, async: true
  
  test "creates custom device profile" do
    # Define custom device behavior
    custom_profile = %{
      name: "Test Switch",
      description: "Custom test switch for unit testing",
      objects: %{
        [1, 3, 6, 1, 2, 1, 1, 1, 0] => "Test Switch v1.0",
        [1, 3, 6, 1, 2, 1, 1, 3, 0] => 12345,  # uptime
        [1, 3, 6, 1, 2, 1, 1, 5, 0] => "test-switch-01",
        # Interface table entries
        [1, 3, 6, 1, 2, 1, 2, 2, 1, 2, 1] => "FastEthernet0/1",
        [1, 3, 6, 1, 2, 1, 2, 2, 1, 2, 2] => "FastEthernet0/2",
        [1, 3, 6, 1, 2, 1, 2, 2, 1, 8, 1] => 1,  # ifOperStatus = up
        [1, 3, 6, 1, 2, 1, 2, 2, 1, 8, 2] => 2   # ifOperStatus = down
      }
    }
    
    {:ok, device} = SnmpKit.Sim.start_device(custom_profile, port: 30001)
    target = "127.0.0.1:30001"
    
    # Test the custom device
    {:ok, description} = SnmpKit.SNMP.get(target, "sysDescr.0")
    assert description == "Test Switch v1.0"
    
    {:ok, if_name} = SnmpKit.SNMP.get(target, "ifDescr.1")
    assert if_name == "FastEthernet0/1"
    
    {:ok, if_status} = SnmpKit.SNMP.get(target, "ifOperStatus.1")
    assert if_status == 1  # up
  end
end

Loading Device Profiles from Files

defmodule MyApp.ProfileTest do
  use ExUnit.Case, async: true
  
  test "loads device profile from walk file" do
    # Create a test walk file
    walk_data = """
    1.3.6.1.2.1.1.1.0 = STRING: "Test Device"
    1.3.6.1.2.1.1.3.0 = Timeticks: (12345) 0:02:03.45
    1.3.6.1.2.1.1.5.0 = STRING: "test-device"
    """
    
    walk_file = Path.join(System.tmp_dir(), "test_device.walk")
    File.write!(walk_file, walk_data)
    
    {:ok, profile} = SnmpKit.SnmpSim.ProfileLoader.load_profile(
      :test_device,
      {:walk_file, walk_file}
    )
    
    {:ok, device} = SnmpKit.Sim.start_device(profile, port: 30002)
    target = "127.0.0.1:30002"
    
    {:ok, description} = SnmpKit.SNMP.get(target, "sysDescr.0")
    assert description == "Test Device"
    
    # Clean up
    File.rm!(walk_file)
  end
end

Performance Testing

Benchmarking SNMP Operations

defmodule MyApp.PerformanceTest do
  use ExUnit.Case
  
  alias MyApp.TestHelper
  
  @moduletag :performance
  
  setup_all do
    # Start multiple devices for load testing
    devices = for i <- 1..10 do
      TestHelper.start_test_device(:generic_router, port: 30000 + i)
    end
    
    %{devices: devices}
  end
  
  test "measures single GET performance", %{devices: [device | _]} do
    target = device.target
    
    # Warm up
    for _ <- 1..10 do
      SnmpKit.SNMP.get(target, "sysDescr.0")
    end
    
    # Measure performance
    {time, results} = :timer.tc(fn ->
      for _ <- 1..100 do
        SnmpKit.SNMP.get(target, "sysDescr.0")
      end
    end)
    
    avg_time = time / 100
    success_count = Enum.count(results, &match?({:ok, _}, &1))
    
    IO.puts("Average GET time: #{avg_time/1000}ms")
    IO.puts("Success rate: #{success_count}/100")
    
    assert avg_time < 50_000  # Should be < 50ms average
    assert success_count == 100  # Should be 100% successful
  end
  
  test "measures concurrent GET performance", %{devices: devices} do
    targets = Enum.map(devices, & &1.target)
    
    {time, results} = :timer.tc(fn ->
      targets
      |> Enum.map(fn target ->
        Task.async(fn ->
          for _ <- 1..50 do
            SnmpKit.SNMP.get(target, "sysDescr.0")
          end
        end)
      end)
      |> Enum.map(&Task.await(&1, 10_000))
    end)
    
    total_requests = length(devices) * 50
    avg_time = time / total_requests
    
    IO.puts("Concurrent average time: #{avg_time/1000}ms")
    IO.puts("Total requests: #{total_requests}")
    IO.puts("Total time: #{time/1_000_000}s")
    
    assert avg_time < 100_000  # Should be < 100ms average under load
  end
  
  test "measures walk performance", %{devices: [device | _]} do
    target = device.target
    
    {time, {:ok, results}} = :timer.tc(fn ->
      SnmpKit.SNMP.walk(target, "system")
    end)
    
    objects_per_ms = length(results) / (time / 1000)
    
    IO.puts("Walk time: #{time/1000}ms")
    IO.puts("Objects retrieved: #{length(results)}")
    IO.puts("Objects per ms: #{objects_per_ms}")
    
    assert time < 1_000_000  # Should complete in < 1 second
    assert length(results) > 0
  end
end

Memory and Resource Testing

defmodule MyApp.ResourceTest do
  use ExUnit.Case
  
  test "handles large result sets without memory issues" do
    device = MyApp.TestHelper.start_test_device(:large_table_device)
    target = device.target
    
    # Monitor memory usage
    initial_memory = :erlang.memory(:total)
    
    # Perform large walk operation
    {:ok, results} = SnmpKit.SNMP.walk(target, "largeTable")
    
    peak_memory = :erlang.memory(:total)
    
    # Force garbage collection
    :erlang.garbage_collect()
    :timer.sleep(100)
    
    final_memory = :erlang.memory(:total)
    
    memory_growth = peak_memory - initial_memory
    memory_retained = final_memory - initial_memory
    
    IO.puts("Results count: #{length(results)}")
    IO.puts("Peak memory growth: #{memory_growth / 1024 / 1024}MB")
    IO.puts("Retained memory: #{memory_retained / 1024 / 1024}MB")
    
    # Memory should be reasonable
    assert memory_growth < 100 * 1024 * 1024  # < 100MB growth
    assert memory_retained < 10 * 1024 * 1024  # < 10MB retained
  end
end

Test Utilities

Custom ExUnit Assertions

defmodule MyApp.SNMPAssertions do
  @moduledoc """
  Custom assertions for SNMP testing.
  """
  
  import ExUnit.Assertions
  
  def assert_snmp_success(result, message \\ nil) do
    case result do
      {:ok, value} -> value
      {:error, reason} -> 
        flunk(message || "Expected SNMP success, got error: #{inspect(reason)}")
    end
  end
  
  def assert_snmp_error(result, expected_error \\ nil, message \\ nil) do
    case result do
      {:error, reason} when expected_error == nil -> reason
      {:error, ^expected_error} -> expected_error
      {:error, reason} when expected_error != nil ->
        flunk(message || "Expected error #{expected_error}, got #{reason}")
      {:ok, value} ->
        flunk(message || "Expected SNMP error, got success: #{inspect(value)}")
    end
  end
  
  def assert_oid_resolved(oid_name, expected_oid \\ nil) do
    case SnmpKit.MIB.resolve(oid_name) do
      {:ok, ^expected_oid} when expected_oid != nil -> expected_oid
      {:ok, resolved_oid} when expected_oid == nil -> resolved_oid
      {:ok, actual_oid} when expected_oid != nil ->
        flunk("Expected OID #{inspect(expected_oid)}, got #{inspect(actual_oid)}")
      {:error, reason} ->
        flunk("Failed to resolve OID #{oid_name}: #{reason}")
    end
  end
  
  def assert_device_responsive(target, timeout \\ 5_000) do
    case SnmpKit.SNMP.get(target, "sysDescr.0", timeout: timeout) do
      {:ok, _} -> :ok
      {:error, reason} ->
        flunk("Device #{target} not responsive: #{reason}")
    end
  end
end

Test Data Generators

defmodule MyApp.TestDataGenerator do
  @moduledoc """
  Generates test data for SNMP testing.
  """
  
  def generate_walk_data(base_oid, count \\ 100) do
    for i <- 1..count do
      oid = base_oid ++ [i]
      value = "Value #{i}"
      {oid, value}
    end
  end
  
  def generate_interface_table(interface_count \\ 24) do
    for i <- 1..interface_count do
      [
        {[1, 3, 6, 1, 2, 1, 2, 2, 1, 1, i], i},  # ifIndex
        {[1, 3, 6, 1, 2, 1, 2, 2, 1, 2, i], "eth#{i}"},  # ifDescr
        {[1, 3, 6, 1, 2, 1, 2, 2, 1, 3, i], 6},  # ifType (ethernet)
        {[1, 3, 6, 1, 2, 1, 2, 2, 1, 5, i], 1_000_000_000},  # ifSpeed
        {[1, 3, 6, 1, 2, 1, 2, 2, 1, 8, i], Enum.random([1, 2])},  # ifOperStatus
        {[1, 3, 6, 1, 2, 1, 2, 2, 1, 10, i], :rand.uniform(1_000_000_000)},  # ifInOctets
        {[1, 3, 6, 1, 2, 1, 2, 2, 1, 16, i], :rand.uniform(1_000_000_000)}   # ifOutOctets
      ]
    end
    |> List.flatten()
  end
  
  def generate_device_profile(type \\ :generic) do
    base_objects = %{
      [1, 3, 6, 1, 2, 1, 1, 1, 0] => device_description(type),
      [1, 3, 6, 1, 2, 1, 1, 2, 0] => device_object_id(type),
      [1, 3, 6, 1, 2, 1, 1, 3, 0] => :rand.uniform(1_000_000),
      [1, 3, 6, 1, 2, 1, 1, 4, 0] => "Test Admin",
      [1, 3, 6, 1, 2, 1, 1, 5, 0] => "test-device-#{:rand.uniform(1000)}",
      [1, 3, 6, 1, 2, 1, 1, 6, 0] => "Test Lab"
    }
    
    interface_objects = 
      generate_interface_table()
      |> Enum.into(%{})
    
    Map.merge(base_objects, interface_objects)
  end
  
  defp device_description(:router), do: "Test Router v1.0"
  defp device_description(:switch), do: "Test Switch v2.0"
  defp device_description(:cable_modem), do: "Test Cable Modem v3.0"
  defp device_description(_), do: "Generic Test Device v1.0"
  
  defp device_object_id(:router), do: [1, 3, 6, 1, 4, 1, 9999, 1, 1]
  defp device_object_id(:switch), do: [1, 3, 6, 1, 4, 1, 9999, 1, 2]
  defp device_object_id(:cable_modem), do: [1, 3, 6, 1, 4, 1, 9999, 1, 3]
  defp device_object_id(_), do: [1, 3, 6, 1, 4, 1, 9999, 1, 0]
end

Continuous Integration

GitHub Actions Configuration

# .github/workflows/test.yml
name: Test

on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main ]

jobs:
  test:
    runs-on: ubuntu-latest
    
    strategy:
      matrix:
        elixir: ['1.14', '1.15']
        otp: ['25', '26']
        
    steps:
    - uses: actions/checkout@v3
    
    - name: Set up Elixir
      uses: erlef/setup-beam@v1
      with:
        elixir-version: ${{ matrix.elixir }}
        otp-version: ${{ matrix.otp }}
        
    - name: Restore dependencies cache
      uses: actions/cache@v3
      with:
        path: deps
        key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
        restore-keys: ${{ runner.os }}-mix-
        
    - name: Install dependencies
      run: mix deps.get
      
    - name: Run tests
      run: mix test --trace
      
    - name: Run integration tests
      run: mix test --include integration
      
    - name: Generate coverage report
      run: mix test --cover
      
    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v3

Test Configuration

# config/test.exs
import Config

config :snmpkit,
  default_timeout: 1_000,  # Faster timeouts for testing
  default_retries: 1

config :snmpkit, :simulation,
  device_profiles_path: "test/fixtures/profiles",
  walk_files_path: "test/fixtures/walks"

# Reduce log noise during testing
config :logger, level: :warning

# Enable async testing
config :ex_unit,
  capture_log: true,
  async: true

Best Practices

1. Use Appropriate Test Types

  • Unit Tests - Test individual functions and modules in isolation
  • Integration Tests - Test interaction between components
  • System Tests - Test complete workflows with simulated devices
  • Performance Tests - Test performance characteristics and limits

2. Design for Testability

# Bad: Hard to test, tightly coupled
def monitor_device(ip) do
  case SnmpKit.SNMP.get(ip, "sysDescr.0") do
    {:ok, description} -> 
      Logger.info("Device #{ip}: #{description}")
      send_notification(description)
    {:error, reason} ->
      Logger.error("Failed to monitor #{ip}: #{reason}")
      raise "Device monitoring failed"
  end
end

# Good: Testable, dependency injection
def monitor_device(ip, snmp_client \\ SnmpKit.SNMP, notifier \\ MyApp.Notifier) do
  case snmp_client.get(ip, "sysDescr.0") do
    {:ok, description} -> 
      Logger.info("Device #{ip}: #{description}")
      notifier.send_notification(description)
      {:ok, description}
    {:error, reason} ->
      Logger.error("Failed to monitor #{ip}: #{reason}")
      {:error, reason}
  end
end

3. Use Simulated Devices Extensively

  • Create realistic device profiles for different device types
  • Test edge cases and error conditions
  • Simulate network conditions (latency, packet loss)
  • Test with various SNMP versions and configurations

4. Test Error Conditions

test "handles various error conditions" do
  test_cases = [
    {:timeout, "unreachable.device"},
    {:no_such_name, "invalid.oid"},
    {:bad_value, "read_only.oid"},
    {:authorization_error, "wrong.community"}
  ]
  
  for {expected_error, scenario} <- test_cases do
    result = perform_test_scenario(scenario)
    assert {:error, ^expected_error} = result
  end
end

5. Use Property-Based Testing

defmodule MyApp.PropertyTest do
  use ExUnit.Case
  use PropCheck
  
  property "OID resolution is bidirectional" do
    forall oid_name <- valid_oid_name() do
      case SnmpKit.MIB.resolve(oid_name) do
        {:ok, oid} ->
          {:ok, reversed_name} = SnmpKit.MIB.reverse_lookup(oid)
          String.contains?(reversed_name, extract_base_name(oid_name))
        {:error, _} ->
          true  # Invalid names are okay
      end
    end
  end
  
  defp valid_oid_name do
    oneof([
      "sysDescr.0",
      "sysUpTime.0", 
      "sysName.0",
      "ifDescr.1",
      "ifInOctets.1"
    ])
  end
end

6. Monitor Test Performance

  • Track test execution times
  • Identify slow tests and optimize them
  • Use async testing where possible
  • Profile memory usage in tests

7. Maintain Test Data

  • Keep device profiles up to date
  • Version test data files
  • Document test scenarios and expected outcomes
  • Clean up test resources properly

For more advanced testing techniques and examples, see the API documentation and example tests.