View Source Pose Estimation: MoveNet

File.cd!(__DIR__)
# for windows JP
System.shell("chcp 65001")

Mix.install([
  {:tfl_interp, path: ".."},
  {:cimg, path: "../../cimg_ex"},
  {:nx, "~> 0.4.0"},
  {:kino, "~> 0.7.0"}
])

0-original-work

0.Original work

Thanks a lot!!!


implementation-with-tflinterp-in-elixir

Implementation with TflInterp in Elixir

1-defining-the-inference-module-movenet

1.Defining the inference module: MoveNet

  • Model<br> yolov4_1_3_608_608_static.onnx

  • Pre-processing<br> Resize the input image to the size {608,608}, normalize to {0.0,1.0} and transpose NCHW.

  • Post-processing<br> Filtering Boxes(output[0]) and Scores(output[1]) with Multi-class Non Maximum Suppression.

defmodule Movenet do
  @width 192
  @height 192

  alias TflInterp, as: NNInterp

  use NNInterp,
    model: "./model/lite-model_movenet_singlepose_lightning_tflite_int8_4.tflite",
    inputs: [u8: {1, @height, @width, 3}],
    outputs: [f32: {1, 1, 17, 3}]

  def apply(img) do
    # preprocess
    input0 =
      CImg.builder(img)
      |> CImg.resize({@width, @height}, :ul, 0)
      |> CImg.to_binary(dtype: "<u1")

    # prediction
    output0 =
      session()
      |> NNInterp.set_input_tensor(0, input0)
      |> NNInterp.invoke()
      |> NNInterp.get_output_tensor(0)
      |> Nx.from_binary(:f32)
      |> Nx.reshape({17, 3})

    # postprocess
    {img_width, img_height, _, _} = CImg.shape(img)

    {:ok, to_bones(output0, {img_width, img_height})}
  end

  @bones [
    {0, 1, {0, 255, 0}},
    {0, 2, {0, 255, 0}},
    {1, 3, {0, 255, 0}},
    {2, 4, {0, 255, 0}},
    {0, 5, {0, 255, 0}},
    {0, 6, {0, 255, 0}},
    {5, 7, {0, 255, 0}},
    {7, 9, {0, 255, 0}},
    {6, 8, {0, 255, 0}},
    {8, 10, {0, 255, 0}},
    {5, 6, {0, 255, 0}},
    {5, 11, {0, 255, 0}},
    {6, 12, {0, 255, 0}},
    {11, 12, {0, 255, 0}},
    {11, 13, {0, 255, 0}},
    {13, 15, {0, 255, 0}},
    {12, 14, {0, 255, 0}},
    {14, 16, {0, 255, 0}}
  ]

  def to_bones(t, {w, h}, threshold \\ 0.11) do
    {scale_x, scale_y} = if w > h, do: {1.0, w / h}, else: {h / w, 1.0}

    Enum.flat_map(@bones, fn {p1, p2, color} ->
      [y1, x1, score1] = Nx.to_flat_list(t[p1])
      [y2, x2, score2] = Nx.to_flat_list(t[p2])

      if score1 > threshold && score2 > threshold do
        [{x1 * scale_x, y1 * scale_y, x2 * scale_x, y2 * scale_y, color}]
      else
        []
      end
    end)
  end
end

Launch Movenet.

# OnnxInterp.stop(Resnet18)
Movenet.start_link([])

Display the properties of the Movenet model.

TflInterp.info(Movenet)

2-defining-execution-module-demomovenet

2.Defining execution module DemoMovenet

defmodule DemoMovenet do
  def run(path) when is_binary(path) do
    img = CImg.load(path)

    with {:ok, res} <- Movenet.apply(img) do
      Enum.reduce(res, CImg.builder(img), &draw_item(&1, &2))
      |> CImg.save("xxx.jpg")
    end
  end

  defp draw_item({x1, y1, x2, y2, color}, canvas) do
    CImg.draw_line(canvas, x1, y1, x2, y2, color)
  end
end

3-let-s-try-it

3.Let's try it

LiveYOLOv4.run("dog.jpg")

appendix

Appendix

&#9633;