defmodule LastfmArchive.Transform do @moduledoc """ This module provides functions for reading and transforming downloaded Lastfm data. """ @default_data_dir "./lastfm_data/" @default_delimiter "\t" @doc """ Transform a downloaded raw JSON page into a list of tab-delimited track data. ### Example ``` # transform a page of scrobbles from the file archive LastfmArchive.Transform.transform("a_lastfm_user", "2007/200_1.gz") ``` A row of tab-delimited track currently contains (if any): - `id` auto-generated by the system to uniquely identify a scrobble - `name` the track name - `scrobble_date` Unix timestamp of the scrobble date - `scrobble_date_iso` scrobble date in ISO 8601 datetime format - `mbid` MusicBrainz identifier for the track - `url` web address of the track on Last.fm - `artist` - `artist_mbid` MusicBrainz identifier for the artist - `artist_url` web address of the artist on Last.fm - `album` - `album_mbid` MusicBrainz identifier for the album """ @spec transform(binary, binary, :atom) :: list(binary) | {:error, :file.posix()} def transform(user, filename, mode \\ :tsv) def transform(user, filename, :tsv) do {status, tracks_data} = read(user, filename) case status do :ok -> index = initial_index(tracks_data["recenttracks"]["@attr"]) [track | tracks] = tracks_data["recenttracks"]["track"] if track["@attr"]["nowplaying"], do: _transform(user, tracks, index, []), else: _transform(user, [track | tracks], index, []) :error -> {:error, tracks_data} end end defp _transform(_user, [], _index, acc), do: acc defp _transform(user, [track | tracks], index, acc) do next_index = index + 1 _transform(user, tracks, next_index, acc ++ [_transform(user, track, index)]) end # id,name,scrobble_date,date_iso,mbid,url,artist,artist_mbid,artist_url,album,album_mbid defp _transform(user, track, index) do id = "#{user}_#{track["date"]["uts"]}_#{index |> to_string}" date_s = track["date"]["uts"] |> DateTime.from_unix!() |> DateTime.to_iso8601() track_info = [id, track["name"] |> String.trim(), track["date"]["uts"], date_s, track["mbid"], track["url"]] artist_info = [track["artist"]["name"], track["artist"]["mbid"], track["artist"]["url"]] album_info = [track["album"]["#text"], track["album"]["mbid"]] Enum.join(track_info ++ artist_info ++ album_info, @default_delimiter) end @doc """ Read and parse a raw Lastfm JSON file from the archive for a Lastfm user. """ @spec read(binary, binary) :: {:ok, map} | {:error, :file.posix()} def read(user, filename) do data_dir = Application.get_env(:lastfm_archive, :data_dir) || @default_data_dir user_data_dir = Path.join(data_dir, user) file_path = Path.join(user_data_dir, filename) {status, file_io} = File.open(file_path, [:read, :compressed, :utf8]) resp = case status do :ok -> {:ok, IO.read(file_io, :line) |> Jason.decode!()} :error -> {:error, file_io} end if is_pid(file_io), do: File.close(file_io) resp end defp initial_index(info), do: (info["page"] - 1) * info["perPage"] + 1 end