diff --git a/config/dev.exs b/config/dev.exs index eb2fd12..5c9c946 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -42,6 +42,9 @@ config :compass_admin, CompassAdmin.Services.QueueSchedule, config :compass_admin, CompassAdmin.Services.CronoCheck, process_name: "compass-web-crontab", supervisor_api: "http://localhost:19999/RPC2", + +config :compass_admin, :apm, + upstream: "http://localhost:4318", basic_auth: "basic_auth" diff --git a/config/prod.exs b/config/prod.exs index 30d13e8..61b51ae 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -48,6 +48,9 @@ config :compass_admin, CompassAdmin.Services.QueueSchedule, config :compass_admin, CompassAdmin.Services.CronoCheck, process_name: "compass-web-crontab", supervisor_api: "http://localhost:19999/RPC2", + +config :compass_admin, :apm, + upstream: "http://localhost:4318", basic_auth: "basic_auth" config :compass_admin, CompassAdmin.Services.ExportMetrics, diff --git a/lib/compass_admin/application.ex b/lib/compass_admin/application.ex index b85e079..26cf984 100644 --- a/lib/compass_admin/application.ex +++ b/lib/compass_admin/application.ex @@ -2,6 +2,7 @@ defmodule CompassAdmin.Application do # See https://hexdocs.pm/elixir/Application.html # for more information on OTP Applications @moduledoc false + @default_log_command "parallel --tagstring \"{}|\" --line-buffer tail -F {} ::: log/*.log" use Application @@ -32,6 +33,7 @@ defmodule CompassAdmin.Application do # Start the Elasticsearch cluster CompassAdmin.Cluster, # Start finch pool + {Finch, name: LogFinch}, {Finch, name: CompassFinch}, # Start the Telemetry supervisor CompassAdminWeb.Telemetry, @@ -55,7 +57,7 @@ defmodule CompassAdmin.Application do [host: redis_host, port: redis_port, auth: auth, database: database] ] ]}, - # {CompassAdmin.Worker, arg} + app_logger(), CompassAdmin.Scheduler, {Highlander, CompassAdmin.GlobalScheduler}, # Exec agent @@ -63,7 +65,7 @@ defmodule CompassAdmin.Application do # Deployment agents {CompassAdmin.Agents.BackendAgent, []}, {CompassAdmin.Agents.FrontendAgent, []} - ] + ] |> Enum.reject(&is_nil/1) CompassAdmin.Plug.MetricsExporter.setup() Metrics.CompassInstrumenter.setup() @@ -89,6 +91,16 @@ defmodule CompassAdmin.Application do :ok end + defp app_logger() do + if String.contains?(to_string(node()), "app") || + String.contains?(to_string(node()), "grimoirelab") do + {CompassAdmin.LogBoardway, [ + Application.get_env(:compass_admin, :apm, %{log_command: @default_log_command})[:log_command] + ] + } + end + end + defp riak_config() do [ name: {:local, CompassAdmin.RiakPool}, diff --git a/lib/compass_admin/log_boardway.ex b/lib/compass_admin/log_boardway.ex new file mode 100644 index 0000000..2d26a1b --- /dev/null +++ b/lib/compass_admin/log_boardway.ex @@ -0,0 +1,129 @@ +defmodule CompassAdmin.LogBoardway do + use Broadway + require Logger + + alias Broadway.Message + alias CompassAdmin.StreamProducer + + @apm_config Application.get_env(:compass_admin, :apm, %{}) + + @app_pattern ~r/method=(\w+) path=(\/[^\s]*) format=(?[^\s]+) controller=([^\s]+) action=([^\s]+) status=(\d+) allocations=(\d+) duration=(\d+\.\d+) view=(\d+\.\d+) db=(\d+\.\d+) host=([^\s]+) remote_ip=([^\s]+) x_forwarded_for=(.*)/ + + @log_meta_pattern ~r/(\w+), \[([^\s]+) (#\d+)\]\s+(\w+)\s+(.*)/ + + def start_link(command) do + Broadway.start_link(__MODULE__, + name: __MODULE__, + producer: [ + concurrency: 1, + module: {StreamProducer, command}, + transformer: {StreamProducer, :transform, []} + ], + processors: [ + default: [ + concurrency: 10, + min_demand: 0, + max_demand: 1 + ] + ] + ) + end + + @impl true + def handle_message(_, %Message{data: data} = message, _) do + String.split(data, "\n", trim: true) + |> Enum.map(fn line -> + + [log_path | rest] = + String.split(line, "|", parts: 2, trim: true) |> Enum.map(&String.trim/1) + + [log_meta | content] = + String.split(to_string(List.first(rest)), " : ", parts: 2, trim: true) |> Enum.map(&String.trim/1) + + {log_path, content} = + if length(content) == 0 do + {log_path, ""} + else + {log_path, IO.iodata_to_binary(content)} + end + + base_document = Map.merge(%{log_path: log_path}, parse_log_meta_line(log_meta)) + + document = + cond do + String.contains?(log_path, "production") -> + Map.merge(base_document, parse_production_log_line(content)) + + String.contains?(log_path, "sneakers") -> + Map.merge(base_document, parse_sneakers_log_line(content)) + + String.contains?(log_path, "development") -> + Map.merge(base_document, parse_development_log_line(content)) + + true -> + Map.merge(base_document, %{content: content}) + end + + Finch.build(:post, + "#{@apm_config[:upstream]}/api/default/default/_json", + [{"Authorization", "Basic #{@apm_config[:basic_auth]}"}], + [Jason.encode!(document)] + ) |> Finch.request(LogFinch) + end) + message + end + + @impl true + def handle_batch(_, messages, _, _) do + messages + end + + def parse_log_meta_line(line) do + case Regex.scan(@log_meta_pattern, line) do + [ + [ + _raw, + short_level, + datetime, + rid, + level, + extra + ] + ] -> + %{ + short_level: short_level, + datetime: datetime, + rid: rid, + level: level, + extra: extra + } + _ -> + %{log_meta: line} + end + end + def parse_sneakers_log_line(line), do: %{content: line} + def parse_production_log_line(line) do + case Regex.scan(@app_pattern, line) do + [[_, method, path, format, controller, action, status, allocations, duration, view, db, host, remote_ip, x_forwarded_for]] -> + %{ + method: method, + path: path, + format: format, + controller: controller, + action: action, + status: String.to_integer(status), + allocations: String.to_integer(allocations), + duration: String.to_float(duration), + view: String.to_float(view), + db: String.to_float(db), + host: host, + remote_ip: remote_ip, + x_forwarded_for: x_forwarded_for + } + + _ -> + %{content: line} + end + end + def parse_development_log_line(line), do: parse_production_log_line(line) +end diff --git a/lib/compass_admin/stream_producer.ex b/lib/compass_admin/stream_producer.ex new file mode 100644 index 0000000..0844fef --- /dev/null +++ b/lib/compass_admin/stream_producer.ex @@ -0,0 +1,39 @@ +defmodule CompassAdmin.StreamProducer do + # See https://hexdocs.pm/broadway/custom-producers.html#example + use GenStage + + alias Broadway.Message + + # Broadway will not call the child_spec/1 or start_link/1 function of the producer. + # That's because Broadway wraps the producer to augment it with extra features. + def start_link(command) do + GenStage.start_link(__MODULE__, command) + end + + # When Broadway starts, the GenStage.init/1 callback will be invoked w the given opts. + def init(command) do + {:producer, Exile.stream!(["bash", "-c"] ++ command, exit_timeout: 1000)} + end + + def handle_demand(demand, stream) when demand > 0 do + {head, tail} = StreamSplit.take_and_drop(stream, demand) + {:noreply, head, tail} + end + + def handle_info(_, state) do + {:noreply, [], state} + end + + # Not part of the behavior, but Broadway req's that we translate the genstage events + # into Broadway msgs + def transform(event, _opts) do + %Message{ + data: event, + acknowledger: {__MODULE__, :ack_id, :ack_data} + } + end + + def ack(:ack_id, _successful, _failed) do + # Write ack code here + end +end diff --git a/lib/compass_admin_web/controllers/debug_controller.ex b/lib/compass_admin_web/controllers/debug_controller.ex index 2028e14..dfba061 100644 --- a/lib/compass_admin_web/controllers/debug_controller.ex +++ b/lib/compass_admin_web/controllers/debug_controller.ex @@ -9,6 +9,7 @@ defmodule CompassAdminWeb.DebugController do import CompassAdminWeb.Helpers @config Application.get_env(:compass_admin, CompassAdmin.Services.ExportMetrics, %{}) + @apm_config Application.get_env(:compass_admin, :apm, %{}) @client_options [proxy: @config[:proxy], timeout: 180_000, recv_timeout: 3_600_000] def webhook(conn, _params) do @@ -23,6 +24,28 @@ defmodule CompassAdminWeb.DebugController do ) end + def apm_proxy(conn, _params) do + upstream = @apm_config[:upstream] + + params = + ReverseProxyPlug.init( + upstream: upstream, + response_mode: :buffer, + preserve_host_header: true + ) + {:ok, body, conn} = Plug.Conn.read_body(conn) + + auth_conn = + %Conn{} + |> Map.merge(conn) + |> Conn.put_req_header("Authorization", "Basic #{@apm_config[:basic_auth]}") + + auth_conn + |> ReverseProxyPlug.request(body, params) + |> ReverseProxyPlug.response(conn, params) + end + + def docker_registry_proxy(conn, _params) do upstream = "https://registry-1.docker.io" diff --git a/lib/compass_admin_web/router.ex b/lib/compass_admin_web/router.ex index 126b997..17fa4f0 100644 --- a/lib/compass_admin_web/router.ex +++ b/lib/compass_admin_web/router.ex @@ -20,6 +20,7 @@ defmodule CompassAdminWeb.Router do scope "/", CompassAdminWeb do pipe_through :api match :*, "/v2/*path", DebugController, :docker_registry_proxy + match :*, "/api/*path", DebugController, :apm_proxy end scope "/admin", CompassAdminWeb do diff --git a/mix.exs b/mix.exs index dbd9816..b502827 100644 --- a/mix.exs +++ b/mix.exs @@ -40,7 +40,9 @@ defmodule CompassAdmin.MixProject do [ {:phoenix, "~> 1.6.6"}, {:phoenix_ecto, "~> 4.4"}, - {:exile, "~> 0.9.1"}, + {:exile, git: "https://github.com/EdmondFrank/exile", tag: "v0.11.1"}, + {:broadway, "~> 1.1"}, + {:stream_split, "~> 0.1.7"}, {:ecto_sql, "~> 3.6"}, {:myxql, "~> 0.6.0"}, {:amqp, "~> 3.2"}, diff --git a/mix.lock b/mix.lock index 31f6235..ed7411a 100644 --- a/mix.lock +++ b/mix.lock @@ -2,6 +2,7 @@ "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm", "11b18c220bcc2eab63b5470c038ef10eb6783bcb1fcdb11aa4137defa5ac1bb8"}, "amqp": {:hex, :amqp, "3.3.0", "056d9f4bac96c3ab5a904b321e70e78b91ba594766a1fc2f32afd9c016d9f43b", [:mix], [{:amqp_client, "~> 3.9", [hex: :amqp_client, repo: "hexpm", optional: false]}], "hexpm", "8d3ae139d2646c630d674a1b8d68c7f85134f9e8b2a1c3dd5621616994b10a8b"}, "amqp_client": {:hex, :amqp_client, "3.12.13", "6fc6a7c681e53fed4cbd3f5bcdda342a2b46976345e460ef85414c63698cfe70", [:make, :rebar3], [{:credentials_obfuscation, "3.4.0", [hex: :credentials_obfuscation, repo: "hexpm", optional: false]}, {:rabbit_common, "3.12.13", [hex: :rabbit_common, repo: "hexpm", optional: false]}], "hexpm", "76f41bff0792193f00e0062128db51eb68bcee0eb8236139247a7d1866438d03"}, + "broadway": {:hex, :broadway, "1.1.0", "8ed3aea01fd6f5640b3e1515b90eca51c4fc1fac15fb954cdcf75dc054ae719c", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.7 or ~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "25e315ef1afe823129485d981dcc6d9b221cea30e625fd5439e9b05f44fb60e4"}, "castore": {:hex, :castore, "0.1.22", "4127549e411bedd012ca3a308dede574f43819fe9394254ca55ab4895abfa1a2", [:mix], [], "hexpm", "c17576df47eb5aa1ee40cc4134316a99f5cad3e215d5c77b8dd3cfef12a22cac"}, "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, @@ -17,13 +18,13 @@ "ecto": {:hex, :ecto, "3.9.6", "2f420c173efcb2e22fa4f8fc41e75e02b3c5bd4cffef12085cae5418c12e530d", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "df17bc06ba6f78a7b764e4a14ef877fe5f4499332c5a105ace11fe7013b72c84"}, "ecto_mysql_extras": {:hex, :ecto_mysql_extras, "0.6.2", "5192d01120c9af22996daf97b602ca5fcef5c38dba3f570411be47b9ad9b5a61", [:mix], [{:ecto_sql, "~> 3.7", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:myxql, "~> 0.5", [hex: :myxql, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1 or ~> 4.0", [hex: :table_rex, repo: "hexpm", optional: true]}], "hexpm", "a5a2482fbf2ee626c2561e794b50066c18067772f43cb43814271c969334c503"}, "ecto_sql": {:hex, :ecto_sql, "3.9.0", "2bb21210a2a13317e098a420a8c1cc58b0c3421ab8e3acfa96417dab7817918c", [:mix], [{:db_connection, "~> 2.5 or ~> 2.4.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a8f3f720073b8b1ac4c978be25fa7960ed7fd44997420c304a4a2e200b596453"}, - "elixir_make": {:hex, :elixir_make, "0.8.3", "d38d7ee1578d722d89b4d452a3e36bcfdc644c618f0d063b874661876e708683", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "5c99a18571a756d4af7a4d89ca75c28ac899e6103af6f223982f09ce44942cc9"}, + "elixir_make": {:hex, :elixir_make, "0.8.4", "4960a03ce79081dee8fe119d80ad372c4e7badb84c493cc75983f9d3bc8bde0f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "6e7f1d619b5f61dfabd0a20aa268e575572b542ac31723293a4c1a567d5ef040"}, "erlsom": {:hex, :erlsom, "1.5.1", "c8fe2babd33ff0846403f6522328b8ab676f896b793634cfe7ef181c05316c03", [:rebar3], [], "hexpm", "7965485494c5844dd127656ac40f141aadfa174839ec1be1074e7edf5b4239eb"}, "esbuild": {:hex, :esbuild, "0.5.0", "d5bb08ff049d7880ee3609ed5c4b864bd2f46445ea40b16b4acead724fb4c4a3", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "f183a0b332d963c4cfaf585477695ea59eef9a6f2204fdd0efa00e099694ffe5"}, "ex_hash_ring": {:hex, :ex_hash_ring, "3.0.0", "da32c83d7c6d964b9537eb52f27bad0a3a6f7012efdc2749e11a5f268b120b6b", [:mix], [], "hexpm", "376e06856f4aaaca2eb65bb0a6b5eaf18778892d782ccc30bc28558e66b440d8"}, "ex_indexea": {:hex, :ex_indexea, "0.1.0", "4621a4b7f3413f20ae9c94056ca81f75d65afd988ec6e85d2e9e2f7d246a400d", [:mix], [{:confex, "~> 3.5", [hex: :confex, repo: "hexpm", optional: false]}, {:httpoison, "~> 2.2", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "dce1d42da336b8eae138f2e0b5775ddbd745331cb6cc8aaab7f8e34a08b081bb"}, "ex_marshal": {:hex, :ex_marshal, "0.0.13", "ee5f756d7f530f44bc18ea71b717e7566004211f1dfca685ed8136719fc9fe98", [:mix], [{:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "f37ebf6f66a4be5769556efde21d76467a33f0e186fa5c0093fbc3d9e43b9920"}, - "exile": {:hex, :exile, "0.9.1", "832b6340cf800661e90e52cebc760b795450f803c0e9265ccdc54150423fbb32", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "553a1847b27118c843d3dc6912adbc36d60336811d15ad70a31b82eb5a416328"}, + "exile": {:git, "https://github.com/EdmondFrank/exile", "2eba6e60fdb9f23a40b3eede6c7b7b94e2f7a293", [tag: "v0.11.1"]}, "fastglobal": {:hex, :fastglobal, "1.0.0", "f3133a0cda8e9408aac7281ec579c4b4a8386ce0e99ca55f746b9f58192f455b", [:mix], [], "hexpm", "cfdb7ed63910bc75f579cd09e2517618fa9418b56731d51d03f7ba4b400798d0"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, "finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"}, @@ -84,6 +85,7 @@ "sitemapper": {:hex, :sitemapper, "0.7.0", "4aee7930327a9a01b1c9b81d1d42f60c1a295e9f420108eb2d130c317415abd7", [:mix], [{:ex_aws_s3, "~> 2.0", [hex: :ex_aws_s3, repo: "hexpm", optional: true]}, {:xml_builder, "~> 2.1", [hex: :xml_builder, repo: "hexpm", optional: false]}], "hexpm", "60f7a684e5e9fe7f10ac5b69f48b0be2bcbba995afafcb3c143fc0c8ef1f223f"}, "snap": {:hex, :snap, "0.8.1", "d95a7ecbc911a5a12f419c7108ce9e44db161b9177d137f14d22a8c1fe85cf4e", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:finch, "~> 0.8", [hex: :finch, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "929e2d3254697c85c0226cabe6b40eda6703cfece4fe8285c4b82f3e66ee6291"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, + "stream_split": {:hex, :stream_split, "0.1.7", "2d3fd1fd21697da7f91926768d65f79409086052c9ec7ae593987388f52425f8", [:mix], [], "hexpm", "1dc072ff507a64404a0ad7af90df97096183fee8eeac7b300320cea7c4679147"}, "swoosh": {:hex, :swoosh, "1.8.0", "51b3cfe5bb08196eba0f64b0e30f589533915b5088dd203fde110f29704289db", [:mix], [{:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:ex_aws, "~> 2.1", [hex: :ex_aws, repo: "hexpm", optional: true]}, {:finch, "~> 0.6", [hex: :finch, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fd77341d3e775a81fccfd26baacb9a47328ac8c27df0fb1d06f0312306721472"}, "tailwind": {:hex, :tailwind, "0.1.9", "25ba09d42f7bfabe170eb67683a76d6ec2061952dc9bd263a52a99ba3d24bd4d", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "9213f87709c458aaec313bb5f2df2b4d2cedc2b630e4ae821bf3c54c47a56d0b"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},