diff --git a/config/config.exs b/config/config.exs
index 888aeb8..51ef5ba 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -31,6 +31,8 @@ server: true,
{'/proxy/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/media/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/static/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
+ {'/api/v1/instance/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
+ {'/api/statusnet/config', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{:_, Plug.Adapters.Cowboy.Handler, {FdWeb.Endpoint, []}}
]}
]
diff --git a/lib/fd/cache.ex b/lib/fd/cache.ex
index 6215a77..b134990 100644
--- a/lib/fd/cache.ex
+++ b/lib/fd/cache.ex
@@ -130,7 +130,6 @@ defmodule Fd.Cache do
for key <- get(ctx)||[] do
[ctx, key]
|> key()
- |> IO.inspect
|> delete()
end
delete(ctx)
diff --git a/lib/fd/domain.ex b/lib/fd/domain.ex
index 0157e45..2fb1d1e 100644
--- a/lib/fd/domain.ex
+++ b/lib/fd/domain.ex
@@ -1,10 +1,8 @@
defmodule Fd.Domain do
def total_count(stats, domain) do
- IO.puts "total count #{inspect domain}"
case Enum.find(stats["domains"], fn({d, _}) -> d == domain end) do
{_, %{"total" => total}} -> total
_ -> 0
end
- |> IO.inspect()
end
end
diff --git a/lib/fd/human_error.ex b/lib/fd/human_error.ex
index 6eaa2f3..e562386 100644
--- a/lib/fd/human_error.ex
+++ b/lib/fd/human_error.ex
@@ -22,7 +22,13 @@ defmodule Fd.HumanError do
end
def format({:error, atom}) when is_atom(atom) do
- to_string(atom)
+ format(atom)
+ end
+
+ def format(atom) when is_atom(atom) do
+ atom
+ |> to_string()
+ |> String.replace("_", " ")
end
def format(error) do
diff --git a/lib/fd/instances/crawler.ex b/lib/fd/instances/crawler.ex
index 20e7093..ec9cfb5 100644
--- a/lib/fd/instances/crawler.ex
+++ b/lib/fd/instances/crawler.ex
@@ -23,11 +23,14 @@ defmodule Fd.Instances.Crawler do
alias Fd.Instances.{Instance, InstanceCheck, Instrumenter}
@hackney_pool :hackney_crawler
- @hackney_pool_opts [{:timeout, 150_000}, {:max_connections, 500}, {:connect_timeout, 300_000}]
+ @hackney_pool_opts [{:timeout, 50_000}, {:max_connections, 200}, {:connect_timeout, 50_000}]
@hackney_mon_pool :hackney_crawler_mon
- @hackney_mon_pool_opts [{:timeout, 150_000}, {:max_connections, 100}, {:connect_timeout, 300_000}]
- @hackney_opts [{:connect_timeout, 50_000}, {:recv_timeout, 50_000}, {:pool, @hackney_pool}]
- @hackney_mon_opt [{:pool, @hackney_pool}]
+ @hackney_mon_pool_opts [{:timeout, 50_000}, {:max_connections, 100}, {:connect_timeout, 50_000}]
+ @hackney_dead_pool :hackney_crawler_dead
+ @hackney_dead_pool_opts [{:timeout, 50_000}, {:max_connections, 100}, {:connect_timeout, 150_000}]
+ @hackney_opts [{:connect_timeout, 150_000}, {:recv_timeout, 50_000}, {:pool, @hackney_pool}]
+ @hackney_mon_opts [{:pool, @hackney_mon_pool}]
+ @hackney_dead_opts [{:pool, @hackney_dead_pool}]
@down_http_codes [301, 410, 502, 503, 504, 505, 520, 521, 522, 523, 524, 525, 526, 527, 530]
@retry_http_codes [500, 502, 503, 504, 505, 520, 521, 522, 523, 524]
@@ -74,11 +77,12 @@ defmodule Fd.Instances.Crawler do
def setup() do
:ok = :hackney_pool.start_pool(@hackney_pool, @hackney_pool_opts)
:ok = :hackney_pool.start_pool(@hackney_mon_pool, @hackney_mon_pool_opts)
+ :ok = :hackney_pool.start_pool(@hackney_dead_pool, @hackney_dead_pool_opts)
end
def run(instance = %Instance{domain: domain}) do
state = %Crawler{instance: instance, halted?: false, has_mastapi?: false, has_statusnet?: false, has_peertubeapi?:
- false, has_nodeinfo?: false, has_misskey?: false, changes: %{}, check: %{}}
+ false, has_nodeinfo?: false, has_misskey?: false, changes: %{}, check: %{}, diffs: %{}}
start = :erlang.monotonic_time
@@ -110,33 +114,34 @@ defmodule Fd.Instances.Crawler do
|> Map.put("last_checked_at", DateTime.utc_now())
|> Map.put("nodeinfo", state.nodeinfo)
+ state = %Crawler{state | changes: changes}
+
debug(state, "changes: #{inspect changes}")
- check = state.check
- check_changeset = InstanceCheck.changeset(%InstanceCheck{instance_id: instance.id}, check)
+ check_changeset = InstanceCheck.changeset(%InstanceCheck{instance_id: instance.id}, state.check)
Fd.Repo.insert!(check_changeset)
- state = case Instances.update_instance(instance, changes) do
+ state = case Instances.update_instance(instance, state.changes) do
{:ok, instance} ->
state = check_for_changes(state)
- if Application.get_env(:fd, :monitoring_alerts, false) && state.instance.monitor && state.instance.settings && state.instance.settings.alerts_to_contact do
- spawn(fn() ->
- became_down? = Map.get(state.diffs, :became_down, false)
- became_up? = Map.get(state.diffs, :became_up, false)
- if became_down? do
- Fd.DownEmail.down_email(state.instance, state.check)
- |> Fd.Mailer.deliver()
- end
- if became_up? do
- Fd.UpEmail.up_email(state.instance)
- |> Fd.Mailer.deliver()
+ if Application.get_env(:fd, :monitoring_alerts, false) && state.instance.monitor && state.instance.settings && state.instance.settings.alerts_to_contact do
+ spawn(fn() ->
+ became_down? = Map.get(state.diffs, :became_down, false)
+ became_up? = Map.get(state.diffs, :became_up, false)
+ if became_down? do
+ Fd.DownEmail.down_email(state.instance, state.check)
+ |> Fd.Mailer.deliver()
+ end
+ if became_up? do
+ Fd.UpEmail.up_email(state.instance)
+ |> Fd.Mailer.deliver()
+ end
+ end)
end
- end)
- end
- info(state, "OK -- updated!")
+ warn(state, "Checked! #{inspect state.diffs}")
state
error ->
error(state, "FAIL: #{inspect error}")
@@ -147,7 +152,7 @@ defmodule Fd.Instances.Crawler do
pipeline_duration = pipeline_stop - start
total_duration = finished - start
- info(state, "finished in #{:erlang.convert_time_unit(total_duration, :native, :millisecond)}ms (pipeline took #{:erlang.convert_time_unit(pipeline_duration, :native, :millisecond)} ms)!")
+ warn(state, "finished in #{:erlang.convert_time_unit(total_duration, :native, :millisecond)}ms (pipeline took #{:erlang.convert_time_unit(pipeline_duration, :native, :millisecond)} ms)!")
spawn(fn() ->
domains = state.m_peers || []
@@ -164,8 +169,19 @@ defmodule Fd.Instances.Crawler do
|> Enum.filter(fn(domain) -> domain end)
|> Enum.reject(fn(domain) -> Enum.member?(existings, domain) end)
- for domain <- new_domains, do: Instances.create_instance(%{"domain" => domain})
+ for domain <- new_domains do
+ case Instances.create_instance(%{"domain" => domain}) do
+ {:ok, instance} ->
+ spawn(fn() ->
+ :timer.sleep(:rand.uniform(60) * 1000)
+ Fd.Instances.Server.crawl(instance.id)
+ end)
+ _ -> :ok
+ end
+ end
end)
+
+ state
end
defp put_public_suffix(crawler) do
@@ -220,10 +236,6 @@ defmodule Fd.Instances.Crawler do
server_changed? = if last_up_check && is_up? do
Map.get(crawler.changes, "server", 0) != last_up_check.server
else false end
-
- diffs = %{new: new?, became_up: became_up?, became_down: became_down?, version_changed: version_changed?,
- server_changed: server_changed?, is_up?: is_up?, was_up?: was_up?}
-
{became_open?, became_closed?} = cond do
signup_changed? && Map.get(crawler.changes, "signup", true) == false ->
{false, true}
@@ -233,13 +245,15 @@ defmodule Fd.Instances.Crawler do
{false, false}
end
- IO.puts Map.get(crawler.changes, "server")
+ diffs = %{new: new?, became_up: became_up?, became_down: became_down?, version_changed: version_changed?,
+ server_changed: server_changed?, is_up?: is_up?, was_up?: was_up?, became_open?: became_open?, became_closed?: became_closed?}
+
+
unless (crawler.instance.hidden || false) or Map.get(crawler.changes, "server") == 0 do
if became_up? do
post("{instance} is back up :)", crawler.instance, [:mon])
end
if became_down? do
- IO.puts "BECAME DOWN"
error = if error = Map.get(crawler.check, "error_s") do
" (#{error})"
else
@@ -285,6 +299,7 @@ defmodule Fd.Instances.Crawler do
post("{instance} upgraded #{server} from #{old_version} to #{new_version}:", crawler.instance, [:watch, :mon])
true -> :nothing_changed
end
+
end
debug(crawler, "Diffs: " <> inspect(diffs))
@@ -813,7 +828,6 @@ defmodule Fd.Instances.Crawler do
def query_html_index(crawler = %Crawler{halted?: false}) do
case request(crawler, "/", [json: false, follow_redirects: true]) do
{:ok, resp = %HTTPoison.Response{status_code: 200, body: body}} ->
- IO.inspect resp
%Crawler{crawler | html: body}
error ->
info(crawler, "failed to get index page #{inspect error}")
@@ -833,11 +847,18 @@ defmodule Fd.Instances.Crawler do
timeout = Keyword.get(options, :timeout, 15_000)
recv_timeout = Keyword.get(options, :recv_timeout, 15_000)
body = Keyword.get(options, :body, "")
- {mon_ua, options} = if crawler.instance.monitor do
- mon_ua = " - monitoring enabled https://fediverse.network/monitoring"
- {mon_ua, [hackney: @hackney_mon_opts]}
- else
- {"", [hackney: @hackney_opts]}
+ {mon_ua, options} = cond do
+ crawler.instance.monitor ->
+ mon_ua = " - monitoring enabled https://fediverse.network/monitoring"
+ {mon_ua, [hackney: @hackney_mon_opts]}
+ crawler.instance.dead || !crawler.instance.last_up_at ->
+ {"", [hackney: @hackney_dead_opts]}
+ crawler.instance.last_up_at && DateTime.diff(DateTime.utc_now(), crawler.instance.last_up_at) >= 2678400 ->
+ {"", [hackney: @hackney_dead_opts]}
+ !crawler.instance.server || crawler.instance.server == 0 ->
+ {"", [hackney: @hackney_dead_opts]}
+ true ->
+ {"", [hackney: @hackney_opts]}
end
options = [timeout: timeout, recv_timeout: recv_timeout, follow_redirect: follow_redirects] ++ options
dev_ua = if @env == :dev, do: " [dev]", else: ""
@@ -851,7 +872,6 @@ defmodule Fd.Instances.Crawler do
Map.put(headers, "Accept", accept)
else headers end
start = :erlang.monotonic_time
- IO.puts "-- #{domain} #{path} #{inspect(headers)}"
case HTTPoison.request(method, "https://#{domain}#{path}", body, headers, options) do
{:ok, response = %HTTPoison.Response{status_code: 200, body: body}} ->
Instrumenter.http_request(path, response, start)
@@ -893,12 +913,12 @@ defmodule Fd.Instances.Crawler do
end
defp retry(crawler, path, options, error, retries) do
- if retries > 5 do
+ if retries > 2 do
error(crawler, "HTTP ERROR (max retries reached): #{inspect error}")
error
else
debug(crawler, "HTTP retry #{inspect retries}: #{inspect error}")
- :timer.sleep(:crypto.rand_uniform(retries*2000, retries*5000))
+ :timer.sleep(:crypto.rand_uniform(retries*500, retries*1000))
Instrumenter.retry_http_request()
request(crawler, path, options, retries + 1)
end
@@ -912,6 +932,10 @@ defmodule Fd.Instances.Crawler do
domain = crawler.instance.domain
Logger.info "Crawler(#{inspect self()} ##{crawler.instance.id} #{domain}): #{message}"
end
+ def warn(crawler, message) do
+ domain = crawler.instance.domain
+ Logger.warn "Crawler(#{inspect self()} ##{crawler.instance.id} #{domain}): #{message}"
+ end
def error(crawler, message) do
domain = crawler.instance.domain
Logger.error "Crawler(#{inspect self()} ##{crawler.instance.id} #{domain}): #{message}"
@@ -919,7 +943,7 @@ defmodule Fd.Instances.Crawler do
defp post(text, instance, accounts, replaces \\ %{}) do
Logger.warn inspect(instance.settings)
- [post_acct | repeat_accts] = if Map.get(instance.settings || %{}, :fedibot) do
+ accounts = if Map.get(instance.settings || %{}, :fedibot) do
[instance.domain] ++ accounts
else
accounts
@@ -944,14 +968,7 @@ defmodule Fd.Instances.Crawler do
|> Enum.filter(&(&1))
|> Enum.join(" - ")
- case Fd.Pleroma.post(post_acct, text) do
- {:ok, activity} ->
- Fd.Pleroma.repeat(activity.id, repeat_accts)
- {:ok, activity}
- {:error, error} ->
- Logger.error "Failed to post status: #{inspect error}"
- {:error, error}
- end
+ Fd.Pleroma.post(accounts, text)
end
def downcase(nil), do: nil
diff --git a/lib/fd/instances/crawler/nodeinfo.ex b/lib/fd/instances/crawler/nodeinfo.ex
index 73ea9f6..7b91315 100644
--- a/lib/fd/instances/crawler/nodeinfo.ex
+++ b/lib/fd/instances/crawler/nodeinfo.ex
@@ -40,19 +40,22 @@ defmodule Fd.Instances.Crawler.Nodeinfo do
href = Map.get(schema, "href")
uri = URI.parse(href)
version = detect_version(Map.get(schema, "rel"))
- if version && uri.host == crawler.instance.domain do
- [{version, uri.path} | acc]
- else
- acc
+ cond do
+ version && uri.host == crawler.instance.domain -> [{version, uri.path} | acc]
+ true -> acc
end
end)
|> Enum.sort_by(fn({v, _}) -> v end, &>=/2)
- {version, path} = List.first(links)
- query_nodeinfo(crawler, version, path)
+ query_nodeinfo(crawler, List.first(links))
end
- defp query_nodeinfo(crawler, version, path) do
- error(crawler, "Should crawl Nodeinfo ver #{inspect version} at path #{inspect path}")
+ defp query_nodeinfo(crawler) do
+ %Crawler{crawler | has_nodeinfo?: false}
+ end
+
+
+ defp query_nodeinfo(crawler, {version, path}) do
+ debug(crawler, "Should crawl Nodeinfo ver #{inspect version} at path #{inspect path}")
case request(crawler, path) do
{:ok, %HTTPoison.Response{status_code: 200, body: body}} ->
debug(crawler, "got nodeinfo#{inspect version} #{inspect path} " <> inspect(body))
@@ -72,6 +75,15 @@ defmodule Fd.Instances.Crawler.Nodeinfo do
end
end
+ defp query_nodeinfo(crawler, nil) do
+ debug(crawler, "no valid nodeinfo")
+ %Crawler{crawler | halted?: true, fatal_error: :invalid_nodeinfo}
+ end
+
+ defp query_nodeinfo(crawler, _) do
+ %Crawler{crawler | has_nodeinfo?: false}
+ end
+
defp detect_version("http://nodeinfo.diaspora.software/ns/schema/"<>float) do
case Float.parse(float) do
{version, _} -> version
diff --git a/lib/fd/instances/instances.ex b/lib/fd/instances/instances.ex
index b0ae003..f2bdcad 100644
--- a/lib/fd/instances/instances.ex
+++ b/lib/fd/instances/instances.ex
@@ -36,6 +36,7 @@ defmodule Fd.Instances do
** (Ecto.NoResultsError)
"""
+
def get_instance!(id), do: Repo.get!(Instance, id)
def get_instance_by_domain!(domain) do
@@ -324,4 +325,25 @@ defmodule Fd.Instances do
|> Map.put_new(:settings, %InstanceSettings{})
|> Instance.changeset(%{})
end
+
+ def delay(%Instance{} = instance) do
+ cond do
+ dead_instance?(instance) -> :instance_dead
+ instance.settings && instance.monitor && instance.settings.keep_calm -> :instance_monitor_calm
+ instance.monitor -> :instance_monitor
+ instance.settings && instance.settings.keep_calm -> :instance_calm
+ instance.server == 0 -> :instance_calm
+ true -> :instance_default
+ end
+ end
+
+ def dead?(%Instance{dead: true}), do: true
+ def dead?(%Instance{} = instance}) do
+ cond do
+ instance.last_up_at && DateTime.diff(DateTime.utc_now(), instance.last_up_at) >= @dead_after_secs -> true
+ !instance.last_up_at && DateTime.diff(DateTime.utc_now(), instance.inserted_at) >= @dead_after_secs -> true
+ true -> false
+ end
+ end
+
end
diff --git a/lib/fd/instances/server.ex b/lib/fd/instances/server.ex
index 36c913c..864a8c6 100644
--- a/lib/fd/instances/server.ex
+++ b/lib/fd/instances/server.ex
@@ -22,16 +22,23 @@ defmodule Fd.Instances.Server do
def init([id]) do
Logger.debug "starting instance #{inspect id}"
instance = Instances.get_instance!(id)
- delay = if instance.last_up_at && DateTime.diff(DateTime.utc_now(), instance.last_up_at) >= 2678400 do
- Logger.info "instance #{instance.domain} dead, waiting"
- {delay, _} = get_delay(instance)
- round(delay / 2)
+ blacklisted? = Enum.any?(Application.get_env(:fd, :blacklist, []), fn(denied) ->
+ String.ends_with?(instance.domain, denied)
+ end)
+ if blacklisted? do
+ :ignore
else
- {min_delay, max_delay} = if instance.monitor, do: {0, 2}, else: {0, 8}
- (:crypto.rand_uniform(min_delay, max_delay) * 60) * 1000
+ delay = if Instances.dead?(instance) do
+ Logger.info "instance #{instance.domain} dead, waiting"
+ {delay, _} = get_delay(instance)
+ (:crypto.rand_uniform(round(delay / 3), round(delay / 2)) * 60) * 1000
+ else
+ {min_delay, max_delay} = if instance.monitor, do: {0, 2}, else: {0, 8}
+ (:crypto.rand_uniform(min_delay, max_delay) * 60) * 1000
+ end
+ {:ok, timer} = :timer.send_after(delay, self(), :crawl)
+ {:ok, %__MODULE__{id: id, instance: instance, timer: timer}}
end
- {:ok, timer} = :timer.send_after(delay, self(), :crawl)
- {:ok, %__MODULE__{id: id, instance: instance, timer: timer}}
end
@dev Mix.env == :dev
@@ -46,11 +53,11 @@ defmodule Fd.Instances.Server do
Fd.Instances.Crawler.run(instance)
rescue
e ->
- Sentry.capture_exception(e, [stacktrace: System.stacktrace(), extra: %{instance_id: id}])
+ Sentry.capture_exception(e, [stacktrace: System.stacktrace(), extra: %{instance: instance.domain}])
Logger.error "Server #{inspect(id)} rescued: #{inspect e}"
catch
e ->
- Sentry.capture_exception(e, [extra: %{instance_id: id}])
+ Sentry.capture_exception(e, [extra: %{instance: instance.domain}])
Logger.error "Server #{inspect(id)} catched: #{inspect e}"
end
end
@@ -74,15 +81,8 @@ defmodule Fd.Instances.Server do
end
defp get_delay(instance) do
- cond do
- instance.dead -> :instance_dead
- instance.settings && instance.monitor && instance.settings.keep_calm -> :instance_monitor_calm
- instance.monitor -> :instance_monitor
- instance.last_up_at && DateTime.diff(DateTime.utc_now(), instance.last_up_at) >= 2678400 -> :instance_dead
- instance.settings && instance.settings.keep_calm -> :instance_calm
- instance.server == 0 -> :instance_calm
- true -> :instance_default
- end
+ instance
+ |> Instances.delay()
|> Fd.Util.get_delay()
|> (fn(delay) ->
Logger.debug "Crawl delay for instance #{to_string(instance.id)} set to #{to_string(delay)}"
diff --git a/lib/fd/instances/server_supervisor.ex b/lib/fd/instances/server_supervisor.ex
index 9b6e9e2..947dcc2 100644
--- a/lib/fd/instances/server_supervisor.ex
+++ b/lib/fd/instances/server_supervisor.ex
@@ -14,7 +14,7 @@ defmodule Fd.Instances.ServerSupervisor do
worker(Fd.Instances.Server, [], restart: :transient)
]
- supervise(children, strategy: :simple_one_for_one)
+ supervise(children, strategy: :simple_one_for_one, max_restarts: 100, max_seconds: 2)
end
end
diff --git a/lib/fd/pleroma.ex b/lib/fd/pleroma.ex
index 96c7ca6..3a34548 100644
--- a/lib/fd/pleroma.ex
+++ b/lib/fd/pleroma.ex
@@ -18,23 +18,24 @@ defmodule Fd.Pleroma do
end
end
+ def get_user(:disabled), do: :disabled
+
def get_user(nick) when is_atom(nick) do
- IO.puts "get user with atom #{inspect nick}"
- with {nick, _, _} <- Map.get(@general_accounts, nick)
+ with {nick, _, _} <- Map.get(@general_accounts, nick),
+ true <- Application.get_env(:fd, :monitoring_alerts, true)
do
get_user(nick)
else
+ _ -> :disabled
_ -> {:error, :not_defined_account}
end
end
def get_user(user=%User{}) do
- IO.puts "Get user with user #{inspect user.id}"
{:ok, user}
end
def get_user(nick) when is_binary(nick) do
- IO.puts "Get user with nick #{inspect nick}"
if user = User.get_by_nickname(sanitize_nick(nick)) do
{:ok, user}
else
@@ -74,32 +75,39 @@ defmodule Fd.Pleroma do
end
end
- def post(nick, status) do
+ def post(nick, status, repeats \\ []) do
with \
- {:ok, user} <- get_user(nick),
- {:allow, _} <- Hammer.check_rate("pleroma:post:#{user.id}", 60_000, 3),
+ [post_nick | repeat_nicks] <- (case nick do
+ nick when is_binary(nick) -> [nick]
+ nick when is_list(nick) -> nick
+ end),
+ {:ok, user} <- get_user(post_nick),
+ {:allow, _} <- Hammer.check_rate("pleroma:post:#{user.id}", 120_000, 30),
{:ok, activity} <- CommonAPI.post(user, %{"status" => status})
do
- Logger.info "#{__MODULE__} Posted to #{nick}: #{inspect status}"
+ Logger.info "#{__MODULE__} Posted to #{post_nick} (repeats: #{inspect(repeat_nicks)}): #{inspect status}"
+ repeat(activity.id, repeat_nicks)
{:ok, activity}
else
+ :disabled -> {:ok, %{id: :disabled}}
{:error, error} -> {:error, error}
{:deny, _} -> {:error, :rate_limited}
end
end
def repeat(_, []), do: []
+ def repeat(_, :disabled), do: :ok
+ def repeat(:disabled, _), do: :ok
def repeat(ap_id, users) when is_list(users) do
- IO.puts "repeat with users #{inspect users}"
users = for user <- users, do: repeat(ap_id, get_user(user))
end
def repeat(ap_id, user) when is_binary(user) or is_atom(user) do
- IO.puts "repeat with binary #{inspect user}"
repeat(ap_id, get_user(user))
end
+
def repeat(ap_id, {:ok, user}), do: repeat(ap_id, user)
def repeat(ap_id, user = %User{}) do
diff --git a/lib/fd_web/controllers/instance_controller.ex b/lib/fd_web/controllers/instance_controller.ex
index 6abff00..cb273da 100644
--- a/lib/fd_web/controllers/instance_controller.ex
+++ b/lib/fd_web/controllers/instance_controller.ex
@@ -76,7 +76,14 @@ defmodule FdWeb.InstanceController do
def index(conn, params) do
{instances, filters, stats} = basic_filter(conn, params)
- render(conn, "index.html", stats: stats, instances: instances, title: "Instances", filters: filters)
+ title = case params do
+ %{"domain" => domain} -> "Instances on #{domain} sub-domains"
+ %{"tld" => tld} -> "Instances on TLD .#{tld}"
+ _ -> "Instances"
+ end
+ conn
+ |> assign(:title, title)
+ |> render("index.html", stats: stats, instances: instances, title: title, filters: filters)
end
def new(conn, _params) do
diff --git a/lib/fd_web/controllers/page_controller.ex b/lib/fd_web/controllers/page_controller.ex
index 551e01d..f5894c3 100644
--- a/lib/fd_web/controllers/page_controller.ex
+++ b/lib/fd_web/controllers/page_controller.ex
@@ -27,6 +27,16 @@ defmodule FdWeb.PageController do
render(conn, "stats.html", stats: stats)
end
+ def trap(conn, _) do
+ conn
+ |> redirect(external: "http://ping.online.net/10000Mo.dat")
+ end
+
+ def not_found(conn, _) do
+ conn
+ |> send_resp(404, "Not found")
+ end
+
defp get_global_stats(params) do
interval = Map.get(params, "interval", "3hour")
diff --git a/lib/fd_web/router.ex b/lib/fd_web/router.ex
index 34c801f..b6fd9f4 100644
--- a/lib/fd_web/router.ex
+++ b/lib/fd_web/router.ex
@@ -40,6 +40,17 @@ defmodule FdWeb.Router do
get "/domain", InstanceController, :domain, as: :instance
get "/domain/:domain", InstanceController, :index, as: :instance_domain
+
+ @traps ~w(wp-login.php)
+ for trap <- @traps do
+ get "/#{trap}", PageController, :trap
+ end
+
+ @notfound ~w(statistics.json siteinfo.json poco)
+ for notfound <- @notfound do
+ get "/#{notfound}", PageController, :not_found
+ end
+
# Manage
get "/manage", ManageController, :index, as: :manage
get "/:instance_id/manage", ManageController, :show, as: :manage
diff --git a/lib/fd_web/templates/instance/domain.html.eex b/lib/fd_web/templates/instance/domain.html.eex
index bb9fc7c..4a29855 100644
--- a/lib/fd_web/templates/instance/domain.html.eex
+++ b/lib/fd_web/templates/instance/domain.html.eex
@@ -3,7 +3,7 @@
Only domains with more than one instance are listed. <%= link "List by TLDs", to: instance_path(@conn, :tld) %>.
- <%= for {tld, stats} <- Map.get(@stats, "domains") do %>
+ <%= for {tld, stats} <- Enum.sort_by(Map.get(@stats, "domains"), &(Map.get(elem(&1, 1), "up", 0)), &>=/2) do %>
<%= link [".", idna(tld)], to: instance_domain_path(@conn, :index, idna(tld)) %>
<%= Map.get(stats, "up", "?") %>
diff --git a/lib/fd_web/templates/instance/show.html.eex b/lib/fd_web/templates/instance/show.html.eex
index 3f55a95..68f1490 100644
--- a/lib/fd_web/templates/instance/show.html.eex
+++ b/lib/fd_web/templates/instance/show.html.eex
@@ -15,7 +15,9 @@
<%= simplify_version(@instance.version || "?") %>
+ <%= version_build_label(@instance.version) %>
+
<%= link "List by domains", to: instance_path(@conn, :domain) %>.
-
- <%= for {tld, stats} <- Map.get(@stats, "tlds") do %>
-
-
<%= link [".", idna(tld)], to: instance_tld_path(@conn, :index, idna(tld)) %>
+
+ <%= for {{tld, stats = %{"up" => up}}, rank} when up > 0 <- Enum.with_index(Enum.sort_by(Map.get(@stats, "tlds"), &(Map.get(elem(&1, 1), "up", 0)), &>=/2)) do %>
+
+
<%= rank+1 %>. <%= link [".", idna(tld)], to: instance_tld_path(@conn, :index, idna(tld)) %>
<%= Map.get(stats, "up", "?") %>
/ <%= Map.get(stats, "total", "?") %>
-
+
<%= for {server, sstats} <- Map.get(stats, "per_server", %{}) |> Enum.sort_by(fn({_,s}) -> Map.get(s,"total",0) end,&>=/2) do %>
<%= unless Map.get(sstats, "total", 0) == 0 do %>
- - <%= link Fd.ServerName.from_int(server), to: instance_tld_path(@conn, :index, tld, server:
+
- <%= link Fd.ServerName.from_int(server), to: instance_tld_path(@conn, :index, tld, server:
Fd.ServerName.from_int(server)|>String.downcase()) %>:
<%= Map.get(sstats, "up", "?") %>/<%= Map.get(sstats, "total", "?") %>
<% end %>
diff --git a/lib/fd_web/templates/page/monitoring.html.md b/lib/fd_web/templates/page/monitoring.html.md
index e24ad02..591cb11 100644
--- a/lib/fd_web/templates/page/monitoring.html.md
+++ b/lib/fd_web/templates/page/monitoring.html.md
@@ -5,6 +5,6 @@
* Instance checked by the crawler every minute/two minute
* E-mail alerts when your instance becomes unavailable, or is available again
* More detailed stats
-* Tested from a location (Hetzner Finland) but other locations may be added in the future
+* Tested from a location (Chicago, USA) but other locations will be added in the future
To enable this option please [login to manage your instance](/manage). :)
diff --git a/lib/fd_web/views/instance_view.ex b/lib/fd_web/views/instance_view.ex
index 31915bb..3266859 100644
--- a/lib/fd_web/views/instance_view.ex
+++ b/lib/fd_web/views/instance_view.ex
@@ -3,21 +3,59 @@ defmodule FdWeb.InstanceView do
alias Fd.Instances.Instance
- def simplify_version(string) when is_binary(string) do
- simple = string
- |> String.split(~r/[\s-]/, parts: 2)
- |> List.first
- |> (fn
- string when byte_size(string) < 15 ->
- string
- _ -> nil
- end).()
+ def version_build_label(string) do
+ with \
+ [_, builds] <- String.split(string, "+", parts: 2),
+ build = String.slice(builds, 0..15)
+ do
+ content_tag(:div, content_tag(:small, build), [title: builds])
+ else
+ err ->
+ ""
+ end
+ end
- cond do
- simple == string -> string
- simple == nil -> simple
+ def simplify_version(nil), do: nil
+ def simplify_version(""), do: ""
+
+ def simplify_version("v"<>string), do: simplify_version(string)
+
+ def simplify_version(string) do
+ full_string = string
+ {string, build} = case String.split(string, "+", parts: 2) do
+ [string, build] -> {string, build}
+ [string] -> {string, nil}
+ end
+
+ {suffix, simple_version} = cond do
+ String.match?(string, ~r/^[0-9a-z]+$/) and String.length(string) > 8 ->
+ << string :: binary-size(8), _ :: binary >> = string
+ {"*", string}
+ String.match?(string, ~r/^[0-9a-z]+$/) ->
+ {nil, string}
true ->
- content_tag(:span, [simple, content_tag(:sup, "*")], [title: string])
+ simplify_version_string(full_string)
+ end
+
+ suffix = if build, do: "+", else: suffix
+
+
+ cond do
+ suffix && simple_version -> content_tag(:span, [simple_version, content_tag(:sup, suffix)], [title: full_string])
+ suffix -> content_tag(:span, content_tag(:sup, suffix), [title: full_string])
+ true -> string
+ end
+ end
+
+ def simplify_version_string(string) do
+ case String.split(string, ~r/[^0-9.]/, parts: 2) do
+ [simple, ext] -> {"*", simple}
+ [version] ->
+ if String.length(version) >= 8 do
+ {"*", nil}
+ else
+ {nil, nil}
+ end
end
end