Skip to content

Commit

Permalink
pleroma inside™
Browse files Browse the repository at this point in the history 
You will need to add the Pleroma config at `config/pleroma.exs` and
create the database.

& random fixes/improvements
  • Loading branch information
@hrefhref
hrefhref committed Nov 26, 2018
1 parent a5b5bd9 commit 0de576b
Show file tree
Hide file tree
Showing 15 changed files with 284 additions and 81 deletions.
32 changes: 30 additions & 2 deletions config/config.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,32 @@ use Mix.Config
config :fd,
ecto_repos: [Fd.Repo]

# Configures the endpoint
# Configures the endpoint
config :fd, FdWeb.Endpoint,
url: [host: "localhost"],
url: [host: "localhost"],
server: true,
http: [
protocol_options: [max_request_line_length: 8192, max_header_value_length: 8192],
dispatch: [
{:_, [
{'/.well-known/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/nodeinfo/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/api/ostatus[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/main/ostatus/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/objects/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/activities/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/notice/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/users/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/push/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/relay/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/inbox/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/proxy/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/media/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{'/static/[...]', Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}},
{:_, Plug.Adapters.Cowboy.Handler, {FdWeb.Endpoint, []}}
]}
]
],
secret_key_base: "UknRWRgXICA0mCWa7QeT/M5pUPZoyZfUmdIsdSbCEh2ObH4mGcwxXwEqR5IgDJYp",
render_errors: [view: FdWeb.ErrorView, accepts: ~w(html json)],
instrumenters: [FdWeb.PhoenixInstrumenter],
Expand Down Expand Up @@ -52,6 +75,10 @@ config :fd, Fd.Cache,
config :fd, Fd.Repo,
loggers: [Fd.Repo.Instrumenter, Ecto.LogEntry]

config :hammer,
backend: {Hammer.Backend.ETS, [expiry_ms: 60_000 * 60 * 4,
cleanup_interval_ms: 60_000 * 10]}

config :sentry,
dsn: "https://edfc2:[email protected]/42",
environment_name: Mix.env(),
Expand All @@ -64,3 +91,4 @@ config :sentry,
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env}.exs"
import_config "pleroma.exs"
4 changes: 2 additions & 2 deletions lib/fd/application.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ defmodule Fd.Application do
opts = [strategy: :one_for_one, name: Fd.Supervisor]
case Supervisor.start_link(children, opts) do
ok = {:ok, sup} ->
Fd.Pleroma.ensure_general_accounts()
if Application.get_env(:fd, :instances)[:autostart], do: spawn(fn -> run_instances() end)
ok
err -> err
Expand All @@ -50,9 +51,8 @@ defmodule Fd.Application do
unknown_instance_ids = from(i in Instance, select: i.id, where: ((is_nil(i.server) or i.server == 0)) and (is_nil(i.monitor) or i.monitor == false), order_by: [asc: i.last_checked_at])
|> Repo.all
for instance_id <- mon_instance_ids ++ instance_ids ++ unknown_instance_ids do
IO.puts "-- starting instance #{instance_id}"
Fd.Instances.ServerSupervisor.start_child(instance_id)
:timer.sleep(:crypto.rand_uniform(150, 500))
:timer.sleep(:crypto.rand_uniform(10, 50))
end
end

Expand Down
8 changes: 4 additions & 4 deletions lib/fd/email/down_email.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,11 @@ defmodule Fd.DownEmail do
The reason seems to be: #{Map.get(check, "error_s", "unknown")}
We're still fixing some bugs -- if this alert was a false positive, sorry about that! Do not hesitate to report it to us.
We're still fixing some bugs -- if this alert was a false positive, sorry about that! Do not hesitate to report it to @[email protected].
---
This e-mail was sent by https://fediverse.network/monitoring
To unsubscribe, please go to https://fediverse.network/manage and disable the monitoring feature.
To unsubscribe, please go to https://fediverse.network/manage
"""
end

Expand All @@ -44,11 +44,11 @@ defmodule Fd.DownEmail do
<br/><br/>
The reason seems to be: <strong>#{Map.get(check, "error_s", "unknown")}</strong>.
<br/><br/>
We're still fixing some bugs -- if this alert was a false positive, sorry about that! Do not hesitate to report it to us.
We're still fixing some bugs -- if this alert was a false positive, sorry about that! Do not hesitate to report it to @[email protected].
<br/><br/>
---<br/>
This e-mail was sent by <a href="https://fediverse.network/monitoring">fediverse.network monitoring</a>.<br/>
To unsubscribe, please <a href="https://fediverse.network/manage">log-in to manage your instance</a> and disable the monitoring feature.<br/>
To unsubscribe, please <a href="https://fediverse.network/manage">log-in to manage your instance</a>.
"""
end

Expand Down
121 changes: 75 additions & 46 deletions lib/fd/instances/crawler.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@ defmodule Fd.Instances.Crawler do
alias Fd.Instances.{Instance, InstanceCheck, Instrumenter}

@hackney_pool :hackney_crawler
@hackney_pool_opts [{:timeout, 150_000}, {:max_connections, 200}, {:connect_timeout, 150_000}]
@hackney_pool_opts [{:timeout, 150_000}, {:max_connections, 500}, {:connect_timeout, 300_000}]
@hackney_mon_pool :hackney_crawler_mon
@hackney_mon_pool_opts [{:timeout, 150_000}, {:max_connections, 50}, {:connect_timeout, 150_000}]
@hackney_opts [{:pool, @hackney_pool}]
@hackney_mon_opts [{:pool, @hackney_pool}]
@hackney_mon_pool_opts [{:timeout, 150_000}, {:max_connections, 100}, {:connect_timeout, 300_000}]
@hackney_opts [{:connect_timeout, 50_000}, {:recv_timeout, 50_000}, {:pool, @hackney_pool}]
@hackney_mon_opt [{:pool, @hackney_pool}]

@down_http_codes [301, 410, 502, 503, 504, 505, 520, 521, 522, 523, 524, 525, 526, 527, 530]
@retry_http_codes [500, 502, 503, 504, 505, 520, 521, 522, 523, 524]
Expand Down Expand Up @@ -100,7 +100,6 @@ defmodule Fd.Instances.Crawler do
|> process_results()
|> put_public_suffix()
|> put_host_info()
|> check_for_changes()

# TODO: If every check is false (and not halted), decide it's NOT a fediverse instance.
# FIXME: Check for AP/OStatus endpoint as last resort before deciding it's not a fediverse instance.
Expand All @@ -117,18 +116,10 @@ defmodule Fd.Instances.Crawler do
check_changeset = InstanceCheck.changeset(%InstanceCheck{instance_id: instance.id}, check)
Fd.Repo.insert!(check_changeset)

case Instances.update_instance(instance, changes) do
state = case Instances.update_instance(instance, changes) do
{:ok, instance} ->
info(state, "OK -- updated!")
error ->
error(state, "FAIL: #{inspect error}")
end

finished = :erlang.monotonic_time
pipeline_duration = pipeline_stop - start
total_duration = finished - start

info(state, "finished in #{:erlang.convert_time_unit(total_duration, :native, :millisecond)}ms (pipeline took #{:erlang.convert_time_unit(pipeline_duration, :native, :millisecond)} ms)!")
state = check_for_changes(state)

if Application.get_env(:fd, :monitoring_alerts, false) && state.instance.monitor && state.instance.settings && state.instance.settings.alerts_to_contact do
spawn(fn() ->
Expand All @@ -145,6 +136,19 @@ defmodule Fd.Instances.Crawler do
end)
end

info(state, "OK -- updated!")
state
error ->
error(state, "FAIL: #{inspect error}")
state
end

finished = :erlang.monotonic_time
pipeline_duration = pipeline_stop - start
total_duration = finished - start

info(state, "finished in #{:erlang.convert_time_unit(total_duration, :native, :millisecond)}ms (pipeline took #{:erlang.convert_time_unit(pipeline_duration, :native, :millisecond)} ms)!")

spawn(fn() ->
domains = state.m_peers || []
existings = Enum.map(Instances.list_instances_by_domains(domains), fn(i) -> String.downcase(i.domain) end)
Expand Down Expand Up @@ -218,7 +222,7 @@ defmodule Fd.Instances.Crawler do
else false end

diffs = %{new: new?, became_up: became_up?, became_down: became_down?, version_changed: version_changed?,
server_changed: server_changed?}
server_changed: server_changed?, is_up?: is_up?, was_up?: was_up?}

{became_open?, became_closed?} = cond do
signup_changed? && Map.get(crawler.changes, "signup", true) == false ->
Expand All @@ -229,36 +233,41 @@ defmodule Fd.Instances.Crawler do
{false, false}
end

unless (crawler.instance.hidden || false) or Map.get(crawler.changes, "server", 0) == 0 do
IO.puts Map.get(crawler.changes, "server")
unless (crawler.instance.hidden || false) or Map.get(crawler.changes, "server") == 0 do
if became_up? do
post("is back up :)", crawler.instance, "[email protected]")
post("{instance} is back up :)", crawler.instance, [:mon])
end
if became_down? do
IO.puts "BECAME DOWN"
error = if error = Map.get(crawler.check, "error_s") do
" (#{error})"
else
""
end
if crawler.instance.settings.maintenance_mode? do
post("is undergoing planned maintenance#{error}", crawler.instance, "[email protected]")
if Map.get(crawler.instance.settings || %{}, :maintenance_mode) do
post("{instance} is undergoing planned maintenance#{error}", crawler.instance, [:mon])
else
post("is down#{error}", crawler.instance, "[email protected]")
post("{instance} is down#{error}", crawler.instance, [:mon])
end
end
if became_closed? do
post("closed registrations", crawler.instance)
post("closed registrations", crawler.instance, "[email protected]")
post("{instance} closed registrations", crawler.instance, [:watch, :mon])
end
if became_open? do
post("opened registrations", crawler.instance)
post("opened registrations", crawler.instance, "[email protected]")
post("{instance} opened registrations", crawler.instance, [:watch, :mon])
end
if new? do
server_id = Map.get(crawler.changes, "server", 0)
unless server_id == 0 do
server = server_id |> Fd.ServerName.from_int()
post("welcome to the fediverse! a new #{server} instance! \o/", crawler.instance)
post("welcome to the fediverse! a new #{server} instance! \o/", crawler.instance, "[email protected]")
[
"{instance}, welcome to the fediverse! a new {server} instance! \\o/",
"\\o/ please welcome {instance} to the fediverse, a new {server} instance",
"one more {server} instance in the fediverse! welcome, {instance}!"
]
|> Enum.random()
|> post(crawler.instance, [:new, :watch, :mon], %{server: server})
end
end
cond do
Expand All @@ -267,15 +276,13 @@ defmodule Fd.Instances.Crawler do
unless last == 0 do
old_server = last_up_check.server |> Fd.ServerName.from_int()
new_server = Map.get(crawler.changes, "server", 0) |> Fd.ServerName.from_int()
post("changed servers from #{old_server} to #{new_server}", crawler.instance)
post("changed servers from #{old_server} to #{new_server}", crawler.instance, "[email protected]")
post("{instance} changed servers from #{old_server} to #{new_server}", crawler.instance, [:watch, :mon])
end
version_changed? ->
server = Map.get(crawler.changes, "server", 0) |> Fd.ServerName.from_int()
old_version = last_up_check.version
new_version = Map.get(crawler.changes, "version", "?")
post("upgraded #{server} from #{old_version} to #{new_version}:", crawler.instance)
post("upgraded #{server} from #{old_version} to #{new_version}:", crawler.instance, "[email protected]")
post("{instance} upgraded #{server} from #{old_version} to #{new_version}:", crawler.instance, [:watch, :mon])
true -> :nothing_changed
end
end
Expand Down Expand Up @@ -514,8 +521,7 @@ defmodule Fd.Instances.Crawler do
defp process_statusnet_version(version), do: {"GNUSocial", version}


defp process_mastapi_version(nil), do: {"Unknown", nil}
defp process_mastapi_version(string) do
defp process_mastapi_version(string) when is_binary(string) do
{server, version} = cond do
# "universal" compatible (pleroma-like) format: "masto_version; compatible ServerName real_version"
# FIXME: it wont work if the server is not in Fd.ServerName
Expand Down Expand Up @@ -544,6 +550,7 @@ defmodule Fd.Instances.Crawler do
end
{downcase(server), version}
end
defp process_mastapi_version(_), do: {"Unknown", nil}

defp clean_string(string) do
string
Expand Down Expand Up @@ -867,7 +874,7 @@ defmodule Fd.Instances.Crawler do
Instrumenter.http_request(path, response, start)
info(crawler, "http ok - #{inspect response.status_code}")
{:ok, response}
{:error, error = %HTTPoison.Error{reason: reason}} when reason in [:timeout, :connect_timeout, :closed, :nxdomain] ->
{:error, error = %HTTPoison.Error{reason: reason}} when reason in [:timeout, :connect_timeout, :closed, :nxdomain, :ehostunreach] ->
Instrumenter.http_request(path, error, start)
retry(crawler, path, options, {:error, error}, retries)
#if retries > 4 do
Expand All @@ -886,12 +893,12 @@ defmodule Fd.Instances.Crawler do
end

defp retry(crawler, path, options, error, retries) do
if retries > 2 do
if retries > 5 do
error(crawler, "HTTP ERROR (max retries reached): #{inspect error}")
error
else
debug(crawler, "HTTP retry #{inspect retries}: #{inspect error}")
:timer.sleep(:crypto.rand_uniform(retries*2000, retries*3000))
:timer.sleep(:crypto.rand_uniform(retries*2000, retries*5000))
Instrumenter.retry_http_request()
request(crawler, path, options, retries + 1)
end
Expand All @@ -910,19 +917,41 @@ defmodule Fd.Instances.Crawler do
Logger.error "Crawler(#{inspect self()} ##{crawler.instance.id} #{domain}): #{message}"
end

defp post(text, instance) do
post(text, instance, "[email protected]")
end
defp post(text, instance, account) do
text = if instance.hidden do
["private instance", text]
defp post(text, instance, accounts, replaces \\ %{}) do
Logger.warn inspect(instance.settings)
[post_acct | repeat_accts] = if Map.get(instance.settings || %{}, :fedibot) do
[instance.domain] ++ accounts
else
[Fd.Util.idna(instance.domain), text, "— https://fediverse.network/#{Fd.Util.idna(instance.domain)}"]
accounts
end
text = text
|> Enum.join(" ")

Fd.Social.async_post(account, text)
{instance_domain, link} = if instance.hidden do
{"[private]", nil}
else
{Fd.Util.idna(instance.domain), "https://fediverse.network/#{Fd.Util.idna(instance.domain)}"}
end

text = text
|> String.replace("{instance}", instance_domain)
|> (fn(text) ->
Enum.reduce(replaces, text, fn({replace, with_text}, text) ->
String.replace(text, "{#{replace}}", with_text)
end)
end).()
|> (fn(text) ->
[text, link]
end).()
|> Enum.filter(&(&1))
|> Enum.join(" - ")

case Fd.Pleroma.post(post_acct, text) do
{:ok, activity} ->
Fd.Pleroma.repeat(activity.id, repeat_accts)
{:ok, activity}
{:error, error} ->
Logger.error "Failed to post status: #{inspect error}"
{:error, error}
end
end

def downcase(nil), do: nil
Expand Down
3 changes: 2 additions & 1 deletion lib/fd/instances/instance_settings.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,14 @@ defmodule Fd.Instances.InstanceSettings do
field :dead_reason, :string
field :alerts_to_contact, :boolean
field :maintenance_mode, :boolean
field :fedibot, :boolean

field :federation_restrictions_link, :string
end

def changeset(settings = %InstanceSettings{}, attrs) do
settings
|> cast(attrs, [:hidden, :alerts_to_contact, :keep_calm, :dead_reason, :maintenance_mode, :federation_restrictions_link])
|> cast(attrs, [:hidden, :alerts_to_contact, :keep_calm, :dead_reason, :maintenance_mode, :federation_restrictions_link, :fedibot])
|> validate_change(:federation_restrictions_link, fn(_, link) ->
case URI.parse(link) do
%URI{scheme: scheme, host: host} when scheme in ["http", "https"] and is_binary(host) -> []
Expand Down
12 changes: 10 additions & 2 deletions lib/fd/instances/server.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,14 @@ defmodule Fd.Instances.Server do
def init([id]) do
Logger.debug "starting instance #{inspect id}"
instance = Instances.get_instance!(id)
{min_delay, max_delay} = if instance.monitor, do: {0, 2}, else: {0, 8}
delay = (:crypto.rand_uniform(min_delay, max_delay) * 60) * 1000
delay = if instance.last_up_at && DateTime.diff(DateTime.utc_now(), instance.last_up_at) >= 2678400 do
Logger.info "instance #{instance.domain} dead, waiting"
{delay, _} = get_delay(instance)
round(delay / 2)
else
{min_delay, max_delay} = if instance.monitor, do: {0, 2}, else: {0, 8}
(:crypto.rand_uniform(min_delay, max_delay) * 60) * 1000
end
{:ok, timer} = :timer.send_after(delay, self(), :crawl)
{:ok, %__MODULE__{id: id, instance: instance, timer: timer}}
end
Expand All @@ -32,6 +38,7 @@ defmodule Fd.Instances.Server do
def handle_info(:crawl, state = %__MODULE__{id: id}) do
if state.timer, do: :timer.cancel(state.timer)
instance = Instances.get_instance!(id)
if Map.get(instance.settings || %{}, :fedibot), do: {:ok, _} = Fd.Pleroma.create_or_get_user(instance.domain, instance.domain, "https://fediverse.network/#{instance.domain}")
if @dev do
Fd.Instances.Crawler.run(instance)
else
Expand Down Expand Up @@ -71,6 +78,7 @@ defmodule Fd.Instances.Server do
instance.dead -> :instance_dead
instance.settings && instance.monitor && instance.settings.keep_calm -> :instance_monitor_calm
instance.monitor -> :instance_monitor
instance.last_up_at && DateTime.diff(DateTime.utc_now(), instance.last_up_at) >= 2678400 -> :instance_dead
instance.settings && instance.settings.keep_calm -> :instance_calm
instance.server == 0 -> :instance_calm
true -> :instance_default
Expand Down
Loading

0 comments on commit 0de576b

Please sign in to comment.