diff --git a/lib/plausible.ex b/lib/plausible.ex new file mode 100644 index 0000000000000000000000000000000000000000..b81c40fa448db840951546aaf79f6510ae33251a --- /dev/null +++ b/lib/plausible.ex @@ -0,0 +1,82 @@ +defmodule Plausible do + @moduledoc """ + Build-related macros + """ + + @ce_builds [:ce, :ce_test, :ce_dev] + + defmacro __using__(_) do + quote do + require Plausible + import Plausible + end + end + + defmacro on_ee(clauses) do + do_on_ee(clauses) + end + + defmacro on_ce(clauses) do + do_on_ce(clauses) + end + + # :erlang.phash2(1, 1) == 0 tricks dialyzer as per: + # https://github.com/elixir-lang/elixir/blob/v1.12.3/lib/elixir/lib/gen_server.ex#L771-L778 + # and also tricks elixir >1.18 type checker + + defmacro always(term) do + quote do + case :erlang.phash2(1, 1) do + 0 -> + unquote(term) + + 1 -> + not unquote(term) + end + end + end + + defmacro ee? do + quote do + always(unquote(Mix.env() not in @ce_builds)) + end + end + + defmacro ce? do + quote do + always(unquote(Mix.env() in @ce_builds)) + end + end + + defp do_on_ce(do: block) do + do_on_ee(do: nil, else: block) + end + + defp do_on_ce(do: do_block, else: else_block) do + do_on_ee(do: else_block, else: do_block) + end + + defp do_on_ee(do: block) do + do_on_ee(do: block, else: nil) + end + + defp do_on_ee(do: do_block, else: else_block) do + if ee?() do + quote do + unquote(do_block) + end + else + quote do + unquote(else_block) + end + end + end + + def product_name do + if ee?() do + "Plausible Analytics" + else + "Plausible CE" + end + end +end diff --git a/lib/plausible/data_migration.ex b/lib/plausible/data_migration.ex new file mode 100644 index 0000000000000000000000000000000000000000..48f847261e5151caf7c1f69af6cedc201180716d --- /dev/null +++ b/lib/plausible/data_migration.ex @@ -0,0 +1,127 @@ +defmodule Plausible.DataMigration do + @moduledoc """ + Base module for coordinated Clickhouse data migrations + run via remote shell or otherwise (TBD). + """ + + defmacro __using__(opts) do + dir = Keyword.fetch!(opts, :dir) + repo = Keyword.get(opts, :repo, Plausible.DataMigration.ClickhouseRepo) + + quote bind_quoted: [dir: dir, repo: repo] do + @dir dir + @repo repo + + def run_sql_confirm(name, assigns \\ [], options \\ []) do + query = unwrap_with_io(name, assigns) + message = Keyword.get(options, :prompt_message, "Execute?") + default_choice = Keyword.get(options, :prompt_default_choice, :yes) + confirm(message, fn -> do_run(name, query) end, default_choice) + end + + def confirm(message, func, default_choice \\ :yes) do + choices = + case default_choice do + :yes -> " [Y/n]: " + :no -> " [y/N]: " + end + + prompt = IO.ANSI.white() <> message <> choices <> IO.ANSI.reset() + answer = String.downcase(String.trim(IO.gets(prompt))) + + skip = fn -> + IO.puts(" #{IO.ANSI.cyan()}Skipped.#{IO.ANSI.reset()}") + {:ok, :skip} + end + + case answer do + "y" -> + func.() + + "n" -> + skip.() + + _ -> + case default_choice do + :yes -> func.() + :no -> skip.() + end + end + end + + def unwrap(name, assigns \\ []) do + :plausible + |> :code.priv_dir() + |> Path.join("data_migrations") + |> Path.join(@dir) + |> Path.join("sql") + |> Path.join(name <> ".sql.eex") + |> EEx.eval_file(assigns: assigns) + end + + @doc """ + Runs a single SQL query in a file. + + Valid options: + - `quiet` - reduces output from running the SQL + - `params` - List of query parameters. + - `query_options` - passed to Repo.query + """ + def run_sql(name, assigns \\ [], options \\ []) do + query = unwrap(name, assigns) + + do_run(name, query, options) + end + + @doc """ + Runs multiple SQL queries from a single file. + + Note that each query must be separated by semicolons. + """ + def run_sql_multi(name, assigns \\ [], options \\ []) do + unwrap(name, assigns) + |> String.trim() + |> String.split(";", trim: true) + |> Enum.with_index(1) + |> Enum.reduce_while(:ok, fn {query, index}, _ -> + case do_run("#{name}-#{index}", query, options) do + {:ok, _} -> {:cont, :ok} + error -> {:halt, error} + end + end) + end + + def do_run(name, query, options \\ []) do + params = Keyword.get(options, :params, []) + query_options = Keyword.get(options, :query_options, []) + + case @repo.query(query, params, [timeout: :infinity] ++ query_options) do + {:ok, res} -> + if not Keyword.get(options, :quiet, false) do + IO.puts( + " #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n" + ) + + IO.puts(String.duplicate("-", 78)) + end + + {:ok, res} + + result -> + result + end + end + + defp unwrap_with_io(name, assigns) do + IO.puts("#{IO.ANSI.yellow()}Running #{name}#{IO.ANSI.reset()}") + query = unwrap(name, assigns) + + IO.puts(""" + -> Query: #{IO.ANSI.blue()}#{String.trim(query)}#{IO.ANSI.reset()} + """) + + query + end + end + end +end diff --git a/lib/plausible/data_migration/postgres_repo.ex b/lib/plausible/data_migration/postgres_repo.ex new file mode 100644 index 0000000000000000000000000000000000000000..351850079893109de4905a696de4119f50cb2f67 --- /dev/null +++ b/lib/plausible/data_migration/postgres_repo.ex @@ -0,0 +1,22 @@ +defmodule Plausible.DataMigration.PostgresRepo do + @moduledoc """ + Ecto.Repo for Posrtgres data migrations, to be started manually, + outside of the main application supervision tree. + """ + use Ecto.Repo, + otp_app: :plausible, + adapter: Ecto.Adapters.Postgres + + def start(url, opts \\ []) when is_binary(url) do + default_config = Plausible.Repo.config() + + start_link( + url: url, + queue_target: 500, + queue_interval: 2000, + pool_size: opts[:pool_size] || 1, + ssl: opts[:ssl] || default_config[:ssl], + ssl_opts: opts[:ssl_opts] || default_config[:ssl_opts] + ) + end +end diff --git a/lib/plausible/data_migration/prefix_tracker_script_configuration_id.ex b/lib/plausible/data_migration/prefix_tracker_script_configuration_id.ex new file mode 100644 index 0000000000000000000000000000000000000000..9932c2d80e47095f83d534b35d4e784104ed8a58 --- /dev/null +++ b/lib/plausible/data_migration/prefix_tracker_script_configuration_id.ex @@ -0,0 +1,96 @@ +defmodule Plausible.DataMigration.PrefixTrackerScriptConfigurationId do + @moduledoc """ + Migration to update tracker script configuration IDs to use the new prefixed format. + + This migration: + 1. Processes tracker configurations in batches to avoid memory issues + 2. Uses batch updates for better performance + 3. Handles the case where tracker configurations might not exist for all sites + 4. Provides progress logging + """ + + import Ecto.Query + alias Plausible.Repo + alias Plausible.Site.TrackerScriptConfiguration + + def run(batch_size \\ 100) do + total_configs = count_total_configurations() + IO.puts("Found #{total_configs} total tracker configurations to process") + + process_batch(nil, total_configs, batch_size, 0) + end + + defp count_total_configurations do + Repo.aggregate( + from(config in TrackerScriptConfiguration, + where: not like(config.id, "pa-%") + ), + :count, + :id + ) + end + + defp process_batch(last_id, total_configs, batch_size, processed_count) do + configs = get_configurations_batch(last_id, batch_size) + + if length(configs) > 0 do + batch_num = div(processed_count, batch_size) + 1 + start_pos = processed_count + 1 + end_pos = processed_count + length(configs) + + IO.puts("Processing batch #{batch_num} (#{start_pos}-#{end_pos} of #{total_configs})") + + process_configurations_batch(configs, batch_num) + + process_batch( + List.last(configs).id, + total_configs, + batch_size, + processed_count + length(configs) + ) + else + IO.puts("Migration completed!") + end + end + + defp get_configurations_batch(last_id, batch_size) do + query = + from(config in TrackerScriptConfiguration, + where: not like(config.id, "pa-%"), + order_by: [asc: config.id], + limit: ^batch_size, + select: %{ + id: config.id + } + ) + + query = + if last_id do + from(config in query, where: config.id > ^last_id) + else + query + end + + Repo.all(query) + end + + defp process_configurations_batch(configs, batch_num) do + config_ids = Enum.map(configs, & &1.id) + + case Repo.query( + "UPDATE tracker_script_configuration SET id = 'pa-' || id WHERE id = ANY($1)", + [config_ids] + ) do + {:ok, %{num_rows: updated_count}} -> + if updated_count != length(configs) do + IO.puts( + "Warning: Batch #{batch_num} - Expected to update #{length(configs)} configurations, but updated #{updated_count}" + ) + end + + {:error, error} -> + IO.puts("Error updating batch #{batch_num}: #{inspect(error)}") + # Continue with next batch instead of crashing + end + end +end diff --git a/lib/plausible/data_migration/site_imports.ex b/lib/plausible/data_migration/site_imports.ex new file mode 100644 index 0000000000000000000000000000000000000000..cf806edd5e25f97ef9c3904d4e872a3d9486cf20 --- /dev/null +++ b/lib/plausible/data_migration/site_imports.ex @@ -0,0 +1,270 @@ +defmodule Plausible.DataMigration.SiteImports do + @moduledoc """ + !!!WARNING!!!: This script is used in migrations. Please take special care + when altering it. + + Site imports migration backfilling SiteImport entries for old imports + and alters import end dates to match actual end date of respective import stats. + """ + + import Ecto.Query + + alias Plausible.{Repo, ClickhouseRepo, Site} + + defmodule SiteImportSnapshot do + @moduledoc """ + A snapshot of the Plausible.Imported.SiteImport schema from April 2024. + """ + + use Ecto.Schema + + schema "site_imports" do + field :start_date, :date + field :end_date, :date + field :label, :string + field :source, Ecto.Enum, values: [:universal_analytics, :google_analytics_4, :csv, :noop] + field :status, Ecto.Enum, values: [:pending, :importing, :completed, :failed] + field :legacy, :boolean, default: false + + belongs_to :site, Plausible.Site + belongs_to :imported_by, Plausible.Auth.User + + timestamps() + end + end + + @imported_tables_april_2024 [ + "imported_visitors", + "imported_sources", + "imported_pages", + "imported_entry_pages", + "imported_exit_pages", + "imported_custom_events", + "imported_locations", + "imported_devices", + "imported_browsers", + "imported_operating_systems" + ] + + def imported_tables_april_2024(), do: @imported_tables_april_2024 + + def run(opts \\ []) do + dry_run? = Keyword.get(opts, :dry_run?, true) + + site_import_query = + from(i in SiteImportSnapshot, + where: i.site_id == parent_as(:site).id and i.status == ^:completed, + select: 1 + ) + + sites_with_only_legacy_import = + from(s in Site, + as: :site, + select: %{id: s.id, imported_data: s.imported_data}, + where: + not is_nil(s.imported_data) and fragment("?->>'status'", s.imported_data) == "ok" and + not exists(site_import_query) + ) + |> Repo.all(log: false) + + site_imports = + from(i in SiteImportSnapshot, where: i.status == ^:completed) + |> Repo.all(log: false) + + legacy_site_imports = backfill_legacy_site_imports(sites_with_only_legacy_import, dry_run?) + + all_site_imports = Repo.preload(site_imports ++ legacy_site_imports, :site) + + adjust_site_import_end_dates(all_site_imports, dry_run?) + + IO.puts("Finished") + end + + defp backfill_legacy_site_imports(sites, dry_run?) do + total = length(sites) + + IO.puts("Backfilling legacy site import across #{total} sites (DRY RUN: #{dry_run?})...") + + legacy_site_imports = + for {site, idx} <- Enum.with_index(sites) do + IO.puts("Creating legacy site import entry for site ID #{site.id} (#{idx + 1}/#{total})") + + params = + site.imported_data + |> from_legacy() + |> Map.put(:site_id, site.id) + |> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id]) + + %SiteImportSnapshot{} + |> Ecto.Changeset.change(params) + |> insert!(dry_run?) + end + + IO.puts("Finished backfilling sites.") + + legacy_site_imports + end + + defp adjust_site_import_end_dates(site_imports, dry_run?) do + total = length(site_imports) + + IO.puts("Adjusting end dates of #{total} site imports (DRY RUN: #{dry_run?})...") + + for {site_import, idx} <- Enum.with_index(site_imports) do + IO.puts( + "Adjusting end date for site import #{site_import.id} (#{idx + 1}/#{total}) (site ID #{site_import.site_id}, start date: #{site_import.start_date}, end date: #{site_import.end_date})" + ) + + import_ids = + if site_import.legacy do + [0, site_import.id] + else + [site_import.id] + end + + end_date = imported_stats_end_date(site_import.site_id, import_ids) + + if !end_date do + IO.puts( + "Site import #{site_import.id} (site ID #{site_import.site_id}) does not have any recorded stats. Removing it." + ) + + if site_import.legacy do + # sanity check that data is correct + "ok" = site_import.site.imported_data.status + + clear_imported_data(site_import.site, dry_run?) + end + + delete!(site_import, dry_run?) + else + case Date.compare(end_date, site_import.end_date) do + :lt -> + IO.puts( + "End date of site import #{site_import.id} (site ID #{site_import.site_id}) is adjusted from #{site_import.end_date} to #{end_date}." + ) + + site_import + |> Ecto.Changeset.change(end_date: end_date) + |> update!(dry_run?) + + # credo:disable-for-next-line Credo.Check.Refactor.Nesting + if site_import.legacy do + # sanity check that data is correct + "ok" = site_import.site.imported_data.status + + site_import.site + |> Ecto.Changeset.change(imported_data: %{end_date: end_date}) + |> update!(dry_run?) + end + + :eq -> + IO.puts( + "End date of site import #{site_import.id} (site ID #{site_import.site_id}) is left unadjusted." + ) + + :gt -> + IO.puts( + "Site import #{site_import.id} (site ID #{site_import.site_id}) computed end date is later than the current one. Skipping." + ) + end + end + end + + IO.puts("Finished adjusting end dates of site imports.") + end + + # Exposed for testing purposes + @doc false + def imported_stats_end_date(site_id, import_ids) do + [first_table | tables] = @imported_tables_april_2024 + + query = + Enum.reduce(tables, max_date_query(first_table, site_id, import_ids), fn table, query -> + from(s in subquery(union_all(query, ^max_date_query(table, site_id, import_ids)))) + end) + + dates = ClickhouseRepo.all(from(q in query, select: q.max_date), log: false) + + if dates != [] do + case Enum.max(dates, Date) do + # no stats for this domain yet + ~D[1970-01-01] -> + nil + + date -> + date + end + else + nil + end + end + + defp insert!(changeset, false = _dry_run?) do + Repo.insert!(changeset) + end + + defp insert!(changeset, true = _dry_run?) do + if changeset.valid? do + changeset + |> Ecto.Changeset.change(id: 0) + |> Ecto.Changeset.apply_changes() + else + raise "Invalid insert: #{inspect(changeset)}" + end + end + + defp clear_imported_data(site, false = _dry_run?) do + Repo.update_all(from(s in Site, where: s.id == ^site.id), set: [imported_data: nil]) + end + + defp clear_imported_data(site, true = _dry_run?) do + %{site | imported_data: nil} + end + + defp update!(changeset, false = _dry_run?) do + Repo.update!(changeset) + end + + defp update!(changeset, true = _dry_run?) do + if changeset.valid? do + Ecto.Changeset.apply_changes(changeset) + else + raise "Invalid update: #{inspect(changeset)}" + end + end + + defp delete!(entity, false = _dry_run?) do + Repo.delete!(entity) + end + + defp delete!(entity, true = _dry_run?) do + entity + end + + defp max_date_query(table, site_id, import_ids) do + from(q in table, + where: q.site_id == ^site_id, + where: q.import_id in ^import_ids, + select: %{max_date: fragment("max(?)", q.date)} + ) + end + + defp from_legacy(%Site.ImportedData{} = data) do + status = + case data.status do + "ok" -> :completed + "error" -> :failed + _ -> :importing + end + + %SiteImportSnapshot{ + id: 0, + legacy: true, + start_date: data.start_date, + end_date: data.end_date, + source: :universal_analytics, + status: status + } + end +end diff --git a/lib/plausible/data_migration/versioned_sessions.ex b/lib/plausible/data_migration/versioned_sessions.ex new file mode 100644 index 0000000000000000000000000000000000000000..90e78adbeebd14e5b66a8ff62b60a00d7f07e538 --- /dev/null +++ b/lib/plausible/data_migration/versioned_sessions.ex @@ -0,0 +1,80 @@ +defmodule Plausible.DataMigration.VersionedSessions do + @moduledoc """ + !!!WARNING!!!: This script is used in migrations. Please take special care + when altering it. + + Sessions CollapsingMergeTree -> VersionedCollapsingMergeTree migration, + SQL files available at: + + priv/data_migrations/VersionedSessions/sql + """ + use Plausible.DataMigration, dir: "VersionedSessions", repo: Plausible.IngestRepo + + @suffix_format "%Y%m%d%H%M%S" + @versioned_table_engines [ + "ReplicatedVersionedCollapsingMergeTree", + "VersionedCollapsingMergeTree" + ] + + def run(opts \\ []) do + run_exchange? = Keyword.get(opts, :run_exchange?, true) + + unique_suffix = DateTime.utc_now() |> Calendar.strftime(@suffix_format) + + cluster? = Plausible.IngestRepo.clustered_table?("sessions_v2") + + {:ok, %{rows: partitions}} = run_sql("list-partitions") + partitions = Enum.map(partitions, fn [part] -> part end) + + {:ok, %{rows: [[current_table_engine, table_settings]]}} = + run_sql("get-sessions-table-settings") + + if Enum.member?(@versioned_table_engines, current_table_engine) do + IO.puts("sessions_v2 table is already versioned, no migration needed") + else + {:ok, _} = run_sql("drop-sessions-tmp-table", cluster?: cluster?) + + {:ok, _} = + run_sql("create-sessions-tmp-table", + cluster?: cluster?, + table_settings: table_settings, + unique_suffix: unique_suffix + ) + + for partition <- partitions do + {:ok, _} = run_sql("attach-partition", partition: partition) + end + + if run_exchange? do + run_exchange(cluster?) + end + + IO.puts("Migration done!") + end + end + + defp run_exchange(cluster?) do + case run_sql("exchange-sessions-tables", cluster?: cluster?) do + {:ok, _} -> + nil + + # Docker containers don't seem to support EXCHANGE TABLE, hack around this with a non-atomic swap + {:error, %Ch.Error{code: code}} when code in [1, 48] -> + IO.puts("Exchanging sessions_v2 and sessions_v2_tmp_versioned non-atomically") + + {:ok, _} = + run_sql("rename-table", + from: "sessions_v2", + to: "sessions_v2_backup", + cluster?: cluster? + ) + + {:ok, _} = + run_sql("rename-table", + from: "sessions_v2_tmp_versioned", + to: "sessions_v2", + cluster?: cluster? + ) + end + end +end diff --git a/lib/plausible/dns_lookup.ex b/lib/plausible/dns_lookup.ex new file mode 100644 index 0000000000000000000000000000000000000000..212ff980463d5a243da316e6e75c5299a1af7b95 --- /dev/null +++ b/lib/plausible/dns_lookup.ex @@ -0,0 +1,31 @@ +defmodule Plausible.DnsLookupInterface do + @moduledoc """ + Behaviour module for DNS lookup operations. + """ + + @callback lookup( + name :: charlist(), + class :: atom(), + type :: atom(), + opts :: list(), + timeout :: integer() + ) :: + list() | [] +end + +defmodule Plausible.DnsLookup do + @moduledoc """ + Thin wrapper around `:inet_res.lookup/5`. + To use, call `Plausible.DnsLookup.impl().lookup/5`, + this allows for mocking DNS lookups in tests. + """ + + @behaviour Plausible.DnsLookupInterface + + @impl Plausible.DnsLookupInterface + def lookup(name, class, type, opts, timeout), + do: :inet_res.lookup(name, class, type, opts, timeout) + + @spec impl() :: Plausible.DnsLookup + def impl(), do: Application.get_env(:plausible, :dns_lookup_impl, __MODULE__) +end diff --git a/lib/plausible/ecto/types/compiled_regex.ex b/lib/plausible/ecto/types/compiled_regex.ex new file mode 100644 index 0000000000000000000000000000000000000000..e748ad6abc787e0aac42a75f0bb772e26d2ab322 --- /dev/null +++ b/lib/plausible/ecto/types/compiled_regex.ex @@ -0,0 +1,14 @@ +defmodule Plausible.Ecto.Types.CompiledRegex do + @moduledoc """ + Ensures that the regex is compiled on load + """ + use Ecto.Type + + def type, do: :string + + def cast(val) when is_binary(val), do: {:ok, val} + def cast(_), do: :error + + def load(val), do: {:ok, Regex.compile!(val, "E")} + def dump(val), do: {:ok, val} +end diff --git a/lib/plausible/ecto/types/nanoid.ex b/lib/plausible/ecto/types/nanoid.ex new file mode 100644 index 0000000000000000000000000000000000000000..9197f5aeec647a79a6254a890cf840fb4f7c2f79 --- /dev/null +++ b/lib/plausible/ecto/types/nanoid.ex @@ -0,0 +1,51 @@ +defmodule Plausible.Ecto.Types.NanoidBase do + @moduledoc """ + Base module for nanoid string types + """ + + use Ecto.Type + + def type(), do: :string + + def cast(value) when is_binary(value), do: {:ok, value} + def cast(_), do: :error + + def load(value), do: {:ok, value} + + def dump(value) when is_binary(value), do: {:ok, value} + def dump(_), do: :error + + @callback autogenerate() :: String.t() +end + +defmodule Plausible.Ecto.Types.Nanoid do + @moduledoc """ + Custom column type for nanoid strings + """ + + @behaviour Plausible.Ecto.Types.NanoidBase + + def type(), do: Plausible.Ecto.Types.NanoidBase.type() + def cast(value), do: Plausible.Ecto.Types.NanoidBase.cast(value) + def load(value), do: Plausible.Ecto.Types.NanoidBase.load(value) + def dump(value), do: Plausible.Ecto.Types.NanoidBase.dump(value) + + @spec autogenerate() :: String.t() + def autogenerate(), do: Nanoid.generate() +end + +defmodule Plausible.Ecto.Types.TrackerScriptNanoid do + @moduledoc """ + Custom column type for tracker script configuration nanoid strings with pa- prefix + """ + + @behaviour Plausible.Ecto.Types.NanoidBase + + def type(), do: Plausible.Ecto.Types.NanoidBase.type() + def cast(value), do: Plausible.Ecto.Types.NanoidBase.cast(value) + def load(value), do: Plausible.Ecto.Types.NanoidBase.load(value) + def dump(value), do: Plausible.Ecto.Types.NanoidBase.dump(value) + + @spec autogenerate() :: String.t() + def autogenerate(), do: "pa-#{Nanoid.generate()}" +end diff --git a/lib/plausible/event/system_events.ex b/lib/plausible/event/system_events.ex new file mode 100644 index 0000000000000000000000000000000000000000..c78d9ed060af7b2c0939489dae440235967b8e3f --- /dev/null +++ b/lib/plausible/event/system_events.ex @@ -0,0 +1,101 @@ +defmodule Plausible.Event.SystemEvents do + @moduledoc """ + System events are events that require at least one form of special treatment by the application. + They are distinguished by event name. + + For some system events, the tracking API may reject events that don't match the expected format + (e.g. engagement events without scroll depth or engagement time defined). + + For other system events, it may accept the event (e.g. "Outbound Link: Click" with the url prop not set). + + System events may have corresponding system-managed goals, in which case the goal name and event name will be the same + (e.g. "Outbound Link: Click" goal is managed by the application). The system will only be able to manage these goals properly + if they are not renamed by the user. + """ + @pageview_event_name "pageview" + @engagement_event_name "engagement" + + @outbound_link_click_event_name "Outbound Link: Click" + @cloaked_link_click_event_name "Cloaked Link: Click" + @file_download_link_click_event_name "File Download" + + @error_404_event_name "404" + @wordpress_form_completions_event_name "WP Form Completions" + @form_submission_event_name "Form: Submission" + + @all_system_events [ + @pageview_event_name, + @engagement_event_name, + @outbound_link_click_event_name, + @cloaked_link_click_event_name, + @file_download_link_click_event_name, + @error_404_event_name, + @wordpress_form_completions_event_name, + @form_submission_event_name + ] + + @interactive_events @all_system_events + + @events_with_url_prop [ + @outbound_link_click_event_name, + @cloaked_link_click_event_name, + @file_download_link_click_event_name + ] + + @events_with_path_prop [ + @error_404_event_name, + @wordpress_form_completions_event_name, + @form_submission_event_name + ] + + @events_with_engagement_props [ + @engagement_event_name + ] + + def events() do + @all_system_events + end + + def events_with_interactive_always_true() do + @interactive_events + end + + def events_with_url_prop() do + @events_with_url_prop + end + + def events_with_path_prop() do + @events_with_path_prop + end + + def events_with_engagement_props() do + @events_with_engagement_props + end + + @spec special_events_for_prop_key(String.t()) :: [String.t()] + def special_events_for_prop_key("url"), do: events_with_url_prop() + def special_events_for_prop_key("path"), do: events_with_path_prop() + + @doc """ + Checks if the event name is for a system event / system goal that should have the event.props.path synced with the event.pathname property. + + ### Examples + iex> sync_props_path_with_pathname?("404", [{"path", "/foo"}]) + false + + Note: Should not override event.props.path if it is set deliberately to nil + iex> sync_props_path_with_pathname?("404", [{"path", nil}]) + false + + iex> sync_props_path_with_pathname?("404", [{"other", "value"}]) + true + + iex> sync_props_path_with_pathname?("404", []) + true + """ + @spec sync_props_path_with_pathname?(String.t(), [{String.t(), String.t()}]) :: boolean() + def sync_props_path_with_pathname?(event_name, props_in_request) do + event_name in events_with_path_prop() and + not Enum.any?(props_in_request, fn {k, _} -> k == "path" end) + end +end diff --git a/lib/plausible/event/write_buffer.ex b/lib/plausible/event/write_buffer.ex new file mode 100644 index 0000000000000000000000000000000000000000..3966b5a3f7e4fbf352c170b5d708032e15f28e78 --- /dev/null +++ b/lib/plausible/event/write_buffer.ex @@ -0,0 +1,38 @@ +defmodule Plausible.Event.WriteBuffer do + @moduledoc false + + %{ + header: header, + insert_sql: insert_sql, + insert_opts: insert_opts, + fields: fields, + encoding_types: encoding_types + } = + Plausible.Ingestion.WriteBuffer.compile_time_prepare(Plausible.ClickhouseEventV2) + + def child_spec(opts) do + opts = + Keyword.merge(opts, + name: __MODULE__, + header: unquote(header), + insert_sql: unquote(insert_sql), + insert_opts: unquote(insert_opts) + ) + + Plausible.Ingestion.WriteBuffer.child_spec(opts) + end + + def insert(event) do + row_binary = + [Enum.map(unquote(fields), fn field -> Map.fetch!(event, field) end)] + |> Ch.RowBinary._encode_rows(unquote(encoding_types)) + |> IO.iodata_to_binary() + + :ok = Plausible.Ingestion.WriteBuffer.insert(__MODULE__, row_binary) + {:ok, event} + end + + def flush do + Plausible.Ingestion.WriteBuffer.flush(__MODULE__) + end +end diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.ex new file mode 100644 index 0000000000000000000000000000000000000000..86734aed19c7eb492fbf63d92bd7ea85a3798c01 --- /dev/null +++ b/lib/plausible/exports.ex @@ -0,0 +1,715 @@ +defmodule Plausible.Exports do + @moduledoc """ + Contains functions to export data for events and sessions as Zip archives. + """ + + use Plausible + use Plausible.Stats.SQL.Fragments + import Ecto.Query + + @doc "Schedules CSV export job to S3 storage" + @spec schedule_s3_export(pos_integer, String.t()) :: + {:ok, Oban.Job.t()} | {:error, :no_data} + def schedule_s3_export(site_id, email_to) do + with :ok <- ensure_has_data(site_id) do + args = %{ + "storage" => "s3", + "site_id" => site_id, + "email_to" => email_to, + "s3_bucket" => Plausible.S3.exports_bucket(), + "s3_path" => s3_export_key(site_id) + } + + {:ok, Oban.insert!(Plausible.Workers.ExportAnalytics.new(args))} + end + end + + @doc "Schedules CSV export job to local storage" + @spec schedule_local_export(pos_integer, String.t()) :: {:ok, Oban.Job.t()} | {:error, :no_data} + def schedule_local_export(site_id, email_to) do + with :ok <- ensure_has_data(site_id) do + args = %{ + "storage" => "local", + "site_id" => site_id, + "email_to" => email_to, + "local_path" => local_export_file(site_id) + } + + {:ok, Oban.insert!(Plausible.Workers.ExportAnalytics.new(args))} + end + end + + @spec ensure_has_data(pos_integer) :: :ok | {:error, :no_data} + defp ensure_has_data(site_id) do + # SELECT true FROM "events_v2" AS e0 WHERE (e0."site_id" = ^site_id) LIMIT 1 + has_data? = Plausible.ClickhouseRepo.exists?(from "events_v2", where: [site_id: ^site_id]) + if has_data?, do: :ok, else: {:error, :no_data} + end + + @doc "Gets last CSV export job for a site" + @spec get_last_export_job(pos_integer) :: Oban.Job.t() | nil + def get_last_export_job(site_id) do + Plausible.Repo.one( + from e in Plausible.Workers.ExportAnalytics.base_query(site_id), + order_by: [desc: :id], + limit: 1 + ) + end + + @doc "Subscribes to CSV export job notifications" + def oban_listen, do: Oban.Notifier.listen(__MODULE__) + @doc false + def oban_notify(site_id), do: Oban.Notifier.notify(__MODULE__, %{"site_id" => site_id}) + + @doc """ + Renders export archive filename. + + Examples: + + iex> archive_filename("plausible.io", _created_on = ~D[2024-12-31]) + "plausible_io_20241231.zip" + + """ + def archive_filename(domain, %Date{} = created_on) do + String.replace(domain, ".", "_") <> "_" <> Calendar.strftime(created_on, "%Y%m%d") <> ".zip" + end + + @doc ~S""" + Safely renders content disposition for an arbitrary export filename. + + Examples: + + iex> content_disposition("plausible_io_20241231.zip") + "attachment; filename=\"plausible_io_20241231.zip\"" + + iex> content_disposition("📊.zip") + "attachment; filename=\"plausible-export.zip\"; filename*=utf-8''%F0%9F%93%8A.zip" + + """ + def content_disposition(filename) do + encoded_filename = URI.encode(filename) + + if encoded_filename == filename do + ~s[attachment; filename="#{filename}"] + else + ~s[attachment; filename="plausible-export.zip"; filename*=utf-8''#{encoded_filename}] + end + end + + @type export :: %{ + path: Path.t(), + name: String.t(), + expires_at: DateTime.t() | nil, + size: pos_integer + } + + @doc "Gets local export for a site" + @spec get_local_export(pos_integer, String.t(), String.t()) :: export | nil + def get_local_export(site_id, domain, timezone) do + path = local_export_file(site_id) + + if File.exists?(path) do + %File.Stat{size: size, mtime: mtime} = File.stat!(path, time: :posix) + created_at = DateTime.from_unix!(mtime) + created_on_in_site_tz = Plausible.Timezones.to_date_in_timezone(created_at, timezone) + name = archive_filename(domain, created_on_in_site_tz) + + %{path: path, name: name, expires_at: nil, size: size} + end + end + + @doc "Deletes local export for a site" + @spec delete_local_export(pos_integer) :: :ok + def delete_local_export(site_id) do + file = local_export_file(site_id) + + if File.exists?(file) do + File.rm!(file) + end + + :ok + end + + @spec local_export_file(pos_integer) :: Path.t() + defp local_export_file(site_id) do + data_dir = Application.get_env(:plausible, :data_dir) + Path.join([data_dir || System.tmp_dir!(), "plausible-exports", Integer.to_string(site_id)]) + end + + @doc "Gets S3 export for a site. Raises if object storage is unavailable." + @spec get_s3_export!(pos_integer, non_neg_integer) :: export | nil + def get_s3_export!(site_id, retries \\ 0) do + path = s3_export_key(site_id) + bucket = Plausible.S3.exports_bucket() + head_object_op = ExAws.S3.head_object(bucket, path) + + case ExAws.request(head_object_op, retries: retries) do + {:ok, %{status_code: 200, headers: headers}} -> + "attachment; filename=" <> filename = :proplists.get_value("content-disposition", headers) + name = String.trim(filename, "\"") + size = :proplists.get_value("content-length", headers, nil) + + expires_at = + if x_amz_expiration = :proplists.get_value("x-amz-expiration", headers, nil) do + ["expiry-date=", expiry_date, ", rule-id=", _rule_id] = + String.split(x_amz_expiration, "\"", trim: true) + + Plausible.Times.parse!(expiry_date, "{RFC1123}") + end + + %{ + path: path, + name: name, + expires_at: expires_at, + size: String.to_integer(size) + } + + {:error, {:http_error, 404, _response}} -> + nil + + {:error, %Finch.TransportError{} = e} -> + raise e + end + end + + @doc "Deletes S3 export for a site. Raises if object storage is unavailable." + @spec delete_s3_export!(pos_integer) :: :ok + def delete_s3_export!(site_id) do + if export = get_s3_export!(site_id) do + exports_bucket = Plausible.S3.exports_bucket() + delete_op = ExAws.S3.delete_object(exports_bucket, export.path) + ExAws.request!(delete_op) + end + + :ok + end + + defp s3_export_key(site_id), do: Integer.to_string(site_id) + + @doc "Returns the date range for the site's events data in site's timezone or `nil` if there is no data" + @spec date_range(pos_integer, String.t()) :: Date.Range.t() | nil + def date_range(site_id, timezone) do + [%Date{} = start_date, %Date{} = end_date] = + Plausible.ClickhouseRepo.one( + from e in "events_v2", + where: [site_id: ^site_id], + select: [ + fragment("toDate(min(?),?)", e.timestamp, ^timezone), + fragment("toDate(max(?),?)", e.timestamp, ^timezone) + ] + ) + + unless end_date == ~D[1970-01-01] do + Date.range(start_date, end_date) + end + end + + @doc """ + Builds Ecto queries to export data from `events_v2` and `sessions_v2` + tables into the format of `imported_*` tables for a website. + """ + @spec export_queries(Plausible.Site.t(), + extname: String.t(), + date_range: Date.Range.t(), + timezone: String.t() + ) :: + %{String.t() => Ecto.Query.t()} + def export_queries(site, opts \\ []) do + extname = opts[:extname] || ".csv" + date_range = opts[:date_range] + timezone = opts[:timezone] || "UTC" + + suffix = + if date_range do + first_date = Calendar.strftime(date_range.first, "%Y%m%d") + last_date = Calendar.strftime(date_range.last, "%Y%m%d") + "_#{first_date}_#{last_date}" <> extname + else + extname + end + + filename = fn name -> name <> suffix end + + %{ + filename.("imported_visitors") => export_visitors_q(site, timezone, date_range), + filename.("imported_sources") => export_sources_q(site, timezone, date_range), + filename.("imported_pages") => export_pages_q(site, timezone, date_range), + filename.("imported_entry_pages") => export_entry_pages_q(site, timezone, date_range), + filename.("imported_exit_pages") => export_exit_pages_q(site, timezone, date_range), + filename.("imported_custom_events") => export_custom_events_q(site, timezone, date_range), + filename.("imported_locations") => export_locations_q(site, timezone, date_range), + filename.("imported_devices") => export_devices_q(site, timezone, date_range), + filename.("imported_browsers") => export_browsers_q(site, timezone, date_range), + filename.("imported_operating_systems") => + export_operating_systems_q(site, timezone, date_range), + filename.("imported_custom_props") => export_custom_props_q(site, timezone, date_range) + } + end + + on_ee do + defp sampled(table) do + Plausible.Stats.Sampling.add_query_hint(from(table)) + end + else + defp sampled(table) do + table + end + end + + defp export_filter(site_id, date_range) do + filter = dynamic([t], t.site_id == ^site_id) + + if date_range do + dynamic( + ^filter and + selected_as(:date) >= ^date_range.first and + selected_as(:date) <= ^date_range.last + ) + else + filter + end + end + + defmacrop date(timestamp, timezone) do + quote do + selected_as( + fragment("toDate(?,?)", unquote(timestamp), unquote(timezone)), + :date + ) + end + end + + defmacrop visit_duration(t) do + quote do + selected_as( + scale_sample(fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).duration)), + :visit_duration + ) + end + end + + defmacrop visitors(t) do + quote do + selected_as( + scale_sample(fragment("uniq(?)", unquote(t).user_id)), + :visitors + ) + end + end + + defmacrop visits(t) do + quote do + selected_as( + scale_sample(fragment("greatest(sum(?),0)", unquote(t).sign)), + :visits + ) + end + end + + defmacrop bounces(t) do + quote do + selected_as( + scale_sample( + fragment( + "greatest(sum(?*?),0)", + unquote(t).sign, + unquote(t).is_bounce + ) + ), + :bounces + ) + end + end + + defmacrop pageviews(t) do + quote do + selected_as( + scale_sample( + fragment( + "greatest(sum(?*?),0)", + unquote(t).sign, + unquote(t).pageviews + ) + ), + :pageviews + ) + end + end + + defp export_visitors_q(site, timezone, date_range) do + visitors_sessions_q = + from s in sampled("sessions_v2"), + where: ^export_filter(site.id, date_range), + group_by: selected_as(:date), + select: %{ + date: date(s.timestamp, ^timezone), + bounces: bounces(s), + visits: visits(s), + visit_duration: visit_duration(s), + visitors: visitors(s) + } + + visitors_events_q = + from e in sampled("events_v2"), + where: ^export_filter(site.id, date_range), + group_by: selected_as(:date), + select: %{ + date: date(e.timestamp, ^timezone), + pageviews: + selected_as( + scale_sample(fragment("countIf(?='pageview')", e.name)), + :pageviews + ) + } + + visitors_q = + "e" + |> with_cte("e", as: ^visitors_events_q) + |> with_cte("s", as: ^visitors_sessions_q) + + from e in visitors_q, + full_join: s in "s", + on: e.date == s.date, + order_by: selected_as(:date), + select: [ + selected_as(fragment("greatest(?,?)", s.date, e.date), :date), + s.visitors, + e.pageviews, + s.bounces, + s.visits, + s.visit_duration + ] + end + + defp export_sources_q(site, timezone, date_range) do + from s in sampled("sessions_v2"), + where: ^export_filter(site.id, date_range), + group_by: [ + selected_as(:date), + selected_as(:source), + s.referrer, + s.utm_source, + s.utm_medium, + s.utm_campaign, + s.utm_content, + s.utm_term + ], + order_by: selected_as(:date), + select: [ + date(s.timestamp, ^timezone), + selected_as(s.referrer_source, :source), + s.referrer, + s.utm_source, + s.utm_medium, + s.utm_campaign, + s.utm_content, + s.utm_term, + pageviews(s), + visitors(s), + visits(s), + visit_duration(s), + bounces(s) + ] + end + + defp export_pages_q(site, timezone, date_range) do + base_q = + from(e in sampled("events_v2"), + where: ^export_filter(site.id, date_range), + where: [name: "pageview"], + group_by: [selected_as(:date), selected_as(:page)], + order_by: selected_as(:date) + ) + + max_scroll_depth_per_session_q = + from(e in "events_v2", + where: ^export_filter(site.id, date_range), + where: e.name == "engagement" and e.scroll_depth <= 100, + select: %{ + date: date(e.timestamp, ^timezone), + page: selected_as(e.pathname, :page), + session_id: e.session_id, + max_scroll_depth: max(e.scroll_depth) + }, + group_by: [e.session_id, selected_as(:date), selected_as(:page)] + ) + + scroll_depth_q = + from(p in subquery(max_scroll_depth_per_session_q), + select: %{ + date: p.date, + page: p.page, + total_scroll_depth: fragment("sum(?)", p.max_scroll_depth), + total_scroll_depth_visits: fragment("uniq(?)", p.session_id) + }, + group_by: [:date, :page] + ) + + from(e in base_q, + left_join: s in subquery(scroll_depth_q), + on: s.date == selected_as(:date) and s.page == selected_as(:page), + select: %{ + date: date(e.timestamp, ^timezone), + hostname: selected_as(fragment("any(?)", e.hostname), :hostname), + page: selected_as(e.pathname, :page), + visits: + selected_as( + scale_sample(fragment("uniq(?)", e.session_id)), + :visits + ), + visitors: visitors(e), + pageviews: selected_as(scale_sample(fragment("count()")), :pageviews), + total_scroll_depth: + selected_as(fragment("any(?)", s.total_scroll_depth), :total_scroll_depth), + total_scroll_depth_visits: + selected_as(fragment("any(?)", s.total_scroll_depth_visits), :total_scroll_depth_visits) + } + ) + |> add_time_on_page_columns(site.id, timezone, date_range) + end + + defp add_time_on_page_columns(q, site_id, timezone, date_range) do + site = Plausible.Repo.get(Plausible.Site, site_id) + + if Plausible.Stats.TimeOnPage.new_time_on_page_visible?(site) do + cutoff = Plausible.Stats.TimeOnPage.legacy_time_on_page_cutoff(site) + + engagements_q = + from(e in sampled("events_v2"), + where: ^export_filter(site_id, date_range), + where: e.name == "engagement", + group_by: [selected_as(:date), selected_as(:page)], + order_by: selected_as(:date), + select: %{ + date: date(e.timestamp, ^timezone), + page: selected_as(e.pathname, :page), + total_time_on_page: + fragment( + "toUInt64(round(sumIf(?, ? >= ?) / 1000))", + e.engagement_time, + e.timestamp, + ^cutoff + ), + total_time_on_page_visits: + fragment("uniqIf(?, ? >= ?)", e.session_id, e.timestamp, ^cutoff) + } + ) + + q + |> join(:left, [], s in subquery(engagements_q), + on: s.date == selected_as(:date) and s.page == selected_as(:page) + ) + |> select_merge_as([..., s], %{ + total_time_on_page: fragment("any(?)", s.total_time_on_page), + total_time_on_page_visits: fragment("any(?)", s.total_time_on_page_visits) + }) + else + q + end + end + + defp export_entry_pages_q(site, timezone, date_range) do + from s in sampled("sessions_v2"), + where: ^export_filter(site.id, date_range), + group_by: [selected_as(:date), s.entry_page], + order_by: selected_as(:date), + select: [ + date(s.timestamp, ^timezone), + s.entry_page, + visitors(s), + selected_as( + scale_sample(fragment("greatest(sum(?),0)", s.sign)), + :entrances + ), + visit_duration(s), + bounces(s), + pageviews(s) + ] + end + + defp export_exit_pages_q(site, timezone, date_range) do + from s in sampled("sessions_v2"), + where: ^export_filter(site.id, date_range), + group_by: [selected_as(:date), s.exit_page], + order_by: selected_as(:date), + select: [ + date(s.timestamp, ^timezone), + s.exit_page, + visitors(s), + visit_duration(s), + selected_as( + scale_sample(fragment("greatest(sum(?),0)", s.sign)), + :exits + ), + bounces(s), + pageviews(s) + ] + end + + defp export_custom_events_q(site, timezone, date_range) do + from e in sampled("events_v2"), + where: ^export_filter(site.id, date_range), + where: e.name != "pageview", + group_by: [ + selected_as(:date), + e.name, + selected_as(:link_url), + selected_as(:path) + ], + order_by: selected_as(:date), + select: [ + date(e.timestamp, ^timezone), + e.name, + selected_as( + fragment( + "if(? in ?, ?, '')", + e.name, + ^Plausible.Event.SystemEvents.events_with_url_prop(), + get_by_key(e, :meta, "url") + ), + :link_url + ), + selected_as( + fragment( + "if(? in ?, ?, '')", + e.name, + ^Plausible.Event.SystemEvents.events_with_path_prop(), + get_by_key(e, :meta, "path") + ), + :path + ), + visitors(e), + selected_as(scale_sample(fragment("count()")), :events) + ] + end + + defp export_custom_props_q(site, timezone, date_range) do + query = + from e in sampled("events_v2"), + join: pv in fragment("arrayZip(`meta.key`, `meta.value`)"), + on: true, + hints: "ARRAY", + where: ^export_filter(site.id, date_range), + group_by: [ + selected_as(:date), + selected_as(:property), + selected_as(:value) + ], + order_by: selected_as(:date), + select: [ + date(e.timestamp, ^timezone), + selected_as(fragment("tupleElement(?, 1)", pv), :property), + selected_as(fragment("tupleElement(?, 2)", pv), :value), + visitors(e), + selected_as(scale_sample(fragment("count()")), :events) + ] + + if Plausible.Billing.Feature.Props.enabled?(site) do + query + else + where(query, [], selected_as(:property) in ^Plausible.Props.internal_keys()) + end + end + + defp export_locations_q(site, timezone, date_range) do + from s in sampled("sessions_v2"), + where: ^export_filter(site.id, date_range), + where: s.country_code != "\0\0" and s.country_code != "ZZ", + group_by: [selected_as(:date), s.country_code, s.subdivision1_code, s.city_geoname_id], + order_by: selected_as(:date), + select: [ + date(s.timestamp, ^timezone), + selected_as(s.country_code, :country), + selected_as(s.subdivision1_code, :region), + selected_as(s.city_geoname_id, :city), + visitors(s), + visits(s), + visit_duration(s), + bounces(s), + pageviews(s) + ] + end + + defp export_devices_q(site, timezone, date_range) do + from s in sampled("sessions_v2"), + where: ^export_filter(site.id, date_range), + group_by: [selected_as(:date), s.screen_size], + order_by: selected_as(:date), + select: [ + date(s.timestamp, ^timezone), + selected_as(s.screen_size, :device), + visitors(s), + visits(s), + visit_duration(s), + bounces(s), + pageviews(s) + ] + end + + defp export_browsers_q(site, timezone, date_range) do + from s in sampled("sessions_v2"), + where: ^export_filter(site.id, date_range), + group_by: [selected_as(:date), s.browser, s.browser_version], + order_by: selected_as(:date), + select: [ + date(s.timestamp, ^timezone), + s.browser, + s.browser_version, + visitors(s), + visits(s), + visit_duration(s), + bounces(s), + pageviews(s) + ] + end + + defp export_operating_systems_q(site, timezone, date_range) do + from s in sampled("sessions_v2"), + where: ^export_filter(site.id, date_range), + group_by: [selected_as(:date), s.operating_system, s.operating_system_version], + order_by: selected_as(:date), + select: [ + date(s.timestamp, ^timezone), + s.operating_system, + s.operating_system_version, + visitors(s), + visits(s), + visit_duration(s), + bounces(s), + pageviews(s) + ] + end + + @doc """ + Creates a streamable Zip archive from the provided (named) Ecto queries. + + Example usage: + + {:ok, pool} = Ch.start_link(pool_size: 1) + + DBConnection.run(pool, fn conn -> + conn + |> stream_archive(export_queries(site), format: "CSVWithNames") + |> Stream.into(File.stream!("export.zip")) + |> Stream.run() + end) + + """ + @spec stream_archive(DBConnection.t(), %{String.t() => Ecto.Query.t()}, [Ch.query_option()]) :: + Enumerable.t() + def stream_archive(conn, named_queries, opts \\ []) do + entries = + Enum.map(named_queries, fn {name, query} -> + {sql, params} = Plausible.ClickhouseRepo.to_sql(:all, query) + + datastream = + conn + |> Ch.stream(sql, params, opts) + |> Stream.map(fn %Ch.Result{data: data} -> data end) + + Zstream.entry(name, datastream, coder: Zstream.Coder.Stored) + end) + + Zstream.zip(entries) + end +end diff --git a/lib/plausible/file.ex b/lib/plausible/file.ex new file mode 100644 index 0000000000000000000000000000000000000000..689134b9453ceb20455f125fdb49346dc65b7e64 --- /dev/null +++ b/lib/plausible/file.ex @@ -0,0 +1,27 @@ +defmodule Plausible.File do + @moduledoc """ + File helpers for Plausible. + """ + + @doc """ + Moves a file from one location to another. + + Tries renaming first, and falls back to copying and deleting the original. + """ + @spec mv!(Path.t(), Path.t()) :: :ok + def mv!(source, destination) do + File.rename!(source, destination) + rescue + e in File.RenameError -> + try do + case e.reason do + # fallback to cp/rm for cross-device moves + # https://github.com/plausible/analytics/issues/4638 + :exdev -> File.cp!(source, destination) + _ -> reraise(e, __STACKTRACE__) + end + after + File.rm(source) + end + end +end diff --git a/lib/plausible/funnel/const.ex b/lib/plausible/funnel/const.ex new file mode 100644 index 0000000000000000000000000000000000000000..05e29e9513c01c472b2aa65fb7b5e03875cdbb56 --- /dev/null +++ b/lib/plausible/funnel/const.ex @@ -0,0 +1,26 @@ +defmodule Plausible.Funnel.Const do + @moduledoc """ + Compile-time convenience constants for funnel characteristics. + """ + @min_steps 2 + @max_steps 8 + + defmacro min_steps() do + quote do + unquote(@min_steps) + end + end + + defmacro max_steps() do + quote do + unquote(@max_steps) + end + end + + defmacro __using__(_opts \\ []) do + quote do + require Plausible.Funnel.Const + alias Plausible.Funnel + end + end +end diff --git a/lib/plausible/geo.ex b/lib/plausible/geo.ex new file mode 100644 index 0000000000000000000000000000000000000000..cb351f82f451a5cfbde448ceb357768e59534dbe --- /dev/null +++ b/lib/plausible/geo.ex @@ -0,0 +1,207 @@ +defmodule Plausible.Geo do + @moduledoc """ + This module provides an API for fetching IP geolocation. + """ + + require Logger + + @db :geolocation + + @doc """ + Starts the geodatabase loading process. Two modes are supported: local file + and MaxMind license key. + + ## Options + + * `:path` - the path to the .mmdb database local file. When present, + `:license_key` and `:edition` are not required. + + * `:license_key` - the [license key](https://support.maxmind.com/hc/en-us/articles/4407111582235-Generate-a-License-Key) + from MaxMind to authenticate requests to MaxMind. + + * `:edition` - the name of the MaxMind database to be downloaded from MaxMind + servers. Defaults to `GeoLite2-City`. + + * `:cache_dir` - if set, the downloaded .mmdb files are cached there across + restarts. + + * `:async` - when used, configures the database loading to run + asynchronously. + + ## Examples + + Loading from a local file: + + iex> load_db(path: "/etc/plausible/dbip-city.mmdb") + :ok + + Downloading a MaxMind DB (this license key is no longer active): + + iex> load_db(license_key: "LNpsJCCKPis6XvBP", edition: "GeoLite2-City", async: true) + :ok + + """ + def load_db(opts) do + cond do + license_key = opts[:license_key] -> + edition = opts[:edition] || "GeoLite2-City" + maxmind_opts = [license_key: license_key] + + loader_opts = + if is_binary(opts[:cache_dir]) do + [ + database_cache_file: + String.to_charlist(Path.join(opts[:cache_dir], edition <> ".mmdb.gz")) + ] + else + [:no_cache] + end + + :ok = :locus.start_loader(@db, {:maxmind, edition}, maxmind_opts ++ loader_opts) + + path = opts[:path] -> + :ok = :locus.start_loader(@db, path) + + true -> + raise "failed to load geolocation db: need :path or :license_key to be provided" + end + + unless opts[:async] do + {:ok, _version} = :locus.await_loader(@db) + end + + :ok + end + + @doc """ + Waits for the database to start after calling `load_db/1` with the async option. + """ + def await_loader, do: :locus.await_loader(@db) + + @doc """ + Returns geodatabase type. + + Used for deciding whether to show the DB-IP disclaimer or not. + + ## Examples + + In the case of a DB-IP database: + + iex> database_type() + "DBIP-City-Lite" + + In the case of a MaxMind database: + + iex> database_type() + "GeoLite2-City" + + """ + def database_type do + case :locus.get_info(@db, :metadata) do + {:ok, %{database_type: type}} -> type + _other -> nil + end + end + + @doc """ + Looks up geo info about an IP address. + + ## Examples + + iex> lookup("8.7.6.5") + %{ + "city" => %{ + "geoname_id" => 5349755, + "names" => %{ + "de" => "Fontana", + "en" => "Fontana", + "ja" => "フォンタナ", + "ru" => "Фонтана" + } + }, + "continent" => %{ + "code" => "NA", + "geoname_id" => 6255149, + "names" => %{ + "de" => "Nordamerika", + "en" => "North America", + "es" => "Norteamérica", + "fr" => "Amérique du Nord", + "ja" => "北アメリカ", + "pt-BR" => "América do Norte", + "ru" => "Северная Америка", + "zh-CN" => "北美洲" + } + }, + "country" => %{ + "geoname_id" => 6252001, + "iso_code" => "US", + "names" => %{ + "de" => "Vereinigte Staaten", + "en" => "United States", + "es" => "Estados Unidos", + "fr" => "États Unis", + "ja" => "アメリカ", + "pt-BR" => "EUA", + "ru" => "США", + "zh-CN" => "美国" + } + }, + "location" => %{ + "accuracy_radius" => 50, + "latitude" => 34.1211, + "longitude" => -117.4362, + "metro_code" => 803, + "time_zone" => "America/Los_Angeles" + }, + "postal" => %{"code" => "92336"}, + "registered_country" => %{ + "geoname_id" => 6252001, + "iso_code" => "US", + "names" => %{ + "de" => "Vereinigte Staaten", + "en" => "United States", + "es" => "Estados Unidos", + "fr" => "États Unis", + "ja" => "アメリカ", + "pt-BR" => "EUA", + "ru" => "США", + "zh-CN" => "美国" + } + }, + "subdivisions" => [ + %{ + "geoname_id" => 5332921, + "iso_code" => "CA", + "names" => %{ + "de" => "Kalifornien", + "en" => "California", + "es" => "California", + "fr" => "Californie", + "ja" => "カリフォルニア州", + "pt-BR" => "Califórnia", + "ru" => "Калифорния", + "zh-CN" => "加州" + } + } + ] + } + + """ + def lookup(ip_address) do + case :locus.lookup(@db, ip_address) do + {:ok, entry} -> + entry + + :not_found -> + nil + + {:error, {:invalid_address, _address}} -> + nil + + {:error, reason} -> + Logger.error("Failed to lookup IP address. Reason: " <> inspect(reason)) + nil + end + end +end diff --git a/lib/plausible/goal.ex b/lib/plausible/goal.ex new file mode 100644 index 0000000000000000000000000000000000000000..1f1a58a087210097c0688c3c11726374c023ed0e --- /dev/null +++ b/lib/plausible/goal.ex @@ -0,0 +1,317 @@ +defmodule Plausible.Goal do + @moduledoc """ + Goal schema. + """ + + use Plausible + use Ecto.Schema + + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + on_ee do + @journey_end_event Plausible.Stats.Exploration.Journey.Step.journey_end_event() + else + @journey_end_event nil + end + + schema "goals" do + field :event_name, :string + field :page_path, :string + field :scroll_threshold, :integer, default: -1 + field :display_name, :string + + on_ee do + field :currency, Ecto.Enum, values: Money.Currency.known_current_currencies() + many_to_many :funnels, Plausible.Funnel, join_through: Plausible.Funnel.Step + else + field :currency, :string, virtual: true, default: nil + field :funnels, {:array, :map}, virtual: true, default: [] + end + + field :custom_props, :map, default: %{} + + belongs_to :site, Plausible.Site + + timestamps() + end + + @fields [ + :event_name, + :page_path, + :scroll_threshold, + :display_name, + :custom_props + ] ++ + on_ee(do: [:currency], else: []) + + @max_event_name_length 120 + + def max_event_name_length(), do: @max_event_name_length + + @max_custom_props_per_goal 3 + + def max_custom_props_per_goal(), do: @max_custom_props_per_goal + + @special_goals [ + "404", + "Outbound Link: Click", + "Cloaked Link: Click", + "File Download", + "Form: Submission", + "WP Search Queries", + "WP Form Completions" + ] + + def special_goals(), do: @special_goals + + @spec special_goal?(t() | String.t() | nil) :: boolean() + def special_goal?(%__MODULE__{event_name: event_name}), do: special_goal?(event_name) + def special_goal?(event_name) when is_binary(event_name), do: event_name in @special_goals + def special_goal?(_), do: false + + def changeset(goal, attrs \\ %{}) do + goal + |> cast(attrs, @fields) + |> update_leading_slash() + |> validate_event_name_and_page_path() + |> validate_page_path_for_scroll_goal() + |> maybe_put_display_name() + |> validate_change(:custom_props, &validate_custom_props/2) + |> unique_constraint(:display_name, name: :goals_display_name_unique) + |> unique_constraint(:event_name, name: :goals_event_config_unique) + |> unique_constraint([:page_path, :scroll_threshold], + name: :goals_pageview_config_unique + ) + |> validate_length(:event_name, max: @max_event_name_length) + |> validate_number(:scroll_threshold, + greater_than_or_equal_to: -1, + less_than_or_equal_to: 100, + message: "Should be -1 (missing) or in range [0, 100]" + ) + |> check_constraint(:event_name, + name: :check_event_name_or_page_path, + message: "cannot co-exist with page_path" + ) + |> maybe_drop_currency() + |> prevent_currency_change() + |> prevent_special_goal_renames() + end + + @spec display_name(t()) :: String.t() + def display_name(goal) do + goal.display_name + end + + @spec type(t()) :: :event | :scroll | :page + def type(goal) do + cond do + is_binary(goal.event_name) -> :event + is_binary(goal.page_path) && goal.scroll_threshold > -1 -> :scroll + is_binary(goal.page_path) -> :page + end + end + + @spec has_custom_props?(t()) :: boolean() + def has_custom_props?(%__MODULE__{custom_props: custom_props}) + when map_size(custom_props) > 0 do + true + end + + def has_custom_props?(_), do: false + + defp update_leading_slash(changeset) do + case get_field(changeset, :page_path) do + "/" <> _ -> + changeset + + page_path when is_binary(page_path) -> + put_change(changeset, :page_path, "/" <> page_path) + + _ -> + changeset + end + end + + defp validate_event_name_and_page_path(changeset) do + case {validate_page_path(changeset), validate_event_name(changeset)} do + {:ok, _} -> + update_change(changeset, :page_path, &String.trim/1) + + {_, :ok} -> + update_change(changeset, :event_name, &String.trim/1) + + {{:error, page_path_error}, {:error, event_name_error}} -> + changeset + |> add_error(:event_name, event_name_error) + |> add_error(:page_path, page_path_error) + end + end + + defp validate_page_path_for_scroll_goal(changeset) do + scroll_threshold = get_field(changeset, :scroll_threshold) + page_path = get_field(changeset, :page_path) + + if scroll_threshold > -1 and is_nil(page_path) do + changeset + |> add_error(:scroll_threshold, "page_path field missing for page scroll goal") + else + changeset + end + end + + defp validate_page_path(changeset) do + value = get_field(changeset, :page_path) + + if value && String.match?(value, ~r/^\/.*/) do + :ok + else + {:error, "this field is required and must start with a /"} + end + end + + defp validate_event_name(changeset) do + value = get_field(changeset, :event_name) + + cond do + value == "engagement" -> + {:error, "The event name 'engagement' is reserved and cannot be used as a goal"} + + journey_end_event?(value) -> + {:error, + "The event name '#{@journey_end_event}' is reserved and cannot be used as a goal"} + + value && String.match?(value, ~r/^.+/) -> + :ok + + true -> + {:error, "this field is required and cannot be blank"} + end + end + + defp maybe_drop_currency(changeset) do + if ee?() and get_field(changeset, :page_path) do + delete_change(changeset, :currency) + else + changeset + end + end + + defp prevent_currency_change(changeset) do + if (ee?() and changeset.data.id) && Map.has_key?(changeset.changes, :currency) do + add_error(changeset, :currency, "cannot change currency of existing goal") + else + changeset + end + end + + defp maybe_put_display_name(changeset) do + clause = + Enum.map([:display_name, :page_path, :event_name], &get_field(changeset, &1)) + + case clause do + [nil, path, _] when is_binary(path) -> + put_change(changeset, :display_name, "Visit " <> path) + + [nil, _, event_name] when is_binary(event_name) -> + put_change(changeset, :display_name, event_name) + + _ -> + changeset + end + |> update_change(:display_name, &String.trim/1) + |> validate_required(:display_name) + end + + defp validate_custom_props(:custom_props, custom_props) when is_map(custom_props) do + cond do + map_size(custom_props) > @max_custom_props_per_goal -> + [custom_props: "use at most #{@max_custom_props_per_goal} properties per goal"] + + not Enum.all?(custom_props, fn {k, v} -> + is_binary(k) and is_binary(v) + end) -> + [custom_props: "must be a map with string keys and string values"] + + Enum.any?(custom_props, fn {k, _v} -> + String.length(k) not in 1..Plausible.Props.max_prop_key_length() + end) -> + [ + custom_props: "key length is 1..#{Plausible.Props.max_prop_key_length()} characters" + ] + + Enum.any?(custom_props, fn {_k, v} -> + String.length(v) not in 1..Plausible.Props.max_prop_value_length() + end) -> + [ + custom_props: "value length is 1..#{Plausible.Props.max_prop_value_length()} characters" + ] + + true -> + [] + end + end + + defp prevent_special_goal_renames(changeset) do + if changeset.data.id && special_goal?(changeset.data.event_name) do + changeset + |> reject_special_goal_field_change(:event_name) + |> reject_special_goal_field_change(:display_name) + else + changeset + end + end + + defp reject_special_goal_field_change(changeset, :event_name) do + if Map.has_key?(changeset.changes, :event_name) do + add_error(changeset, :event_name, "cannot be changed for an automated goal") + else + changeset + end + end + + defp reject_special_goal_field_change(changeset, :display_name) do + new_value = get_change(changeset, :display_name) + canonical = changeset.data.event_name + + if new_value && new_value != canonical do + add_error(changeset, :display_name, "cannot be changed for an automated goal") + else + changeset + end + end + + on_ee do + defp journey_end_event?(name) do + name == @journey_end_event + end + else + defp journey_end_event?(_name), do: always(false) + end +end + +defimpl Jason.Encoder, for: Plausible.Goal do + def encode(value, opts) do + domain = value.site.domain + + value + |> Map.put(:goal_type, Plausible.Goal.type(value)) + |> Map.take([:id, :goal_type, :event_name, :page_path, :custom_props]) + |> Map.put(:domain, domain) + |> Map.put(:display_name, value.display_name) + |> Jason.Encode.map(opts) + end +end + +defimpl String.Chars, for: Plausible.Goal do + def to_string(goal) do + goal.display_name + end +end + +defimpl Phoenix.HTML.Safe, for: Plausible.Goal do + def to_iodata(data) do + data |> to_string() |> Phoenix.HTML.Engine.html_escape() + end +end diff --git a/lib/plausible/goals/goals.ex b/lib/plausible/goals/goals.ex new file mode 100644 index 0000000000000000000000000000000000000000..e79e278927c9174cdbf64032ceb70830c023d18e --- /dev/null +++ b/lib/plausible/goals/goals.ex @@ -0,0 +1,490 @@ +defmodule Plausible.Goals do + @moduledoc """ + Goals API. + """ + + use Plausible + use Plausible.Repo + use Plausible.Funnel.Const + + import Ecto.Query + + alias Plausible.Goal + alias Ecto.Multi + alias Ecto.Changeset + + @max_goals_per_site 1_000 + @spec max_goals_per_site(Keyword.t()) :: pos_integer() + def max_goals_per_site(opts \\ []) do + override = Keyword.get(opts, :max_goals_per_site) + + if override do + override + else + # see: config/test.exs - you can steer this limit for tests + # by providing `max_goals_per_site` option to e.g. create/3 + Application.get_env(:plausible, :max_goals_per_site, @max_goals_per_site) + end + end + + @spec get(Plausible.Site.t(), pos_integer()) :: nil | Plausible.Goal.t() + def get(site, id) when is_integer(id) do + q = + from g in Plausible.Goal, + where: g.site_id == ^site.id, + order_by: [desc: g.id], + where: g.id == ^id + + Repo.one(q) + end + + @spec create(Plausible.Site.t(), map(), Keyword.t()) :: + {:ok, Goal.t()} + | {:error, Changeset.t()} + | {:error, :upgrade_required} + | {:error, :revenue_goals_unavailable} + @doc """ + Creates a Goal for a site. + + If the created goal is a revenue goal, it sets site.updated_at to be + refreshed by the sites cache, as revenue goals are used during ingestion. + + Returns `{:ok, goal}` or `{:error, changeset}` when creation fails due to + invalid fields. It can also return: + + * `{:error, :upgrade_required}` - Adding a revenue goal is not allowed + for team's subscription. + + * `{:error, :revenue_goals_unavailable}` - When the site is a consolidated + view and the goal created is a revenue goal. Revenue goal creation is not + allowed for consolidated views due to the inability to force a single + currency on a goal across all consolidated sites. + """ + def create(site, params, opts \\ []) do + Repo.transaction(fn -> + case insert_goal(site, params, opts) do + {:ok, :insert, goal} -> + on_ee do + now = Keyword.get(opts, :now, DateTime.utc_now()) + # credo:disable-for-next-line Credo.Check.Refactor.Nesting + if Plausible.Goal.Revenue.revenue?(goal) do + Plausible.Site.Cache.touch_site!(site, now) + end + end + + Repo.preload(goal, :site) + + {:ok, :upsert, goal} -> + Repo.preload(goal, :site) + + {:error, cause} -> + Repo.rollback(cause) + end + end) + end + + @spec update(Plausible.Goal.t(), map()) :: + {:ok, Goal.t()} | {:error, Changeset.t()} | {:error, :upgrade_required} + def update(goal, params) do + changeset = Goal.changeset(goal, params) + + Repo.transaction(fn -> + site = Repo.preload(goal, :site).site + + with :ok <- maybe_check_feature_access(site, changeset), + {:ok, updated_goal} <- Repo.update(changeset), + :ok <- Plausible.Segments.update_goal_in_segments(site, goal, updated_goal) do + on_ee do + Repo.preload(updated_goal, :funnels) + else + updated_goal + end + else + {:error, %Changeset{} = changeset} -> + Repo.rollback(changeset) + + {:error, :upgrade_required} -> + Repo.rollback(:upgrade_required) + end + end) + end + + def find_or_create(site, params, opts \\ []) + + def find_or_create( + site, + %{ + "goal_type" => "event", + "event_name" => event_name, + "currency" => currency + } = params, + opts + ) + when is_binary(event_name) and is_binary(currency) do + with {:ok, goal} <- create(site, params, do_upsert(opts)) do + if to_string(goal.currency) == currency do + {:ok, goal} + else + # we must disallow creation of the same goal name with different currency + changeset = + goal + |> Goal.changeset() + |> Changeset.add_error( + :event_name, + "'#{goal.event_name}' (with currency: #{goal.currency}) has already been taken" + ) + + {:error, changeset} + end + end + end + + def find_or_create(site, %{"goal_type" => "event", "event_name" => event_name} = params, opts) + when is_binary(event_name) do + create(site, params, do_upsert(opts)) + end + + def find_or_create(_, %{"goal_type" => "event"}, _), do: {:missing, "event_name"} + + def find_or_create(site, %{"goal_type" => "page", "page_path" => _} = params, opts) do + create(site, params, do_upsert(opts)) + end + + def find_or_create(_, %{"goal_type" => "page"}, _), do: {:missing, "page_path"} + + def list_revenue_goals(site) do + from(g in Plausible.Goal, + where: g.site_id == ^site.id and not is_nil(g.currency), + select: %{display_name: g.display_name, currency: g.currency}, + order_by: [desc: g.id], + limit: ^max_goals_per_site() + ) + |> Plausible.Repo.all() + end + + def for_site(site, opts \\ []) do + site + |> for_site_query(opts) + |> Repo.all() + |> Enum.map(&maybe_trim/1) + end + + def for_site_query(site \\ nil, opts \\ []) do + query = + from g in Goal, + order_by: [desc: g.id], + limit: ^max_goals_per_site(opts) + + query = + if site do + from g in query, + inner_join: assoc(g, :site), + where: g.site_id == ^site.id, + preload: [:site] + else + query + end + + query = + if Keyword.get(opts, :include_goals_with_custom_props?, true) == false do + from g in query, where: g.custom_props == ^%{} + else + query + end + + if ee?() and opts[:preload_funnels?] == true do + from(g in query, + left_join: assoc(g, :funnels), + group_by: g.id, + preload: [:funnels] + ) + else + query + end + end + + def batch_create_event_goals(names, site, opts \\ []) do + Enum.reduce(names, [], fn name, acc -> + case insert_goal(site, %{event_name: name}, do_upsert(opts)) do + {:ok, _, goal} -> acc ++ [goal] + _ -> acc + end + end) + end + + @doc """ + If a goal belongs to funnel(s), we need to inspect their number of steps. + + If it exceeds the minimum allowed (defined via `Plausible.Funnel.min_steps/0`), + the funnel will be reduced (i.e. a step associated with the goal to be deleted + is removed), so that the minimum number of steps is preserved. This is done + implicitly, by postgres, as per on_delete: :delete_all. + + Otherwise, for associated funnel(s) consisting of minimum number steps only, + funnel record(s) are removed completely along with the targeted goal. + """ + def delete(id, %Plausible.Site{id: site_id}) do + delete(id, site_id) + end + + def delete(id, site_id) do + goal_query = + from(g in Goal, + where: g.id == ^id, + where: g.site_id == ^site_id + ) + + goal_query = on_ee(do: preload(goal_query, funnels: :steps), else: goal_query) + + result = + Multi.new() + |> Multi.one( + :goal, + goal_query + ) + |> Multi.run(:funnel_ids_to_wipe, fn + _, %{goal: nil} -> + {:error, :not_found} + + _, %{goal: %{funnels: []}} -> + {:ok, []} + + _, %{goal: %{funnels: funnels}} -> + funnels_to_wipe = + funnels + |> Enum.filter(&(Enum.count(&1.steps) == Funnel.Const.min_steps())) + |> Enum.map(& &1.id) + + {:ok, funnels_to_wipe} + end) + |> Multi.merge(fn + %{funnel_ids_to_wipe: []} -> + Ecto.Multi.new() + + %{funnel_ids_to_wipe: [_ | _] = funnel_ids} -> + Ecto.Multi.new() + |> Multi.delete_all( + :delete_funnels, + from(f in "funnels", + where: f.id in ^funnel_ids + ) + ) + end) + |> Multi.delete_all( + :delete_goals, + fn _ -> + from g in Goal, + where: g.id == ^id, + where: g.site_id == ^site_id + end + ) + |> Repo.transaction() + + case result do + {:ok, _} -> :ok + {:error, _step, reason, _context} -> {:error, reason} + end + end + + @spec count(Plausible.Site.t()) :: non_neg_integer() + def count(site) do + Repo.aggregate( + from( + g in Goal, + where: g.site_id == ^site.id + ), + :count + ) + end + + @spec create_outbound_links(Plausible.Site.t()) :: :ok + def create_outbound_links(%Plausible.Site{} = site) do + create(site, %{"event_name" => "Outbound Link: Click"}, do_upsert()) + :ok + end + + @spec create_file_downloads(Plausible.Site.t()) :: :ok + def create_file_downloads(%Plausible.Site{} = site) do + create(site, %{"event_name" => "File Download"}, do_upsert()) + :ok + end + + @spec create_form_submissions(Plausible.Site.t()) :: :ok + def create_form_submissions(%Plausible.Site{} = site) do + create(site, %{"event_name" => "Form: Submission"}, do_upsert()) + :ok + end + + @spec create_404(Plausible.Site.t()) :: :ok + def create_404(%Plausible.Site{} = site) do + create(site, %{"event_name" => "404"}, do_upsert()) + :ok + end + + @spec delete_outbound_links(Plausible.Site.t()) :: :ok + def delete_outbound_links(%Plausible.Site{} = site) do + q = + from g in Goal, + where: g.site_id == ^site.id, + where: g.event_name == "Outbound Link: Click" + + Repo.delete_all(q) + :ok + end + + @spec delete_file_downloads(Plausible.Site.t()) :: :ok + def delete_file_downloads(%Plausible.Site{} = site) do + q = + from g in Goal, + where: g.site_id == ^site.id, + where: g.event_name == "File Download" + + Repo.delete_all(q) + :ok + end + + @spec delete_404(Plausible.Site.t()) :: :ok + def delete_404(%Plausible.Site{} = site) do + q = + from g in Goal, + where: g.site_id == ^site.id, + where: g.event_name == "404" + + Repo.delete_all(q) + :ok + end + + @spec delete_form_submissions(Plausible.Site.t()) :: :ok + def delete_form_submissions(%Plausible.Site{} = site) do + q = + from g in Goal, + where: g.site_id == ^site.id, + where: g.event_name == "Form: Submission" + + Repo.delete_all(q) + :ok + end + + defp insert_goal(site, params, opts) do + params = Map.delete(params, "site_id") + + insert_opts = + if upsert?(opts) do + [on_conflict: :nothing] + else + [] + end + + changeset = + %Goal{} + |> Goal.changeset(params) + |> Ecto.Changeset.put_assoc(:site, site) + + with :ok <- maybe_check_feature_access(site, changeset), + :ok <- check_no_currency_if_consolidated(site, changeset), + :ok <- check_goals_limit(site, changeset, opts), + {:ok, goal} <- Repo.insert(changeset, insert_opts) do + # Upsert with `on_conflict: :nothing` strategy + # will result in goal struct missing primary key field + # which is generated by the database. + if goal.id do + {:ok, :insert, goal} + else + get_params = + goal + |> Map.take([:site_id, :event_name, :page_path]) + |> Enum.reject(fn {_, value} -> is_nil(value) end) + + {:ok, :upsert, Repo.get_by!(Goal, get_params)} + end + end + end + + defp maybe_check_feature_access(site, changeset) do + with :ok <- revenue_goals_access_check(site, changeset) do + custom_props_goals_access_check(site, changeset) + end + end + + defp revenue_goals_access_check(site, changeset) do + if Changeset.get_field(changeset, :currency) do + site = Plausible.Repo.preload(site, :team) + Plausible.Billing.Feature.RevenueGoals.check_availability(site.team) + else + :ok + end + end + + defp custom_props_goals_access_check(site, changeset) do + if map_size(Changeset.get_field(changeset, :custom_props)) > 0 do + site = Plausible.Repo.preload(site, :team) + Plausible.Billing.Feature.Props.check_availability(site.team) + else + :ok + end + end + + defp check_goals_limit(site, changeset, opts) do + if upsert?(opts) and goal_exists_for_upsert?(site, changeset) do + :ok + else + if count(site) >= max_goals_per_site(opts) and changeset.valid? do + changeset + |> Changeset.add_error(:page_path, "Maximum number of goals reached") + |> Changeset.add_error(:event_name, "Maximum number of goals reached") + |> Changeset.apply_action(:insert) + else + :ok + end + end + end + + defp check_no_currency_if_consolidated(site, changeset) do + if Plausible.Sites.consolidated?(site) && Changeset.get_field(changeset, :currency) do + {:error, :revenue_goals_unavailable} + else + :ok + end + end + + defp maybe_trim(%Goal{} = goal) do + # we make sure that even if we saved goals erroneously with trailing + # space, it's removed during fetch + goal + |> Map.update!(:event_name, &maybe_trim/1) + |> Map.update!(:page_path, &maybe_trim/1) + end + + defp maybe_trim(s) when is_binary(s) do + String.trim(s) + end + + defp maybe_trim(other) do + other + end + + defp upsert?(opts) do + Keyword.get(opts, :upsert?, false) + end + + defp do_upsert(opts \\ []) do + Keyword.put(opts, :upsert?, true) + end + + defp goal_exists_for_upsert?(site, changeset) do + event_name = Changeset.get_field(changeset, :event_name) + page_path = Changeset.get_field(changeset, :page_path) + scroll_threshold = Changeset.get_field(changeset, :scroll_threshold) + + query_params = + [site_id: site.id] + |> maybe_add_filter(:event_name, event_name) + |> maybe_add_filter(:page_path, page_path) + |> maybe_add_filter(:scroll_threshold, scroll_threshold) + + Repo.exists?(from(g in Goal, where: ^query_params)) + end + + defp maybe_add_filter(params, _key, nil), do: params + defp maybe_add_filter(params, key, value), do: Keyword.put(params, key, value) +end diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex new file mode 100644 index 0000000000000000000000000000000000000000..8dde546d031fda1179bba0dcae13638909dcc52f --- /dev/null +++ b/lib/plausible/google/api.ex @@ -0,0 +1,201 @@ +defmodule Plausible.Google.API do + @moduledoc """ + API to Google services. + """ + + alias Plausible.Google.HTTP + alias Plausible.Google.SearchConsole + alias Plausible.Stats.Query + + @search_console_scope URI.encode_www_form( + "email https://www.googleapis.com/auth/webmasters.readonly" + ) + @import_scope URI.encode_www_form("email https://www.googleapis.com/auth/analytics.readonly") + + @verified_permission_levels ["siteOwner", "siteFullUser", "siteRestrictedUser"] + + @valid_oauth_contexts ["search-console", "import"] + + @oauth_salt "google-oauth-state" + + def search_console_authorize_url(site) do + "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@search_console_scope}&state=" <> + sign_oauth_state(site, "search-console") + end + + def import_authorize_url(site) do + "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <> + sign_oauth_state(site, "import") + end + + @doc false + def sign_oauth_state(site, context) do + Phoenix.Token.sign(PlausibleWeb.Endpoint, @oauth_salt, [site.id, context]) + end + + def verify_oauth_state(state) do + case Phoenix.Token.verify(PlausibleWeb.Endpoint, @oauth_salt, state, max_age: 3600) do + {:ok, [site_id, context]} when context in @valid_oauth_contexts -> + if site = Plausible.Repo.get(Plausible.Site, site_id) do + {:ok, %{site: site, context: context}} + else + {:error, {:invalid_oauth_state, :missing_site}} + end + + {:error, token_error} -> + {:error, {:invalid_oauth_state, token_error}} + end + end + + def fetch_access_token!(code) do + HTTP.fetch_access_token!(code) + end + + def list_properties(access_token) do + Plausible.Google.GA4.API.list_properties(access_token) + end + + def get_property(access_token, property) do + Plausible.Google.GA4.API.get_property(access_token, property) + end + + def get_analytics_start_date(access_token, property) do + Plausible.Google.GA4.API.get_analytics_start_date(access_token, property) + end + + def get_analytics_end_date(access_token, property) do + Plausible.Google.GA4.API.get_analytics_end_date(access_token, property) + end + + def fetch_verified_properties(auth) do + with {:ok, access_token} <- maybe_refresh_token(auth), + {:ok, sites} <- Plausible.Google.HTTP.list_sites(access_token) do + sites + |> Map.get("siteEntry", []) + |> Enum.filter(fn site -> site["permissionLevel"] in @verified_permission_levels end) + |> Enum.map(fn site -> site["siteUrl"] end) + |> Enum.map(fn url -> String.trim_trailing(url, "/") end) + |> then(&{:ok, &1}) + end + end + + def fetch_stats(site, query, pagination, search) do + with {:ok, site} <- ensure_search_console_property(site), + {:ok, access_token} <- maybe_refresh_token(site.google_auth), + {:ok, gsc_filters} <- + SearchConsole.Filters.transform(site.google_auth.property, query.filters, search), + {:ok, stats} <- + HTTP.list_stats( + access_token, + site.google_auth.property, + Query.date_range(query), + pagination, + gsc_filters + ) do + stats + |> Map.get("rows", []) + |> Enum.map(&search_console_row/1) + |> then(&{:ok, &1}) + else + :google_property_not_configured -> {:error, :google_property_not_configured} + :unsupported_filters -> {:error, :unsupported_filters} + {:error, error} -> {:error, error} + end + end + + def maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do + with true <- needs_to_refresh_token?(auth.expires), + {:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token), + changeset <- + Plausible.Site.GoogleAuth.changeset(auth, %{ + access_token: new_access_token, + expires: expires_at + }), + {:ok, _google_auth} <- Plausible.Repo.update(changeset) do + {:ok, new_access_token} + else + false -> {:ok, auth.access_token} + {:error, cause} -> {:error, cause} + end + end + + def maybe_refresh_token({access_token, refresh_token, expires_at}) do + with true <- needs_to_refresh_token?(expires_at), + {:ok, {new_access_token, _expires_at}} <- do_refresh_token(refresh_token) do + {:ok, new_access_token} + else + false -> {:ok, access_token} + {:error, cause} -> {:error, cause} + end + end + + def property?(value), do: String.starts_with?(value, "properties/") + + defp do_refresh_token(refresh_token) do + case HTTP.refresh_auth_token(refresh_token) do + {:ok, %{"access_token" => new_access_token, "expires_in" => expires_in}} -> + expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in) + {:ok, {new_access_token, expires_at}} + + {:error, cause} -> + {:error, cause} + end + end + + defp needs_to_refresh_token?(expires_at) when is_binary(expires_at) do + expires_at + |> NaiveDateTime.from_iso8601!() + |> needs_to_refresh_token?() + end + + defp needs_to_refresh_token?(%NaiveDateTime{} = expires_at) do + thirty_seconds_ago = DateTime.shift(DateTime.utc_now(), second: 30) + NaiveDateTime.before?(expires_at, thirty_seconds_ago) + end + + defp ensure_search_console_property(site) do + site = Plausible.Repo.preload(site, :google_auth) + + if site.google_auth && site.google_auth.property do + {:ok, site} + else + :google_property_not_configured + end + end + + defp search_console_row(row) do + %{ + # We always request just one dimension at a time (`query`) + name: row["keys"] |> List.first(), + visitors: row["clicks"], + impressions: row["impressions"], + ctr: rounded_ctr(row["ctr"]), + position: rounded_position(row["position"]) + } + end + + defp rounded_ctr(ctr) do + {:ok, decimal} = Decimal.cast(ctr) + + decimal + |> Decimal.mult(100) + |> Decimal.round(1) + |> Decimal.to_float() + end + + defp rounded_position(position) do + {:ok, decimal} = Decimal.cast(position) + + decimal + |> Decimal.round(1) + |> Decimal.to_float() + end + + defp client_id() do + Keyword.fetch!(Application.get_env(:plausible, :google), :client_id) + end + + defp redirect_uri() do + PlausibleWeb.Endpoint.url() <> "/auth/google/callback" + end +end diff --git a/lib/plausible/google/ga4/api.ex b/lib/plausible/google/ga4/api.ex new file mode 100644 index 0000000000000000000000000000000000000000..2dc43328464c2cd45bf56ffb25a0db9ba09f8987 --- /dev/null +++ b/lib/plausible/google/ga4/api.ex @@ -0,0 +1,215 @@ +defmodule Plausible.Google.GA4.API do + @moduledoc """ + API for Google Analytics 4. + """ + + alias Plausible.Google + alias Plausible.Google.GA4 + + require Logger + + @type import_auth :: { + access_token :: String.t(), + refresh_token :: String.t(), + expires_at :: String.t() + } + + @per_page 200_000 + @backoff_factor :timer.seconds(10) + @max_attempts 5 + + def list_properties(access_token) do + case GA4.HTTP.list_accounts_for_user(access_token) do + {:ok, %{"accountSummaries" => accounts}} -> + accounts = + accounts + |> Enum.filter(& &1["propertySummaries"]) + |> Enum.map(fn account -> + %{"account" => account_id, "displayName" => account_name} = account + + {"#{account_name} (#{account_id})", + Enum.map(account["propertySummaries"], fn property -> + %{"displayName" => property_name, "property" => property_id} = property + + {"#{property_name} (#{property_id})", property_id} + end)} + end) + + {:ok, accounts} + + {:ok, _} -> + {:ok, []} + + {:error, cause} -> + {:error, cause} + end + end + + def get_property(access_token, lookup_property) do + case GA4.HTTP.get_property(access_token, lookup_property) do + {:ok, property} -> + %{"displayName" => property_name, "name" => property_id, "account" => account_id} = + property + + {:ok, + %{ + id: property_id, + name: "#{property_name} (#{property_id})", + account_id: account_id + }} + + {:error, cause} -> + {:error, cause} + end + end + + def get_analytics_start_date(access_token, property) do + GA4.HTTP.get_analytics_start_date(access_token, property) + end + + def get_analytics_end_date(access_token, property) do + GA4.HTTP.get_analytics_end_date(access_token, property) + end + + def import_analytics(date_range, property, auth, opts) do + persist_fn = Keyword.fetch!(opts, :persist_fn) + fetch_opts = Keyword.get(opts, :fetch_opts, []) + resume_opts = Keyword.get(opts, :resume_opts, []) + + Logger.debug( + "[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}" + ) + + with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do + do_import_analytics(date_range, property, access_token, persist_fn, fetch_opts, resume_opts) + end + end + + defp do_import_analytics( + date_range, + property, + access_token, + persist_fn, + fetch_opts, + [] = _resume_opts + ) do + Enum.reduce_while(GA4.ReportRequest.full_report(), :ok, fn report_request, :ok -> + Logger.debug( + "[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}" + ) + + report_request = prepare_request(report_request, date_range, property, access_token) + + case fetch_and_persist(report_request, persist_fn: persist_fn, fetch_opts: fetch_opts) do + :ok -> {:cont, :ok} + {:error, _} = error -> {:halt, error} + end + end) + end + + defp do_import_analytics( + date_range, + property, + access_token, + persist_fn, + fetch_opts, + resume_opts + ) do + dataset = Keyword.fetch!(resume_opts, :dataset) + offset = Keyword.fetch!(resume_opts, :offset) + + GA4.ReportRequest.full_report() + |> Enum.drop_while(&(&1.dataset != dataset)) + |> Enum.reduce_while(:ok, fn report_request, :ok -> + Logger.debug( + "[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}" + ) + + request_offset = + if report_request.dataset == dataset do + offset + else + 0 + end + + report_request = + report_request + |> prepare_request(date_range, property, access_token) + |> Map.put(:offset, request_offset) + + case fetch_and_persist(report_request, persist_fn: persist_fn, fetch_opts: fetch_opts) do + :ok -> {:cont, :ok} + {:error, _} = error -> {:halt, error} + end + end) + end + + @spec fetch_and_persist(GA4.ReportRequest.t(), Keyword.t()) :: + :ok | {:error, term()} + def fetch_and_persist(%GA4.ReportRequest{} = report_request, opts \\ []) do + persist_fn = Keyword.fetch!(opts, :persist_fn) + attempt = Keyword.get(opts, :attempt, 1) + fetch_opts = Keyword.get(opts, :fetch_opts, []) + max_attempts = Keyword.get(fetch_opts, :max_attempts, @max_attempts) + sleep_time = Keyword.get(fetch_opts, :sleep_time, @backoff_factor) + + case GA4.HTTP.get_report(report_request) do + {:ok, {_, 0}} -> + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Fetched empty response for #{report_request.dataset}" + ) + + :ok + + {:ok, {rows, row_count}} -> + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Fetched #{length(rows)} rows of total #{row_count} with offset #{report_request.offset} for #{report_request.dataset}" + ) + + :ok = persist_fn.(report_request.dataset, rows) + + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Persisted #{length(rows)} for #{report_request.dataset}" + ) + + if report_request.offset + @per_page < row_count do + fetch_and_persist( + %GA4.ReportRequest{report_request | offset: report_request.offset + @per_page}, + opts + ) + else + :ok + end + + {:error, {:rate_limit_exceeded, details}} -> + {:error, {:rate_limit_exceeded, details}} + + {:error, cause} -> + if attempt >= max_attempts do + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Terminating." + ) + + {:error, cause} + else + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Will retry." + ) + + Process.sleep(attempt * sleep_time) + fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1)) + end + end + end + + defp prepare_request(%GA4.ReportRequest{} = report_request, date_range, property, access_token) do + %GA4.ReportRequest{ + report_request + | date_range: date_range, + property: property, + access_token: access_token, + offset: 0, + limit: @per_page + } + end +end diff --git a/lib/plausible/google/ga4/http.ex b/lib/plausible/google/ga4/http.ex new file mode 100644 index 0000000000000000000000000000000000000000..5f3a1fccc8a99618f331558312b929362d4715ad --- /dev/null +++ b/lib/plausible/google/ga4/http.ex @@ -0,0 +1,330 @@ +defmodule Plausible.Google.GA4.HTTP do + @moduledoc """ + HTTP client implementation for Google Analytics 4 API. + """ + + use Plausible + + alias Plausible.HTTPClient + + require Logger + + @spec get_report(Plausible.Google.GA4.ReportRequest.t()) :: + {:ok, {[map()], non_neg_integer()}} | {:error, any()} + def get_report(%Plausible.Google.GA4.ReportRequest{} = report_request) do + params = %{ + requests: [ + %{ + property: report_request.property, + dateRanges: [ + %{ + startDate: report_request.date_range.first, + endDate: report_request.date_range.last + } + ], + dimensions: Enum.map(report_request.dimensions, &%{name: &1}), + metrics: Enum.map(report_request.metrics, &build_metric/1), + orderBys: [ + %{ + dimension: %{ + dimensionName: "date", + orderType: "ALPHANUMERIC" + }, + desc: true + } + ], + dimensionFilter: report_request.dimension_filter, + limit: report_request.limit, + offset: report_request.offset + } + ] + } + + url = + "#{reporting_api_url()}/v1beta/#{report_request.property}:batchRunReports" + + response = + HTTPClient.impl().post( + url, + [{"Authorization", "Bearer #{report_request.access_token}"}], + params, + receive_timeout: 80_000 + ) + + with {:ok, %{body: body}} <- response, + {:ok, report} <- parse_report_from_response(body), + row_count <- Map.get(report, "rowCount", 0), + {:ok, report} <- convert_to_maps(report) do + {:ok, {report, row_count}} + else + {:error, %{reason: %{status: 429, body: body}} = error} -> + log_ce_error("retrieving report for #{report_request.dataset}", error) + + Logger.debug( + "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} due to exceeding rate limit." + ) + + Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: 429}}) + + {:error, + {:rate_limit_exceeded, dataset: report_request.dataset, offset: report_request.offset}} + + {:error, %{reason: %{status: status, body: body}} = error} when status >= 500 -> + log_ce_error("retrieving report for #{report_request.dataset}", error) + + Logger.warning( + "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}" + ) + + Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}}) + + {:error, {:server_failed, dataset: report_request.dataset, offset: report_request.offset}} + + {:error, %{reason: %{status: status, body: body}} = error} -> + log_ce_error("retrieving report for #{report_request.dataset}", error) + + Logger.warning( + "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}" + ) + + Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}}) + + {:error, :request_failed} + + {:error, reason} -> + log_ce_error("retrieving report for #{report_request.dataset}", reason) + + Logger.warning( + "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}: #{inspect(reason)}" + ) + + Sentry.Context.set_extra_context(%{ga_response: %{body: inspect(reason), status: 0}}) + + {:error, {:socket_failed, dataset: report_request.dataset, offset: report_request.offset}} + end + end + + defp build_metric(expression) do + case String.split(expression, " = ") do + [name, expression] -> + %{ + name: name, + expression: expression + } + + [name] -> + %{name: name} + end + end + + defp parse_report_from_response(%{"reports" => [report | _]}) do + {:ok, report} + end + + defp parse_report_from_response(body) do + Sentry.Context.set_extra_context(%{google_analytics4_response: body}) + + Logger.error( + "Google Analytics 4: Failed to find report in response. Reason: #{inspect(body)}" + ) + + {:error, {:invalid_response, body}} + end + + defp convert_to_maps(%{ + "rows" => rows, + "dimensionHeaders" => dimension_headers, + "metricHeaders" => metric_headers + }) + when is_list(rows) do + dimension_headers = Enum.map(dimension_headers, & &1["name"]) + metric_headers = Enum.map(metric_headers, & &1["name"]) + + report = + Enum.map(rows, fn %{"dimensionValues" => dimensions, "metricValues" => metrics} -> + dimension_values = Enum.map(dimensions, & &1["value"]) + metric_values = Enum.map(metrics, & &1["value"]) + metrics = Enum.zip(metric_headers, metric_values) + dimensions = Enum.zip(dimension_headers, dimension_values) + %{metrics: Map.new(metrics), dimensions: Map.new(dimensions)} + end) + + {:ok, report} + end + + defp convert_to_maps(%{"dimensionHeaders" => _, "metricHeaders" => _}) do + {:ok, []} + end + + defp convert_to_maps(response) do + Logger.error( + "Google Analytics 4: Failed to read report in response. Reason: #{inspect(response)}" + ) + + Sentry.Context.set_extra_context(%{google_analytics4_response: response}) + {:error, {:invalid_response, response}} + end + + def list_accounts_for_user(access_token) do + url = "#{admin_api_url()}/v1beta/accountSummaries?pageSize=200" + + headers = [{"Authorization", "Bearer #{access_token}"}] + + case HTTPClient.impl().get(url, headers) do + {:ok, %Finch.Response{body: body, status: 200}} -> + {:ok, body} + + {:error, %HTTPClient.Non200Error{reason: %{status: 429}} = error} -> + log_ce_error("listing accounts for user", error) + {:error, :rate_limit_exceeded} + + {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> + log_ce_error("listing accounts for user", error) + {:error, authentication_failed(error)} + + {:error, %{reason: :timeout} = error} -> + log_ce_error("listing accounts for user", error) + {:error, :timeout} + + {:error, error} -> + log_ce_error("listing accounts for user", error) + Sentry.capture_message("Error listing GA4 accounts for user", extra: %{error: error}) + {:error, :unknown} + end + end + + def get_property(access_token, property) do + url = "#{admin_api_url()}/v1beta/#{property}" + + headers = [{"Authorization", "Bearer #{access_token}"}] + + case HTTPClient.impl().get(url, headers) do + {:ok, %Finch.Response{body: body, status: 200}} -> + {:ok, body} + + {:error, %HTTPClient.Non200Error{reason: %{status: 429}} = error} -> + log_ce_error("retrieving property #{property}", error) + {:error, :rate_limit_exceeded} + + {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> + log_ce_error("retrieving property #{property}", error) + {:error, authentication_failed(error)} + + {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [404] -> + log_ce_error("retrieving property #{property}", error) + {:error, :not_found} + + {:error, %{reason: :timeout} = error} -> + log_ce_error("retrieving property #{property}", error) + {:error, :timeout} + + {:error, error} -> + log_ce_error("retrieving property #{property}", error) + + Sentry.capture_message("Error retrieving GA4 property #{property}", + extra: %{error: error} + ) + + {:error, :unknown} + end + end + + @earliest_valid_date "2015-08-14" + + def get_analytics_start_date(access_token, property) do + get_analytics_boundary_date(access_token, property, :start) + end + + def get_analytics_end_date(access_token, property) do + get_analytics_boundary_date(access_token, property, :end) + end + + defp get_analytics_boundary_date(access_token, property, edge) do + descending? = edge == :end + + params = %{ + requests: [ + %{ + property: "#{property}", + dateRanges: [ + %{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Date.utc_today())} + ], + dimensions: [%{name: "date"}], + metrics: [%{name: "screenPageViews"}], + orderBys: [ + %{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: descending?} + ], + limit: 1 + } + ] + } + + url = "#{reporting_api_url()}/v1beta/#{property}:batchRunReports" + headers = [{"Authorization", "Bearer #{access_token}"}] + + case HTTPClient.impl().post(url, headers, params) do + {:ok, %Finch.Response{body: body, status: 200}} -> + report = List.first(body["reports"]) + + date = + case report["rows"] do + [%{"dimensionValues" => [%{"value" => date_str}]}] -> + Plausible.Times.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date() + + _ -> + nil + end + + {:ok, date} + + {:error, %HTTPClient.Non200Error{reason: %{status: 429}} = error} -> + log_ce_error("retrieving #{edge} date", error) + {:error, :rate_limit_exceeded} + + {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] -> + log_ce_error("retrieving #{edge} date", error) + {:error, authentication_failed(error)} + + {:error, %{reason: :timeout} = error} -> + log_ce_error("retrieving #{edge} date", error) + {:error, :timeout} + + {:error, error} -> + log_ce_error("retrieving #{edge} date", error) + + Sentry.capture_message("Error retrieving GA4 #{edge} date", + extra: %{error: error} + ) + + {:error, :unknown} + end + end + + defp reporting_api_url, do: "https://analyticsdata.googleapis.com" + defp admin_api_url, do: "https://analyticsadmin.googleapis.com" + + @spec authentication_failed(HTTPClient.Non200Error.t()) :: + {:authentication_failed, String.t() | nil} + defp authentication_failed(error) do + message = + case error.reason.body do + %{"error" => %{"message" => message}} when is_binary(message) -> message + _ -> nil + end + + {:authentication_failed, message} + end + + @spec log_ce_error(String.t(), any) :: :ok + defp log_ce_error(action, error) + + on_ce do + defp log_ce_error(action, error) do + Logger.error("Google Analytics 4: Failed when #{action}. Reason: #{inspect(error)}") + end + end + + on_ee do + defp log_ce_error(_action, _error), do: :ok + end +end diff --git a/lib/plausible/google/ga4/report_request.ex b/lib/plausible/google/ga4/report_request.ex new file mode 100644 index 0000000000000000000000000000000000000000..2019320f4c2bb726300aa42e12e8c35070b2c08e --- /dev/null +++ b/lib/plausible/google/ga4/report_request.ex @@ -0,0 +1,168 @@ +defmodule Plausible.Google.GA4.ReportRequest do + @moduledoc """ + Report request struct for Google Analytics 4 API + """ + + @excluded_event_names [ + "page_view", + "session_start", + "first_visit", + "user_engagement" + ] + + defstruct [ + :dataset, + :dimensions, + :metrics, + :date_range, + :property, + :access_token, + :offset, + :dimension_filter, + :limit + ] + + @type t() :: %__MODULE__{ + dataset: String.t(), + dimensions: [String.t()], + metrics: [String.t()], + date_range: Date.Range.t(), + property: term(), + access_token: String.t(), + offset: non_neg_integer(), + limit: non_neg_integer() + } + + def full_report do + [ + %__MODULE__{ + dataset: "imported_visitors", + dimensions: ["date"], + metrics: [ + "totalUsers", + "screenPageViews", + "bounces = sessions - engagedSessions", + "sessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_sources", + dimensions: [ + "date", + "sessionSource", + "sessionDefaultChannelGroup", + "sessionMedium", + "sessionCampaignName", + "sessionManualAdContent", + "sessionGoogleAdsKeyword" + ], + metrics: [ + "screenPageViews", + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_pages", + dimensions: ["date", "hostName", "pagePath"], + # NOTE: no exits as GA4 DATA API does not provide that metric + metrics: [ + "totalUsers", + "activeUsers", + "screenPageViews", + "sessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_entry_pages", + dimensions: ["date", "landingPage"], + metrics: [ + "screenPageViews", + "totalUsers", + "sessions", + "userEngagementDuration", + "bounces = sessions - engagedSessions" + ] + }, + # NOTE: Skipping for now as there's no dimension directly mapping to exit page path + # %__MODULE__{ + # dataset: "imported_exit_pages", + # dimensions: ["date", "ga:exitPagePath"], + # metrics: [ + # "totalUsers", + # "sessions", + # "screenPageViews", + # "userEngagementDuration", + # "bounces = sessions - engagedSessions" + # ] + # }, + %__MODULE__{ + dataset: "imported_custom_events", + dimensions: ["date", "eventName", "linkUrl"], + metrics: [ + "totalUsers", + "eventCount" + ], + dimension_filter: %{ + "notExpression" => %{ + "filter" => %{ + "fieldName" => "eventName", + "inListFilter" => %{ + "values" => @excluded_event_names, + "caseSensitive" => true + } + } + } + } + }, + %__MODULE__{ + dataset: "imported_locations", + dimensions: ["date", "countryId", "region", "city"], + metrics: [ + "screenPageViews", + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_devices", + dimensions: ["date", "deviceCategory"], + metrics: [ + "screenPageViews", + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_browsers", + dimensions: ["date", "browser"], + metrics: [ + "screenPageViews", + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + }, + %__MODULE__{ + dataset: "imported_operating_systems", + dimensions: ["date", "operatingSystem", "operatingSystemVersion"], + metrics: [ + "screenPageViews", + "totalUsers", + "sessions", + "bounces = sessions - engagedSessions", + "userEngagementDuration" + ] + } + ] + end +end diff --git a/lib/plausible/google/http.ex b/lib/plausible/google/http.ex new file mode 100644 index 0000000000000000000000000000000000000000..843f1dc1e8c918472234a85fb231835d702cbbf5 --- /dev/null +++ b/lib/plausible/google/http.ex @@ -0,0 +1,110 @@ +defmodule Plausible.Google.HTTP do + @moduledoc """ + HTTP client implementation for Google APIs. + """ + + require Logger + alias Plausible.HTTPClient + + def list_sites(access_token) do + url = "#{api_url()}/webmasters/v3/sites" + headers = [{"Content-Type", "application/json"}, {"Authorization", "Bearer #{access_token}"}] + + case HTTPClient.impl().get(url, headers) do + {:ok, %{body: body}} -> + {:ok, body} + + {:error, %{reason: %{status: s}}} when s in [401, 403] -> + {:error, "google_auth_error"} + + {:error, %{reason: %{body: %{"error" => error}}}} -> + {:error, error} + + {:error, reason} -> + Logger.error("Google Analytics: failed to list sites: #{inspect(reason)}") + {:error, "failed_to_list_sites"} + end + end + + def fetch_access_token!(code) do + url = "#{api_url()}/oauth2/v4/token" + headers = [{"Content-Type", "application/x-www-form-urlencoded"}] + + params = %{ + client_id: client_id(), + client_secret: client_secret(), + code: code, + grant_type: :authorization_code, + redirect_uri: redirect_uri() + } + + {:ok, response} = HTTPClient.impl().post(url, headers, params) + + response.body + end + + def list_stats(access_token, property, date_range, pagination, search_console_filters) do + {limit, page} = pagination + + params = %{ + startDate: Date.to_iso8601(date_range.first), + endDate: Date.to_iso8601(date_range.last), + dimensions: ["query"], + rowLimit: limit, + startRow: page * limit, + dimensionFilterGroups: search_console_filters + } + + url = + "#{api_url()}/webmasters/v3/sites/#{URI.encode_www_form(property)}/searchAnalytics/query" + + headers = [{"Authorization", "Bearer #{access_token}"}] + + case HTTPClient.impl().post(url, headers, params) do + {:ok, %Finch.Response{body: body, status: 200}} -> + {:ok, body} + + {:error, %{reason: %Finch.Response{body: _body, status: status}}} + when status in [401, 403] -> + {:error, "google_auth_error"} + + {:error, %{reason: %{body: %{"error" => error}}}} -> + {:error, error} + + {:error, reason} -> + Logger.error("Google Search Console: failed to list stats: #{inspect(reason)}") + {:error, "failed_to_list_stats"} + end + end + + def refresh_auth_token(refresh_token) do + url = "#{api_url()}/oauth2/v4/token" + headers = [{"content-type", "application/x-www-form-urlencoded"}] + + params = %{ + client_id: client_id(), + client_secret: client_secret(), + refresh_token: refresh_token, + grant_type: :refresh_token, + redirect_uri: redirect_uri() + } + + case HTTPClient.impl().post(url, headers, params) do + {:ok, %Finch.Response{body: body, status: 200}} -> + {:ok, body} + + {:error, %{reason: %Finch.Response{body: %{"error" => error}, status: _non_http_200}}} -> + {:error, error} + + {:error, %{reason: _} = e} -> + Sentry.capture_message("Error fetching Google queries", extra: %{error: inspect(e)}) + {:error, :unknown_error} + end + end + + defp config, do: Application.get_env(:plausible, :google) + defp client_id, do: Keyword.fetch!(config(), :client_id) + defp client_secret, do: Keyword.fetch!(config(), :client_secret) + defp api_url, do: Keyword.fetch!(config(), :api_url) + defp redirect_uri, do: PlausibleWeb.Endpoint.url() <> "/auth/google/callback" +end diff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.ex new file mode 100644 index 0000000000000000000000000000000000000000..1ea9c4a8fa2dac122402363cfe48477a8ab5bbac --- /dev/null +++ b/lib/plausible/google/search_console/filters.ex @@ -0,0 +1,77 @@ +defmodule Plausible.Google.SearchConsole.Filters do + @moduledoc false + import Plausible.Stats.Filters.Utils, only: [page_regex: 1] + + def transform(property, plausible_filters, search) do + gsc_filters = + Enum.reduce_while(plausible_filters, [], fn plausible_filter, gsc_filters -> + case transform_filter(property, plausible_filter) do + :unsupported -> {:halt, :unsupported_filters} + :ignore -> {:cont, gsc_filters} + gsc_filter -> {:cont, [gsc_filter | gsc_filters]} + end + end) + |> maybe_add_search_filter(search) + + case gsc_filters do + :unsupported_filters -> :unsupported_filters + [] -> {:ok, []} + filters when is_list(filters) -> {:ok, [%{filters: filters}]} + end + end + + defp transform_filter(property, [op, "event:page" | rest]) do + transform_filter(property, [op, "visit:entry_page" | rest]) + end + + # :TODO: Should also work case-insensitive, if not, blacklist. + defp transform_filter(property, [:is, "visit:entry_page", pages | _]) when is_list(pages) do + expression = + Enum.map_join(pages, "|", fn page -> property_url(property, Regex.escape(page)) end) + + %{dimension: "page", operator: "includingRegex", expression: expression} + end + + defp transform_filter(property, [:matches_wildcard, "visit:entry_page", pages | _]) + when is_list(pages) do + expression = + Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end) + + %{dimension: "page", operator: "includingRegex", expression: expression} + end + + defp transform_filter(_property, [:is, "visit:screen", devices | _]) when is_list(devices) do + expression = Enum.map_join(devices, "|", &search_console_device/1) + %{dimension: "device", operator: "includingRegex", expression: expression} + end + + defp transform_filter(_property, [:is, "visit:country", countries | _]) + when is_list(countries) do + expression = Enum.map_join(countries, "|", &search_console_country/1) + %{dimension: "country", operator: "includingRegex", expression: expression} + end + + defp transform_filter(_, [_, "visit:source" | _rest]), do: :ignore + + defp transform_filter(_, [_, "visit:channel" | _rest]), do: :ignore + + defp transform_filter(_, _filter), do: :unsupported + + defp property_url("sc-domain:" <> domain, page), do: "https://" <> domain <> page + defp property_url(url, page), do: url <> page + + defp search_console_device("Desktop"), do: "DESKTOP" + defp search_console_device("Mobile"), do: "MOBILE" + defp search_console_device("Tablet"), do: "TABLET" + + defp search_console_country(alpha_2) do + country = Location.Country.get_country(alpha_2) + country.alpha_3 + end + + defp maybe_add_search_filter(gsc_filters, search) when byte_size(search) > 0 do + [%{operator: "includingRegex", expression: search, dimension: "query"} | gsc_filters] + end + + defp maybe_add_search_filter(gsc_filters, _search), do: gsc_filters +end diff --git a/lib/plausible/helpers/changeset.ex b/lib/plausible/helpers/changeset.ex new file mode 100644 index 0000000000000000000000000000000000000000..41fb0556899e8a9705a3e9570a3d54f4d79db508 --- /dev/null +++ b/lib/plausible/helpers/changeset.ex @@ -0,0 +1,34 @@ +defmodule Plausible.ChangesetHelpers do + @moduledoc "Helper function for working with Ecto changesets" + + def traverse_errors(changeset) do + Ecto.Changeset.traverse_errors(changeset, fn {msg, opts} -> + Regex.replace(~r"%{(\w+)}", msg, fn _, key -> + opts + |> Keyword.get(String.to_existing_atom(key), key) + |> to_string() + end) + end) + end + + @doc """ + iex> serialize_first_error([{"name", {"should be at most %{count} byte(s)", [count: 255]}}]) + "name should be at most 255 byte(s)" + """ + def serialize_first_error(errors) do + {field, {message, opts}} = List.first(errors) + + formatted_message = + Enum.reduce(opts, message, fn {key, value}, acc -> + placeholder = "%{#{key}}" + + if String.contains?(acc, placeholder) do + String.replace(acc, placeholder, to_string(value)) + else + acc + end + end) + + "#{field} #{formatted_message}" + end +end diff --git a/lib/plausible/helpers/config.ex b/lib/plausible/helpers/config.ex new file mode 100644 index 0000000000000000000000000000000000000000..ed5833bd088b2f5e7952ef40373e147014b3b915 --- /dev/null +++ b/lib/plausible/helpers/config.ex @@ -0,0 +1,53 @@ +defmodule Plausible.ConfigHelpers do + def get_var_from_path_or_env(config_dir, var_name, default \\ nil) do + var_path = Path.join(config_dir, var_name) + + if File.exists?(var_path) do + File.read!(var_path) |> String.trim() + else + System.get_env(var_name, default) + end + end + + def get_int_from_path_or_env(config_dir, var_name, default \\ nil) do + var = get_var_from_path_or_env(config_dir, var_name) + + case var do + nil -> + default + + var -> + case Integer.parse(var) do + {int, ""} -> int + _ -> raise "Config variable #{var_name} must be an integer. Got #{var}" + end + end + end + + def get_bool_from_path_or_env(config_dir, var_name, default \\ nil) do + case get_var_from_path_or_env(config_dir, var_name) do + nil -> default + var -> parse_bool(var) + end + end + + @var_true ["1", "t", "true", "y", "yes", "on"] + @var_false ["0", "f", "false", "n", "no", "off"] + @var_bool_message Enum.zip_with(@var_true, @var_false, fn t, f -> [t, f] end) + |> List.flatten() + |> Enum.join(", ") + + defp parse_bool(var) do + case String.downcase(var) do + t when t in @var_true -> + true + + f when f in @var_false -> + false + + _ -> + raise ArgumentError, + "Invalid boolean value: #{inspect(var)}. Expected one of: " <> @var_bool_message + end + end +end diff --git a/lib/plausible/helpers/json.ex b/lib/plausible/helpers/json.ex new file mode 100644 index 0000000000000000000000000000000000000000..098522c2870cee634c82e7e87f9f62ac2fd39cfd --- /dev/null +++ b/lib/plausible/helpers/json.ex @@ -0,0 +1,15 @@ +defmodule Plausible.Helpers.JSON do + @moduledoc """ + Common helpers for JSON handling + """ + + def decode_or_fallback(raw) do + with raw when is_binary(raw) <- raw, + {:ok, %{} = decoded} <- Jason.decode(raw) do + decoded + else + already_a_map when is_map(already_a_map) -> already_a_map + _any -> %{} + end + end +end diff --git a/lib/plausible/http_client.ex b/lib/plausible/http_client.ex new file mode 100644 index 0000000000000000000000000000000000000000..f6c17efa6ad7fea3e56464155b6b3c6f4d4e4cc6 --- /dev/null +++ b/lib/plausible/http_client.ex @@ -0,0 +1,189 @@ +defmodule Plausible.HTTPClient.Non200Error do + @moduledoc false + + defstruct reason: nil + + @type t :: %__MODULE__{reason: Finch.Response.t()} + + @spec new(Finch.Response.t()) :: t() + def new(%Finch.Response{status: status} = response) + when is_integer(status) and (status < 200 or status >= 300) do + %__MODULE__{reason: response} + end +end + +defmodule Plausible.HTTPClient.Interface do + @moduledoc false + + @type t() :: module() + + @type finch_request_opts() :: Keyword.t() + @type url() :: Finch.Request.url() + @type headers() :: Finch.Request.headers() + @type params() :: Finch.Request.body() | map() + @type response() :: + {:ok, Finch.Response.t()} + | {:error, Finch.error() | Plausible.HTTPClient.Non200Error.t()} + + @callback get(url(), headers(), params()) :: response() + @callback get(url(), headers()) :: response() + @callback get(url()) :: response() + @callback post(url(), headers(), params()) :: response() + @callback post(url(), headers(), params(), finch_request_opts()) :: response() +end + +defmodule Plausible.HTTPClient do + @moduledoc """ + HTTP Client built on top of Finch. + + By default, request parameters are json-encoded. + + If a raw binary value is supplied, no encoding is performed. + If x-www-form-urlencoded content-type is set in headers, + URL encoding is invoked. + """ + + require OpenTelemetry.Tracer + alias Plausible.HTTPClient.Non200Error + + @doc """ + Make a POST request + """ + @behaviour Plausible.HTTPClient.Interface + + @impl Plausible.HTTPClient.Interface + def post(url, headers \\ [], params \\ nil, finch_req_opts \\ []) do + call(:post, url, headers, params, finch_req_opts) + end + + @doc """ + Make a GET request + """ + @impl Plausible.HTTPClient.Interface + def get(url, headers \\ [], params \\ nil) do + call(:get, url, headers, params) + end + + @spec impl() :: Plausible.HTTPClient.Interface.t() + def impl() do + Application.get_env(:plausible, :http_impl, __MODULE__) + end + + defp call(method, url, headers, params, finch_req_opts \\ []) do + OpenTelemetry.Tracer.with_span "http_client.request" do + {params, headers} = maybe_encode_params(params, headers) + + method + |> build_request(url, headers, params) + |> trace_request() + |> do_request(finch_req_opts) + |> maybe_decode_body() + |> tag_error() + |> trace_response() + end + end + + defp build_request(method, url, headers, params) do + Finch.build(method, url, headers, params) + end + + defp do_request(request, finch_req_opts) do + Finch.request(request, Plausible.Finch, finch_req_opts) + end + + defp maybe_encode_params(params, headers) when is_binary(params) or is_nil(params) do + {params, headers} + end + + defp maybe_encode_params(params, headers) when is_map(params) do + content_type = + Enum.find_value(headers, "", fn {k, v} -> + if String.downcase(k) == "content-type" do + v + end + end) + + case String.downcase(content_type) do + "application/x-www-form-urlencoded" -> + {URI.encode_query(params), headers} + + "application/json" -> + {Jason.encode!(params), headers} + + _ -> + {Jason.encode!(params), [{"content-type", "application/json"} | headers]} + end + end + + defp tag_error({:ok, %Finch.Response{status: status}} = ok) + when is_integer(status) and status >= 200 and status < 300 do + ok + end + + defp tag_error({:ok, %Finch.Response{status: _} = response}) do + {:error, Non200Error.new(response)} + end + + defp tag_error({:error, _} = error) do + error + end + + defp maybe_decode_body({:ok, %{headers: headers, body: body} = resp}) + when is_binary(body) and body != "" do + if json?(headers) do + {:ok, update_in(resp.body, &Jason.decode!/1)} + else + {:ok, resp} + end + end + + defp maybe_decode_body(response), do: response + + defp json?(headers) do + found = + Enum.find(headers, fn + {"content-type", "application/json" <> _} -> + true + + _ -> + false + end) + + is_tuple(found) + end + + defp trace_request(%Finch.Request{} = request) do + OpenTelemetry.Tracer.set_attributes([ + {"http_client.request.host", request.host}, + {"http_client.request.method", request.method}, + {"http_client.request.path", request.path} + ]) + + request + end + + defp trace_response(response) do + case response do + {:ok, %{status: status}} -> + OpenTelemetry.Tracer.set_attributes([ + {"http_client.response.status", status} + ]) + + {:error, %Non200Error{reason: %{status: status}}} -> + OpenTelemetry.Tracer.set_attributes([ + {"http_client.request.failure", "non_200"}, + {"http_client.response.status", status} + ]) + + {:error, exception} when is_exception(exception) -> + OpenTelemetry.Tracer.set_attributes([ + {"http_client.request.failure", Exception.message(exception)} + ]) + + _any -> + :skip + end + + response + end +end diff --git a/lib/plausible/import_deletion_repo.ex b/lib/plausible/import_deletion_repo.ex new file mode 100644 index 0000000000000000000000000000000000000000..3c8a012d8a24368e86872cb315e2155ccec39625 --- /dev/null +++ b/lib/plausible/import_deletion_repo.ex @@ -0,0 +1,17 @@ +defmodule Plausible.ImportDeletionRepo do + @moduledoc """ + A dedicated repo for import related mutations + """ + + use Ecto.Repo, + otp_app: :plausible, + adapter: Ecto.Adapters.ClickHouse + + defmacro __using__(_) do + quote do + alias Plausible.ImportDeletionRepo + import Ecto + import Ecto.Query, only: [from: 1, from: 2] + end + end +end diff --git a/lib/plausible/imported.ex b/lib/plausible/imported.ex new file mode 100644 index 0000000000000000000000000000000000000000..0c6f0eab7786440ac6ae7b55c4cdafaf599d8e58 --- /dev/null +++ b/lib/plausible/imported.ex @@ -0,0 +1,212 @@ +defmodule Plausible.Imported do + @moduledoc """ + Context for managing site statistics imports. + + For list of currently supported import sources see `Plausible.Imported.ImportSources`. + + For more information on implementing importers, see `Plausible.Imported.Importer`. + """ + + import Ecto.Query + + alias Plausible.{Site, Repo, Imported} + alias Plausible.Imported.SiteImport + alias Plausible.Stats.Query + + require Plausible.Imported.SiteImport + + @tables [ + Imported.Visitor, + Imported.Source, + Imported.Page, + Imported.EntryPage, + Imported.ExitPage, + Imported.CustomEvent, + Imported.Location, + Imported.Device, + Imported.Browser, + Imported.OperatingSystem + ] + + @table_names Enum.map(@tables, & &1.__schema__(:source)) + # Maximum number of complete imports to account for when querying stats + @max_complete_imports 5 + + @spec schemas() :: [module()] + def schemas, do: @tables + + @spec tables() :: [String.t()] + def tables, do: @table_names + + @spec max_complete_imports() :: non_neg_integer() + def max_complete_imports() do + @max_complete_imports + end + + @spec imported_custom_props() :: [String.t()] + def imported_custom_props do + # NOTE: Keep up to date with `Plausible.Props.internal_keys/1`, + # but _ignore_ unsupported keys. Currently, `search_query` is + # not supported in imported queries. + Enum.map(~w(url path), &("event:props:" <> &1)) + end + + @spec any_completed_imports?(Site.t()) :: boolean() + def any_completed_imports?(site) do + get_completed_imports(site) != [] + end + + @spec earliest_import_start_date(Site.t()) :: Date.t() | nil + def earliest_import_start_date(site) do + site + |> get_completed_imports() + |> Enum.map(& &1.start_date) + |> Enum.min(Date, fn -> nil end) + end + + @spec complete_import_ids(Site.t()) :: [non_neg_integer()] + def complete_import_ids(site) do + imports = get_completed_imports(site) + has_legacy? = Enum.any?(imports, fn %{legacy: legacy?} -> legacy? end) + ids = Enum.map(imports, fn %{id: id} -> id end) + + # account for legacy imports as well + if has_legacy? do + [0 | ids] + else + ids + end + end + + @spec completed_imports_in_query_range(Site.t(), Query.t()) :: [SiteImport.t()] + def completed_imports_in_query_range(%Site{} = site, %Query{} = query) do + date_range = Query.date_range(query) + + site + |> get_completed_imports() + |> Enum.reject(fn site_import -> + Date.after?(site_import.start_date, date_range.last) or + Date.before?(site_import.end_date, date_range.first) + end) + end + + @spec get_import(Site.t(), non_neg_integer()) :: SiteImport.t() | nil + def get_import(site, import_id) do + Repo.get_by(SiteImport, id: import_id, site_id: site.id) + end + + defdelegate listen(), to: Imported.Importer + + @spec list_all_imports(Site.t(), atom()) :: [SiteImport.t()] + def list_all_imports(site, status \\ nil) do + from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at]) + |> maybe_filter_by_status(status) + |> Repo.all() + end + + @spec other_imports_in_progress?(SiteImport.t()) :: boolean() + def other_imports_in_progress?(site_import) do + Repo.exists?( + from(i in SiteImport, + where: i.site_id == ^site_import.site_id and i.id != ^site_import.id, + where: i.status in ^[SiteImport.pending(), SiteImport.importing()] + ) + ) + end + + defp maybe_filter_by_status(query, nil), do: query + + defp maybe_filter_by_status(query, status) do + where(query, [i], i.status == ^status) + end + + @spec delete_imports_for_site(Site.t()) :: :ok + def delete_imports_for_site(site) do + Repo.delete_all(from i in SiteImport, where: i.site_id == ^site.id) + + :ok + end + + @spec clamp_dates(Site.t(), Date.t(), Date.t()) :: + {:ok, Date.t(), Date.t()} | {:error, :no_time_window} + def clamp_dates(site, start_date, end_date) do + cutoff_date = get_cutoff_date(site) + occupied_ranges = get_occupied_date_ranges(site) + + clamp_dates(occupied_ranges, cutoff_date, start_date, end_date) + end + + @spec clamp_dates([Date.Range.t()], Date.t(), Date.t(), Date.t()) :: + {:ok, Date.t(), Date.t()} | {:error, :no_time_window} + def clamp_dates(occupied_ranges, cutoff_date, start_date, end_date) do + end_date = Enum.min([end_date, cutoff_date], Date) + + with true <- Date.diff(end_date, start_date) >= 2, + [_ | _] = free_ranges <- find_free_ranges(start_date, end_date, occupied_ranges) do + longest = Enum.max_by(free_ranges, &Date.diff(&1.last, &1.first)) + {:ok, longest.first, longest.last} + else + _ -> {:error, :no_time_window} + end + end + + @spec get_occupied_date_ranges(Site.t()) :: [Date.Range.t()] + def get_occupied_date_ranges(site) do + site + |> Imported.list_all_imports(Imported.SiteImport.completed()) + |> Enum.reject(&(Date.diff(&1.end_date, &1.start_date) < 2)) + |> Enum.map(&Date.range(&1.start_date, &1.end_date)) + |> Enum.sort_by(& &1.first, Date) + end + + @spec get_cutoff_date(Site.t()) :: Date.t() + def get_cutoff_date(site) do + Plausible.Sites.native_stats_start_date(site) || + DateTime.to_date(DateTime.now!(site.timezone)) + end + + defp find_free_ranges(start_date, end_date, occupied_ranges) do + Date.range(start_date, end_date) + |> free_ranges(start_date, occupied_ranges, []) + end + + # This function recursively finds open ranges that are not yet occupied + # by existing imported data. The idea is that we keep moving a dynamic + # date index `d` from start until the end of `imported_range`, hopping + # over each occupied range, and capturing the open ranges step-by-step + # in the `result` array. + defp free_ranges(import_range, d, [occupied_range | rest_of_occupied_ranges], result) do + cond do + Date.diff(occupied_range.last, d) <= 0 -> + free_ranges(import_range, d, rest_of_occupied_ranges, result) + + in_range?(d, occupied_range) || Date.diff(occupied_range.first, d) < 2 -> + d = occupied_range.last + free_ranges(import_range, d, rest_of_occupied_ranges, result) + + true -> + free_range = Date.range(d, occupied_range.first) + result = result ++ [free_range] + d = occupied_range.last + free_ranges(import_range, d, rest_of_occupied_ranges, result) + end + end + + defp free_ranges(import_range, d, [], result) do + if Date.diff(import_range.last, d) < 2 do + result + else + result ++ [Date.range(d, import_range.last)] + end + end + + defp in_range?(date, range) do + Date.before?(range.first, date) && Date.after?(range.last, date) + end + + defp get_completed_imports(site) do + site + |> Repo.preload(:completed_imports) + |> Map.fetch!(:completed_imports) + end +end diff --git a/lib/plausible/imported/browser.ex b/lib/plausible/imported/browser.ex new file mode 100644 index 0000000000000000000000000000000000000000..bf39470e51ea3653a3b1bb08bd0064948b95b7a7 --- /dev/null +++ b/lib/plausible/imported/browser.ex @@ -0,0 +1,18 @@ +defmodule Plausible.Imported.Browser do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_browsers" do + field :site_id, Ch, type: "UInt64" + field :import_id, Ch, type: "UInt64" + field :date, :date + field :browser, :string + field :browser_version, :string + field :visitors, Ch, type: "UInt64" + field :visits, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" + field :pageviews, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt32" + end +end diff --git a/lib/plausible/imported/buffer.ex b/lib/plausible/imported/buffer.ex new file mode 100644 index 0000000000000000000000000000000000000000..1945a6081236855a4b415f4c0881db133d59be98 --- /dev/null +++ b/lib/plausible/imported/buffer.ex @@ -0,0 +1,115 @@ +defmodule Plausible.Imported.Buffer do + @moduledoc """ + This GenServer inserts records into Clickhouse `imported_*` tables. Multiple buffers are + automatically created for each table. Records are flushed when the table buffer reaches the + maximum size, defined by `max_buffer_size/0`. + """ + + use GenServer + require Logger + + def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, opts) + end + + def init(opts) do + flush_interval = Keyword.get(opts, :flush_interval_ms, 1000) + {:ok, %{flush_interval: flush_interval, buffers: %{}}} + end + + @spec insert_many(pid(), term(), [map()]) :: :ok + @doc """ + Puts the given records into the table buffer. + """ + def insert_many(pid, table_name, records) do + GenServer.call(pid, {:insert_many, table_name, records}) + end + + @spec size(pid(), term()) :: non_neg_integer() + @doc """ + Returns the total count of items in the given table buffer. + """ + def size(pid, table_name) do + GenServer.call(pid, {:get_size, table_name}) + end + + @spec flush(pid()) :: :ok + @doc """ + Flushes all table buffers to Clickhouse. + """ + def flush(pid, timeout \\ :infinity) do + GenServer.call(pid, :flush_all_buffers, timeout) + end + + def stop(pid) do + GenServer.stop(pid) + end + + def handle_call({:get_size, table_name}, _from, %{buffers: buffers} = state) do + size = + buffers + |> Map.get(table_name, []) + |> length() + + {:reply, size, state} + end + + def handle_call({:insert_many, table_name, records}, _from, %{buffers: buffers} = state) do + Logger.notice("Import: Adding #{length(records)} to #{table_name} buffer") + + new_buffer = Map.get(buffers, table_name, []) ++ records + new_state = put_in(state.buffers[table_name], new_buffer) + + if length(new_buffer) >= max_buffer_size() do + {:reply, :ok, new_state, {:continue, {:flush, table_name}}} + else + {:reply, :ok, new_state} + end + end + + def handle_call(:flush_all_buffers, _from, state) do + Enum.each(state.buffers, fn {table_name, records} -> + flush_buffer(records, table_name, state.flush_interval) + end) + + {:reply, :ok, put_in(state.buffers, %{})} + end + + def handle_continue({:flush, table_name}, state) do + flush_buffer(state.buffers[table_name], table_name, state.flush_interval) + {:noreply, put_in(state.buffers[table_name], [])} + end + + defp max_buffer_size do + :plausible + |> Application.fetch_env!(:imported) + |> Keyword.fetch!(:max_buffer_size) + end + + defp flush_buffer(records, table_name, flush_interval) do + # Clickhouse does not recommend sending more than 1 INSERT operation per second, and this + # sleep call slows down the flushing + Process.sleep(flush_interval) + + Logger.notice("Import: Flushing #{length(records)} from #{table_name} buffer") + insert_all(table_name, records) + end + + # used in tests `setup` + @doc false + def insert_all(table_name, records) do + schema = table_schema(table_name) + Plausible.IngestRepo.insert_all(schema, records) + end + + defp table_schema("imported_visitors"), do: Plausible.Imported.Visitor + defp table_schema("imported_sources"), do: Plausible.Imported.Source + defp table_schema("imported_pages"), do: Plausible.Imported.Page + defp table_schema("imported_entry_pages"), do: Plausible.Imported.EntryPage + defp table_schema("imported_exit_pages"), do: Plausible.Imported.ExitPage + defp table_schema("imported_custom_events"), do: Plausible.Imported.CustomEvent + defp table_schema("imported_locations"), do: Plausible.Imported.Location + defp table_schema("imported_devices"), do: Plausible.Imported.Device + defp table_schema("imported_browsers"), do: Plausible.Imported.Browser + defp table_schema("imported_operating_systems"), do: Plausible.Imported.OperatingSystem +end diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex new file mode 100644 index 0000000000000000000000000000000000000000..ab51268b8bae561d6c2a45bc9778583432feaedc --- /dev/null +++ b/lib/plausible/imported/csv_importer.ex @@ -0,0 +1,372 @@ +defmodule Plausible.Imported.CSVImporter do + @moduledoc """ + CSV importer from either S3 for which it uses ClickHouse [s3 table function](https://clickhouse.com/docs/en/sql-reference/table-functions/s3) + or from local storage for which it uses [input function.](https://clickhouse.com/docs/en/sql-reference/table-functions/input) + """ + + use Plausible.Imported.Importer + import Ecto.Query, only: [from: 2] + + @impl true + def name(), do: :csv + + @impl true + def label(), do: "CSV" + + @impl true + def email_template(), do: "csv_import.html" + + @impl true + def parse_args(%{"uploads" => uploads, "storage" => storage}) do + uploads = Enum.reject(uploads, &String.starts_with?(&1["filename"], "imported_custom_props_")) + + [uploads: uploads, storage: storage] + end + + @impl true + def import_data(site_import, opts) do + storage = Keyword.fetch!(opts, :storage) + uploads = Keyword.fetch!(opts, :uploads) + + if storage == "local" do + # we need to remove the imported files from local storage + # after the importer has completed or ran out of attempts + paths = Enum.map(uploads, &Map.fetch!(&1, "local_path")) + + Oban.insert!( + Plausible.Workers.LocalImportAnalyticsCleaner.new( + %{"import_id" => site_import.id, "paths" => paths}, + schedule_in: _one_hour = 3600 + ) + ) + end + + {:ok, ch} = + Plausible.IngestRepo.config() + |> Keyword.replace!(:pool_size, 1) + |> Ch.start_link() + + case storage do + "s3" -> import_s3(ch, site_import, uploads) + "local" -> import_local(ch, site_import, uploads) + end + rescue + # we are cancelling on any argument or ClickHouse errors, assuming they are permanent + e in [ArgumentError, Ch.Error] -> + # see Plausible.Imported.Importer for more details on transient vs permanent errors + {:error, Exception.message(e)} + end + + def on_success(site_import, _extra_data) do + has_scroll_depth? = + Plausible.ClickhouseRepo.exists?( + from(i in "imported_pages", + where: i.site_id == ^site_import.site_id, + where: i.import_id == ^site_import.id, + where: i.total_scroll_depth_visits > 0, + select: 1 + ) + ) + + if has_scroll_depth? do + site_import + |> Ecto.Changeset.change(%{has_scroll_depth: true}) + |> Plausible.Repo.update!() + end + + :ok + end + + defp import_s3(ch, site_import, uploads) do + %{ + id: import_id, + site_id: site_id, + start_date: start_date, + end_date: end_date + } = site_import + + %{access_key_id: s3_access_key_id, secret_access_key: s3_secret_access_key} = + Plausible.S3.import_clickhouse_credentials() + + Enum.each(uploads, fn upload -> + %{"filename" => filename, "s3_url" => s3_url} = upload + + {table, _, _} = parse_filename!(filename) + s3_structure = input_structure!(table) + s3_columns = input_columns!(table) + + statement = + """ + INSERT INTO {table:Identifier}(site_id,import_id,#{s3_columns}) \ + SELECT {site_id:UInt64}, {import_id:UInt64}, #{s3_columns} \ + FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String}) \ + WHERE date >= {start_date:Date} AND date <= {end_date:Date}\ + """ + + params = + %{ + "table" => table, + "site_id" => site_id, + "import_id" => import_id, + "s3_url" => s3_url, + "s3_access_key_id" => s3_access_key_id, + "s3_secret_access_key" => s3_secret_access_key, + "s3_format" => "CSVWithNames", + "s3_structure" => s3_structure, + "start_date" => start_date, + "end_date" => end_date + } + + Ch.query!(ch, statement, params, timeout: :infinity) + end) + end + + defp import_local(ch, site_import, uploads) do + %{ + id: import_id, + site_id: site_id, + start_date: start_date, + end_date: end_date + } = site_import + + DBConnection.run( + ch, + fn conn -> + Enum.each(uploads, fn upload -> + %{"filename" => filename, "local_path" => local_path} = upload + + {table, _, _} = parse_filename!(filename) + input_structure = input_structure!(table) + input_columns = input_columns!(table) + + statement = + """ + INSERT INTO {table:Identifier}(site_id,import_id,#{input_columns}) \ + SELECT {site_id:UInt64}, {import_id:UInt64}, #{input_columns} \ + FROM input({input_structure:String}) \ + WHERE date >= {start_date:Date} AND date <= {end_date:Date} \ + FORMAT CSVWithNames + """ + + params = %{ + "table" => table, + "site_id" => site_id, + "import_id" => import_id, + "input_structure" => input_structure, + "start_date" => start_date, + "end_date" => end_date + } + + # we are reading in 512KB chunks for better performance + # the default would've been line by line (not great for a CSV) + File.stream!(local_path, 512_000) + |> Stream.into(Ch.stream(conn, statement, params)) + |> Stream.run() + end) + end, + timeout: :infinity + ) + end + + input_structures = %{ + "imported_browsers" => + "date Date, browser String, browser_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", + "imported_devices" => + "date Date, device String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", + "imported_entry_pages" => + "date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", + "imported_exit_pages" => + "date Date, exit_page String, visitors UInt64, visit_duration UInt64, exits UInt64, bounces UInt32, pageviews UInt64", + "imported_custom_events" => + "date Date, name String, link_url String, path String, visitors UInt64, events UInt64", + "imported_locations" => + "date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", + "imported_operating_systems" => + "date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64", + "imported_pages" => + "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, total_scroll_depth UInt64, total_scroll_depth_visits UInt64, total_time_on_page UInt64, total_time_on_page_visits UInt64", + "imported_sources" => + "date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32", + "imported_visitors" => + "date Date, visitors UInt64, pageviews UInt64, bounces UInt64, visits UInt64, visit_duration UInt64" + } + + @doc """ + Extracts min/max date range from a list of uploads. + + Examples: + + iex> date_range([ + ...> %{"filename" => "imported_devices_20190101_20210101.csv"}, + ...> "pages_20200101_20220101.csv" + ...> ]) + Date.range(~D[2019-01-01], ~D[2022-01-01]) + + iex> date_range([]) + nil + + """ + @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t() | nil + def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil) + def date_range([]), do: nil + + defp date_range( + [%{"filename" => "imported_custom_props_" <> _} | uploads], + prev_start_date, + prev_end_date + ) do + date_range(uploads, prev_start_date, prev_end_date) + end + + defp date_range([upload | uploads], prev_start_date, prev_end_date) do + filename = + case upload do + %{"filename" => filename} -> filename + filename when is_binary(filename) -> filename + end + + {_table, start_date, end_date} = parse_filename!(filename) + + start_date = + if prev_start_date do + Enum.min([start_date, prev_start_date], Date) + else + start_date + end + + end_date = + if prev_end_date do + Enum.max([end_date, prev_end_date], Date) + else + end_date + end + + date_range(uploads, start_date, end_date) + end + + defp date_range([], first, last), do: Date.range(first, last) + + @spec parse_date!(String.t()) :: Date.t() + defp parse_date!(<>) do + Date.from_iso8601!("#{yyyy}-#{mm}-#{dd}") + end + + @doc """ + Extracts table name and min/max dates from the filename. + + Examples: + + iex> parse_filename!("my_data.csv") + ** (ArgumentError) invalid filename + + iex> parse_filename!("imported_devices_00010101_20250101.csv") + {"imported_devices", ~D[0001-01-01], ~D[2025-01-01]} + + iex> parse_filename!("devices_00010101_20250101.csv") + {"imported_devices", ~D[0001-01-01], ~D[2025-01-01]} + + """ + @spec parse_filename!(String.t()) :: + {table :: String.t(), start_date :: Date.t(), end_date :: Date.t()} + def parse_filename!(filename) + + for {table, input_structure} <- input_structures do + defp input_structure!(unquote(table)), do: unquote(input_structure) + + input_columns = + input_structure + |> String.split(",", trim: true) + |> Enum.map_join(",", fn kv -> + [col, _type] = String.split(kv) + String.trim(col) + end) + + defp input_columns!(unquote(table)), do: unquote(input_columns) + + def parse_filename!( + <> + ) do + {unquote(table), parse_date!(start_date), parse_date!(end_date)} + end + + "imported_" <> name = table + + def parse_filename!( + <> + ) do + {unquote(table), parse_date!(start_date), parse_date!(end_date)} + end + end + + def parse_filename!(_filename) do + raise ArgumentError, "invalid filename" + end + + @doc """ + Checks if the provided filename conforms to the expected format. + + Examples: + + iex> valid_filename?("my_data.csv") + false + + iex> valid_filename?("imported_devices_00010101_20250101.csv") + true + + iex> valid_filename?("devices_00010101_20250101.csv") + true + + """ + @spec valid_filename?(String.t()) :: boolean + def valid_filename?(filename) do + try do + parse_filename!(filename) + else + _ -> true + rescue + _ -> false + end + end + + @doc """ + Extracts the table name from the provided filename. + + Raises if the filename doesn't conform to the expected format. + + Examples: + + iex> extract_table("my_data.csv") + ** (ArgumentError) invalid filename + + iex> extract_table("imported_devices_00010101_20250101.csv") + "imported_devices" + + iex> extract_table("devices_00010101_20250101.csv") + "imported_devices" + + """ + @spec extract_table(String.t()) :: String.t() + def extract_table(filename) do + {table, _start_date, _end_date} = parse_filename!(filename) + table + end + + @doc """ + Returns local directory for CSV imports storage. + + Builds upon `$DATA_DIR`, `$PERSISTENT_CACHE_DIR` or `$DEFAULT_DATA_DIR` (if set) and falls back to /tmp. + + `$DEFAULT_DATA_DIR` is set to `/var/lib/plausible` in container images. + + Examples: + + iex> local_dir = local_dir(_site_id = 37) + iex> String.ends_with?(local_dir, "/plausible-imports/37") + true + + """ + def local_dir(site_id) do + data_dir = Application.get_env(:plausible, :data_dir) + Path.join([data_dir || System.tmp_dir!(), "plausible-imports", Integer.to_string(site_id)]) + end +end diff --git a/lib/plausible/imported/custom_event.ex b/lib/plausible/imported/custom_event.ex new file mode 100644 index 0000000000000000000000000000000000000000..5b31227b8053d32b07aad47fd4723d082fde0332 --- /dev/null +++ b/lib/plausible/imported/custom_event.ex @@ -0,0 +1,16 @@ +defmodule Plausible.Imported.CustomEvent do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_custom_events" do + field :site_id, Ch, type: "UInt64" + field :import_id, Ch, type: "UInt64" + field :date, :date + field :name, :string + field :link_url, :string + field :path, :string + field :visitors, Ch, type: "UInt64" + field :events, Ch, type: "UInt64" + end +end diff --git a/lib/plausible/imported/device.ex b/lib/plausible/imported/device.ex new file mode 100644 index 0000000000000000000000000000000000000000..c07e058e48f7c0636be32174a1fb605c7b6fd09d --- /dev/null +++ b/lib/plausible/imported/device.ex @@ -0,0 +1,17 @@ +defmodule Plausible.Imported.Device do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_devices" do + field :site_id, Ch, type: "UInt64" + field :import_id, Ch, type: "UInt64" + field :date, :date + field :device, :string + field :visitors, Ch, type: "UInt64" + field :visits, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" + field :pageviews, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt32" + end +end diff --git a/lib/plausible/imported/entry_page.ex b/lib/plausible/imported/entry_page.ex new file mode 100644 index 0000000000000000000000000000000000000000..294e9b05756d99ce7ea40e3392fa570dc661ea50 --- /dev/null +++ b/lib/plausible/imported/entry_page.ex @@ -0,0 +1,17 @@ +defmodule Plausible.Imported.EntryPage do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_entry_pages" do + field :site_id, Ch, type: "UInt64" + field :import_id, Ch, type: "UInt64" + field :date, :date + field :entry_page, :string + field :visitors, Ch, type: "UInt64" + field :entrances, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" + field :pageviews, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt32" + end +end diff --git a/lib/plausible/imported/exit_page.ex b/lib/plausible/imported/exit_page.ex new file mode 100644 index 0000000000000000000000000000000000000000..a057fe721f2b13edf52bd2043880e519d0a4ac03 --- /dev/null +++ b/lib/plausible/imported/exit_page.ex @@ -0,0 +1,17 @@ +defmodule Plausible.Imported.ExitPage do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_exit_pages" do + field :site_id, Ch, type: "UInt64" + field :import_id, Ch, type: "UInt64" + field :date, :date + field :exit_page, :string + field :exits, Ch, type: "UInt64" + field :visitors, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" + field :pageviews, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt32" + end +end diff --git a/lib/plausible/imported/google_analytics4.ex b/lib/plausible/imported/google_analytics4.ex new file mode 100644 index 0000000000000000000000000000000000000000..2ecb7cb40f97f5af3e754620f5c61e7d233a0e89 --- /dev/null +++ b/lib/plausible/imported/google_analytics4.ex @@ -0,0 +1,357 @@ +defmodule Plausible.Imported.GoogleAnalytics4 do + @moduledoc """ + Import implementation for Google Analytics 4. + """ + + use Plausible.Imported.Importer + + alias Plausible.Imported + alias Plausible.Repo + + @recoverable_errors [:rate_limit_exceeded, :socket_failed, :server_failed] + @missing_values ["(none)", "(not set)", "(not provided)", "(other)"] + + @impl true + def name(), do: :google_analytics_4 + + @impl true + def label(), do: "Google Analytics 4" + + @impl true + def email_template(), do: "google_analytics_import.html" + + @impl true + def before_start(site_import, opts) do + site_import = Repo.preload(site_import, :site) + + if import_id = Keyword.get(opts, :resume_from_import_id) do + if existing_site_import = Imported.get_import(site_import.site, import_id) do + Repo.delete!(site_import) + {:ok, existing_site_import} + else + # NOTE: shouldn't happen under normal circumsatnces + {:error, {:no_import_to_resume, import_id}} + end + else + {:ok, site_import} + end + end + + @impl true + def parse_args(%{"resume_from_dataset" => dataset, "resume_from_offset" => offset} = args) do + args + |> Map.drop(["resume_from_dataset", "resume_from_offset"]) + |> parse_args() + |> Keyword.put(:dataset, dataset) + |> Keyword.put(:offset, offset) + end + + def parse_args( + %{"property" => property, "start_date" => start_date, "end_date" => end_date} = args + ) do + start_date = Date.from_iso8601!(start_date) + end_date = Date.from_iso8601!(end_date) + date_range = Date.range(start_date, end_date) + + auth = { + Map.fetch!(args, "access_token"), + Map.fetch!(args, "refresh_token"), + Map.fetch!(args, "token_expires_at") + } + + [ + property: property, + date_range: date_range, + auth: auth + ] + end + + @doc """ + Imports stats from a Google Analytics 4 property to a Plausible site. + + This function fetches Google Analytics 4 reports which are then passed in batches + to Clickhouse by the `Plausible.Imported.Buffer` process. + """ + @impl true + def import_data(site_import, opts) do + date_range = Keyword.fetch!(opts, :date_range) + property = Keyword.fetch!(opts, :property) + auth = Keyword.fetch!(opts, :auth) + flush_interval_ms = Keyword.get(opts, :flush_interval_ms, 1000) + + {:ok, buffer} = Plausible.Imported.Buffer.start_link(flush_interval_ms: flush_interval_ms) + + persist_fn = fn table, rows -> + records = from_report(rows, site_import.site_id, site_import.id, table) + Plausible.Imported.Buffer.insert_many(buffer, table, records) + end + + resume_opts = Keyword.take(opts, [:dataset, :offset]) + fetch_opts = Keyword.get(opts, :fetch_opts, []) + + try do + result = + Plausible.Google.GA4.API.import_analytics(date_range, property, auth, + persist_fn: persist_fn, + fetch_opts: fetch_opts, + resume_opts: resume_opts + ) + + case result do + {:error, {error, details}} when error in @recoverable_errors -> + site_import = Repo.preload(site_import, [:site, :imported_by]) + dataset = Keyword.fetch!(details, :dataset) + offset = Keyword.fetch!(details, :offset) + {access_token, refresh_token, token_expires_at} = auth + + resume_import_opts = [ + property: property, + label: property, + start_date: date_range.first, + end_date: date_range.last, + access_token: access_token, + refresh_token: refresh_token, + token_expires_at: token_expires_at, + resume_from_import_id: site_import.id, + resume_from_dataset: dataset, + resume_from_offset: offset, + job_opts: [schedule_in: {65, :minutes}, unique: nil] + ] + + new_import( + site_import.site, + site_import.imported_by, + resume_import_opts + ) + + {:error, error, skip_purge?: true, skip_mark_failed?: true} + + other -> + other + end + after + Plausible.Imported.Buffer.flush(buffer) + Plausible.Imported.Buffer.stop(buffer) + end + end + + def from_report(nil, _site_id, _import_id, _metric), do: nil + + def from_report(data, site_id, import_id, table) do + Enum.reduce(data, [], fn row, acc -> + if Map.get(row.dimensions, "date") in @missing_values do + acc + else + [new_from_report(site_id, import_id, table, row) | acc] + end + end) + end + + defp parse_number(nr) do + {float, ""} = Float.parse(nr) + round(float) + end + + defp maybe_override_event_name("file_download"), do: "File Download" + defp maybe_override_event_name("click"), do: "Outbound Link: Click" + defp maybe_override_event_name(name), do: name + + defp new_from_report(site_id, import_id, "imported_visitors", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + defp new_from_report(site_id, import_id, "imported_sources", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + source: row.dimensions |> Map.fetch!("sessionSource") |> parse_source(), + # GA4 channels map 1-1 to Plausible channels + channel: row.dimensions |> Map.fetch!("sessionDefaultChannelGroup"), + referrer: nil, + # Only `source` exists in GA4 API + utm_source: nil, + utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(), + utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(), + utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(), + utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + defp new_from_report(site_id, import_id, "imported_pages", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""), + page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), + # NOTE: no exits metric in GA4 API currently + exits: 0, + total_time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(), + total_time_on_page_visits: row.metrics |> Map.fetch!("sessions") |> parse_number() + } + end + + defp new_from_report(site_id, import_id, "imported_entry_pages", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + entry_page: row.dimensions |> Map.fetch!("landingPage"), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(), + pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number() + } + end + + # NOTE: no exit pages metrics in GA4 API available for now + # defp new_from_report(site_id, import_id, "imported_exit_pages", row) do + # %{ + # site_id: site_id, + # import_id: import_id, + # date: get_date(row), + # exit_page: Map.fetch!(row.dimensions, "exitPage"), + # visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + # exits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + # visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(), + # pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), + # bounces: row.metrics |> Map.fetch!("bounces") |> parse_number() + # } + # end + + defp new_from_report(site_id, import_id, "imported_custom_events", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + name: row.dimensions |> Map.fetch!("eventName") |> maybe_override_event_name(), + link_url: row.dimensions |> Map.fetch!("linkUrl"), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + events: row.metrics |> Map.fetch!("eventCount") |> parse_number() + } + end + + defp new_from_report(site_id, import_id, "imported_locations", row) do + country_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("") + city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("") + city_data = Location.get_city(city_name, country_code) + + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + country: country_code, + region: row.dimensions |> Map.fetch!("region") |> default_if_missing(""), + city: city_data && city_data.id, + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + defp new_from_report(site_id, import_id, "imported_devices", row) do + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + @browser_google_to_plausible %{ + "User-Agent:Opera" => "Opera", + "Mozilla Compatible Agent" => "Mobile App", + "Android Webview" => "Mobile App", + "Android Browser" => "Mobile App", + "Safari (in-app)" => "Mobile App", + "User-Agent: Mozilla" => "Firefox", + "(not set)" => "" + } + + defp new_from_report(site_id, import_id, "imported_browsers", row) do + browser = Map.fetch!(row.dimensions, "browser") + + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + browser: Map.get(@browser_google_to_plausible, browser, browser), + # Does not exist in GA4 API + browser_version: nil, + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + @os_google_to_plausible %{ + "Macintosh" => "Mac", + "Linux" => "GNU/Linux", + "(not set)" => "" + } + + defp new_from_report(site_id, import_id, "imported_operating_systems", row) do + os = Map.fetch!(row.dimensions, "operatingSystem") + + %{ + site_id: site_id, + import_id: import_id, + date: get_date(row), + operating_system: Map.get(@os_google_to_plausible, os, os), + operating_system_version: row.dimensions |> Map.fetch!("operatingSystemVersion"), + visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(), + visits: row.metrics |> Map.fetch!("sessions") |> parse_number(), + pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(), + bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(), + visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number() + } + end + + defp get_date(%{dimensions: %{"date" => date}}) do + date + |> Plausible.Times.parse!("%Y%m%d", :strftime) + |> NaiveDateTime.to_date() + end + + defp default_if_missing(value, default \\ nil) + defp default_if_missing(value, default) when value in @missing_values, do: default + defp default_if_missing(value, _default), do: value + + defp parse_source(nil), do: nil + defp parse_source("(direct)"), do: nil + defp parse_source("google"), do: "Google" + defp parse_source("bing"), do: "Bing" + defp parse_source("duckduckgo"), do: "DuckDuckGo" + + defp parse_source(ref) do + Plausible.Ingestion.Source.from_referrer("https://" <> ref) + end +end diff --git a/lib/plausible/imported/import_sources.ex b/lib/plausible/imported/import_sources.ex new file mode 100644 index 0000000000000000000000000000000000000000..96a149b19ba50d3e76506ac2f358e5349641604c --- /dev/null +++ b/lib/plausible/imported/import_sources.ex @@ -0,0 +1,24 @@ +defmodule Plausible.Imported.ImportSources do + @moduledoc """ + Definitions of import sources. + """ + + @sources [ + Plausible.Imported.GoogleAnalytics4, + Plausible.Imported.UniversalAnalytics, + Plausible.Imported.NoopImporter, + Plausible.Imported.CSVImporter + ] + + @sources_map Map.new(@sources, &{&1.name(), &1}) + + @source_names Enum.map(@sources, & &1.name()) + + @spec names() :: [atom()] + def names(), do: @source_names + + @spec by_name(atom()) :: module() + def by_name(name) do + Map.fetch!(@sources_map, name) + end +end diff --git a/lib/plausible/imported/importer.ex b/lib/plausible/imported/importer.ex new file mode 100644 index 0000000000000000000000000000000000000000..f8a20e20850525a6f402e69044b896bcef636543 --- /dev/null +++ b/lib/plausible/imported/importer.ex @@ -0,0 +1,261 @@ +defmodule Plausible.Imported.Importer do + @moduledoc """ + Behaviour that should be implemented for each import source. + + All imports are executed as background jobs run via `Plausible.Workers.ImportAnalytics` + Oban worker. Each import source must define a module conforming `Importer` behaviour. + + The callbacks that need to be implemented: + + * `name/0` - Returns import source name as an atom. Example: `:universal_analytics`. + * `label/0` - Descriptive, display friendly name of the source. + Example: "Google Analytics". + * `email_template/0` - Name of the email template to use for notifications in + `PlausibleWeb.Email` (`import_success` and `import_failure`). The template + should have content customized for a particular source. + * `parse_args/1` - Receives Oban job arguments coming from `new_import/3`. Whatever + options were passed to `new_import/3` will be present in the input map with string + keys and values serialized to primitives. If, for instance `start_date: ~D[2024-01-03]` + is passed as an option, `parse_args/1` receives `%{..., "start_date" => "2024-01-03"}`. + The expectation is parsing the map values producing a keyword list of options to + pass to `import_data/2`. + * `import_data/2` - Receives site import struct and options produced by `parse_args/1`. + This is where all the import processing is done. The way the import is implemented + is entirely arbitrary except the requirement that the process as a whole must + by synchronous. The callback is expected to return either `:ok` or `{:ok, %{...}}` + on successful import or `{:error, ...}` on failure. The map in success tuple is + used for updating site import struct and is passed to `on_success/2` callback. + Please note that error tuple should be only returned on errors that can't be + recovered from. For transient errors, the import should throw an exception or + simply crash. The error tuple has an alternative `{error, reason, opts}` form, + where `opts` allow to skip purging imported data so far via `skip_purge?` flag + and skip marking the import as failed and notifying the user via `skip_mark_failed?` + flag. Both flags are booleans. + * `before_start/2` - Optional callback run right before scheduling import job. It's + expected to either return `{:ok, site_import}` for the import to proceed + or `{:error, ...}` tuple, which will be returned from `new_import/3` call. + The `site_import` can be altered or replaced at this stage. The second argument + are opts passed to `new_import/3`. + * `on_success/2` - Optional callback run once site import is completed. Receives map + returned from `import_data/2`. Expected to always return `:ok`. + * `on_failure/1` - Optional callback run when import job fails permanently. + + All sources must be added to the list in `Plausible.Imported.ImportSources`. + + In order to schedule a new import job using a given source, respective importer's + `new_import/3` function must be called. It accepts site, user who is doing the import + and any options necessary to carry out the import. + + There's an expectation that `start_date` and `end_date` are provided either as options + passed to `new_import/3` or data in map returned from `import_data/2`. If these parameters + are not provided, the import will eventually crash. These parameters define time range + of imported data which is in turn used for efficient querying. + + Logic running inside `import_data/2` is expected to populated all `imported_*` tables + in ClickHouse with `import_id` column set to site import's ID. + + Managing any configuration or authentication prior to running import is outside of + scope of importer logic and is expected to be implemented separately. + + ## Running import fully synchronously + + In case it's necessary to run the whole import job fully synchronously, the + `Plausible.Workers.ImportAnalytics` worker sends an `Oban.Notifier` message + on completion, failure or transient failure of the import. + + A basic usage scenario looks like this: + + ```elixir + {:ok, job} = Plausible.Imported.NoopImporter.new_import( + site, + user, + start_date: ~D[2005-01-01], + end_date: Date.utc_today(), + # this option is necessary to setup the calling process as listener + listen?: true + ) + + import_id = job.args[:import_id] + + receive do + {:notification, :analytics_imports_jobs, %{"event" => "complete", "import_id" => ^import_id}} -> + IO.puts("Job completed") + + {:notification, :analytics_imports_jobs, %{"event" => "transient_fail", "import_id" => ^import_id}} -> + IO.puts("Job failed transiently") + + {:notification, :analytics_imports_jobs, %{"event" => "fail", "import_id" => ^import_id}} -> + IO.puts("Job failed permanently") + after + 15_000 -> + IO.puts("Job didn't finish in 15 seconds") + end + ``` + + In a more realistic scenario, job scheduling will be done inside a GenServer process + like LiveView, where notifications can be listened for via `handle_info/2`. + """ + + alias Plausible.Imported + alias Plausible.Imported.SiteImport + alias Plausible.Repo + + @callback name() :: atom() + @callback label() :: String.t() + @callback email_template() :: String.t() + @callback parse_args(map()) :: Keyword.t() + @callback import_data(SiteImport.t(), Keyword.t()) :: + :ok | {:error, any()} | {:error, any(), Keyword.t()} + @callback before_start(SiteImport.t(), Keyword.t()) :: {:ok, SiteImport.t()} | {:error, any()} + @callback on_success(SiteImport.t(), map()) :: :ok + @callback on_failure(SiteImport.t()) :: :ok + + defmacro __using__(_opts) do + quote do + @behaviour Plausible.Imported.Importer + + @spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) :: + {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t() | :import_in_progress | any()} + def new_import(site, user, opts) do + Plausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/2) + end + + @doc false + @spec run_import(SiteImport.t(), map()) :: {:ok, SiteImport.t()} | {:error, :any} + def run_import(site_import, args) do + Plausible.Imported.Importer.run_import( + site_import, + args, + &parse_args/1, + &import_data/2, + &on_success/2 + ) + end + + @doc false + @spec mark_failed(SiteImport.t()) :: SiteImport.t() + def mark_failed(site_import) do + site_import = + site_import + |> SiteImport.fail_changeset() + |> Repo.update!() + + :ok = on_failure(site_import) + + site_import + end + + @impl true + def before_start(site_import, _opts), do: {:ok, site_import} + + @impl true + def on_success(_site_import, _extra_data), do: :ok + + @impl true + def on_failure(_site_import), do: :ok + + defoverridable before_start: 2, on_success: 2, on_failure: 1 + end + end + + @doc false + def new_import(source, site, user, opts, before_start_fun) do + import_params = + opts + |> Keyword.take([:label, :start_date, :end_date, :legacy]) + |> Keyword.put(:source, source) + |> Map.new() + + Repo.transaction(fn -> + result = + site + |> SiteImport.create_changeset(user, import_params) + |> Repo.insert() + + with {:ok, site_import} <- result, + {:ok, site_import} <- before_start_fun.(site_import, opts), + {:ok, job} <- schedule_job(site_import, opts) do + job + else + {:error, error} -> + Repo.rollback(error) + end + end) + end + + @doc false + def run_import(site_import, args, parse_args_fun, import_fun, on_success_fun) do + site_import = + site_import + |> SiteImport.start_changeset() + |> Repo.update!() + |> Repo.preload(:site) + + import_opts = parse_args_fun.(args) + + case import_fun.(site_import, import_opts) do + :ok -> + {:ok, mark_complete(site_import, %{}, on_success_fun)} + + {:ok, extra_data} -> + {:ok, mark_complete(site_import, extra_data, on_success_fun)} + + {:error, error} -> + {:error, error, []} + + {:error, error, opts} -> + {:error, error, opts} + end + end + + @oban_channel :analytics_imports_jobs + + @doc false + def notify(site_import, event) do + Oban.Notifier.notify(Oban, @oban_channel, %{ + "event" => event, + "import_id" => site_import.id, + "site_id" => site_import.site_id + }) + end + + @doc """ + Allows to explicitly start listening for importer job notifications. + + Listener must explicitly filter out a subset of imports that apply to the given context. + """ + @spec listen() :: :ok + def listen() do + :ok = Oban.Notifier.listen([@oban_channel]) + end + + defp schedule_job(site_import, opts) do + {listen?, opts} = Keyword.pop(opts, :listen?, false) + {job_opts, opts} = Keyword.pop(opts, :job_opts, []) + + if listen? do + :ok = listen() + end + + if not Imported.other_imports_in_progress?(site_import) do + opts + |> Keyword.put(:import_id, site_import.id) + |> Map.new() + |> Plausible.Workers.ImportAnalytics.new(job_opts) + |> Oban.insert() + else + {:error, :import_in_progress} + end + end + + defp mark_complete(site_import, extra_data, on_success_fun) do + site_import = + site_import + |> SiteImport.complete_changeset(extra_data) + |> Repo.update!() + + :ok = on_success_fun.(site_import, extra_data) + + site_import + end +end diff --git a/lib/plausible/imported/location.ex b/lib/plausible/imported/location.ex new file mode 100644 index 0000000000000000000000000000000000000000..2d7825bbe87a77f2f5fefd4199cc0905577470b3 --- /dev/null +++ b/lib/plausible/imported/location.ex @@ -0,0 +1,19 @@ +defmodule Plausible.Imported.Location do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_locations" do + field :site_id, Ch, type: "UInt64" + field :import_id, Ch, type: "UInt64" + field :date, :date + field :country, :string + field :region, :string + field :city, Ch, type: "UInt64" + field :visitors, Ch, type: "UInt64" + field :visits, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" + field :pageviews, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt32" + end +end diff --git a/lib/plausible/imported/noop_importer.ex b/lib/plausible/imported/noop_importer.ex new file mode 100644 index 0000000000000000000000000000000000000000..01adeccc931df46ea5c6d803f314e688e9029568 --- /dev/null +++ b/lib/plausible/imported/noop_importer.ex @@ -0,0 +1,46 @@ +defmodule Plausible.Imported.NoopImporter do + @moduledoc """ + Stub import implementation. + """ + + use Plausible.Imported.Importer + + @impl true + def name(), do: :noop + + @impl true + def label(), do: "Noop" + + # reusing existing template from another source + @impl true + def email_template(), do: "google_analytics_import.html" + + @impl true + def parse_args(opts), do: opts + + @impl true + def import_data(_site_import, %{"error" => true}), do: {:error, "Something went wrong"} + def import_data(_site_import, %{"crash" => true}), do: raise("boom") + def import_data(_site_import, _opts), do: :ok + + @impl true + def before_start(site_import, _opts) do + send(self(), {:before_start, site_import.id}) + + {:ok, site_import} + end + + @impl true + def on_success(site_import, _extra_data) do + send(self(), {:on_success, site_import.id}) + + :ok + end + + @impl true + def on_failure(site_import) do + send(self(), {:on_failure, site_import.id}) + + :ok + end +end diff --git a/lib/plausible/imported/operating_system.ex b/lib/plausible/imported/operating_system.ex new file mode 100644 index 0000000000000000000000000000000000000000..90666de76acfdd902f1c8a0e02f2aa8671e0b248 --- /dev/null +++ b/lib/plausible/imported/operating_system.ex @@ -0,0 +1,18 @@ +defmodule Plausible.Imported.OperatingSystem do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_operating_systems" do + field :site_id, Ch, type: "UInt64" + field :import_id, Ch, type: "UInt64" + field :date, :date + field :operating_system, :string + field :operating_system_version, :string + field :visitors, Ch, type: "UInt64" + field :visits, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" + field :pageviews, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt32" + end +end diff --git a/lib/plausible/imported/page.ex b/lib/plausible/imported/page.ex new file mode 100644 index 0000000000000000000000000000000000000000..3b854ec104dbc0223951021cb83e91dff7ccdb1a --- /dev/null +++ b/lib/plausible/imported/page.ex @@ -0,0 +1,21 @@ +defmodule Plausible.Imported.Page do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_pages" do + field :site_id, Ch, type: "UInt64" + field :import_id, Ch, type: "UInt64" + field :date, :date + field :hostname, :string + field :page, :string + field :visits, Ch, type: "UInt64" + field :visitors, Ch, type: "UInt64" + field :pageviews, Ch, type: "UInt64" + field :exits, Ch, type: "UInt64" + field :total_scroll_depth, Ch, type: "UInt64" + field :total_scroll_depth_visits, Ch, type: "UInt64" + field :total_time_on_page, Ch, type: "UInt64" + field :total_time_on_page_visits, Ch, type: "UInt64" + end +end diff --git a/lib/plausible/imported/site_import.ex b/lib/plausible/imported/site_import.ex new file mode 100644 index 0000000000000000000000000000000000000000..48fae49254a55ec3bdf06d6490310743ee9717c4 --- /dev/null +++ b/lib/plausible/imported/site_import.ex @@ -0,0 +1,73 @@ +defmodule Plausible.Imported.SiteImport do + @moduledoc """ + Site import schema. + """ + + use Ecto.Schema + + import Ecto.Changeset + + alias Plausible.Auth.User + alias Plausible.Imported.ImportSources + alias Plausible.Site + + @statuses [:pending, :importing, :completed, :failed] + + @type t() :: %__MODULE__{} + + schema "site_imports" do + field :start_date, :date + field :end_date, :date + field :label, :string + field :source, Ecto.Enum, values: ImportSources.names() + field :status, Ecto.Enum, values: @statuses + field :legacy, :boolean, default: false + field :has_scroll_depth, :boolean, default: false + + belongs_to :site, Site + belongs_to :imported_by, User + + timestamps() + end + + for status <- @statuses do + defmacro unquote(status)(), do: unquote(status) + end + + @spec label(t()) :: String.t() + def label(%{source: source, label: label}) do + build_label(ImportSources.by_name(source).label(), label) + end + + @spec create_changeset(Site.t(), User.t(), map()) :: Ecto.Changeset.t() + def create_changeset(site, user, params) do + %__MODULE__{} + |> cast(params, [:label, :source, :start_date, :end_date, :legacy]) + |> validate_required([:source]) + |> put_assoc(:site, site) + |> put_assoc(:imported_by, user) + |> put_change(:status, pending()) + end + + @spec start_changeset(t()) :: Ecto.Changeset.t() + def start_changeset(site_import) do + site_import + |> change(status: importing()) + end + + @spec complete_changeset(t(), map()) :: Ecto.Changeset.t() + def complete_changeset(site_import, params \\ %{}) do + site_import + |> cast(params, [:start_date, :end_date]) + |> put_change(:status, completed()) + |> validate_required([:start_date, :end_date]) + end + + @spec fail_changeset(t()) :: Ecto.Changeset.t() + def fail_changeset(site_import) do + change(site_import, status: failed()) + end + + defp build_label(source, nil), do: source + defp build_label(source, label), do: "#{source} (#{label})" +end diff --git a/lib/plausible/imported/source.ex b/lib/plausible/imported/source.ex new file mode 100644 index 0000000000000000000000000000000000000000..03f61dacbb453c8b30e4c151318e9d531d900462 --- /dev/null +++ b/lib/plausible/imported/source.ex @@ -0,0 +1,24 @@ +defmodule Plausible.Imported.Source do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_sources" do + field :site_id, Ch, type: "UInt64" + field :import_id, Ch, type: "UInt64" + field :date, :date + field :source, :string + field :channel, Ch, type: "LowCardinality(String)" + field :referrer, :string + field :utm_source, :string + field :utm_medium, :string + field :utm_campaign, :string + field :utm_content, :string + field :utm_term, :string + field :visitors, Ch, type: "UInt64" + field :visits, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" + field :pageviews, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt32" + end +end diff --git a/lib/plausible/imported/universal_analytics.ex b/lib/plausible/imported/universal_analytics.ex new file mode 100644 index 0000000000000000000000000000000000000000..608cd7b7f9e6234279b0df490646482b2c3fd6b0 --- /dev/null +++ b/lib/plausible/imported/universal_analytics.ex @@ -0,0 +1,29 @@ +defmodule Plausible.Imported.UniversalAnalytics do + @moduledoc """ + Import implementation for Universal Analytics. + + NOTE: As importing from UA is no longer supported, this module + is only used to support rendering existing imports. + """ + + use Plausible.Imported.Importer + + @impl true + def name(), do: :universal_analytics + + @impl true + def label(), do: "Google Analytics" + + @impl true + def email_template(), do: "google_analytics_import.html" + + @impl true + def parse_args(_args) do + raise "Importing data not supported" + end + + @impl true + def import_data(_site_import, _opts) do + raise "Importing data not supported" + end +end diff --git a/lib/plausible/imported/visitor.ex b/lib/plausible/imported/visitor.ex new file mode 100644 index 0000000000000000000000000000000000000000..2e4d4d26d73c65d006dbcdcf5f9ee4beed357c7d --- /dev/null +++ b/lib/plausible/imported/visitor.ex @@ -0,0 +1,16 @@ +defmodule Plausible.Imported.Visitor do + @moduledoc false + use Ecto.Schema + + @primary_key false + schema "imported_visitors" do + field :site_id, Ch, type: "UInt64" + field :import_id, Ch, type: "UInt64" + field :date, :date + field :visitors, Ch, type: "UInt64" + field :pageviews, Ch, type: "UInt64" + field :bounces, Ch, type: "UInt64" + field :visits, Ch, type: "UInt64" + field :visit_duration, Ch, type: "UInt64" + end +end diff --git a/lib/plausible/ingest_repo.ex b/lib/plausible/ingest_repo.ex new file mode 100644 index 0000000000000000000000000000000000000000..79c825647251a4569c90d2e3a1102a5d34198e1b --- /dev/null +++ b/lib/plausible/ingest_repo.ex @@ -0,0 +1,28 @@ +defmodule Plausible.IngestRepo do + @moduledoc """ + Write-centric Clickhouse access interface + """ + + use Ecto.Repo, + otp_app: :plausible, + adapter: Ecto.Adapters.ClickHouse + + defmacro __using__(_) do + quote do + alias Plausible.IngestRepo + import Ecto + import Ecto.Query, only: [from: 1, from: 2] + end + end + + def clustered_table?(table) do + replica_count(table) > 1 + end + + def replica_count(table) do + {:ok, %{rows: [[count]]}} = + query("SELECT sum(active_replicas) FROM system.replicas WHERE table = '#{table}'") + + count + end +end diff --git a/lib/plausible/ingestion/acquisition.ex b/lib/plausible/ingestion/acquisition.ex new file mode 100644 index 0000000000000000000000000000000000000000..74b8df274428dc46f77e1a11fb47d3f7dc0391f7 --- /dev/null +++ b/lib/plausible/ingestion/acquisition.ex @@ -0,0 +1,218 @@ +defmodule Plausible.Ingestion.Acquisition do + @moduledoc """ + This module is responsible for figuring out acquisition channel from event referrer_source. + + Acquisition channel is the marketing channel where people come from and convert and help + users to understand and improve their marketing flow. + + Note it uses priv/ga4-source-categories.csv as a source, which comes from https://support.google.com/analytics/answer/9756891?hl=en. + + Notable differences from GA4 that have been implemented just for Plausible: + 1. The @custom_source_categories module attribute contains a list of custom source categories that we have manually + added based on our own judgement and user feedback. For example we group AI assistants (ChatGPT, Claude, Gemini, etc.) + into their own "AI Assistants" channel. + 2. Google is in a privileged position to analyze paid traffic from within their own network. The biggest use-case is auto-tagged adwords campaigns. + We do our best by categorizing as paid search when source is Google and the url has `gclid` parameter. Same for source Bing and `msclkid` url parameter. + 3. The @paid_sources module attribute in Plausible.Ingestion.Source contains a list of utm_sources that we will automatically categorize as paid traffic + regardless of the medium. Examples are `yt-ads`, `facebook_ad`, `adwords`, etc. See also: Plausible.Ingestion.Source.paid_source?/1 + """ + + @external_resource "priv/ga4-source-categories.csv" + @custom_source_categories [ + {"hacker news", "SOURCE_CATEGORY_SOCIAL"}, + {"yahoo!", "SOURCE_CATEGORY_SEARCH"}, + {"gmail", "SOURCE_CATEGORY_EMAIL"}, + {"telegram", "SOURCE_CATEGORY_SOCIAL"}, + {"slack", "SOURCE_CATEGORY_SOCIAL"}, + {"producthunt", "SOURCE_CATEGORY_SOCIAL"}, + {"github", "SOURCE_CATEGORY_SOCIAL"}, + {"steamcommunity.com", "SOURCE_CATEGORY_SOCIAL"}, + {"statics.teams.cdn.office.net", "SOURCE_CATEGORY_SOCIAL"}, + {"vkontakte", "SOURCE_CATEGORY_SOCIAL"}, + {"threads", "SOURCE_CATEGORY_SOCIAL"}, + {"ecosia", "SOURCE_CATEGORY_SEARCH"}, + {"kagi", "SOURCE_CATEGORY_SEARCH"}, + {"brave", "SOURCE_CATEGORY_SEARCH"}, + {"temu.com", "SOURCE_CATEGORY_SHOPPING"}, + {"discord", "SOURCE_CATEGORY_SOCIAL"}, + {"sogou", "SOURCE_CATEGORY_SEARCH"}, + {"microsoft teams", "SOURCE_CATEGORY_SOCIAL"}, + {"bluesky", "SOURCE_CATEGORY_SOCIAL"}, + {"mastodon", "SOURCE_CATEGORY_SOCIAL"}, + {"chatgpt", "SOURCE_CATEGORY_AI_ASSISTANTS"}, + {"claude", "SOURCE_CATEGORY_AI_ASSISTANTS"}, + {"phind", "SOURCE_CATEGORY_AI_ASSISTANTS"}, + {"deepseek", "SOURCE_CATEGORY_AI_ASSISTANTS"}, + {"microsoft copilot", "SOURCE_CATEGORY_AI_ASSISTANTS"}, + {"grok", "SOURCE_CATEGORY_AI_ASSISTANTS"}, + {"google gemini", "SOURCE_CATEGORY_AI_ASSISTANTS"}, + {"perplexity", "SOURCE_CATEGORY_AI_ASSISTANTS"}, + {"x (twitter)", "SOURCE_CATEGORY_SOCIAL"} + ] + @source_categories Application.app_dir(:plausible, "priv/ga4-source-categories.csv") + |> File.read!() + |> NimbleCSV.RFC4180.parse_string(skip_headers: false) + |> Enum.map(fn [source, category] -> {source, category} end) + |> then(&(@custom_source_categories ++ &1)) + |> Enum.into(%{}) + + def source_categories(), do: @source_categories + + def get_channel(source, utm_medium, utm_campaign, utm_source, click_id_param) do + get_channel_lowered( + String.downcase(source || ""), + String.downcase(utm_medium || ""), + String.downcase(utm_campaign || ""), + String.downcase(utm_source || ""), + click_id_param + ) + end + + defp get_channel_lowered(source, utm_medium, utm_campaign, utm_source, click_id_param) do + cond do + cross_network?(utm_campaign) -> "Cross-network" + paid_shopping?(source, utm_campaign, utm_medium) -> "Paid Shopping" + paid_search?(source, utm_medium, utm_source, click_id_param) -> "Paid Search" + paid_social?(source, utm_medium, utm_source) -> "Paid Social" + paid_video?(source, utm_medium, utm_source) -> "Paid Video" + display?(utm_medium) -> "Display" + paid_other?(utm_medium) -> "Paid Other" + organic_shopping?(source, utm_campaign) -> "Organic Shopping" + organic_social?(source, utm_medium) -> "Organic Social" + organic_video?(source, utm_medium) -> "Organic Video" + ai_assistants?(source) -> "AI Assistants" + search_source?(source) -> "Organic Search" + email?(source, utm_source, utm_medium) -> "Email" + affiliates?(utm_medium) -> "Affiliates" + audio?(utm_medium) -> "Audio" + sms?(utm_source, utm_medium) -> "SMS" + mobile_push_notifications?(source, utm_medium) -> "Mobile Push Notifications" + referral?(source, utm_medium) -> "Referral" + true -> "Direct" + end + end + + defp cross_network?(utm_campaign) do + String.contains?(utm_campaign, "cross-network") + end + + defp paid_shopping?(source, utm_campaign, utm_medium) do + (shopping_source?(source) or shopping_campaign?(utm_campaign)) and paid_medium?(utm_medium) + end + + defp paid_search?(source, utm_medium, utm_source, click_id_param) do + (search_source?(source) and paid_medium?(utm_medium)) or + (search_source?(source) and paid_source?(utm_source)) or + (source == "google" and click_id_param == "gclid") or + (source == "bing" and click_id_param == "msclkid") + end + + defp paid_social?(source, utm_medium, utm_source) do + (social_source?(source) and paid_medium?(utm_medium)) or + (social_source?(source) and paid_source?(utm_source)) + end + + defp paid_video?(source, utm_medium, utm_source) do + (video_source?(source) and paid_medium?(utm_medium)) or + (video_source?(source) and paid_source?(utm_source)) + end + + defp display?(utm_medium) do + utm_medium in [ + "display", + "banner", + "expandable", + "interstitial", + "cpm" + ] + end + + defp paid_other?(utm_medium) do + paid_medium?(utm_medium) + end + + defp organic_shopping?(source, utm_campaign) do + shopping_source?(source) or shopping_campaign?(utm_campaign) + end + + defp organic_social?(source, utm_medium) do + social_source?(source) or + utm_medium in [ + "social", + "social-network", + "social-media", + "sm", + "social network", + "social media" + ] + end + + defp organic_video?(source, utm_medium) do + video_source?(source) or String.contains?(utm_medium, "video") + end + + defp referral?(source, utm_medium) do + utm_medium in ["referral", "app", "link"] or source !== "" + end + + @email_tags ["email", "e-mail", "e_mail", "e mail", "newsletter"] + defp email?(source, utm_source, utm_medium) do + email_source?(source) or + String.contains?(utm_source, @email_tags) or + String.contains?(utm_medium, @email_tags) + end + + defp affiliates?(utm_medium) do + utm_medium == "affiliate" + end + + defp audio?(utm_medium) do + utm_medium == "audio" + end + + defp sms?(utm_source, utm_medium) do + utm_source == "sms" or utm_medium == "sms" + end + + defp mobile_push_notifications?(source, utm_medium) do + String.ends_with?(utm_medium, "push") or + String.contains?(utm_medium, ["mobile", "notification"]) or + source == "firebase" + end + + defp shopping_source?(source) do + @source_categories[source] == "SOURCE_CATEGORY_SHOPPING" + end + + defp search_source?(source) do + @source_categories[source] == "SOURCE_CATEGORY_SEARCH" + end + + defp social_source?(source) do + @source_categories[source] == "SOURCE_CATEGORY_SOCIAL" + end + + defp video_source?(source) do + @source_categories[source] == "SOURCE_CATEGORY_VIDEO" + end + + defp email_source?(source) do + @source_categories[source] == "SOURCE_CATEGORY_EMAIL" + end + + defp ai_assistants?(source) do + @source_categories[source] == "SOURCE_CATEGORY_AI_ASSISTANTS" + end + + defp shopping_campaign?(utm_campaign) do + Regex.match?(~r/^(.*(([^a-df-z]|^)shop|shopping).*)$/, utm_campaign) + end + + defp paid_medium?(utm_medium) do + Regex.match?(~r/^(.*cp.*|ppc|retargeting|paid.*)$/, utm_medium) + end + + defp paid_source?(utm_source) do + Plausible.Ingestion.Source.paid_source?(utm_source) + end +end diff --git a/lib/plausible/ingestion/city_overrides.ex b/lib/plausible/ingestion/city_overrides.ex new file mode 100644 index 0000000000000000000000000000000000000000..a00eb24ab9f8093ca7979fbe8dbcc5587b0cc9f3 --- /dev/null +++ b/lib/plausible/ingestion/city_overrides.ex @@ -0,0 +1,202 @@ +defmodule Plausible.Ingestion.CityOverrides do + @moduledoc false + + @overrides %{ + # Austria + # Gemeindebezirk Floridsdorf -> Vienna + 2_779_467 => 2_761_369, + # Gemeindebezirk Leopoldstadt -> Vienna + 2_772_614 => 2_761_369, + # Gemeindebezirk Landstrasse -> Vienna + 2_773_040 => 2_761_369, + # Gemeindebezirk Donaustadt -> Vienna + 2_780_851 => 2_761_369, + # Gemeindebezirk Favoriten -> Vienna + 2_779_776 => 2_761_369, + # Gemeindebezirk Währing -> Vienna + 2_762_091 => 2_761_369, + # Gemeindebezirk Wieden -> Vienna + 2_761_393 => 2_761_369, + # Gemeindebezirk Innere Stadt -> Vienna + 2_775_259 => 2_761_369, + # Gemeindebezirk Alsergrund -> Vienna + 2_782_729 => 2_761_369, + # Gemeindebezirk Liesing -> Vienna + 2_772_484 => 2_761_369, + # Urfahr -> Linz + 2_762_518 => 2_772_400, + + # Canada + # Old Toronto -> Toronto + 8_436_019 => 6_167_865, + # Etobicoke -> Toronto + 5_950_267 => 6_167_865, + # East York -> Toronto + 5_946_235 => 6_167_865, + # Scarborough -> Toronto + 6_948_711 => 6_167_865, + # North York -> Toronto + 6_091_104 => 6_167_865, + + # Czech republic + # Praha 5 -> Prague + 11_951_220 => 3_067_696, + # Praha 4 -> Prague + 11_951_218 => 3_067_696, + # Praha 11 -> Prague + 11_951_232 => 3_067_696, + # Praha 10 -> Prague + 11_951_210 => 3_067_696, + # Praha 4 -> Prague + 8_378_772 => 3_067_696, + + # Denmark + # København SV -> Copenhagen + 11_747_123 => 2_618_425, + # København NV -> Copenhagen + 11_746_894 => 2_618_425, + # Odense S -> Odense + 11_746_825 => 2_615_876, + # Odense M -> Odense + 11_746_974 => 2_615_876, + # Odense SØ -> Odense + 11_746_888 => 2_615_876, + # Aarhus C -> Aarhus + 11_746_746 => 2_624_652, + # Aarhus N -> Aarhus + 11_746_890 => 2_624_652, + + # Estonia + # Kristiine linnaosa -> Tallinn + 11_050_530 => 588_409, + # Kesklinna linnaosa -> Tallinn + 11_053_706 => 588_409, + # Lasnamäe linnaosa -> Tallinn + 11_050_526 => 588_409, + # Põhja-Tallinna linnaosa -> Tallinn + 11_049_594 => 588_409, + # Mustamäe linnaosa -> Tallinn + 11_050_531 => 588_409, + # Haabersti linnaosa -> Tallinn + 11_053_707 => 588_409, + # Viimsi -> Tallinn + 587_629 => 588_409, + + # Germany + # Bezirk Tempelhof-Schöneberg -> Berlin + 3_336_297 => 2_950_159, + # Bezirk Mitte -> Berlin + 2_870_912 => 2_950_159, + # Bezirk Charlottenburg-Wilmersdorf -> Berlin + 3_336_294 => 2_950_159, + # Bezirk Friedrichshain-Kreuzberg -> Berlin + 3_336_295 => 2_950_159, + # Moosach -> Munich + 8_351_447 => 2_867_714, + # Schwabing-Freimann -> Munich + 8_351_448 => 2_867_714, + # Stadtbezirk 06 -> Düsseldorf + 6_947_276 => 2_934_246, + # Stadtbezirk 04 -> Düsseldorf + 6_947_274 => 2_934_246, + # Köln-Ehrenfeld -> Köln + 6_947_479 => 2_886_242, + # Köln-Lindenthal- -> Köln + 6_947_481 => 2_886_242, + # Beuel -> Bonn + 2_949_619 => 2_946_447, + # Innenstadt I -> Frankfurt am Main + 6_946_225 => 2_925_533, + # Innenstadt II -> Frankfurt am Main + 6_946_226 => 2_925_533, + # Innenstadt III -> Frankfurt am Main + 6_946_227 => 2_925_533, + + # India + # Navi Mumbai -> Mumbai + 6_619_347 => 1_275_339, + + # Mexico + # Miguel Hidalgo Villa Olímpica -> Mexico city + 11_561_026 => 3_530_597, + # Zedec Santa Fe -> Mexico city + 3_517_471 => 3_530_597, + # Fuentes del Pedregal-> Mexico city + 11_562_596 => 3_530_597, + # Centro -> Mexico city + 9_179_691 => 3_530_597, + # Cuauhtémoc-> Mexico city + 12_266_959 => 3_530_597, + + # Netherlands + # Schiphol-Rijk -> Amsterdam + 10_173_838 => 2_759_794, + # Westpoort -> Amsterdam + 11_525_047 => 2_759_794, + # Amsterdam-Zuidoost -> Amsterdam + 6_544_881 => 2_759_794, + # Loosduinen -> The Hague + 11_525_037 => 2_747_373, + # Laak -> The Hague + 11_525_042 => 2_747_373, + + # Norway + # Nordre Aker District -> Oslo + 6_940_981 => 3_143_244, + + # Romania + # Sector 1 -> Bucharest, + 11_055_041 => 683_506, + # Sector 2 -> Bucharest + 11_055_040 => 683_506, + # Sector 3 -> Bucharest + 11_055_044 => 683_506, + # Sector 4 -> Bucharest + 11_055_042 => 683_506, + # Sector 5 -> Bucharest + 11_055_043 => 683_506, + # Sector 6 -> Bucharest + 11_055_039 => 683_506, + # Bucuresti -> Bucharest + 6_691_781 => 683_506, + + # Slovakia + # Bratislava -> Bratislava + 3_343_955 => 3_060_972, + + # Sweden + # Södermalm -> Stockholm + 2_676_209 => 2_673_730, + + # Switzerland + # Vorstädte -> Basel + 11_789_440 => 2_661_604, + # Zürich (Kreis 11) / Oerlikon -> Zürich + 2_659_310 => 2_657_896, + # Zürich (Kreis 3) / Alt-Wiedikon -> Zürich + 2_658_007 => 2_657_896, + # Zürich (Kreis 5) -> Zürich + 6_295_521 => 2_657_896, + # Zürich (Kreis 1) / Hochschulen -> Zürich + 6_295_489 => 2_657_896, + + # UK + # Shadwell -> London + 6_690_595 => 2_643_743, + # City of London -> London + 2_643_741 => 2_643_743, + # South Bank -> London + 6_545_251 => 2_643_743, + # Soho -> London + 6_545_173 => 2_643_743, + # Whitechapel -> London + 2_634_112 => 2_643_743, + # King's Cross -> London + 6_690_589 => 2_643_743, + # Poplar -> London + 2_640_091 => 2_643_743, + # Hackney -> London + 2_647_694 => 2_643_743 + } + def get(key, default), do: Map.get(@overrides, key, default) +end diff --git a/lib/plausible/ingestion/counters.ex b/lib/plausible/ingestion/counters.ex new file mode 100644 index 0000000000000000000000000000000000000000..818f86937a3f41e0ffa88012dbcf7dc03d313de2 --- /dev/null +++ b/lib/plausible/ingestion/counters.ex @@ -0,0 +1,128 @@ +defmodule Plausible.Ingestion.Counters do + @moduledoc """ + This is instrumentation necessary for keeping track of per-domain + internal metrics. Due to metric labels cardinality (domain x metric_name), + these statistics are not suitable for prometheus/grafana exposure, + hence an internal storage is used. + + The module installs `Counters.TelemetryHandler` and periodically + flushes the internal counter aggregates via `Counters.Buffer` interface. + + The underlying database schema is running `SummingMergeTree` engine. + To take advantage of automatic roll-ups it provides, upon dispatching the + buffered records to Clickhouse this module transforms each `event_timebucket` + aggregate into a 1-minute resolution. + + Clickhouse connection is set to insert counters asynchronously every time + a pool checkout is made. Those properties are reverted once the insert is done + (or naturally, if the connection crashes). + """ + + @behaviour :gen_cycle + + alias Plausible.Ingestion.Counters.Buffer + alias Plausible.Ingestion.Counters.Record + alias Plausible.Ingestion.Counters.TelemetryHandler + alias Plausible.AsyncInsertRepo + + @interval :timer.seconds(10) + + @spec child_spec(Keyword.t()) :: Supervisor.child_spec() | :ignore + def child_spec(opts) do + buffer_name = Keyword.get(opts, :buffer_name, __MODULE__) + + %{ + id: buffer_name, + start: {:gen_cycle, :start_link, [{:local, buffer_name}, __MODULE__, opts]} + } + end + + @spec enabled?() :: boolean() + def enabled?() do + Application.fetch_env!(:plausible, __MODULE__)[:enabled] == true + end + + @impl true + def init_cycle(opts) do + Process.flag(:trap_exit, true) + buffer_name = Keyword.get(opts, :buffer_name, __MODULE__) + force_start? = Keyword.get(opts, :force_start?, false) + + if enabled?() or force_start? do + buffer = Buffer.new(buffer_name, opts) + :ok = TelemetryHandler.install(buffer) + + interval = Keyword.get(opts, :interval, @interval) + + {:ok, {interval, buffer}} + else + :ignore + end + end + + @impl true + def handle_cycle(buffer, now \\ DateTime.utc_now()) do + case Buffer.flush(buffer, now) do + [] -> + :noop + + records -> + records = + Enum.map(records, fn {bucket, metric, domain, tracker_script_version, value} -> + %{ + event_timebucket: to_0_minute_datetime(bucket), + metric: metric, + site_id: Plausible.Site.Cache.get_site_id(domain), + domain: domain, + tracker_script_version: tracker_script_version, + value: value + } + end) + + try do + {_, _} = AsyncInsertRepo.insert_all(Record, records) + catch + _, thrown -> + Sentry.capture_message( + "Caught an error when trying to flush ingest counters.", + extra: %{ + number_of_records: Enum.count(records), + error: inspect(thrown) + } + ) + end + end + + {:continue_hibernated, buffer} + end + + @impl true + def handle_info(:stop, _state) do + {:stop, :normal} + end + + def handle_info(_msg, state) do + {:continue, state} + end + + @impl true + def terminate(_reason, buffer) do + # we'll travel in time to flush everything regardless of current bucket completion + future = DateTime.utc_now() |> DateTime.add(60, :second) + handle_cycle(buffer, future) + :ok + end + + @spec stop(pid()) :: :ok + def stop(pid) do + send(pid, :stop) + :ok + end + + defp to_0_minute_datetime(unix_ts) when is_integer(unix_ts) do + unix_ts + |> DateTime.from_unix!() + |> DateTime.truncate(:second) + |> Map.replace(:second, 0) + end +end diff --git a/lib/plausible/ingestion/counters/buffer.ex b/lib/plausible/ingestion/counters/buffer.ex new file mode 100644 index 0000000000000000000000000000000000000000..ea8ffffa1bbb931097efc8a21c70e236961ed8c5 --- /dev/null +++ b/lib/plausible/ingestion/counters/buffer.ex @@ -0,0 +1,114 @@ +defmodule Plausible.Ingestion.Counters.Buffer do + @moduledoc """ + A buffer aggregating counters for internal metrics, within 10 seconds time buckets. + + See `Plausible.Ingestion.Counters` for integration. + + Flushing is by default possible only once the 10s bucket is complete + (its window has moved). This is to avoid race conditions + when clearing up the buffer on dequeue - because there is no atomic "get and delete", + and items are buffered concurrently, there is a gap between get and delete + in which items written may disappear otherwise. + + `aggregate_bucket_fn` and `flush_boundary_fn` control that semantics and + are configurable only for test purposes. + """ + + defstruct [:buffer_name, :aggregate_bucket_fn, :flush_boundary_fn] + + alias Plausible.Ingestion.Counters.Record + + @type t() :: %__MODULE__{} + @type unix_timestamp() :: pos_integer() + @type bucket_fn_opt() :: + {:aggregate_bucket_fn, (NaiveDateTime.t() -> unix_timestamp())} + | {:flush_boundary_fn, (DateTime.t() -> unix_timestamp())} + + @ets_opts [ + :public, + :ordered_set, + :named_table, + write_concurrency: true + ] + + @spec new(atom(), [bucket_fn_opt()]) :: t() + def new(buffer_name, opts \\ []) do + ^buffer_name = :ets.new(buffer_name, @ets_opts) + + aggregate_bucket_fn = Keyword.get(opts, :aggregate_bucket_fn, &bucket_10s/1) + flush_boundary_fn = Keyword.get(opts, :flush_boundary_fn, &previous_10s/1) + + %__MODULE__{ + buffer_name: buffer_name, + aggregate_bucket_fn: aggregate_bucket_fn, + flush_boundary_fn: flush_boundary_fn + } + end + + @spec aggregate( + t(), + binary(), + binary(), + timestamp :: NaiveDateTime.t(), + tracker_script_version :: integer() + ) :: t() + def aggregate( + %__MODULE__{buffer_name: buffer_name, aggregate_bucket_fn: bucket_fn} = buffer, + metric, + domain, + timestamp, + tracker_script_version + ) do + bucket = + bucket_fn.(timestamp) + + :ets.update_counter( + buffer_name, + {bucket, metric, domain, tracker_script_version}, + {2, 1}, + {{bucket, metric, domain, tracker_script_version}, 0} + ) + + buffer + end + + @spec flush(t(), now :: DateTime.t()) :: [Record.t()] + def flush( + %__MODULE__{buffer_name: buffer_name, flush_boundary_fn: flush_boundary_fn}, + now \\ DateTime.utc_now() + ) do + boundary = flush_boundary_fn.(now) + + match = {{:"$1", :"$2", :"$3", :"$4"}, :"$5"} + guard = {:"=<", :"$1", boundary} + select = {{:"$1", :"$2", :"$3", :"$4", :"$5"}} + + match_specs_read = [{match, [guard], [select]}] + match_specs_delete = [{match, [guard], [true]}] + + case :ets.select(buffer_name, match_specs_read) do + [] -> + [] + + data -> + :ets.select_delete(buffer_name, match_specs_delete) + data + end + end + + @spec bucket_10s(NaiveDateTime.t()) :: unix_timestamp() + def bucket_10s(datetime) do + datetime + |> DateTime.from_naive!("Etc/UTC") + |> DateTime.truncate(:second) + |> Map.replace(:second, div(datetime.second, 10) * 10) + |> DateTime.to_unix() + end + + @spec previous_10s(DateTime.t()) :: unix_timestamp() + def previous_10s(datetime) do + datetime + |> DateTime.add(-10, :second) + |> DateTime.to_unix() + end +end diff --git a/lib/plausible/ingestion/counters/record.ex b/lib/plausible/ingestion/counters/record.ex new file mode 100644 index 0000000000000000000000000000000000000000..ae8ce399b304df6c31777c28a56fafd206e190f9 --- /dev/null +++ b/lib/plausible/ingestion/counters/record.ex @@ -0,0 +1,18 @@ +defmodule Plausible.Ingestion.Counters.Record do + @moduledoc """ + Clickhouse schema for storing ingest counter metrics + """ + use Ecto.Schema + + @type t() :: %__MODULE__{} + + @primary_key false + schema "ingest_counters" do + field :event_timebucket, :utc_datetime + field :site_id, Ch, type: "Nullable(UInt64)" + field :domain, Ch, type: "LowCardinality(String)" + field :metric, Ch, type: "LowCardinality(String)" + field :value, Ch, type: "UInt64" + field :tracker_script_version, Ch, type: "UInt16" + end +end diff --git a/lib/plausible/ingestion/counters/telemetry_handler.ex b/lib/plausible/ingestion/counters/telemetry_handler.ex new file mode 100644 index 0000000000000000000000000000000000000000..8caa37681e9d488e65b1d9ca5ac5f1fa4307dc73 --- /dev/null +++ b/lib/plausible/ingestion/counters/telemetry_handler.ex @@ -0,0 +1,65 @@ +defmodule Plausible.Ingestion.Counters.TelemetryHandler do + @moduledoc """ + Subscribes to telemetry events emitted by `Plausible.Ingestion.Event`. + Every time a request derived event is either dispatched to clickhouse or dropped, + a telemetry event is emitted respectively. That event is captured here, + its metadata is extracted and sent for internal stats aggregation via + `Counters.Buffer` interface. + """ + alias Plausible.Ingestion.Counters + alias Plausible.Ingestion.Event + + @event_dropped Event.telemetry_event_dropped() + @event_buffered Event.telemetry_event_buffered() + + @telemetry_events [@event_dropped, @event_buffered] + @telemetry_handler &__MODULE__.handle_event/4 + + @spec install(Counters.Buffer.t()) :: :ok + def install(%Counters.Buffer{buffer_name: buffer_name} = buffer) do + :ok = + :telemetry.attach_many( + "ingest-counters-#{buffer_name}", + @telemetry_events, + @telemetry_handler, + buffer + ) + end + + @spec handle_event([atom()], any(), map(), Counters.Buffer.t()) :: :ok + def handle_event( + @event_dropped, + _measurements, + %{ + domain: domain, + reason: reason, + request_timestamp: timestamp, + tracker_script_version: tracker_script_version + }, + buffer + ) do + Counters.Buffer.aggregate( + buffer, + "dropped_#{reason}", + domain, + timestamp, + tracker_script_version + ) + + :ok + end + + def handle_event( + @event_buffered, + _measurements, + %{ + domain: domain, + request_timestamp: timestamp, + tracker_script_version: tracker_script_version + }, + buffer + ) do + Counters.Buffer.aggregate(buffer, "buffered", domain, timestamp, tracker_script_version) + :ok + end +end diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex new file mode 100644 index 0000000000000000000000000000000000000000..ee4d99e1f8c8ac02d50fc333ece669691071fe3f --- /dev/null +++ b/lib/plausible/ingestion/event.ex @@ -0,0 +1,588 @@ +defmodule Plausible.Ingestion.Event do + @moduledoc """ + This module exposes the `build_and_buffer/1` function capable of + turning %Plausible.Ingestion.Request{} into a series of events that in turn + are uniformly either buffered in batches (to Clickhouse) or dropped + (e.g. due to spam blocklist) from the processing pipeline. + """ + use Plausible + alias Plausible.Ingestion.Request + alias Plausible.ClickhouseEventV2 + alias Plausible.Site.GateKeeper + + defstruct domain: nil, + site: nil, + clickhouse_event_attrs: %{}, + clickhouse_session_attrs: %{}, + clickhouse_event: nil, + dropped?: false, + drop_reason: nil, + request: nil, + salts: nil, + changeset: nil + + @type drop_reason() :: + :bot + | :spam_referrer + | GateKeeper.policy() + | :invalid + | :dc_ip + | :threat_ip + | :site_ip_blocklist + | :site_country_blocklist + | :site_page_blocklist + | :site_hostname_allowlist + | :verification_agent + | :lock_timeout + | :no_session_for_engagement + | :persist_timeout + | :persist_error + | :persist_decode_error + + @type t() :: %__MODULE__{ + domain: String.t() | nil, + site: %Plausible.Site{} | nil, + clickhouse_event_attrs: map(), + clickhouse_session_attrs: map(), + clickhouse_event: %ClickhouseEventV2{} | nil, + dropped?: boolean(), + drop_reason: drop_reason(), + request: Request.t(), + salts: map(), + changeset: %Ecto.Changeset{} + } + + @spec build_and_buffer(Request.t(), Keyword.t()) :: {:ok, %{buffered: [t()], dropped: [t()]}} + def build_and_buffer(%Request{domains: domains} = request, context \\ []) do + processed_events = + if spam_referrer?(request) do + for domain <- domains, do: drop(new(domain, request), :spam_referrer) + else + Enum.reduce(domains, [], fn domain, acc -> + # credo:disable-for-next-line Credo.Check.Refactor.Nesting + case GateKeeper.check(domain) do + {:allow, site} -> + processed = + domain + |> new(site, request) + |> process_unless_dropped(pipeline(), context) + + [processed | acc] + + {:deny, reason} -> + [drop(new(domain, request), reason) | acc] + end + end) + end + + {dropped, buffered} = Enum.split_with(processed_events, & &1.dropped?) + {:ok, %{dropped: dropped, buffered: buffered}} + end + + @spec telemetry_event_buffered() :: [atom()] + def telemetry_event_buffered() do + [:plausible, :ingest, :event, :buffered] + end + + @spec telemetry_event_dropped() :: [atom()] + def telemetry_event_dropped() do + [:plausible, :ingest, :event, :dropped] + end + + @spec telemetry_pipeline_step_duration() :: [atom()] + def telemetry_pipeline_step_duration() do + [:plausible, :ingest, :pipeline, :step] + end + + @spec telemetry_ua_parse_timeout() :: [atom()] + def telemetry_ua_parse_timeout() do + [:plausible, :ingest, :user_agent_parse, :timeout] + end + + @spec emit_telemetry_ua_parse_timeout() :: :ok + def emit_telemetry_ua_parse_timeout() do + :telemetry.execute(telemetry_ua_parse_timeout(), %{}, %{}) + end + + @spec emit_telemetry_buffered(t()) :: :ok + def emit_telemetry_buffered(event) do + :telemetry.execute(telemetry_event_buffered(), %{}, %{ + domain: event.domain, + request_timestamp: event.request.timestamp, + tracker_script_version: event.request.tracker_script_version + }) + end + + @spec emit_telemetry_dropped(t(), drop_reason()) :: :ok + def emit_telemetry_dropped(event, reason) do + :telemetry.execute( + telemetry_event_dropped(), + %{}, + %{ + domain: event.domain, + reason: reason, + request_timestamp: event.request.timestamp, + tracker_script_version: event.request.tracker_script_version + } + ) + end + + defp pipeline() do + [ + drop_verification_agent: &drop_verification_agent/2, + drop_datacenter_ip: &drop_datacenter_ip/2, + drop_threat_ip: &drop_threat_ip/2, + drop_shield_rule_hostname: &drop_shield_rule_hostname/2, + drop_shield_rule_page: &drop_shield_rule_page/2, + drop_shield_rule_ip: &drop_shield_rule_ip/2, + put_geolocation: &put_geolocation/2, + drop_shield_rule_country: &drop_shield_rule_country/2, + put_user_agent: &put_user_agent/2, + put_basic_info: &put_basic_info/2, + put_source_info: &put_source_info/2, + maybe_infer_medium: &maybe_infer_medium/2, + put_props: &put_props/2, + put_revenue: &put_revenue/2, + put_salts: &put_salts/2, + put_user_id: &put_user_id/2, + validate_clickhouse_event: &validate_clickhouse_event/2, + register_session: ®ister_session/2 + ] + end + + defp process_unless_dropped(%__MODULE__{} = initial_event, pipeline, context) do + Enum.reduce_while(pipeline, initial_event, fn {step_name, step_fn}, acc_event -> + Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration( + telemetry_pipeline_step_duration(), + fn -> execute_step(step_fn, acc_event, context) end, + %{step: "#{step_name}"} + ) + end) + end + + defp execute_step(step_fn, acc_event, context) do + case step_fn.(acc_event, context) do + %__MODULE__{dropped?: true} = dropped -> {:halt, dropped} + %__MODULE__{dropped?: false} = event -> {:cont, event} + end + end + + defp new(domain, request) do + struct!(__MODULE__, domain: domain, request: request) + end + + defp new(domain, site, request) do + struct!(__MODULE__, domain: domain, site: site, request: request) + end + + defp drop(%__MODULE__{} = event, reason, attrs \\ []) do + fields = + attrs + |> Keyword.put(:dropped?, true) + |> Keyword.put(:drop_reason, reason) + + emit_telemetry_dropped(event, reason) + struct!(event, fields) + end + + defp update_event_attrs(%__MODULE__{} = event, %{} = attrs) do + struct!(event, clickhouse_event_attrs: Map.merge(event.clickhouse_event_attrs, attrs)) + end + + defp update_session_attrs(%__MODULE__{} = event, %{} = attrs) do + struct!(event, clickhouse_session_attrs: Map.merge(event.clickhouse_session_attrs, attrs)) + end + + on_ee do + @verification_user_agent Plausible.InstallationSupport.user_agent() + + defp drop_verification_agent(%__MODULE__{} = event, _context) do + case event.request.user_agent do + @verification_user_agent -> + drop(event, :verification_agent) + + _ -> + event + end + end + else + defp drop_verification_agent(%__MODULE__{} = event, _context), do: event + end + + defp drop_datacenter_ip(%__MODULE__{} = event, _context) do + case event.request.ip_classification do + "dc_ip" -> + drop(event, :dc_ip) + + _any -> + event + end + end + + defp drop_threat_ip(%__MODULE__{} = event, _context) do + case event.request.ip_classification do + "threat_ip" -> + drop(event, :threat_ip) + + _any -> + event + end + end + + defp drop_shield_rule_ip(%__MODULE__{} = event, _context) do + if Plausible.Shields.ip_blocked?(event.domain, event.request.remote_ip) do + drop(event, :site_ip_blocklist) + else + event + end + end + + defp drop_shield_rule_hostname(%__MODULE__{} = event, _context) do + if Plausible.Shields.hostname_allowed?(event.domain, event.request.hostname) do + event + else + drop(event, :site_hostname_allowlist) + end + end + + defp drop_shield_rule_page(%__MODULE__{} = event, _context) do + if Plausible.Shields.page_blocked?(event.domain, event.request.pathname) do + drop(event, :site_page_blocklist) + else + event + end + end + + defp put_user_agent(%__MODULE__{} = event, _context) do + case parse_user_agent(event.request) do + {:ok, %UAInspector.Result{client: %UAInspector.Result.Client{name: "Headless Chrome"}}} -> + drop(event, :bot) + + {:ok, %UAInspector.Result.Bot{}} -> + drop(event, :bot) + + {:ok, %UAInspector.Result{} = user_agent} -> + update_session_attrs(event, %{ + operating_system: os_name(user_agent), + operating_system_version: os_version(user_agent), + browser: browser_name(user_agent), + browser_version: browser_version(user_agent), + screen_size: screen_size(user_agent) + }) + + _any -> + event + end + end + + defp put_basic_info(%__MODULE__{} = event, _context) do + update_event_attrs(event, %{ + domain: event.domain, + site_id: event.site.id, + timestamp: event.request.timestamp, + name: event.request.event_name, + hostname: event.request.hostname, + pathname: event.request.pathname, + scroll_depth: event.request.scroll_depth, + engagement_time: event.request.engagement_time, + interactive?: event.request.interactive? + }) + end + + defp put_source_info(%__MODULE__{} = event, _context) do + query_params = event.request.query_params + + tagged_source = + query_params["utm_source"] || + query_params["source"] || + query_params["ref"] + + update_session_attrs(event, %{ + referrer_source: Plausible.Ingestion.Source.resolve(event.request), + referrer: Plausible.Ingestion.Source.format_referrer(event.request), + click_id_param: get_click_id_param(event.request.query_params), + utm_source: tagged_source, + utm_medium: query_params["utm_medium"], + utm_campaign: query_params["utm_campaign"], + utm_content: query_params["utm_content"], + utm_term: query_params["utm_term"] + }) + end + + defp maybe_infer_medium(%__MODULE__{} = event, _context) do + inferred_medium = + case event.clickhouse_session_attrs do + %{utm_medium: medium} when is_binary(medium) -> medium + %{utm_medium: nil, referrer_source: "Google", click_id_param: "gclid"} -> "(gclid)" + %{utm_medium: nil, referrer_source: "Bing", click_id_param: "msclkid"} -> "(msclkid)" + _ -> nil + end + + update_session_attrs(event, %{utm_medium: inferred_medium}) + end + + defp put_geolocation(%__MODULE__{} = event, _context) do + case event.request.ip_classification do + "anonymous_vpn_ip" -> + update_session_attrs(event, %{country_code: "A1"}) + + _any -> + result = Plausible.Ingestion.Geolocation.lookup(event.request.remote_ip) || %{} + update_session_attrs(event, result) + end + end + + defp drop_shield_rule_country( + %__MODULE__{domain: domain, clickhouse_session_attrs: %{country_code: cc}} = event, + _context + ) + when is_binary(domain) and is_binary(cc) do + if Plausible.Shields.country_blocked?(domain, cc) do + drop(event, :site_country_blocklist) + else + event + end + end + + defp drop_shield_rule_country( + %__MODULE__{domain: domain} = event, + _context + ) + when is_binary(domain) do + if Plausible.Shields.country_blocked?( + domain, + Plausible.Shield.CountryRule.unknown_country_code() + ) do + drop(event, :site_country_blocklist) + else + event + end + end + + defp put_props(%__MODULE__{request: %{props: %{} = props}} = event, _context) do + # defensive: ensuring the keys/values are always in the same order + {keys, values} = Enum.unzip(props) + + update_event_attrs(event, %{ + "meta.key": keys, + "meta.value": values + }) + end + + defp put_props(%__MODULE__{} = event, _context), do: event + + defp put_revenue(event, _context) do + on_ee do + attrs = Plausible.Ingestion.Event.Revenue.get_revenue_attrs(event) + update_event_attrs(event, attrs) + else + event + end + end + + defp put_salts(%__MODULE__{} = event, _context) do + %{event | salts: Plausible.Session.Salts.fetch()} + end + + defp put_user_id(%__MODULE__{} = event, _context) do + update_event_attrs(event, %{ + user_id: + generate_user_id( + event.request, + event.domain, + event.clickhouse_event_attrs.hostname, + event.salts.current + ) + }) + end + + defp validate_clickhouse_event(%__MODULE__{} = event, _context) do + clickhouse_event = + event + |> Map.fetch!(:clickhouse_event_attrs) + |> ClickhouseEventV2.new() + + case Ecto.Changeset.apply_action(clickhouse_event, nil) do + {:ok, valid_clickhouse_event} -> + %{event | clickhouse_event: valid_clickhouse_event} + + {:error, changeset} -> + drop(event, :invalid, changeset: changeset) + end + end + + defp register_session(%__MODULE__{} = event, context) do + persistor_opts = Keyword.get(context, :persistor_opts, []) + + previous_user_id = + generate_user_id( + event.request, + event.domain, + event.clickhouse_event.hostname, + event.salts.previous + ) + + case Plausible.Ingestion.Persistor.persist_event(event, previous_user_id, persistor_opts) do + {:ok, event} -> + emit_telemetry_buffered(event) + event + + {:error, reason} -> + drop(event, reason) + end + end + + @click_id_params ["gclid", "gbraid", "wbraid", "msclkid", "fbclid", "twclid"] + + defp get_click_id_param(nil), do: nil + + defp get_click_id_param(query_params) do + @click_id_params + |> Enum.find(fn param_name -> Map.has_key?(query_params, param_name) end) + end + + @parse_user_agent_timeout 200 + defp parse_user_agent(%Request{user_agent: user_agent}) when is_binary(user_agent) do + Plausible.Cache.Adapter.fetch(:user_agents, user_agent, fn -> + parse_user_agent_safe(user_agent) + end) + end + + defp parse_user_agent(request), do: request + + defp parse_user_agent_safe(user_agent) do + task = + Task.Supervisor.async_nolink( + {:via, PartitionSupervisor, {Plausible.UserAgentParseTaskSupervisor, self()}}, + fn -> + UAInspector.parse(user_agent) + end + ) + + case Task.yield(task, @parse_user_agent_timeout) || Task.shutdown(task) do + {:ok, result} -> + {:ok, result} + + nil -> + emit_telemetry_ua_parse_timeout() + {:error, :timeout} + end + end + + defp browser_name(ua) do + case ua.client do + :unknown -> "" + %UAInspector.Result.Client{name: "Mobile Safari"} -> "Safari" + %UAInspector.Result.Client{name: "Chrome Mobile"} -> "Chrome" + %UAInspector.Result.Client{name: "Chrome Mobile iOS"} -> "Chrome" + %UAInspector.Result.Client{name: "Firefox Mobile"} -> "Firefox" + %UAInspector.Result.Client{name: "Firefox Mobile iOS"} -> "Firefox" + %UAInspector.Result.Client{name: "Opera Mobile"} -> "Opera" + %UAInspector.Result.Client{name: "Opera Mini"} -> "Opera" + %UAInspector.Result.Client{name: "Opera Mini iOS"} -> "Opera" + %UAInspector.Result.Client{name: "Yandex Browser Lite"} -> "Yandex Browser" + %UAInspector.Result.Client{name: "Chrome Webview"} -> "Mobile App" + %UAInspector.Result.Client{type: "mobile app"} -> "Mobile App" + client -> client.name + end + end + + @mobile_types [ + "smartphone", + "feature phone", + "portable media player", + "phablet", + "wearable", + "camera" + ] + @tablet_types ["car browser", "tablet"] + @desktop_types ["tv", "console", "desktop"] + alias UAInspector.Result.Device + + defp screen_size(ua) do + case ua.device do + %Device{type: t} when t in @mobile_types -> + "Mobile" + + %Device{type: t} when t in @tablet_types -> + "Tablet" + + %Device{type: t} when t in @desktop_types -> + "Desktop" + + _ -> + nil + end + end + + defp browser_version(ua) do + case ua.client do + :unknown -> "" + %UAInspector.Result.Client{type: "mobile app"} -> "" + client -> major_minor(client.version) + end + end + + defp os_name(ua) do + case ua.os do + :unknown -> "" + os -> os.name + end + end + + defp os_version(ua) do + case ua.os do + :unknown -> "" + os -> major_minor(os.version) + end + end + + defp major_minor(version) do + case version do + :unknown -> + "" + + version -> + version + |> String.split(".") + |> Enum.take(2) + |> Enum.join(".") + end + end + + defp generate_user_id(request, domain, hostname, salt) do + cond do + is_nil(salt) -> + nil + + is_nil(domain) -> + nil + + true -> + user_agent = request.user_agent || "" + root_domain = get_root_domain(hostname) + + SipHash.hash!(salt, user_agent <> request.remote_ip <> domain <> root_domain) + end + end + + defp get_root_domain("(none)"), do: "(none)" + + defp get_root_domain(hostname) do + case :inet.parse_ipv4_address(String.to_charlist(hostname)) do + {:ok, _} -> + hostname + + {:error, :einval} -> + PublicSuffix.registrable_domain(hostname) || hostname + end + end + + defp spam_referrer?(%Request{referrer: referrer}) when is_binary(referrer) do + URI.parse(referrer).host + |> Request.sanitize_hostname() + |> ReferrerBlocklist.is_spammer?() + end + + defp spam_referrer?(_), do: false +end diff --git a/lib/plausible/ingestion/geolocation.ex b/lib/plausible/ingestion/geolocation.ex new file mode 100644 index 0000000000000000000000000000000000000000..43a564de63da1349fa9e992d9e7d7c12f92bd2c4 --- /dev/null +++ b/lib/plausible/ingestion/geolocation.ex @@ -0,0 +1,53 @@ +defmodule Plausible.Ingestion.Geolocation do + @moduledoc false + + def lookup(ip_address) do + case Plausible.Geo.lookup(ip_address) do + %{} = entry -> + country_code = + entry + |> get_in(["country", "iso_code"]) + |> ignore_unknown_country() + + city_geoname_id = country_code && get_in(entry, ["city", "geoname_id"]) + city_geoname_id = Plausible.Ingestion.CityOverrides.get(city_geoname_id, city_geoname_id) + + %{ + country_code: country_code, + subdivision1_code: subdivision1_code(country_code, entry), + subdivision2_code: subdivision2_code(country_code, entry), + city_geoname_id: city_geoname_id + } + + nil -> + nil + end + end + + defp subdivision1_code(country_code, %{"subdivisions" => [%{"iso_code" => iso_code} | _rest]}) + when not is_nil(country_code) do + country_code <> "-" <> iso_code + end + + defp subdivision1_code(_, _), do: nil + + defp subdivision2_code(country_code, %{ + "subdivisions" => [_first, %{"iso_code" => iso_code} | _rest] + }) + when not is_nil(country_code) do + country_code <> "-" <> iso_code + end + + defp subdivision2_code(_, _), do: nil + + @ignored_countries [ + # Worldwide + "ZZ", + # Disputed territory + "XX", + # Tor exit node + "T1" + ] + defp ignore_unknown_country(code) when code in @ignored_countries, do: nil + defp ignore_unknown_country(country), do: country +end diff --git a/lib/plausible/ingestion/persistor.ex b/lib/plausible/ingestion/persistor.ex new file mode 100644 index 0000000000000000000000000000000000000000..6e8b8f3a72072827061cb942ec3050d3163c54c0 --- /dev/null +++ b/lib/plausible/ingestion/persistor.ex @@ -0,0 +1,39 @@ +defmodule Plausible.Ingestion.Persistor do + @moduledoc """ + Registers and persists sessions and events. + """ + + @fallback_backend Plausible.Ingestion.Persistor.Embedded + + def persist_event(event, previous_user_id, opts) do + {backend_override, opts} = Keyword.pop(opts, :backend) + user_id = event.clickhouse_event.user_id + + backend(backend_override, user_id).persist_event(event, previous_user_id, opts) + end + + defp backend(nil, user_id) do + percent_enabled = + :plausible + |> Application.fetch_env!(__MODULE__) + |> Keyword.fetch!(:backend_percent_enabled) + + backend = + :plausible + |> Application.fetch_env!(__MODULE__) + |> Keyword.fetch!(:backend) + + cond do + backend == @fallback_backend or percent_enabled >= 100 -> + backend + + :erlang.phash2(user_id, 100) + 1 <= percent_enabled -> + backend + + true -> + @fallback_backend + end + end + + defp backend(override, _user_id), do: override +end diff --git a/lib/plausible/ingestion/persistor/embedded.ex b/lib/plausible/ingestion/persistor/embedded.ex new file mode 100644 index 0000000000000000000000000000000000000000..b11c471ada7fb5fc0c7530594271a579d2dab7e8 --- /dev/null +++ b/lib/plausible/ingestion/persistor/embedded.ex @@ -0,0 +1,40 @@ +defmodule Plausible.Ingestion.Persistor.Embedded do + @moduledoc """ + Embedded implementation of session and event persistence. + """ + + alias Plausible.ClickhouseEventV2 + + def persist_event(ingest_event, previous_user_id, opts) do + event = ingest_event.clickhouse_event + session_attrs = ingest_event.clickhouse_session_attrs + + session_write_buffer_insert = + Keyword.get(opts, :session_write_buffer_insert, &Plausible.Session.WriteBuffer.insert/1) + + event_write_buffer_insert = + Keyword.get(opts, :event_write_buffer_insert, &Plausible.Event.WriteBuffer.insert/1) + + session_result = + Plausible.Session.CacheStore.on_event( + event, + session_attrs, + previous_user_id, + buffer_insert: session_write_buffer_insert + ) + + case session_result do + {:ok, :no_session_for_engagement} -> + {:error, :no_session_for_engagement} + + {:error, :timeout} -> + {:error, :lock_timeout} + + {:ok, session} -> + event = ClickhouseEventV2.merge_session(event, session) + {:ok, _} = event_write_buffer_insert.(event) + + {:ok, %{ingest_event | clickhouse_event: event}} + end + end +end diff --git a/lib/plausible/ingestion/persistor/embedded_with_relay.ex b/lib/plausible/ingestion/persistor/embedded_with_relay.ex new file mode 100644 index 0000000000000000000000000000000000000000..0fff74aeb94f74cbb7feab8a73a89088a7604095 --- /dev/null +++ b/lib/plausible/ingestion/persistor/embedded_with_relay.ex @@ -0,0 +1,86 @@ +defmodule Plausible.Ingestion.Persistor.EmbeddedWithRelay do + @moduledoc """ + Embedded implementation with async relay to remote. + """ + + alias Plausible.Ingestion.Persistor + + def persist_event(event, previous_user_id, opts) do + {sync?, opts} = Keyword.pop(opts, :sync?, false) + + if sync? do + persist_remote_event_sync(event, previous_user_id, opts) + else + persist_remote_event_async(event, previous_user_id, opts) + end + + Persistor.Embedded.persist_event(event, previous_user_id, opts) + end + + defp persist_remote_event_async(event, previous_user_id, opts) do + ctx = OpenTelemetry.Ctx.get_current() + + Task.start(fn -> + OpenTelemetry.Ctx.attach(ctx) + + Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration( + telemetry_pipeline_step_duration(), + fn -> do_persist_event(event, previous_user_id, opts) end, + %{step: "remote_register_session"} + ) + end) + end + + defp persist_remote_event_sync(event, previous_user_id, opts) do + Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration( + telemetry_pipeline_step_duration(), + fn -> do_persist_event(event, previous_user_id, opts) end, + %{step: "remote_register_session"} + ) + end + + defp do_persist_event(event, previous_user_id, opts) do + result = Persistor.Remote.persist_event(event, previous_user_id, opts) + + case result do + {:ok, event} -> + emit_telemetry_buffered(event) + + {:error, reason} -> + emit_telemetry_dropped(event, reason) + end + end + + def emit_telemetry_buffered(event) do + :telemetry.execute(telemetry_event_buffered(), %{}, %{ + domain: event.domain, + request_timestamp: event.request.timestamp, + tracker_script_version: event.request.tracker_script_version + }) + end + + def emit_telemetry_dropped(event, reason) do + :telemetry.execute( + telemetry_event_dropped(), + %{}, + %{ + domain: event.domain, + reason: reason, + request_timestamp: event.request.timestamp, + tracker_script_version: event.request.tracker_script_version + } + ) + end + + def telemetry_event_buffered() do + [:plausible, :remote_ingest, :event, :buffered] + end + + def telemetry_event_dropped() do + [:plausible, :remote_ingest, :event, :dropped] + end + + def telemetry_pipeline_step_duration() do + [:plausible, :remote_ingest, :pipeline, :step] + end +end diff --git a/lib/plausible/ingestion/persistor/remote.ex b/lib/plausible/ingestion/persistor/remote.ex new file mode 100644 index 0000000000000000000000000000000000000000..afddbe682172d327c2be22aae0fcf4badd38e525 --- /dev/null +++ b/lib/plausible/ingestion/persistor/remote.ex @@ -0,0 +1,163 @@ +defmodule Plausible.Ingestion.Persistor.Remote do + @moduledoc """ + Remote implementation of session and event persistence. + """ + + require Logger + require OpenTelemetry.Tracer + + @max_transient_retries 3 + + def persist_event(ingest_event, previous_user_id, opts) do + event = ingest_event.clickhouse_event + session_attrs = ingest_event.clickhouse_session_attrs + + site_id = event.site_id + current_user_id = event.user_id + override_url = Keyword.get(opts, :url) + + headers = [ + {"x-site-id", site_id}, + {"x-current-user-id", current_user_id}, + {"x-previous-user-id", previous_user_id} + ] + + case request(override_url, event, session_attrs, headers) do + {:ok, %{status: 200, body: event_payload}} -> + case decode_payload(event_payload) do + {:ok, event} -> + {:ok, %{ingest_event | clickhouse_event: event}} + + {:error, decode_error} -> + log_error(site_id, current_user_id, previous_user_id, decode_error) + {:error, :persist_decode_error} + end + + {:ok, %{body: "no_session_for_engagement"}} -> + {:error, :no_session_for_engagement} + + {:ok, %{body: error}} -> + log_error(site_id, current_user_id, previous_user_id, error) + {:error, decode_error(error)} + + {:error, %{reason: :timeout}} -> + {:error, :persist_timeout} + + {:error, error} -> + log_error(site_id, current_user_id, previous_user_id, error) + + {:error, :persist_error} + end + end + + def telemetry_request_duration() do + [:plausible, :remote_ingest, :request, :duration] + end + + def telemetry_decode_duration() do + [:plausible, :remote_ingest, :decode, :duration] + end + + defp request(override_url, event, session_attrs, headers) do + Plausible.PromEx.Plugins.PlausibleMetrics.measure_duration( + telemetry_request_duration(), + fn -> + url = persistor_url(override_url) + + OpenTelemetry.Tracer.with_span "persistor.remote.request", + %{attributes: persistor_span_attributes(headers)} do + result = + Req.new( + finch: Plausible.Finch, + body: encode_payload(event, session_attrs), + headers: headers, + retry: &handle_transient_error/2, + max_retries: @max_transient_retries + ) + |> OpentelemetryReq.attach(propagate_trace_headers: true) + |> Req.post(url: url, span_name: "persistor.remote.post") + + trace_result(result) + result + end + end + ) + end + + defp handle_transient_error(_request, %Req.HTTPError{protocol: :http2, reason: :disconnected}) do + true + end + + defp handle_transient_error(_request, %Req.HTTPError{protocol: :http2, reason: :unprocessed}) do + true + end + + defp handle_transient_error(_reqeust, _response), do: false + + defp persistor_span_attributes(headers) do + site_id = find_header(headers, "x-site-id") + user_id = find_header(headers, "x-current-user-id") + + [{"plausible.persistor.site_id", site_id}, {"plausible.persistor.user_id", user_id}] + end + + defp find_header(headers, name) do + case List.keyfind(headers, name, 0) do + {_, value} -> value + nil -> "" + end + end + + defp trace_result({:ok, %{status: status}}) do + OpenTelemetry.Tracer.set_attributes([{"http.response.status_code", status}]) + end + + defp trace_result({:error, error}) do + OpenTelemetry.Tracer.set_status(:error, inspect(error)) + end + + defp encode_payload(event, session_attrs) do + event_data = + event + |> Map.from_struct() + |> Map.delete(:__meta__) + + {event_data, session_attrs} + |> :erlang.term_to_binary() + |> Base.encode64(padding: false) + end + + defp decode_payload(payload) do + case Base.decode64(payload, padding: false) do + {:ok, data} -> + event_data = :erlang.binary_to_term(data) + event = struct(Plausible.ClickhouseEventV2, event_data) + + {:ok, event} + + _ -> + {:error, :invalid_web_encoding} + end + catch + _, _ -> + {:error, :invalid_payload} + end + + defp decode_error("no_session_for_engagement"), do: :no_session_for_engagement + defp decode_error("lock_timeout"), do: :lock_timeout + defp decode_error(_), do: :persist_error + + defp persistor_url(nil) do + Keyword.fetch!(Application.fetch_env!(:plausible, __MODULE__), :url) + end + + defp persistor_url(url) when is_binary(url) do + url + end + + defp log_error(site_id, current_user_id, previous_user_id, error) do + Logger.warning( + "Persisting event for (#{site_id};#{current_user_id},#{previous_user_id}) failed: #{inspect(error)}" + ) + end +end diff --git a/lib/plausible/ingestion/persistor/telemetry_handler.ex b/lib/plausible/ingestion/persistor/telemetry_handler.ex new file mode 100644 index 0000000000000000000000000000000000000000..7d95c34973965c9b97b843de701967bbb7f0a69a --- /dev/null +++ b/lib/plausible/ingestion/persistor/telemetry_handler.ex @@ -0,0 +1,177 @@ +defmodule Plausible.Ingestion.Persistor.TelemetryHandler do + @moduledoc """ + Sets up telemetry for remote calls to persistor via `Finch.Telemetry` events. + + Emits persistor specific telemetry events for tracking metrics. + """ + + @finch_request_event [:finch, :request, :stop] + @finch_connect_event [:finch, :connect, :stop] + @finch_send_event [:finch, :send, :stop] + @finch_receive_event [:finch, :recv, :stop] + + @persistor_request_event [:persistor, :remote, :request] + @persistor_connect_event [:persistor, :remote, :connect] + @persistor_send_event [:persistor, :remote, :send] + @persistor_receive_event [:persistor, :remote, :receive] + + @telemetry_events [ + @finch_request_event, + @finch_connect_event, + @finch_send_event, + @finch_receive_event + ] + + @telemetry_handler &__MODULE__.handle_event/4 + + def request_event(), do: @persistor_request_event + def connect_event(), do: @persistor_connect_event + def send_event(), do: @persistor_send_event + def receive_event(), do: @persistor_receive_event + + @spec install() :: :ok + def install() do + if persistor_backend() in [ + Plausible.Ingestion.Persistor.Remote, + Plausible.Ingestion.Persistor.EmbeddedWithRelay + ] do + persistor_host = + persistor_url() + |> URI.parse() + |> Map.fetch!(:host) + + persistor_count = persistor_count() + + if is_binary(persistor_host) do + :ok = + :telemetry.attach_many( + "persistor-remote-finch-metrics", + @telemetry_events, + @telemetry_handler, + %{remote_host: persistor_host, pool_size: persistor_count} + ) + else + :ok + end + else + :ok + end + end + + @spec handle_event([atom()], any(), map(), any()) :: :ok + def handle_event( + @finch_request_event, + %{duration: duration}, + %{request: request, result: result}, + config + ) do + if request.host == config.remote_host do + result = + case result do + {:ok, %Finch.Response{status: status}} when status >= 200 and status < 300 -> + "ok" + + {:ok, %Finch.Response{}} -> + "bad_status" + + {:ok, _} -> + "stream" + + {:error, %{reason: :timeout}} -> + "timeout_error" + + {:error, _} -> + "unknown_error" + end + + :telemetry.execute( + @persistor_request_event, + %{duration: duration}, + %{result: result, path: normalize_path(request.path)} + ) + end + + :ok + end + + def handle_event( + @finch_connect_event, + %{duration: duration}, + %{host: host} = meta, + config + ) do + if host == config.remote_host do + :telemetry.execute( + @persistor_connect_event, + %{duration: duration}, + %{status: if(meta[:error], do: "error", else: "ok")} + ) + end + + :ok + end + + def handle_event( + @finch_send_event, + %{duration: duration}, + %{request: request} = meta, + config + ) do + if request.host == config.remote_host do + :telemetry.execute( + @persistor_send_event, + %{duration: duration}, + %{status: if(meta[:error], do: "error", else: "ok")} + ) + end + + :ok + end + + def handle_event( + @finch_receive_event, + %{duration: duration}, + %{request: request} = meta, + config + ) do + if request.host == config.remote_host do + status = meta[:status] || 0 + + :telemetry.execute( + @persistor_receive_event, + %{duration: duration}, + %{ + status: if(meta[:error] || status < 200 || status > 299, do: "error", else: "ok") + } + ) + end + + :ok + end + + defp normalize_path(path) do + if path == "/event" do + "/event" + else + "unknown" + end + end + + defp persistor_url() do + :plausible + |> Application.fetch_env!(Plausible.Ingestion.Persistor.Remote) + |> Keyword.fetch!(:url) + end + + defp persistor_backend() do + :plausible + |> Application.fetch_env!(Plausible.Ingestion.Persistor) + |> Keyword.fetch!(:backend) + end + + defp persistor_count() do + :plausible + |> Application.fetch_env!(Plausible.Ingestion.Persistor.Remote) + |> Keyword.fetch!(:count) + end +end diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex new file mode 100644 index 0000000000000000000000000000000000000000..e8b15a97f1527630c9fa99ef95ea717fa5a37202 --- /dev/null +++ b/lib/plausible/ingestion/request.ex @@ -0,0 +1,433 @@ +defmodule Plausible.Ecto.EventName do + @moduledoc """ + Custom type for event name. Accepts Strings and Integers and stores them as String. Returns + cast error if any other type is provided. Accepting integers is important for 404 tracking. + """ + + use Ecto.Type + def type, do: :string + + def cast(val) when is_binary(val), do: {:ok, val} + def cast(val) when is_integer(val), do: {:ok, Integer.to_string(val)} + + def cast(_), do: :error + def load(val), do: {:ok, val} + def dump(val), do: {:ok, val} +end + +defmodule Plausible.Ingestion.Request do + @moduledoc """ + The %Plausible.Ingestion.Request{} struct stores all needed fields + to create an event downstream. Pre-eliminary validation is made + to detect user errors early. + """ + + use Ecto.Schema + use Plausible + alias Ecto.Changeset + + @max_url_size 2_000 + @missing_scroll_depth 255 + @missing_engagement_time 0 + + # :KLUDGE: Old version of tracker script sent huge values for engagement time. Ignore + # these while users might still have the old script cached. + @too_large_engagement_time :timer.hours(30 * 24) + + @blank_engagement_error_message "engagement event requires a valid integer value for at least one of 'sd' or 'e' fields" + + def too_large_engagement_time(), do: @too_large_engagement_time + def blank_engagement_error_message(), do: @blank_engagement_error_message + + @primary_key false + embedded_schema do + field :remote_ip, :string + field :user_agent, :string + field :event_name, Plausible.Ecto.EventName + field :uri, :map + field :hostname, :string + field :referrer, :string + field :domains, {:array, :string} + field :ip_classification, :string + field :hash_mode, :integer + field :pathname, :string + field :props, :map + field :scroll_depth, :integer + field :engagement_time, :integer + field :tracker_script_version, :integer, default: 0 + field :interactive?, :boolean, default: true + + on_ee do + field :revenue_source, :map + end + + field :query_params, :map + + field :timestamp, :naive_datetime + end + + @type t() :: %__MODULE__{} + + @spec build(Plug.Conn.t(), NaiveDateTime.t()) :: + {:ok, t(), Plug.Conn.t()} | {:error, Changeset.t()} + @doc """ + Builds and initially validates %Plausible.Ingestion.Request{} struct from %Plug.Conn{}. + """ + def build(%Plug.Conn{} = conn, now \\ NaiveDateTime.utc_now()) do + changeset = + %__MODULE__{} + |> Changeset.change() + |> Changeset.put_change( + :timestamp, + NaiveDateTime.truncate(now, :second) + ) + + case parse_body(conn) do + {:ok, request_body, conn} -> + request = + changeset + |> put_ip_classification(conn) + |> put_remote_ip(conn) + |> put_uri(request_body) + |> put_hostname() + |> put_user_agent(conn) + |> put_request_params(request_body) + |> put_referrer(request_body) + |> put_pathname() + |> put_props(request_body) + |> put_engagement_fields(request_body) + |> put_query_params() + |> put_revenue_source(request_body) + |> put_interactive(request_body) + |> put_tracker_script_version(request_body) + |> map_domains(request_body) + |> Changeset.validate_required([ + :event_name, + :hostname, + :pathname, + :timestamp + ]) + |> Changeset.validate_length(:event_name, max: 120) + |> Changeset.apply_action(nil) + + case request do + {:ok, request} -> + {:ok, request, conn} + + {:error, _} = error -> + error + end + + {:error, :invalid_json} -> + {:error, Changeset.add_error(changeset, :request, "Unable to parse request body as json")} + end + end + + on_ee do + defp put_revenue_source(changeset, request_body) do + Plausible.Ingestion.Request.Revenue.put_revenue_source(changeset, request_body) + end + else + defp put_revenue_source(changeset, _request_body), do: changeset + end + + defp put_remote_ip(changeset, conn) do + Changeset.put_change(changeset, :remote_ip, PlausibleWeb.RemoteIP.get(conn)) + end + + defp parse_body(conn) do + case conn.body_params do + %Plug.Conn.Unfetched{} -> + with max_length <- conn.assigns[:read_body_limit] || 1_000_000, + {:ok, body, conn} <- + Plug.Conn.read_body(conn, length: max_length, read_length: max_length), + {:ok, params} when is_map(params) <- Jason.decode(body) do + {:ok, params, conn} + else + _ -> {:error, :invalid_json} + end + + params -> + {:ok, params, conn} + end + end + + defp put_request_params(changeset, %{} = request_body) do + Changeset.cast( + changeset, + %{ + event_name: request_body["n"] || request_body["name"], + hash_mode: request_body["h"] || request_body["hashMode"] + }, + [:event_name, :hash_mode] + ) + end + + defp put_referrer(changeset, %{} = request_body) do + referrer = request_body["r"] || request_body["referrer"] + + if is_binary(referrer) do + referrer = String.slice(referrer, 0..(@max_url_size - 1)) + Changeset.put_change(changeset, :referrer, referrer) + else + changeset + end + end + + defp put_pathname(changeset) do + uri = Changeset.get_field(changeset, :uri) + hash_mode = Changeset.get_field(changeset, :hash_mode) + pathname = get_pathname(uri, hash_mode) + Changeset.put_change(changeset, :pathname, pathname) + end + + defp maybe_set_props_path_to_pathname(props_in_request, changeset) do + if Plausible.Event.SystemEvents.sync_props_path_with_pathname?( + Changeset.get_field(changeset, :event_name), + props_in_request + ) do + # "path" props is added to the head of the props enum to avoid it being cut off + [{"path", Changeset.get_field(changeset, :pathname)}] ++ props_in_request + else + props_in_request + end + end + + defp map_domains(changeset, %{} = request_body) do + raw = request_body["d"] || request_body["domain"] + raw = if is_binary(raw), do: String.trim(raw) + + case raw do + "" -> + Changeset.add_error(changeset, :domain, "can't be blank") + + raw when is_binary(raw) -> + domains = + raw + |> String.split(",") + |> Enum.map(&sanitize_hostname/1) + + Changeset.put_change(changeset, :domains, domains) + + nil -> + from_uri = sanitize_hostname(Changeset.get_field(changeset, :uri)) + + if from_uri do + Changeset.put_change(changeset, :domains, [from_uri]) + else + Changeset.add_error(changeset, :domain, "can't be blank") + end + end + end + + @disallowed_schemes ~w(data) + defp put_uri(changeset, %{} = request_body) do + with url when is_binary(url) <- request_body["u"] || request_body["url"], + url when byte_size(url) <= @max_url_size <- url, + %URI{} = uri when uri.scheme not in @disallowed_schemes <- URI.parse(url) do + Changeset.put_change(changeset, :uri, uri) + else + nil -> Changeset.add_error(changeset, :url, "is required") + %URI{} -> Changeset.add_error(changeset, :url, "scheme is not allowed") + _ -> Changeset.add_error(changeset, :url, "must be a valid url") + end + end + + defp put_hostname(changeset) do + host = + case Changeset.get_field(changeset, :uri) do + %{host: host} when is_binary(host) and host != "" -> sanitize_hostname(host) + _ -> "(none)" + end + + Changeset.put_change(changeset, :hostname, host) + end + + @max_props 30 + defp put_props(changeset, %{} = request_body) do + props = + (request_body["m"] || request_body["meta"] || request_body["p"] || request_body["props"]) + |> Plausible.Helpers.JSON.decode_or_fallback() + |> Enum.reduce([], &filter_bad_props/2) + |> maybe_set_props_path_to_pathname(changeset) + |> Enum.take(@max_props) + |> Map.new() + + changeset + |> Changeset.put_change(:props, props) + |> validate_props() + end + + defp put_interactive(changeset, %{} = request_body) do + can_be_marked_as_non_interactive? = + Changeset.get_field(changeset, :event_name) not in Plausible.Event.SystemEvents.events_with_interactive_always_true() + + if can_be_marked_as_non_interactive? and + (request_body["i"] == false or + request_body["interactive"] == false) do + Changeset.put_change(changeset, :interactive?, false) + else + changeset + end + end + + defp filter_bad_props({k, v}, acc) do + cond do + Enum.any?([k, v], &(is_list(&1) or is_map(&1))) -> acc + Enum.any?([k, v], &(String.trim_leading(to_string(&1)) == "")) -> acc + true -> [{to_string(k), to_string(v)} | acc] + end + end + + @max_prop_key_length Plausible.Props.max_prop_key_length() + @max_prop_value_length Plausible.Props.max_prop_value_length() + defp validate_props(changeset) do + case Changeset.get_field(changeset, :props) do + props -> + Enum.reduce_while(props, changeset, fn + {key, value}, changeset + when byte_size(key) > @max_prop_key_length or + byte_size(value) > @max_prop_value_length -> + {:halt, + Changeset.add_error( + changeset, + :props, + "keys should have at most #{@max_prop_key_length} bytes and values #{@max_prop_value_length} bytes" + )} + + _, changeset -> + {:cont, changeset} + end) + end + end + + defp put_engagement_fields(changeset, %{} = request_body) do + if Changeset.get_field(changeset, :event_name) == "engagement" do + scroll_depth = parse_scroll_depth(request_body["sd"]) + engagement_time = parse_engagement_time(request_body["e"]) + + case {scroll_depth, engagement_time} do + {@missing_scroll_depth, @missing_engagement_time} -> + changeset + |> Changeset.add_error( + :event_name, + "engagement event requires a valid integer value for at least one of 'sd' or 'e' fields" + ) + + _ -> + changeset + |> Changeset.put_change(:scroll_depth, scroll_depth) + |> Changeset.put_change(:engagement_time, engagement_time) + end + else + changeset + end + end + + defp put_tracker_script_version(changeset, %{} = request_body) do + case request_body["v"] do + version when is_integer(version) -> + Changeset.put_change(changeset, :tracker_script_version, version) + + _ -> + changeset + end + end + + defp put_query_params(changeset) do + case Changeset.get_field(changeset, :uri) do + %{query: query} when is_binary(query) -> + Changeset.put_change(changeset, :query_params, URI.decode_query(query)) + + _any -> + changeset + end + end + + defp put_ip_classification(changeset, %Plug.Conn{} = conn) do + value = + conn + |> Plug.Conn.get_req_header("x-plausible-ip-type") + |> List.first() + + Changeset.put_change(changeset, :ip_classification, value) + end + + defp put_user_agent(changeset, %Plug.Conn{} = conn) do + user_agent = + conn + |> Plug.Conn.get_req_header("user-agent") + |> List.first() + + Changeset.put_change(changeset, :user_agent, user_agent) + end + + defp get_pathname(nil, _hash_mode), do: "/" + + defp get_pathname(uri, hash_mode) do + pathname = + (uri.path || "/") + |> URI.decode() + |> String.trim_trailing() + + if hash_mode == 1 && uri.fragment do + pathname <> "#" <> URI.decode(uri.fragment) + else + pathname + end + end + + @doc """ + Removes the "www" part of a hostname. + """ + def sanitize_hostname(%URI{host: hostname}) do + sanitize_hostname(hostname) + end + + def sanitize_hostname(hostname) when is_binary(hostname) do + hostname + |> String.trim() + |> String.replace_prefix("www.", "") + end + + def sanitize_hostname(nil) do + nil + end + + defp parse_scroll_depth(sd) when is_binary(sd) do + case Integer.parse(sd) do + {sd_int, ""} -> parse_scroll_depth(sd_int) + _ -> @missing_scroll_depth + end + end + + defp parse_scroll_depth(sd) when is_integer(sd) and sd >= 0 and sd <= 100, do: sd + defp parse_scroll_depth(sd) when is_integer(sd) and sd > 100, do: 100 + defp parse_scroll_depth(_), do: @missing_scroll_depth + + defp parse_engagement_time(et) when is_binary(et) do + case Integer.parse(et) do + {et_int, ""} -> parse_engagement_time(et_int) + _ -> @missing_engagement_time + end + end + + defp parse_engagement_time(et) + when is_integer(et) and et >= 0 and et < @too_large_engagement_time, + do: et + + defp parse_engagement_time(_), do: @missing_engagement_time +end + +defimpl Jason.Encoder, for: URI do + def encode(uri, _opts), do: [?", URI.to_string(uri), ?"] +end + +defimpl Jason.Encoder, for: Plausible.Ingestion.Request do + @fields Plausible.Ingestion.Request.__schema__(:fields) + def encode(request, opts) do + request + |> Map.take(@fields) + |> Jason.Encode.map(opts) + end +end diff --git a/lib/plausible/ingestion/source.ex b/lib/plausible/ingestion/source.ex new file mode 100644 index 0000000000000000000000000000000000000000..d80bbf2548e924053f8b16a42ad35c65262069d8 --- /dev/null +++ b/lib/plausible/ingestion/source.ex @@ -0,0 +1,159 @@ +defmodule Plausible.Ingestion.Source do + @moduledoc """ + Resolves the `source` dimension from a combination of `referer` header and either `utm_source`, `source`, or `ref` query parameter. + + """ + alias Plausible.Ingestion.Request + + @external_resource "priv/custom_sources.json" + @custom_sources Application.app_dir(:plausible, "priv/custom_sources.json") + |> File.read!() + |> Jason.decode!() + + @paid_sources Map.keys(@custom_sources) + |> Enum.filter(&String.ends_with?(&1, ["ads", "ad"])) + |> then(&["adwords" | &1]) + |> MapSet.new() + + @custom_source_suffixes %{ + "officeapps.live.com" => "Microsoft 365", + "wikipedia.org" => "Wikipedia" + } + + @external_resource "priv/ref_inspector/referers.yml" + @referers_yaml Application.app_dir(:plausible, "priv/ref_inspector/referers.yml") + + yaml_reader = RefInspector.Config.yaml_file_reader() + {:ok, db} = RefInspector.Database.Loader.load(@referers_yaml, yaml_reader) + db = RefInspector.Database.Parser.parse(db) + + lookup = + Enum.reduce(db, Map.new(), fn {_, entries}, lookup -> + Enum.reduce(entries, lookup, fn {_, _, _, _, _, _, name}, lookup_inner -> + Map.put(lookup_inner, String.downcase(name), name) + end) + end) + + lookup = + Enum.reduce(@custom_sources, lookup, fn {key, val}, lookup -> + lookup + |> Map.put(key, val) + |> Map.put(String.downcase(val), val) + end) + + @spec from_custom_sources(String.t()) :: String.t() | nil + for {k, v} <- Enum.sort(lookup) do + def from_custom_sources(unquote(k)), do: unquote(v) + end + + def from_custom_sources(_), do: nil + + def paid_sources() do + @paid_sources |> MapSet.to_list() + end + + def paid_source?(source) do + MapSet.member?(@paid_sources, source) + end + + @doc """ + Resolves the source of a session based on query params and the `Referer` header. + + When a query parameter like `utm_source` is present, it will be prioritized over the `Referer` header. When the URL does not contain a source tag, we fall + back to using `Referer` to determine the source. This module also takes care of certain transformations to make the data more useful for the user: + 1. The RefInspector library is used to categorize referrers into "known" sources. For example, when the referrer is google.com or google.co.uk, + it will always be stored as "Google" which is more useful for marketers. + 2. On top of the standard RefInspector behaviour, we also keep a list of `custom_sources.json` which extends it with referrers that we have seen in the wild. + For example, Wikipedia has many domains that need to be combined into a single known source. These could all in theory be [upstreamed](https://github.com/snowplow-referer-parser/referer-parser). + 3. When a known source is supplied in utm_source (or source, ref) query parameter, we merge it with our known sources in a case-insensitive manner. + 4. Our list of `custom_sources.json` also contains some commonly used utm_source shorthands for certain sources. URL tagging is a mess, and we can never do it + perfectly, but at least we're making an effort for the most commonly used ones. For example, `ig -> Instagram` and `adwords -> Google`. + """ + def resolve(request) do + cond do + tagged = tagged_param(request) -> canonical(tagged) + has_valid_referral?(request) -> from_referrer(request.referrer) + true -> nil + end + end + + @doc """ + Resolves a source from a bare referrer URL. This is also the entry point used + when importing referrers from Google Analytics, so that imported data + is normalized identically to live ingestion. + """ + def from_referrer(referrer) do + host = + referrer + |> String.trim() + |> URI.parse() + |> format_referrer_host() + + downcased_host = String.downcase(host) + + # Prefer custom source overrides over RefInspector so subdomain matches win + # (e.g. gemini.google.com resolves to Google Gemini, not Google). Exact host + # matches take priority, then suffix families, then RefInspector. + cond do + name = from_custom_sources(downcased_host) -> + name + + name = from_custom_source_suffix(downcased_host) -> + name + + true -> + case RefInspector.parse(referrer).source do + :unknown -> host + # Normalize RefInspector names through our aliases (e.g. Twitter to X (Twitter)). + name -> canonical(name) + end + end + end + + defp from_custom_source_suffix(host) do + Enum.find_value(@custom_source_suffixes, fn {suffix, name} -> + if host == suffix or String.ends_with?(host, "." <> suffix), do: name + end) + end + + defp tagged_param(request) do + request.query_params["utm_source"] || + request.query_params["source"] || + request.query_params["ref"] + end + + defp canonical(source) do + from_custom_sources(String.downcase(source)) || source + end + + def format_referrer(request) do + if has_valid_referral?(request) do + referrer_uri = URI.parse(request.referrer) + path = String.trim_trailing(referrer_uri.path || "", "/") + format_referrer_host(referrer_uri) <> path + end + end + + defp has_valid_referral?(%Request{referrer: nil}), do: false + + defp has_valid_referral?(%Request{referrer: referrer, uri: uri}) do + referrer_uri = URI.parse(referrer) + + valid_scheme? = referrer_uri.scheme in ["http", "https", "android-app"] + valid_host? = !is_nil(referrer_uri.host) && byte_size(referrer_uri.host) > 0 + + internal? = + Request.sanitize_hostname(referrer_uri.host) == Request.sanitize_hostname(uri.host) + + local? = referrer_uri.host == "localhost" + + valid_scheme? and valid_host? and not internal? and not local? + end + + defp format_referrer_host(uri) do + protocol = if uri.scheme == "android-app", do: "android-app://", else: "" + host = String.replace_prefix(uri.host, "www.", "") + + protocol <> host + end +end diff --git a/lib/plausible/ingestion/write_buffer.ex b/lib/plausible/ingestion/write_buffer.ex new file mode 100644 index 0000000000000000000000000000000000000000..26ace34dd0fe6d56b38ae1db971164400b7a79bb --- /dev/null +++ b/lib/plausible/ingestion/write_buffer.ex @@ -0,0 +1,159 @@ +defmodule Plausible.Ingestion.WriteBuffer do + @moduledoc false + use GenServer + require Logger + + alias Plausible.IngestRepo + + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: Keyword.fetch!(opts, :name)) + end + + def insert(server, row_binary) do + GenServer.cast(server, {:insert, row_binary}) + end + + def flush(server) do + GenServer.call(server, :flush, :infinity) + end + + @impl true + def init(opts) do + buffer = opts[:buffer] || [] + max_buffer_size = opts[:max_buffer_size] || default_max_buffer_size() + flush_interval_ms = opts[:flush_interval_ms] || default_flush_interval_ms() + + Process.flag(:trap_exit, true) + timer = Process.send_after(self(), :tick, flush_interval_ms) + + {:ok, + %{ + buffer: buffer, + timer: timer, + name: Keyword.fetch!(opts, :name), + insert_sql: Keyword.fetch!(opts, :insert_sql), + insert_opts: Keyword.fetch!(opts, :insert_opts), + header: Keyword.fetch!(opts, :header), + buffer_size: IO.iodata_length(buffer), + max_buffer_size: max_buffer_size, + flush_interval_ms: flush_interval_ms + }} + end + + @impl true + def handle_cast({:insert, row_binary}, state) do + state = %{ + state + | buffer: [state.buffer | row_binary], + buffer_size: state.buffer_size + IO.iodata_length(row_binary) + } + + if state.buffer_size >= state.max_buffer_size do + Logger.notice("#{state.name} buffer full, flushing to ClickHouse") + Process.cancel_timer(state.timer) + do_flush(state) + new_timer = Process.send_after(self(), :tick, state.flush_interval_ms) + {:noreply, %{state | buffer: [], timer: new_timer, buffer_size: 0}} + else + {:noreply, state} + end + end + + @impl true + def handle_info(:tick, state) do + do_flush(state) + timer = Process.send_after(self(), :tick, state.flush_interval_ms) + {:noreply, %{state | buffer: [], buffer_size: 0, timer: timer}} + end + + def handle_info({:EXIT, _from, reason}, state) do + Logger.warning("#{state.name} received EXIT, keeping buffer alive: #{inspect(reason)}") + {:noreply, state} + end + + @impl true + def handle_call(:flush, _from, state) do + %{timer: timer, flush_interval_ms: flush_interval_ms} = state + Process.cancel_timer(timer) + do_flush(state) + new_timer = Process.send_after(self(), :tick, flush_interval_ms) + {:reply, :ok, %{state | buffer: [], buffer_size: 0, timer: new_timer}} + end + + @impl true + def terminate(_reason, %{name: name} = state) do + Logger.notice("Flushing #{name} buffer before shutdown...") + do_flush(state) + end + + defp do_flush(state) do + %{ + buffer: buffer, + buffer_size: buffer_size, + insert_opts: insert_opts, + insert_sql: insert_sql, + header: header, + name: name + } = state + + case buffer do + [] -> + nil + + _not_empty -> + Logger.notice("Flushing #{buffer_size} byte(s) RowBinary from #{name}") + IngestRepo.query!(insert_sql, [header | buffer], insert_opts) + end + end + + defp default_flush_interval_ms do + Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms) + end + + defp default_max_buffer_size do + Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size) + end + + @doc false + def compile_time_prepare(schema) do + fields = + schema.__schema__(:fields) + |> Enum.reject(&(&1 in fields_to_ignore())) + + types = + Enum.map(fields, fn field -> + type = schema.__schema__(:type, field) || raise "missing type for #{field}" + + type + |> Ecto.Type.type() + |> Ecto.Adapters.ClickHouse.Schema.remap_type(schema, field) + end) + + encoding_types = Ch.RowBinary.encoding_types(types) + + header = + fields + |> Enum.map(&to_string/1) + |> Ch.RowBinary.encode_names_and_types(types) + |> IO.iodata_to_binary() + + insert_sql = + "INSERT INTO #{schema.__schema__(:source)} (#{Enum.join(fields, ", ")}) FORMAT RowBinaryWithNamesAndTypes" + + %{ + fields: fields, + types: types, + encoding_types: encoding_types, + header: header, + insert_sql: insert_sql, + insert_opts: [ + command: :insert, + encode: false, + source: schema.__schema__(:source), + cast_params: [] + ] + } + end + + defp fields_to_ignore(), do: [:acquisition_channel, :interactive?] +end diff --git a/lib/plausible/internal_stats_api_version.ex b/lib/plausible/internal_stats_api_version.ex new file mode 100644 index 0000000000000000000000000000000000000000..ded0ce3a52823b0d2430da7fd24be004ecd1b36f --- /dev/null +++ b/lib/plausible/internal_stats_api_version.ex @@ -0,0 +1,77 @@ +defmodule Plausible.InternalStatsApiVersion do + @moduledoc """ + Tracks the effective internal stats API version across the cluster. + + Increment `@api_version` when deploying a change that breaks dashboards + already loaded from a previous deployment. The FE, upon detecting a + version mismatch, reloads the page to fetch the new dashboard code. + + Each app node has `@api_version` compiled in. The effective version served + to clients is the minimum across all connected nodes, fetched via + `:rpc.multicall` and refreshed every 30 seconds. This means the version + only advances once every node in a rolling deploy is running the new code, + avoiding repeated dashboard reloads during the deployment window. + """ + use GenServer + + @api_version 1 + + @refresh_interval :timer.seconds(30) + + @spec api_version() :: non_neg_integer() + def api_version, do: @api_version + + @spec effective_version() :: non_neg_integer() + def effective_version() do + # Use 0 as a placeholder version until the first multicall completes. + # The FE only reloads when the received version exceeds its compiled-in + # expectation, so 0 is always safe regardless of current @api_version. + case :ets.lookup(__MODULE__, :version) do + [{:version, v}] -> v + _ -> 0 + end + end + + def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @impl GenServer + def init(_opts) do + __MODULE__ = + :ets.new(__MODULE__, [ + :named_table, + :set, + :protected, + {:read_concurrency, true} + ]) + + {:ok, nil, {:continue, :fetch}} + end + + @impl GenServer + def handle_continue(:fetch, state) do + Process.send_after(self(), :refresh, @refresh_interval) + :ets.insert(__MODULE__, {:version, fetch_cluster_min()}) + {:noreply, state} + end + + @impl GenServer + def handle_info(:refresh, state) do + Process.send_after(self(), :refresh, @refresh_interval) + :ets.insert(__MODULE__, {:version, fetch_cluster_min()}) + {:noreply, state} + end + + defp fetch_cluster_min() do + {results, _bad_nodes} = :rpc.multicall(__MODULE__, :api_version, [], :timer.seconds(5)) + cluster_min(results) + end + + def cluster_min(results) do + case Enum.filter(results, &is_integer/1) do + [] -> @api_version + versions -> Enum.min(versions) + end + end +end diff --git a/lib/plausible/mailer.ex b/lib/plausible/mailer.ex new file mode 100644 index 0000000000000000000000000000000000000000..6fc23fb97ff2ee6a6f11b54337b327262fc6d7e1 --- /dev/null +++ b/lib/plausible/mailer.ex @@ -0,0 +1,22 @@ +defmodule Plausible.Mailer do + use Bamboo.Mailer, otp_app: :plausible + require Logger + + @spec send(Bamboo.Email.t()) :: :ok | {:error, :unknown_error} + def send(email) do + try do + deliver_now!(email) + rescue + e -> + # this message is ignored by Sentry, only appears in logs + log = "Failed to send e-mail:\n\n " <> Exception.format(:error, e, __STACKTRACE__) + # Sentry report is built entirely from crash_reason + crash_reason = {e, __STACKTRACE__} + + Logger.error(log, crash_reason: crash_reason) + {:error, :unknown_error} + else + _sent_email -> :ok + end + end +end diff --git a/lib/plausible/migration_utils.ex b/lib/plausible/migration_utils.ex new file mode 100644 index 0000000000000000000000000000000000000000..01e884deeb9d5a081d65d8784064c3ea1fb1070b --- /dev/null +++ b/lib/plausible/migration_utils.ex @@ -0,0 +1,49 @@ +defmodule Plausible.MigrationUtils do + @moduledoc """ + Base module for to use in Clickhouse migrations + """ + + use Plausible + + alias Plausible.IngestRepo + + def on_cluster_statement(table) do + if(IngestRepo.clustered_table?(table), do: "ON CLUSTER '{cluster}'", else: "") + end + + # See https://clickhouse.com/docs/en/sql-reference/dictionaries#clickhouse for context + def dictionary_connection_params() do + IngestRepo.config() + |> Enum.map(fn + {:database, database} -> "DB '#{database}'" + {:username, username} -> "USER '#{username}'" + {:password, password} -> "PASSWORD '#{password}'" + _ -> nil + end) + |> Enum.reject(&is_nil/1) + |> Enum.join(" ") + end + + def table_settings() do + IngestRepo.config() + |> Keyword.get(:table_settings) + |> Enum.reject(fn {_, v} -> is_nil(v) end) + end + + def table_settings_expr(type \\ :prefix) do + expr = Enum.map_join(table_settings(), ", ", fn {k, v} -> "#{k} = #{encode(v)}" end) + + case {table_settings(), type} do + {[], _} -> "" + {_, :prefix} -> "SETTINGS #{expr}" + {_, :suffix} -> ", #{expr}" + end + end + + def enterprise_edition?(), do: ee?() + + def community_edition?(), do: ce?() + + defp encode(value) when is_number(value), do: value + defp encode(value) when is_binary(value), do: "'#{value}'" +end diff --git a/lib/plausible/open_telemetry.ex b/lib/plausible/open_telemetry.ex new file mode 100644 index 0000000000000000000000000000000000000000..1112b785e8c3980a8fee7dfbfb66d6002323b197 --- /dev/null +++ b/lib/plausible/open_telemetry.ex @@ -0,0 +1,63 @@ +defmodule Plausible.OpenTelemetry do + @moduledoc false + + alias OpenTelemetry.Tracer + + def current_trace_id do + case Tracer.current_span_ctx() do + :undefined -> + nil + + span_ctx -> + span_ctx + |> OpenTelemetry.Span.trace_id() + |> Integer.to_string(16) + |> String.downcase() + end + end + + def add_site_attributes(site) do + case site do + %Plausible.Site{} = site -> + Tracer.set_attributes([ + {"plausible.site.id", site.id}, + {"plausible.site.domain", site.domain}, + {"plausible.site.team_id", site.team_id} + ]) + + id when is_integer(id) -> + Tracer.set_attributes([{"plausible.site.id", id}]) + + _any -> + :ignore + end + end + + def add_user_attributes(user) do + case user do + %Plausible.Auth.User{} = user -> + Tracer.set_attributes([ + {"plausible.user.id", user.id}, + {"plausible.user.name", user.name}, + {"plausible.user.email", user.email} + ]) + + id when is_integer(id) -> + Tracer.set_attributes([{"plausible.user.id", id}]) + + _any -> + :ignore + end + end + + # https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/semantic_conventions/README.md#service + def resource_attributes(runtime_metadata) do + [ + {"service.name", "analytics"}, + {"service.namespace", "plausible"}, + {"service.instance.app_host", runtime_metadata[:app_host]}, + {"service.instance.id", runtime_metadata[:host]}, + {"service.version", runtime_metadata[:version]} + ] + end +end diff --git a/lib/plausible/open_telemetry/beam_metrics.ex b/lib/plausible/open_telemetry/beam_metrics.ex new file mode 100644 index 0000000000000000000000000000000000000000..2d99d82ab96068c60c622feb85a889aac735c37b --- /dev/null +++ b/lib/plausible/open_telemetry/beam_metrics.ex @@ -0,0 +1,107 @@ +defmodule Plausible.OpenTelemetry.BeamMetrics do + @moduledoc """ + Periodic BEAM process sampling via OpenTelemetry observable gauges. + + Uses `:recon.proc_count/2` to sample top-N processes by memory, reductions, + and message queue length. Emits data as OTel observable gauge observations, + exported via OTLP to the configured OTel Collector. + + Disabled by default. Enable with `BEAM_METRICS_ENABLED=true`. + Collection interval controlled by `BEAM_METRICS_INTERVAL_MS` (default: 5000). + """ + + require Logger + + @top_n 20 + @metrics [:memory, :reductions, :message_queue_len] + @process_info_keys [ + :registered_name, + :current_function, + :initial_call, + :memory, + :reductions, + :message_queue_len + ] + + @instruments %{ + memory: + {:"beam.top_process.memory", + %{description: "Memory usage of top BEAM processes", unit: :bytes}}, + reductions: + {:"beam.top_process.reductions", + %{description: "Reductions of top BEAM processes", unit: :"1"}}, + message_queue_len: + {:"beam.top_process.message_queue_len", + %{description: "Message queue length of top BEAM processes", unit: :"1"}} + } + + @doc """ + Registers OTel observable gauge instruments and a shared callback. + + Should be called once during application startup when BEAM metrics are enabled. + """ + def setup do + scope = :opentelemetry.instrumentation_scope("plausible_beam_metrics", "0.1.0", :undefined) + meter = :opentelemetry_experimental.get_meter(scope) + + gauges = + Enum.map(@metrics, fn metric -> + {name, opts} = Map.fetch!(@instruments, metric) + :otel_meter.create_observable_gauge(meter, name, opts) + end) + + :otel_meter.register_callback(meter, gauges, &observe_top_processes/1, []) + + Logger.info("BEAM metrics setup complete — sampling top #{@top_n} processes per metric") + :ok + end + + @doc """ + Callback invoked by the OTel Metric Reader on each collection cycle. + + Returns named observations for all three gauge instruments. + """ + def observe_top_processes(_callback_args) do + Enum.map(@metrics, fn metric -> + {gauge_name, _opts} = Map.fetch!(@instruments, metric) + observations = collect_observations(metric) + {gauge_name, observations} + end) + end + + defp collect_observations(metric) do + metric + |> :recon.proc_count(@top_n) + |> Enum.flat_map(fn {pid, value, _info} -> + case Process.info(pid, @process_info_keys) do + nil -> [] + info -> [{value, build_attributes(pid, info)}] + end + end) + end + + defp build_attributes(pid, info) do + registered_name = + case Keyword.get(info, :registered_name) do + [] -> "" + name when is_atom(name) -> Atom.to_string(name) + _ -> "" + end + + current_function = format_mfa(Keyword.get(info, :current_function)) + initial_call = format_mfa(Keyword.get(info, :initial_call)) + + %{ + "beam.process.pid" => inspect(pid), + "beam.process.registered_name" => registered_name, + "beam.process.current_function" => current_function, + "beam.process.initial_call" => initial_call, + "beam.process.memory" => to_string(Keyword.get(info, :memory, 0)), + "beam.process.reductions" => to_string(Keyword.get(info, :reductions, 0)), + "beam.process.message_queue_len" => to_string(Keyword.get(info, :message_queue_len, 0)) + } + end + + defp format_mfa({m, f, a}), do: Exception.format_mfa(m, f, a) + defp format_mfa(_), do: "" +end diff --git a/lib/plausible/open_telemetry/logger.ex b/lib/plausible/open_telemetry/logger.ex new file mode 100644 index 0000000000000000000000000000000000000000..1511887bbae2e7744e01abf8715f159f12e20684 --- /dev/null +++ b/lib/plausible/open_telemetry/logger.ex @@ -0,0 +1,34 @@ +defmodule Plausible.OpenTelemetry.Logger do + @moduledoc """ + Telemetry handler that adds the OpenTelemetry trace_id to Logger metadata. + + This enables correlation between log lines and distributed traces. + The trace_id is extracted from the current OTel span context and added + to the Logger metadata when Phoenix router dispatch starts. + """ + + @doc """ + Attaches telemetry handlers to set trace_id in Logger metadata. + + Should be called during application startup, after OpentelemetryPhoenix.setup(). + """ + def setup do + :telemetry.attach( + "plausible-otel-logger-metadata", + [:phoenix, :router_dispatch, :start], + &__MODULE__.handle_router_dispatch_start/4, + %{} + ) + end + + @doc false + def handle_router_dispatch_start(_event, _measurements, _metadata, _config) do + case Plausible.OpenTelemetry.current_trace_id() do + nil -> + :ok + + trace_id_hex -> + Logger.metadata(trace_id: trace_id_hex) + end + end +end diff --git a/lib/plausible/pagination.ex b/lib/plausible/pagination.ex new file mode 100644 index 0000000000000000000000000000000000000000..730086d337b3ad5527eb1d7f57a77edf33888b7e --- /dev/null +++ b/lib/plausible/pagination.ex @@ -0,0 +1,45 @@ +defmodule Plausible.Pagination do + @moduledoc """ + Cursor-based pagination. + """ + + @limit 10 + @maximum_limit 100 + + @spec paginate(Ecto.Queryable.t(), map(), Keyword.t(), Keyword.t()) :: Paginator.Page.t() + def paginate(queryable, params, opts, repo_opts \\ []) do + opts = Keyword.merge([limit: @limit, maximum_limit: @maximum_limit], opts) + + Paginator.paginate( + queryable, + Keyword.merge(opts, to_pagination_opts(params)), + Plausible.Repo, + repo_opts + ) + end + + defp to_pagination_opts(params) do + Enum.reduce(params, Keyword.new(), fn + {"after", cursor}, acc -> + Keyword.put(acc, :after, cursor) + + {"before", cursor}, acc -> + Keyword.put(acc, :before, cursor) + + {"limit", limit}, acc -> + limit = to_int(limit) + + if limit > 0 and limit <= @maximum_limit do + Keyword.put(acc, :limit, limit) + else + acc + end + + _, acc -> + acc + end) + end + + defp to_int(x) when is_binary(x), do: String.to_integer(x) + defp to_int(x) when is_integer(x), do: x +end diff --git a/lib/plausible/plugins/api/capabilities.ex b/lib/plausible/plugins/api/capabilities.ex new file mode 100644 index 0000000000000000000000000000000000000000..5f7665b81d776199463f3626e31e6985a634e7fc --- /dev/null +++ b/lib/plausible/plugins/api/capabilities.ex @@ -0,0 +1,35 @@ +defmodule Plausible.Plugins.API.Capabilities do + @moduledoc """ + Context module for querying API capabilities + """ + require Plausible.Billing.Feature + alias Plausible.Billing.Feature + + @spec get(Plug.Conn.t()) :: {:ok, map()} + def get(conn) do + conn = PlausibleWeb.Plugs.AuthorizePluginsAPI.call(conn, send_error?: false) + + site = conn.assigns[:authorized_site] + + features = + if site do + site = Plausible.Repo.preload(site, :team) + allowed_features = Plausible.Teams.Billing.allowed_features_for(site.team) + + Feature.list() + |> Enum.map(fn mod -> + feature = mod |> Module.split() |> List.last() + {feature, mod in allowed_features} + end) + else + Enum.map(Feature.list_short_names(), &{&1, false}) + end + + {:ok, + %{ + authorized: not is_nil(site), + data_domain: site && site.domain, + features: Enum.into(features, %{}) + }} + end +end diff --git a/lib/plausible/plugins/api/custom_props.ex b/lib/plausible/plugins/api/custom_props.ex new file mode 100644 index 0000000000000000000000000000000000000000..b60437014a50b8cd85c15e2f6ba4c8fb0c241acf --- /dev/null +++ b/lib/plausible/plugins/api/custom_props.ex @@ -0,0 +1,27 @@ +defmodule Plausible.Plugins.API.CustomProps do + @moduledoc """ + Plugins API context module for Custom Props. + All high level Custom Props operations should be implemented here. + """ + + @spec enable(Plausible.Site.t(), String.t() | [String.t()]) :: + {:ok, [String.t()]} | {:error, :upgrade_required | Ecto.Changeset.t()} + def enable(site, prop_or_props) do + case Plausible.Props.allow(site, prop_or_props) do + {:ok, site} -> + {:ok, site.allowed_event_props} + + error -> + error + end + end + + @spec disable(Plausible.Site.t(), String.t() | [String.t()]) :: + :ok | {:error, Ecto.Changeset.t()} + def disable(site, prop_or_props) do + case Plausible.Props.disallow(site, prop_or_props) do + {:ok, _site} -> :ok + error -> error + end + end +end diff --git a/lib/plausible/plugins/api/goals.ex b/lib/plausible/plugins/api/goals.ex new file mode 100644 index 0000000000000000000000000000000000000000..37c6a7ff0aff8e6e7f8ef5bbc93538a0d31e7535 --- /dev/null +++ b/lib/plausible/plugins/api/goals.ex @@ -0,0 +1,110 @@ +defmodule Plausible.Plugins.API.Goals do + @moduledoc """ + Plugins API context module for Goals. + All high level Goal operations should be implemented here. + """ + use Plausible + + import Plausible.Pagination + + alias Plausible.Repo + alias PlausibleWeb.Plugins.API.Schemas.Goal.CreateRequest + + @type create_request() :: + CreateRequest.CustomEvent.t() + | CreateRequest.Revenue.t() + | CreateRequest.Pageview.t() + + @spec create( + Plausible.Site.t(), + create_request() | list(create_request()) + ) :: + {:ok, list(Plausible.Goal.t())} + | {:error, Ecto.Changeset.t()} + | {:error, :upgrade_required} + def create(site, goal_or_goals) do + Repo.transaction(fn -> find_or_create(site, goal_or_goals) end) + end + + @spec get_goals(Plausible.Site.t(), map()) :: {:ok, Paginator.Page.t()} + def get_goals(site, params) do + query = Plausible.Goals.for_site_query(site, preload_funnels?: false) + + {:ok, paginate(query, params, cursor_fields: [{:id, :desc}])} + end + + @spec get(Plausible.Site.t(), pos_integer()) :: nil | Plausible.Goal.t() + def get(site, id) when is_integer(id) do + Plausible.Goals.get(site, id) + end + + @spec delete(Plausible.Site.t(), [pos_integer()] | pos_integer()) :: :ok + def delete(site, id_or_ids) do + Plausible.Repo.transaction(fn -> + id_or_ids + |> List.wrap() + |> Enum.each(fn id when is_integer(id) -> + Plausible.Goals.delete(id, site) + end) + end) + + :ok + end + + defp convert_to_create_params(%CreateRequest.CustomEvent{ + goal: %{event_name: event_name, custom_props: custom_props} + }) + when is_map(custom_props) do + %{"goal_type" => "event", "event_name" => event_name, "custom_props" => custom_props} + end + + defp convert_to_create_params(%CreateRequest.CustomEvent{goal: %{event_name: event_name}}) do + %{"goal_type" => "event", "event_name" => event_name} + end + + defp convert_to_create_params(%CreateRequest.Revenue{ + goal: %{event_name: event_name, currency: currency, custom_props: custom_props} + }) + when is_map(custom_props) do + %{ + "goal_type" => "event", + "event_name" => event_name, + "currency" => currency, + "custom_props" => custom_props + } + end + + defp convert_to_create_params(%CreateRequest.Revenue{ + goal: %{event_name: event_name, currency: currency} + }) do + %{"goal_type" => "event", "event_name" => event_name, "currency" => currency} + end + + defp convert_to_create_params(%CreateRequest.Pageview{ + goal: %{path: page_path, custom_props: custom_props} + }) + when is_map(custom_props) do + %{"goal_type" => "page", "page_path" => page_path, "custom_props" => custom_props} + end + + defp convert_to_create_params(%CreateRequest.Pageview{goal: %{path: page_path}}) do + %{"goal_type" => "page", "page_path" => page_path} + end + + defp find_or_create(site, goal_or_goals) do + goal_or_goals + |> List.wrap() + |> Enum.map(fn goal -> + case Plausible.Goals.find_or_create(site, convert_to_create_params(goal)) do + {:ok, goal} -> + goal + + {:error, :upgrade_required} -> + Repo.rollback(:upgrade_required) + + {:error, changeset} -> + Repo.rollback(changeset) + end + end) + end +end diff --git a/lib/plausible/plugins/api/shared_links.ex b/lib/plausible/plugins/api/shared_links.ex new file mode 100644 index 0000000000000000000000000000000000000000..e38fcf723ee67634576ee0aca4e7a7337ac22873 --- /dev/null +++ b/lib/plausible/plugins/api/shared_links.ex @@ -0,0 +1,61 @@ +defmodule Plausible.Plugins.API.SharedLinks do + @moduledoc """ + Plugins API context module for Shared Links. + All high level Shared Links operations should be implemented here. + """ + import Ecto.Query + import Plausible.Pagination + + alias Plausible.Repo + + @spec get_shared_links(Plausible.Site.t(), map()) :: {:ok, Paginator.Page.t()} + def get_shared_links(site, params) do + query = + from l in Plausible.Site.SharedLink, + where: l.site_id == ^site.id, + order_by: [desc: l.id] + + {:ok, paginate(query, params, cursor_fields: [{:id, :desc}])} + end + + @spec get(Plausible.Site.t(), pos_integer() | String.t()) :: nil | Plausible.Site.SharedLink.t() + def get(site, id) when is_integer(id) do + get_by_id(site, id) + end + + def get(site, name) when is_binary(name) do + get_by_name(site, name) + end + + @spec get_or_create(Plausible.Site.t(), String.t(), String.t() | nil) :: + {:ok, Plausible.Site.SharedLink.t()} + def get_or_create(site, name, password \\ nil) do + case get_by_name(site, name) do + nil -> + Plausible.Sites.create_shared_link(site, name, + password: password, + skip_feature_check?: true, + skip_special_name_check?: true + ) + + shared_link -> + {:ok, shared_link} + end + end + + defp get_by_id(site, id) do + Repo.one( + from l in Plausible.Site.SharedLink, + where: l.site_id == ^site.id, + where: l.id == ^id + ) + end + + defp get_by_name(site, name) do + Repo.one( + from l in Plausible.Site.SharedLink, + where: l.site_id == ^site.id, + where: l.name == ^name + ) + end +end diff --git a/lib/plausible/plugins/api/token.ex b/lib/plausible/plugins/api/token.ex new file mode 100644 index 0000000000000000000000000000000000000000..2b8cc2284c3d22530bc9eab4b36465831dd68d63 --- /dev/null +++ b/lib/plausible/plugins/api/token.ex @@ -0,0 +1,106 @@ +defmodule Plausible.Plugins.API.Token do + @moduledoc """ + Ecto schema for Plugins API Tokens. + Tokens are stored hashed and require a description. + + Tokens are considered secret, although the Plugins API + by nature will expose very little, if any, destructive/insecure operations. + + The raw token version is meant to be presented to the user upon creation. + It is prefixed with a plain text identifier allowing source scanning + for leaked secrets. + """ + use Plausible + use Ecto.Schema + import Ecto.Changeset + + alias Plausible.Site + + @type t() :: %__MODULE__{} + + @primary_key {:id, :binary_id, autogenerate: true} + schema "plugins_api_tokens" do + timestamps() + field(:token_hash, :binary) + field(:description, :string) + field(:hint, :string) + field(:last_used_at, :naive_datetime) + + belongs_to(:site, Site) + end + + @spec generate(String.t()) :: map() + def generate(random_bytes \\ random_bytes()) do + raw = prefixed(random_bytes) + hash = hash(raw) + + %{ + raw: raw, + hash: hash + } + end + + @spec hash(String.t()) :: binary() + def hash(raw) do + :crypto.hash(:sha256, raw) + end + + @fields [:description, :site_id] + @required_fields [:description, :site, :token_hash, :hint] + + @spec insert_changeset(Site.t(), map(), map()) :: Ecto.Changeset.t() + def insert_changeset(site, %{hash: hash, raw: raw}, attrs \\ %{}) do + %__MODULE__{} + |> cast(attrs, @fields) + |> put_change(:token_hash, hash) + |> put_change(:hint, String.slice(raw, -4, 4)) + |> put_assoc(:site, site) + |> validate_required(@required_fields) + end + + @doc """ + Raw tokens are prefixed so that tools like + https://docs.github.com/en/code-security/secret-scanning/about-secret-scanning + can scan repositories for accidental secret commits. + """ + def prefix() do + on_ee do + env = Application.get_env(:plausible, :environment) + + case env do + "prod" -> "plausible-plugin" + env -> "plausible-plugin-#{env}" + end + else + "plausible-plugin-selfhost" + end + end + + @spec last_used_humanize(t()) :: String.t() + def last_used_humanize(token) do + diff = + if token.last_used_at do + now = NaiveDateTime.utc_now() + NaiveDateTime.diff(now, token.last_used_at, :minute) + end + + cond do + is_nil(diff) -> "Not yet" + diff < 5 -> "Just recently" + diff < 30 -> "Several minutes ago" + diff < 70 -> "An hour ago" + diff < 24 * 60 -> "Hours ago" + diff < 24 * 60 * 2 -> "Yesterday" + diff < 24 * 60 * 7 -> "Sometime this week" + true -> "Long time ago" + end + end + + defp prefixed(random_bytes) do + Enum.join([prefix(), random_bytes], "-") + end + + defp random_bytes() do + 30 |> :crypto.strong_rand_bytes() |> Base.encode64() + end +end diff --git a/lib/plausible/plugins/api/tokens.ex b/lib/plausible/plugins/api/tokens.ex new file mode 100644 index 0000000000000000000000000000000000000000..8eb6d78a483a5cc8a95ebe66576c68963bc89a10 --- /dev/null +++ b/lib/plausible/plugins/api/tokens.ex @@ -0,0 +1,74 @@ +defmodule Plausible.Plugins.API.Tokens do + @moduledoc """ + Context module for Plugins API Tokens. + Exposes high-level operation for token-based authentication flows. + """ + alias Plausible.Plugins.API.Token + alias Plausible.Site + alias Plausible.Repo + + import Ecto.Query + + @spec create(Site.t(), String.t()) :: + {:ok, Token.t(), String.t()} | {:error, Ecto.Changeset.t()} + def create(%Site{} = site, description, generated_token \\ Token.generate()) do + with changeset <- Token.insert_changeset(site, generated_token, %{description: description}), + {:ok, saved_token} <- Repo.insert(changeset) do + {:ok, saved_token, generated_token.raw} + end + end + + @spec find(String.t()) :: {:ok, Token.t()} | {:error, :not_found} + def find(raw) do + found = + Repo.one( + from(t in Token, + inner_join: s in Site, + on: s.id == t.site_id, + where: t.token_hash == ^Token.hash(raw), + preload: [:site] + ) + ) + + if found do + {:ok, found} + else + {:error, :not_found} + end + end + + @spec delete(Site.t(), String.t()) :: :ok + def delete(site, token_id) do + Repo.delete_all(from(t in Token, where: t.site_id == ^site.id and t.id == ^token_id)) + :ok + end + + @spec list(Site.t()) :: {:ok, [Token.t()]} + def list(site) do + Repo.all( + from(t in Token, where: t.site_id == ^site.id, order_by: [desc: t.inserted_at, desc: t.id]) + ) + end + + @spec any?(Site.t()) :: boolean() + def any?(site) do + Repo.exists?(from(t in Token, where: t.site_id == ^site.id)) + end + + @spec update_last_seen(Token.t(), NaiveDateTime.t()) :: {:ok, Token.t()} + def update_last_seen(token, now \\ NaiveDateTime.utc_now()) do + # we don't need very precise timestamp tracking, and to spare postgres we only + # update that timestamp in 5m windows - this is mostly to help users reason + # about what token they have, in case of rotations + now = NaiveDateTime.truncate(now, :second) + last_used = token.last_used_at + + if is_nil(last_used) or NaiveDateTime.diff(now, last_used, :minute) > 5 do + token + |> Ecto.Changeset.change(%{last_used_at: now}) + |> Repo.update() + else + {:ok, token} + end + end +end diff --git a/lib/plausible/prom_ex.ex b/lib/plausible/prom_ex.ex new file mode 100644 index 0000000000000000000000000000000000000000..f2186e0206a6bf73cedde6bc5c42fb716e46260f --- /dev/null +++ b/lib/plausible/prom_ex.ex @@ -0,0 +1,53 @@ +defmodule Plausible.PromEx do + use PromEx, otp_app: :plausible + + alias PromEx.Plugins + + @plugins [ + Plugins.Application, + Plugins.Beam, + Plugins.PhoenixLiveView, + {Plugins.Phoenix, router: PlausibleWeb.Router, endpoint: PlausibleWeb.Endpoint}, + {Plugins.Ecto, + repos: [ + Plausible.Repo, + Plausible.ClickhouseRepo, + Plausible.IngestRepo, + Plausible.AsyncInsertRepo + ]}, + Plausible.PromEx.Plugins.PlausibleMetrics + ] + + @impl true + if Mix.env() in [:test, :ce_test, :e2e_test] do + # PromEx tries to query Oban's DB tables in order to retrieve metrics. + # During tests, however, this is pointless as Oban is in manual mode, + # and that leads to connection ownership clashes. + def plugins do + @plugins + end + else + def plugins do + [Plugins.Oban | @plugins] + end + end + + @impl true + def dashboard_assigns do + [ + datasource_id: "grafanacloud-prom", + default_selected_interval: "30s" + ] + end + + @impl true + def dashboards do + [ + {:prom_ex, "application.json"}, + {:prom_ex, "beam.json"}, + {:prom_ex, "phoenix.json"}, + {:prom_ex, "ecto.json"}, + {:prom_ex, "oban.json"} + ] + end +end diff --git a/lib/plausible/prom_ex/buckets.ex b/lib/plausible/prom_ex/buckets.ex new file mode 100644 index 0000000000000000000000000000000000000000..cf477664ad7e750642d82c8b7e432fcad21ffb3c --- /dev/null +++ b/lib/plausible/prom_ex/buckets.ex @@ -0,0 +1,85 @@ +defmodule Plausible.PromEx.Buckets do + @moduledoc """ + Adapts `Peep` for `PromEx`. + + Based on `Peep.Buckets.Custom` and `TelemetryMetricsPrometheus.Core`. + """ + + @behaviour Peep.Buckets + + @impl true + def config(%Telemetry.Metrics.Distribution{reporter_options: reporter_options}) do + # PromEx configures buckets with `:reporter_options` + buckets = Keyword.fetch!(reporter_options, :buckets) + + if Enum.empty?(buckets) do + raise ArgumentError, "expected buckets list to be non-empty, got #{inspect(buckets)}" + end + + unless Enum.all?(buckets, &is_number/1) do + raise ArgumentError, + "expected buckets list to contain only numbers, got #{inspect(buckets)}" + end + + unless buckets == Enum.uniq(Enum.sort(buckets)) do + raise ArgumentError, "expected buckets to be ordered ascending, got #{inspect(buckets)}" + end + + number_of_buckets = length(buckets) + + int_tree = :gb_trees.from_orddict(int_buckets(buckets, nil, 0)) + + float_tree = + Enum.map(buckets, &(&1 * 1.0)) + |> Enum.with_index() + |> :gb_trees.from_orddict() + + upper_bound = + buckets + |> Enum.with_index() + |> Map.new(fn {boundary, bucket_idx} -> {bucket_idx, to_string(boundary * 1.0)} end) + + %{ + number_of_buckets: number_of_buckets, + int_tree: int_tree, + float_tree: float_tree, + upper_bound: upper_bound + } + end + + @impl true + def number_of_buckets(config) do + config.number_of_buckets + end + + @impl true + def bucket_for(number, config) when is_integer(number) do + case :gb_trees.larger(number, config.int_tree) do + {_, bucket_idx} -> bucket_idx + :none -> config.number_of_buckets + end + end + + def bucket_for(number, config) when is_float(number) do + case :gb_trees.larger(number, config.float_tree) do + {_, bucket_idx} -> bucket_idx + :none -> config.number_of_buckets + end + end + + @impl true + def upper_bound(bucket_idx, config) do + Map.get(config.upper_bound, bucket_idx, "+Inf") + end + + defp int_buckets([], _prev, _counter) do + [] + end + + defp int_buckets([curr | tail], prev, counter) do + case ceil(curr) do + ^prev -> int_buckets(tail, prev, counter + 1) + curr -> [{curr, counter} | int_buckets(tail, curr, counter + 1)] + end + end +end diff --git a/lib/plausible/prom_ex/striped_peep.ex b/lib/plausible/prom_ex/striped_peep.ex new file mode 100644 index 0000000000000000000000000000000000000000..613c09ec0e11da57a43e2752a050e84cfad6e993 --- /dev/null +++ b/lib/plausible/prom_ex/striped_peep.ex @@ -0,0 +1,25 @@ +defmodule Plausible.PromEx.StripedPeep do + @moduledoc """ + "Striped" storage based on `PromEx.Storage.Peep`. + """ + + @behaviour PromEx.Storage + + @impl true + def scrape(name) do + (Peep.get_all_metrics(name) || []) + |> Peep.Prometheus.export() + |> IO.iodata_to_binary() + end + + @impl true + def child_spec(name, metrics) do + opts = [ + name: name, + metrics: metrics, + storage: :striped + ] + + Peep.child_spec(opts) + end +end diff --git a/lib/plausible/props.ex b/lib/plausible/props.ex new file mode 100644 index 0000000000000000000000000000000000000000..5130c95f5307153900123b840bfc3da911e46259 --- /dev/null +++ b/lib/plausible/props.ex @@ -0,0 +1,199 @@ +defmodule Plausible.Props do + @moduledoc """ + Context module for handling custom event props. + """ + use Plausible + import Ecto.Query + + @type prop :: String.t() + + @max_props 300 + def max_props, do: @max_props + + @max_prop_key_length 300 + def max_prop_key_length, do: @max_prop_key_length + + @max_prop_value_length 2000 + def max_prop_value_length, do: @max_prop_value_length + + # NOTE: Keep up to date with `Plausible.Imported.imported_custom_props/0`. + @internal_keys ~w(url path search_query) + + @doc """ + Lists prop keys used internally. + + These props should be allowed by default, and should not be displayed in the + props settings page. For example, `url` is a special prop key used for file + downloads and outbound links. It doesn't make sense to remove this prop key + from the allow list, or to suggest users to add this prop key. + """ + def internal_keys, do: @internal_keys + + @doc """ + Returns the custom props allowed in queries for the given site. There are + two factors deciding whether a custom property is allowed for a site. + + ### 1. Subscription plan including the props feature. + + Internally used keys (i.e. `#{inspect(@internal_keys)}`) are always allowed, + even for plans that don't include props. For any other props, access to the + Custom Properties feature is required. + + ### 2. The site having an `allowed_event_props` list configured. + + For customers with a configured `allowed_event_props` list, this function + returns that list (+ internally used keys). That helps to filter out garbage + props which people might not want to see in their dashboards. + + With the `bypass_setup?` boolean option you can override the requirement of + the site having set up props in the `allowed_event_props` list. For example, + this is currently used for fetching allowed properties in Stats API queries + in order to ensure the props feature access. + + Since `allowed_event_props` was added after the props feature had already + been used for a while, there are sites with `allowed_event_props = nil`. For + those sites, all custom properties that exist in the database are allowed to + be queried. + """ + @spec allowed_for(Plausible.Site.t()) :: [prop()] | :all + def allowed_for(site, opts \\ []) do + site = Plausible.Repo.preload(site, :team) + internal_keys = Plausible.Props.internal_keys() + props_enabled? = Plausible.Billing.Feature.Props.check_availability(site.team) == :ok + bypass_setup? = Keyword.get(opts, :bypass_setup?) + + cond do + props_enabled? && is_nil(site.allowed_event_props) -> :all + props_enabled? && bypass_setup? -> :all + props_enabled? -> site.allowed_event_props ++ internal_keys + true -> internal_keys + end + end + + @spec allow(Plausible.Site.t(), [prop()] | prop()) :: + {:ok, Plausible.Site.t()} | {:error, Ecto.Changeset.t()} | {:error, :upgrade_required} + @doc """ + Allows a prop key or a list of props keys to be included in ClickHouse + queries. Allowing prop keys does not affect ingestion, as we don't want any + data to be dropped or lost. + """ + def allow(site, prop_or_props) do + with site <- Plausible.Repo.preload(site, :team), + :ok <- ensure_prop_key_accessible(prop_or_props, site.team) do + site + |> allow_changeset(prop_or_props) + |> Plausible.Repo.update() + end + end + + def allow_changeset(site, prop_or_props) do + old_props = site.allowed_event_props || [] + new_props = List.wrap(prop_or_props) ++ old_props + + changeset(site, new_props) + end + + @spec disallow(Plausible.Site.t(), [prop()] | prop()) :: + {:ok, Plausible.Site.t()} | {:error, Ecto.Changeset.t()} + @doc """ + Removes previously allowed prop key(s) from the allow list. This means this + prop key won't be included in ClickHouse queries. This doesn't drop any + ClickHouse data, nor affects ingestion. + """ + def disallow(site, prop_or_props) do + allowed_event_props = site.allowed_event_props || [] + + site + |> changeset(allowed_event_props -- List.wrap(prop_or_props)) + |> Plausible.Repo.update() + end + + defp changeset(site, props) do + props = + props + |> Enum.map(&String.trim/1) + |> Enum.uniq() + + site + |> Ecto.Changeset.change(allowed_event_props: props) + |> Ecto.Changeset.validate_length(:allowed_event_props, max: @max_props) + |> Ecto.Changeset.validate_change(:allowed_event_props, fn field, allowed_props -> + if Enum.all?(allowed_props, &valid?/1), + do: [], + else: [ + {field, + {"must be between 1 and #{@max_prop_key_length} characters", + validation: :length, type: :string}} + ] + end) + end + + @spec allow_existing_props(Plausible.Site.t()) :: + {:ok, Plausible.Site.t()} | {:error, :upgrade_required} + @doc """ + Allows the #{@max_props} most frequent props keys for a specific site over + the past 6 months. + """ + def allow_existing_props(%Plausible.Site{} = site) do + props_to_allow = + site + |> suggest_keys_to_allow() + |> Enum.filter(&valid?/1) + + allow(site, props_to_allow) + end + + def ensure_prop_key_accessible(prop_keys, team) when is_list(prop_keys) do + Enum.reduce_while(prop_keys, :ok, fn prop_key, :ok -> + case ensure_prop_key_accessible(prop_key, team) do + :ok -> {:cont, :ok} + error -> {:halt, error} + end + end) + end + + def ensure_prop_key_accessible(prop_key, team) do + if prop_key in @internal_keys do + :ok + else + Plausible.Billing.Feature.Props.check_availability(team) + end + end + + @spec suggest_keys_to_allow(Plausible.Site.t(), non_neg_integer()) :: [String.t()] + @doc """ + Queries the events table to fetch the #{@max_props} most frequent prop keys + for a specific site over the past 6 months, excluding keys that are already + allowed. + """ + def suggest_keys_to_allow(%Plausible.Site{} = site, limit \\ @max_props) do + allowed_event_props = site.allowed_event_props || [] + + unnested_keys = + from e in Plausible.ClickhouseEventV2, + where: ^Plausible.Sites.site_id_query_filter(site), + where: fragment("? > (NOW() - INTERVAL 6 MONTH)", e.timestamp), + select: %{key: fragment("arrayJoin(?)", field(e, :"meta.key"))} + + Plausible.ClickhouseRepo.all( + from uk in subquery(unnested_keys), + where: uk.key not in ^allowed_event_props, + where: uk.key not in ^@internal_keys, + group_by: uk.key, + select: uk.key, + order_by: {:desc, count(uk.key)}, + limit: ^limit + ) + end + + defp valid?(key) do + String.length(key) in 1..@max_prop_key_length + end + + @doc """ + Returns whether the site has configured custom props or not. + """ + def configured?(%Plausible.Site{allowed_event_props: allowed_event_props}) do + is_list(allowed_event_props) && length(allowed_event_props) > 0 + end +end diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.ex new file mode 100644 index 0000000000000000000000000000000000000000..aa072bade88a6ced1f64f072b55cd54cc330cdd5 --- /dev/null +++ b/lib/plausible/purge.ex @@ -0,0 +1,100 @@ +defmodule Plausible.Purge do + @moduledoc """ + Deletes data from a site. + + Stats are stored on Clickhouse, and unlike other databases data deletion is + done asynchronously. + + All import tables have MergeTree's deduplication mechanism _disabled_ by setting + `replicated_deduplication_window` from default 100 to 0. When enabled, every insert + into a given table is compared against hashes of 100 previous inserts (as complete + parts, not concrete rows) and ignored when match is found. The prupose of that + mechanism is making inserts of exact same batches idempotent when retrying them + shortly after - for instance due to timeout, when the client can't easily tell if + previous insert succeeded or not. Deduplication, however, only considers inserts, + not mutations. Deletions do not affect stored hashes, so further inserts of parts + that were deleted will still be treated as duplicates. That's why this feature + is disabled for import tables. + + Although deletions are asynchronous, the parts to delete are "remembered", so there's + no risk of overlapping deletion causing problems with import following right after it. + + IMPORTANT: Deletion requires revision if/when import tables get moved to sharded CH + cluster setup. Mutation queries, which have to be run with `ON CLUSTER` in such setup, + dispatch independent queries across shards and those queries can start at different + times. This in turn means risk of deletions corrupting data of follow-up inserts + in some edge cases. Ideally, imported entries should be unique for a given import + - an extra `import_id` column can be introduced, holding identifier. Last processed + import identifier should be stored with other site data and should be used for scoping + imported stats queries. No longer used imports can then be safely removed fully + asynchronously. + + - [Clickhouse `ALTER TABLE ... DELETE` Statement](https://clickhouse.com/docs/en/sql-reference/statements/alter/delete) + - [Synchronicity of `ALTER` Queries](https://clickhouse.com/docs/en/sql-reference/statements/alter/#synchronicity-of-alter-queries) + """ + + alias Plausible.Repo + + @spec delete_imported_stats!(Plausible.Site.t() | Plausible.Imported.SiteImport.t()) :: :ok + @doc """ + Deletes imported stats from and clears the `stats_start_date` field. + + The `stats_start_date` is expected to get repopulated the next time + `Plausible.Sites.stats_start_date/1` is called. + + If the input argument is a site, all imported stats are deleted. If it's a site import, + only imported stats for that import are deleted. + """ + def delete_imported_stats!(%Plausible.Site{} = site) do + Enum.each(Plausible.Imported.tables(), fn table -> + sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64}" + Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id]) + end) + + Plausible.Sites.clear_stats_start_date!(site) + + :ok + end + + def delete_imported_stats!(%Plausible.Imported.SiteImport{} = site_import) do + site_import = Repo.preload(site_import, :site) + delete_imported_stats!(site_import.site, site_import.id) + + if site_import.legacy do + delete_imported_stats!(site_import.site, 0) + end + + :ok + end + + def delete_imported_stats!(%Plausible.Site{} = site, import_id) do + Enum.each(Plausible.Imported.tables(), fn table -> + sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64} AND import_id = {$1:UInt64}" + + Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id, import_id]) + end) + + Plausible.Sites.clear_stats_start_date!(site) + + :ok + end + + @spec delete_native_stats!(Plausible.Site.t()) :: :ok + @doc """ + Move stats pointers so that no historical stats are available. + """ + def delete_native_stats!(site) do + reset!(site) + + :ok + end + + def reset!(site) do + site + |> Ecto.Changeset.change( + native_stats_start_at: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second), + stats_start_date: nil + ) + |> Plausible.Repo.update!() + end +end diff --git a/lib/plausible/rate_limit.ex b/lib/plausible/rate_limit.ex new file mode 100644 index 0000000000000000000000000000000000000000..a92f5798426f1a343e1464c5a143c8c2db4fc8cb --- /dev/null +++ b/lib/plausible/rate_limit.ex @@ -0,0 +1,98 @@ +defmodule Plausible.RateLimit do + @moduledoc """ + Thin wrapper around `:ets.update_counter/4` and a + clean-up process to act as a rate limiter. + """ + + use GenServer + + @doc """ + Starts the process that creates and cleans the ETS table. + + Accepts the following options: + - `GenServer.option()` + - `:table` for the ETS table name, defaults to `#{__MODULE__}` + - `:clean_period` for how often to perform garbage collection + """ + @spec start_link([GenServer.option() | {:table, atom} | {:clean_period, pos_integer}]) :: + GenServer.on_start() + def start_link(opts) do + {gen_opts, opts} = Keyword.split(opts, [:debug, :name, :spawn_opt, :hibernate_after]) + GenServer.start_link(__MODULE__, opts, gen_opts) + end + + @doc """ + Checks the rate-limit for a key. + """ + @spec check_rate(:ets.table(), key, scale, limit, increment) :: {:allow, count} | {:deny, limit} + when key: term, + scale: pos_integer, + limit: pos_integer, + increment: pos_integer, + count: pos_integer + def check_rate(table \\ __MODULE__, key, scale, limit, increment \\ 1) do + bucket = div(now(), scale) + full_key = {key, bucket} + expires_at = (bucket + 1) * scale + + count = + case :ets.lookup(table, full_key) do + [{_, counter, _expires_at}] -> + :atomics.add_get(counter, 1, increment) + + [] -> + counter = :atomics.new(1, signed: false) + + case :ets.insert_new(table, {full_key, counter, expires_at}) do + true -> + :atomics.add_get(counter, 1, increment) + + false -> + [{_, counter, _expires_at}] = :ets.lookup(table, full_key) + :atomics.add_get(counter, 1, increment) + end + end + + if count <= limit, do: {:allow, count}, else: {:deny, limit} + end + + @impl true + def init(opts) do + clean_period = Keyword.fetch!(opts, :clean_period) + table = Keyword.get(opts, :table, __MODULE__) + + ^table = + :ets.new(table, [ + :named_table, + :set, + :public, + {:read_concurrency, true}, + {:write_concurrency, :auto}, + {:decentralized_counters, true} + ]) + + schedule(clean_period) + {:ok, %{table: table, clean_period: clean_period}} + end + + @impl true + def handle_info(:clean, state) do + clean(state.table) + schedule(state.clean_period) + {:noreply, state} + end + + defp schedule(clean_period) do + Process.send_after(self(), :clean, clean_period) + end + + defp clean(table) do + ms = [{{{:_, :_}, :_, :"$1"}, [], [{:<, :"$1", {:const, now()}}]}] + :ets.select_delete(table, ms) + end + + @compile inline: [now: 0] + defp now do + System.system_time(:millisecond) + end +end diff --git a/lib/plausible/repo.ex b/lib/plausible/repo.ex new file mode 100644 index 0000000000000000000000000000000000000000..6a3e17c9879ea48ecec39740465838680e825ebc --- /dev/null +++ b/lib/plausible/repo.ex @@ -0,0 +1,15 @@ +defmodule Plausible.Repo do + use Ecto.Repo, + otp_app: :plausible, + adapter: Ecto.Adapters.Postgres + + use Plausible.Audit.Repo + + defmacro __using__(_) do + quote do + alias Plausible.Repo + import Ecto + import Ecto.Query, only: [from: 1, from: 2] + end + end +end diff --git a/lib/plausible/request_logger.ex b/lib/plausible/request_logger.ex new file mode 100644 index 0000000000000000000000000000000000000000..747d66f10d577b6d8fe397239469966d9bb1ecbe --- /dev/null +++ b/lib/plausible/request_logger.ex @@ -0,0 +1,19 @@ +defmodule Plausible.RequestLogger do + @moduledoc """ + Custom request logger which: + - Includes query parameters on the same line + - Includes request duration on the same line + """ + + require Logger + + def log_request(_, %{duration: duration}, %{conn: conn}, _) do + duration_ms = System.convert_time_unit(duration, :native, :millisecond) + path = path_with_params(conn.request_path, conn.query_string) + + Logger.info("(#{conn.status}) #{conn.method} #{path} took #{duration_ms}ms") + end + + defp path_with_params(request_path, ""), do: request_path + defp path_with_params(request_path, query_string), do: request_path <> "?" <> query_string +end diff --git a/lib/plausible/s3.ex b/lib/plausible/s3.ex new file mode 100644 index 0000000000000000000000000000000000000000..1d471a43fd24df70bd677901699a4b10686723c1 --- /dev/null +++ b/lib/plausible/s3.ex @@ -0,0 +1,152 @@ +defmodule Plausible.S3 do + @moduledoc """ + Helper functions for S3 exports/imports. + """ + + @doc """ + Returns the pre-configured S3 bucket for CSV exports. + + config :plausible, Plausible.S3, + exports_bucket: System.fetch_env!("S3_EXPORTS_BUCKET") + + Example: + + iex> exports_bucket() + "test-exports" + + """ + @spec exports_bucket :: String.t() + def exports_bucket, do: config(:exports_bucket) + + @doc """ + Returns the pre-configured S3 bucket for CSV imports. + + config :plausible, Plausible.S3, + imports_bucket: System.fetch_env!("S3_IMPORTS_BUCKET") + + Example: + + iex> imports_bucket() + "test-imports" + + """ + @spec imports_bucket :: String.t() + def imports_bucket, do: config(:imports_bucket) + + defp config, do: Application.fetch_env!(:plausible, __MODULE__) + defp config(key), do: Keyword.fetch!(config(), key) + + @doc """ + Presigns an upload for an imported file. + + In the current implementation the bucket always goes into the path component. + + Example: + + iex> upload = import_presign_upload(_site_id = 123, _filename = "imported_browsers.csv") + iex> true = String.ends_with?(upload.s3_url, "/test-imports/123/imported_browsers.csv") + iex> true = String.contains?(upload.presigned_url, "/test-imports/123/imported_browsers.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&") + + """ + def import_presign_upload(site_id, filename) do + config = ExAws.Config.new(:s3) + s3_path = Path.join(Integer.to_string(site_id), filename) + bucket = imports_bucket() + {:ok, presigned_url} = ExAws.S3.presigned_url(config, :put, bucket, s3_path) + %{s3_url: extract_s3_url(presigned_url), presigned_url: presigned_url} + end + + # to make ClickHouse see MinIO in dev and test envs we replace + # the host in the S3 URL with host.docker.internal or whatever's set in $MINIO_HOST_FOR_CLICKHOUSE + if Mix.env() in [:dev, :test, :ce_dev, :ce_test, :e2e_test] do + defp extract_s3_url(presigned_url) do + [s3_url, _] = String.split(presigned_url, "?") + default_ch_host = unless System.get_env("CI"), do: "host.docker.internal" + ch_host = System.get_env("MINIO_HOST_FOR_CLICKHOUSE", default_ch_host) + URI.to_string(%URI{URI.parse(s3_url) | host: ch_host}) + end + else + defp extract_s3_url(presigned_url) do + [s3_url, _] = String.split(presigned_url, "?") + s3_url + end + end + + @doc """ + Chunks and uploads Zip archive to the provided S3 destination. + + In the current implementation the bucket always goes into the path component. + """ + @spec export_upload_multipart(Enumerable.t(), String.t(), Path.t(), String.t()) :: :ok + def export_upload_multipart(stream, s3_bucket, s3_path, filename) do + # 5 MiB is the smallest chunk size AWS S3 supports + chunk_into_parts(stream, 5 * 1024 * 1024) + |> ExAws.S3.upload(s3_bucket, s3_path, + content_disposition: Plausible.Exports.content_disposition(filename), + content_type: "application/zip", + timeout: :infinity + ) + |> ExAws.request!() + + :ok + end + + @doc """ + Returns a presigned URL to download the exported Zip archive from S3. + The URL expires in 300 seconds, which should be enough for a redirect. + + In the current implementation the bucket always goes into the path component. + """ + @spec download_url(String.t(), Path.t()) :: :uri_string.uri_string() + def download_url(s3_bucket, s3_path) do + config = ExAws.Config.new(:s3) + + {:ok, download_url} = + ExAws.S3.presigned_url(config, :get, s3_bucket, s3_path, expires_in: 300) + + download_url + end + + defp chunk_into_parts(stream, min_part_size) do + Stream.chunk_while( + stream, + _acc = %{buffer_size: 0, buffer: [], min_part_size: min_part_size}, + _chunk_fun = &buffer_until_big_enough/2, + _after_fun = &flush_leftovers/1 + ) + end + + defp buffer_until_big_enough(data, acc) do + %{buffer_size: prev_buffer_size, buffer: prev_buffer, min_part_size: min_part_size} = acc + new_buffer_size = prev_buffer_size + IO.iodata_length(data) + new_buffer = [prev_buffer | data] + + if new_buffer_size > min_part_size do + # NOTE: PR to make ExAws.Operation.ExAws.Operation.S3.put_content_length_header/3 accept iodata + {:cont, IO.iodata_to_binary(new_buffer), %{acc | buffer_size: 0, buffer: []}} + else + {:cont, %{acc | buffer_size: new_buffer_size, buffer: new_buffer}} + end + end + + defp flush_leftovers(acc) do + # NOTE: PR to make ExAws.Operation.ExAws.Operation.S3.put_content_length_header/3 accept iodata + {:cont, IO.iodata_to_binary(acc.buffer), %{acc | buffer_size: 0, buffer: []}} + end + + @doc """ + Returns `access_key_id` and `secret_access_key` to be used by ClickHouse during imports from S3. + + Example: + + iex> import_clickhouse_credentials() + %{access_key_id: "minioadmin", secret_access_key: "minioadmin"} + + """ + @spec import_clickhouse_credentials :: + %{access_key_id: String.t(), secret_access_key: String.t()} + def import_clickhouse_credentials do + %{access_key_id: access_key_id, secret_access_key: secret_access_key} = ExAws.Config.new(:s3) + %{access_key_id: access_key_id, secret_access_key: secret_access_key} + end +end diff --git a/lib/plausible/s3/client.ex b/lib/plausible/s3/client.ex new file mode 100644 index 0000000000000000000000000000000000000000..54817c8da41cf9b39bdcd4be005204e64a35c0c7 --- /dev/null +++ b/lib/plausible/s3/client.ex @@ -0,0 +1,17 @@ +defmodule Plausible.S3.Client do + @moduledoc false + @behaviour ExAws.Request.HttpClient + + @impl true + def request(method, url, body, headers, opts) do + req = Finch.build(method, url, headers, body) + + case Finch.request(req, Plausible.Finch, opts) do + {:ok, %Finch.Response{status: status, headers: headers, body: body}} -> + {:ok, %{status_code: status, headers: headers, body: body}} + + {:error, reason} -> + {:error, %{reason: reason}} + end + end +end diff --git a/lib/plausible/segments/filters.ex b/lib/plausible/segments/filters.ex new file mode 100644 index 0000000000000000000000000000000000000000..bec2c14f9e612810808915eaa678e25505983ae1 --- /dev/null +++ b/lib/plausible/segments/filters.ex @@ -0,0 +1,135 @@ +defmodule Plausible.Segments.Filters do + @moduledoc """ + This module contains functions that enable resolving segments in filters. + """ + alias Plausible.Segments + alias Plausible.Stats.{Filters, ApiQueryParser, QueryError} + + @max_segment_filters_count 10 + + @doc """ + Finds unique segment IDs used in query filters. + + ## Examples + iex> get_segment_ids([[:not, [:is, "segment", [10, 20]]], [:contains, "visit:entry_page", ["blog"]]]) + {:ok, [10, 20]} + + iex> get_segment_ids([[:and, [[:is, "segment", Enum.to_list(1..6)], [:is, "segment", Enum.to_list(1..6)]]]]) + {:error, %Plausible.Stats.QueryError{code: :invalid_filters, message: "Invalid filters. You can only use up to 10 segment filters in a query."}} + """ + def get_segment_ids(filters) do + ids = + filters + |> Filters.traverse() + |> Enum.flat_map(fn + {[_operation, "segment", clauses], _} -> clauses + _ -> [] + end) + + if length(ids) > @max_segment_filters_count do + {:error, + %QueryError{ + code: :invalid_filters, + message: + "Invalid filters. You can only use up to #{@max_segment_filters_count} segment filters in a query." + }} + else + {:ok, Enum.uniq(ids)} + end + end + + def preload_needed_segments(%Plausible.Site{} = site, filters) do + with {:ok, segment_ids} <- get_segment_ids(filters), + {:ok, segments} <- + Segments.get_many( + site, + segment_ids, + fields: [:id, :segment_data] + ), + {:ok, segments_by_id} <- + {:ok, + Enum.into( + segments, + %{}, + fn %Segments.Segment{id: id, segment_data: segment_data} -> + case ApiQueryParser.parse_filters(segment_data["filters"]) do + {:ok, filters} -> {id, filters} + _ -> {id, nil} + end + end + )}, + :ok <- + if(Enum.any?(segment_ids, fn id -> is_nil(Map.get(segments_by_id, id)) end), + do: + {:error, + %QueryError{ + code: :invalid_filters, + message: "Invalid filters. Some segments don't exist or aren't accessible." + }}, + else: :ok + ) do + {:ok, segments_by_id} + end + end + + defp expand_first_level_and_filters(filter) do + case filter do + [:and, clauses] -> clauses + filter -> [filter] + end + end + + defp replace_segment_with_filter_tree([_, "segment", clauses], preloaded_segments) do + if length(clauses) == 1 do + [[:and, Map.get(preloaded_segments, Enum.at(clauses, 0))]] + else + [[:or, Enum.map(clauses, fn id -> [:and, Map.get(preloaded_segments, id)] end)]] + end + end + + defp replace_segment_with_filter_tree(_filter, _preloaded_segments) do + nil + end + + @doc """ + ## Examples + + iex> resolve_segments([[:is, "visit:entry_page", ["/home"]]], %{}) + {:ok, [[:is, "visit:entry_page", ["/home"]]]} + + iex> resolve_segments([[:is, "visit:entry_page", ["/home"]], [:is, "segment", [1]]], %{1 => [[:contains, "visit:entry_page", ["blog"]], [:is, "visit:country", ["PL"]]]}) + {:ok, [ + [:is, "visit:entry_page", ["/home"]], + [:contains, "visit:entry_page", ["blog"]], + [:is, "visit:country", ["PL"]] + ]} + + iex> resolve_segments([[:is, "visit:entry_page", ["/home"]], [:is, "segment", [1]], [:is, "segment", [2]]], %{1 => [[:is, "visit:country", ["PL"]]], 2 => [[:is, "event:goal", ["Signup"]]]}) + {:ok, [ + [:is, "visit:entry_page", ["/home"]], + [:is, "visit:country", ["PL"]], + [:is, "event:goal", ["Signup"]] + ]} + + + iex> resolve_segments([[:is, "segment", [1, 2]]], %{1 => [[:contains, "event:goal", ["Singup"]], [:is, "visit:country", ["PL"]]], 2 => [[:contains, "event:goal", ["Sauna"]], [:is, "visit:country", ["EE"]]]}) + {:ok, [ + [:or, [ + [:and, [[:contains, "event:goal", ["Singup"]], [:is, "visit:country", ["PL"]]]], + [:and, [[:contains, "event:goal", ["Sauna"]], [:is, "visit:country", ["EE"]]]]] + ] + ]} + """ + def resolve_segments(original_filters, preloaded_segments) do + if map_size(preloaded_segments) > 0 do + {:ok, + original_filters + |> Filters.transform_filters(fn f -> + replace_segment_with_filter_tree(f, preloaded_segments) + end) + |> Filters.transform_filters(&expand_first_level_and_filters/1)} + else + {:ok, original_filters} + end + end +end diff --git a/lib/plausible/segments/segment.ex b/lib/plausible/segments/segment.ex new file mode 100644 index 0000000000000000000000000000000000000000..1187714013533e02463cc6fc5c194b2203c233da --- /dev/null +++ b/lib/plausible/segments/segment.ex @@ -0,0 +1,155 @@ +defmodule Plausible.Segments.Segment do + @moduledoc """ + Schema for segments. Segments are saved filter combinations. + """ + use Plausible + use Ecto.Schema + import Ecto.Changeset + alias Plausible.Stats.{ApiQueryParser, QueryBuilder, ParsedQueryParams, QueryError} + + @segment_types [:personal, :site] + + @type t() :: %__MODULE__{} + + schema "segments" do + field :name, :string + field :type, Ecto.Enum, values: @segment_types + field :segment_data, :map + + # owner ID can be null (aka segment is dangling) when the original owner is deassociated from the site + # the segment is dangling until another user edits it: the editor becomes the new owner + belongs_to :owner, Plausible.Auth.User, foreign_key: :owner_id + belongs_to :site, Plausible.Site + + timestamps() + end + + def changeset(segment, attrs) do + segment + |> cast(attrs, [ + :name, + :segment_data, + :type, + :owner_id + ]) + |> validate_required([:name, :segment_data, :site_id, :type, :owner_id]) + |> validate_length(:name, count: :bytes, min: 1, max: 255) + |> foreign_key_constraint(:site_id) + |> foreign_key_constraint(:owner_id) + |> validate_only_known_properties_present() + |> validate_segment_data_filters() + |> validate_segment_data_labels() + |> validate_json_byte_length(:segment_data, max: 5 * 1024) + end + + defp validate_only_known_properties_present(%Ecto.Changeset{} = changeset) do + case get_field(changeset, :segment_data) do + segment_data when is_map(segment_data) -> + if Enum.any?(Map.keys(segment_data) -- ["filters", "labels"]) do + add_error( + changeset, + :segment_data, + "must not contain any other property except \"filters\" and \"labels\"" + ) + else + changeset + end + + _ -> + changeset + end + end + + defp validate_segment_data_filters(%Ecto.Changeset{} = changeset) do + case get_field(changeset, :segment_data) do + %{"filters" => filters} when is_list(filters) and length(filters) > 0 -> + changeset + + _ -> + add_error( + changeset, + :segment_data, + "property \"filters\" must be an array with at least one member" + ) + end + end + + defp validate_segment_data_labels(%Ecto.Changeset{} = changeset) do + case get_field(changeset, :segment_data) do + %{"labels" => labels} when not is_map(labels) -> + add_error(changeset, :segment_data, "property \"labels\" must be map or nil") + + _ -> + changeset + end + end + + defp validate_json_byte_length(%Ecto.Changeset{} = changeset, field_key, opts) do + field = get_field(changeset, field_key) + max = Keyword.get(opts, :max, 0) + + if :erlang.byte_size(Jason.encode!(field)) > max do + add_error(changeset, field_key, "should be at most %{count} byte(s)", count: max) + else + changeset + end + end + + def validate_segment_data_if_exists( + %Plausible.Site{} = _site, + nil = _segment_data, + _restricted_depth? + ), + do: :ok + + def validate_segment_data_if_exists(%Plausible.Site{} = site, segment_data, restricted_depth?) do + validate_segment_data(site, segment_data, restricted_depth?) + end + + @spec validate_segment_data(Plausible.Site.t(), map(), boolean()) :: + :ok | {:error, {:invalid_filters, String.t()}} + @doc """ + Checks whether the filters in segment_data are valid by attempting to build + a `%Query{}` struct with them. The `metrics` and `input_date_range` values + do not matter as long as they don't affect filters being the only field + affecting query validity. + """ + def validate_segment_data(%Plausible.Site{} = site, %{"filters" => filters}, restricted_depth?) do + with {:ok, parsed_filters} <- ApiQueryParser.parse_filters(filters), + {:ok, _} <- + QueryBuilder.build(site, %ParsedQueryParams{ + metrics: [:visitors], + input_date_range: {:last_n_days, 7}, + filters: parsed_filters + }), + :ok <- maybe_validate_filters_depth(parsed_filters, restricted_depth?) do + :ok + else + {:error, %QueryError{code: :invalid_filters, message: message}} -> + {:error, {:invalid_filters, message}} + + {:error, :deep_filters_not_supported} -> + {:error, {:invalid_filters, "Invalid filters. Deep filters are not supported."}} + end + end + + defp maybe_validate_filters_depth(filters, restricted_depth?) + + defp maybe_validate_filters_depth(_filters, false), do: :ok + + defp maybe_validate_filters_depth(filters, true) do + if Enum.all?(filters, &dashboard_compatible_filter?/1) do + :ok + else + {:error, :deep_filters_not_supported} + end + end + + defp dashboard_compatible_filter?(filter) do + case filter do + [operation, dimension, _clauses] when is_atom(operation) and is_binary(dimension) -> true + [:has_not_done, _] -> true + _ -> false + end + end +end diff --git a/lib/plausible/segments/segments.ex b/lib/plausible/segments/segments.ex new file mode 100644 index 0000000000000000000000000000000000000000..8b1556e41c88fdd378d2dadd01206e1ae4247ab0 --- /dev/null +++ b/lib/plausible/segments/segments.ex @@ -0,0 +1,464 @@ +defmodule Plausible.Segments do + @moduledoc """ + Module for accessing Segments. + """ + alias Plausible.Segments.Segment + alias Plausible.Repo + import Ecto.Query + + @roles_with_personal_segments [:billing, :viewer, :editor, :admin, :owner, :super_admin] + @roles_with_maybe_site_segments [:editor, :admin, :owner, :super_admin] + + @type error_not_enough_permissions() :: {:error, :not_enough_permissions} + @type error_segment_not_found() :: {:error, :segment_not_found} + @type error_segment_limit_reached() :: {:error, :segment_limit_reached} + @type error_invalid_segment() :: {:error, {:invalid_segment, Keyword.t()}} + @type unknown_error() :: {:error, any()} + + @max_segments 500 + + def get_all_for_site(%Plausible.Site{} = site, site_role) do + fields = [:id, :name, :type, :inserted_at, :updated_at, :segment_data] + + cond do + site_role in [:public] -> + {:ok, + Repo.all( + from(segment in Segment, + select: ^fields, + where: segment.site_id == ^site.id, + order_by: [desc: segment.updated_at, desc: segment.id] + ) + )} + + site_role in roles_with_personal_segments() or site_role in roles_with_maybe_site_segments() -> + fields = fields ++ [:owner_id] + + {:ok, + Repo.all( + from(segment in Segment, + select: ^fields, + where: segment.site_id == ^site.id, + order_by: [desc: segment.updated_at, desc: segment.id], + preload: [:owner] + ) + )} + + true -> + {:error, :not_enough_permissions} + end + end + + @spec get_many(Plausible.Site.t(), list(pos_integer()), Keyword.t()) :: + {:ok, [Segment.t()]} + def get_many(%Plausible.Site{} = _site, segment_ids, _opts) + when segment_ids == [] do + {:ok, []} + end + + def get_many(%Plausible.Site{} = site, segment_ids, opts) + when is_list(segment_ids) do + fields = Keyword.get(opts, :fields, [:id]) + + query = + from(segment in Segment, + select: ^fields, + where: segment.site_id == ^site.id, + where: segment.id in ^segment_ids + ) + + {:ok, Repo.all(query)} + end + + def search_by_name(%Plausible.Site{} = site, input) do + type = :site + fields = [:id, :name] + + input_empty? = is_nil(input) or (is_binary(input) and String.trim(input) == "") + + base_query = + from(segment in Segment, + where: segment.site_id == ^site.id, + where: segment.type == ^type, + limit: 20 + ) + + query = + if input_empty? do + from([segment] in base_query, + select: ^fields, + order_by: [desc: segment.updated_at] + ) + else + from([segment] in base_query, + select: %{ + id: segment.id, + name: segment.name, + match_rank: + fragment( + "CASE + WHEN lower(?) = lower(?) THEN 0 -- exact match + WHEN lower(?) LIKE lower(?) || '%' THEN 1 -- starts with + WHEN lower(?) LIKE '% ' || lower(?) || '%' THEN 2 -- after a space + WHEN lower(?) ILIKE ? THEN 3 -- anywhere + END AS match_rank", + segment.name, + ^input, + segment.name, + ^input, + segment.name, + ^input, + segment.name, + ^"%#{input}%" + ), + pos: + fragment( + "position(lower(?) IN lower(?)) AS pos", + ^input, + segment.name + ), + len_diff: fragment("abs(length(?) - length(?)) AS len_diff", segment.name, ^input) + }, + where: fragment("? ilike ?", segment.name, ^"%#{input}%"), + order_by: fragment("match_rank asc, pos asc, len_diff asc, updated_at desc") + ) + end + + {:ok, Repo.all(query)} + end + + @spec get_one(pos_integer(), Plausible.Site.t(), atom(), pos_integer() | nil) :: + {:ok, Segment.t()} + | error_not_enough_permissions() + | error_segment_not_found() + def get_one(user_id, site, site_role, segment_id) do + if site_role in roles_with_personal_segments() do + case do_get_one(user_id, site.id, segment_id) do + %Segment{} = segment -> {:ok, segment} + nil -> {:error, :segment_not_found} + end + else + {:error, :not_enough_permissions} + end + end + + @spec insert_one(pos_integer(), Plausible.Site.t(), atom(), map()) :: + {:ok, Segment.t()} + | error_not_enough_permissions() + | error_invalid_segment() + | error_segment_limit_reached() + | unknown_error() + + def insert_one( + user_id, + %Plausible.Site{} = site, + site_role, + %{} = params + ) do + with :ok <- can_insert_one?(site, site_role, params), + %{valid?: true} = changeset <- + Segment.changeset( + %Segment{site_id: site.id}, + Map.merge(params, %{"owner_id" => user_id}) + ), + :ok <- + Segment.validate_segment_data(site, params["segment_data"], true) do + {:ok, changeset |> Repo.insert!() |> Repo.preload(:owner)} + else + %{valid?: false, errors: errors} -> + {:error, {:invalid_segment, errors}} + + {:error, {:invalid_filters, message}} -> + {:error, {:invalid_segment, segment_data: {message, []}}} + + {:error, _type} = error -> + error + end + end + + @spec update_one(pos_integer(), Plausible.Site.t(), atom(), pos_integer(), map()) :: + {:ok, Segment.t()} + | error_not_enough_permissions() + | error_invalid_segment() + | unknown_error() + + def update_one( + user_id, + %Plausible.Site{} = site, + site_role, + segment_id, + %{} = params + ) do + with {:ok, segment} <- get_one(user_id, site, site_role, segment_id), + :ok <- can_update_one?(site, site_role, params, segment.type), + %{valid?: true} = changeset <- + Segment.changeset( + segment, + Map.merge(params, %{"owner_id" => user_id}) + ), + :ok <- + Segment.validate_segment_data_if_exists( + site, + params["segment_data"], + true + ) do + Repo.update!(changeset) + + {:ok, Repo.reload!(segment) |> Repo.preload(:owner)} + else + %{valid?: false, errors: errors} -> + {:error, {:invalid_segment, errors}} + + {:error, {:invalid_filters, message}} -> + {:error, {:invalid_segment, segment_data: {message, []}}} + + {:error, _type} = error -> + error + end + end + + def update_goal_in_segments( + %Plausible.Site{} = site, + %Plausible.Goal{} = stale_goal, + %Plausible.Goal{} = updated_goal + ) do + # Looks for a pattern like ["is", "event:goal", [......]] in the filters structure. + # Added a bunch of whitespace matchers to make sure it's tolerant of valid JSON formatting + goal_filter_regex = + ~s(.*?\\[\s*"is",\s*"event:goal",\s*\\[.*?"#{Regex.escape(stale_goal.display_name)}".*?\\]\s*\\].*?) + + segments_to_update = + from( + s in Segment, + where: s.site_id == ^site.id, + where: fragment("?['filters']::text ~ ?", s.segment_data, ^goal_filter_regex) + ) + + stale_goal_name = stale_goal.display_name + + for segment <- Repo.all(segments_to_update) do + updated_filters = + Plausible.Stats.Filters.transform_filters(segment.segment_data["filters"], fn + ["is", "event:goal", clauses] -> + new_clauses = + Enum.map(clauses, fn + ^stale_goal_name -> updated_goal.display_name + clause -> clause + end) + + [["is", "event:goal", new_clauses]] + + _ -> + nil + end) + + updated_segment_data = Map.put(segment.segment_data, "filters", updated_filters) + + from( + s in Segment, + where: s.id == ^segment.id, + update: [set: [segment_data: ^updated_segment_data]] + ) + |> Repo.update_all([]) + end + + :ok + end + + def after_user_removed_from_site(site, user) do + Repo.delete_all( + from segment in Segment, + where: segment.site_id == ^site.id, + where: segment.owner_id == ^user.id, + where: segment.type == :personal + ) + + Repo.update_all( + from(segment in Segment, + where: segment.site_id == ^site.id, + where: segment.owner_id == ^user.id, + where: segment.type == :site, + update: [set: [owner_id: nil]] + ), + [] + ) + end + + def after_user_removed_from_team(team, user) do + team_sites_q = + from( + site in Plausible.Site, + where: site.team_id == ^team.id, + where: parent_as(:segment).site_id == site.id + ) + + Repo.delete_all( + from segment in Segment, + as: :segment, + where: segment.owner_id == ^user.id, + where: segment.type == :personal, + where: exists(team_sites_q) + ) + + Repo.update_all( + from(segment in Segment, + as: :segment, + where: segment.owner_id == ^user.id, + where: segment.type == :site, + where: exists(team_sites_q), + update: [set: [owner_id: nil]] + ), + [] + ) + end + + def user_removed(user) do + Repo.delete_all( + from segment in Segment, + as: :segment, + where: segment.owner_id == ^user.id, + where: segment.type == :personal + ) + + # Site segments are set to owner=null via ON DELETE SET NULL + end + + def delete_one(user_id, %Plausible.Site{} = site, site_role, segment_id) do + with {:ok, segment} <- get_one(user_id, site, site_role, segment_id) do + cond do + segment.type == :site and site_role in roles_with_maybe_site_segments() -> + {:ok, do_delete_one(segment)} + + segment.type == :personal and site_role in roles_with_personal_segments() -> + {:ok, do_delete_one(segment)} + + true -> + {:error, :not_enough_permissions} + end + end + end + + def get_related_shared_links( + _site, + _site_role, + nil + ) do + {:error, :segment_not_found} + end + + def get_related_shared_links( + %Plausible.Site{} = site, + site_role, + segment_id + ) do + if site_role in roles_with_personal_segments() do + {:ok, + Repo.all( + from(shared_link in Plausible.Site.SharedLink, + select: [:name], + where: shared_link.segment_id == ^segment_id, + where: shared_link.site_id == ^site.id + ) + )} + else + {:error, :not_enough_permissions} + end + end + + @spec do_get_one(pos_integer(), pos_integer(), pos_integer() | nil) :: + Segment.t() | nil + defp do_get_one(user_id, site_id, segment_id) + + defp do_get_one(_user_id, _site_id, nil) do + nil + end + + defp do_get_one(user_id, site_id, segment_id) do + query = + from(segment in Segment, + where: segment.site_id == ^site_id, + where: segment.id == ^segment_id, + where: segment.type == :site or segment.owner_id == ^user_id, + preload: [:owner] + ) + + Repo.one(query) + end + + defp do_delete_one(segment) do + Repo.delete!(segment) + segment + end + + defp can_update_one?(%Plausible.Site{} = site, site_role, params, existing_segment_type) do + updating_to_site_segment? = params["type"] == "site" + + cond do + (existing_segment_type == :site or + updating_to_site_segment?) and site_role in roles_with_maybe_site_segments() and + site_segments_available?(site) -> + :ok + + existing_segment_type == :personal and not updating_to_site_segment? and + site_role in roles_with_personal_segments() -> + :ok + + true -> + {:error, :not_enough_permissions} + end + end + + defp can_insert_one?(%Plausible.Site{} = site, site_role, params) do + cond do + count_segments(site.id) >= @max_segments -> + {:error, :segment_limit_reached} + + params["type"] == "site" and site_role in roles_with_maybe_site_segments() and + site_segments_available?(site) -> + :ok + + params["type"] == "personal" and + site_role in roles_with_personal_segments() -> + :ok + + true -> + {:error, :not_enough_permissions} + end + end + + defp count_segments(site_id) do + from(segment in Segment, + where: segment.site_id == ^site_id + ) + |> Repo.aggregate(:count, :id) + end + + def to_response_map(%Segment{} = segment, %Plausible.Site{} = site) do + %{ + id: segment.id, + name: segment.name, + type: segment.type, + segment_data: segment.segment_data, + owner_id: segment.owner_id, + owner_name: owner_name(segment), + inserted_at: shift_to_site_tz(segment.inserted_at, site.timezone), + updated_at: shift_to_site_tz(segment.updated_at, site.timezone) + } + end + + defp owner_name(%Segment{owner_id: nil}), do: nil + defp owner_name(%Segment{owner: %Plausible.Auth.User{name: name}}), do: name + defp owner_name(%Segment{}), do: nil + + defp shift_to_site_tz(%NaiveDateTime{} = naive_utc, timezone) do + naive_utc + |> DateTime.from_naive!("Etc/UTC") + |> Plausible.Times.to_naive_datetime!(timezone) + end + + def roles_with_personal_segments(), do: @roles_with_personal_segments + def roles_with_maybe_site_segments(), do: @roles_with_maybe_site_segments + + def site_segments_available?(%Plausible.Site{} = site), + do: Plausible.Billing.Feature.SiteSegments.check_availability(site.team) == :ok +end diff --git a/lib/plausible/sentry/client.ex b/lib/plausible/sentry/client.ex new file mode 100644 index 0000000000000000000000000000000000000000..7cbf28839a934797f0297fca352d17375a69e60d --- /dev/null +++ b/lib/plausible/sentry/client.ex @@ -0,0 +1,32 @@ +defmodule Plausible.Sentry.Client do + @behaviour Sentry.HTTPClient + + defguardp is_redirect(status) when is_integer(status) and status >= 300 and status < 400 + + require Logger + + def post(url, headers, body) do + req_opts = Application.get_env(:plausible, __MODULE__)[:finch_request_opts] || [] + + :post + |> Finch.build(url, headers, body) + |> Finch.request(Plausible.Finch, req_opts) + |> handle_response() + end + + defp handle_response(resp) do + case resp do + {:ok, %{status: status, headers: _}} when is_redirect(status) -> + # Just playing safe here. hackney client didn't support those; redirects are opt-in in hackney + Logger.warning("Sentry returned a redirect that is not handled yet.") + {:error, :stop} + + {:ok, %{status: status, body: body, headers: headers}} -> + {:ok, status, headers, body} + + {:error, error} = e -> + Logger.warning("Sentry call failed with: #{inspect(error)}") + e + end + end +end diff --git a/lib/plausible/session/balancer.ex b/lib/plausible/session/balancer.ex new file mode 100644 index 0000000000000000000000000000000000000000..11235adcd0331870257728543bc98ef5603e2178 --- /dev/null +++ b/lib/plausible/session/balancer.ex @@ -0,0 +1,39 @@ +defmodule Plausible.Session.Balancer do + @moduledoc "Serialize session processing to avoid explicit locks" + use GenServer + + def start_link(id) do + GenServer.start_link(__MODULE__, id, name: via(id)) + end + + @impl true + def init(id) do + {:ok, %{id: id}} + end + + def dispatch(user_id, fun, opts) do + timeout = Keyword.fetch!(opts, :timeout) + local? = Keyword.get(opts, :local?, false) + + if local? do + fun.() + else + worker = :erlang.phash2(user_id, Plausible.Session.BalancerSupervisor.size()) + 1 + [{pid, _}] = Registry.lookup(Plausible.Session.Balancer.Registry, worker) + GenServer.call(pid, {:process, fun}, timeout) + end + end + + @impl true + def handle_call({:process, fun}, _from, state) do + try do + response = fun.() + {:reply, response, state} + rescue + e -> + {:reply, {:error, e}, state} + end + end + + defp via(id), do: {:via, Registry, {Plausible.Session.Balancer.Registry, id}} +end diff --git a/lib/plausible/session/balancer_supervisor.ex b/lib/plausible/session/balancer_supervisor.ex new file mode 100644 index 0000000000000000000000000000000000000000..2c456b7d1ae00aca6dcbae47ab122169073ac373 --- /dev/null +++ b/lib/plausible/session/balancer_supervisor.ex @@ -0,0 +1,33 @@ +defmodule Plausible.Session.BalancerSupervisor do + @moduledoc "Serialize session processing to avoid explicit locks" + use Supervisor + + if Mix.env() in [:test, :ce_test, :e2e_test] do + def size(), + do: 10 + else + def size(), do: 100 + end + + def start_link(_) do + Supervisor.start_link(__MODULE__, size(), name: __MODULE__) + end + + def init(size) do + children = + for id <- 1..size do + %{ + id: id, + start: {Plausible.Session.Balancer, :start_link, [id]}, + restart: :permanent + } + end + + Supervisor.init( + [ + {Registry, [keys: :unique, name: Plausible.Session.Balancer.Registry]} | children + ], + strategy: :one_for_one + ) + end +end diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.ex new file mode 100644 index 0000000000000000000000000000000000000000..05b3998e06cebff09364cc9a7cfd8841207f8f37 --- /dev/null +++ b/lib/plausible/session/cache_store.ex @@ -0,0 +1,165 @@ +defmodule Plausible.Session.CacheStore do + @moduledoc """ + Session management on the basis of incoming events. + """ + + alias Plausible.Session.WriteBuffer + + @lock_timeout 1000 + + @lock_telemetry_event [:plausible, :sessions, :cache, :lock] + + def lock_telemetry_event, do: @lock_telemetry_event + + def on_event(event, session_attributes, prev_user_id, opts \\ []) do + buffer_insert = Keyword.get(opts, :buffer_insert, &WriteBuffer.insert/1) + skip_balancer? = Keyword.get(opts, :skip_balancer?, false) + lock_requested_at = System.monotonic_time() + + try do + response = + Plausible.Session.Balancer.dispatch( + event.user_id, + fn -> + lock_duration = System.monotonic_time() - lock_requested_at + :telemetry.execute(@lock_telemetry_event, %{duration: lock_duration}, %{}) + + found_session = + find_session(event, event.user_id) || find_session(event, prev_user_id) + + handle_event(event, found_session, session_attributes, buffer_insert) + end, + timeout: @lock_timeout, + local?: skip_balancer? + ) + + case response do + {:error, e} -> raise e + _ -> {:ok, response} + end + catch + :exit, {:timeout, _} -> + Sentry.capture_message("Timeout while handling session event") + {:error, :timeout} + end + end + + defp handle_event(%{name: "engagement"} = event, found_session, _, _) do + if found_session do + # Make sure the session is kept active in the in-memory session cache + refresh_session_cache(found_session, event.timestamp) + + found_session + else + :no_session_for_engagement + end + end + + defp handle_event(event, found_session, session_attributes, buffer_insert) do + if found_session do + updated_session = update_session(found_session, event) + buffer_insert.([%{found_session | sign: -1}, %{updated_session | sign: 1}]) + update_session_cache(updated_session) + else + new_session = new_session_from_event(event, session_attributes) + buffer_insert.([new_session]) + update_session_cache(new_session) + end + end + + defp find_session(_domain, nil), do: nil + + defp find_session(event, user_id) do + from_cache = Plausible.Cache.Adapter.get(:sessions, {event.site_id, user_id}) + + case from_cache do + nil -> + nil + + session -> + if NaiveDateTime.diff(event.timestamp, session.timestamp, :minute) <= 30 do + session + end + end + end + + defp update_session_cache(session) do + key = {session.site_id, session.user_id} + Plausible.Cache.Adapter.put(:sessions, key, session, dirty?: true) + session + end + + defp refresh_session_cache(session, timestamp) do + session + |> Map.put(:timestamp, timestamp) + |> update_session_cache() + end + + defp update_session(session, event) do + pageview? = event.name == "pageview" + pageviews = if(pageview?, do: session.pageviews + 1, else: session.pageviews) + + %{ + session + | timestamp: event.timestamp, + entry_page: + if(session.entry_page == "" and pageview?, + do: event.pathname, + else: session.entry_page + ), + hostname: + if(pageview? and session.hostname == "", + do: event.hostname, + else: session.hostname + ), + exit_page: if(pageview?, do: event.pathname, else: session.exit_page), + exit_page_hostname: if(pageview?, do: event.hostname, else: session.exit_page_hostname), + is_bounce: + if(session.is_bounce, + do: not (pageviews >= 2 or (event.interactive? and not pageview?)), + else: session.is_bounce + ), + duration: NaiveDateTime.diff(event.timestamp, session.start) |> abs, + pageviews: pageviews, + events: session.events + 1 + } + end + + defp new_session_from_event(event, session_attributes) do + %Plausible.ClickhouseSessionV2{ + sign: 1, + session_id: Plausible.ClickhouseSessionV2.random_uint64(), + hostname: if(event.name == "pageview", do: event.hostname, else: ""), + site_id: event.site_id, + user_id: event.user_id, + entry_page: if(event.name == "pageview", do: event.pathname, else: ""), + exit_page: if(event.name == "pageview", do: event.pathname, else: ""), + exit_page_hostname: if(event.name == "pageview", do: event.hostname, else: ""), + is_bounce: event.name == "pageview" or not event.interactive?, + duration: 0, + pageviews: if(event.name == "pageview", do: 1, else: 0), + events: 1, + referrer: Map.get(session_attributes, :referrer), + click_id_param: Map.get(session_attributes, :click_id_param), + referrer_source: Map.get(session_attributes, :referrer_source), + utm_medium: Map.get(session_attributes, :utm_medium), + utm_source: Map.get(session_attributes, :utm_source), + utm_campaign: Map.get(session_attributes, :utm_campaign), + utm_content: Map.get(session_attributes, :utm_content), + utm_term: Map.get(session_attributes, :utm_term), + country_code: Map.get(session_attributes, :country_code), + subdivision1_code: Map.get(session_attributes, :subdivision1_code), + subdivision2_code: Map.get(session_attributes, :subdivision2_code), + city_geoname_id: Map.get(session_attributes, :city_geoname_id), + screen_size: Map.get(session_attributes, :screen_size), + operating_system: Map.get(session_attributes, :operating_system), + operating_system_version: Map.get(session_attributes, :operating_system_version), + browser: Map.get(session_attributes, :browser), + browser_version: Map.get(session_attributes, :browser_version), + timestamp: event.timestamp, + start: event.timestamp, + "entry_meta.key": Map.get(event, :"meta.key"), + "entry_meta.value": Map.get(event, :"meta.value") + } + end +end diff --git a/lib/plausible/session/salts.ex b/lib/plausible/session/salts.ex new file mode 100644 index 0000000000000000000000000000000000000000..a7fd2eca49fb3f918f2077df917b6319b0696feb --- /dev/null +++ b/lib/plausible/session/salts.ex @@ -0,0 +1,92 @@ +defmodule Plausible.Session.Salts do + use GenServer + use Plausible.Repo + + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: opts[:name] || __MODULE__) + end + + @impl true + def init(opts) do + name = opts[:name] || __MODULE__ + now = opts[:now] || DateTime.utc_now() + clean_old_salts(now) + + ^name = + :ets.new(name, [ + :named_table, + :set, + :protected, + {:read_concurrency, true} + ]) + + refresh(name, now) + + {:ok, name} + end + + def refresh(name, now) do + salts = Repo.all(from s in "salts", select: s.salt, order_by: [desc: s.id], limit: 2) + + state = + case salts do + [current, prev] -> + %{previous: prev, current: current} + + [current] -> + %{previous: nil, current: current} + + [] -> + new = generate_and_persist_new_salt(now) + %{previous: nil, current: new} + end + + true = :ets.insert(name, {:state, state}) + interval = Application.get_env(:plausible, __MODULE__)[:interval] || :timer.seconds(90) + Process.send_after(self(), {:refresh, now}, interval) + :ok + end + + def rotate(name \\ __MODULE__, now \\ DateTime.utc_now()) do + GenServer.call(name, {:rotate, now}) + end + + def fetch(name \\ __MODULE__) do + [state: state] = :ets.lookup(name, :state) + state + end + + @impl true + def handle_call({:rotate, now}, _from, name) do + current = fetch(name).current + clean_old_salts(now) + + state = + %{ + current: generate_and_persist_new_salt(now), + previous: current + } + + true = :ets.insert(name, {:state, state}) + {:reply, :ok, name} + end + + @impl true + def handle_info({:refresh, now}, name) do + refresh(name, now) + {:noreply, name} + end + + defp generate_and_persist_new_salt(now) do + salt = :crypto.strong_rand_bytes(16) + Repo.insert_all("salts", [%{salt: salt, inserted_at: now}]) + salt + end + + defp clean_old_salts(now) do + h48_ago = + DateTime.shift(now, hour: -48) + + Repo.delete_all(from s in "salts", where: s.inserted_at < ^h48_ago) + end +end diff --git a/lib/plausible/session/transfer.ex b/lib/plausible/session/transfer.ex new file mode 100644 index 0000000000000000000000000000000000000000..ab0103af84fa9ef681c4813e163948ed0bfd702f --- /dev/null +++ b/lib/plausible/session/transfer.ex @@ -0,0 +1,187 @@ +defmodule Plausible.Session.Transfer do + @moduledoc """ + Cross-deployment transfer for `:sessions` cache. + + It works by establishing a client-server architecture where: + - The "replica" one-time task retrieves `:sessions` data from other OS processes via Unix domain sockets + - The "primary" server process responds to requests for `:sessions` data via Unix domain sockets + - The "alive" process waits on shutdown for at least one replica, for 15 seconds + """ + + @behaviour Supervisor + + require Logger + alias Plausible.Session.Transfer.{TinySock, Alive} + alias Plausible.{Cache, ClickhouseSessionV2, Session} + + @cmd_list_cache_names :list + @cmd_dump_cache :get + @cmd_takeover_done :done + + def telemetry_event, do: [:plausible, :sessions, :takeover] + + @doc """ + Starts the `:sessions` transfer supervisor. + + Options: + - `:name` - the name of the supervisor (default: `Plausible.Session.Transfer`) + - `:base_path` - the base path for the Unix domain sockets (required) + """ + def start_link(opts) do + name = Keyword.get(opts, :name, __MODULE__) + base_path = Keyword.fetch!(opts, :base_path) + Supervisor.start_link(__MODULE__, base_path, name: name) + end + + @impl true + def init(nil) do + Logger.notice( + "Session transfer: ignoring, no socket base path configured (make sure ENABLE_SESSION_TRANSFER/PERSISTENT_CACHE_DIR are set)" + ) + + :ignore + end + + def init(base_path) do + File.mkdir_p!(base_path) + + replica = + Supervisor.child_spec( + {Task, fn -> init_takeover(base_path) end}, + id: :transfer_replica + ) + + given_counter = :counters.new(1, []) + parent = self() + + primary = + {TinySock, + base_path: base_path, + handler: fn message -> handle_replica(message, parent, given_counter) end} + + alive = + Supervisor.child_spec( + {Alive, + _until = fn -> + result = :counters.get(given_counter, 1) > 0 + + Logger.notice( + "Session transfer delayed shut down. Checking if session takeover happened?: #{result}" + ) + + result + end}, + shutdown: :timer.seconds(15) + ) + + Logger.notice("Session transfer init: #{base_path}") + Supervisor.init([replica, primary, alive], strategy: :one_for_one) + end + + @doc """ + Returns `true` if the transfer has been attempted (successfully or not). + Returns `false` if the transfer is still in progress. + """ + def attempted?(transfer_sup \\ __MODULE__) do + result = not replica_alive?(transfer_sup) + Logger.notice("Session transfer attempted?: #{result}") + result + end + + @doc """ + Returns the child specification for the `:sessions` transfer supervisor. + See `start_link/1` for options. + """ + def child_spec(opts) do + %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [opts]}, + type: :supervisor, + restart: :temporary + } + end + + defp handle_replica(request, parent, given_counter) do + Logger.notice( + "Session transfer message received at #{node()}: #{inspect(request, limit: 10)}" + ) + + case request do + {@cmd_list_cache_names, session_version} -> + if session_version == session_version() and attempted?(parent) do + Cache.Adapter.get_names(:sessions) + else + [] + end + + {@cmd_dump_cache, cache} -> + Cache.Adapter.cache2list(cache) + + @cmd_takeover_done -> + :counters.add(given_counter, 1, 1) + end + end + + defp init_takeover(base_path) do + started = System.monotonic_time() + + base_path + |> TinySock.list!() + |> Enum.sort_by(&file_stat_ctime/1, :asc) + |> Enum.each(&request_takeover/1) + + :telemetry.execute(telemetry_event(), %{duration: System.monotonic_time() - started}) + end + + defp request_takeover(sock) do + Logger.notice("Session transfer: requesting takeover at #{node()}") + + with {:ok, names} <- TinySock.call(sock, {@cmd_list_cache_names, session_version()}) do + tasks = Enum.map(names, fn name -> Task.async(fn -> takeover_cache(sock, name) end) end) + Task.await_many(tasks, :timer.seconds(10)) + end + after + Logger.notice("Session transfer: marking takeover as done at #{node()}") + TinySock.call(sock, @cmd_takeover_done) + end + + defp takeover_cache(sock, cache) do + Logger.notice("Session transfer: requesting cache #{cache} dump at #{node()}") + + with {:ok, records} <- TinySock.call(sock, {@cmd_dump_cache, cache}) do + Enum.each(records, fn record -> + {key, %ClickhouseSessionV2{} = session} = record + Cache.Adapter.put(:sessions, key, session) + end) + + Logger.notice("Session transfer: restored cache #{cache} at #{node()}") + end + end + + defp file_stat_ctime(path) do + case File.stat(path) do + {:ok, stat} -> stat.ctime + {:error, _} -> nil + end + end + + defp session_version do + [ + ClickhouseSessionV2.module_info(:md5), + Cache.Adapter.module_info(:md5), + Session.CacheStore.module_info(:md5), + Session.Transfer.module_info(:md5) + ] + end + + defp replica_alive?(transfer_sup) do + children = Supervisor.which_children(transfer_sup) + + replica = + Enum.find_value(children, fn {id, pid, _, _} -> id == :transfer_replica && pid end) + + is_pid(replica) and Process.alive?(replica) + catch + :exit, {:noproc, _} -> false + end +end diff --git a/lib/plausible/session/transfer/alive.ex b/lib/plausible/session/transfer/alive.ex new file mode 100644 index 0000000000000000000000000000000000000000..bfaab0ccdb3dd0ef8c79d19edc2af9b8fee83bbc --- /dev/null +++ b/lib/plausible/session/transfer/alive.ex @@ -0,0 +1,31 @@ +defmodule Plausible.Session.Transfer.Alive do + @moduledoc false + use GenServer + + @spec start_link((-> boolean)) :: GenServer.on_start() + def start_link(until) do + GenServer.start_link(__MODULE__, until) + end + + @impl true + def init(until) when is_function(until, 0) do + Process.flag(:trap_exit, true) + {:ok, until} + end + + @impl true + def terminate(_reason, until) do + loop(until) + end + + defp loop(until) do + case until.() do + true -> + :ok + + false -> + :timer.sleep(500) + loop(until) + end + end +end diff --git a/lib/plausible/session/transfer/tinysock.ex b/lib/plausible/session/transfer/tinysock.ex new file mode 100644 index 0000000000000000000000000000000000000000..f409895e054651b704dd8813f5babb61610f6952 --- /dev/null +++ b/lib/plausible/session/transfer/tinysock.ex @@ -0,0 +1,170 @@ +defmodule Plausible.Session.Transfer.TinySock do + @moduledoc false + use GenServer + require Logger + + @tag_data "tinysock" + @tag_size byte_size(@tag_data) + + def start_link(opts) do + base_path = Keyword.fetch!(opts, :base_path) + handler = Keyword.fetch!(opts, :handler) + GenServer.start_link(__MODULE__, {base_path, handler}) + end + + @impl true + def init({base_path, handler}) do + File.mkdir_p!(base_path) + socket = sock_listen_or_retry!(base_path) + + Process.flag(:trap_exit, true) + state = %{socket: socket, handler: handler} + for _ <- 1..10, do: spawn_acceptor(state) + + {:ok, state} + end + + @spec list!(Path.t()) :: [Path.t()] + def list!(base_path) do + for @tag_data <> _rand = name <- File.ls!(base_path) do + Path.join(base_path, name) + end + end + + @spec call(Path.t(), term, timeout) :: {:ok, reply :: term} | {:error, :timeout | :inet.posix()} + def call(sock_path, message, timeout \\ :timer.seconds(5)) do + with {:ok, socket} <- sock_connect_or_rm(sock_path, timeout) do + try do + with :ok <- sock_send(socket, :erlang.term_to_iovec(message)) do + sock_recv(socket, timeout) + end + after + sock_shut_and_close(socket) + end + end + end + + @impl true + def handle_cast(:accepted, state) do + spawn_acceptor(state) + {:noreply, state} + end + + @impl true + def handle_info({:EXIT, pid, reason}, state) do + case reason do + :normal -> + {:noreply, state} + + :emfile -> + Logger.error("tinysock ran out of file descriptors, stopping") + {:stop, reason, state} + + reason -> + log = "tinysock request handler #{inspect(pid)} terminating\n" <> format_error(reason) + Logger.error(log, crash_reason: reason) + {:noreply, state} + end + end + + defp format_error(reason) do + case reason do + {e, stacktrace} when is_list(stacktrace) -> Exception.format(:error, e, stacktrace) + _other -> Exception.format(:exit, reason) + end + end + + @impl true + def terminate(reason, %{socket: socket}) do + with {:ok, {:local, path}} <- :inet.sockname(socket), do: File.rm(path) + reason + end + + defp spawn_acceptor(%{socket: socket, handler: handler}) do + :proc_lib.spawn_link(__MODULE__, :accept, [_parent = self(), socket, handler]) + end + + @doc false + def accept(parent, listen_socket, handler) do + case :gen_tcp.accept(listen_socket, :infinity) do + {:ok, socket} -> + GenServer.cast(parent, :accepted) + handle_message(socket, handler) + + {:error, reason} -> + exit(reason) + end + end + + defp handle_message(socket, handler) do + {:ok, message} = sock_recv(socket, _timeout = :timer.seconds(5)) + sock_send(socket, :erlang.term_to_iovec(handler.(message))) + after + sock_shut_and_close(socket) + end + + @listen_opts mode: :binary, active: false, backlog: 1024 + @connect_opts mode: :binary, active: false + + defp sock_listen_or_retry!(base_path) do + sock_name = @tag_data <> Base.url_encode64(:crypto.strong_rand_bytes(4), padding: false) + sock_path = Path.join(base_path, sock_name) + + case :gen_tcp.listen(0, [{:ifaddr, {:local, sock_path}} | @listen_opts]) do + {:ok, socket} -> socket + {:error, :eaddrinuse} -> sock_listen_or_retry!(base_path) + {:error, reason} -> raise File.Error, path: sock_path, reason: reason, action: "bind" + end + end + + defp sock_connect_or_rm(sock_path, timeout) do + case :gen_tcp.connect({:local, sock_path}, 0, @connect_opts, timeout) do + {:ok, socket} -> + {:ok, socket} + + {:error, reason} = error -> + if reason != :timeout do + # removes stale socket file + # possible - but unlikely - race condition + File.rm(sock_path) + end + + error + end + end + + @dialyzer :no_improper_lists + defp sock_send(socket, data) do + :gen_tcp.send(socket, [<<@tag_data, IO.iodata_length(data)::64-little>> | data]) + end + + defp sock_recv(socket, timeout) do + with {:ok, <<@tag_data, size::64-little>>} <- :gen_tcp.recv(socket, @tag_size + 8, timeout), + {:ok, binary} <- sock_recv_continue(socket, size, timeout, []) do + try do + {:ok, :erlang.binary_to_term(binary, [:safe])} + rescue + e -> {:error, e} + end + end + end + + @five_mb 5 * 1024 * 1024 + + # for larger messages (>64MB) we need to read in chunks or we get {:error, :enomem} + defp sock_recv_continue(socket, size, timeout, acc) do + with {:ok, data} <- :gen_tcp.recv(socket, min(size, @five_mb), timeout) do + acc = [acc | data] + + case size - byte_size(data) do + 0 -> {:ok, IO.iodata_to_binary(acc)} + left -> sock_recv_continue(socket, left, timeout, acc) + end + end + end + + defp sock_shut_and_close(socket) do + :gen_tcp.shutdown(socket, :read_write) + :gen_tcp.close(socket) + end +end diff --git a/lib/plausible/session/write_buffer.ex b/lib/plausible/session/write_buffer.ex new file mode 100644 index 0000000000000000000000000000000000000000..1adcbbb67d530442fbab0d9097cd14589f70015a --- /dev/null +++ b/lib/plausible/session/write_buffer.ex @@ -0,0 +1,43 @@ +defmodule Plausible.Session.WriteBuffer do + @moduledoc false + + %{ + header: header, + insert_sql: insert_sql, + insert_opts: insert_opts, + fields: fields, + encoding_types: encoding_types + } = + Plausible.Ingestion.WriteBuffer.compile_time_prepare(Plausible.ClickhouseSessionV2) + + def child_spec(opts) do + opts = + Keyword.merge(opts, + name: __MODULE__, + header: unquote(header), + insert_sql: unquote(insert_sql), + insert_opts: unquote(insert_opts) + ) + + Plausible.Ingestion.WriteBuffer.child_spec(opts) + end + + def insert(sessions) do + row_binary = + sessions + |> Enum.map(fn %{is_bounce: is_bounce} = session -> + {:ok, is_bounce} = Plausible.ClickhouseSessionV2.BoolUInt8.dump(is_bounce) + session = %{session | is_bounce: is_bounce} + Enum.map(unquote(fields), fn field -> Map.fetch!(session, field) end) + end) + |> Ch.RowBinary._encode_rows(unquote(encoding_types)) + |> IO.iodata_to_binary() + + :ok = Plausible.Ingestion.WriteBuffer.insert(__MODULE__, row_binary) + {:ok, sessions} + end + + def flush do + Plausible.Ingestion.WriteBuffer.flush(__MODULE__) + end +end diff --git a/lib/plausible/shield/country_rule.ex b/lib/plausible/shield/country_rule.ex new file mode 100644 index 0000000000000000000000000000000000000000..a9234363eb876af5972ac1b5bd6c396e206366b5 --- /dev/null +++ b/lib/plausible/shield/country_rule.ex @@ -0,0 +1,45 @@ +defmodule Plausible.Shield.CountryRule do + @moduledoc """ + Schema for Country Block List + """ + use Ecto.Schema + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + @primary_key {:id, :binary_id, autogenerate: true} + schema "shield_rules_country" do + belongs_to :site, Plausible.Site + field :country_code, :string + field :action, Ecto.Enum, values: [:deny, :allow], default: :deny + field :added_by, :string + + # If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Shield.Rules.Country.Cache` + field :from_cache?, :boolean, virtual: true, default: false + timestamps() + end + + @unknown_country_code "ZZ" + + @spec unknown_country_code() :: String.t() + def unknown_country_code(), do: @unknown_country_code + + def changeset(rule, attrs) do + rule + |> cast(attrs, [:country_code]) + |> validate_required([:country_code]) + |> validate_length(:country_code, is: 2) + |> validate_change(:country_code, fn :country_code, cc -> + valid_codes = [@unknown_country_code | Enum.map(Location.Country.all(), & &1.alpha_2)] + + if cc in valid_codes do + [] + else + [country_code: "is invalid"] + end + end) + |> unique_constraint(:country_code, + name: :shield_rules_country_site_id_country_code_index + ) + end +end diff --git a/lib/plausible/shield/country_rule_cache.ex b/lib/plausible/shield/country_rule_cache.ex new file mode 100644 index 0000000000000000000000000000000000000000..9ac9563f49fe12ff22a602c5f4f46f7a82234a1a --- /dev/null +++ b/lib/plausible/shield/country_rule_cache.ex @@ -0,0 +1,65 @@ +defmodule Plausible.Shield.CountryRuleCache do + @moduledoc """ + Allows retrieving Country Rules by domain and country code + """ + alias Plausible.Shield.CountryRule + + import Ecto.Query + use Plausible.Cache + + @cache_name :country_blocklist_by_domain + + @cached_schema_fields ~w( + id + country_code + action + )a + + @impl true + def name(), do: @cache_name + + @impl true + def child_id(), do: :cache_country_blocklist + + @impl true + def count_all() do + Plausible.Repo.aggregate(CountryRule, :count) + end + + @impl true + def base_db_query() do + from rule in CountryRule, + inner_join: s in assoc(rule, :site), + select: { + s.domain, + s.domain_changed_from, + %{struct(rule, ^@cached_schema_fields) | from_cache?: true} + } + end + + @impl true + def get_from_source({domain, country_code}) do + query = + base_db_query() + |> where([rule, site], rule.country_code == ^country_code and site.domain == ^domain) + + case Plausible.Repo.one(query) do + {_, _, rule = %CountryRule{}} -> %CountryRule{rule | from_cache?: false} + _any -> nil + end + end + + @impl true + def unwrap_cache_keys(items) do + Enum.reduce(items, [], fn + {domain, nil, object}, acc -> + [{{domain, String.upcase(object.country_code)}, object} | acc] + + {domain, domain_changed_from, object}, acc -> + [ + {{domain, String.upcase(object.country_code)}, object}, + {{domain_changed_from, String.upcase(object.country_code)}, object} | acc + ] + end) + end +end diff --git a/lib/plausible/shield/hostname_rule.ex b/lib/plausible/shield/hostname_rule.ex new file mode 100644 index 0000000000000000000000000000000000000000..2f7f5295cd01983ac75e4829b5d9c59426f038b8 --- /dev/null +++ b/lib/plausible/shield/hostname_rule.ex @@ -0,0 +1,67 @@ +defmodule Plausible.Shield.HostnameRule do + @moduledoc """ + Schema for Hostnames allow list + """ + use Ecto.Schema + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + @primary_key {:id, :binary_id, autogenerate: true} + schema "shield_rules_hostname" do + belongs_to :site, Plausible.Site + field :hostname, :string + field :hostname_pattern, Plausible.Ecto.Types.CompiledRegex + field :action, Ecto.Enum, values: [:deny, :allow], default: :allow + field :added_by, :string + + # If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Shield.Rules.IP.Cache` + field :from_cache?, :boolean, virtual: true, default: false + timestamps() + end + + def changeset(rule \\ %__MODULE__{}, attrs) do + rule + |> cast(attrs, [:hostname]) + |> validate_required([:hostname]) + |> validate_length(:hostname, max: 250) + |> store_regex() + |> unique_constraint(:hostname_pattern, + name: :shield_rules_hostname_site_id_hostname_pattern_index, + error_key: :hostname, + message: "rule already exists" + ) + end + + defp store_regex(changeset) do + case fetch_change(changeset, :hostname) do + {:ok, hostname} -> + hostname + |> build_regex() + |> verify_and_put_regex(changeset) + + :error -> + changeset + end + end + + defp build_regex(hostname) do + regex = + hostname + |> Regex.escape() + |> String.replace("\\*\\*", ".*") + |> String.replace("\\*", ".*") + + "^#{regex}$" + end + + defp verify_and_put_regex(regex, changeset) do + case Regex.compile(regex) do + {:ok, _} -> + put_change(changeset, :hostname_pattern, regex) + + {:error, _} -> + add_error(changeset, :hostname, "could not compile regular expression") + end + end +end diff --git a/lib/plausible/shield/hostname_rule_cache.ex b/lib/plausible/shield/hostname_rule_cache.ex new file mode 100644 index 0000000000000000000000000000000000000000..e5ce6a067a8e9dad09b491c6dc85c573b9c61e5d --- /dev/null +++ b/lib/plausible/shield/hostname_rule_cache.ex @@ -0,0 +1,70 @@ +defmodule Plausible.Shield.HostnameRuleCache do + @moduledoc """ + Allows retrieving Hostname Rules by domain + """ + alias Plausible.Shield.HostnameRule + + import Ecto.Query + use Plausible.Cache + + @cache_name :hostname_allowlist_by_domain + + @cached_schema_fields ~w( + id + hostname_pattern + action + )a + + @impl true + def name(), do: @cache_name + + @impl true + def child_id(), do: :cache_hostname_blocklist + + @impl true + def count_all() do + Plausible.Repo.aggregate(HostnameRule, :count) + end + + @impl true + def base_db_query() do + from rule in HostnameRule, + inner_join: s in assoc(rule, :site), + select: { + s.domain, + s.domain_changed_from, + %{struct(rule, ^@cached_schema_fields) | from_cache?: true} + } + end + + @impl true + def get_from_source(domain) do + query = + base_db_query() + |> where([..., site], site.domain == ^domain) + + case Plausible.Repo.all(query) do + [_ | _] = results -> + Enum.map(results, fn {_, _, rule = %HostnameRule{}} -> + %HostnameRule{rule | from_cache?: false} + end) + + _ -> + nil + end + end + + @impl true + def unwrap_cache_keys(items) do + Enum.reduce(items, [], fn + {domain, nil, object}, acc -> + [{domain, object} | acc] + + {domain, domain_changed_from, object}, acc -> + [ + {domain, object}, + {domain_changed_from, object} | acc + ] + end) + end +end diff --git a/lib/plausible/shield/ip_rule.ex b/lib/plausible/shield/ip_rule.ex new file mode 100644 index 0000000000000000000000000000000000000000..fa725e99e3f81ff08503d9464613012c0232ed61 --- /dev/null +++ b/lib/plausible/shield/ip_rule.ex @@ -0,0 +1,42 @@ +defmodule Plausible.Shield.IPRule do + @moduledoc """ + Schema for IP block list + """ + use Ecto.Schema + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + @primary_key {:id, :binary_id, autogenerate: true} + schema "shield_rules_ip" do + belongs_to :site, Plausible.Site + field :inet, EctoNetwork.INET + field :action, Ecto.Enum, values: [:deny, :allow], default: :deny + field :description, :string + field :added_by, :string + + # If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Shield.Rules.IP.Cache` + field :from_cache?, :boolean, virtual: true, default: false + timestamps() + end + + def changeset(rule, attrs) do + rule + |> cast(attrs, [:inet, :description]) + |> validate_required([:inet]) + |> disallow_netmask(:inet) + |> unique_constraint(:inet, + name: :shield_rules_ip_site_id_inet_index + ) + end + + defp disallow_netmask(changeset, field) do + case get_field(changeset, field) do + %Postgrex.INET{netmask: netmask} when netmask != 32 and netmask != 128 -> + add_error(changeset, field, "netmask unsupported") + + _ -> + changeset + end + end +end diff --git a/lib/plausible/shield/ip_rule_cache.ex b/lib/plausible/shield/ip_rule_cache.ex new file mode 100644 index 0000000000000000000000000000000000000000..43946fcf3f89390c4b5771098e26674b9ce0e563 --- /dev/null +++ b/lib/plausible/shield/ip_rule_cache.ex @@ -0,0 +1,65 @@ +defmodule Plausible.Shield.IPRuleCache do + @moduledoc """ + Allows retrieving IP Rules by domain and IP + """ + alias Plausible.Shield.IPRule + + import Ecto.Query + use Plausible.Cache + + @cache_name :ip_blocklist_by_domain + + @cached_schema_fields ~w( + id + inet + action + )a + + @impl true + def name(), do: @cache_name + + @impl true + def child_id(), do: :cache_ip_blocklist + + @impl true + def count_all() do + Plausible.Repo.aggregate(IPRule, :count) + end + + @impl true + def base_db_query() do + from rule in IPRule, + inner_join: s in assoc(rule, :site), + select: { + s.domain, + s.domain_changed_from, + %{struct(rule, ^@cached_schema_fields) | from_cache?: true} + } + end + + @impl true + def get_from_source({domain, address}) do + query = + base_db_query() + |> where([rule, site], rule.inet == ^address and site.domain == ^domain) + + case Plausible.Repo.one(query) do + {_, _, rule = %IPRule{}} -> %IPRule{rule | from_cache?: false} + _any -> nil + end + end + + @impl true + def unwrap_cache_keys(items) do + Enum.reduce(items, [], fn + {domain, nil, object}, acc -> + [{{domain, to_string(object.inet)}, object} | acc] + + {domain, domain_changed_from, object}, acc -> + [ + {{domain, to_string(object.inet)}, object}, + {{domain_changed_from, to_string(object.inet)}, object} | acc + ] + end) + end +end diff --git a/lib/plausible/shield/page_rule.ex b/lib/plausible/shield/page_rule.ex new file mode 100644 index 0000000000000000000000000000000000000000..d56c8f2fece8be4d7dbb7cb31361fa8f870ad582 --- /dev/null +++ b/lib/plausible/shield/page_rule.ex @@ -0,0 +1,70 @@ +defmodule Plausible.Shield.PageRule do + @moduledoc """ + Schema for Pages block list + """ + use Ecto.Schema + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + @primary_key {:id, :binary_id, autogenerate: true} + schema "shield_rules_page" do + belongs_to :site, Plausible.Site + field :page_path, :string + field :page_path_pattern, Plausible.Ecto.Types.CompiledRegex + field :action, Ecto.Enum, values: [:deny, :allow], default: :deny + field :added_by, :string + + # If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Shield.Rules.IP.Cache` + field :from_cache?, :boolean, virtual: true, default: false + timestamps() + end + + def changeset(rule \\ %__MODULE__{}, attrs) do + rule + |> cast(attrs, [:page_path]) + |> validate_required([:page_path]) + |> validate_length(:page_path, max: 250) + |> validate_change(:page_path, fn :page_path, p -> + if not String.starts_with?(p, "/") do + [page_path: "must start with /"] + else + [] + end + end) + |> store_regex() + |> unique_constraint(:page_path_pattern, + name: :shield_rules_page_site_id_page_path_pattern_index, + error_key: :page_path, + message: "rule already exists" + ) + end + + defp store_regex(changeset) do + case get_field(changeset, :page_path) do + "/" <> _ = page_path -> + regex = + page_path + |> Regex.escape() + |> String.replace("\\*\\*", ".*") + |> String.replace("\\*", ".*") + + regex = "^#{regex}$" + + verify_valid_regex(changeset, regex) + + _ -> + changeset + end + end + + defp verify_valid_regex(changeset, regex) do + case Regex.compile(regex) do + {:ok, _} -> + put_change(changeset, :page_path_pattern, regex) + + {:error, _} -> + add_error(changeset, :page_path, "could not compile regular expression") + end + end +end diff --git a/lib/plausible/shield/page_rule_cache.ex b/lib/plausible/shield/page_rule_cache.ex new file mode 100644 index 0000000000000000000000000000000000000000..4c7dab0d24d96529c1925bb7a7856caedc152441 --- /dev/null +++ b/lib/plausible/shield/page_rule_cache.ex @@ -0,0 +1,65 @@ +defmodule Plausible.Shield.PageRuleCache do + @moduledoc """ + Allows retrieving Page Rules by domain + """ + alias Plausible.Shield.PageRule + + import Ecto.Query + use Plausible.Cache + + @cache_name :page_blocklist_by_domain + + @cached_schema_fields ~w( + id + page_path_pattern + action + )a + + @impl true + def name(), do: @cache_name + + @impl true + def child_id(), do: :cache_page_blocklist + + @impl true + def count_all() do + Plausible.Repo.aggregate(PageRule, :count) + end + + @impl true + def base_db_query() do + from rule in PageRule, + inner_join: s in assoc(rule, :site), + select: { + s.domain, + s.domain_changed_from, + %{struct(rule, ^@cached_schema_fields) | from_cache?: true} + } + end + + @impl true + def get_from_source(domain) do + query = + base_db_query() + |> where([..., site], site.domain == ^domain) + + case Plausible.Repo.one(query) do + {_, _, rule = %PageRule{}} -> %PageRule{rule | from_cache?: false} + _any -> nil + end + end + + @impl true + def unwrap_cache_keys(items) do + Enum.reduce(items, [], fn + {domain, nil, object}, acc -> + [{domain, object} | acc] + + {domain, domain_changed_from, object}, acc -> + [ + {domain, object}, + {domain_changed_from, object} | acc + ] + end) + end +end diff --git a/lib/plausible/shields.ex b/lib/plausible/shields.ex new file mode 100644 index 0000000000000000000000000000000000000000..0bf16f79f962ab7eb35fdfd49140169b66a84a69 --- /dev/null +++ b/lib/plausible/shields.ex @@ -0,0 +1,260 @@ +defmodule Plausible.Shields do + @moduledoc """ + Contextual interface for shields. + """ + import Ecto.Query + alias Plausible.Repo + alias Plausible.Shield + alias Plausible.Site + + @maximum_ip_rules 30 + def maximum_ip_rules(), do: @maximum_ip_rules + + @maximum_country_rules 30 + def maximum_country_rules(), do: @maximum_country_rules + + @maximum_page_rules 30 + def maximum_page_rules(), do: @maximum_page_rules + + @maximum_hostname_rules 10 + def maximum_hostname_rules(), do: @maximum_hostname_rules + + @spec list_ip_rules(Site.t() | non_neg_integer()) :: [Shield.IPRule.t()] + def list_ip_rules(site_or_id) do + list(Shield.IPRule, site_or_id) + end + + @spec hostname_allowed?(Site.t() | String.t(), String.t()) :: boolean() + def hostname_allowed?(%Site{domain: domain}, hostname) do + hostname_allowed?(domain, hostname) + end + + def hostname_allowed?(domain, hostname) when is_binary(domain) and is_binary(hostname) do + hostname_rules = Shield.HostnameRuleCache.get(domain) + + if hostname_rules do + hostname_rules + |> List.wrap() + |> Enum.find_value(false, fn rule -> + rule.action == :allow and + Regex.match?(rule.hostname_pattern, hostname) + end) + else + true + end + end + + @spec allowed_hostname_patterns(Site.t() | String.t()) :: list(String.t()) | :all + def allowed_hostname_patterns(%Site{domain: domain}) do + allowed_hostname_patterns(domain) + end + + def allowed_hostname_patterns(domain) when is_binary(domain) do + hostname_rules = Shield.HostnameRuleCache.get(domain) + + if hostname_rules do + hostname_rules + |> List.wrap() + |> Enum.map(&Regex.source(&1.hostname_pattern)) + else + :all + end + end + + @spec ip_blocked?(Site.t() | String.t(), String.t()) :: boolean() + def ip_blocked?(%Site{domain: domain}, address) do + ip_blocked?(domain, address) + end + + def ip_blocked?(domain, address) when is_binary(domain) and is_binary(address) do + case Shield.IPRuleCache.get({domain, address}) do + %Shield.IPRule{action: :deny} -> + true + + _ -> + false + end + end + + @spec page_blocked?(Site.t() | String.t(), String.t()) :: boolean() + def page_blocked?(%Site{domain: domain}, address) do + page_blocked?(domain, address) + end + + def page_blocked?(domain, pathname) when is_binary(domain) and is_binary(pathname) do + page_rules = Shield.PageRuleCache.get(domain) + + if page_rules do + page_rules + |> List.wrap() + |> Enum.find_value(false, fn rule -> + rule.action == :deny and Regex.match?(rule.page_path_pattern, pathname) + end) + else + false + end + end + + @spec country_blocked?(Site.t() | String.t(), String.t()) :: boolean() + def country_blocked?(%Site{domain: domain}, country_code) do + country_blocked?(domain, country_code) + end + + def country_blocked?(domain, country_code) when is_binary(domain) and is_binary(country_code) do + case Shield.CountryRuleCache.get({domain, String.upcase(country_code)}) do + %Shield.CountryRule{action: :deny} -> + true + + _ -> + false + end + end + + @spec add_ip_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) :: + {:ok, Shield.IPRule.t()} | {:error, Ecto.Changeset.t()} + def add_ip_rule(site_or_id, params, opts \\ []) do + opts = + Keyword.put(opts, :limit, {:inet, @maximum_ip_rules}) + + add(Shield.IPRule, site_or_id, params, opts) + end + + @spec remove_ip_rule(Site.t() | non_neg_integer(), String.t()) :: :ok + def remove_ip_rule(site_or_id, rule_id) do + remove(Shield.IPRule, site_or_id, rule_id) + end + + @spec count_ip_rules(Site.t() | non_neg_integer()) :: non_neg_integer() + def count_ip_rules(site_or_id) do + count(Shield.IPRule, site_or_id) + end + + @spec list_country_rules(Site.t() | non_neg_integer()) :: [Shield.CountryRule.t()] + def list_country_rules(site_or_id) do + list(Shield.CountryRule, site_or_id) + end + + @spec add_country_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) :: + {:ok, Shield.CountryRule.t()} | {:error, Ecto.Changeset.t()} + def add_country_rule(site_or_id, params, opts \\ []) do + opts = Keyword.put(opts, :limit, {:country_code, @maximum_country_rules}) + add(Shield.CountryRule, site_or_id, params, opts) + end + + @spec remove_country_rule(Site.t() | non_neg_integer(), String.t()) :: :ok + def remove_country_rule(site_or_id, rule_id) do + remove(Shield.CountryRule, site_or_id, rule_id) + end + + @spec count_country_rules(Site.t() | non_neg_integer()) :: non_neg_integer() + def count_country_rules(site_or_id) do + count(Shield.CountryRule, site_or_id) + end + + @spec list_page_rules(Site.t() | non_neg_integer()) :: [Shield.PageRule.t()] + def list_page_rules(site_or_id) do + list(Shield.PageRule, site_or_id) + end + + @spec add_page_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) :: + {:ok, Shield.PageRule.t()} | {:error, Ecto.Changeset.t()} + def add_page_rule(site_or_id, params, opts \\ []) do + opts = Keyword.put(opts, :limit, {:page_path, @maximum_page_rules}) + add(Shield.PageRule, site_or_id, params, opts) + end + + @spec remove_page_rule(Site.t() | non_neg_integer(), String.t()) :: :ok + def remove_page_rule(site_or_id, rule_id) do + remove(Shield.PageRule, site_or_id, rule_id) + end + + @spec count_page_rules(Site.t() | non_neg_integer()) :: non_neg_integer() + def count_page_rules(site_or_id) do + count(Shield.PageRule, site_or_id) + end + + @spec list_hostname_rules(Site.t() | non_neg_integer()) :: [Shield.HostnameRule.t()] + def list_hostname_rules(site_or_id) do + list(Shield.HostnameRule, site_or_id) + end + + @spec add_hostname_rule(Site.t() | non_neg_integer(), map(), Keyword.t()) :: + {:ok, Shield.HostnameRule.t()} | {:error, Ecto.Changeset.t()} + def add_hostname_rule(site_or_id, params, opts \\ []) do + opts = Keyword.put(opts, :limit, {:hostname, @maximum_hostname_rules}) + add(Shield.HostnameRule, site_or_id, params, opts) + end + + @spec remove_hostname_rule(Site.t() | non_neg_integer(), String.t()) :: :ok + def remove_hostname_rule(site_or_id, rule_id) do + remove(Shield.HostnameRule, site_or_id, rule_id) + end + + @spec count_hostname_rules(Site.t() | non_neg_integer()) :: non_neg_integer() + def count_hostname_rules(site_or_id) do + count(Shield.HostnameRule, site_or_id) + end + + defp list(schema, %Site{id: id}) do + list(schema, id) + end + + defp list(schema, site_id) when is_integer(site_id) do + Repo.all( + from r in schema, + where: r.site_id == ^site_id, + order_by: [desc: r.inserted_at] + ) + end + + defp add(schema, %Site{id: id}, params, opts) do + add(schema, id, params, opts) + end + + defp add(schema, site_id, params, opts) when is_integer(site_id) do + {field, max} = Keyword.fetch!(opts, :limit) + + Repo.transaction(fn -> + result = + if count(schema, site_id) >= max do + changeset = + schema + |> struct(site_id: site_id) + |> schema.changeset(params) + |> Ecto.Changeset.add_error(field, "maximum reached") + + {:error, changeset} + else + schema + |> struct(site_id: site_id, added_by: format_added_by(opts[:added_by])) + |> schema.changeset(params) + |> Repo.insert() + end + + case result do + {:ok, rule} -> rule + {:error, changeset} -> Repo.rollback(changeset) + end + end) + end + + defp remove(schema, %Site{id: id}, rule_id) do + remove(schema, id, rule_id) + end + + defp remove(schema, site_id, rule_id) when is_integer(site_id) do + Repo.delete_all(from(r in schema, where: r.site_id == ^site_id and r.id == ^rule_id)) + :ok + end + + defp count(schema, %Site{id: id}) do + count(schema, id) + end + + defp count(schema, site_id) when is_integer(site_id) do + Repo.aggregate(from(r in schema, where: r.site_id == ^site_id), :count) + end + + defp format_added_by(nil), do: "" + defp format_added_by(%Plausible.Auth.User{} = user), do: "#{user.name} <#{user.email}>" +end diff --git a/lib/plausible/site.ex b/lib/plausible/site.ex new file mode 100644 index 0000000000000000000000000000000000000000..f7c76f589bad6f948e7450ef2526a05f2a9a8492 --- /dev/null +++ b/lib/plausible/site.ex @@ -0,0 +1,250 @@ +defmodule Plausible.Site do + @moduledoc """ + Site schema + """ + use Ecto.Schema + use Plausible + import Ecto.Changeset + import Ecto.Query, only: [from: 2] + alias Plausible.Site.GoogleAuth + + @type t() :: %__MODULE__{} + + @derive {Jason.Encoder, only: [:domain, :timezone]} + schema "sites" do + field :domain, :string + field :timezone, :string, default: "Etc/UTC" + field :public, :boolean + field :stats_start_date, :date + field :native_stats_start_at, :naive_datetime + field :allowed_event_props, {:array, :string} + field :conversions_enabled, :boolean, default: true + field :props_enabled, :boolean, default: true + field :funnels_enabled, :boolean, default: true + field :legacy_time_on_page_cutoff, :date, default: ~D[1970-01-01] + + field :consolidated, :boolean, default: false + + field :ingest_rate_limit_scale_seconds, :integer, default: 60 + # default is set via changeset/2 + field :ingest_rate_limit_threshold, :integer + + field :domain_changed_from, :string + field :domain_changed_at, :naive_datetime + + # NOTE: needed by `SiteImports` data migration script + embeds_one :imported_data, Plausible.Site.ImportedData, on_replace: :update + + # NOTE: new teams relations + belongs_to :team, Plausible.Teams.Team + has_many :guest_memberships, Plausible.Teams.GuestMembership + has_many :guest_invitations, Plausible.Teams.GuestInvitation + + has_one :tracker_script_configuration, Plausible.Site.TrackerScriptConfiguration + + has_many :goals, Plausible.Goal, preload_order: [desc: :id] + has_many :revenue_goals, Plausible.Goal, where: [currency: {:not, nil}] + has_one :google_auth, GoogleAuth + has_one :weekly_report, Plausible.Site.WeeklyReport + has_one :monthly_report, Plausible.Site.MonthlyReport + has_many :ownerships, through: [:team, :ownerships], preload_order: [asc: :id] + has_many :owners, through: [:team, :owners] + + # If `from_cache?` is set, the struct might be incomplete - see `Plausible.Site.Cache`. + # Use `Plausible.Repo.reload!(cached_site)` to pre-fill missing fields if + # strictly necessary. + field :from_cache?, :boolean, virtual: true, default: false + + # Used in the context of paginated sites list to order in relation to + # user's membership state. Currently it can be either "invitation", + # "pinned_site" or "site", where invitations are first. + field :entry_type, :string, virtual: true + field :memberships, {:array, :map}, virtual: true + field :invitations, {:array, :map}, virtual: true + field :pinned_at, :naive_datetime, virtual: true + + has_many :completed_imports, Plausible.Imported.SiteImport, where: [status: :completed] + + timestamps() + end + + def regular(q \\ __MODULE__) do + from s in q, where: not s.consolidated + end + + def new_for_team(team, params) do + params + |> new() + |> put_assoc(:team, team) + end + + def new(params), do: changeset(%__MODULE__{}, params) + + on_ee do + @domain_unique_error """ + This domain cannot be registered. Perhaps one of your colleagues registered it? If that's not the case, please contact support@plausible.io + """ + else + @domain_unique_error """ + This domain cannot be registered. Perhaps one of your colleagues registered it? + """ + end + + on_ee do + @changeset_cast_fields [:domain, :consolidated, :timezone, :legacy_time_on_page_cutoff] + else + @changeset_cast_fields [:domain, :timezone, :legacy_time_on_page_cutoff] + end + + def changeset(site, attrs \\ %{}) do + site + |> cast(attrs, @changeset_cast_fields) + |> clean_domain() + |> validate_required([:domain, :timezone]) + |> validate_timezone() + |> validate_domain_format() + |> validate_domain_reserved_characters() + |> unique_constraint(:domain, + message: @domain_unique_error + ) + |> unique_constraint(:domain, + name: "domain_change_disallowed", + message: @domain_unique_error + ) + |> put_change( + :ingest_rate_limit_threshold, + Application.get_env(:plausible, __MODULE__)[:default_ingest_threshold] + ) + end + + def update_changeset(site, attrs \\ %{}, opts \\ []) do + at = + opts + |> Keyword.get(:at, NaiveDateTime.utc_now()) + |> NaiveDateTime.truncate(:second) + + site + |> changeset(attrs) + |> handle_domain_change(at) + end + + def crm_changeset(site, attrs) do + site + |> cast(attrs, [ + :timezone, + :public, + :native_stats_start_at, + :ingest_rate_limit_threshold, + :ingest_rate_limit_scale_seconds + ]) + |> validate_required([:timezone, :public]) + |> validate_number(:ingest_rate_limit_scale_seconds, + greater_than_or_equal_to: 1, + message: "must be at least 1 second" + ) + |> validate_number(:ingest_rate_limit_threshold, + greater_than_or_equal_to: 0, + message: "must be empty, zero or positive" + ) + end + + def tz_offset(site, utc_now \\ DateTime.utc_now()) do + case DateTime.shift_zone(utc_now, site.timezone) do + {:ok, datetime} -> + datetime.utc_offset + datetime.std_offset + + res -> + Sentry.capture_message("Unable to determine timezone offset for", + extra: %{site: site, result: res} + ) + + 0 + end + end + + def make_public(site) do + change(site, public: true) + end + + def make_private(site) do + change(site, public: false) + end + + def set_stats_start_date(site, val) do + change(site, stats_start_date: val) + end + + def set_native_stats_start_at(site, val) do + change(site, native_stats_start_at: val) + end + + defp clean_domain(changeset) do + clean_domain = + (get_field(changeset, :domain) || "") + |> String.downcase() + |> String.trim() + |> String.replace_leading("http://", "") + |> String.replace_leading("https://", "") + |> String.trim("/") + |> String.replace_leading("www.", "") + + change(changeset, %{domain: clean_domain}) + end + + # https://tools.ietf.org/html/rfc3986#section-2.2 + @uri_reserved_chars ~w(: ? # [ ] @ ! $ & ' \( \) * + , ; =) + defp validate_domain_reserved_characters(changeset) do + domain = get_field(changeset, :domain) || "" + + if String.contains?(domain, @uri_reserved_chars) do + add_error( + changeset, + :domain, + "must not contain URI reserved characters #{@uri_reserved_chars}" + ) + else + changeset + end + end + + defp validate_domain_format(changeset) do + validate_format(changeset, :domain, ~r/^[-\.\\\/_:\p{L}\d]*$/u, + message: "only letters, numbers, slashes, underscores and period allowed" + ) + end + + defp handle_domain_change(changeset, at) do + new_domain = get_change(changeset, :domain) + + if new_domain do + changeset + |> put_change(:domain_changed_from, changeset.data.domain) + |> put_change(:domain_changed_at, at) + |> unique_constraint(:domain, + name: "domain_change_disallowed", + message: @domain_unique_error + ) + |> unique_constraint(:domain_changed_from, + message: @domain_unique_error + ) + else + changeset + end + end + + defp validate_timezone(changeset) do + tz = get_field(changeset, :timezone) + + if Plausible.Timezones.valid?(tz) do + changeset + else + add_error(changeset, :timezone, "is invalid") + end + end +end + +defimpl FunWithFlags.Actor, for: Plausible.Site do + def id(%{domain: domain}) do + "site:#{domain}" + end +end diff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.ex new file mode 100644 index 0000000000000000000000000000000000000000..2c559f66ca221c9f5431cfb29e17fa161a412ca9 --- /dev/null +++ b/lib/plausible/site/cache.ex @@ -0,0 +1,104 @@ +defmodule Plausible.Site.Cache do + @moduledoc """ + The cache allows lookups by both `domain` and `domain_changed_from` + fields - this is to allow traffic from sites whose domains changed within a certain + grace period (see: `Plausible.Site.Transfer`). + + To differentiate cached Site structs from those retrieved directly from the + database, a virtual schema field `from_cache?: true` is set. + This indicates the `Plausible.Site` struct is incomplete in comparison to its + database counterpart -- to spare bandwidth and query execution time, + only selected database columns are retrieved and cached. + + The `@cached_schema_fields` attribute defines the list of DB columns + queried on each cache refresh. + + Also see tests for more comprehensive examples. + """ + require Logger + + import Ecto.Query + + alias Plausible.Site + + use Plausible.Cache + + @cache_name :sites_by_domain + + @cached_schema_fields ~w( + id + domain + domain_changed_from + ingest_rate_limit_scale_seconds + ingest_rate_limit_threshold + )a + + @impl true + def name(), do: @cache_name + + @impl true + def child_id(), do: :cache_sites + + @impl true + def count_all() do + from(s in Site.regular()) + |> Plausible.Repo.aggregate(:count) + end + + @impl true + def base_db_query() do + from s in Site.regular(), + left_join: rg in assoc(s, :revenue_goals), + inner_join: team in assoc(s, :team), + select: { + s.domain, + s.domain_changed_from, + %{struct(s, ^@cached_schema_fields) | from_cache?: true} + }, + preload: [revenue_goals: rg, team: team] + end + + @impl true + def get_from_source(domain) do + query = from s in base_db_query(), where: s.domain == ^domain + + case Plausible.Repo.one(query) do + {_, _, site = %Site{}} -> %Site{site | from_cache?: false} + _any -> nil + end + end + + @spec get_site_id(String.t(), Keyword.t()) :: pos_integer() | nil + def get_site_id(domain, opts \\ []) do + case get(domain, opts) do + %{id: site_id} -> + site_id + + nil -> + nil + end + end + + @spec touch_site!(Site.t(), DateTime.t()) :: Site.t() + def touch_site!(site, now) do + now = + now + |> DateTime.truncate(:second) + |> DateTime.to_naive() + + site + |> Ecto.Changeset.change(updated_at: now) + |> Plausible.Repo.update!() + end + + @impl true + def unwrap_cache_keys(items) do + Enum.reduce(items, [], fn + {domain, nil, object}, acc -> + [{domain, object} | acc] + + {domain, domain_changed_from, object}, acc -> + [{domain, object}, {domain_changed_from, object} | acc] + end) + end +end diff --git a/lib/plausible/site/domain.ex b/lib/plausible/site/domain.ex new file mode 100644 index 0000000000000000000000000000000000000000..9767b09b076cd53c8941ca0db0f8fb483626c73e --- /dev/null +++ b/lib/plausible/site/domain.ex @@ -0,0 +1,62 @@ +defmodule Plausible.Site.Domain do + @expire_threshold_hours 72 + + @moduledoc """ + Basic interface for domain changes. + + Once `Plausible.DataMigration.NumericIDs` schema migration is ready, + domain change operation will be enabled, accessible to the users. + + We will set a transition period of #{@expire_threshold_hours} hours + during which, both old and new domains, will be accepted as traffic + identifiers to the same site. + + A periodic worker will call the `expire/0` function to end it where applicable. + See: `Plausible.Workers.ExpireDomainChangeTransitions`. + + The underlying changeset for domain change (see: `Plausible.Site`) relies + on database trigger installed via `Plausible.Repo.Migrations.AllowDomainChange` + Postgres migration. The trigger checks if either `domain` or `domain_changed_from` + exist to ensure unicity. + """ + + alias Plausible.Site + alias Plausible.Repo + + import Ecto.Query + + @spec expire_change_transitions(integer()) :: {:ok, non_neg_integer()} + def expire_change_transitions(expire_threshold_hours \\ @expire_threshold_hours) do + {updated, _} = + Repo.update_all( + from(s in Site.regular(), + where: s.domain_changed_at < ago(^expire_threshold_hours, "hour") + ), + set: [ + domain_changed_from: nil, + domain_changed_at: nil + ] + ) + + {:ok, updated} + end + + @spec change(Site.t(), String.t(), Keyword.t()) :: + {:ok, Site.t()} | {:error, Ecto.Changeset.t()} + def change(%Site{} = site, new_domain, opts \\ []) do + changeset = Site.update_changeset(site, %{domain: new_domain}, opts) + + changeset = + if is_nil(changeset.errors[:domain]) and is_nil(changeset.changes[:domain]) do + Ecto.Changeset.add_error( + changeset, + :domain, + "New domain must be different than the current one" + ) + else + changeset + end + + Repo.update(changeset) + end +end diff --git a/lib/plausible/site/gate_keeper.ex b/lib/plausible/site/gate_keeper.ex new file mode 100644 index 0000000000000000000000000000000000000000..67f79d2790bcea1b068bd71e31ed4a0452e6bca6 --- /dev/null +++ b/lib/plausible/site/gate_keeper.ex @@ -0,0 +1,77 @@ +defmodule Plausible.Site.GateKeeper do + @type policy() :: :allow | :not_found | :block | :throttle | :payment_required + @policy_for_non_existing_sites :not_found + + @type t() :: {:allow, Plausible.Site.t()} | {:deny, policy()} + + @moduledoc """ + Thin wrapper around `Plausible.RateLimit` for gate keeping domain-specific events + during the ingestion phase. When the site is allowed, gate keeping + check returns `:allow`, otherwise a `:deny` tagged tuple is returned + with one of the following policy markers: + * `:not_found` (indicates site not found in cache) + * `:block` (indicates disabled sites) + * `:throttle` (indicates rate limiting) + + Rate Limiting buckets are configured per site (externally via the CRM). + See: `Plausible.Site` + + To look up each site's configuration, the RateLimiter fetches + a Site by domain using `Plausible.Cache` interface. + + The module defines two policies outside the regular bucket inspection: + * when the site is not found in cache: #{@policy_for_non_existing_sites} + * when the underlying rate limiting mechanism returns + an internal error: :allow + """ + alias Plausible.{Site, RateLimit} + alias Plausible.Site.Cache + + @spec check(String.t(), Keyword.t()) :: t() + def check(domain, opts \\ []) when is_binary(domain) do + case policy(domain, opts) do + {:allow, site} -> {:allow, site} + other -> {:deny, other} + end + end + + @spec key(String.t()) :: String.t() + def key(domain) do + "ingest:site:#{domain}" + end + + defp policy(domain, opts) when is_binary(domain) do + with %Site{team: %{accept_traffic_until: accept_traffic_until}} = site <- + Cache.get(domain, Keyword.get(opts, :cache_opts, [])), + true <- Plausible.Sites.regular?(site) do + if not is_nil(accept_traffic_until) and + Date.after?(Date.utc_today(), accept_traffic_until) do + :payment_required + else + check_rate_limit(site, opts) + end + else + _ -> + @policy_for_non_existing_sites + end + end + + defp check_rate_limit(%Site{ingest_rate_limit_threshold: nil} = site, _opts) do + {:allow, site} + end + + defp check_rate_limit(%Site{ingest_rate_limit_threshold: 0}, _opts) do + :block + end + + defp check_rate_limit(%Site{ingest_rate_limit_threshold: threshold} = site, opts) + when is_integer(threshold) do + key = Keyword.get(opts, :key, key(site.domain)) + scale_ms = site.ingest_rate_limit_scale_seconds * 1_000 + + case RateLimit.check_rate(key, scale_ms, threshold) do + {:deny, _} -> :throttle + {:allow, _} -> {:allow, site} + end + end +end diff --git a/lib/plausible/site/google_auth.ex b/lib/plausible/site/google_auth.ex new file mode 100644 index 0000000000000000000000000000000000000000..29ea929124ceea04c8ca0a91604d14498ce1c76a --- /dev/null +++ b/lib/plausible/site/google_auth.ex @@ -0,0 +1,33 @@ +defmodule Plausible.Site.GoogleAuth do + @moduledoc """ + Struct for storing google auth token info used by search console. + """ + + use Ecto.Schema + import Ecto.Changeset + + schema "google_auth" do + field :email, :string + field :property, :string + field :refresh_token, :string + field :access_token, :string + field :expires, :naive_datetime + + belongs_to :user, Plausible.Auth.User + belongs_to :site, Plausible.Site + + timestamps() + end + + def changeset(auth, attrs \\ %{}) do + auth + |> cast(attrs, [:refresh_token, :access_token, :expires, :email]) + |> validate_required([:refresh_token, :access_token, :expires, :email]) + |> unique_constraint(:site) + end + + def set_property(auth, attrs \\ %{}) do + auth + |> cast(attrs, [:property]) + end +end diff --git a/lib/plausible/site/imported_data.ex b/lib/plausible/site/imported_data.ex new file mode 100644 index 0000000000000000000000000000000000000000..710cc80d44e594808a97d87daede66fcac45dafb --- /dev/null +++ b/lib/plausible/site/imported_data.ex @@ -0,0 +1,17 @@ +defmodule Plausible.Site.ImportedData do + @moduledoc """ + Embedded schema for analytics imports + + NOTE: needed by `SiteImports` data migration script + """ + use Ecto.Schema + + @type t() :: %__MODULE__{} + + embedded_schema do + field :start_date, :date + field :end_date, :date + field :source, :string + field :status, :string + end +end diff --git a/lib/plausible/site/monthly_report.ex b/lib/plausible/site/monthly_report.ex new file mode 100644 index 0000000000000000000000000000000000000000..464a6d60396a5ca8f9c72a94aedd0e29f8f34172 --- /dev/null +++ b/lib/plausible/site/monthly_report.ex @@ -0,0 +1,28 @@ +defmodule Plausible.Site.MonthlyReport do + use Ecto.Schema + import Ecto.Changeset + + schema "monthly_reports" do + field :recipients, {:array, :string} + belongs_to :site, Plausible.Site + + timestamps() + end + + def changeset(settings, attrs \\ %{}) do + settings + |> cast(attrs, [:site_id, :recipients]) + |> validate_required([:site_id, :recipients]) + |> unique_constraint(:site_id) + end + + def add_recipient(report, recipient) do + report + |> change(recipients: report.recipients ++ [recipient]) + end + + def remove_recipient(report, recipient) do + report + |> change(recipients: List.delete(report.recipients, recipient)) + end +end diff --git a/lib/plausible/site/removal.ex b/lib/plausible/site/removal.ex new file mode 100644 index 0000000000000000000000000000000000000000..1d1f27512fa8128cd89360fc2974a3734fbdad23 --- /dev/null +++ b/lib/plausible/site/removal.ex @@ -0,0 +1,33 @@ +defmodule Plausible.Site.Removal do + @moduledoc """ + A site deletion service stub. + """ + use Plausible + + alias Plausible.Repo + alias Plausible.Teams + + import Ecto.Query + + @spec run(Plausible.Site.t()) :: {:ok, map()} + def run(site) do + Repo.transaction(fn -> + site = Repo.preload(site, :team) + + result = Repo.delete_all(from(s in Plausible.Site, where: s.domain == ^site.domain)) + + Teams.Memberships.prune_guests(site.team) + Teams.Invitations.prune_guest_invitations(site.team) + + on_ee do + if Plausible.Sites.regular?(site) do + :ok = Plausible.ConsolidatedView.reset_if_enabled(site.team) + end + + Plausible.Billing.SiteLocker.update_for(site.team, send_email?: false) + end + + %{delete_all: result} + end) + end +end diff --git a/lib/plausible/site/shared_link.ex b/lib/plausible/site/shared_link.ex new file mode 100644 index 0000000000000000000000000000000000000000..7e4df1399e79add5deef7ad0ad39c9d4131510cb --- /dev/null +++ b/lib/plausible/site/shared_link.ex @@ -0,0 +1,61 @@ +defmodule Plausible.Site.SharedLink do + @moduledoc """ + Schema for shared links. + """ + use Ecto.Schema + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + schema "shared_links" do + belongs_to :site, Plausible.Site + field :name, :string + field :slug, :string + field :password_hash, :string + field :password, :string, virtual: true + belongs_to :segment, Plausible.Segments.Segment + + timestamps() + end + + def changeset(link, attrs \\ %{}, opts \\ []) do + link + |> cast(attrs, [:password, :name]) + |> validate_required([:slug, :name]) + |> validate_special_name(opts) + |> unique_constraint(:slug) + |> unique_constraint(:name, name: :shared_links_site_id_name_index) + |> foreign_key_constraint(:segment_id) + |> hash_password() + end + + defp validate_special_name(changeset, opts) do + name = get_change(changeset, :name) + + if name not in Plausible.Sites.shared_link_special_names() || + Keyword.get(opts, :skip_special_name_check?, false) do + changeset + else + changeset |> add_error(:name, "This name is reserved. Please choose another one") + end + end + + defp hash_password(link) do + case link.changes[:password] do + nil -> + link + + password -> + hash = Plausible.Auth.Password.hash(password) + change(link, password_hash: hash) + end + end + + def password_protected?(%__MODULE__{password_hash: hash}) when not is_nil(hash), do: true + def password_protected?(%__MODULE__{}), do: false + + def limited_to_segment?(%__MODULE__{segment_id: segment_id}) when is_integer(segment_id), + do: true + + def limited_to_segment?(%__MODULE__{}), do: false +end diff --git a/lib/plausible/site/tracker_script_cache.ex b/lib/plausible/site/tracker_script_cache.ex new file mode 100644 index 0000000000000000000000000000000000000000..1e86efa22048ec8756563421b2c3edb9bc16adb8 --- /dev/null +++ b/lib/plausible/site/tracker_script_cache.ex @@ -0,0 +1,55 @@ +defmodule Plausible.Site.TrackerScriptCache do + @moduledoc """ + Caches the full tracker script. + Must be refreshed whenever the TrackerScriptConfiguration associated with the script changes. + """ + alias Plausible.Site.TrackerScriptConfiguration + alias PlausibleWeb.Tracker + + import Ecto.Query + use Plausible + use Plausible.Cache + + @cache_name :tracker_script_cache + + @impl true + def name(), do: @cache_name + + @impl true + def child_id(), do: :cache_tracker_script + + def cache_content( + %TrackerScriptConfiguration{site: %{domain: _domain}} = tracker_script_configuration + ), + do: Tracker.build_script(tracker_script_configuration) + + @impl true + def count_all() do + Plausible.Repo.aggregate(TrackerScriptConfiguration, :count) + end + + @impl true + def base_db_query(), do: Tracker.get_tracker_script_configuration_base_query() + + @impl true + def get_from_source(id) do + case Tracker.get_tracker_script_configuration_by_id(id) do + %TrackerScriptConfiguration{site: %{domain: _domain}} = tracker_script_configuration -> + cache_content(tracker_script_configuration) + + _ -> + nil + end + end + + @impl true + def unwrap_cache_keys(items) do + Enum.reduce(items, [], fn + tracker_script_configuration, acc -> + [ + {tracker_script_configuration.id, cache_content(tracker_script_configuration)} + | acc + ] + end) + end +end diff --git a/lib/plausible/site/tracker_script_configuration.ex b/lib/plausible/site/tracker_script_configuration.ex new file mode 100644 index 0000000000000000000000000000000000000000..cdd4e0ecefcd7e8453d4043715717943acfb2459 --- /dev/null +++ b/lib/plausible/site/tracker_script_configuration.ex @@ -0,0 +1,70 @@ +defmodule Plausible.Site.TrackerScriptConfiguration do + @moduledoc """ + Schema for tracker script configuration + """ + use Ecto.Schema + import Ecto.Changeset + + @type t() :: %__MODULE__{} + @derive {Jason.Encoder, + only: [ + :id, + :installation_type, + :track_404_pages, + :hash_based_routing, + :outbound_links, + :file_downloads, + :revenue_tracking, + :tagged_events, + :form_submissions, + :pageview_props + ]} + + @primary_key {:id, Plausible.Ecto.Types.TrackerScriptNanoid, autogenerate: true} + schema "tracker_script_configuration" do + field :installation_type, Ecto.Enum, values: [:manual, :wordpress, :gtm, :npm, nil] + + field :track_404_pages, :boolean, default: false + field :hash_based_routing, :boolean, default: false + field :outbound_links, :boolean, default: false + field :file_downloads, :boolean, default: false + field :revenue_tracking, :boolean, default: false + field :tagged_events, :boolean, default: false + field :form_submissions, :boolean, default: false + field :pageview_props, :boolean, default: false + + belongs_to :site, Plausible.Site + + timestamps() + end + + def installation_changeset(struct, params) do + struct + |> cast(params, [ + :installation_type, + :track_404_pages, + :hash_based_routing, + :outbound_links, + :file_downloads, + :revenue_tracking, + :tagged_events, + :form_submissions, + :pageview_props, + :site_id + ]) + |> validate_required([:site_id]) + end + + def plugins_api_changeset(struct, params) do + struct + |> cast(params, [ + :installation_type, + :hash_based_routing, + :outbound_links, + :file_downloads, + :form_submissions, + :site_id + ]) + |> validate_required([:site_id]) + end +end diff --git a/lib/plausible/site/traffic_change_notification.ex b/lib/plausible/site/traffic_change_notification.ex new file mode 100644 index 0000000000000000000000000000000000000000..d84af9787ff876627e3ab10b895c9f754a716189 --- /dev/null +++ b/lib/plausible/site/traffic_change_notification.ex @@ -0,0 +1,40 @@ +defmodule Plausible.Site.TrafficChangeNotification do + @moduledoc """ + Configuration schema for site-specific traffic change notifications. + """ + use Ecto.Schema + import Ecto.Changeset + + # legacy table name since traffic drop notifications were introduced + schema "spike_notifications" do + field :recipients, {:array, :string} + field :threshold, :integer + field :last_sent, :naive_datetime + field :type, Ecto.Enum, values: [:spike, :drop], default: :spike + belongs_to :site, Plausible.Site + + timestamps() + end + + def changeset(schema, attrs) do + schema + |> cast(attrs, [:site_id, :recipients, :threshold, :type]) + |> validate_required([:site_id, :recipients, :threshold, :type]) + |> validate_number(:threshold, greater_than_or_equal_to: 1) + |> unique_constraint([:site_id, :type]) + end + + def add_recipient(schema, recipient) do + schema + |> change(recipients: schema.recipients ++ [recipient]) + end + + def remove_recipient(schema, recipient) do + schema + |> change(recipients: List.delete(schema.recipients, recipient)) + end + + def was_sent(schema, now) do + schema |> change(last_sent: now) + end +end diff --git a/lib/plausible/site/user_preference.ex b/lib/plausible/site/user_preference.ex new file mode 100644 index 0000000000000000000000000000000000000000..89e80e2f2b70b0a20bd674fedbb8ab33fcb15769 --- /dev/null +++ b/lib/plausible/site/user_preference.ex @@ -0,0 +1,30 @@ +defmodule Plausible.Site.UserPreference do + @moduledoc """ + User-specific site preferences schema + """ + + use Ecto.Schema + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + @options [:pinned_at] + + schema "site_user_preferences" do + field :pinned_at, :naive_datetime + + belongs_to :user, Plausible.Auth.User + belongs_to :site, Plausible.Site + + timestamps() + end + + defmacro options, do: @options + + def changeset(user, site, attrs \\ %{}) do + %__MODULE__{} + |> cast(attrs, @options) + |> put_assoc(:user, user) + |> put_assoc(:site, site) + end +end diff --git a/lib/plausible/site/weekly_report.ex b/lib/plausible/site/weekly_report.ex new file mode 100644 index 0000000000000000000000000000000000000000..1348b4f154c9fab1f6dcaff56932db23ddfc3c56 --- /dev/null +++ b/lib/plausible/site/weekly_report.ex @@ -0,0 +1,28 @@ +defmodule Plausible.Site.WeeklyReport do + use Ecto.Schema + import Ecto.Changeset + + schema "weekly_reports" do + field :recipients, {:array, :string} + belongs_to :site, Plausible.Site + + timestamps() + end + + def changeset(settings, attrs \\ %{}) do + settings + |> cast(attrs, [:site_id, :recipients]) + |> validate_required([:site_id, :recipients]) + |> unique_constraint(:site_id) + end + + def add_recipient(report, recipient) do + report + |> change(recipients: report.recipients ++ [recipient]) + end + + def remove_recipient(report, recipient) do + report + |> change(recipients: List.delete(report.recipients, recipient)) + end +end diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex new file mode 100644 index 0000000000000000000000000000000000000000..aa9ca3ccdd8492fd28ad0840a86acb4d2e8bb190 --- /dev/null +++ b/lib/plausible/sites.ex @@ -0,0 +1,549 @@ +defmodule Plausible.Sites do + @moduledoc """ + Sites context functions. + """ + use Plausible + + import Ecto.Query + + alias Plausible.{Auth, Repo, Site, Teams, Billing} + alias Plausible.Billing.Feature.SharedLinks + alias Plausible.Site.SharedLink + + require Plausible.Site.UserPreference + + on_ee do + @spec regular?(Site.t()) :: boolean() + def regular?(%Site{} = site), do: not site.consolidated + + @spec consolidated?(Site.t()) :: boolean() + def consolidated?(%Site{} = site), do: site.consolidated + + def site_id_query_filter(%Site{} = site) do + if consolidated?(site) do + site_ids = Plausible.ConsolidatedView.Cache.get(site.domain) + dynamic([x], fragment("? in ?", x.site_id, ^site_ids)) + else + dynamic([x], x.site_id == ^site.id) + end + end + else + @spec regular?(Site.t()) :: boolean() + def regular?(%Site{}), do: always(true) + + @spec consolidated?(Site.t()) :: boolean() + def consolidated?(%Site{}), do: always(false) + + def site_id_query_filter(%Site{} = site) do + dynamic([x], x.site_id == ^site.id) + end + end + + def display_name(%Site{} = site, opts \\ []) do + if consolidated?(site) do + "#{if opts[:capitalize_consolidated], do: "C", else: "c"}onsolidated view" + else + site.domain + end + end + + @shared_link_special_names ["WordPress - Shared Dashboard"] + @doc """ + Special shared link names are used to distinguish between those + created by the Plugins API, and those created in any other way + (i.e. via Sites API or in the Dashboard Site Settings UI). + + The intent is to give our WP plugin the ability to display an + embedded dashboard even when the user's subscription does not + support the shared links feature. + + A shared link with a special name can only be created via the + plugins API, and it will not show up under the list of shared + links in Site Settings > Visibility. + + Once created with the special name, the link will be accessible + even when the team does not have access to SharedLinks feature. + """ + def shared_link_special_names(), do: @shared_link_special_names + + def get_by_domain(domain, opts \\ []) do + include_consolidated? = Keyword.get(opts, :include_consolidated?, false) + + if include_consolidated? do + Repo.get_by(Site, domain: domain) + else + Repo.get_by(Site.regular(), domain: domain) + end + end + + def get_by_domain!(domain, opts \\ []) do + include_consolidated? = Keyword.get(opts, :include_consolidated?, false) + + if include_consolidated? do + Repo.get_by!(Site, domain: domain) + else + Repo.get_by!(Site.regular(), domain: domain) + end + end + + @spec toggle_pin(Auth.User.t(), Site.t()) :: + {:ok, Site.UserPreference.t()} | {:error, :too_many_pins} + def toggle_pin(user, site) do + pinned_at = + if site.pinned_at do + nil + else + NaiveDateTime.utc_now() + end + + with :ok <- check_user_pin_limit(user, pinned_at) do + {:ok, set_option(user, site, :pinned_at, pinned_at)} + end + end + + @pins_limit 9 + + defp check_user_pin_limit(_user, nil), do: :ok + + defp check_user_pin_limit(user, _) do + pins_count = + from(up in Site.UserPreference, + where: up.user_id == ^user.id and not is_nil(up.pinned_at) + ) + |> Repo.aggregate(:count) + + if pins_count + 1 > @pins_limit do + {:error, :too_many_pins} + else + :ok + end + end + + @spec set_option(Auth.User.t(), Site.t(), atom(), any()) :: Site.UserPreference.t() + def set_option(user, site, option, value) when option in Site.UserPreference.options() do + get_for_user!(user, site.domain, + roles: [ + :owner, + :admin, + :editor, + :viewer, + :billing + ] + ) + + user + |> Site.UserPreference.changeset(site, %{option => value}) + |> Repo.insert!( + conflict_target: [:user_id, :site_id], + # This way of conflict handling enables doing upserts of options leaving + # existing, unrelated values intact. + on_conflict: from(p in Site.UserPreference, update: [set: [{^option, ^value}]]), + returning: true + ) + end + + defdelegate get_for_user_by_ids(user, site_ids, opts \\ []), to: Plausible.Teams.Sites + + def list_people(site) do + owner_memberships = + from( + tm in Teams.Membership, + inner_join: u in assoc(tm, :user), + where: tm.team_id == ^site.team_id, + where: tm.role == :owner, + select: %{ + user: u, + role: tm.role + } + ) + |> Repo.all() + + memberships = + from( + gm in Teams.GuestMembership, + inner_join: tm in assoc(gm, :team_membership), + inner_join: u in assoc(tm, :user), + where: gm.site_id == ^site.id, + select: %{ + user: u, + role: gm.role + } + ) + |> Repo.all() + + memberships = owner_memberships ++ memberships + + invitations = + from( + gi in Teams.GuestInvitation, + inner_join: ti in assoc(gi, :team_invitation), + where: gi.site_id == ^site.id, + select: %{ + invitation_id: gi.invitation_id, + email: ti.email, + role: gi.role + } + ) + |> Repo.all() + + site_transfers = + from( + st in Teams.SiteTransfer, + where: st.site_id == ^site.id, + select: %{ + invitation_id: st.transfer_id, + email: st.email, + role: :owner + } + ) + |> Repo.all() + + %{memberships: memberships, invitations: site_transfers ++ invitations} + end + + @spec list_guests_query(Site.t(), Keyword.t()) :: Ecto.Query.t() + def list_guests_query(site, opts \\ []) do + guest_memberships = + from( + gm in Teams.GuestMembership, + inner_join: tm in assoc(gm, :team_membership), + inner_join: u in assoc(tm, :user), + as: :user, + where: gm.site_id == ^site.id, + select: %{ + id: gm.id, + sort_index: -gm.id, + inserted_at: gm.inserted_at, + email: u.email, + role: gm.role, + status: "accepted" + } + ) + + guest_memberships = + if email = opts[:email] do + guest_memberships |> where([user: u], u.email == ^email) + else + guest_memberships + end + + guest_invitations = + from( + gi in Teams.GuestInvitation, + inner_join: ti in assoc(gi, :team_invitation), + as: :team_invitation, + where: gi.site_id == ^site.id, + select: %{ + id: gi.id, + sort_index: gi.id, + inserted_at: gi.inserted_at, + email: ti.email, + role: gi.role, + status: "invited" + } + ) + + guest_invitations = + if email = opts[:email] do + guest_invitations |> where([team_invitation: ti], ti.email == ^email) + else + guest_invitations + end + + guests = union_all(guest_memberships, ^guest_invitations) + + from(g in subquery(guests), + select: %{ + id: g.id, + sort_index: g.sort_index, + inserted_at: g.inserted_at, + email: g.email, + role: g.role, + status: g.status + }, + order_by: [asc: g.sort_index] + ) + end + + @spec for_user_query(Auth.User.t(), Teams.Team.t() | nil) :: Ecto.Query.t() + def for_user_query(user, team \\ nil) do + query = + from(s in Site.regular(), + as: :site, + inner_join: t in assoc(s, :team), + as: :team, + inner_join: tm in assoc(t, :team_memberships), + as: :team_memberships, + left_join: gm in assoc(tm, :guest_memberships), + as: :guest_memberships, + where: tm.user_id == ^user.id, + order_by: [desc: s.id] + ) + + if team do + where( + query, + [team_memberships: tm, guest_memberships: gm, site: s], + tm.role != :guest and tm.team_id == ^team.id + ) + else + where( + query, + [team_memberships: tm, guest_memberships: gm, site: s], + tm.role != :guest or gm.site_id == s.id + ) + end + end + + def create(user, params, team \\ nil) do + Ecto.Multi.new() + |> Ecto.Multi.put(:site_changeset, Site.new(params)) + |> Ecto.Multi.run(:create_team, fn _repo, _context -> + cond do + team && Teams.Memberships.can_add_site?(team, user) -> + {:ok, Teams.with_subscription(team)} + + is_nil(team) -> + with {:ok, team} <- Teams.get_or_create(user) do + {:ok, Teams.with_subscription(team)} + end + + true -> + {:error, :permission_denied} + end + end) + |> Ecto.Multi.run(:ensure_can_add_new_site, fn _repo, %{create_team: team} -> + case Teams.Billing.ensure_can_add_new_site(team) do + :ok -> {:ok, :proceed} + error -> error + end + end) + |> Ecto.Multi.run(:clear_changed_from, fn + _repo, %{site_changeset: %{changes: %{domain: domain}}} -> + case get_for_user(user, domain, roles: [:owner]) do + %Site{domain_changed_from: ^domain} = site -> + site + |> Ecto.Changeset.change() + |> Ecto.Changeset.put_change(:domain_changed_from, nil) + |> Ecto.Changeset.put_change(:domain_changed_at, nil) + |> Repo.update() + + _ -> + {:ok, :ignore} + end + + _repo, _context -> + {:ok, :ignore} + end) + |> Ecto.Multi.insert(:site, fn %{site_changeset: site, create_team: team} -> + Ecto.Changeset.put_assoc(site, :team, team) + end) + |> Ecto.Multi.run(:trial, fn _repo, %{create_team: team} -> + if is_nil(team.trial_expiry_date) and is_nil(team.subscription) do + Teams.start_trial(team) + {:ok, :trial_started} + else + {:ok, :trial_already_started} + end + end) + |> Ecto.Multi.run(:updated_lock, fn _repo, %{create_team: team} -> + lock_state = + if ee?() do + Billing.SiteLocker.update_for(team, send_email?: false) + else + :unlocked + end + + {:ok, lock_state} + end) + |> Repo.transaction() + end + + @spec clear_stats_start_date!(Site.t()) :: Site.t() + def clear_stats_start_date!(site) do + site + |> Ecto.Changeset.change(stats_start_date: nil) + |> Plausible.Repo.update!() + end + + @spec stats_start_date(Site.t()) :: Date.t() | nil + @doc """ + Returns the date of the first event of the given site, or `nil` if the site + does not have stats yet. + + If this is the first time the function is called for the site, it queries + imported stats and Clickhouse, choosing the earliest start date and saves + it in the sites table. + """ + def stats_start_date(site) + + on_ee do + # for now, we're going to always update consolidated views + def stats_start_date(%Site{consolidated: true} = site) do + team = Repo.preload(site, :team).team + + site + |> Plausible.ConsolidatedView.change_stats_dates(team) + |> Repo.update!() + |> Map.fetch!(:stats_start_date) + end + end + + def stats_start_date(%Site{stats_start_date: %Date{} = date}) do + date + end + + def stats_start_date(%Site{} = site) do + start_date = + [ + Plausible.Imported.earliest_import_start_date(site), + native_stats_start_date(site) + ] + |> Enum.reject(&is_nil/1) + |> Enum.min(Date, fn -> nil end) + + if start_date do + updated_site = + site + |> Site.set_stats_start_date(start_date) + |> Repo.update!() + + updated_site.stats_start_date + end + end + + @spec native_stats_start_date(Site.t()) :: Date.t() | nil + def native_stats_start_date(site) do + Plausible.Stats.Clickhouse.pageview_start_date_local(site) + end + + def has_stats?(site) do + !!stats_start_date(site) + end + + def create_shared_link(site, name, opts \\ []) do + password = Keyword.get(opts, :password) + + site = Plausible.Repo.preload(site, :team) + skip_feature_check? = Keyword.get(opts, :skip_feature_check?, false) + + if not skip_feature_check? and SharedLinks.check_availability(site.team) != :ok do + {:error, :upgrade_required} + else + segment_id = fetch_segment_id_for_site(Keyword.get(opts, :segment_id), site) + + %SharedLink{site_id: site.id, slug: Nanoid.generate()} + |> SharedLink.changeset( + %{name: name, password: password}, + Keyword.take(opts, [:skip_special_name_check?]) + ) + |> Ecto.Changeset.put_change(:segment_id, segment_id) + |> Repo.insert() + end + end + + def update_shared_link(shared_link, site, params) do + # Set segment_id via put_change, not cast, to enforce site-scoped ownership. + segment_id = fetch_segment_id_for_site(params["segment_id"], site) + link_params = Map.drop(params, ["segment_id"]) + + shared_link + |> SharedLink.changeset(link_params) + |> Ecto.Changeset.put_change(:segment_id, segment_id) + |> Repo.update() + end + + defp fetch_segment_id_for_site(id, _site) when id in [nil, ""], do: nil + + defp fetch_segment_id_for_site(id, site) do + with {int, ""} when int > 0 <- Integer.parse(to_string(id)), + %{id: segment_id} <- + Repo.get_by(Plausible.Segments.Segment, id: int, site_id: site.id) do + segment_id + else + _ -> nil + end + end + + def shared_link_url(site, link) do + base = PlausibleWeb.Endpoint.url() + domain = "/share/#{URI.encode_www_form(site.domain)}" + base <> domain <> "?auth=" <> link.slug + end + + def update_legacy_time_on_page_cutoff!(site, cutoff) do + site + |> Ecto.Changeset.change() + |> Ecto.Changeset.put_change(:legacy_time_on_page_cutoff, cutoff) + |> Repo.update!() + end + + def has_goals?(site) do + Repo.exists?( + from(g in Plausible.Goal, + where: g.site_id == ^site.id + ) + ) + end + + def get_for_user!(user, domain, opts \\ []) do + opts = default_get_for_user_opts(opts) + roles = Keyword.fetch!(opts, :roles) + include_consolidated? = Keyword.fetch!(opts, :include_consolidated?) + + site = + if :super_admin in roles and Plausible.Auth.super_admin?(user.id) do + get_by_domain!(domain, include_consolidated?: include_consolidated?) + else + user.id + |> get_for_user_query(domain, List.delete(roles, :super_admin), opts) + |> Repo.one!() + end + + Repo.preload(site, :team) + end + + def get_for_user(user, domain, opts \\ []) do + opts = default_get_for_user_opts(opts) + roles = Keyword.fetch!(opts, :roles) + include_consolidated? = Keyword.fetch!(opts, :include_consolidated?) + + if :super_admin in roles and Plausible.Auth.super_admin?(user.id) do + get_by_domain(domain, include_consolidated?: include_consolidated?) + else + user.id + |> get_for_user_query(domain, List.delete(roles, :super_admin), opts) + |> Repo.one() + end + end + + defp get_for_user_query(user_id, domain, roles, opts) do + include_consolidated? = Keyword.fetch!(opts, :include_consolidated?) + roles = Enum.map(roles, &to_string/1) + + q = + from(s in Site, + join: t in assoc(s, :team), + join: tm in assoc(t, :team_memberships), + left_join: gm in assoc(tm, :guest_memberships), + where: tm.user_id == ^user_id, + where: coalesce(gm.role, tm.role) in ^roles, + where: s.domain == ^domain or s.domain_changed_from == ^domain, + where: is_nil(gm.id) or gm.site_id == s.id, + select: s + ) + + if include_consolidated? do + q + else + from(s in Site.regular(q)) + end + end + + defp default_get_for_user_opts(opts) do + Keyword.merge( + [include_consolidated?: false, roles: [:owner, :admin, :editor, :viewer]], + opts + ) + end +end diff --git a/lib/plausible/sites/index.ex b/lib/plausible/sites/index.ex new file mode 100644 index 0000000000000000000000000000000000000000..191e70775abffe507fd39b8044fdbd72a9952986 --- /dev/null +++ b/lib/plausible/sites/index.ex @@ -0,0 +1,411 @@ +defmodule Plausible.Sites.Index do + @moduledoc """ + Site index query module: fetches a sorted, paginated list of site IDs for a + user, supporting alphanumeric or traffic-based sorting over the last 24 hours. + """ + + import Ecto.Query + + alias Plausible.Auth + alias Plausible.Repo + alias Plausible.ClickhouseRepo + alias Plausible.Site + alias Plausible.Teams + alias Plausible.Teams.Sites + + def sort_direction_values, do: [:asc, :desc] + def sort_by_values, do: [:traffic, :alnum] + + @type sort_by() :: :alnum | :traffic + @type sort_direction() :: :asc | :desc + + @type index_option() :: + {:team, Teams.Team.t() | nil} + | {:sort_by, sort_by()} + | {:sort_direction, sort_direction()} + + @type pagination_option() :: + {:page, pos_integer() | String.t() | nil} + | {:page_size, pos_integer() | String.t() | nil} + | {:filter_by_domain, String.t() | nil} + + defmodule Page do + @moduledoc """ + A single page of results, drop-in replacement for Scrivener.Page + """ + + @type t() :: %__MODULE__{ + entries: [pos_integer()], + page_number: pos_integer(), + page_size: pos_integer(), + total_entries: non_neg_integer(), + total_pages: pos_integer() + } + + @enforce_keys [:entries, :page_number, :page_size, :total_entries, :total_pages] + defstruct [:entries, :page_number, :page_size, :total_entries, :total_pages] + end + + defmodule State do + @moduledoc """ + In-memory representation of a fully-sorted site index for one user + """ + + @type t() :: %__MODULE__{ + user: Auth.User.t(), + opts: %{team: Plausible.Teams.Team.t() | nil}, + ordered_ids: [pos_integer()], + pins: %{pos_integer() => NaiveDateTime.t() | nil}, + domains: %{pos_integer() => String.t()}, + traffic: %{pos_integer() => non_neg_integer()}, + sort_by: Plausible.Sites.Index.sort_by(), + sort_direction: Plausible.Sites.Index.sort_direction() + } + + @enforce_keys [:user, :opts, :ordered_ids, :pins, :sort_by, :sort_direction] + defstruct [ + :user, + :opts, + :ordered_ids, + :pins, + :sort_by, + :sort_direction, + domains: %{}, + traffic: %{} + ] + end + + @doc """ + Builds an `Index.State` for `user` by running all necessary queries + """ + @spec build(Auth.User.t(), [index_option()] | map()) :: State.t() + def build(user, opts \\ []) + + def build(user, %{__struct__: _} = opts) do + build(user, Map.from_struct(opts)) + end + + def build(user, opts) when is_map(opts) do + build(user, Keyword.new(opts)) + end + + def build(user, opts) do + sort_by = Keyword.get(opts, :sort_by, :alnum) + sort_direction = Keyword.get(opts, :sort_direction, :asc) + + site_ids = fetch_site_ids(user, opts) + pins = fetch_pins(user, site_ids) + domains = fetch_domains(site_ids) + + %State{ + user: user, + opts: %{ + team: Keyword.get(opts, :team) + }, + ordered_ids: site_ids, + pins: pins, + domains: domains, + traffic: %{}, + sort_by: sort_by, + sort_direction: sort_direction + } + |> sort(opts) + end + + @spec paginate( + State.t(), + [pagination_option()] + ) :: Page.t() + def paginate(%State{ordered_ids: ordered_ids} = state, opts \\ []) do + page_number = + opts + |> Keyword.get(:page) + |> cast_int(min: 1, max: :unlimited, default: 1) + + page_size = + opts + |> Keyword.get(:page_size) + |> cast_int(min: 1, max: 100, default: 24) + + filter_by_domain = Keyword.get(opts, :filter_by_domain) + + filtered_ids = apply_domain_filter(ordered_ids, state.domains, filter_by_domain) + + total_entries = length(filtered_ids) + total_pages = max(1, ceil(total_entries / page_size)) + page_number = min(page_number, total_pages) + + entries = + filtered_ids + |> Enum.drop((page_number - 1) * page_size) + |> Enum.take(page_size) + + %Page{ + entries: entries, + page_number: page_number, + page_size: page_size, + total_entries: total_entries, + total_pages: total_pages + } + end + + @spec refresh_pins(State.t()) :: State.t() + def refresh_pins( + %State{ + user: user, + ordered_ids: ordered_ids, + domains: domains, + traffic: traffic, + sort_by: sort_by, + sort_direction: sort_direction + } = state + ) do + new_pins = fetch_pins(user, ordered_ids) + + new_ordered_ids = + if sort_by == :traffic do + sort_by_traffic(ordered_ids, new_pins, traffic, sort_direction) + else + sort_alnum(ordered_ids, new_pins, domains, sort_direction) + end + + %State{state | pins: new_pins, ordered_ids: new_ordered_ids} + end + + @spec sort(State.t(), [index_option()] | map()) :: State.t() + + def sort(user, %{__struct__: _} = opts) do + sort(user, Map.from_struct(opts)) + end + + def sort(user, opts) when is_map(opts) do + sort(user, Keyword.new(opts)) + end + + def sort(%State{} = state, opts) do + sort_by = Keyword.get(opts, :sort_by, state.sort_by) + sort_direction = Keyword.get(opts, :sort_direction, state.sort_direction) + + apply_order(state, sort_by, sort_direction) + end + + # no traffic data yet, fetch + defp apply_order(%State{traffic: t} = state, :traffic, sort_direction) + when map_size(t) == 0 do + fetch_traffic_and_sort(state, sort_direction) + end + + # same sort_by, flip direction + defp apply_order( + %State{sort_by: :traffic, sort_direction: old_direction} = state, + :traffic, + new_direction + ) + when old_direction != new_direction do + ordered_ids = flip_order(state.ordered_ids, state.pins) + + %State{ + state + | sort_direction: new_direction, + ordered_ids: ordered_ids + } + end + + # alnum flip + defp apply_order( + %State{sort_by: :alnum, sort_direction: old_direction} = state, + :alnum, + sort_direction + ) + when old_direction != sort_direction do + ordered_ids = flip_order(state.ordered_ids, state.pins) + + %State{ + state + | sort_direction: sort_direction, + traffic: %{}, + ordered_ids: ordered_ids + } + end + + # alnum + defp apply_order(%State{} = state, :alnum, sort_direction) do + ordered_ids = sort_alnum(state.ordered_ids, state.pins, state.domains, sort_direction) + + %State{ + state + | sort_by: :alnum, + sort_direction: sort_direction, + traffic: %{}, + ordered_ids: ordered_ids + } + end + + # no-op + defp apply_order( + %State{ + sort_by: sort_by, + sort_direction: sort_direction + } = state, + sort_by, + sort_direction + ) do + state + end + + defp fetch_traffic_and_sort(%State{} = state, sort_direction) do + traffic_list = traffic_for_site_ids(state.ordered_ids) + traffic_map = Map.new(traffic_list) + ordered_ids = sort_by_traffic(state.ordered_ids, state.pins, traffic_map, sort_direction) + + %State{ + state + | sort_by: :traffic, + sort_direction: sort_direction, + traffic: traffic_map, + ordered_ids: ordered_ids + } + end + + defp flip_order(site_ids, pins) do + {pinned, unpinned} = split_pinned(site_ids, pins) + pinned ++ Enum.reverse(unpinned) + end + + @spec fetch_site_ids(Auth.User.t(), [index_option()]) :: [pos_integer()] + def fetch_site_ids(user, opts \\ []) do + team = Keyword.get(opts, :team) + + Repo.all( + from(u in subquery(Sites.accessible_by(user, team)), + inner_join: s in ^Site.regular(), + on: u.site_id == s.id, + select: s.id + ) + ) + end + + @spec traffic_for_site_ids([pos_integer()]) :: [{pos_integer(), non_neg_integer()}] + def traffic_for_site_ids([]), do: [] + + def traffic_for_site_ids(site_ids) do + now = DateTime.utc_now() + utc_first_dt = DateTime.shift(now, hour: -24) + + utc_first = DateTime.to_naive(utc_first_dt) + utc_last = DateTime.to_naive(now) + utc_first_minus_7d = NaiveDateTime.add(utc_first, -7, :day) + + site_ids + |> Enum.chunk_every(1_000) + |> Task.async_stream( + fn batch -> + try do + ClickhouseRepo.all( + from(s in "sessions_v2", + where: s.site_id in ^batch, + where: s.sign == 1, + where: s.start >= ^utc_first_minus_7d, + where: s.timestamp >= ^utc_first, + where: s.start <= ^utc_last, + group_by: s.site_id, + select: + {s.site_id, fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id)} + ) + ) + rescue + e -> + Sentry.capture_message("traffic_for_site_ids: batch query failed", + extra: %{ + first_site_id: List.first(batch), + last_site_id: List.last(batch), + batch_size: length(batch), + error: Exception.message(e) + } + ) + + [] + end + end, + max_concurrency: 4, + ordered: false + ) + |> Enum.flat_map(fn {:ok, rows} -> rows end) + end + + defp fetch_pins(_user, []), do: %{} + + defp fetch_pins(user, site_ids) do + pinned = + from(up in Site.UserPreference, + where: up.user_id == ^user.id and up.site_id in ^site_ids and not is_nil(up.pinned_at), + select: {up.site_id, up.pinned_at} + ) + |> Repo.all() + |> Map.new() + + Map.new(site_ids, fn id -> {id, Map.get(pinned, id)} end) + end + + defp sort_by_traffic(site_ids, pins, traffic_map, sort_direction) do + sort_ids(site_ids, pins, &Map.get(traffic_map, &1, 0), sort_direction) + end + + defp sort_alnum(site_ids, pins, domains, sort_direction) do + sort_ids(site_ids, pins, &Map.get(domains, &1, ""), sort_direction) + end + + defp sort_ids(site_ids, pins, key_fn, direction) do + {pinned, unpinned} = split_pinned(site_ids, pins) + pinned ++ Enum.sort_by(unpinned, key_fn, direction) + end + + defp split_pinned(site_ids, pins) do + {pinned_pairs, unpinned} = + Enum.split_with(site_ids, fn id -> not is_nil(Map.get(pins, id)) end) + + pinned = + pinned_pairs + |> Enum.sort_by(fn id -> pins[id] end, {:desc, NaiveDateTime}) + + {pinned, unpinned} + end + + defp fetch_domains([]), do: %{} + + defp fetch_domains(site_ids) do + from(s in Site.regular(), + where: s.id in ^site_ids, + select: {s.id, s.domain} + ) + |> Repo.all() + |> Map.new() + end + + defp apply_domain_filter(site_ids, _domains, filter) + when filter in [nil, ""] do + site_ids + end + + defp apply_domain_filter(site_ids, domains, filter) when is_binary(filter) do + needle = String.downcase(filter) + + Enum.filter(site_ids, fn id -> + String.contains?(String.downcase(Map.get(domains, id, "")), needle) + end) + end + + defp cast_int(value, min: min, max: max, default: default) when is_binary(value) do + case Integer.parse(value) do + {int, ""} when int >= min and int <= max -> int + _ -> default + end + end + + defp cast_int(value, min: min, max: max, default: _default) + when is_integer(value) and value >= min and value <= max do + value + end + + defp cast_int(_value, min: _min, max: _max, default: default), do: default +end diff --git a/lib/plausible/sites/index/user_preference.ex b/lib/plausible/sites/index/user_preference.ex new file mode 100644 index 0000000000000000000000000000000000000000..53b2d832d47da49f17b7f55b81b18b7200a2270a --- /dev/null +++ b/lib/plausible/sites/index/user_preference.ex @@ -0,0 +1,25 @@ +defmodule Plausible.Sites.Index.UserPreference do + @moduledoc """ + User preference persistence schema for the sites index + """ + + use Ecto.Schema + import Ecto.Changeset + alias Plausible.Sites.Index + + @primary_key false + embedded_schema do + field :sort_by, Ecto.Enum, values: Index.sort_by_values(), default: :traffic + field :sort_direction, Ecto.Enum, values: Index.sort_direction_values(), default: :desc + end + + def changeset(struct \\ %__MODULE__{}, attrs) do + cast(struct, attrs, [:sort_by, :sort_direction]) + end + + def new(attrs) do + attrs |> changeset() |> apply_changes() + end + + def default(), do: %__MODULE__{} +end diff --git a/lib/plausible/stats.ex b/lib/plausible/stats.ex new file mode 100644 index 0000000000000000000000000000000000000000..2dd9229dd2a50e653a62801a80198f74bdf7df2f --- /dev/null +++ b/lib/plausible/stats.ex @@ -0,0 +1,34 @@ +defmodule Plausible.Stats do + @moduledoc false + + use Plausible + use Plausible.ClickhouseRepo + + alias Plausible.Stats.{ + CurrentVisitors, + FilterSuggestions, + QueryRunner + } + + def query(site, query) do + QueryRunner.run(site, query) + end + + def current_visitors(site, duration \\ Duration.new!(minute: -5)) do + CurrentVisitors.current_visitors(site, duration) + end + + on_ee do + def funnel(site, query, funnel) do + Plausible.Stats.Funnel.funnel(site, query, funnel) + end + end + + def filter_suggestions(site, query, filter_name, filter_search) do + FilterSuggestions.filter_suggestions(site, query, filter_name, filter_search) + end + + def custom_prop_value_filter_suggestions(site, query, prop_key, filter_search) do + FilterSuggestions.custom_prop_value_filter_suggestions(site, query, prop_key, filter_search) + end +end diff --git a/lib/plausible/stats/api_query_parser.ex b/lib/plausible/stats/api_query_parser.ex new file mode 100644 index 0000000000000000000000000000000000000000..c8d16cc1f72b1cdb5a46de85e47ca7c6817b77f7 --- /dev/null +++ b/lib/plausible/stats/api_query_parser.ex @@ -0,0 +1,391 @@ +defmodule Plausible.Stats.ApiQueryParser do + @moduledoc false + + use Plausible + + alias Plausible.Stats.{Filters, Metrics, JSONSchema, QueryError} + + @default_include %Plausible.Stats.QueryInclude{ + imports: false, + imports_meta: false, + time_labels: false, + total_rows: false, + trim_relative_date_range: false, + compare: nil, + compare_match_day_of_week: false, + legacy_time_on_page_cutoff: nil + } + + def default_include(), do: @default_include + + @default_pagination %{limit: 10_000, offset: 0} + + def default_pagination(), do: @default_pagination + + def parse(params, opts \\ []) when is_map(params) do + with :ok <- JSONSchema.validate(params), + {:ok, input_date_range} <- parse_input_date_range(params["date_range"]), + {:ok, metrics} <- parse_metrics(Map.fetch!(params, "metrics")), + {:ok, filters} <- parse_filters(params["filters"]), + {:ok, dimensions} <- parse_dimensions(params["dimensions"]), + {:ok, order_by} <- parse_order_by(params["order_by"]), + {:ok, pagination} <- parse_pagination(params["pagination"]), + {:ok, include} <- parse_include(params["include"]) do + {:ok, + Plausible.Stats.ParsedQueryParams.new!(%{ + input_date_range: input_date_range, + metrics: metrics, + filters: filters, + dimensions: dimensions, + order_by: order_by, + pagination: pagination, + include: include, + now: Keyword.get(opts, :now) + })} + end + end + + def parse_metrics(metrics) when is_list(metrics) do + parse_list(metrics, &parse_metric/1) + end + + defp parse_metric(metric_str) do + case Metrics.from_string(metric_str) do + {:ok, metric} -> + {:ok, metric} + + _ -> + {:error, + %QueryError{code: :invalid_metrics, message: "Unknown metric '#{i(metric_str)}'."}} + end + end + + def parse_filters(filters) when is_list(filters) do + parse_list(filters, &parse_filter/1) + end + + def parse_filters(nil), do: {:ok, []} + + defp parse_filter(filter) do + with {:ok, operator} <- parse_operator(filter), + {:ok, second} <- parse_filter_second(operator, filter), + {:ok, rest} <- parse_filter_rest(operator, filter) do + {:ok, [operator, second | rest]} + end + end + + defp parse_operator(["is" | _rest]), do: {:ok, :is} + defp parse_operator(["is_not" | _rest]), do: {:ok, :is_not} + defp parse_operator(["matches" | _rest]), do: {:ok, :matches} + defp parse_operator(["matches_not" | _rest]), do: {:ok, :matches_not} + defp parse_operator(["matches_wildcard" | _rest]), do: {:ok, :matches_wildcard} + defp parse_operator(["matches_wildcard_not" | _rest]), do: {:ok, :matches_wildcard_not} + defp parse_operator(["contains" | _rest]), do: {:ok, :contains} + defp parse_operator(["contains_not" | _rest]), do: {:ok, :contains_not} + defp parse_operator(["and" | _rest]), do: {:ok, :and} + defp parse_operator(["or" | _rest]), do: {:ok, :or} + defp parse_operator(["not" | _rest]), do: {:ok, :not} + defp parse_operator(["has_done" | _rest]), do: {:ok, :has_done} + defp parse_operator(["has_not_done" | _rest]), do: {:ok, :has_not_done} + + defp parse_operator(filter), + do: + {:error, + %QueryError{code: :invalid_filters, message: "Unknown operator for filter '#{i(filter)}'."}} + + def parse_filter_second(operator, [_, filters | _rest]) when operator in [:and, :or], + do: parse_filters(filters) + + def parse_filter_second(operator, [_, filter | _rest]) + when operator in [:not, :has_done, :has_not_done], + do: parse_filter(filter) + + def parse_filter_second(_operator, filter), do: parse_filter_dimension(filter) + + defp parse_filter_dimension([_operator, filter_dimension | _rest] = filter) do + parse_filter_dimension_string(filter_dimension, "Invalid filter '#{i(filter)}'.") + end + + defp parse_filter_dimension(filter), + do: {:error, %QueryError{code: :invalid_filters, message: "Invalid filter '#{i(filter)}'."}} + + defp parse_filter_rest(operator, filter) + when operator in [ + :is, + :is_not, + :matches, + :matches_not, + :matches_wildcard, + :matches_wildcard_not, + :contains, + :contains_not + ] do + with {:ok, clauses} <- parse_clauses_list(filter), + {:ok, modifiers} <- parse_filter_modifiers(Enum.at(filter, 3)) do + {:ok, [clauses | modifiers]} + end + end + + defp parse_filter_rest(operator, _filter) + when operator in [:not, :and, :or, :has_done, :has_not_done], + do: {:ok, []} + + defp parse_clauses_list([operator, dimension, list | _rest] = filter) when is_list(list) do + all_strings? = Enum.all?(list, &is_binary/1) + all_integers? = Enum.all?(list, &is_integer/1) + + case {dimension, all_strings?} do + {"visit:city", false} when all_integers? -> + {:ok, list} + + {"visit:country", true} when operator in ["is", "is_not"] -> + if Enum.all?(list, &(String.length(&1) == 2)) do + {:ok, list} + else + {:error, + %QueryError{ + code: :invalid_filters, + message: + "Invalid visit:country filter, visit:country needs to be a valid 2-letter country code." + }} + end + + {"segment", _} when all_integers? -> + {:ok, list} + + {_, true} when dimension !== "segment" -> + {:ok, list} + + _ -> + {:error, %QueryError{code: :invalid_filters, message: "Invalid filter '#{i(filter)}'."}} + end + end + + defp parse_clauses_list(filter), + do: {:error, %QueryError{code: :invalid_filters, message: "Invalid filter '#{i(filter)}'."}} + + defp parse_filter_modifiers(modifiers) when is_map(modifiers) do + {:ok, [atomize_keys(modifiers)]} + end + + defp parse_filter_modifiers(nil) do + {:ok, []} + end + + def parse_input_date_range("day"), do: {:ok, :day} + def parse_input_date_range("24h"), do: {:ok, :"24h"} + def parse_input_date_range("month"), do: {:ok, :month} + def parse_input_date_range("year"), do: {:ok, :year} + def parse_input_date_range("all"), do: {:ok, :all} + + def parse_input_date_range(shorthand) when is_binary(shorthand) do + case Integer.parse(shorthand) do + {n, "d"} when n > 0 and n <= 5_000 -> + {:ok, {:last_n_days, n}} + + {n, "mo"} when n > 0 and n <= 100 -> + {:ok, {:last_n_months, n}} + + _ -> + {:error, + %QueryError{code: :invalid_date_range, message: "Invalid date_range #{i(shorthand)}"}} + end + end + + def parse_input_date_range([from, to]) when is_binary(from) and is_binary(to) do + case parse_date_strings(from, to) do + {:ok, dates} -> {:ok, dates} + {:error, _} -> parse_timestamp_strings(from, to) + end + end + + def parse_input_date_range(unknown) do + {:error, %QueryError{code: :invalid_date_range, message: "Invalid date_range #{i(unknown)}"}} + end + + def parse_date_strings(from, to) do + with {:ok, from_date} <- Date.from_iso8601(from), + {:ok, to_date} <- Date.from_iso8601(to) do + {:ok, {:date_range, from_date, to_date}} + end + end + + defp parse_timestamp_strings(from, to) do + with {:ok, from_datetime, _offset} <- DateTime.from_iso8601(from), + {:ok, to_datetime, _offset} <- DateTime.from_iso8601(to) do + {:ok, {:datetime_range, from_datetime, to_datetime}} + else + _ -> + {:error, + %QueryError{code: :invalid_date_range, message: "Invalid date_range '#{i([from, to])}'."}} + end + end + + def parse_dimensions(dimensions) when is_list(dimensions) do + parse_list( + dimensions, + &parse_dimension_entry(&1, "Invalid dimensions '#{i(dimensions)}'") + ) + end + + def parse_dimensions(nil), do: {:ok, []} + + def parse_order_by(order_by) when is_list(order_by) do + parse_list(order_by, &parse_order_by_entry/1) + end + + def parse_order_by(nil), do: {:ok, nil} + + def parse_order_by(order_by), + do: + {:error, + %QueryError{code: :invalid_order_by, message: "Invalid order_by '#{i(order_by)}'."}} + + defp parse_order_by_entry(entry) do + with {:ok, value} <- parse_metric_or_dimension(entry), + {:ok, order_direction} <- parse_order_direction(entry) do + {:ok, {value, order_direction}} + end + end + + defp parse_dimension_entry(key, error_message) do + case { + parse_time_dimension(key), + parse_non_time_dimension(key) + } do + {{:ok, time}, _} -> {:ok, time} + {_, {:ok, dimension}} -> {:ok, dimension} + _ -> {:error, %QueryError{code: :invalid_dimensions, message: error_message}} + end + end + + defp parse_metric_or_dimension([value, _] = entry) do + case { + parse_time_dimension(value), + parse_metric(value), + parse_non_time_dimension(value) + } do + {{:ok, time_dimension}, _, _} -> + {:ok, time_dimension} + + {_, {:ok, metric}, _} -> + {:ok, metric} + + {_, _, {:ok, non_time_dimension}} -> + {:ok, non_time_dimension} + + _ -> + {:error, + %QueryError{code: :invalid_order_by, message: "Invalid order_by entry '#{i(entry)}'."}} + end + end + + defp parse_time_dimension("time"), do: {:ok, "time"} + defp parse_time_dimension("time:minute"), do: {:ok, "time:minute"} + defp parse_time_dimension("time:hour"), do: {:ok, "time:hour"} + defp parse_time_dimension("time:day"), do: {:ok, "time:day"} + defp parse_time_dimension("time:week"), do: {:ok, "time:week"} + defp parse_time_dimension("time:month"), do: {:ok, "time:month"} + defp parse_time_dimension(_), do: :error + + @breakdown_only_non_time_dimensions ["event:prop_key"] + + defp parse_non_time_dimension(dimension) do + if dimension in @breakdown_only_non_time_dimensions do + {:ok, dimension} + else + parse_filter_dimension_string(dimension) + end + end + + defp parse_order_direction([_, "asc"]), do: {:ok, :asc} + defp parse_order_direction([_, "desc"]), do: {:ok, :desc} + + defp parse_order_direction(entry), + do: + {:error, + %QueryError{code: :invalid_order_by, message: "Invalid order_by entry '#{i(entry)}'."}} + + def parse_include(include) when is_map(include) do + parsed_include_params_or_error = + include + |> Enum.reduce_while({:ok, []}, fn {key, value}, {:ok, acc} -> + case parse_include_entry(key, value) do + {:ok, parsed_tuple} -> {:cont, {:ok, acc ++ [parsed_tuple]}} + {:error, query_error} -> {:halt, {:error, query_error}} + end + end) + + with {:ok, parsed_include_params} <- parsed_include_params_or_error do + {:ok, struct!(@default_include, parsed_include_params)} + end + end + + def parse_include(nil), do: {:ok, @default_include} + + def parse_include(include), + do: {:error, %QueryError{code: :invalid_include, message: "Invalid include '#{i(include)}'."}} + + @allowed_include_keys Enum.map(Map.keys(@default_include), &Atom.to_string/1) + + defp parse_include_entry(key, value) when key in @allowed_include_keys do + {:ok, {String.to_existing_atom(key), value}} + end + + defp parse_include_entry(key, _value), + do: {:error, %QueryError{code: :invalid_include, message: "Invalid include key'#{i(key)}'."}} + + def parse_pagination(pagination) when is_map(pagination) do + {:ok, Map.merge(@default_pagination, atomize_keys(pagination))} + end + + def parse_pagination(nil), do: {:ok, @default_pagination} + + defp atomize_keys(map) when is_map(map) do + Map.new(map, fn {key, value} -> + {String.to_existing_atom(key), value} + end) + end + + defp parse_filter_dimension_string(dimension, error_message \\ "") do + case dimension do + "event:props:" <> property_name -> + if String.length(property_name) > 0 do + {:ok, dimension} + else + {:error, %QueryError{code: :invalid_filters, message: error_message}} + end + + "event:" <> key -> + if key in Filters.event_props() do + {:ok, dimension} + else + {:error, %QueryError{code: :invalid_filters, message: error_message}} + end + + "visit:" <> key -> + if key in Filters.visit_props() do + {:ok, dimension} + else + {:error, %QueryError{code: :invalid_filters, message: error_message}} + end + + "segment" -> + {:ok, dimension} + + _ -> + {:error, %QueryError{code: :invalid_filters, message: error_message}} + end + end + + defp i(value), do: inspect(value, charlists: :as_lists) + + defp parse_list(list, parser_function) do + Enum.reduce_while(list, {:ok, []}, fn value, {:ok, results} -> + case parser_function.(value) do + {:ok, result} -> {:cont, {:ok, results ++ [result]}} + {:error, _} = error -> {:halt, error} + end + end) + end +end diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex new file mode 100644 index 0000000000000000000000000000000000000000..68c9391846da5ba0089da8bc5c730d0f5736c0eb --- /dev/null +++ b/lib/plausible/stats/base.ex @@ -0,0 +1,52 @@ +defmodule Plausible.Stats.Base do + use Plausible + + alias Plausible.Stats.{TableDecider, SQL} + import Ecto.Query + + def base_event_query(query) do + events_q = query_events(query) + + if TableDecider.events_join_sessions?(query) do + sessions_q = + from( + s in query_sessions(query), + select: %{session_id: s.session_id}, + where: s.sign == 1, + group_by: s.session_id + ) + + from( + e in events_q, + join: sq in subquery(sessions_q), + on: e.session_id == sq.session_id + ) + else + events_q + end + end + + defp query_events(query) do + q = + from(e in "events_v2", + where: ^SQL.WhereBuilder.build(:events, query), + where: ^SQL.WhereBuilder.derived_name_filter(query) + ) + + on_ee do + q = Plausible.Stats.Sampling.add_query_hint(q, query) + end + + q + end + + def query_sessions(query) do + q = from(s in "sessions_v2", where: ^SQL.WhereBuilder.build(:sessions, query)) + + on_ee do + q = Plausible.Stats.Sampling.add_query_hint(q, query) + end + + q + end +end diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex new file mode 100644 index 0000000000000000000000000000000000000000..375c9e9f47fc9afd0e0f116a42fa1a461e791fdf --- /dev/null +++ b/lib/plausible/stats/clickhouse.ex @@ -0,0 +1,210 @@ +defmodule Plausible.Stats.Clickhouse do + @moduledoc """ + Clickhouse utility functions + """ + + use Plausible + use Plausible.Repo + use Plausible.ClickhouseRepo + use Plausible.Stats.SQL.Fragments + + import Ecto.Query, only: [from: 2, subquery: 1] + + alias Plausible.Timezones + alias Plausible.Stats + + @spec pageview_start_date_local(Plausible.Site.t()) :: Date.t() | nil + def pageview_start_date_local(site) do + datetime = + ClickhouseRepo.one( + from(e in "events_v2", + select: fragment("min(?)", e.timestamp), + where: e.site_id == ^site.id, + where: e.timestamp >= ^site.native_stats_start_at + ) + ) + + case datetime do + # no stats for this domain yet + ~N[1970-01-01 00:00:00] -> + nil + + _ -> + Timezones.to_date_in_timezone(datetime, site.timezone) + end + end + + def imported_pageview_count(site) do + Plausible.ClickhouseRepo.one( + from(i in "imported_visitors", + where: i.site_id == ^site.id, + select: sum(i.pageviews) + ) + ) + end + + @spec imported_pageview_counts(Plausible.Site.t()) :: %{non_neg_integer() => non_neg_integer()} + def imported_pageview_counts(site) do + from(i in "imported_visitors", + where: i.site_id == ^site.id, + group_by: i.import_id, + select: {i.import_id, sum(i.pageviews)} + ) + |> Plausible.ClickhouseRepo.all() + |> Map.new() + end + + def usage_breakdown([sid | _] = site_ids, date_range) when is_integer(sid) do + Enum.chunk_every(site_ids, 1000) + |> Enum.map(fn site_ids -> + fn -> + ClickhouseRepo.one( + from(e in "events_v2", + where: e.site_id in ^site_ids, + where: e.name != "engagement", + where: fragment("toDate(?)", e.timestamp) >= ^date_range.first, + where: fragment("toDate(?)", e.timestamp) <= ^date_range.last, + select: { + fragment("countIf(? = 'pageview')", e.name), + fragment("countIf(? != 'pageview')", e.name) + } + ) + ) + end + end) + |> ClickhouseRepo.parallel_tasks(max_concurrency: 10) + |> Enum.reduce(fn {pageviews, custom_events}, {pageviews_total, custom_events_total} -> + {pageviews_total + pageviews, custom_events_total + custom_events} + end) + end + + def usage_breakdown([], _date_range), do: {0, 0} + + @doc """ + Estimates a team's monthly traffic from a partial sample, computed entirely in + ClickHouse with one light query scoped to the team's own sites. Used by + `Plausible.Workers.ScoreTrialProspects`. + + Billable events = pageviews + custom events (everything except engagement). + The window is the trailing `@window_days` complete days (up to yesterday). + Within it, `min(toDate(timestamp))` is the team's `first_data_day`, so the + elapsed complete days since then give `observed_days` (clamped to + `[1, @window_days]`), and the monthly figure is + `round(events_in_window / observed_days * @days_in_month)` — all in the DB. + + Returns `%{first_data_day, events_in_window, observed_days, estimated_monthly}`. + `events_in_window == 0` means the team had no complete day of traffic. + """ + # trailing complete days sampled to estimate the monthly run rate + @window_days 30 + @days_in_month 30 + @spec trial_traffic([pos_integer()]) :: %{ + first_data_day: Date.t() | nil, + events_in_window: non_neg_integer(), + observed_days: non_neg_integer(), + estimated_monthly: non_neg_integer() + } + def trial_traffic([]) do + %{first_data_day: nil, events_in_window: 0, observed_days: 0, estimated_monthly: 0} + end + + def trial_traffic(site_ids) do + last_complete_day = Date.add(Date.utc_today(), -1) + first_day = Date.add(last_complete_day, -(@window_days - 1)) + + aggregates = + from(e in "events_v2", + where: e.site_id in ^site_ids, + where: e.name != "engagement", + where: fragment("toDate(?)", e.timestamp) >= ^first_day, + where: fragment("toDate(?)", e.timestamp) <= ^last_complete_day, + select: %{ + first_data_day: fragment("min(toDate(?))", e.timestamp), + events_in_window: fragment("count(*)"), + # elapsed complete days since first traffic, clamped to [1, @window_days] + observed_days: + fragment( + "least(greatest(dateDiff('day', min(toDate(?)), ?) + 1, 1), ?)", + e.timestamp, + ^last_complete_day, + ^@window_days + ) + } + ) + + ClickhouseRepo.one( + from(t in subquery(aggregates), + select: %{ + first_data_day: t.first_data_day, + events_in_window: t.events_in_window, + observed_days: t.observed_days, + estimated_monthly: + fragment( + "toUInt64(round(? / ? * ?))", + t.events_in_window, + t.observed_days, + ^@days_in_month + ) + } + ) + ) + end + + def per_site_usage_breakdown( + site_ids, + date_range, + limit \\ Plausible.Teams.Billing.max_sites_for_usage_breakdown() + ) + + def per_site_usage_breakdown([], _date_range, _limit), do: [] + + def per_site_usage_breakdown( + [sid | _] = site_ids, + date_range, + limit + ) + when is_integer(sid) and length(site_ids) <= limit do + ClickhouseRepo.all( + from(e in "events_v2", + where: e.site_id in ^site_ids, + where: e.name != "engagement", + where: fragment("toDate(?)", e.timestamp) >= ^date_range.first, + where: fragment("toDate(?)", e.timestamp) <= ^date_range.last, + group_by: e.site_id, + order_by: [ + desc: fragment("countIf(? = 'pageview') + countIf(? != 'pageview')", e.name, e.name) + ], + limit: ^limit, + select: { + e.site_id, + fragment("countIf(? = 'pageview')", e.name), + fragment("countIf(? != 'pageview')", e.name) + } + ) + ) + end + + def current_visitors(site) do + Stats.current_visitors(site) + end + + def current_visitors_12h(site) do + Stats.current_visitors(site, Duration.new!(hour: -12)) + end + + def has_pageviews?(site) do + # This function is currently only used in installation verification + # which is not accessible for consolidated views. + true = Plausible.Sites.regular?(site) + + ClickhouseRepo.exists?( + from(e in "events_v2", + where: + e.site_id == ^site.id and + e.name == "pageview" and + e.timestamp >= + ^site.native_stats_start_at + ) + ) + end +end diff --git a/lib/plausible/stats/compare.ex b/lib/plausible/stats/compare.ex new file mode 100644 index 0000000000000000000000000000000000000000..e492b5deb012257e4c524d49f530ea58016868ca --- /dev/null +++ b/lib/plausible/stats/compare.ex @@ -0,0 +1,39 @@ +defmodule Plausible.Stats.Compare do + def calculate_change(:conversion_rate, old_value, new_value) do + Float.round(new_value - old_value, 1) + end + + def calculate_change(:exit_rate, old_value, new_value) do + if is_float(old_value) and is_float(new_value) do + Float.round(new_value - old_value, 1) + end + end + + def calculate_change(:bounce_rate, old_count, new_count) do + if old_count > 0, do: new_count - old_count + end + + def calculate_change(_metric, old_count, new_count) do + percent_change(old_count, new_count) + end + + def percent_change(nil, _new_count), do: nil + def percent_change(_old_count, nil), do: nil + + def percent_change(%{value: old_count}, %{value: new_count}) do + percent_change(old_count, new_count) + end + + def percent_change(old_count, new_count) do + cond do + old_count == 0 and new_count > 0 -> + 100 + + old_count == 0 and new_count == 0 -> + 0 + + true -> + round((new_count - old_count) / old_count * 100) + end + end +end diff --git a/lib/plausible/stats/comparisons.ex b/lib/plausible/stats/comparisons.ex new file mode 100644 index 0000000000000000000000000000000000000000..4a5ba9c6d4ec9978341151d0d769b4157a0a80d0 --- /dev/null +++ b/lib/plausible/stats/comparisons.ex @@ -0,0 +1,265 @@ +defmodule Plausible.Stats.Comparisons do + @moduledoc """ + This module provides functions for comparing query periods. + + It allows you to compare a given period with a previous period or with the + same period from the previous year. For example, you can compare this month's + main graph with last month or with the same month from last year. + """ + + alias Plausible.Stats + alias Plausible.Stats.{DateTimeRange, Query, Time} + alias Plausible.Times + + @spec get_comparison_utc_time_range(Stats.Query.t()) :: DateTimeRange.t() + @doc """ + Generates a `DateTimeRange` representing the comparison period of a given + `%Query{}` struct (i.e. the `source_query`). + + There are different modes and options that determine the outcome of the + resulting DateTimeRange. Those are specified under `source_query.include`. + + Currently only historical periods are supported for comparisons (not `realtime` + and `30m` periods). + + ## Modes (`source_query.include.compare` field) + + * `:previous_period` - shifts back the query by the same number of days the + source query has. + + * `:year_over_year` - shifts back the query by 1 year. + + * `{:date_range, from, to}` - compares the query using a custom date range. + + ## Options + + * `source_query.include.compare_match_day_of_week` + + Determines whether the comparison query should be adjusted to match the + day of the week of the source query. When this option is set to true, the + comparison query is shifted to start on the same day of the week as the + source query, rather than on the exact same date. + + Example: if the source query starts on Sunday, January 1st, 2023 and the + `year_over_year` comparison query is configured to `match_day_of_week`, it + will be shifted to start on Sunday, January 2nd, 2022 instead of January 1st. + + Note: this option has no effect when custom date range mode is used. + + """ + def get_comparison_utc_time_range(%Stats.Query{} = source_query) do + datetime_range = + case source_query.include.compare do + {:datetime_range, from, to} -> + DateTimeRange.new!(from, to) + + _ -> + # For 24h period and today, work directly with datetime ranges to preserve time precision + if use_datetime_for_comparison?(source_query) do + get_comparison_datetime_range(source_query) + else + comparison_date_range = get_comparison_date_range(source_query) + + DateTimeRange.new!( + comparison_date_range.first, + comparison_date_range.last, + source_query.timezone + ) + end + end + + DateTimeRange.to_timezone(datetime_range, "Etc/UTC") + end + + defp use_datetime_for_comparison?(query) do + if query.input_date_range == :day do + today_from_now = Times.to_date(query.now, query.timezone) + day_from_range = Times.to_date(query.utc_time_range.first, query.timezone) + + Date.compare(today_from_now, day_from_range) == :eq + else + query.input_date_range == :"24h" + end + end + + def get_comparison_query( + %Query{comparison_utc_time_range: %DateTimeRange{} = comparison_range} = source_query + ) do + source_query + |> Query.set(utc_time_range: comparison_range) + end + + @doc """ + Builds comparison query that specifically filters for values appearing in the main query results. + + When querying for comparisons with dimensions and pagination, extra + filters are added to ensure comparison query returns same set of results + as main query. + """ + def add_comparison_filters(comparison_query, main_results_list) do + comparison_filters = + Enum.flat_map(main_results_list, &build_comparison_filter(&1, comparison_query)) + + comparison_query + |> add_query_filters(comparison_filters) + end + + defp add_query_filters(query, []), do: query + + defp add_query_filters(query, [filter]) do + query + |> Query.add_filter([:ignore_in_totals_query, filter]) + |> Query.set(pagination: nil) + end + + defp add_query_filters(query, filters) do + query + |> Query.add_filter([:ignore_in_totals_query, [:or, filters]]) + |> Query.set(pagination: nil) + end + + defp build_comparison_filter(%{dimensions: dimension_labels}, query) do + query_filters = + query.dimensions + |> Enum.zip(dimension_labels) + |> Enum.reject(fn {dimension, _label} -> Time.time_dimension?(dimension) end) + |> Enum.map(fn {dimension, label} -> [:is, dimension, [label]] end) + + case query_filters do + [] -> [] + [filter] -> [filter] + filters -> [[:and, filters]] + end + end + + # For 24h and today periods, shift the datetime range directly to preserve time precision + defp get_comparison_datetime_range( + %Query{ + input_date_range: input_range, + include: %{compare: :previous_period} = include + } = source_query + ) + when input_range in [:"24h", :day] do + offset = + if include.compare_match_day_of_week do + [day: -7] + else + [hour: -24] + end + + comparison_start = DateTime.shift(source_query.utc_time_range.first, offset) + comparison_end = DateTime.shift(source_query.utc_time_range.last, offset) + + DateTimeRange.new!(comparison_start, comparison_end) + end + + defp get_comparison_datetime_range( + %Query{ + input_date_range: input_range, + include: %{compare: :year_over_year} = include + } = source_query + ) + when input_range in [:"24h", :day] do + comparison_start = DateTime.shift(source_query.utc_time_range.first, year: -1) + comparison_end = DateTime.shift(source_query.utc_time_range.last, year: -1) + + if include.compare_match_day_of_week do + source_first_date = Times.to_date(source_query.utc_time_range.first, source_query.timezone) + comparison_first_date = Times.to_date(comparison_start, source_query.timezone) + + day_to_match = Date.day_of_week(source_first_date) + matched_date = shift_to_nearest(day_to_match, comparison_first_date, source_first_date) + days_shifted = Date.diff(matched_date, comparison_first_date) + + DateTimeRange.new!( + DateTime.shift(comparison_start, day: days_shifted), + DateTime.shift(comparison_end, day: days_shifted) + ) + else + DateTimeRange.new!(comparison_start, comparison_end) + end + end + + defp get_comparison_datetime_range( + %Query{ + input_date_range: input_range, + include: %{compare: {:date_range, from_date, to_date}} + } = source_query + ) + when input_range in [:"24h", :day] do + DateTimeRange.new!(from_date, to_date, source_query.timezone) + end + + defp get_comparison_date_range(%Query{include: %{compare: :year_over_year}} = source_query) do + source_date_range = Query.date_range(source_query, trim_trailing: true) + + start_date = source_date_range.first |> Date.shift(year: -1) + diff_in_days = Date.diff(source_date_range.last, source_date_range.first) + end_date = Date.add(start_date, diff_in_days) + + Date.range(start_date, end_date) + |> maybe_match_day_of_week(source_date_range, source_query) + end + + defp get_comparison_date_range(%Query{include: %{compare: :previous_period}} = source_query) do + source_date_range = Query.date_range(source_query, trim_trailing: true) + + last = source_date_range.last + diff_in_days = Date.diff(source_date_range.first, last) - 1 + + new_first = Date.add(source_date_range.first, diff_in_days) + new_last = Date.add(last, diff_in_days) + + Date.range(new_first, new_last) + |> maybe_match_day_of_week(source_date_range, source_query) + end + + defp get_comparison_date_range(%Query{include: %{compare: {:date_range, from_date, to_date}}}) do + Date.range(from_date, to_date) + end + + defp maybe_match_day_of_week(comparison_date_range, source_date_range, source_query) do + if source_query.include.compare_match_day_of_week do + day_to_match = Date.day_of_week(source_date_range.first) + + new_first = + shift_to_nearest( + day_to_match, + comparison_date_range.first, + source_date_range.first + ) + + days_shifted = Date.diff(new_first, comparison_date_range.first) + new_last = Date.add(comparison_date_range.last, days_shifted) + + Date.range(new_first, new_last) + else + comparison_date_range + end + end + + defp shift_to_nearest(day_of_week, date, reject) do + if Date.day_of_week(date) == day_of_week do + date + else + [next_occurring(day_of_week, date), previous_occurring(day_of_week, date)] + |> Enum.sort_by(&Date.diff(date, &1)) + |> Enum.reject(&(&1 == reject)) + |> List.first() + end + end + + defp next_occurring(day_of_week, date) do + days_to_add = day_of_week - Date.day_of_week(date) + days_to_add = if days_to_add > 0, do: days_to_add, else: days_to_add + 7 + + Date.add(date, days_to_add) + end + + defp previous_occurring(day_of_week, date) do + days_to_subtract = Date.day_of_week(date) - day_of_week + days_to_subtract = if days_to_subtract > 0, do: days_to_subtract, else: days_to_subtract + 7 + + Date.add(date, -days_to_subtract) + end +end diff --git a/lib/plausible/stats/current_visitors.ex b/lib/plausible/stats/current_visitors.ex new file mode 100644 index 0000000000000000000000000000000000000000..73b294ac4f14fb1245c23e62e9c9a604d24eb667 --- /dev/null +++ b/lib/plausible/stats/current_visitors.ex @@ -0,0 +1,20 @@ +defmodule Plausible.Stats.CurrentVisitors do + use Plausible.ClickhouseRepo + use Plausible.Stats.SQL.Fragments + + @spec current_visitors(Plausible.Site.t(), Duration.duration()) :: non_neg_integer + def current_visitors(site, duration \\ Duration.new!(minute: -5)) do + first_datetime = + NaiveDateTime.utc_now() + |> NaiveDateTime.shift(duration) + |> NaiveDateTime.truncate(:second) + + ClickhouseRepo.one( + from e in "events_v2", + where: ^Plausible.Sites.site_id_query_filter(site), + where: e.timestamp >= ^first_datetime, + where: e.name != "engagement", + select: uniq(e.user_id) + ) + end +end diff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.ex new file mode 100644 index 0000000000000000000000000000000000000000..8190d9caecf5f394bf8ec00a676233a5fd9abc33 --- /dev/null +++ b/lib/plausible/stats/custom_props.ex @@ -0,0 +1,34 @@ +defmodule Plausible.Stats.CustomProps do + @moduledoc """ + Module for querying user defined 'custom properties'. + """ + + alias Plausible.Stats.Filters + use Plausible.ClickhouseRepo + import Plausible.Stats.Base + + def fetch_prop_names(site, query) do + case Filters.get_toplevel_filter(query, "event:props:") do + [_op, "event:props:" <> key | _rest] -> + [key] + + _ -> + from(e in base_event_query(query), + join: meta in fragment("meta"), + hints: "ARRAY", + on: true, + select: meta.key, + distinct: true + ) + |> maybe_allowed_props_only(site) + |> ClickhouseRepo.all() + end + end + + def maybe_allowed_props_only(q, site) do + case Plausible.Props.allowed_for(site) do + :all -> q + allowed_props -> from [..., m] in q, where: m.key in ^allowed_props + end + end +end diff --git a/lib/plausible/stats/dashboard/csv_export.ex b/lib/plausible/stats/dashboard/csv_export.ex new file mode 100644 index 0000000000000000000000000000000000000000..869b6e8cf1a71cb9d47dfae74d11087848c6841f --- /dev/null +++ b/lib/plausible/stats/dashboard/csv_export.ex @@ -0,0 +1,329 @@ +defmodule Plausible.Stats.Dashboard.CsvExport do + @moduledoc false + + alias Plausible.Stats.Dashboard.QueryParser + alias Plausible.Stats.{Time, ParsedQueryParams, Query, QueryBuilder, QueryRunner, QueryResult} + + @csv_filenames [ + "visitors.csv", + "pages.csv", + "entry_pages.csv", + "exit_pages.csv", + "browsers.csv", + "browser_versions.csv", + "operating_systems.csv", + "operating_system_versions.csv", + "devices.csv", + "channels.csv", + "sources.csv", + "referrers.csv", + "utm_mediums.csv", + "utm_sources.csv", + "utm_campaigns.csv", + "utm_contents.csv", + "utm_terms.csv", + "countries.csv", + "regions.csv", + "cities.csv", + "custom_props.csv", + "conversions.csv" + ] + + def get_csvs(site, params, debug_metadata) do + with {:ok, tasks_by_filename} <- create_tasks_by_filename(site, params, debug_metadata) do + csv_values = + tasks_by_filename + |> Enum.map(fn {_filename, task} -> task end) + |> Plausible.ClickhouseRepo.parallel_tasks() + + csvs = + tasks_by_filename + |> Enum.map(fn {filename, _task} -> filename end) + |> Enum.zip(csv_values) + |> Enum.reject(fn {_filename, csv_values} -> is_nil(csv_values) end) + |> Enum.map(fn {k, v} -> {String.to_charlist(k), IO.iodata_to_binary(v)} end) + + {:ok, csvs} + end + end + + defp create_tasks_by_filename(site, params, debug_metadata) do + requested = Map.keys(params["reports"] || %{}) |> Enum.filter(&(&1 in @csv_filenames)) + + Enum.reduce_while(requested, {:ok, []}, fn filename, acc -> + case create_task(site, filename, params, debug_metadata) do + {:ok, task} -> {:cont, {:ok, elem(acc, 1) ++ [{filename, task}]}} + error_tuple -> {:halt, error_tuple} + end + end) + end + + defp create_task(site, "custom_props.csv", params, debug_metadata) do + with {:ok, prop_keys_query} <- get_custom_prop_key_query(site, params, debug_metadata), + prop_keys = get_custom_prop_keys(site, prop_keys_query), + {:ok, prop_value_queries_by_prop_key} <- + get_custom_prop_queries(site, params, debug_metadata, prop_keys) do + {:ok, fn -> get_custom_props_csv(site, prop_value_queries_by_prop_key) end} + end + end + + defp create_task(site, filename, params, debug_metadata) do + with {:ok, query} <- parse_csv_query(site, filename, params, debug_metadata) do + {:ok, fn -> run_query_into_csv(site, filename, query) end} + end + end + + defp parse_csv_query(site, filename, params, debug_metadata) do + params = construct_raw_query_params(filename, params) + + with {:ok, %ParsedQueryParams{} = params} <- QueryParser.parse(params) do + QueryBuilder.build(site, params, debug_metadata) + end + end + + defp construct_raw_query_params(filename, params) do + report_params = params["reports"][filename] + + filters = + case report_params["always_on_filters"] do + always_on when is_list(always_on) -> params["filters"] ++ always_on + _ -> params["filters"] + end + + %{ + "dimensions" => report_params["dimensions"], + "metrics" => report_params["metrics"], + "date_range" => params["date_range"], + "relative_date" => params["relative_date"], + "filters" => filters, + "include" => params["include"] + } + end + + defp run_query_into_csv(site, "visitors.csv", query) do + query = + query + |> Query.set(order_by: [{Time.time_dimension(query), :asc}]) + |> Query.set_include(:time_labels, true) + |> Query.set_include(:empty_metrics, true) + + QueryRunner.run(site, query) |> timeseries_query_result_to_csv() + end + + defp run_query_into_csv(site, filename, query) do + # The export is limited to 300 entries for other reports and 100 entries for + # pages because bigger result sets start causing failures. Since we request + # data like time on page or bounce_rate for pages in a separate query using + # the IN filter, it causes the requests to balloon in payload size. + limit = if filename in ["pages.csv", "exit_pages.csv"], do: 100, else: 300 + + query = + query + |> Query.set(order_by: get_order_by(query.dimensions)) + |> Query.set(pagination: %{limit: limit, offset: 0}) + + QueryRunner.run(site, query) |> query_result_to_csv() + end + + defp query_result_to_csv(%QueryResult{results: results, query: query}) do + results + |> Enum.reduce([csv_first_row(query)], fn row, acc -> + acc ++ [csv_dimension_values(query[:dimensions], row) ++ row.metrics] + end) + |> NimbleCSV.RFC4180.dump_to_iodata() + end + + defp timeseries_query_result_to_csv(%QueryResult{results: results, query: query, meta: meta}) do + meta[:time_labels] + |> Enum.reduce([csv_first_row(query)], fn timelabel, acc -> + if result_item = find_result_item_by_timelabel(results, timelabel) do + acc ++ [[timelabel | result_item.metrics]] + else + acc ++ [[timelabel | meta[:empty_metrics]]] + end + end) + |> NimbleCSV.RFC4180.dump_to_iodata() + end + + defp find_result_item_by_timelabel(results, timelabel) do + results + |> Map.new(fn entry -> {Enum.at(entry.dimensions, 0), entry} end) + |> Map.get(timelabel) + end + + defp get_order_by(["event:goal"]), do: [{:visitors, :desc}] + + defp get_order_by(dimensions) do + [{:visitors, :desc} | Enum.map(dimensions, &{&1, :asc})] + end + + defp csv_first_row(query) do + csv_dimension_labels(query[:dimensions]) ++ csv_metric_labels(query) + end + + defp csv_dimension_labels(["time:" <> _]) do + [:date] + end + + defp csv_dimension_labels(["visit:browser_version", "visit:browser"]) do + [:name, :version] + end + + defp csv_dimension_labels(["visit:os_version", "visit:os"]) do + [:name, :version] + end + + defp csv_dimension_labels([_dimension]) do + [:name] + end + + defp get_goal_filtered_metric_label_overrides(query) do + case query[:dimensions] do + ["time:" <> _] -> + %{ + visitors: :unique_conversions, + events: :total_conversions, + group_conversion_rate: :conversion_rate + } + + ["event:goal"] -> + %{ + visitors: :unique_conversions, + events: :total_conversions + } + + _ -> + %{ + visitors: :conversions, + group_conversion_rate: :conversion_rate + } + end + end + + @dimension_specific_metric_label_overrides %{ + ["visit:entry_page"] => %{ + visitors: :unique_entrances, + visits: :total_entrances + }, + ["visit:exit_page"] => %{ + visitors: :unique_exits, + visits: :total_exits + }, + ["event:goal"] => %{ + visitors: :unique_conversions, + events: :total_conversions + } + } + + defp csv_metric_labels(query) do + goal_filter? = + Enum.any?(query[:filters], fn [_operator, key | _rest] -> + key == "event:goal" + end) + + metric_label_overrides = + if goal_filter? do + get_goal_filtered_metric_label_overrides(query) + else + Map.get(@dimension_specific_metric_label_overrides, query[:dimensions], %{}) + end + + Enum.map(query[:metrics], &(metric_label_overrides[&1] || &1)) + end + + defp csv_dimension_values(["visit:browser_version", "visit:browser"], row) do + row[:dimensions] |> Enum.reverse() + end + + defp csv_dimension_values(["visit:os_version", "visit:os"], row) do + row[:dimensions] |> Enum.reverse() + end + + defp csv_dimension_values([_dimension], row) do + row[:dimensions] + end + + defp get_custom_prop_key_query(site, params, debug_metadata) do + raw_params = + construct_raw_query_params("custom_props.csv", params) + |> Map.merge(%{ + "dimensions" => ["event:prop_key"], + "metrics" => ["visitors"], + "pagination" => %{"limit" => 25} + }) + + with {:ok, parsed} <- QueryParser.parse(raw_params), + {:ok, query} <- QueryBuilder.build(site, parsed, debug_metadata) do + {:ok, Query.set(query, order_by: [{:visitors, :desc}, {"event:prop_key", :asc}])} + end + end + + defp get_custom_prop_keys(site, query) do + case Plausible.Stats.Filters.get_toplevel_filter(query, "event:props:") do + [_op, "event:props:" <> key | _rest] -> + [key] + + _ -> + %QueryResult{results: results} = QueryRunner.run(site, query) + + Enum.map(results, fn r -> hd(r.dimensions) end) + |> maybe_allowed_props_only(site) + end + end + + defp get_custom_prop_queries(site, params, debug_metadata, prop_keys) do + Enum.reduce_while(prop_keys, {:ok, []}, fn prop_key, acc -> + dimension = "event:props:#{prop_key}" + + raw_params = + construct_raw_query_params("custom_props.csv", params) + |> Map.put("dimensions", [dimension]) + + with {:ok, parsed} <- QueryParser.parse(raw_params), + {:ok, query} <- QueryBuilder.build(site, parsed, debug_metadata) do + query = + query + |> Query.set(order_by: [{:visitors, :desc}, {dimension, :asc}]) + |> Query.set(pagination: %{limit: 300, offset: 0}) + + {:cont, {:ok, elem(acc, 1) ++ [{prop_key, query}]}} + else + error_tuple -> {:halt, error_tuple} + end + end) + end + + defp get_custom_props_csv(_site, []), do: nil + + defp get_custom_props_csv(site, prop_value_queries_by_prop_key) do + [{_prop_key, %Query{metrics: metrics}} | _] = prop_value_queries_by_prop_key + header_row = format_custom_props_header_row(metrics) + data_rows = custom_prop_queries_into_data_rows(site, prop_value_queries_by_prop_key) + + NimbleCSV.RFC4180.dump_to_iodata([header_row] ++ data_rows) + end + + defp custom_prop_queries_into_data_rows(site, prop_value_queries_by_prop_key) do + prop_value_queries_by_prop_key + |> Enum.map(fn {prop_key, query} -> + fn -> + %QueryResult{results: results} = QueryRunner.run(site, query) + Enum.map(results, &format_custom_props_data_row(prop_key, &1)) + end + end) + |> Plausible.ClickhouseRepo.parallel_tasks() + |> Enum.concat() + end + + defp format_custom_props_header_row(metrics), do: [:property, :value] ++ metrics + + defp format_custom_props_data_row(prop_key, row), + do: [prop_key | row.dimensions] ++ row.metrics + + defp maybe_allowed_props_only(prop_keys, site) do + case Plausible.Props.allowed_for(site) do + :all -> prop_keys + allowed -> Enum.filter(prop_keys, &(&1 in allowed)) + end + end +end diff --git a/lib/plausible/stats/dashboard/query_parser.ex b/lib/plausible/stats/dashboard/query_parser.ex new file mode 100644 index 0000000000000000000000000000000000000000..ebf4ca4591bae7d59d48b28bfe9fdfb943478568 --- /dev/null +++ b/lib/plausible/stats/dashboard/query_parser.ex @@ -0,0 +1,127 @@ +defmodule Plausible.Stats.Dashboard.QueryParser do + @moduledoc """ + Parses a dashboard query string into `%ParsedQueryParams{}`. Note that + `metrics` and `dimensions` do not exist at this step yet, and are expected + to be filled in by each specific report. + """ + + alias Plausible.Stats.{ParsedQueryParams, QueryInclude, ApiQueryParser, QueryError} + + @valid_comparison_shorthands %{ + "previous_period" => :previous_period, + "year_over_year" => :year_over_year + } + + @valid_comparison_shorthand_keys Map.keys(@valid_comparison_shorthands) + + def parse(params, opts \\ []) do + with {:ok, input_date_range} <- parse_input_date_range(params), + {:ok, relative_date} <- parse_relative_date(params), + {:ok, dimensions} <- ApiQueryParser.parse_dimensions(params["dimensions"]), + {:ok, filters} <- ApiQueryParser.parse_filters(params["filters"]), + {:ok, metrics} <- parse_metrics(params), + {:ok, order_by} <- ApiQueryParser.parse_order_by(params["order_by"]), + {:ok, pagination} <- ApiQueryParser.parse_pagination(params["pagination"]), + {:ok, include} <- parse_include(params) do + {:ok, + ParsedQueryParams.new!(%{ + input_date_range: input_date_range, + relative_date: relative_date, + dimensions: dimensions, + filters: filters, + metrics: metrics, + order_by: order_by, + pagination: pagination, + include: include, + skip_goal_existence_check: true, + now: Keyword.get(opts, :now) + })} + end + end + + defp parse_input_date_range(%{"date_range" => date_range}) do + case date_range do + "realtime" -> {:ok, :realtime} + "realtime_30m" -> {:ok, :realtime_30m} + date_range -> ApiQueryParser.parse_input_date_range(date_range) + end + end + + defp parse_input_date_range(_) do + {:error, + %QueryError{code: :invalid_date_range, message: "Required 'date_range' parameter missing"}} + end + + defp parse_relative_date(%{"relative_date" => date}) when is_binary(date) do + case Date.from_iso8601(date) do + {:ok, date} -> + {:ok, date} + + _ -> + {:error, + %QueryError{code: :invalid_relative_date, message: "Failed to convert '#{date}' to date"}} + end + end + + defp parse_relative_date(_), do: {:ok, nil} + + defp parse_metrics(%{"metrics" => [_ | _] = metrics}) do + ApiQueryParser.parse_metrics(metrics) + end + + defp parse_metrics(_), + do: + {:error, + %QueryError{ + code: :invalid_metrics, + message: "Expected at least one valid metric in 'metrics'" + }} + + defp parse_include(params) do + with {:ok, compare} <- parse_include_compare(params["include"]) do + {:ok, + %QueryInclude{ + imports: params["include"]["imports"] == true, + imports_meta: params["include"]["imports_meta"] == true, + compare: compare, + compare_match_day_of_week: params["include"]["compare_match_day_of_week"] == true, + time_labels: params["include"]["time_labels"] == true, + partial_time_labels: params["include"]["partial_time_labels"] == true, + present_index: params["include"]["present_index"] == true, + empty_metrics: params["include"]["empty_metrics"] == true, + trim_relative_date_range: true, + drop_unavailable_time_on_page: true, + drop_unavailable_revenue_metrics: true + }} + end + end + + defp parse_include_compare(%{"compare" => compare}) + when compare in @valid_comparison_shorthand_keys do + {:ok, @valid_comparison_shorthands[compare]} + end + + defp parse_include_compare(%{"compare" => [from, to] = compare}) + when is_binary(from) and is_binary(to) do + case ApiQueryParser.parse_date_strings(from, to) do + {:ok, compare} -> + {:ok, compare} + + {:error, _} -> + {:error, + %QueryError{ + code: :invalid_include, + message: "Invalid include.compare '#{inspect(compare)}'" + }} + end + end + + defp parse_include_compare(%{"compare" => compare}) when not is_nil(compare) do + {:error, + %QueryError{code: :invalid_include, message: "Invalid include.compare '#{inspect(compare)}'"}} + end + + defp parse_include_compare(_) do + {:ok, nil} + end +end diff --git a/lib/plausible/stats/datetime_range.ex b/lib/plausible/stats/datetime_range.ex new file mode 100644 index 0000000000000000000000000000000000000000..6cee97fa1c0b81738d6cee531a8bfb44b45d5964 --- /dev/null +++ b/lib/plausible/stats/datetime_range.ex @@ -0,0 +1,79 @@ +defmodule Plausible.Stats.DateTimeRange do + @moduledoc """ + Defines a struct similar `Date.Range`, but with `DateTime` instead of `Date`. + + The structs should be created with the `new!/2` function. + """ + + @enforce_keys [:first, :last] + defstruct [:first, :last] + + @type t() :: %__MODULE__{ + first: %DateTime{}, + last: %DateTime{} + } + + @doc """ + Creates a `DateTimeRange` struct from the given `%Date{}` structs. + + The first datetime will become the first date at 00:00:00, and the last datetime + will become the last date at 23:59:59. Both dates will be turned into `%DateTime{}` + structs in the given timezone. + """ + def new!(%Date{} = first, last, timezone) do + first = + case DateTime.new(first, ~T[00:00:00], timezone) do + {:ok, datetime} -> datetime + {:gap, _just_before, just_after} -> just_after + {:ambiguous, _first_datetime, second_datetime} -> second_datetime + end + + new!(first, last, timezone) + end + + def new!(first, %Date{} = last, timezone) do + last = + case DateTime.new(last, ~T[23:59:59], timezone) do + {:ok, datetime} -> datetime + {:gap, just_before, _just_after} -> just_before + {:ambiguous, first_datetime, _second_datetime} -> first_datetime + end + + new!(first, last, timezone) + end + + def new!(%DateTime{} = first, %DateTime{} = last, _timezone) do + new!(first, last) + end + + def new!(%DateTime{} = first, %DateTime{} = last) do + first = DateTime.truncate(first, :second) + last = DateTime.truncate(last, :second) + + if DateTime.before?(first, last) do + %__MODULE__{first: first, last: last} + else + %__MODULE__{first: last, last: first} + end + end + + def to_timezone(%__MODULE__{first: first, last: last}, timezone) do + first = DateTime.shift_zone!(first, timezone) + last = DateTime.shift_zone!(last, timezone) + + %__MODULE__{first: first, last: last} + end + + def to_date_range(datetime_range, timezone) do + %__MODULE__{first: first, last: last} = to_timezone(datetime_range, timezone) + + first = DateTime.to_date(first) + last = DateTime.to_date(last) + + Date.range(first, last) + end + + def length(%__MODULE__{first: first, last: last}, unit) do + DateTime.diff(last, first, unit) + end +end diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex new file mode 100644 index 0000000000000000000000000000000000000000..e890c7f36bdc3b6d123c28e3a23564a2c524e4c2 --- /dev/null +++ b/lib/plausible/stats/filter_suggestions.ex @@ -0,0 +1,305 @@ +defmodule Plausible.Stats.FilterSuggestions do + use Plausible.Repo + use Plausible.ClickhouseRepo + use Plausible.Stats.SQL.Fragments + + import Plausible.Stats.Base + import Ecto.Query + + alias Plausible.Stats.Query + alias Plausible.Stats.Imported + + def filter_suggestions(site, query, "country", filter_search) do + matches = Location.search_country(filter_search) + + q = + from( + e in query_sessions(query), + group_by: e.country_code, + order_by: [desc: fragment("count(*)")], + select: e.country_code + ) + |> Imported.merge_imported_country_suggestions(site, query) + + ClickhouseRepo.all(q) + |> Enum.map(fn c -> Enum.find(matches, fn x -> x.alpha_2 == c end) end) + |> Enum.filter(& &1) + |> Enum.slice(0..24) + |> Enum.map(fn match -> + %{ + value: match.alpha_2, + label: match.name + } + end) + end + + def filter_suggestions(site, query, "region", "") do + from( + e in query_sessions(query), + group_by: e.subdivision1_code, + order_by: [desc: fragment("count(*)")], + select: e.subdivision1_code, + where: e.subdivision1_code != "" + ) + |> Imported.merge_imported_region_suggestions(site, query) + |> limit(24) + |> ClickhouseRepo.all() + |> Enum.map(fn c -> + subdiv = Location.get_subdivision(c) + + if subdiv do + %{ + value: c, + label: subdiv.name + } + else + %{ + value: c, + label: c + } + end + end) + end + + def filter_suggestions(site, query, "region", filter_search) do + matches = Location.search_subdivision(filter_search) + filter_search = String.downcase(filter_search) + + q = + from( + e in query_sessions(query), + group_by: e.subdivision1_code, + order_by: [desc: fragment("count(*)")], + select: e.subdivision1_code, + where: e.subdivision1_code != "" + ) + |> Imported.merge_imported_region_suggestions(site, query) + + ClickhouseRepo.all(q) + |> Enum.map(fn c -> + match = Enum.find(matches, fn x -> x.code == c end) + + cond do + match -> + match + + String.contains?(String.downcase(c), filter_search) -> + %{ + code: c, + name: c + } + + true -> + nil + end + end) + |> Enum.filter(& &1) + |> Enum.slice(0..24) + |> Enum.map(fn subdiv -> + %{ + value: subdiv.code, + label: subdiv.name + } + end) + end + + def filter_suggestions(site, query, "city", "") do + from( + e in query_sessions(query), + group_by: e.city_geoname_id, + order_by: [desc: fragment("count(*)")], + select: e.city_geoname_id, + where: e.city_geoname_id != 0 + ) + |> Imported.merge_imported_city_suggestions(site, query) + |> limit(24) + |> ClickhouseRepo.all() + |> Enum.map(fn c -> + city = Location.get_city(c) + + %{ + value: c, + label: (city && city.name) || "N/A" + } + end) + end + + def filter_suggestions(site, query, "city", filter_search) do + filter_search = String.downcase(filter_search) + + q = + from( + e in query_sessions(query), + group_by: e.city_geoname_id, + order_by: [desc: fragment("count(*)")], + select: e.city_geoname_id, + where: e.city_geoname_id != 0 + ) + |> Imported.merge_imported_city_suggestions(site, query) + |> limit(5000) + + ClickhouseRepo.all(q) + |> Enum.map(fn c -> Location.get_city(c) end) + |> Enum.filter(fn city -> + city && String.contains?(String.downcase(city.name), filter_search) + end) + |> Enum.slice(0..24) + |> Enum.map(fn c -> %{value: c.id, label: c.name} end) + end + + def filter_suggestions(site, _query, "goal", filter_search) do + site = Plausible.Repo.preload(site, :team) + props_available? = Plausible.Billing.Feature.Props.check_availability(site.team) == :ok + + site + |> Plausible.Goals.for_site(include_goals_with_custom_props?: props_available?) + |> Enum.map(& &1.display_name) + |> Enum.filter(fn goal -> + String.contains?( + String.downcase(goal), + String.downcase(filter_search) + ) + end) + |> wrap_suggestions() + end + + def filter_suggestions(site, query, "prop_key", filter_search) do + filter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%" + + from(e in base_event_query(query), + join: meta in "meta", + hints: "ARRAY", + on: true, + as: :meta, + select: meta.key, + where: fragment("? ilike ?", meta.key, ^filter_query), + group_by: meta.key, + order_by: [desc: fragment("count(*)")], + limit: 25 + ) + |> Plausible.Stats.CustomProps.maybe_allowed_props_only(site) + |> ClickhouseRepo.all() + |> wrap_suggestions() + end + + def filter_suggestions(site, query, filter_name, filter_search) do + filter_search = if filter_search == nil, do: "", else: filter_search + + filter_query = + if Enum.member?(["entry_page", "page", "exit_page"], filter_name), + do: "%#{String.replace(filter_search, "*", "")}%", + else: "%#{filter_search}%" + + filter_name = + case filter_name do + "page" -> :pathname + "entry_page" -> :entry_page + "source" -> :referrer_source + "channel" -> :acquisition_channel + "os" -> :operating_system + "os_version" -> :operating_system_version + "screen" -> :screen_size + "exit_page" -> :exit_page + "utm_source" -> :utm_source + "utm_medium" -> :utm_medium + "utm_campaign" -> :utm_campaign + "utm_content" -> :utm_content + "utm_term" -> :utm_term + "referrer" -> :referrer + "browser" -> :browser + "browser_version" -> :browser_version + "operating_system" -> :operating_system + "operating_system_version" -> :operating_system_version + "screen_size" -> :screen_size + "hostname" -> :hostname + end + + base_q = + if filter_name in [:pathname, :hostname] do + base_event_query(query) + else + query_sessions(query) + end + + from(e in base_q, + where: fragment("? ilike ?", field(e, ^filter_name), ^filter_query), + select: field(e, ^filter_name), + group_by: ^filter_name, + order_by: [desc: fragment("count(*)")] + ) + |> apply_additional_filters(filter_name, site) + |> Imported.merge_imported_filter_suggestions( + site, + query, + filter_name, + filter_query + ) + |> limit(25) + |> ClickhouseRepo.all() + |> add_direct_suggestion(filter_name, filter_search) + |> Enum.filter(fn suggestion -> suggestion != "" end) + |> wrap_suggestions() + end + + def custom_prop_value_filter_suggestions(_site, query, prop_key, filter_search) do + filter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%" + + none_q = + from( + e in base_event_query(Query.remove_top_level_filters(query, ["event:props:#{prop_key}"])), + select: "(none)", + where: not has_key(e, :meta, ^prop_key), + limit: 1 + ) + + search_q = + from(e in base_event_query(query), + select: get_by_key(e, :meta, ^prop_key), + where: + has_key(e, :meta, ^prop_key) and + fragment( + "? ilike ?", + get_by_key(e, :meta, ^prop_key), + ^filter_query + ), + group_by: get_by_key(e, :meta, ^prop_key), + order_by: [desc: fragment("count(*)")], + limit: 25 + ) + + ClickhouseRepo.all(none_q) + |> Kernel.++(ClickhouseRepo.all(search_q)) + |> wrap_suggestions() + end + + defp apply_additional_filters(q, :hostname, site) do + case Plausible.Shields.allowed_hostname_patterns(site.domain) do + :all -> + q + + limited_to when is_list(limited_to) -> + from(e in q, + where: fragment("multiMatchAny(?, ?)", e.hostname, ^limited_to) + ) + end + end + + defp apply_additional_filters(q, _, _), do: q + + defp wrap_suggestions(list) do + list |> Enum.uniq() |> Enum.map(fn val -> %{value: val, label: val} end) + end + + @no_ref "Direct / None" + defp add_direct_suggestion(suggestions, :referrer_source, filter_search) do + if String.contains?(String.downcase(@no_ref), String.downcase(filter_search)) do + [@no_ref | suggestions] + else + suggestions + end + end + + defp add_direct_suggestion(suggestions, _filter_name, _filter_search) do + suggestions + end +end diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex new file mode 100644 index 0000000000000000000000000000000000000000..9bcfa2f545dcd131b50a53e57315bb18876074dc --- /dev/null +++ b/lib/plausible/stats/filters/filters.ex @@ -0,0 +1,218 @@ +defmodule Plausible.Stats.Filters do + @moduledoc """ + A module for parsing filters used in stat queries. + """ + + alias Plausible.Stats.Query + alias Plausible.Stats.ApiQueryParser + alias Plausible.Stats.Filters.LegacyStatsAPIFilterParser + + @visit_props [ + :source, + :channel, + :referrer, + :utm_medium, + :utm_source, + :utm_campaign, + :utm_content, + :utm_term, + :screen, + :device, + :browser, + :browser_version, + :os, + :os_version, + :country, + :region, + :city, + :country_name, + :region_name, + :city_name, + :entry_page, + :exit_page, + :entry_page_hostname, + :exit_page_hostname + ] + def visit_props(), do: @visit_props |> Enum.map(&to_string/1) + + @event_table_visit_props @visit_props -- + [ + :entry_page, + :exit_page, + :entry_page_hostname, + :exit_page_hostname + ] + def event_table_visit_props(), do: @event_table_visit_props |> Enum.map(&to_string/1) + + @event_props [:name, :page, :goal, :hostname] + + def event_props(), do: @event_props |> Enum.map(&to_string/1) + + @doc """ + Parses different filter formats. + + Depending on the format and type of the `filters` argument, returns: + + * a parsed filter list, when `filters` is a filter expression string + * the same list, when `filters` is a map + + Returns an empty list when argument type is unexpected (e.g. `nil`). + + ## Examples: + + iex> Filters.parse("visit:browser!=Chrome") + [[:is_not, "visit:browser", ["Chrome"]]] + + iex> Filters.parse(nil) + [] + """ + def parse(filters) when is_binary(filters) do + case Jason.decode(filters) do + {:ok, filters} when is_list(filters) -> parse(filters) + {:ok, _} -> [] + {:error, err} -> LegacyStatsAPIFilterParser.parse_filter_expression(err.data) + end + end + + def parse(filters) when is_list(filters) do + {:ok, parsed_filters} = ApiQueryParser.parse_filters(filters) + parsed_filters + end + + def parse(_), do: [] + + def without_prefix(dimension) do + dimension + |> String.split(":") + |> List.last() + |> String.to_existing_atom() + end + + def dimensions_used_in_filters(filters, opts \\ []) do + min_depth = Keyword.get(opts, :min_depth, 0) + max_depth = Keyword.get(opts, :max_depth, 999) + # :ignore or :only + behavioral_filter_option = Keyword.get(opts, :behavioral_filters, nil) + + filters + |> traverse( + {0, false}, + fn {depth, is_behavioral_filter}, operator -> + {depth + 1, is_behavioral_filter or operator in [:has_done, :has_not_done]} + end + ) + |> Enum.filter(fn {_filter, {depth, is_behavioral_filter}} -> + matches_behavioral_filter_option? = + case behavioral_filter_option do + :ignore -> not is_behavioral_filter + :only -> is_behavioral_filter + _ -> true + end + + depth >= min_depth and depth <= max_depth and matches_behavioral_filter_option? + end) + |> Enum.map(fn {[_operator, dimension | _rest], _depth} -> dimension end) + end + + def filtering_on_dimension?(query_or_filters, dimension, opts \\ []) do + filters = + case query_or_filters do + %Query{filters: filters} -> filters + %{filters: filters} -> filters + filters when is_list(filters) -> filters + end + + dimension in dimensions_used_in_filters(filters, opts) + end + + def all_leaf_filters(filters) do + filters + |> traverse(nil, fn _, _ -> nil end) + |> Enum.map(fn {filter, _} -> filter end) + end + + @doc """ + Gets the first top level filter with matching dimension (or nil). + + Only use in cases where it's known that filters are only set on the top level as it + does not handle AND/OR/NOT! + """ + def get_toplevel_filter(query, prefix) do + Enum.find(query.filters, fn [_op, dimension | _rest] -> + is_binary(dimension) and String.starts_with?(dimension, prefix) + end) + end + + def rename_dimensions_used_in_filter(filters, renames) do + transform_filters(filters, fn + [operation, dimension | rest] -> + [[operation, Map.get(renames, dimension, dimension) | rest]] + + _subtree -> + nil + end) + end + + @doc """ + Updates filters via `transformer`. + + Transformer will receive each node (filter, and/or/not subtree) of + query and must return a list of nodes to replace it with or nil + to ignore and look deeper. + + ## Examples + iex> Filters.transform_filters([[:is, "visit:os", ["Linux"]], [:and, [[:is, "segment", [1]], [:is, "segment", [2]]]]], fn + ...> [_, "segment", _] -> [[:is, "segment", ["changed"]]] + ...> _ -> nil + ...> end) + [[:is, "visit:os", ["Linux"]], [:and, [[:is, "segment", ["changed"]], [:is, "segment", ["changed"]]]]] + """ + def transform_filters(filters, transformer) do + filters + |> Enum.flat_map(&transform_tree(&1, transformer)) + end + + defp transform_tree(filter, transformer) do + case {transformer.(filter), filter} do + # Transformer did not return that value - transform that subtree + {nil, [operator, child_filter]} + when operator in [:not, :ignore_in_totals_query, :has_done, :has_not_done] -> + [transformed_child] = transform_tree(child_filter, transformer) + [[operator, transformed_child]] + + {nil, [operator, filters]} when operator in [:and, :or] -> + [[operator, transform_filters(filters, transformer)]] + + # Reached a leaf node, return existing value + {nil, filter} -> + [filter] + + # Transformer returned a value - don't transform that subtree + {transformed_filters, _filter} -> + transformed_filters + end + end + + @doc """ + Traverses a filter tree while accumulating state. + """ + def traverse(filters, state \\ nil, state_transformer \\ fn state, _ -> state end) do + filters + |> Enum.flat_map(&traverse_tree(&1, state, state_transformer)) + end + + defp traverse_tree(filter, state, state_transformer) do + case filter do + [operation, child_filter] + when operation in [:not, :ignore_in_totals_query, :has_done, :has_not_done] -> + traverse_tree(child_filter, state_transformer.(state, operation), state_transformer) + + [operation, filters] when operation in [:and, :or] -> + traverse(filters, state_transformer.(state, operation), state_transformer) + + # Leaf node + _ -> + [{filter, state}] + end + end +end diff --git a/lib/plausible/stats/filters/legacy_stats_api_filter_parser.ex b/lib/plausible/stats/filters/legacy_stats_api_filter_parser.ex new file mode 100644 index 0000000000000000000000000000000000000000..189553aeaf35ad49f7226b9aa0c51a93f8e73cf8 --- /dev/null +++ b/lib/plausible/stats/filters/legacy_stats_api_filter_parser.ex @@ -0,0 +1,93 @@ +defmodule Plausible.Stats.Filters.LegacyStatsAPIFilterParser do + @moduledoc """ + Parser for legacy filter format used in Stats API v1. + """ + + @non_escaped_pipe_regex ~r/(? Enum.reject(fn parsed -> parsed == :error end) + end + + defp parse_single_filter(str) do + case to_kv(str) do + ["event:goal" = key, raw_value] -> + parse_goal_filter(key, raw_value) + + [key, raw_value] -> + is_negated? = String.contains?(str, "!=") + is_list? = list_expression?(raw_value) + is_wildcard? = wildcard_expression?(raw_value) + + final_value = remove_escape_chars(raw_value) + + cond do + is_wildcard? && is_negated? -> [:matches_wildcard_not, key, [raw_value]] + is_wildcard? -> [:matches_wildcard, key, [raw_value]] + is_list? -> [:is, key, parse_member_list(raw_value)] + is_negated? -> [:is_not, key, [final_value]] + true -> [:is, key, [final_value]] + end + |> reject_invalid_country_codes() + + _ -> + :error + end + end + + defp reject_invalid_country_codes([_op, "visit:country", code_or_codes | _rest] = filter) do + code_or_codes + |> List.wrap() + |> Enum.reduce_while(filter, fn + value, _ when byte_size(value) == 2 -> {:cont, filter} + _, _ -> {:halt, :error} + end) + end + + defp reject_invalid_country_codes(filter), do: filter + + defp to_kv(str) do + str + |> String.trim() + |> String.split(["==", "!="], trim: true) + |> Enum.map(&String.trim/1) + end + + defp parse_goal_filter(key, value) do + is_list? = list_expression?(value) + + value = + if is_list? do + parse_member_list(value) + else + remove_escape_chars(value) + end + + [:is, key, List.wrap(value)] + end + + defp list_expression?(expression) do + Regex.match?(@non_escaped_pipe_regex, expression) + end + + defp wildcard_expression?(expression) do + String.contains?(expression, "*") + end + + defp parse_member_list(raw_value) do + raw_value + |> String.split(@non_escaped_pipe_regex) + |> Enum.map(&remove_escape_chars/1) + end + + defp remove_escape_chars(value) do + String.replace(value, "\\|", "|") + end +end diff --git a/lib/plausible/stats/filters/utils.ex b/lib/plausible/stats/filters/utils.ex new file mode 100644 index 0000000000000000000000000000000000000000..48dd501a870ca0fb66a2824064a725b6068c74f3 --- /dev/null +++ b/lib/plausible/stats/filters/utils.ex @@ -0,0 +1,14 @@ +defmodule Plausible.Stats.Filters.Utils do + @moduledoc false + + def page_regex(expr) do + escaped = + expr + |> Regex.escape() + |> String.replace("\\|", "|") + |> String.replace("\\*\\*", ".*") + |> String.replace("\\*", ".*") + + "^#{escaped}$" + end +end diff --git a/lib/plausible/stats/goal_suggestions.ex b/lib/plausible/stats/goal_suggestions.ex new file mode 100644 index 0000000000000000000000000000000000000000..397b6f78a736994021758f2020aa52c8b411b124 --- /dev/null +++ b/lib/plausible/stats/goal_suggestions.ex @@ -0,0 +1,187 @@ +defmodule Plausible.Stats.GoalSuggestions do + @moduledoc false + + use Plausible.Stats.SQL.Fragments + + alias Plausible.{Repo, ClickhouseRepo} + alias Plausible.Stats.{Query, QueryBuilder} + import Plausible.Stats.Base + import Ecto.Query + + # As excluded goal names are interpolated as separate + # parameters in the query, there's a risk of running + # against max parameters limit. Given the failure mode + # in this case is suggesting an event that is already + # added as a goal, which will be validated when creating, + # it's safe to trim exclusions list. + @max_excluded 1000 + + defmacrop visitors(e) do + quote do + selected_as( + fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", unquote(e).user_id), + :visitors + ) + end + end + + def suggest_event_names(site, search_input, opts \\ []) do + matches = "%#{search_input}%" + + site = Repo.preload(site, goals: Plausible.Goals.for_site_query()) + + excluded = + opts + |> Keyword.get(:exclude, []) + |> Enum.take(@max_excluded) + + limit = Keyword.get(opts, :limit, 25) + + to_date = Date.utc_today() + from_date = Date.shift(to_date, month: -6) + + query = + QueryBuilder.build!(site, + input_date_range: {:date_range, from_date, to_date}, + metrics: [:pageviews], + include: [imports: true] + ) + + native_q = + from(e in base_event_query(query), + where: fragment("? ilike ?", e.name, ^matches), + where: e.name not in ["pageview", "engagement"], + where: fragment("trim(?)", e.name) != "", + where: e.name == fragment("trim(?)", e.name), + where: e.name not in ^excluded, + select: %{ + name: e.name, + visitors: visitors(e) + }, + order_by: selected_as(:visitors), + group_by: e.name + ) + |> maybe_set_limit(limit) + + date_range = Query.date_range(query) + + imported_q = + from(i in "imported_custom_events", + where: i.site_id == ^site.id, + where: i.import_id in ^Plausible.Imported.complete_import_ids(site), + where: i.date >= ^date_range.first and i.date <= ^date_range.last, + where: i.visitors > 0, + where: fragment("? ilike ?", i.name, ^matches), + where: fragment("trim(?)", i.name) != "", + where: i.name == fragment("trim(?)", i.name), + where: i.name not in ^excluded, + select: %{ + name: i.name, + visitors: selected_as(sum(i.visitors), :visitors) + }, + order_by: selected_as(:visitors), + group_by: i.name + ) + |> maybe_set_limit(limit) + + from(e in Ecto.Query.subquery(native_q), + full_join: i in subquery(imported_q), + on: e.name == i.name, + select: selected_as(fragment("if(empty(?), ?, ?)", e.name, i.name, e.name), :name), + order_by: [desc: e.visitors + i.visitors] + ) + |> maybe_set_limit(limit) + |> ClickhouseRepo.all() + |> Enum.reject(&(String.length(&1) > Plausible.Goal.max_event_name_length())) + end + + def suggest_custom_property_names(site, search_input, _opts \\ []) do + filter_query = if search_input == nil, do: "%", else: "%#{search_input}%" + + query = custom_props_query_30d(site) + + search_q = + from(e in base_event_query(query), + join: meta in "meta", + hints: "ARRAY", + on: true, + as: :meta, + select: meta.key, + where: fragment("? ilike ?", meta.key, ^filter_query), + group_by: meta.key, + order_by: [desc: fragment("count(*)")], + limit: 25 + ) + + event_prop_names = ClickhouseRepo.all(search_q) + + allowed_props = site.allowed_event_props || [] + + allowed_prop_names = + if search_input == nil or search_input == "" do + allowed_props + else + search_lower = String.downcase(search_input) + + Enum.filter(allowed_props, fn prop -> + String.contains?(String.downcase(prop), search_lower) + end) + end + + # Combine results, prioritizing event_prop_names (they have usage data), + # then append allowed_prop_names that aren't already in event_prop_names + event_prop_set = MapSet.new(event_prop_names) + + allowed_only = + allowed_prop_names + |> Enum.reject(&MapSet.member?(event_prop_set, &1)) + + event_prop_names ++ Enum.sort(allowed_only) + end + + def suggest_custom_property_values(site, prop_key, search_input) do + filter_query = if search_input == nil, do: "%", else: "%#{search_input}%" + + query = custom_props_query_30d(site) + + search_q = + from(e in base_event_query(query), + select: get_by_key(e, :meta, ^prop_key), + where: + has_key(e, :meta, ^prop_key) and + fragment( + "? ilike ?", + get_by_key(e, :meta, ^prop_key), + ^filter_query + ), + group_by: get_by_key(e, :meta, ^prop_key), + order_by: [desc: fragment("count(*)")], + limit: 25 + ) + + ClickhouseRepo.all(search_q) + end + + defp custom_props_query_30d(site) do + Plausible.Stats.Query.parse_and_build!( + site, + %{ + "site_id" => site.domain, + "date_range" => [ + Date.to_iso8601(Date.shift(Date.utc_today(), day: -30)), + Date.to_iso8601(Date.utc_today()) + ], + "metrics" => ["pageviews"], + "include" => %{"imports" => true} + } + ) + end + + defp maybe_set_limit(q, :unlimited) do + q + end + + defp maybe_set_limit(q, limit) when is_integer(limit) and limit > 0 do + limit(q, ^limit) + end +end diff --git a/lib/plausible/stats/goals.ex b/lib/plausible/stats/goals.ex new file mode 100644 index 0000000000000000000000000000000000000000..806635f3509e7e4a4b7bd257fc06ef58ff2b10fa --- /dev/null +++ b/lib/plausible/stats/goals.ex @@ -0,0 +1,277 @@ +defmodule Plausible.Stats.Goals do + @moduledoc """ + Stats code related to filtering and grouping by goals. + """ + + import Ecto.Query + import Plausible.Stats.Filters.Utils, only: [page_regex: 1] + + alias Plausible.Stats.Filters + + @doc """ + Preloads goals data if needed for query-building and related work. + """ + def preload_needed_goals(site, dimensions, filters) do + if Enum.member?(dimensions, "event:goal") or + Filters.filtering_on_dimension?(filters, "event:goal") do + site = Plausible.Repo.preload(site, :team) + props_available? = Plausible.Billing.Feature.Props.check_availability(site.team) == :ok + goals = Plausible.Goals.for_site(site, include_goals_with_custom_props?: props_available?) + + %{ + # When grouping by event:goal, later pipeline needs to know which goals match filters exactly. + # This can affect both calculations whether all goals have the same revenue currency and + # whether we should skip imports. + matching_toplevel_filters: goals_matching_toplevel_filters(goals, filters), + all: goals + } + else + %{ + all: [], + matching_toplevel_filters: [] + } + end + end + + @doc """ + Translates an event:goal filter into SQL. Similarly to other `add_filter` clauses in + `Plausible.Stats.SQL.WhereBuilder`, returns an `Ecto.Query.dynamic` expression. + + Compared to other dimensions, filtering by goals works differently. First, we expect + all goals to be preloaded into the query - this list of goals is what actually gets + filtered by `operator` and `clauses`. + + With the resulting filtered list of goals, we build conditions for whether that goal + was completed or not, and join those conditions with a logical OR. + + ### Options + + * `imported?` - when `true`, builds conditions on the `page` db field rather than + `pathname`, and also skips the `e.name == "pageview"` check. + """ + def add_filter(query, [operation, "event:goal", clauses | _] = filter, opts \\ []) + when operation in [:is, :contains] do + imported? = Keyword.get(opts, :imported?, false) + + Enum.reduce(clauses, false, fn clause, dynamic_statement -> + condition = + query.preloaded_goals.all + |> filter_preloaded(filter, clause) + |> build_condition(imported?) + + dynamic([e], ^condition or ^dynamic_statement) + end) + end + + @type goal_join_data() :: %{ + indices: [non_neg_integer()], + types: [String.t()], + event_names_imports: [String.t()], + event_names_by_type: [String.t()], + page_regexes: [String.t()], + scroll_thresholds: [non_neg_integer()], + custom_props_keys: [[String.t()]], + custom_props_values: [[String.t()]] + } + + @doc """ + Returns data needed to perform a GROUP BY on goals in an ecto query. + """ + @spec goal_join_data(Plausible.Stats.Query.t()) :: goal_join_data() + def goal_join_data(query) do + goals = query.preloaded_goals.matching_toplevel_filters + + goals + |> Enum.with_index(1) + |> Enum.reduce( + %{ + indices: [], + types: [], + event_names_imports: [], + event_names_by_type: [], + page_regexes: [], + scroll_thresholds: [], + custom_props_keys: [], + custom_props_values: [] + }, + fn {goal, idx}, acc -> + goal_type = Plausible.Goal.type(goal) + {prop_keys, prop_values} = Enum.unzip(goal.custom_props) + + %{ + indices: [idx | acc.indices], + types: [to_string(goal_type) | acc.types], + # This will contain "" for non-event goals + event_names_imports: [to_string(goal.event_name) | acc.event_names_imports], + event_names_by_type: [event_name_by_type(goal_type, goal) | acc.event_names_by_type], + # Event goals are considered to match everything for the sake of efficient queries in query_builder.ex + # See also Plausible.Stats.SQL.Expression.event_goal_join/1 + page_regexes: [page_regex_for_goal(goal_type, goal) | acc.page_regexes], + scroll_thresholds: [goal.scroll_threshold | acc.scroll_thresholds], + custom_props_keys: [prop_keys | acc.custom_props_keys], + custom_props_values: [prop_values | acc.custom_props_values] + } + end + ) + |> Enum.map(fn {key, list} -> {key, Enum.reverse(list)} end) + |> Map.new() + end + + defp event_name_by_type(:event, goal), do: goal.event_name + defp event_name_by_type(:page, _goal), do: "pageview" + defp event_name_by_type(:scroll, _goal), do: "engagement" + + defp page_regex_for_goal(:event, _goal), do: ".?" + defp page_regex_for_goal(:page, goal), do: Filters.Utils.page_regex(goal.page_path) + defp page_regex_for_goal(:scroll, goal), do: Filters.Utils.page_regex(goal.page_path) + + def toplevel_scroll_goal_filters?(query) do + goal_filters? = + Enum.any?(query.filters, fn + [_, "event:goal", _] -> true + _ -> false + end) + + any_scroll_goals_preloaded? = + query.preloaded_goals.matching_toplevel_filters + |> Enum.any?(fn goal -> Plausible.Goal.type(goal) == :scroll end) + + goal_filters? and any_scroll_goals_preloaded? + end + + defp filter_preloaded(goals, filter, clause) do + Enum.filter(goals, fn goal -> matches?(goal, filter, clause) end) + end + + defp goals_matching_toplevel_filters(goals, filters) do + Enum.reduce(filters, goals, fn + [_, "event:goal" | _] = filter, goals -> + goals_matching_any_clause(goals, filter) + + _filter, goals -> + goals + end) + end + + defp goals_matching_any_clause(goals, [_, _, clauses | _] = filter) do + goals + |> Enum.filter(fn goal -> + Enum.any?(clauses, fn clause -> matches?(goal, filter, clause) end) + end) + end + + defp matches?(goal, [operation | _rest] = filter, clause) do + goal_name = + goal + |> Plausible.Goal.display_name() + |> mod(filter) + + clause = mod(clause, filter) + + case operation do + :is -> + goal_name == clause + + :contains -> + String.contains?(goal_name, clause) + end + end + + defp mod(str, filter) do + case filter do + [_, _, _, %{case_sensitive: false}] -> String.downcase(str) + _ -> str + end + end + + defp build_condition(filtered_goals, imported?) do + Enum.reduce(filtered_goals, false, fn goal, dynamic_statement -> + if is_nil(goal) do + dynamic([e], ^dynamic_statement) + else + dynamic([e], ^goal_condition(goal, imported?) or ^dynamic_statement) + end + end) + end + + def goal_condition(goal, imported? \\ false) do + type = Plausible.Goal.type(goal) + goal_condition(type, goal, imported?) + end + + defp goal_condition(:event, goal, _) do + name_condition = dynamic([e], e.name == ^goal.event_name) + + if Plausible.Goal.has_custom_props?(goal) do + custom_props_condition = build_custom_props_condition(goal.custom_props) + dynamic([e], ^name_condition and ^custom_props_condition) + else + name_condition + end + end + + defp goal_condition(:scroll, goal, false = _imported?) do + pathname_condition = page_path_condition(goal.page_path, _imported? = false) + name_condition = dynamic([e], e.name == "engagement") + + scroll_condition = + dynamic([e], e.scroll_depth <= 100 and e.scroll_depth >= ^goal.scroll_threshold) + + base_condition = dynamic([e], ^pathname_condition and ^name_condition and ^scroll_condition) + + if Plausible.Goal.has_custom_props?(goal) do + custom_props_condition = build_custom_props_condition(goal.custom_props) + dynamic([e], ^base_condition and ^custom_props_condition) + else + base_condition + end + end + + defp goal_condition(:page, goal, true = _imported?) do + page_path_condition(goal.page_path, _imported? = true) + end + + defp goal_condition(:page, goal, false = _imported?) do + name_condition = dynamic([e], e.name == "pageview") + pathname_condition = page_path_condition(goal.page_path, _imported? = false) + base_condition = dynamic([e], ^pathname_condition and ^name_condition) + + if Plausible.Goal.has_custom_props?(goal) do + custom_props_condition = build_custom_props_condition(goal.custom_props) + dynamic([e], ^base_condition and ^custom_props_condition) + else + base_condition + end + end + + defp page_path_condition(page_path, imported?) do + db_field = page_path_db_field(imported?) + + if String.contains?(page_path, "*") do + dynamic([e], fragment("match(?, ?)", field(e, ^db_field), ^page_regex(page_path))) + else + dynamic([e], field(e, ^db_field) == ^page_path) + end + end + + defp build_custom_props_condition(custom_props) do + Enum.reduce(custom_props, true, fn {prop_key, prop_value}, acc -> + condition = + dynamic( + [e], + fragment( + "?[indexOf(?, ?)] = ?", + field(e, :"meta.value"), + field(e, :"meta.key"), + ^prop_key, + ^prop_value + ) + ) + + dynamic([e], ^acc and ^condition) + end) + end + + def page_path_db_field(true = _imported?), do: :page + def page_path_db_field(false = _imported?), do: :pathname +end diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex new file mode 100644 index 0000000000000000000000000000000000000000..1580b9e42df3b6121ac499120ededa5eb5ddd98f --- /dev/null +++ b/lib/plausible/stats/imported/base.ex @@ -0,0 +1,209 @@ +defmodule Plausible.Stats.Imported.Base do + @moduledoc """ + A module for building the base of an imported stats query + """ + + import Ecto.Query + + alias Plausible.Imported + alias Plausible.Stats.Query + + import Plausible.Stats.Filters, + only: [dimensions_used_in_filters: 1, dimensions_used_in_filters: 2] + + @property_to_table_mappings %{ + "visit:source" => "imported_sources", + "visit:channel" => "imported_sources", + "visit:referrer" => "imported_sources", + "visit:utm_source" => "imported_sources", + "visit:utm_medium" => "imported_sources", + "visit:utm_campaign" => "imported_sources", + "visit:utm_term" => "imported_sources", + "visit:utm_content" => "imported_sources", + "visit:entry_page" => "imported_entry_pages", + "visit:exit_page" => "imported_exit_pages", + "visit:country" => "imported_locations", + "visit:region" => "imported_locations", + "visit:city" => "imported_locations", + "visit:country_name" => "imported_locations", + "visit:region_name" => "imported_locations", + "visit:city_name" => "imported_locations", + "visit:device" => "imported_devices", + "visit:browser" => "imported_browsers", + "visit:browser_version" => "imported_browsers", + "visit:os" => "imported_operating_systems", + "visit:os_version" => "imported_operating_systems", + "event:page" => "imported_pages", + "event:name" => "imported_custom_events", + + # NOTE: these dimensions can be only filtered by + "visit:screen" => "imported_devices", + "event:hostname" => "imported_pages", + + # NOTE: These dimensions are only used in group by + "time:month" => "imported_visitors", + "time:week" => "imported_visitors", + "time:day" => "imported_visitors", + "time:hour" => "imported_visitors" + } + + @queriable_time_dimensions ["time:month", "time:week", "time:day", "time:hour"] + + @imported_custom_props Imported.imported_custom_props() + + def property_to_table_mappings(), do: @property_to_table_mappings + + def query_imported(site, query) do + [table] = decide_tables(query) + + query_imported(table, site, query) + end + + def query_imported(table, site, query) do + import_ids = Imported.complete_import_ids(site) + # Assumption: dates in imported table are in user-local timezone. + %{first: date_from, last: date_to} = Query.date_range(query) + + from(i in table, + where: i.site_id == ^site.id, + where: i.import_id in ^import_ids, + where: i.date >= ^date_from, + where: i.date <= ^date_to, + where: ^Plausible.Stats.Imported.SQL.WhereBuilder.build(query), + select: %{} + ) + end + + def decide_tables(query) do + behavioral_filters = dimensions_used_in_filters(query.filters, behavioral_filters: :only) + + cond do + # Behavioral filters cannot be emulated via aggregated imported stats + length(behavioral_filters) > 0 -> + [] + + custom_prop_query?(query) -> + do_decide_custom_prop_table(query) + + true -> + do_decide_tables(query) + end + end + + defp custom_prop_query?(query) do + dimensions_used_in_filters(query.filters) + |> Enum.concat(query.dimensions) + |> Enum.any?(&(&1 in @imported_custom_props)) + end + + defp do_decide_custom_prop_table(%{dimensions: [dimension]} = query) + when dimension in @imported_custom_props do + do_decide_custom_prop_table(query, dimension) + end + + @queriable_custom_prop_dimensions ["event:goal", "event:name"] ++ @queriable_time_dimensions + defp do_decide_custom_prop_table(%{dimensions: dimensions} = query) do + if dimensions == [] or + (length(dimensions) == 1 and hd(dimensions) in @queriable_custom_prop_dimensions) do + custom_prop_filters = + dimensions_used_in_filters(query.filters) + |> Enum.filter(&(&1 in @imported_custom_props)) + |> Enum.uniq() + + case custom_prop_filters do + [custom_prop_filter] -> + do_decide_custom_prop_table(query, custom_prop_filter) + + _ -> + [] + end + else + [] + end + end + + defp do_decide_custom_prop_table(query, property) do + "event:props:" <> prop_key = property + + has_required_event_or_goal_name_filter? = + query.filters + |> Enum.flat_map(fn + [:is, "event:name", event_names | _rest] -> event_names + [:is, "event:goal", goal_names | _rest] -> goal_names + _ -> [] + end) + |> Enum.any?(fn event_or_goal_name -> + event_or_goal_name in Plausible.Event.SystemEvents.special_events_for_prop_key(prop_key) + end) + + has_unsupported_filters? = + query.filters + |> dimensions_used_in_filters() + |> Enum.any?(&(&1 not in [property, "event:name", "event:goal"])) + + if has_required_event_or_goal_name_filter? and + not has_unsupported_filters? do + ["imported_custom_events"] + else + [] + end + end + + defp do_decide_tables(%Query{filters: [], dimensions: []}), do: ["imported_visitors"] + + defp do_decide_tables(%Query{filters: [], dimensions: ["event:goal"]}) do + ["imported_pages", "imported_custom_events"] + end + + defp do_decide_tables(%Query{dimensions: ["event:goal"]} = query) do + filter_dimensions = dimensions_used_in_filters(query.filters) + + filter_goals = query.preloaded_goals.matching_toplevel_filters + + any_event_goals? = Enum.any?(filter_goals, fn goal -> Plausible.Goal.type(goal) == :event end) + + any_pageview_goals? = + Enum.any?(filter_goals, fn goal -> Plausible.Goal.type(goal) == :page end) + + any_event_name_filters? = "event:name" in filter_dimensions or any_event_goals? + any_page_filters? = "event:page" in filter_dimensions or any_pageview_goals? + + any_other_filters? = + Enum.any?(filter_dimensions, &(&1 not in ["event:page", "event:name", "event:goal"])) + + cond do + any_other_filters? -> [] + any_event_name_filters? and not any_page_filters? -> ["imported_custom_events"] + any_page_filters? and not any_event_name_filters? -> ["imported_pages"] + true -> [] + end + end + + defp do_decide_tables(query) do + table_candidates = + dimensions_used_in_filters(query.filters) + |> Enum.concat(query.dimensions) + |> Enum.reject(&(&1 in @queriable_time_dimensions or &1 == "event:goal")) + |> Enum.flat_map(fn + "visit:screen" -> ["visit:device"] + dimension -> [dimension] + end) + |> Enum.map(&@property_to_table_mappings[&1]) + + filter_goal_table_candidates = + query.preloaded_goals.matching_toplevel_filters + |> Enum.map(&Plausible.Goal.type/1) + |> Enum.map(fn + :event -> "imported_custom_events" + :page -> "imported_pages" + :scroll -> nil + end) + + case Enum.uniq(table_candidates ++ filter_goal_table_candidates) do + [] -> ["imported_visitors"] + [nil] -> [] + [candidate] -> [candidate] + _ -> [] + end + end +end diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex new file mode 100644 index 0000000000000000000000000000000000000000..88a923e565b4bd81ccf92c7521fde7b5d3baa6a3 --- /dev/null +++ b/lib/plausible/stats/imported/imported.ex @@ -0,0 +1,380 @@ +defmodule Plausible.Stats.Imported do + @moduledoc """ + Module defining functions that merge imported query into a native one, + and also functions that decide whether imported data is eligible for + a given query. + """ + use Plausible.ClickhouseRepo + use Plausible.Stats.SQL.Fragments + + import Ecto.Query + import Plausible.Stats.Imported.SQL.Expression + + alias Plausible.Stats.Imported + alias Plausible.Stats.Query + alias Plausible.Stats.SQL.QueryBuilder + + @property_to_table_mappings Imported.Base.property_to_table_mappings() + + @doc """ + Returns a boolean indicating whether the combination of filters and + breakdown property is possible to query from the imported tables. + + Usually, when no filters are used, the imported schema supports the + query. There is one exception though - breakdown by a custom property. + We are currently importing only two custom properties - `url` and `path`. + Both these properties can only be used with their particular `event:name` + filter or the corresponding goal filter (see Plausible.Event.SystemEvents). + """ + def schema_supports_query?(query) do + length(Imported.Base.decide_tables(query)) > 0 + end + + def schema_supports_interval?(query) do + "time:minute" not in query.dimensions and + "time:hour" not in query.dimensions + end + + def merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{ + include_imported: false + }) do + native_q + end + + def merge_imported_country_suggestions(native_q, site, query) do + supports_filter_set? = + Enum.all?(query.filters, fn filter -> + [_, filtered_prop | _] = filter + @property_to_table_mappings[filtered_prop] == "imported_locations" + end) + + if supports_filter_set? do + native_q = + native_q + |> exclude(:order_by) + |> exclude(:select) + |> select([e], %{country_code: e.country_code, count: fragment("count(*)")}) + + imported_q = + from i in Imported.Base.query_imported("imported_locations", site, query), + group_by: i.country, + select_merge: %{country_code: i.country, count: fragment("sum(?)", i.pageviews)} + + from(s in subquery(native_q), + full_join: i in subquery(imported_q), + on: s.country_code == i.country_code, + select: + fragment("if(not empty(?), ?, ?)", s.country_code, s.country_code, i.country_code), + order_by: [desc: fragment("? + ?", s.count, i.count)] + ) + else + native_q + end + end + + def merge_imported_region_suggestions(native_q, _site, %Plausible.Stats.Query{ + include_imported: false + }) do + native_q + end + + def merge_imported_region_suggestions(native_q, site, query) do + supports_filter_set? = + Enum.all?(query.filters, fn filter -> + [_, filtered_prop | _] = filter + @property_to_table_mappings[filtered_prop] == "imported_locations" + end) + + if supports_filter_set? do + native_q = + native_q + |> exclude(:order_by) + |> exclude(:select) + |> select([e], %{region_code: e.subdivision1_code, count: fragment("count(*)")}) + + imported_q = + from i in Imported.Base.query_imported("imported_locations", site, query), + where: i.region != "", + group_by: i.region, + select_merge: %{region_code: i.region, count: fragment("sum(?)", i.pageviews)} + + from(s in subquery(native_q), + full_join: i in subquery(imported_q), + on: s.region_code == i.region_code, + select: fragment("if(not empty(?), ?, ?)", s.region_code, s.region_code, i.region_code), + order_by: [desc: fragment("? + ?", s.count, i.count)] + ) + else + native_q + end + end + + def merge_imported_city_suggestions(native_q, _site, %Plausible.Stats.Query{ + include_imported: false + }) do + native_q + end + + def merge_imported_city_suggestions(native_q, site, query) do + supports_filter_set? = + Enum.all?(query.filters, fn filter -> + [_, filtered_prop | _] = filter + @property_to_table_mappings[filtered_prop] == "imported_locations" + end) + + if supports_filter_set? do + native_q = + native_q + |> exclude(:order_by) + |> exclude(:select) + |> select([e], %{city_id: e.city_geoname_id, count: fragment("count(*)")}) + + imported_q = + from i in Imported.Base.query_imported("imported_locations", site, query), + where: i.city != 0, + group_by: i.city, + select_merge: %{city_id: i.city, count: fragment("sum(?)", i.pageviews)} + + from(s in subquery(native_q), + full_join: i in subquery(imported_q), + on: s.city_id == i.city_id, + select: fragment("if(? > 0, ?, ?)", s.city_id, s.city_id, i.city_id), + order_by: [desc: fragment("? + ?", s.count, i.count)] + ) + else + native_q + end + end + + def merge_imported_filter_suggestions( + native_q, + _site, + %Plausible.Stats.Query{include_imported: false}, + _filter_name, + _filter_search + ) do + native_q + end + + def merge_imported_filter_suggestions( + native_q, + site, + query, + filter_name, + filter_query + ) do + {table, db_field} = expand_suggestions_field(filter_name) + + supports_filter_set? = + Enum.all?(query.filters, fn filter -> + [_, filtered_prop | _] = filter + @property_to_table_mappings[filtered_prop] == table + end) + + if supports_filter_set? do + native_q = + native_q + |> exclude(:order_by) + |> exclude(:select) + |> select([e], %{name: field(e, ^filter_name), count: fragment("count(*)")}) + + imported_q = + from i in Imported.Base.query_imported(table, site, query), + where: fragment("? ilike ?", field(i, ^db_field), ^filter_query), + group_by: field(i, ^db_field), + select_merge: %{name: field(i, ^db_field), count: fragment("sum(?)", i.pageviews)} + + from(s in subquery(native_q), + full_join: i in subquery(imported_q), + on: s.name == i.name, + select: fragment("if(not empty(?), ?, ?)", s.name, s.name, i.name), + order_by: [desc: fragment("? + ?", s.count, i.count)], + limit: 25 + ) + else + native_q + end + end + + @filter_suggestions_mapping %{ + referrer_source: :source, + acquisition_channel: :channel, + screen_size: :device, + pathname: :page + } + + defp expand_suggestions_field(filter_name) do + db_field = Map.get(@filter_suggestions_mapping, filter_name, filter_name) + + property = + case db_field do + :operating_system -> :os + :operating_system_version -> :os_version + other -> other + end + + table_by_visit = Map.get(@property_to_table_mappings, "visit:#{property}") + table_by_event = Map.get(@property_to_table_mappings, "event:#{property}") + table = table_by_visit || table_by_event + + {table, db_field} + end + + def merge_imported(q, _, %Query{include_imported: false}), do: q + + def merge_imported(q, site, %Query{dimensions: []} = query) do + q = paginate_optimization(q, query) + + imported_q = + site + |> Imported.Base.query_imported(query) + |> select_imported_metrics(query) + |> paginate_optimization(query) + + from( + s in subquery(q), + cross_join: i in subquery(imported_q), + select: %{} + ) + |> select_joined_metrics(query) + end + + def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query) do + goal_join_data = Plausible.Stats.Goals.goal_join_data(query) + + Imported.Base.decide_tables(query) + |> Enum.map(fn + "imported_custom_events" -> + Imported.Base.query_imported("imported_custom_events", site, query) + |> where([i], i.visitors > 0) + |> select_merge_as([i], %{ + dim0: + fragment( + "indexOf(?, ?)", + type(^goal_join_data.event_names_imports, {:array, :string}), + i.name + ) + }) + |> select_imported_metrics(query) + |> group_by([], selected_as(:dim0)) + |> where([], selected_as(:dim0) != 0) + + "imported_pages" -> + Imported.Base.query_imported("imported_pages", site, query) + |> where([i], i.visitors > 0) + |> where( + [i], + fragment( + """ + notEmpty( + arrayFilter( + goal_idx -> ?[goal_idx] = 'page' AND match(?, ?[goal_idx]), + ? + ) as indices + ) + """, + type(^goal_join_data.types, {:array, :string}), + i.page, + type(^goal_join_data.page_regexes, {:array, :string}), + type(^goal_join_data.indices, {:array, :integer}) + ) + ) + |> join(:inner, [_i], index in fragment("indices"), hints: "ARRAY", on: true) + |> group_by([_i, index], index) + |> select_merge_as([_i, index], %{ + dim0: fragment("CAST(?, 'UInt64')", index) + }) + |> select_imported_metrics(query) + end) + |> Enum.reduce(q, fn imports_q, q -> + naive_dimension_join(q, imports_q, query) + end) + end + + def merge_imported(q, site, query) do + if schema_supports_query?(query) do + q = paginate_optimization(q, query) + + imported_q = + site + |> Imported.Base.query_imported(query) + |> where([i], i.visitors > 0) + |> group_imported_by(query) + |> select_imported_metrics(query) + |> paginate_optimization(query) + + from(s in subquery(q), + full_join: i in subquery(imported_q), + on: ^QueryBuilder.build_group_by_join(query), + select: %{} + ) + |> select_joined_dimensions(query) + |> select_joined_metrics(query) + else + q + end + end + + def total_imported_visitors(site, query) do + site + |> Imported.Base.query_imported(query) + |> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)}) + end + + defp naive_dimension_join(q1, q2, query) do + from(a in subquery(q1), + full_join: b in subquery(q2), + on: a.dim0 == b.dim0, + select: %{} + ) + |> select_merge_as([a, b], %{ + dim0: fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0) + }) + |> select_joined_metrics(query) + end + + # Optimization for cases when grouping by a very high cardinality column. + # + # Instead of joining all rows from main and imported tables, we limit the number of rows + # in both tables to LIMIT N * 100. + # + # This speeds up cases where a site has millions of unique pathnames, reducing the time spent + # JOINing tables by an order of magnitude. + # + # Note that this optimization is lossy as the true top N values can arise from outside the top C + # items of either subquery. In practice though, this will give plausible results. + # + # We only apply this optimization in cases where we can deterministically ORDER BY. This covers + # opening Plausible dashboard but not more complicated use-cases. + defp paginate_optimization(q, query) do + if is_map(query.pagination) and can_order_by?(query) do + n = (query.pagination.limit + query.pagination.offset) * 100 + + q + |> QueryBuilder.build_order_by(query) + |> limit(^n) + else + q + end + end + + @cannot_optimize_metrics [ + :exit_rate, + :scroll_depth, + :percentage, + :bounce_rate, + :conversion_rate, + :group_conversion_rate, + :time_on_page, + :total_revenue, + :average_revenue + ] + + defp can_order_by?(query) do + Enum.all?(query.order_by, fn + {metric, _direction} when metric in @cannot_optimize_metrics -> false + {metric, _direction} when is_atom(metric) -> metric in query.metrics + _ -> true + end) + end +end diff --git a/lib/plausible/stats/imported/sql/expression.ex b/lib/plausible/stats/imported/sql/expression.ex new file mode 100644 index 0000000000000000000000000000000000000000..f96cdbbb839654aae3c042c3e9a5140da092d6ad --- /dev/null +++ b/lib/plausible/stats/imported/sql/expression.ex @@ -0,0 +1,480 @@ +defmodule Plausible.Stats.Imported.SQL.Expression do + @moduledoc """ + This module is responsible for generating SQL/Ecto expressions + for dimensions, filters and metrics used in import table queries + """ + + use Plausible.Stats.SQL.Fragments + + import Plausible.Stats.Util, only: [shortname: 2] + import Ecto.Query + + alias Plausible.Stats.Query + + @no_ref "Direct / None" + @not_set "(not set)" + @none "(none)" + + def select_imported_metrics( + %Ecto.Query{from: %Ecto.Query.FromExpr{source: {table, _}}} = q, + query + ) do + select_clause = + query.metrics + |> Enum.map(&select_metric(&1, table, query)) + |> Enum.reduce(%{}, &Map.merge/2) + + q + |> select_merge(q, ^select_clause) + |> filter_pageviews(query.metrics, table) + end + + defp filter_pageviews(q, metrics, table) do + should_filter = :pageviews in metrics or :views_per_visit in metrics + + case {should_filter, table} do + {_, "imported_custom_events"} -> q + {true, _} -> q |> where([i], i.pageviews > 0) + {false, _} -> q + end + end + + defp select_metric(:visitors, _table, _query) do + wrap_alias([i], %{visitors: sum(i.visitors)}) + end + + defp select_metric(:events, "imported_custom_events", _query) do + wrap_alias([i], %{events: sum(i.events)}) + end + + defp select_metric(:events, _table, _query) do + wrap_alias([i], %{events: sum(i.pageviews)}) + end + + defp select_metric(:visits, "imported_exit_pages", _query) do + wrap_alias([i], %{visits: sum(i.exits)}) + end + + defp select_metric(:visits, "imported_entry_pages", _query) do + wrap_alias([i], %{visits: sum(i.entrances)}) + end + + defp select_metric(:visits, _table, _query) do + wrap_alias([i], %{visits: sum(i.visits)}) + end + + defp select_metric(:pageviews, "imported_custom_events", _query) do + wrap_alias([i], %{pageviews: 0}) + end + + defp select_metric(:pageviews, _table, _query) do + wrap_alias([i], %{pageviews: sum(i.pageviews)}) + end + + defp select_metric(:bounce_rate, "imported_pages", _query) do + wrap_alias([i], %{bounces: fragment("any(0)"), __internal_visits: fragment("any(0)")}) + end + + defp select_metric(:bounce_rate, "imported_exit_pages", _query) do + wrap_alias([i], %{bounces: sum(i.bounces), __internal_visits: sum(i.exits)}) + end + + defp select_metric(:bounce_rate, "imported_entry_pages", _query) do + wrap_alias([i], %{bounces: sum(i.bounces), __internal_visits: sum(i.entrances)}) + end + + defp select_metric(:bounce_rate, _table, _query) do + wrap_alias([i], %{bounces: sum(i.bounces), __internal_visits: sum(i.visits)}) + end + + defp select_metric(:exit_rate, "imported_exit_pages", _query) do + wrap_alias([i], %{__internal_visits: sum(i.exits)}) + end + + defp select_metric(:visit_duration, "imported_pages", _query) do + wrap_alias([i], %{visit_duration: 0, total_visit_duration: 0}) + end + + defp select_metric(:visit_duration, "imported_exit_pages", _query) do + wrap_alias([i], %{ + visit_duration: + fragment("ifNotFinite(round(? / ?), 0)", sum(i.visit_duration), sum(i.exits)), + total_visit_duration: sum(i.visit_duration), + __internal_visits: sum(i.exits) + }) + end + + defp select_metric(:visit_duration, "imported_entry_pages", _query) do + wrap_alias([i], %{ + visit_duration: + fragment("ifNotFinite(round(? / ?), 0)", sum(i.visit_duration), sum(i.entrances)), + total_visit_duration: sum(i.visit_duration), + __internal_visits: sum(i.entrances) + }) + end + + defp select_metric(:visit_duration, _table, _query) do + wrap_alias([i], %{ + visit_duration: + fragment("ifNotFinite(round(? / ?), 0)", sum(i.visit_duration), sum(i.visits)), + total_visit_duration: sum(i.visit_duration), + __internal_visits: sum(i.visits) + }) + end + + defp select_metric(:views_per_visit, "imported_exit_pages", _query) do + wrap_alias([i], %{ + views_per_visit: + fragment("ifNotFinite(round(? / ?, 2), 0)", sum(i.pageviews), sum(i.exits)), + pageviews: sum(i.pageviews), + __internal_visits: sum(i.exits) + }) + end + + defp select_metric(:views_per_visit, "imported_entry_pages", _query) do + wrap_alias([i], %{ + views_per_visit: + fragment("ifNotFinite(round(? / ?, 2), 0)", sum(i.pageviews), sum(i.entrances)), + pageviews: sum(i.pageviews), + __internal_visits: sum(i.entrances) + }) + end + + defp select_metric(:views_per_visit, _table, _query) do + wrap_alias([i], %{ + views_per_visit: + fragment("ifNotFinite(round(? / ?, 2), 0)", sum(i.pageviews), sum(i.visits)), + pageviews: sum(i.pageviews), + __internal_visits: sum(i.visits) + }) + end + + defp select_metric(:scroll_depth, "imported_pages", _query) do + wrap_alias([i], %{ + total_scroll_depth: sum(i.total_scroll_depth), + total_scroll_depth_visits: sum(i.total_scroll_depth_visits) + }) + end + + defp select_metric(:time_on_page, "imported_pages", query) do + case query.time_on_page_data do + %{include_new_metric: false} -> + wrap_alias([i], %{ + total_time_on_page: 0, + total_time_on_page_visits: 0 + }) + + %{include_new_metric: true, cutoff: nil} -> + wrap_alias([i], %{ + total_time_on_page: sum(i.total_time_on_page), + total_time_on_page_visits: sum(i.total_time_on_page_visits) + }) + + %{include_new_metric: true, cutoff: cutoff} -> + cutoff_date = cutoff |> DateTime.shift_zone!(query.timezone) |> DateTime.to_date() + + wrap_alias([i], %{ + total_time_on_page: + fragment("sumIf(?, ? >= ?)", i.total_time_on_page, i.date, ^cutoff_date), + total_time_on_page_visits: + fragment("sumIf(?, ? >= ?)", i.total_time_on_page_visits, i.date, ^cutoff_date) + }) + end + end + + defp select_metric(_metric, _table, _query), do: %{} + + def group_imported_by(q, query) do + Enum.reduce(query.dimensions, q, fn dimension, q -> + q + |> select_group_fields(dimension, shortname(query, dimension), query) + |> filter_group_values(dimension) + |> group_by([], selected_as(^shortname(query, dimension))) + end) + end + + defp select_group_fields(q, dimension, key, _query) + when dimension in ["visit:source", "visit:referrer"] do + select_merge_as(q, [i], %{ + key => + fragment( + "if(empty(?), ?, ?)", + field(i, ^dim(dimension)), + @no_ref, + field(i, ^dim(dimension)) + ) + }) + end + + defp select_group_fields(q, "event:page", key, _query) do + select_merge_as(q, [i], %{key => i.page}) + end + + defp select_group_fields(q, dimension, key, _query) + when dimension in ["visit:device", "visit:browser", "visit:channel"] do + select_merge_as(q, [i], %{ + key => + fragment( + "if(empty(?), ?, ?)", + field(i, ^dim(dimension)), + @not_set, + field(i, ^dim(dimension)) + ) + }) + end + + defp select_group_fields(q, "visit:browser_version", key, _query) do + select_merge_as(q, [i], %{ + key => fragment("if(empty(?), ?, ?)", i.browser_version, @not_set, i.browser_version) + }) + end + + defp select_group_fields(q, "visit:os", key, _query) do + select_merge_as(q, [i], %{ + key => fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system) + }) + end + + defp select_group_fields(q, "visit:os_version", key, _query) do + select_merge_as(q, [i], %{ + key => + fragment( + "if(empty(?), ?, ?)", + i.operating_system_version, + @not_set, + i.operating_system_version + ) + }) + end + + defp select_group_fields(q, "event:props:url", key, _query) do + select_merge_as(q, [i], %{ + key => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none) + }) + end + + defp select_group_fields(q, "event:props:path", key, _query) do + select_merge_as(q, [i], %{ + key => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none) + }) + end + + defp select_group_fields(q, "time:month", key, _query) do + select_merge_as(q, [i], %{key => fragment("toStartOfMonth(?)", i.date)}) + end + + defp select_group_fields(q, dimension, key, _query) + when dimension in ["time:hour", "time:day"] do + select_merge_as(q, [i], %{key => i.date}) + end + + defp select_group_fields(q, "time:week", key, query) do + date_range = Query.date_range(query) + + select_merge_as(q, [i], %{ + key => weekstart_not_before(i.date, ^date_range.first) + }) + end + + # Rows imported from Plausible store `imported_locations.region` as ISO 3166-2 code. + # Rows imported from GA4 have `imported_locations.region` set to region name + # (e.g. "California"). + # + # imported_locations.region_name is an alias column that looks up the name by the ISO code. + # Lookup by values imported from GA4 (e.g. "California"), will yield an empty string. + # + # This function ensures that in those scenarios, the region name is set to the region code. + defp select_group_fields(q, "visit:region_name", key, _query) do + select_merge_as(q, [i], %{ + key => fragment("if(empty(?), ?, ?)", i.region_name, i.region, i.region_name) + }) + end + + defp select_group_fields(q, dimension, key, _query) do + select_merge_as(q, [i], %{key => field(i, ^dim(dimension))}) + end + + @utm_dimensions [ + "visit:utm_source", + "visit:utm_medium", + "visit:utm_campaign", + "visit:utm_term", + "visit:utm_content" + ] + defp filter_group_values(q, dimension) when dimension in @utm_dimensions do + dim = Plausible.Stats.Filters.without_prefix(dimension) + + where(q, [i], fragment("not empty(?)", field(i, ^dim))) + end + + defp filter_group_values(q, "visit:country"), do: where(q, [i], i.country != "ZZ") + defp filter_group_values(q, "visit:region"), do: where(q, [i], i.region != "") + defp filter_group_values(q, "visit:city"), do: where(q, [i], i.city != 0 and not is_nil(i.city)) + + defp filter_group_values(q, "visit:country_name"), do: where(q, [i], i.country_name != "ZZ") + defp filter_group_values(q, "visit:region_name"), do: where(q, [i], i.region != "") + defp filter_group_values(q, "visit:city_name"), do: where(q, [i], i.city_name != "") + + defp filter_group_values(q, _dimension), do: q + + def select_joined_dimensions(q, query) do + Enum.reduce(query.dimensions, q, fn dimension, q -> + select_joined_dimension(q, dimension, shortname(query, dimension)) + end) + end + + defp select_joined_dimension(q, "visit:city", key) do + select_merge_as(q, [s, i], %{ + key => fragment("greatest(?,?)", field(i, ^key), field(s, ^key)) + }) + end + + defp select_joined_dimension(q, "time:" <> _, key) do + select_merge_as(q, [s, i], %{ + key => fragment("greatest(?, ?)", field(i, ^key), field(s, ^key)) + }) + end + + defp select_joined_dimension(q, _dimension, key) do + select_merge_as(q, [s, i], %{ + key => fragment("if(empty(?), ?, ?)", field(s, ^key), field(i, ^key), field(s, ^key)) + }) + end + + def select_joined_metrics(q, query) do + select_metrics = + query.metrics + |> Enum.map(&joined_metric(&1, query)) + |> Enum.reduce(%{}, &Map.merge/2) + + select_merge(q, ^select_metrics) + end + + # NOTE: Reverse-engineering the native data bounces and total visit + # durations to combine with imported data is inefficient. Instead both + # queries should fetch bounces/total_visit_duration and visits and be + # used as subqueries to a main query that then find the bounce rate/avg + # visit_duration. + + defp joined_metric(:visits, _query) do + wrap_alias([s, i], %{visits: s.visits + i.visits}) + end + + defp joined_metric(:visitors, _query) do + wrap_alias([s, i], %{visitors: s.visitors + i.visitors}) + end + + defp joined_metric(:events, _query) do + wrap_alias([s, i], %{events: s.events + i.events}) + end + + defp joined_metric(:pageviews, _query) do + wrap_alias([s, i], %{pageviews: s.pageviews + i.pageviews}) + end + + defp joined_metric(:views_per_visit, _query) do + wrap_alias([s, i], %{ + views_per_visit: + fragment( + "if(? + ? > 0, round((? + ? * ?) / (? + ?), 2), 0)", + s.__internal_visits, + i.__internal_visits, + i.pageviews, + s.views_per_visit, + s.__internal_visits, + i.__internal_visits, + s.__internal_visits + ) + }) + end + + defp joined_metric(:bounce_rate, _query) do + wrap_alias([s, i], %{ + bounce_rate: + fragment( + "if(? + ? > 0, round(100 * (? + (? * ? / 100)) / (? + ?)), 0)", + s.__internal_visits, + i.__internal_visits, + i.bounces, + s.bounce_rate, + s.__internal_visits, + i.__internal_visits, + s.__internal_visits + ) + }) + end + + defp joined_metric(:visit_duration, _query) do + wrap_alias([s, i], %{ + visit_duration: + fragment( + """ + if( + ? + ? > 0, + round((? + ? * ?) / (? + ?), 0), + 0 + ) + """, + s.__internal_visits, + i.__internal_visits, + i.total_visit_duration, + s.visit_duration, + s.__internal_visits, + s.__internal_visits, + i.__internal_visits + ) + }) + end + + # The final `scroll_depth` gets selected at a later querybuilding step + # (in `Plausible.Stats.SQL.SpecialMetrics.add/3`). But in order to avoid + # having to join with imported data there again, we select the required + # information from imported data here already. + defp joined_metric(:scroll_depth, _query) do + wrap_alias([s, i], %{ + __imported_total_scroll_depth: i.total_scroll_depth, + __imported_total_scroll_depth_visits: i.total_scroll_depth_visits + }) + end + + defp joined_metric(:time_on_page, query) do + wrap_alias([s, i], %{ + __internal_total_time_on_page: s.__internal_total_time_on_page + i.total_time_on_page, + __internal_total_time_on_page_visits: + s.__internal_total_time_on_page_visits + i.total_time_on_page_visits + }) + |> Map.merge(time_on_page_metric(query)) + end + + defp joined_metric(:exit_rate, _query) do + wrap_alias([s, i], %{ + __internal_visits: s.__internal_visits + i.__internal_visits + }) + end + + # Ignored as it's calculated separately + defp joined_metric(metric, _query) + when metric in [:conversion_rate, :group_conversion_rate, :percentage] do + %{} + end + + defp joined_metric(metric, _query) do + wrap_alias([s, i], %{metric => field(s, ^metric)}) + end + + defp dim(dimension), do: Plausible.Stats.Filters.without_prefix(dimension) + + defp time_on_page_metric(query) do + if query.time_on_page_data.include_legacy_metric do + %{} + else + wrap_alias([], %{ + time_on_page: + time_on_page( + selected_as(:__internal_total_time_on_page), + selected_as(:__internal_total_time_on_page_visits) + ) + }) + end + end +end diff --git a/lib/plausible/stats/imported/sql/where_builder.ex b/lib/plausible/stats/imported/sql/where_builder.ex new file mode 100644 index 0000000000000000000000000000000000000000..a9b8437265af1c80a3e98356814ca9b00e208c72 --- /dev/null +++ b/lib/plausible/stats/imported/sql/where_builder.ex @@ -0,0 +1,63 @@ +defmodule Plausible.Stats.Imported.SQL.WhereBuilder do + @moduledoc """ + A module for building an ecto where clause of a query out of a query for Imported tables. + """ + + import Ecto.Query + + alias Plausible.Stats.Query + + @db_field_mappings %{ + referrer_source: :source, + screen_size: :device, + screen: :device, + os: :operating_system, + os_version: :operating_system_version, + country_code: :country, + subdivision1_code: :region, + city_geoname_id: :city, + entry_page_hostname: :hostname, + pathname: :page, + url: :link_url + } + + def build(%Query{filters: []}), do: true + + def build(query) do + query.filters + |> Enum.map(&add_filter(query, &1)) + |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end) + end + + defp add_filter(query, [:ignore_in_totals_query, filter]) do + add_filter(query, filter) + end + + defp add_filter(query, [:not, filter]) do + dynamic([i], not (^add_filter(query, filter))) + end + + defp add_filter(query, [:and, filters]) do + filters + |> Enum.map(&add_filter(query, &1)) + |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end) + end + + defp add_filter(query, [:or, filters]) do + filters + |> Enum.map(&add_filter(query, &1)) + |> Enum.reduce(fn condition, acc -> dynamic([], ^acc or ^condition) end) + end + + defp add_filter(query, [_operation, dimension, _clauses | _rest] = filter) do + db_field = Plausible.Stats.Filters.without_prefix(dimension) + + if db_field == :goal do + Plausible.Stats.Goals.add_filter(query, filter, imported?: true) + else + mapped_db_field = Map.get(@db_field_mappings, db_field, db_field) + + Plausible.Stats.SQL.WhereBuilder.build_condition(mapped_db_field, filter) + end + end +end diff --git a/lib/plausible/stats/interval.ex b/lib/plausible/stats/interval.ex new file mode 100644 index 0000000000000000000000000000000000000000..e86ffbbdbb59d8010e65a78a4bdbbc9a902efb4d --- /dev/null +++ b/lib/plausible/stats/interval.ex @@ -0,0 +1,47 @@ +defmodule Plausible.Stats.Interval do + @moduledoc """ + [DEPRECATED] Stats API v2 handles "intervals" as time dimensions. + See `Plausible.Stats.ApiQueryParser.parse_dimensions/1`. + """ + + alias Plausible.Stats.{DateTimeRange, Query} + + @intervals ["minute", "hour", "day", "week", "month"] + + def valid?(interval) do + interval in @intervals + end + + @doc """ + Returns the suggested interval for a given Stats API v1 (legacy) query. + """ + def default_for_query(query) + + def default_for_query(%Query{ + input_date_range: :all, + utc_time_range: %DateTimeRange{first: first, last: last} + }) do + cond do + Plausible.Times.diff(last, first, :month) > 0 -> + "month" + + DateTime.diff(last, first, :day) > 0 -> + "day" + + true -> + "hour" + end + end + + def default_for_query(%Query{} = query) do + case query.input_date_range do + period when period in [:realtime, :realtime_30m] -> "minute" + :day -> "hour" + :"24h" -> "hour" + {:last_n_days, _} -> "day" + period when period in [:custom, :month] -> "day" + {:last_n_months, _} -> "month" + :year -> "month" + end + end +end diff --git a/lib/plausible/stats/json-schema/utils.ex b/lib/plausible/stats/json-schema/utils.ex new file mode 100644 index 0000000000000000000000000000000000000000..f5a33a6aafd13d1504c54102d273c95d00efde87 --- /dev/null +++ b/lib/plausible/stats/json-schema/utils.ex @@ -0,0 +1,44 @@ +defmodule Plausible.Stats.JSONSchema.Utils do + @moduledoc """ + Module for traversing and modifying JSON schemas. + """ + + @type json :: map() | list() | String.t() | number() | boolean() | nil + @type transform_fun :: (json() -> json() | :remove) + + @spec traverse(map(), transform_fun()) :: map() | :remove + def traverse(json, fun) when is_map(json) do + result = + Enum.reduce(json, %{}, fn {k, v}, acc -> + case traverse(v, fun) do + :remove -> acc + transformed_v -> Map.put(acc, k, transformed_v) + end + end) + + case result do + map when map_size(map) == 0 -> fun.(%{}) + map -> fun.(map) + end + end + + @spec traverse(list(), transform_fun()) :: list() | :remove + def traverse(json, fun) when is_list(json) do + result = + Enum.reduce(json, [], fn v, acc -> + case traverse(v, fun) do + :remove -> acc + transformed_v -> [transformed_v | acc] + end + end) + |> Enum.reverse() + + case result do + [] -> fun.([]) + list -> fun.(list) + end + end + + @spec traverse(String.t() | number() | boolean() | nil, transform_fun()) :: json() | :remove + def traverse(value, fun), do: fun.(value) +end diff --git a/lib/plausible/stats/json_schema.ex b/lib/plausible/stats/json_schema.ex new file mode 100644 index 0000000000000000000000000000000000000000..c12b65e473fd7200bf6a0a4fbe5dad62de37f1e5 --- /dev/null +++ b/lib/plausible/stats/json_schema.ex @@ -0,0 +1,64 @@ +defmodule Plausible.Stats.JSONSchema do + @moduledoc """ + Module for validating query parameters against JSON schema. + + Note that `internal` queries expose some metrics, filter types and other features not + available on the public API. + """ + use Plausible + alias Plausible.Stats.JSONSchema.Utils + alias Plausible.Stats.QueryError + + @json_schema_filepath "priv/json-schemas/query-api-schema.json" + @external_resource @json_schema_filepath + + @raw_schema Application.app_dir(:plausible, @json_schema_filepath) + |> File.read!() + |> Jason.decode!() + |> Utils.traverse(fn + %{"$comment" => "only :ee"} = value -> + if(ee?(), do: Map.delete(value, "$comment"), else: :remove) + + value -> + value + end) + + def raw_schema(), do: @raw_schema + + @query_schema ExJsonSchema.Schema.resolve(@raw_schema) + + def validate(params) do + case ExJsonSchema.Validator.validate(@query_schema, params) do + :ok -> + :ok + + {:error, errors} -> + {:error, + %QueryError{code: :failed_schema_validation, message: format_errors(errors, params)}} + end + end + + defp format_errors(errors, params) do + errors + |> Enum.map_join("\n", fn {error, path} -> + value = JSONPointer.get!(params, path) + + "#{path}: #{reword(path, error, value)}" + end) + end + + @no_matches "Expected exactly one of the schemata to match, but none of them did." + + defp reword("#/dimensions/" <> _, @no_matches, value), do: "Invalid dimension #{i(value)}" + defp reword("#/metrics/" <> _, @no_matches, value), do: "Invalid metric #{i(value)}" + defp reword("#/filters/" <> _, @no_matches, value), do: "Invalid filter #{i(value)}" + defp reword("#/date_range", @no_matches, value), do: "Invalid date range #{i(value)}" + + defp reword("#/order_by/" <> _, @no_matches, value) do + "Invalid value in order_by #{i(value)}" + end + + defp reword(_path, error, _value), do: error + + defp i(value), do: inspect(value, charlists: :as_lists) +end diff --git a/lib/plausible/stats/legacy/aggregate.ex b/lib/plausible/stats/legacy/aggregate.ex new file mode 100644 index 0000000000000000000000000000000000000000..cc633af96890ca839fe15ac8c3110aed50cfba3d --- /dev/null +++ b/lib/plausible/stats/legacy/aggregate.ex @@ -0,0 +1,61 @@ +defmodule Plausible.Stats.Legacy.Aggregate do + @moduledoc """ + Builds aggregate results for the Stats API v1. Avoid adding new logic here. + """ + + use Plausible.ClickhouseRepo + alias Plausible.Stats.{Query, QueryRunner, QueryResult, QueryOptimizer} + + def aggregate(site, query, metrics) do + query = + query + |> Query.set(metrics: metrics) + |> Query.set_include(:drop_unavailable_revenue_metrics, true) + |> QueryOptimizer.optimize() + + %QueryResult{results: [entry], meta: meta} = QueryRunner.run(site, query) + + results = + query.metrics + |> Enum.with_index() + |> Enum.map(fn {metric, index} -> + { + metric, + metric_map(entry, index, metric) + } + end) + |> Enum.into(%{}) + + %{results: results, meta: meta} + end + + def metric_map( + %{metrics: metrics, comparison: %{metrics: comparison_metrics, change: change}}, + index, + metric + ) do + %{ + value: get_value(metrics, index, metric), + comparison_value: get_value(comparison_metrics, index, metric), + change: Enum.at(change, index) + } + end + + def metric_map(%{metrics: metrics}, index, metric) do + %{ + value: get_value(metrics, index, metric) + } + end + + def get_value(metric_list, index, metric) do + metric_list + |> Enum.at(index) + |> maybe_round_value(metric) + end + + @metrics_to_round [:bounce_rate, :visit_duration, :sample_percent] + + defp maybe_round_value(nil, _metric), do: nil + defp maybe_round_value(value, metric) when metric in @metrics_to_round, do: round(value) + defp maybe_round_value(value, _metric), do: value +end diff --git a/lib/plausible/stats/legacy/breakdown.ex b/lib/plausible/stats/legacy/breakdown.ex new file mode 100644 index 0000000000000000000000000000000000000000..364a5e05df5b91c39a39f42faf1d3b8f5da0a11a --- /dev/null +++ b/lib/plausible/stats/legacy/breakdown.ex @@ -0,0 +1,183 @@ +defmodule Plausible.Stats.Legacy.Breakdown do + @moduledoc """ + Builds breakdown results for the Stats API v1. Avoid adding new logic here. + """ + + use Plausible.ClickhouseRepo + use Plausible.Stats.SQL.Fragments + + alias Plausible.Stats.{Query, QueryRunner, QueryResult, Comparisons} + + def breakdown( + site, + %Query{dimensions: [dimension], order_by: order_by} = query, + metrics, + {limit, page}, + _opts \\ [] + ) do + transformed_metrics = transform_metrics(metrics, dimension) + transformed_order_by = transform_order_by(order_by || [], dimension) + + query_with_metrics = + query + |> Query.set( + metrics: transformed_metrics, + # Concat client requested order with default order, overriding only if client explicitly requests it + order_by: + Enum.concat(transformed_order_by, infer_order_by(transformed_metrics, dimension)) + |> Enum.uniq_by(&elem(&1, 0)), + dimensions: transform_dimensions(dimension), + filters: query.filters ++ dimension_filters(dimension), + pagination: %{limit: limit, offset: (page - 1) * limit}, + # Allow pageview and event metrics to be queried off of sessions table + legacy_breakdown: true + ) + |> Query.set_include(:drop_unavailable_revenue_metrics, true) + + %QueryResult{results: results, meta: meta} = QueryRunner.run(site, query_with_metrics) + + %{ + results: build_breakdown_result(results, query_with_metrics, metrics), + meta: meta + } + end + + def formatted_date_ranges(query) do + formatted = %{ + date_range_label: format_date_range(query) + } + + if query.include.compare do + comparison_date_range_label = + query + |> Comparisons.get_comparison_query() + |> format_date_range() + + Map.put( + formatted, + :comparison_date_range_label, + comparison_date_range_label + ) + else + formatted + end + end + + defp build_breakdown_result(results, query, metrics) do + dimension_keys = query.dimensions |> Enum.map(&result_key/1) + + results + |> Enum.map(fn entry -> + comparison_map = + if entry[:comparison] do + comparison = + build_map(metrics, entry.comparison.metrics) + |> Map.put(:change, build_map(metrics, entry.comparison.change)) + + %{comparison: comparison} + else + %{} + end + + build_map(dimension_keys, entry.dimensions) + |> Map.merge(build_map(metrics, entry.metrics)) + |> Map.merge(comparison_map) + end) + end + + defp build_map(keys, values) do + Enum.zip(keys, values) |> Map.new() + end + + defp result_key("event:props:" <> custom_property), do: custom_property + defp result_key("event:" <> key), do: key |> String.to_existing_atom() + defp result_key("visit:" <> key), do: key |> String.to_existing_atom() + defp result_key(dimension), do: dimension + + defp maybe_remap_to_group_conversion_rate(metric, dimension) do + case {metric, dimension} do + {:conversion_rate, "event:props:" <> _} -> :conversion_rate + {:conversion_rate, "event:goal"} -> :conversion_rate + {:conversion_rate, _} -> :group_conversion_rate + _ -> metric + end + end + + defp transform_metrics(metrics, dimension) do + metrics = + if is_nil(metric_to_order_by(metrics)) do + metrics ++ [:visitors] + else + metrics + end + + Enum.map(metrics, fn metric -> maybe_remap_to_group_conversion_rate(metric, dimension) end) + end + + defp transform_order_by(order_by, dimension) do + Enum.map(order_by, fn {metric, direction} -> + {maybe_remap_to_group_conversion_rate(metric, dimension), direction} + end) + end + + defp infer_order_by(metrics, "event:goal"), + do: [{metric_to_order_by(metrics), :desc}] + + defp infer_order_by(metrics, dimension), + do: [{metric_to_order_by(metrics), :desc}, {dimension, :asc}] + + defp metric_to_order_by(metrics) do + Enum.find(metrics, &(&1 != :time_on_page)) + end + + def transform_dimensions("visit:browser_version"), + do: ["visit:browser", "visit:browser_version"] + + def transform_dimensions("visit:os_version"), do: ["visit:os", "visit:os_version"] + def transform_dimensions(dimension), do: [dimension] + + @filter_dimensions_not %{ + "visit:city" => [0], + "visit:country" => ["\0\0", "ZZ"], + "visit:region" => [""], + "visit:utm_medium" => [""], + "visit:utm_source" => [""], + "visit:utm_campaign" => [""], + "visit:utm_content" => [""], + "visit:utm_term" => [""], + "visit:entry_page" => [""], + "visit:exit_page" => [""] + } + + @extra_filter_dimensions Map.keys(@filter_dimensions_not) + + defp dimension_filters(dimension) when dimension in @extra_filter_dimensions do + [[:is_not, dimension, Map.get(@filter_dimensions_not, dimension)]] + end + + defp dimension_filters(_), do: [] + + defp format_date_range(%Query{} = query) do + year = query.now.year + %Date.Range{first: first, last: last} = Query.date_range(query, trim_trailing: true) + + cond do + first == last -> + strfdate(first, first.year != year) + + first.year == last.year -> + "#{strfdate(first, false)} - #{strfdate(last, year != last.year)}" + + true -> + "#{strfdate(first, true)} - #{strfdate(last, true)}" + end + end + + defp strfdate(date, true = _include_year) do + Calendar.strftime(date, "%-d %b %Y") + end + + defp strfdate(date, false = _include_year) do + Calendar.strftime(date, "%-d %b") + end +end diff --git a/lib/plausible/stats/legacy/legacy_dimensions.ex b/lib/plausible/stats/legacy/legacy_dimensions.ex new file mode 100644 index 0000000000000000000000000000000000000000..d8b03d90061fd55f1f1f1cf3666cba739c37412c --- /dev/null +++ b/lib/plausible/stats/legacy/legacy_dimensions.ex @@ -0,0 +1,31 @@ +defmodule Plausible.Stats.Legacy.Dimensions do + @moduledoc """ + Deprecated module. See ApiQueryParser for list of valid dimensions + """ + @event_props ["event:page", "event:name", "event:goal", "event:hostname"] + @session_props [ + "visit:source", + "visit:channel", + "visit:country", + "visit:region", + "visit:city", + "visit:entry_page", + "visit:exit_page", + "visit:referrer", + "visit:utm_medium", + "visit:utm_source", + "visit:utm_campaign", + "visit:utm_content", + "visit:utm_term", + "visit:device", + "visit:os", + "visit:os_version", + "visit:browser", + "visit:browser_version" + ] + + def valid?(prop) when prop in @event_props, do: true + def valid?(prop) when prop in @session_props, do: true + def valid?("event:props:" <> prop) when byte_size(prop) > 0, do: true + def valid?(_), do: false +end diff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.ex new file mode 100644 index 0000000000000000000000000000000000000000..f942d27d1bcd6a710af8f83458146347adf94e76 --- /dev/null +++ b/lib/plausible/stats/legacy/legacy_query_builder.ex @@ -0,0 +1,351 @@ +defmodule Plausible.Stats.Legacy.QueryBuilder do + @moduledoc """ + Module used to parse URL search params to a valid Query, used to power the API for the dashboard + and v1 of our Stats API. + + @deprecated + """ + + use Plausible + + alias Plausible.Stats.{Filters, Query, ApiQueryParser, QueryBuilder, DateTimeRange} + alias Plausible.Times + + def from(site, params, debug_metadata, now \\ nil) do + now = now || DateTime.utc_now(:second) + + query = + Query + |> struct!( + now: now, + debug_metadata: debug_metadata, + site_id: site.id, + site_native_stats_start_at: site.native_stats_start_at + ) + |> put_input_date_range(site, params) + |> put_timezone(site) + |> put_dimensions(params) + |> put_parsed_filters(params) + |> resolve_segments(site) + |> preload_goals_and_revenue(site) + |> put_consolidated_site_ids(site) + |> put_order_by(params) + |> put_include(params) + |> QueryBuilder.put_comparison_utc_time_range() + |> Query.put_imported_opts(site) + |> QueryBuilder.set_time_on_page_data(site) + + on_ee do + query = Plausible.Stats.Sampling.put_threshold(query, site, params) + end + + query + end + + on_ee do + defp put_consolidated_site_ids(query, %Plausible.Site{} = site) do + if Plausible.Sites.consolidated?(site) do + site_ids = Plausible.ConsolidatedView.Cache.get(site.domain) + struct!(query, consolidated_site_ids: site_ids) + else + query + end + end + else + defp put_consolidated_site_ids(query, _site), do: query + end + + defp resolve_segments(query, site) do + with {:ok, preloaded_segments} <- + Plausible.Segments.Filters.preload_needed_segments(site, query.filters), + {:ok, filters} <- + Plausible.Segments.Filters.resolve_segments(query.filters, preloaded_segments) do + struct!(query, + filters: filters + ) + end + end + + defp preload_goals_and_revenue(query, site) do + {preloaded_goals, revenue_warning, revenue_currencies} = + Plausible.Stats.QueryBuilder.preload_goals_and_revenue( + site, + query.metrics, + query.filters, + query.dimensions + ) + + struct!(query, + preloaded_goals: preloaded_goals, + revenue_warning: revenue_warning, + revenue_currencies: revenue_currencies + ) + end + + defp put_input_date_range(%Query{now: now} = query, _site, %{"period" => period}) + when period in ["realtime", "30m"] do + {duration_minutes, input_date_range_atom} = + case period do + "realtime" -> {5, :realtime} + "30m" -> {30, :realtime_30m} + end + + first_datetime = DateTime.shift(now, minute: -duration_minutes) + last_datetime = DateTime.shift(now, second: 5) + + datetime_range = + DateTimeRange.new!(first_datetime, last_datetime) |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, input_date_range: input_date_range_atom, utc_time_range: datetime_range) + end + + defp put_input_date_range(%Query{now: now} = query, site, %{"period" => "day"} = params) do + date = parse_single_date(query, params) + + datetime_range = + if Date.compare(Times.to_date(now, site.timezone), date) == :eq do + DateTimeRange.new!(date, now, site.timezone) + else + DateTimeRange.new!(date, date, site.timezone) + end + |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, input_date_range: :day, utc_time_range: datetime_range) + end + + defp put_input_date_range(%Query{now: now} = query, _site, %{"period" => "24h"}) do + from = DateTime.shift(now, hour: -24) + datetime_range = DateTimeRange.new!(from, now) + struct!(query, input_date_range: :"24h", utc_time_range: datetime_range) + end + + defp put_input_date_range(query, site, %{"period" => period} = params) + when period in ["7d", "28d", "30d", "91d"] do + {days, "d"} = Integer.parse(period) + + end_date = parse_single_date(query, params) |> Date.shift(day: -1) + start_date = end_date |> Date.shift(day: 1 - days) + + datetime_range = + DateTimeRange.new!(start_date, end_date, site.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, input_date_range: {:last_n_days, days}, utc_time_range: datetime_range) + end + + defp put_input_date_range(query, site, %{"period" => "month"} = params) do + date = parse_single_date(query, params) + start_date = Date.beginning_of_month(date) + end_date = Date.end_of_month(date) + + datetime_range = + DateTimeRange.new!(start_date, end_date, site.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, input_date_range: :month, utc_time_range: datetime_range) + end + + defp put_input_date_range(query, site, %{"period" => "6mo"} = params) do + end_date = + parse_single_date(query, params) + |> Date.shift(month: -1) + |> Date.end_of_month() + + start_date = + Date.shift(end_date, month: -5) + |> Date.beginning_of_month() + + datetime_range = + DateTimeRange.new!(start_date, end_date, site.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, input_date_range: {:last_n_months, 6}, utc_time_range: datetime_range) + end + + defp put_input_date_range(query, site, %{"period" => "12mo"} = params) do + end_date = + parse_single_date(query, params) + |> Date.shift(month: -1) + |> Date.end_of_month() + + start_date = + Date.shift(end_date, month: -11) + |> Date.beginning_of_month() + + datetime_range = + DateTimeRange.new!(start_date, end_date, site.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, input_date_range: {:last_n_months, 12}, utc_time_range: datetime_range) + end + + defp put_input_date_range(query, site, %{"period" => "year"} = params) do + end_date = + parse_single_date(query, params) + |> Plausible.Times.end_of_year() + + start_date = Plausible.Times.beginning_of_year(end_date) + + datetime_range = + DateTimeRange.new!(start_date, end_date, site.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, input_date_range: :year, utc_time_range: datetime_range) + end + + defp put_input_date_range(query, site, %{"period" => "all"}) do + today = today(query) + start_date = Plausible.Sites.stats_start_date(site) || today + + datetime_range = + DateTimeRange.new!(start_date, today, site.timezone) |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, input_date_range: :all, utc_time_range: datetime_range) + end + + defp put_input_date_range( + query, + site, + %{"period" => "custom", "from" => from, "to" => to} = params + ) do + new_params = + params + |> Map.drop(["from", "to"]) + |> Map.put("date", Enum.join([from, to], ",")) + + put_input_date_range(query, site, new_params) + end + + defp put_input_date_range(query, site, %{"period" => "custom", "date" => date}) do + [from, to] = String.split(date, ",") + from_date = Date.from_iso8601!(String.trim(from)) + to_date = Date.from_iso8601!(String.trim(to)) + + datetime_range = + DateTimeRange.new!(from_date, to_date, site.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") + + struct!(query, input_date_range: :custom, utc_time_range: datetime_range) + end + + defp put_input_date_range(query, site, params) do + put_input_date_range(query, site, Map.merge(params, %{"period" => "30d"})) + end + + defp put_timezone(query, site) do + struct!(query, timezone: site.timezone) + end + + defp put_dimensions(query, params) do + if not is_nil(params["property"]) do + struct!(query, dimensions: [params["property"]]) + else + struct!(query, dimensions: Map.get(params, "dimensions", [])) + end + end + + defp put_include(query, params) do + include = parse_include(params["include"]) + + query + |> struct!(include: include) + |> Query.set_include(:compare, parse_include_compare(params)) + |> Query.set_include(:compare_match_day_of_week, params["match_day_of_week"] == "true") + |> Query.set_include(:imports, params["with_imported"] == "true") + end + + @doc """ + ### Examples: + iex> QueryBuilder.parse_order_by(nil) + [] + + iex> QueryBuilder.parse_order_by("") + [] + + iex> QueryBuilder.parse_order_by("0") + [] + + iex> QueryBuilder.parse_order_by("[}") + [] + + iex> QueryBuilder.parse_order_by(~s({"any":"object"})) + [] + + iex> QueryBuilder.parse_order_by(~s([["visitors","invalid"]])) + [] + + iex> QueryBuilder.parse_order_by(~s([["visitors","desc"]])) + [{:visitors, :desc}] + + iex> QueryBuilder.parse_order_by(~s([["visitors","asc"],["visit:source","desc"]])) + [{:visitors, :asc}, {"visit:source", :desc}] + """ + def parse_order_by(order_by) do + with true <- is_binary(order_by), + {:ok, order_by} <- JSON.decode(order_by), + {:ok, order_by} <- ApiQueryParser.parse_order_by(order_by) do + order_by + else + _ -> [] + end + end + + @doc """ + ### Examples: + iex> Plausible.Stats.Legacy.QueryBuilder.parse_include(nil) + Plausible.Stats.ApiQueryParser.default_include() + + iex> Plausible.Stats.Legacy.QueryBuilder.parse_include(~s({"total_rows": true})) + Map.merge(Plausible.Stats.ApiQueryParser.default_include(), %{total_rows: true}) + """ + def parse_include(include) do + include = + with true <- is_binary(include), + {:ok, include} <- JSON.decode(include), + {:ok, include} <- ApiQueryParser.parse_include(include) do + include + else + _ -> %{} + end + + Plausible.Stats.ApiQueryParser.default_include() + |> Map.merge(include) + end + + defp put_order_by(query, %{} = params) do + struct!(query, order_by: parse_order_by(params["order_by"])) + end + + defp put_parsed_filters(query, params) do + struct!(query, filters: Filters.parse(params["filters"])) + end + + defp today(query) do + query.now |> DateTime.to_date() + end + + defp parse_single_date(query, params) do + case params["date"] do + "today" -> query.now |> DateTime.to_date() + date when is_binary(date) -> Date.from_iso8601!(date) + _ -> today(query) + end + end + + defp parse_include_compare(%{"period" => period}) when period in ~w(realtime all), + do: nil + + defp parse_include_compare(%{"comparison" => "previous_period"}), do: :previous_period + defp parse_include_compare(%{"comparison" => "year_over_year"}), do: :year_over_year + + # Stats API v1 + defp parse_include_compare(%{"compare" => "previous_period"}), do: :previous_period + + defp parse_include_compare(%{"comparison" => "custom"} = params) do + from_date = Date.from_iso8601!(params["compare_from"]) + to_date = Date.from_iso8601!(params["compare_to"]) + {:date_range, from_date, to_date} + end + + defp parse_include_compare(_options), do: nil +end diff --git a/lib/plausible/stats/legacy/time_on_page.ex b/lib/plausible/stats/legacy/time_on_page.ex new file mode 100644 index 0000000000000000000000000000000000000000..488df28e1e17e6a93f7384f4a0252cc784bded99 --- /dev/null +++ b/lib/plausible/stats/legacy/time_on_page.ex @@ -0,0 +1,214 @@ +defmodule Plausible.Stats.Legacy.TimeOnPage do + @moduledoc """ + Calculation methods for `legacy` time_on_page metric. `Legacy` calculation methods + are used when site does not have engagement data for the requested dates. + + Query `include.legacy_time_on_page_cutoff` is used to determine what time range + to use for the legacy time_on_page calculations. + + Legacy metric is not exposed in the public API. + """ + + use Plausible.ClickhouseRepo + use Plausible.Stats.SQL.Fragments + + import Ecto.Query + import Plausible.Stats.Util + + alias Plausible.Stats.{Base, Filters, Query, SQL} + + def can_merge_legacy_time_on_page?(query) do + :time_on_page in query.metrics and query.dimensions in [[], ["event:page"]] + end + + def merge_legacy_time_on_page(q, query) do + if :time_on_page in query.metrics and query.time_on_page_data.include_legacy_metric do + q |> merge_legacy_time_on_page(query, query.dimensions) + else + q + end + end + + defp merge_legacy_time_on_page(q, query, []) do + from( + e in subquery(q), + inner_join: t in subquery(aggregate_time_on_page_q(query)), + on: true + ) + |> select_metrics_and_dimensions(query) + end + + defp merge_legacy_time_on_page(q, query, ["event:page"]) do + from( + e in subquery(q), + left_join: t in subquery(breakdown_q(query)), + on: e.dim0 == t.pathname + ) + |> select_metrics_and_dimensions(query) + end + + defp merge_legacy_time_on_page(q, _query, _dimensions) do + q + end + + defp select_metrics_and_dimensions(q, query) do + q + |> select_join_fields(query, List.delete(query.metrics, :time_on_page), e) + |> select_join_fields(query, query.dimensions, e) + |> select_merge_as([e, t], %{ + time_on_page: + time_on_page( + e.__internal_total_time_on_page + t.total_time_on_page, + e.__internal_total_time_on_page_visits + t.transition_count + ) + }) + end + + defp aggregate_time_on_page_q(query) do + windowed_pages_q = + from(e in Base.base_event_query(Query.remove_top_level_filters(query, ["event:page"])), + where: e.name != "engagement", + where: ^filter_by_cutoff(query), + select: %{ + next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon), + next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon), + timestamp: e.timestamp, + pathname: e.pathname, + session_id: e.session_id + }, + windows: [ + event_horizon: [ + partition_by: e.session_id, + order_by: e.timestamp, + frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING") + ] + ] + ) + + event_page_filter = Filters.get_toplevel_filter(query, "event:page") + + timed_page_transitions_q = + from e in Ecto.Query.subquery(windowed_pages_q), + group_by: [e.pathname, e.next_pathname, e.session_id], + where: ^SQL.WhereBuilder.build_condition(:pathname, event_page_filter), + where: e.next_timestamp != 0, + select: %{ + pathname: e.pathname, + transition: e.next_pathname != e.pathname, + duration: sum(e.next_timestamp - e.timestamp) + } + + avg_time_per_page_transition_q = + from e in Ecto.Query.subquery(timed_page_transitions_q), + select: %{ + avg: fragment("sum(?)/countIf(?)", e.duration, e.transition), + duration: fragment("sum(?)", e.duration), + transition_count: fragment("countIf(?)", e.transition) + }, + group_by: e.pathname + + from( + e in subquery(avg_time_per_page_transition_q), + select: %{ + total_time_on_page: fragment("sum(?)", e.duration), + transition_count: fragment("sum(?)", e.transition_count) + } + ) + end + + defp breakdown_q(query) do + windowed_pages_q = + from( + e in Base.base_event_query( + Query.remove_top_level_filters(query, ["event:page", "event:props"]) + ), + where: e.name != "engagement", + where: ^filter_by_cutoff(query), + select: %{ + next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon), + next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon), + timestamp: e.timestamp, + pathname: e.pathname, + session_id: e.session_id + }, + windows: [ + event_horizon: [ + partition_by: e.session_id, + order_by: e.timestamp, + frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING") + ] + ] + ) + + timed_page_transitions_q = + from(e in subquery(windowed_pages_q), + group_by: [e.pathname, e.next_pathname, e.session_id], + where: e.next_timestamp != 0, + select: %{ + pathname: e.pathname, + transition: e.next_pathname != e.pathname, + duration: sum(e.next_timestamp - e.timestamp) + } + ) + + no_select_timed_pages_q = + from e in subquery(timed_page_transitions_q), + group_by: e.pathname + + date_range = Query.date_range(query) + + if query.include_imported do + # Imported page views have pre-calculated values + imported_timed_pages_q = + from(i in "imported_pages", + group_by: i.page, + where: i.site_id == ^query.site_id, + where: i.date >= ^date_range.first and i.date <= ^date_range.last, + select: %{ + page: i.page, + time_on_page: sum(i.total_time_on_page), + visits: sum(i.pageviews) - sum(i.exits) + } + ) + + timed_pages_q = + from e in no_select_timed_pages_q, + select: %{ + page: e.pathname, + time_on_page: fragment("sum(?)", e.duration), + visits: fragment("countIf(?)", e.transition) + } + + "timed_pages" + |> with_cte("timed_pages", as: ^timed_pages_q) + |> with_cte("imported_timed_pages", as: ^imported_timed_pages_q) + |> join(:full, [t], i in "imported_timed_pages", on: t.page == i.page) + |> select( + [t, i], + %{ + pathname: fragment("if(empty(?),?,?)", t.page, i.page, t.page), + total_time_on_page: t.time_on_page + i.time_on_page, + transition_count: t.visits + i.visits + } + ) + else + from(e in no_select_timed_pages_q, + select: %{ + pathname: e.pathname, + total_time_on_page: fragment("sum(?)", e.duration), + transition_count: fragment("countIf(?)", e.transition) + } + ) + end + end + + defp filter_by_cutoff(query) do + case query.time_on_page_data do + %{include_legacy_metric: true, include_new_metric: true, cutoff: cutoff} -> + dynamic([e], e.timestamp < ^cutoff) + + _ -> + true + end + end +end diff --git a/lib/plausible/stats/legacy/timeseries.ex b/lib/plausible/stats/legacy/timeseries.ex new file mode 100644 index 0000000000000000000000000000000000000000..93eb1ba5129d1a017fd64776d07d9b6c95636181 --- /dev/null +++ b/lib/plausible/stats/legacy/timeseries.ex @@ -0,0 +1,86 @@ +defmodule Plausible.Stats.Legacy.Timeseries do + @moduledoc """ + Builds timeseries results for the Stats API v1. Avoid adding new logic here. + """ + + use Plausible + use Plausible.ClickhouseRepo + alias Plausible.Stats.{Query, QueryRunner, Metrics, Time, QueryOptimizer} + + def timeseries(site, query, metrics) do + [time_dimension] = query.dimensions + + query = + query + |> Query.set( + metrics: transform_metrics(metrics, %{conversion_rate: :group_conversion_rate}), + order_by: [{time_dimension, :asc}] + ) + |> Query.set_include(:drop_unavailable_revenue_metrics, true) + |> QueryOptimizer.optimize() + + query_result = QueryRunner.run(site, query) + + { + build_result(query_result, query), + query_result.meta + } + end + + # Given a query result, build a legacy timeseries result + # Format is %{ date => %{ date: date_string, [metric] => value } } with a bunch of special cases for the UI + defp build_result(query_result, %Query{} = query) do + query_result.results + |> Enum.map(fn + %{dimensions: [time_dimension_value], metrics: metrics} -> + metrics_map = Enum.zip(query.metrics, metrics) |> Map.new() + + { + time_dimension_value, + Map.put(metrics_map, :date, time_dimension_value) + } + end) + |> Map.new() + |> add_labels(query) + end + + defp add_labels(results_map, query) do + query + |> Time.time_labels() + |> Enum.map(fn key -> + Map.get( + results_map, + key, + empty_row(key, query.metrics, query) + ) + end) + |> transform_realtime_labels(query) + |> transform_keys(%{group_conversion_rate: :conversion_rate}) + end + + defp empty_row(date, metrics, query) do + metrics + |> Map.new(fn metric -> {metric, Metrics.default_value(metric, query, [date])} end) + |> Map.put(:date, date) + end + + defp transform_metrics(metrics, to_replace) do + Enum.map(metrics, &Map.get(to_replace, &1, &1)) + end + + defp transform_keys(results, keys_to_replace) do + Enum.map(results, fn map -> + Enum.map(map, fn {key, val} -> + {Map.get(keys_to_replace, key, key), val} + end) + |> Enum.into(%{}) + end) + end + + defp transform_realtime_labels(results, %Query{input_date_range: :realtime_30m}) do + Enum.with_index(results) + |> Enum.map(fn {entry, index} -> %{entry | date: -30 + index} end) + end + + defp transform_realtime_labels(results, _query), do: results +end diff --git a/lib/plausible/stats/metrics.ex b/lib/plausible/stats/metrics.ex new file mode 100644 index 0000000000000000000000000000000000000000..191b3a363421aaea00a6a0c4fd60d966d0a038d2 --- /dev/null +++ b/lib/plausible/stats/metrics.ex @@ -0,0 +1,75 @@ +defmodule Plausible.Stats.Metrics do + @moduledoc """ + A module listing all available metrics in Plausible. + + Useful for an explicit string to atom conversion. + """ + + use Plausible + + @revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: []) + + @all_metrics [ + :visitors, + :visits, + :pageviews, + :exit_rate, + :views_per_visit, + :bounce_rate, + :visit_duration, + :events, + :conversion_rate, + :total_visitors, + :group_conversion_rate, + :time_on_page, + :percentage, + :scroll_depth + ] ++ @revenue_metrics + + @metric_mappings Enum.into(@all_metrics, %{}, fn metric -> {to_string(metric), metric} end) + + def metric?(value), do: Enum.member?(@all_metrics, value) + + on_ee do + # Default value in a goal breakdown depends on per-row currency + def default_value(metric, query, row_dimensions) when metric in @revenue_metrics do + Plausible.Stats.Goal.Revenue.format_revenue_metric(nil, query, row_dimensions) + end + end + + def default_value(metric, _query, _dimensions), do: default_value(metric) + + on_ee do + # When revenue metrics are queried without event:goal dimension, + # a single default currency is expected. + def default_value(metric, query) when metric in @revenue_metrics do + currency = query.revenue_currencies.default + Plausible.Stats.Goal.Revenue.format_revenue_metric(nil, currency) + end + end + + def default_value(metric, _query), do: default_value(metric) + + def default_value(:visit_duration), do: nil + def default_value(:exit_rate), do: nil + def default_value(:scroll_depth), do: nil + def default_value(:time_on_page), do: nil + + @float_metrics [ + :views_per_visit, + :bounce_rate, + :percentage, + :conversion_rate, + :group_conversion_rate + ] + def default_value(metric) when metric in @float_metrics, do: 0.0 + def default_value(_metric), do: 0 + + def from_string!(str) do + Map.fetch!(@metric_mappings, str) + end + + def from_string(str) do + Map.fetch(@metric_mappings, str) + end +end diff --git a/lib/plausible/stats/parsed_query_params.ex b/lib/plausible/stats/parsed_query_params.ex new file mode 100644 index 0000000000000000000000000000000000000000..86689ec8638d23880b48225b18d98355f0dcad0b --- /dev/null +++ b/lib/plausible/stats/parsed_query_params.ex @@ -0,0 +1,66 @@ +defmodule Plausible.Stats.ParsedQueryParams do + @moduledoc false + + alias Plausible.Stats.Query + + defstruct input_date_range: nil, + # `relative_date` is a convenience currently exclusive to the internal + # dashboard API for constructing datetime ranges. It adds the ability + # to use `day`, `month` and `year` periods relative to a specific date. + # E.g.: `?period=month&date=2021-01-15` will query the entire month of + # January 2021. In the public API it is always today in site.timezone. + relative_date: nil, + metrics: [], + filters: [], + dimensions: [], + order_by: nil, + pagination: nil, + now: nil, + include: %Plausible.Stats.QueryInclude{}, + # When true, skips the validation that goal names in `is` filters must be + # configured for the site. Missing goals simply match nothing in SQL, matching + # the behaviour of legacy endpoints like top_stats. + skip_goal_existence_check: false + + def to_query!(%__MODULE__{} = parsed_query_params) do + query_fields = %Query{} |> Map.from_struct() |> Map.keys() + struct!(%Query{}, Map.take(parsed_query_params, query_fields)) + end + + def new!(params) when is_map(params) do + struct!(__MODULE__, Map.to_list(params)) + end + + def set(params, keywords) do + struct!(params, keywords) + end + + def set_include(params, key, value) do + struct!(params, include: struct!(params.include, [{key, value}])) + end + + @props_prefix "event:props:" + + def add_or_replace_filter(%__MODULE__{filters: filters} = parsed_query_params, new_filter) do + [_, new_filter_dimension, _] = new_filter + + prop_filter? = String.starts_with?(new_filter_dimension, @props_prefix) + + new_filters = + filters + |> Enum.reject(fn [_, existing_filter_dimension, _] -> + existing_filter_dimension == new_filter_dimension or + (prop_filter? and String.starts_with?(existing_filter_dimension, @props_prefix)) + end) + |> Enum.concat([new_filter]) + + struct!(parsed_query_params, filters: new_filters) + end + + def conversion_goal_filter?(%__MODULE__{filters: filters}) do + Plausible.Stats.Filters.filtering_on_dimension?(filters, "event:goal", + max_depth: 0, + behavioral_filters: :ignore + ) + end +end diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex new file mode 100644 index 0000000000000000000000000000000000000000..5330b7bd0c7a21af29a13185ceec12812bd166c8 --- /dev/null +++ b/lib/plausible/stats/query.ex @@ -0,0 +1,232 @@ +defmodule Plausible.Stats.Query do + @moduledoc false + use Plausible + + defstruct utc_time_range: nil, + comparison_utc_time_range: nil, + input_date_range: nil, + dimensions: [], + filters: [], + sample_threshold: nil, + imports_exist: false, + imports_in_range: [], + include_imported: false, + skip_imported_reason: nil, + now: nil, + metrics: [], + order_by: nil, + timezone: nil, + legacy_breakdown: false, + preloaded_goals: [], + include: Plausible.Stats.ApiQueryParser.default_include(), + debug_metadata: %{}, + pagination: nil, + # Revenue metric specific metadata + revenue_currencies: %{}, + revenue_warning: nil, + site_id: nil, + consolidated_site_ids: nil, + site_native_stats_start_at: nil, + # Contains information to determine how to combine legacy and new time on page metrics + time_on_page_data: %{}, + sql_join_type: :left, + smear_session_metrics: false + + alias OpenTelemetry.Tracer + + alias Plausible.Stats.{ + DateTimeRange, + Imported, + Legacy, + Comparisons, + ApiQueryParser, + ParsedQueryParams, + QueryBuilder, + QueryError + } + + @type t :: %__MODULE__{} + + def parse_and_build( + %Plausible.Site{domain: domain} = site, + %{"site_id" => domain} = params, + opts \\ [] + ) do + with {:ok, %ParsedQueryParams{} = parsed_query_params} <- + ApiQueryParser.parse(params, opts) do + QueryBuilder.build(site, parsed_query_params, Keyword.get(opts, :debug_metadata, %{})) + end + end + + def parse_and_build!(site, params, opts \\ []) do + case parse_and_build(site, params, opts) do + {:ok, query} -> + query + + {:error, %QueryError{message: message}} -> + raise "Failed to build query: #{inspect(message)}" + end + end + + @doc """ + Builds query from old-style stats APIv1 params. New code should use `Query.parse_and_build` + or `QueryBuilder.build` with already parsed params. + """ + def from(site, params, opts \\ []) do + Legacy.QueryBuilder.from( + site, + params, + Keyword.get(opts, :debug_metadata, %{}), + Keyword.get(opts, :now) + ) + end + + def date_range(query, options \\ []) do + date_range = DateTimeRange.to_date_range(query.utc_time_range, query.timezone) + + if Keyword.get(options, :trim_trailing) do + today = query.now |> DateTime.shift_zone!(query.timezone) |> DateTime.to_date() + + Date.range( + date_range.first, + clamp(today, date_range) + ) + else + date_range + end + end + + defp clamp(date, date_range) do + cond do + date in date_range -> date + Date.before?(date, date_range.first) -> date_range.first + Date.after?(date, date_range.last) -> date_range.last + end + end + + def set(query, keywords) do + new_query = struct!(query, keywords) + + if Keyword.has_key?(keywords, :include_imported) do + new_query + else + refresh_imported_opts(new_query) + end + end + + def set_include(query, key, value) do + struct!(query, include: struct!(query.include, [{key, value}])) + end + + def add_filter(query, filter) do + query + |> struct!(filters: query.filters ++ [filter]) + |> refresh_imported_opts() + end + + @doc """ + Removes top level filters matching any of passed prefix from the query. + + Note that this doesn't handle cases with AND/OR/NOT and as such is discouraged + from use. + """ + def remove_top_level_filters(query, prefixes) do + new_filters = + Enum.reject(query.filters, fn [_, dimension_or_filter_tree | _rest] -> + is_binary(dimension_or_filter_tree) and + Enum.any?(prefixes, &String.starts_with?(dimension_or_filter_tree, &1)) + end) + + query + |> struct!(filters: new_filters) + |> refresh_imported_opts() + end + + defp refresh_imported_opts(query) do + put_imported_opts(query, nil) + end + + def put_imported_opts(query, site) do + requested? = query.include.imports + + query = + if site && Imported.schema_supports_interval?(query) do + site = Plausible.Repo.preload(site, :completed_imports) + + struct!(query, + imports_exist: Plausible.Imported.any_completed_imports?(site), + imports_in_range: get_imports_in_range(site, query) + ) + else + query + end + + skip_imported_reason = get_skip_imported_reason(query) + + struct!(query, + include_imported: requested? and is_nil(skip_imported_reason), + skip_imported_reason: skip_imported_reason + ) + end + + defp get_imports_in_range(_site, %__MODULE__{input_date_range: period}) + when period in [:realtime, :realtime_30m] do + [] + end + + defp get_imports_in_range(site, query) do + in_range = Plausible.Imported.completed_imports_in_query_range(site, query) + + in_comparison_range = + if query.include.compare do + comparison_query = Comparisons.get_comparison_query(query) + Plausible.Imported.completed_imports_in_query_range(site, comparison_query) + else + [] + end + + in_comparison_range ++ in_range + end + + @spec get_skip_imported_reason(t()) :: + nil | :no_imported_data | :out_of_range | :unsupported_interval | :unsupported_query + def get_skip_imported_reason(query) do + cond do + not Imported.schema_supports_interval?(query) -> + :unsupported_interval + + not query.imports_exist -> + :no_imported_data + + query.imports_in_range == [] -> + :out_of_range + + not Imported.schema_supports_query?(query) -> + :unsupported_query + + true -> + nil + end + end + + @spec trace(%__MODULE__{}, [atom()]) :: %__MODULE__{} + def trace(%__MODULE__{} = query, metrics) do + filter_dimensions = + query.filters + |> Plausible.Stats.Filters.dimensions_used_in_filters() + |> Enum.sort() + |> Enum.uniq() + |> Enum.join(";") + + metrics = metrics |> Enum.sort() |> Enum.join(";") + + Tracer.set_attributes([ + {"plausible.query.dimensions", query.dimensions |> Enum.join(";")}, + {"plausible.query.include_imported", query.include_imported}, + {"plausible.query.filter_keys", filter_dimensions}, + {"plausible.query.metrics", metrics} + ]) + + query + end +end diff --git a/lib/plausible/stats/query_builder.ex b/lib/plausible/stats/query_builder.ex new file mode 100644 index 0000000000000000000000000000000000000000..0784bac368765fcb9481ca8530de78b1127449b4 --- /dev/null +++ b/lib/plausible/stats/query_builder.ex @@ -0,0 +1,550 @@ +defmodule Plausible.Stats.QueryBuilder do + @moduledoc """ + A module used for building the Query struct from already parsed params. + """ + + use Plausible + alias Plausible.Segments + + alias Plausible.Stats.{ + Query, + ParsedQueryParams, + Comparisons, + Filters, + Time, + TableDecider, + DateTimeRange, + QueryError, + QueryInclude, + QueryPeriod + } + + @doc """ + Runs various validations and builds a `%Query{}` from already parsed params. + + For convenience, the "parsed params" can also be given as a keyword list, in + which case it gets turned in to `ParsedQueryParams` by the function itself. + """ + def build(site, parsed_query_params, debug_metadata \\ %{}) + + def build(site, %ParsedQueryParams{} = parsed_query_params, debug_metadata) do + with {:ok, parsed_query_params} <- resolve_segments_in_filters(parsed_query_params, site), + {:ok, query} <- do_build(parsed_query_params, site, debug_metadata), + :ok <- validate_order_by(query), + :ok <- validate_custom_props_access(site, query), + :ok <- validate_case_sensitive_filter_modifier(query), + :ok <- validate_toplevel_only_filter_dimension(query), + :ok <- validate_time_dimension_granularity(query), + :ok <- validate_special_metrics_filters(query), + :ok <- validate_behavioral_filters(query), + :ok <- validate_filtered_goals_exist(query, parsed_query_params), + :ok <- validate_revenue_metrics_access(site, query), + :ok <- validate_metrics(query), + :ok <- validate_include(query) do + query = + query + |> set_time_on_page_data(site) + |> put_comparison_utc_time_range() + |> Query.put_imported_opts(site) + + on_ee do + # NOTE: The Query API schema does not allow the sample_threshold param + # and it looks like it's not used as a parameter anymore. We might want + # to clean this up. + query = Plausible.Stats.Sampling.put_threshold(query, site, %{}) + end + + {:ok, query} + end + end + + def build(site, params, debug_metadata) when is_list(params) do + include = struct!(QueryInclude, Keyword.get(params, :include, [])) + parsed_query_params = struct!(ParsedQueryParams, Keyword.put(params, :include, include)) + + build(site, parsed_query_params, debug_metadata) + end + + def build!(site, params, debug_metadata \\ %{}) do + case build(site, params, debug_metadata) do + {:ok, query} -> + query + + {:error, %QueryError{message: message}} -> + raise "Failed to build query: #{inspect(message)}" + end + end + + defp resolve_segments_in_filters(%ParsedQueryParams{} = parsed_query_params, site) do + with {:ok, preloaded_segments} <- + Segments.Filters.preload_needed_segments(site, parsed_query_params.filters), + {:ok, filters} <- + Segments.Filters.resolve_segments(parsed_query_params.filters, preloaded_segments) do + {:ok, struct!(parsed_query_params, filters: filters)} + end + end + + defp do_build(parsed_query_params, site, debug_metadata) do + parsed_query_params + |> ParsedQueryParams.to_query!() + |> set_now() + |> set_utc_time_range(site, Map.get(parsed_query_params, :relative_date)) + |> set_preloaded_goals_and_revenue(site) + |> Query.set( + site_id: site.id, + site_native_stats_start_at: site.native_stats_start_at, + timezone: site.timezone, + consolidated_site_ids: get_consolidated_site_ids(site), + debug_metadata: debug_metadata + ) + |> maybe_drop_revenue_metrics() + end + + defp set_now(%Query{now: nil} = query), do: Query.set(query, now: DateTime.utc_now(:second)) + defp set_now(query), do: query + + defp set_utc_time_range(query, site, relative_date) do + utc_time_range = + query.input_date_range + |> QueryPeriod.build_range_for_site(site, relative_date, query.now) + |> DateTimeRange.to_timezone("Etc/UTC") + + Query.set(query, utc_time_range: utc_time_range) + end + + defp set_preloaded_goals_and_revenue(query, site) do + {preloaded_goals, revenue_warning, revenue_currencies} = + preload_goals_and_revenue(site, query.metrics, query.filters, query.dimensions) + + Query.set(query, + preloaded_goals: preloaded_goals, + revenue_warning: revenue_warning, + revenue_currencies: revenue_currencies + ) + end + + def preload_goals_and_revenue(site, metrics, filters, dimensions) do + preloaded_goals = + Plausible.Stats.Goals.preload_needed_goals(site, dimensions, filters) + + {revenue_warning, revenue_currencies} = + preload_revenue(site, preloaded_goals, metrics, dimensions) + + { + preloaded_goals, + revenue_warning, + revenue_currencies + } + end + + on_ee do + def get_consolidated_site_ids(%Plausible.Site{} = site) do + if Plausible.Sites.consolidated?(site) do + Plausible.ConsolidatedView.Cache.get(site.domain) + end + end + else + def get_consolidated_site_ids(_site), do: nil + end + + def set_time_on_page_data(query, site) do + struct!(query, + time_on_page_data: %{ + new_metric_visible: Plausible.Stats.TimeOnPage.new_time_on_page_visible?(site), + cutoff_date: site.legacy_time_on_page_cutoff + } + ) + end + + def put_comparison_utc_time_range(%Query{include: %{compare: nil}} = query), do: query + + def put_comparison_utc_time_range(%Query{} = query) do + datetime_range = Comparisons.get_comparison_utc_time_range(query) + struct!(query, comparison_utc_time_range: datetime_range) + end + + on_ee do + alias Plausible.Stats.Goal.Revenue + + def preload_revenue(site, preloaded_goals, metrics, dimensions) do + Revenue.preload(site, preloaded_goals, metrics, dimensions) + end + + defp validate_revenue_metrics_access(site, query) do + if Revenue.requested?(query.metrics) and not Revenue.available?(site) do + {:error, + %QueryError{ + code: :feature_access, + message: "The owner of this site does not have access to the revenue metrics feature." + }} + else + :ok + end + end + + defp maybe_drop_revenue_metrics( + %Query{ + include: %QueryInclude{drop_unavailable_revenue_metrics: true}, + revenue_currencies: revenue_currencies + } = query + ) + when map_size(revenue_currencies) == 0 do + if Enum.all?(query.metrics, &(&1 in Revenue.revenue_metrics())) do + {:error, + %QueryError{ + code: :all_metrics_dropped, + message: "Revenue metrics were dropped and no other metrics were left to query." + }} + else + {:ok, Query.set(query, metrics: query.metrics -- Revenue.revenue_metrics())} + end + end + + defp maybe_drop_revenue_metrics(%Query{} = query), do: {:ok, query} + else + defp preload_revenue(_site, _preloaded_goals, _metrics, _dimensions), do: {nil, %{}} + + defp validate_revenue_metrics_access(_site, _query), do: :ok + + defp maybe_drop_revenue_metrics(query), do: {:ok, query} + end + + defp validate_order_by(query) do + if query.order_by do + valid_values = query.metrics ++ query.dimensions + + invalid_entry = + Enum.find(query.order_by, fn {value, _direction} -> + not Enum.member?(valid_values, value) + end) + + case invalid_entry do + nil -> + :ok + + _ -> + {:error, + %QueryError{ + code: :invalid_order_by, + message: + "Invalid order_by entry '#{i(invalid_entry)}'. Entry is not a queried metric or dimension." + }} + end + else + :ok + end + end + + @pattern_filter_operators [:matches, :matches_not, :matches_wildcard, :matches_wildcard_not] + defp validate_case_sensitive_filter_modifier(query) do + invalid_filter = + query.filters + |> Filters.all_leaf_filters() + |> Enum.find(fn filter -> + case filter do + [operator, _, _, modifiers] when operator in @pattern_filter_operators -> + is_map_key(modifiers, :case_insensitive) + + _ -> + false + end + end) + + if invalid_filter do + {:error, + %QueryError{ + code: :invalid_filters, + message: + "Invalid filters. The case_sensitive modifier is not allowed with pattern operators (#{i(invalid_filter)})" + }} + else + :ok + end + end + + @only_toplevel ["event:goal", "event:hostname"] + defp validate_toplevel_only_filter_dimension(query) do + not_toplevel = + query.filters + |> Filters.dimensions_used_in_filters(min_depth: 1, behavioral_filters: :ignore) + |> Enum.filter(&(&1 in @only_toplevel)) + + if Enum.count(not_toplevel) > 0 do + {:error, + %QueryError{ + code: :invalid_filters, + message: + "Invalid filters. Dimension `#{List.first(not_toplevel)}` can only be filtered at the top level." + }} + else + :ok + end + end + + @max_hours_for_minute_interval 30 + + defp validate_time_dimension_granularity(query) do + if Time.time_dimension(query) == "time:minute" and + DateTimeRange.length(query.utc_time_range, :minute) > @max_hours_for_minute_interval * 60 do + {:error, + %QueryError{ + code: :invalid_dimensions, + message: + "Invalid dimensions. Dimension `time:minute` is only supported for time ranges up to 30 hours." + }} + else + :ok + end + end + + @special_metrics [:conversion_rate, :group_conversion_rate] + defp validate_special_metrics_filters(query) do + special_metric? = Enum.any?(@special_metrics, &(&1 in query.metrics)) + + deep_custom_property? = + query.filters + |> Filters.dimensions_used_in_filters(min_depth: 1) + |> Enum.any?(fn dimension -> String.starts_with?(dimension, "event:props:") end) + + if special_metric? and deep_custom_property? do + {:error, + %QueryError{ + code: :invalid_filters, + message: + "Invalid filters. When `conversion_rate` or `group_conversion_rate` metrics are used, custom property filters can only be used on top level." + }} + else + :ok + end + end + + defp validate_behavioral_filters(query) do + query.filters + |> Filters.traverse(0, fn behavioral_depth, operator -> + if operator in [:has_done, :has_not_done] do + behavioral_depth + 1 + else + behavioral_depth + end + end) + |> Enum.reduce_while(:ok, fn {[_operator, dimension | _rest], behavioral_depth}, :ok -> + cond do + behavioral_depth == 0 -> + # ignore non-behavioral filters + {:cont, :ok} + + behavioral_depth > 1 -> + {:halt, + {:error, + %QueryError{ + code: :invalid_filters, + message: + "Invalid filters. Behavioral filters (has_done, has_not_done) cannot be nested." + }}} + + not String.starts_with?(dimension, "event:") -> + {:halt, + {:error, + %QueryError{ + code: :invalid_filters, + message: + "Invalid filters. Behavioral filters (has_done, has_not_done) can only be used with event dimension filters." + }}} + + true -> + {:cont, :ok} + end + end) + end + + defp validate_filtered_goals_exist(_query, %ParsedQueryParams{skip_goal_existence_check: true}), + do: :ok + + defp validate_filtered_goals_exist(query, %ParsedQueryParams{}) do + # Note: We don't check :contains goal filters since it's acceptable if they match nothing. + goal_filter_clauses = + query.filters + |> Filters.all_leaf_filters() + |> Enum.flat_map(fn + [:is, "event:goal", clauses] -> clauses + _ -> [] + end) + + if length(goal_filter_clauses) > 0 do + configured_goal_names = + query.preloaded_goals.all + |> Enum.map(&Plausible.Goal.display_name/1) + + validate_list(goal_filter_clauses, &validate_goal_filter(&1, configured_goal_names)) + else + :ok + end + end + + defp validate_goal_filter(clause, configured_goal_names) do + if Enum.member?(configured_goal_names, clause) do + :ok + else + {:error, + %QueryError{ + code: :invalid_filters, + message: + "Invalid filters. The goal `#{clause}` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals" + }} + end + end + + defp validate_custom_props_access(site, query) do + allowed_props = Plausible.Props.allowed_for(site, bypass_setup?: true) + + validate_custom_props_access(site, query, allowed_props) + end + + defp validate_custom_props_access(_site, _query, :all), do: :ok + + defp validate_custom_props_access(_site, query, allowed_props) do + valid? = + query.filters + |> Filters.dimensions_used_in_filters() + |> Enum.concat(query.dimensions) + |> Enum.all?(fn + "event:props:" <> prop -> prop in allowed_props + _ -> true + end) + + if valid? do + :ok + else + {:error, + %QueryError{ + code: :feature_access, + message: "The owner of this site does not have access to the custom properties feature." + }} + end + end + + defp validate_metrics(query) do + with :ok <- validate_list(query.metrics, &validate_metric(&1, query)) do + TableDecider.validate_no_metrics_dimensions_conflict(query) + end + end + + defp validate_metric(metric, query) when metric in [:conversion_rate, :group_conversion_rate] do + if Enum.member?(query.dimensions, "event:goal") or + Filters.filtering_on_dimension?(query, "event:goal", behavioral_filters: :ignore) do + :ok + else + {:error, + %QueryError{ + code: :invalid_metrics, + message: "Metric `#{metric}` can only be queried with event:goal filters or dimensions." + }} + end + end + + defp validate_metric(:scroll_depth = metric, query) do + page_dimension? = Enum.member?(query.dimensions, "event:page") + toplevel_page_filter? = not is_nil(Filters.get_toplevel_filter(query, "event:page")) + + if page_dimension? or toplevel_page_filter? do + :ok + else + {:error, + %QueryError{ + code: :invalid_metrics, + message: "Metric `#{metric}` can only be queried with event:page filters or dimensions." + }} + end + end + + defp validate_metric(:exit_rate = metric, query) do + case {Enum.sort(query.dimensions), TableDecider.sessions_join_events?(query)} do + {["visit:exit_page"], false} -> + :ok + + {["visit:exit_page", "visit:exit_page_hostname"], false} -> + :ok + + {["visit:exit_page"], true} -> + {:error, + %QueryError{ + code: :invalid_metrics, + message: "Metric `#{metric}` cannot be queried when filtering on event dimensions." + }} + + _ -> + {:error, + %QueryError{ + code: :invalid_metrics, + message: + "Metric `#{metric}` requires a `\"visit:exit_page\"` dimension. No other dimensions are allowed." + }} + end + end + + defp validate_metric(:views_per_visit = metric, query) do + cond do + Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore) -> + {:error, + %QueryError{ + code: :invalid_metrics, + message: "Metric `#{metric}` cannot be queried with a filter on `event:page`." + }} + + Enum.any?(query.dimensions, &(not Time.time_dimension?(&1))) -> + {:error, + %QueryError{ + code: :invalid_metrics, + message: "Metric `#{metric}` cannot be queried with non-time dimensions." + }} + + true -> + :ok + end + end + + defp validate_metric(:time_on_page = metric, query) do + cond do + Enum.member?(query.dimensions, "event:page") -> + :ok + + Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore) -> + :ok + + true -> + {:error, + %QueryError{ + code: :invalid_metrics, + message: + "Metric `#{metric}` can only be queried with event:page filters or dimensions." + }} + end + end + + defp validate_metric(_, _), do: :ok + + defp validate_include(query) do + time_dimension? = Enum.any?(query.dimensions, &Time.time_dimension?/1) + + if query.include.time_labels and not time_dimension? do + {:error, + %QueryError{ + code: :invalid_include, + message: "Invalid include.time_labels: requires a time dimension." + }} + else + :ok + end + end + + defp i(value), do: inspect(value, charlists: :as_lists) + + defp validate_list(list, parser_function) do + Enum.reduce_while(list, :ok, fn value, :ok -> + case parser_function.(value) do + :ok -> {:cont, :ok} + {:error, _} = error -> {:halt, error} + end + end) + end +end diff --git a/lib/plausible/stats/query_error.ex b/lib/plausible/stats/query_error.ex new file mode 100644 index 0000000000000000000000000000000000000000..3ec8a9e6ef1eca13329a272fe85b73be38284910 --- /dev/null +++ b/lib/plausible/stats/query_error.ex @@ -0,0 +1,4 @@ +defmodule Plausible.Stats.QueryError do + @moduledoc false + defstruct [:code, :message] +end diff --git a/lib/plausible/stats/query_include.ex b/lib/plausible/stats/query_include.ex new file mode 100644 index 0000000000000000000000000000000000000000..49f72637feebabe779778c65d6e0405def5f7e2e --- /dev/null +++ b/lib/plausible/stats/query_include.ex @@ -0,0 +1,45 @@ +defmodule Plausible.Stats.QueryInclude do + @moduledoc false + + defstruct imports: false, + imports_meta: false, + time_labels: false, + # `time_label_result_indices` is a convenience for our main graph component. It + # is not yet ready for a public API release because it should also account for + # breakdowns by multiple dimensions (time + non-time). Also, at this point it is + # still unclear whether `time_labels` will stay in the public API or not. + time_label_result_indices: false, + # Another flag to simplify frontend code by not having to repeat the logic defining + # default values for metrics (especially revenue metrics). + empty_metrics: false, + present_index: false, + partial_time_labels: false, + total_rows: false, + trim_relative_date_range: false, + compare: nil, + compare_match_day_of_week: false, + legacy_time_on_page_cutoff: nil, + drop_unavailable_time_on_page: false, + drop_unavailable_revenue_metrics: false + + @type date_range_tuple() :: {:date_range, Date.t(), Date.t()} + @type datetime_range_tuple() :: {:datetime_range, DateTime.t(), DateTime.t()} + + @type t() :: %__MODULE__{ + imports: boolean(), + imports_meta: boolean(), + time_labels: boolean(), + time_label_result_indices: boolean(), + empty_metrics: boolean(), + present_index: boolean(), + partial_time_labels: boolean(), + total_rows: boolean(), + trim_relative_date_range: boolean(), + compare: + nil | :previous_period | :year_over_year | date_range_tuple() | datetime_range_tuple(), + compare_match_day_of_week: boolean(), + legacy_time_on_page_cutoff: any(), + drop_unavailable_time_on_page: boolean(), + drop_unavailable_revenue_metrics: boolean() + } +end diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.ex new file mode 100644 index 0000000000000000000000000000000000000000..f72d01ed5bffea67db0ab2d4699bef8570f7ee38 --- /dev/null +++ b/lib/plausible/stats/query_optimizer.ex @@ -0,0 +1,355 @@ +defmodule Plausible.Stats.QueryOptimizer do + @moduledoc """ + Methods to manipulate Query for business logic reasons before building an ecto query. + """ + + use Plausible + + alias Plausible.Stats.{ + DateTimeRange, + Filters, + Query, + TableDecider, + TimeOnPage, + Util, + Time, + Legacy + } + + @doc """ + This module manipulates an existing query, updating it according to business logic. + + For example, it: + 1. Figures out what the right granularity to group by time is + 2. Adds a missing order_by clause to a query + 3. Updating "time" dimension in order_by to the right granularity + 4. Updates event:hostname filters to also apply on visit level for sane results. + 5. [DEPRECATED AND WILL BE REMOVED] Removes revenue metrics from legacy queries if ineligible + 6. Trims the date range to the current time if query.include.trim_relative_date_range is true. + 7. Sets the join_type for the query based on the query. + + """ + def optimize(query) do + Enum.reduce(pipeline(), query, fn step, acc -> step.(acc) end) + end + + @doc """ + Splits a query into event and sessions subcomponents as not all metrics can be + queried from a single table. + + event:page dimension is treated in a special way, doing a breakdown of visit:entry_page + for sessions. + """ + def split(query) do + query.metrics + |> Util.maybe_add_visitors_metric() + |> TableDecider.partition_metrics(query) + |> Enum.map(fn {table_type, metrics} -> + build_split_query(table_type, metrics, query) + end) + end + + defp pipeline() do + [ + &update_group_by_time/1, + &add_missing_order_by/1, + &update_time_in_order_by/1, + &extend_hostname_filters_to_visit/1, + &set_time_on_page_data/1, + &remove_time_on_page_if_unavailable/1, + &remove_revenue_metrics_if_unavailable/1, + &trim_relative_date_range/1, + &set_sql_join_type/1 + ] + end + + defp add_missing_order_by(%Query{order_by: nil} = query) do + order_by = + case time_dimension(query) do + nil -> [{hd(query.metrics), :desc}] + time_dimension -> [{time_dimension, :asc}, {hd(query.metrics), :desc}] + end + + %Query{query | order_by: order_by} + end + + defp add_missing_order_by(query), do: query + + defp update_group_by_time( + %Query{ + utc_time_range: %DateTimeRange{first: first, last: last} + } = query + ) do + dimensions = + query.dimensions + |> Enum.map(fn + "time" -> resolve_time_dimension(first, last) + entry -> entry + end) + + %Query{query | dimensions: dimensions} + end + + defp update_group_by_time(query), do: query + + defp resolve_time_dimension(first, last) do + cond do + DateTime.diff(last, first, :hour) <= 48 -> "time:hour" + DateTime.diff(last, first, :day) <= 40 -> "time:day" + Plausible.Times.diff(last, first, :week) <= 52 -> "time:week" + true -> "time:month" + end + end + + defp update_time_in_order_by(%Query{} = query) do + order_by = + query.order_by + |> Enum.map(fn + {"time", direction} -> {time_dimension(query), direction} + entry -> entry + end) + + %Query{query | order_by: order_by} + end + + @dimensions_hostname_map %{ + "visit:source" => "visit:entry_page_hostname", + "visit:entry_page" => "visit:entry_page_hostname", + "visit:utm_medium" => "visit:entry_page_hostname", + "visit:utm_source" => "visit:entry_page_hostname", + "visit:utm_campaign" => "visit:entry_page_hostname", + "visit:utm_content" => "visit:entry_page_hostname", + "visit:utm_term" => "visit:entry_page_hostname", + "visit:referrer" => "visit:entry_page_hostname", + "visit:exit_page" => "visit:exit_page_hostname" + } + + # To avoid showing referrers across hostnames when event:hostname + # filter is present for breakdowns, add entry/exit page hostname + # filters + defp extend_hostname_filters_to_visit(%Query{} = query) do + # Note: Only works since event:hostname is only allowed as a top level filter + hostname_filters = + query.filters + |> Enum.filter(fn [_operation, dimension | _rest] -> dimension == "event:hostname" end) + + if length(hostname_filters) > 0 do + extra_filters = + query.dimensions + |> Enum.flat_map(&hostname_filters_for_dimension(&1, hostname_filters)) + + %Query{query | filters: query.filters ++ extra_filters} + else + query + end + end + + defp hostname_filters_for_dimension(dimension, hostname_filters) do + if Map.has_key?(@dimensions_hostname_map, dimension) do + dimension = Map.get(@dimensions_hostname_map, dimension) + + hostname_filters + |> Enum.map(fn [operation, _dimension | rest] -> [operation, dimension | rest] end) + else + [] + end + end + + defp time_dimension(query) do + Enum.find(query.dimensions, &Time.time_dimension?/1) + end + + defp build_split_query(:events, metrics, query) do + { + :events, + Query.set(query, + metrics: metrics, + include_imported: query.include_imported + ) + } + end + + @event_page_dim_renames %{ + "event:page" => "visit:entry_page", + "event:hostname" => "visit:entry_page_hostname" + } + + defp build_split_query(:sessions, metrics, query) do + active_renames = Map.take(@event_page_dim_renames, query.dimensions) + + dimensions = Enum.map(query.dimensions, &Map.get(active_renames, &1, &1)) + + filters = + if map_size(active_renames) > 0 do + Filters.rename_dimensions_used_in_filter(query.filters, active_renames) + else + query.filters + end + + { + :sessions, + Query.set(query, + filters: filters, + metrics: metrics, + dimensions: dimensions, + include_imported: query.include_imported + ) + } + end + + defp build_split_query(:sessions_smeared, metrics, query) do + {_, query} = build_split_query(:sessions, metrics, query) + + {:sessions, Query.set(query, smear_session_metrics: true)} + end + + on_ee do + defp remove_revenue_metrics_if_unavailable(query) do + if query.include.drop_unavailable_revenue_metrics and + map_size(query.revenue_currencies) == 0 do + Query.set(query, metrics: query.metrics -- Plausible.Stats.Goal.Revenue.revenue_metrics()) + else + query + end + end + else + defp remove_revenue_metrics_if_unavailable(query), do: query + end + + # Unavailable in this context means not implemented. Imported data is ignored in the + # aggregate legacy time on page query, while the legacy breakdown query includes it. + # We drop the metric to avoid reporting different numbers in different reports. + defp remove_time_on_page_if_unavailable(query) do + if query.include.drop_unavailable_time_on_page and time_on_page_unavailable?(query) do + Query.set(query, metrics: query.metrics -- [:time_on_page]) + else + query + end + end + + defp time_on_page_unavailable?(%Query{ + include_imported: true, + dimensions: [], + time_on_page_data: %{include_legacy_metric: true} + }) do + true + end + + defp time_on_page_unavailable?(_), do: false + + defp set_time_on_page_data(query) do + case {:time_on_page in query.metrics, query.time_on_page_data} do + {true, %{new_metric_visible: true, cutoff_date: cutoff_date}} -> + cutoff = + cutoff_date + |> TimeOnPage.cutoff_datetime(query.timezone) + |> DateTime.shift_zone!("Etc/UTC") + |> DateTime.truncate(:second) + + Query.set( + query, + time_on_page_data: + Map.merge(query.time_on_page_data, %{ + include_new_metric: DateTime.before?(cutoff, query.utc_time_range.last), + include_legacy_metric: + DateTime.after?(cutoff, query.utc_time_range.first) and + Legacy.TimeOnPage.can_merge_legacy_time_on_page?(query), + cutoff: + if(DateTime.after?(cutoff, query.utc_time_range.first), do: cutoff, else: nil) + }) + ) + + _ -> + Query.set( + query, + time_on_page_data: + Map.merge(query.time_on_page_data, %{ + include_new_metric: false, + include_legacy_metric: true, + cutoff: nil + }) + ) + end + end + + defp trim_relative_date_range(%Query{include: %{trim_relative_date_range: true}} = query) do + # This is here to trim future bucket labels on the main graph + if should_trim_date_range?(query) do + trimmed_range = trim_date_range_to_now(query) + %Query{query | utc_time_range: trimmed_range} + else + query + end + end + + defp trim_relative_date_range(query), do: query + + defp should_trim_date_range?(%Query{input_date_range: :month} = query) do + today = query.now |> DateTime.shift_zone!(query.timezone) |> DateTime.to_date() + date_range = Query.date_range(query) + + current_month_start = Date.beginning_of_month(today) + current_month_end = Date.end_of_month(today) + + date_range.first == current_month_start and date_range.last == current_month_end + end + + defp should_trim_date_range?(%Query{input_date_range: :year} = query) do + today = query.now |> DateTime.shift_zone!(query.timezone) |> DateTime.to_date() + date_range = Query.date_range(query) + + current_year_start = Date.new!(today.year, 1, 1) + current_year_end = Date.new!(today.year, 12, 31) + + date_range.first == current_year_start and date_range.last == current_year_end + end + + defp should_trim_date_range?(%Query{input_date_range: :day} = query) do + today = query.now |> DateTime.shift_zone!(query.timezone) |> DateTime.to_date() + date_range = Query.date_range(query) + + is_nil(query.include.compare) and date_range.first == today and date_range.last == today + end + + defp should_trim_date_range?(_query), do: false + + defp trim_date_range_to_now(query) do + if query.input_date_range == :day do + time_range = query.utc_time_range |> DateTimeRange.to_timezone(query.timezone) + + current_hour = + query.now + |> DateTime.shift_zone!(query.timezone) + |> Map.merge(%{minute: 59, second: 59, millisecond: 999}) + + time_range.first + |> DateTimeRange.new!(current_hour) + |> DateTimeRange.to_timezone("Etc/UTC") + else + date_range = Query.date_range(query) + today = query.now |> DateTime.shift_zone!(query.timezone) |> DateTime.to_date() + + trimmed_to_date = + Enum.min([date_range.last, today], Date) + + date_range.first + |> DateTimeRange.new!(trimmed_to_date, query.timezone) + |> DateTimeRange.to_timezone("Etc/UTC") + end + end + + # Normally we can always LEFT JOIN as this is more performant and tables + # are expected to contain the same dimensions. + + # The only exception is using the "time:minute"/"time:hour" dimension where the sessions + # subquery might return more rows than the events one. That's because we're + # counting sessions in all time buckets they were active in even if no event + # occurred during that particular bucket. + defp set_sql_join_type(query) do + if "time:minute" in query.dimensions or "time:hour" in query.dimensions do + Query.set(query, sql_join_type: :full) + else + query + end + end +end diff --git a/lib/plausible/stats/query_period.ex b/lib/plausible/stats/query_period.ex new file mode 100644 index 0000000000000000000000000000000000000000..4b06b10ba5ab2b8541bf1ea9fee144c83140ebda --- /dev/null +++ b/lib/plausible/stats/query_period.ex @@ -0,0 +1,228 @@ +defmodule Plausible.Stats.QueryPeriod do + @moduledoc """ + Builds the time range covered by a stats query from parsed parameters. + """ + + alias Plausible.Stats.DateTimeRange + alias Plausible.Times + + @doc """ + Resolves an `input_date_range` for the site and builds the corresponding + `%DateTimeRange{}` anchored to the site's timezone. + + `relative_date` may be `nil`, in which case it defaults to today in the + site's timezone. + """ + def build_range_for_site(input_date_range, %Plausible.Site{} = site, relative_date, now) do + relative_date = relative_date || Times.to_date(now, site.timezone) + + input_date_range + |> resolve_input_date_range(site, relative_date) + |> build_datetime_range(site.timezone, relative_date, now) + end + + @doc """ + For some query periods, `now` or `relative_date` or both are irrelevant + for resolving the date range. This dispatcher ensures that + """ + def build_datetime_range(:realtime, timezone, _relative_date, now), + do: build_realtime_range(timezone, now, 5) + + def build_datetime_range(:realtime_30m, timezone, _relative_date, now), + do: build_realtime_range(timezone, now, 30) + + def build_datetime_range(:day, timezone, relative_date, now), + do: build_day_range(timezone, relative_date, now) + + def build_datetime_range(:"24h", timezone, _relative_date, now), + do: build_24h_range(timezone, now) + + def build_datetime_range(:month, timezone, relative_date, _now), + do: build_month_range(timezone, relative_date) + + def build_datetime_range(:year, timezone, relative_date, _now), + do: build_year_range(timezone, relative_date) + + def build_datetime_range({:last_n_days, n}, timezone, relative_date, _now), + do: build_last_n_days_range(timezone, n, relative_date) + + def build_datetime_range({:last_n_months, n}, timezone, relative_date, _now), + do: build_last_n_months_range(timezone, n, relative_date) + + def build_datetime_range({:date_range, from, to}, timezone, _relative_date, _now), + do: build_date_range(timezone, from, to) + + def build_datetime_range({:datetime_range, from, to}, timezone, _relative_date, _now), + do: build_range_from_datetimes(timezone, from, to) + + @doc """ + Resolves any site-dependent shape (currently only `:all`, which needs the + site's stats start date) into a shape `build_datetime_range/4` can handle + from a timezone alone. Other shapes pass through unchanged. + """ + def resolve_input_date_range(:all, %Plausible.Site{} = site, relative_date) do + start_date = Plausible.Sites.stats_start_date(site) || relative_date + {:date_range, start_date, relative_date} + end + + def resolve_input_date_range(input_date_range, _site, _relative_date), + do: input_date_range + + @doc """ + Builds a realtime window of `duration_minutes` ending slightly after `now`. + + ## Examples + + iex> QueryPeriod.build_realtime_range("Etc/UTC", ~U[2026-05-05 12:30:00Z], 5) + %DateTimeRange{first: ~U[2026-05-05 12:25:00Z], last: ~U[2026-05-05 12:30:05Z]} + + iex> QueryPeriod.build_realtime_range("Etc/UTC", ~U[2026-05-05 12:30:00Z], 30) + %DateTimeRange{first: ~U[2026-05-05 12:00:00Z], last: ~U[2026-05-05 12:30:05Z]} + """ + def build_realtime_range(timezone, now, duration_minutes) do + first_datetime = DateTime.shift(now, minute: -duration_minutes) + last_datetime = DateTime.shift(now, second: 5) + + DateTimeRange.new!(first_datetime, last_datetime) + |> DateTimeRange.to_timezone(timezone) + end + + @doc """ + Builds the range for a single calendar day. When `date` matches today in + `timezone`, the range is truncated at `now`; otherwise it spans the full + local day. + + ## Examples + + iex> QueryPeriod.build_day_range("Etc/UTC", ~D[2026-05-05], ~U[2026-05-05 12:30:00Z]) + %DateTimeRange{first: ~U[2026-05-05 00:00:00Z], last: ~U[2026-05-05 12:30:00Z]} + + iex> QueryPeriod.build_day_range("Etc/UTC", ~D[2026-05-04], ~U[2999-01-01 00:00:00Z]) + %DateTimeRange{first: ~U[2026-05-04 00:00:00Z], last: ~U[2026-05-04 23:59:59Z]} + + iex> QueryPeriod.build_day_range("Europe/Tallinn", ~D[2026-01-15], ~U[2999-01-01 00:00:00Z]) |> DateTimeRange.to_timezone("Etc/UTC") + %DateTimeRange{first: ~U[2026-01-14 22:00:00Z], last: ~U[2026-01-15 21:59:59Z]} + """ + def build_day_range(timezone, date, now) do + if Date.compare(Times.to_date(now, timezone), date) == :eq do + DateTimeRange.new!(date, now, timezone) + else + DateTimeRange.new!(date, date, timezone) + end + end + + @doc """ + Builds the 24-hour window ending at `now`. + + ## Examples + + iex> QueryPeriod.build_24h_range("Etc/UTC", ~U[2026-05-05 12:30:00Z]) + %DateTimeRange{first: ~U[2026-05-04 12:30:00Z], last: ~U[2026-05-05 12:30:00Z]} + """ + def build_24h_range(timezone, now) do + from = DateTime.shift(now, hour: -24) + + DateTimeRange.new!(from, now) + |> DateTimeRange.to_timezone(timezone) + end + + @doc """ + Builds the range spanning the calendar month containing `date`. + + ## Examples + + iex> QueryPeriod.build_month_range("Etc/UTC", ~D[2026-05-15]) + %DateTimeRange{first: ~U[2026-05-01 00:00:00Z], last: ~U[2026-05-31 23:59:59Z]} + + iex> QueryPeriod.build_month_range("Europe/Tallinn", ~D[2026-01-15]) |> DateTimeRange.to_timezone("Etc/UTC") + %DateTimeRange{first: ~U[2025-12-31 22:00:00Z], last: ~U[2026-01-31 21:59:59Z]} + """ + def build_month_range(timezone, date) do + first = Date.beginning_of_month(date) + last = Date.end_of_month(date) + DateTimeRange.new!(first, last, timezone) + end + + @doc """ + Builds the range spanning the calendar year containing `date`. + + ## Examples + + iex> QueryPeriod.build_year_range("Etc/UTC", ~D[2026-05-15]) + %DateTimeRange{first: ~U[2026-01-01 00:00:00Z], last: ~U[2026-12-31 23:59:59Z]} + + iex> QueryPeriod.build_year_range("Europe/Tallinn", ~D[2026-05-15]) |> DateTimeRange.to_timezone("Etc/UTC") + %DateTimeRange{first: ~U[2025-12-31 22:00:00Z], last: ~U[2026-12-31 21:59:59Z]} + """ + def build_year_range(timezone, date) do + first = Times.beginning_of_year(date) + last = Times.end_of_year(date) + DateTimeRange.new!(first, last, timezone) + end + + @doc """ + Builds a range spanning `n` full calendar days ending the day before + `end_date`. + + ## Examples + + iex> QueryPeriod.build_last_n_days_range("Etc/UTC", 7, ~D[2026-05-08]) + %DateTimeRange{first: ~U[2026-05-01 00:00:00Z], last: ~U[2026-05-07 23:59:59Z]} + + iex> QueryPeriod.build_last_n_days_range("Europe/Tallinn", 7, ~D[2026-05-08]) |> DateTimeRange.to_timezone("Etc/UTC") + %DateTimeRange{first: ~U[2026-04-30 21:00:00Z], last: ~U[2026-05-07 20:59:59Z]} + """ + def build_last_n_days_range(timezone, n, end_date) do + last = Date.add(end_date, -1) + first = Date.add(end_date, -n) + DateTimeRange.new!(first, last, timezone) + end + + @doc """ + Builds a range spanning `n` full calendar months ending the month + before `end_date`'s month. + + ## Examples + + iex> QueryPeriod.build_last_n_months_range("Etc/UTC", 3, ~D[2026-05-15]) + %DateTimeRange{first: ~U[2026-02-01 00:00:00Z], last: ~U[2026-04-30 23:59:59Z]} + + iex> QueryPeriod.build_last_n_months_range("Europe/Tallinn", 3, ~D[2026-05-15]) |> DateTimeRange.to_timezone("Etc/UTC") + %DateTimeRange{first: ~U[2026-01-31 22:00:00Z], last: ~U[2026-04-30 20:59:59Z]} + """ + def build_last_n_months_range(timezone, n, end_date) do + last = end_date |> Date.shift(month: -1) |> Date.end_of_month() + first = end_date |> Date.shift(month: -n) |> Date.beginning_of_month() + DateTimeRange.new!(first, last, timezone) + end + + @doc """ + Builds a range from explicit start/end calendar dates, + from start of first day to end of last day in `timezone`. + + ## Examples + + iex> QueryPeriod.build_date_range("Etc/UTC", ~D[2026-05-01], ~D[2026-05-10]) + %DateTimeRange{first: ~U[2026-05-01 00:00:00Z], last: ~U[2026-05-10 23:59:59Z]} + + iex> QueryPeriod.build_date_range("Europe/Tallinn", ~D[2026-05-01], ~D[2026-05-10]) |> DateTimeRange.to_timezone("Etc/UTC") + %DateTimeRange{first: ~U[2026-04-30 21:00:00Z], last: ~U[2026-05-10 20:59:59Z]} + """ + def build_date_range(timezone, from, to) do + DateTimeRange.new!(from, to, timezone) + end + + @doc """ + Builds a range from explicit start/end datetimes, which may have any time zone, + then shifts them to be in `timezone`. + + ## Examples + + iex> QueryPeriod.build_range_from_datetimes("Etc/UTC", ~U[2026-05-01 06:00:00Z], ~U[2026-05-01 18:00:00Z]) + %DateTimeRange{first: ~U[2026-05-01 06:00:00Z], last: ~U[2026-05-01 18:00:00Z]} + """ + def build_range_from_datetimes(timezone, from, to) do + DateTimeRange.new!(from, to) + |> DateTimeRange.to_timezone(timezone) + end +end diff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.ex new file mode 100644 index 0000000000000000000000000000000000000000..6f16b760c9a7546c5e0c1ab660c1c16dde25e70f --- /dev/null +++ b/lib/plausible/stats/query_result.ex @@ -0,0 +1,352 @@ +defmodule Plausible.Stats.QueryResult do + @moduledoc """ + This struct contains the (JSON-encodable) response for a query and + is responsible for building it from database query results. + + For the convenience of API docs and consumers, the JSON result + produced by Jason.encode(query_result) is ordered. + """ + + use Plausible + alias Plausible.Stats.{Query, QueryRunner, Filters} + + defstruct results: [], + comparison_results: nil, + meta: %{}, + query: nil + + @imports_warnings %{ + unsupported_query: + "Imported stats are not included in the results because query parameters are not supported. " <> + "For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats", + unsupported_interval: + "Imported stats are not included because the time dimension (i.e. the interval) is too short." + } + + def imports_warnings(), do: @imports_warnings + + @no_imported_scroll_depth_warning %{ + code: :no_imported_scroll_depth, + warning: "No imports with scroll depth data were found" + } + + def no_imported_scroll_depth_warning(), do: @no_imported_scroll_depth_warning + + @no_imported_bounce_rate_warning %{ + code: :no_imported_bounce_rate, + warning: "imported bounce_rate is not available when using a page filter" + } + + def no_imported_bounce_rate_warning(), do: @no_imported_bounce_rate_warning + + @doc """ + Builds full JSON-serializable query response. + + `results` should already-built by Plausible.Stats.QueryRunner + """ + def from(%QueryRunner{results: results, comparison_results: comparison_results} = runner) do + struct!( + __MODULE__, + results: results, + comparison_results: comparison_results, + meta: meta(runner) |> Jason.OrderedObject.new(), + query: query(runner) |> Jason.OrderedObject.new() + ) + end + + defp meta(%QueryRunner{} = runner) do + %{} + |> add_imports_meta(runner.main_query) + |> add_metric_warnings_meta(runner.main_query) + |> add_empty_metrics_meta(runner.main_query) + |> add_time_labels_meta(runner) + |> add_time_labels_result_indices_meta(runner) + |> add_comparison_time_labels_meta(runner) + |> add_comparison_time_label_result_indices_meta(runner) + |> add_present_index_meta(runner.main_query) + |> add_partial_time_labels_meta(runner.main_query) + |> add_comparison_partial_time_labels_meta(runner) + |> add_total_rows_meta(runner.main_query, runner.total_rows) + |> Enum.sort_by(&elem(&1, 0)) + end + + defp add_imports_meta(meta, %Query{include: include} = query) do + if include.imports or include.imports_meta do + %{ + imports_included: query.include_imported, + imports_skip_reason: query.skip_imported_reason, + imports_warning: @imports_warnings[query.skip_imported_reason] + } + |> Map.reject(fn {_key, value} -> is_nil(value) end) + |> Map.merge(meta) + else + meta + end + end + + defp add_metric_warnings_meta(meta, query) do + warnings = metric_warnings(query) + + if map_size(warnings) > 0 do + Map.put(meta, :metric_warnings, warnings) + else + meta + end + end + + defp add_empty_metrics_meta(meta, query) do + if query.include.empty_metrics and "event:goal" not in query.dimensions do + Map.put( + meta, + :empty_metrics, + Enum.map(query.metrics, &Plausible.Stats.Metrics.default_value(&1, query)) + ) + else + meta + end + end + + defp add_time_labels_meta(meta, %QueryRunner{main_query: query}) do + if query.include.time_labels do + Map.put(meta, :time_labels, Plausible.Stats.Time.time_labels(query)) + else + meta + end + end + + defp add_comparison_time_labels_meta(meta, %QueryRunner{main_query: query} = runner) do + if query.include.time_labels && query.include.compare do + Map.put( + meta, + :comparison_time_labels, + Plausible.Stats.Time.time_labels(runner.comparison_query) + ) + else + meta + end + end + + defp add_time_labels_result_indices_meta(meta, %QueryRunner{main_query: query} = runner) do + time_labels = meta[:time_labels] + + if query.include.time_label_result_indices and is_list(time_labels) do + Map.put( + meta, + :time_label_result_indices, + result_indices_for_time_labels(time_labels, runner.main_results) + ) + else + meta + end + end + + defp add_comparison_time_label_result_indices_meta( + meta, + %QueryRunner{main_query: query} = runner + ) do + comp_time_labels = meta[:comparison_time_labels] + + if query.include.time_label_result_indices and is_list(comp_time_labels) do + Map.put( + meta, + :comparison_time_label_result_indices, + result_indices_for_time_labels(comp_time_labels, runner.comparison_results) + ) + else + meta + end + end + + defp add_present_index_meta(meta, query) do + time_labels = meta[:time_labels] + + if query.include.present_index and is_list(time_labels) do + Map.put(meta, :present_index, Plausible.Stats.Time.present_index(time_labels, query)) + else + meta + end + end + + defp add_partial_time_labels_meta(meta, query) do + time_labels = meta[:time_labels] + + if query.include.partial_time_labels and is_list(time_labels) do + Map.put( + meta, + :partial_time_labels, + Plausible.Stats.Time.partial_time_labels(time_labels, query) + ) + else + meta + end + end + + defp add_comparison_partial_time_labels_meta(meta, %QueryRunner{main_query: query} = runner) do + comparison_time_labels = meta[:comparison_time_labels] + + if query.include.partial_time_labels and is_list(comparison_time_labels) do + Map.put( + meta, + :comparison_partial_time_labels, + Plausible.Stats.Time.partial_time_labels(comparison_time_labels, runner.comparison_query) + ) + else + meta + end + end + + defp add_total_rows_meta(meta, query, total_rows) do + if query.include.total_rows do + Map.put(meta, :total_rows, total_rows) + else + meta + end + end + + defp query(%QueryRunner{site: site, main_query: query}) do + [ + site_id: site.domain, + metrics: query.metrics, + date_range: [ + to_iso8601(query.utc_time_range.first, query.timezone), + to_iso8601(query.utc_time_range.last, query.timezone) + ], + comparison_date_range: + if(query.include.compare, + do: [ + to_iso8601(query.comparison_utc_time_range.first, query.timezone), + to_iso8601(query.comparison_utc_time_range.last, query.timezone) + ] + ), + filters: query.filters, + dimensions: query.dimensions, + order_by: query.order_by |> Enum.map(&Tuple.to_list/1), + include: include(query) |> Map.filter(fn {_key, val} -> val end), + pagination: query.pagination + ] + |> Enum.reject(fn {_k, v} -> is_nil(v) end) + end + + defp include(query) do + case query.include.compare do + {:date_range, first, last} -> + struct!(query.include, compare: [Date.to_iso8601(first), Date.to_iso8601(last)]) + + _ -> + query.include + end + |> Map.from_struct() + end + + defp metric_warnings(%Query{} = query) do + Enum.reduce(query.metrics, %{}, fn metric, acc -> + case metric_warning(metric, query) do + nil -> acc + %{} = warning -> Map.put(acc, metric, warning) + end + end) + end + + on_ee do + @revenue_metrics Plausible.Stats.Goal.Revenue.revenue_metrics() + + @revenue_metrics_warnings %{ + revenue_goals_unavailable: + "The owner of this site does not have access to the revenue metrics feature.", + no_single_revenue_currency: + "Revenue metrics are null as there are multiple currencies for the selected event:goals.", + no_revenue_goals_matching: + "Revenue metrics are null as there are no matching revenue goals." + } + + defp metric_warning(metric, %Query{} = query) + when metric in @revenue_metrics do + if query.revenue_warning do + %{ + code: query.revenue_warning, + warning: @revenue_metrics_warnings[query.revenue_warning] + } + else + nil + end + end + end + + defp metric_warning(:scroll_depth, %Query{} = query) do + if query.include_imported and not Enum.any?(query.imports_in_range, & &1.has_scroll_depth) do + @no_imported_scroll_depth_warning + end + end + + defp metric_warning(:time_on_page, %Query{} = query) do + case query.time_on_page_data do + %{new_metric_visible: true, include_legacy_metric: true, cutoff: cutoff} -> + cutoff_date = + cutoff |> DateTime.shift_zone!(query.timezone) |> Calendar.strftime("%Y-%m-%d") + + %{ + code: :legacy_time_on_page_used, + message: + "This period includes data calculated with the legacy time on page method up to #{cutoff_date}" + } + + _ -> + nil + end + end + + # Native queries (i.e. ones that don't include imported data) allow querying bounce rate + # with an `event:page` filter or dimension. In those cases, an `event:page` gets treated + # as `visit:entry_page`. While theoretically possible, this behaviour does not yet exist + # for imported data, which is why we're returning a metric warning here. + defp metric_warning(:bounce_rate, %Query{} = query) do + page_filter_or_dimension? = + Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore) or + "event:page" in query.dimensions + + if query.include_imported and page_filter_or_dimension? do + @no_imported_bounce_rate_warning + end + end + + defp metric_warning(_metric, _query), do: nil + + defp result_indices_for_time_labels(time_labels, results_list) do + index_lookup_map = + results_list + |> Enum.with_index() + |> Map.new(fn {%{dimensions: [dim]}, idx} -> {dim, idx} end) + + Enum.map(time_labels, &Map.get(index_lookup_map, &1)) + end + + defp to_iso8601(datetime, timezone) do + datetime + |> DateTime.shift_zone!(timezone) + |> DateTime.to_iso8601(:extended) + end +end + +defimpl Jason.Encoder, for: Plausible.Stats.QueryResult do + def encode( + %Plausible.Stats.QueryResult{ + results: results, + comparison_results: comparison_results, + meta: meta, + query: query + }, + opts + ) do + if comparison_results do + Jason.OrderedObject.new( + results: results, + comparison_results: comparison_results, + meta: meta, + query: query + ) + else + Jason.OrderedObject.new(results: results, meta: meta, query: query) + end + |> Jason.Encoder.encode(opts) + end +end diff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.ex new file mode 100644 index 0000000000000000000000000000000000000000..62628909921844576e22551abed3de9b7f59c0ff --- /dev/null +++ b/lib/plausible/stats/query_runner.ex @@ -0,0 +1,265 @@ +defmodule Plausible.Stats.QueryRunner do + @moduledoc """ + This module is responsible for executing a Plausible.Stats.Query + and gathering results. + + Some secondary responsibilities are: + 1. Dealing with comparison queries and combining results with main + 2. Dealing with time-on-page + 3. Getting total_rows from ClickHouse results + """ + + use Plausible + use Plausible.ClickhouseRepo + + alias Plausible.Stats.{ + Comparisons, + Compare, + Query, + QueryOptimizer, + QueryResult, + Metrics, + SQL, + Util, + Time + } + + defstruct [ + :site, + :main_query, + :main_results, + :comparison_query, + :comparison_results, + :total_rows, + :results + ] + + def run(site, query) do + optimized_query = QueryOptimizer.optimize(query) + + Query.trace(optimized_query, optimized_query.metrics) + + %__MODULE__{main_query: optimized_query, site: site} + |> execute_main_query() + |> add_comparison_query() + |> execute_comparison_query() + |> build_results_list() + |> QueryResult.from() + end + + defp execute_main_query(%__MODULE__{main_query: query, site: site} = runner) do + ch_results = execute_query(query, site) + + main_results = build_from_ch(ch_results, query) + + runner = struct!(runner, main_results: main_results) + + if query.include.total_rows do + struct!(runner, total_rows: total_rows(ch_results)) + else + runner + end + end + + defp add_comparison_query(%__MODULE__{main_query: query, main_results: main_results} = runner) + when not is_nil(query.include.compare) do + comparison_query = + query + |> Comparisons.get_comparison_query() + |> Comparisons.add_comparison_filters(main_results) + |> QueryOptimizer.optimize() + + struct!(runner, comparison_query: comparison_query) + end + + defp add_comparison_query(runner), do: runner + + defp execute_comparison_query( + %__MODULE__{comparison_query: comparison_query, site: site} = runner + ) do + if comparison_query do + ch_results = execute_query(comparison_query, site) + comparison_results = build_from_ch(ch_results, comparison_query) + + struct!(runner, comparison_results: comparison_results) + else + runner + end + end + + # Assembles the final results, optionally attaching comparison data. + # + # Without a comparison, main results are returned as-is and comparison_results + # is nil. + # + # With comparisons, timeseries and non-time-dimension breakdowns are handled + # separately because they have fundamentally different shapes: + # + # - Non-time breakdowns (e.g. by page, source) return one row per dimension + # group. The comparison query is filtered to the same set of dimension + # values as the main query, so every comparison result is guaranteed to + # have a matching main result. Comparison data is merged inline into each + # result row; comparison_results is nil. + # + # - Timeseries (single "time:*" dimension) keep results and comparison_results + # as separate lists of only non-empty rows. Each comparison row carries a + # `change` field computed against the positionally-aligned original bucket + # (or nil when there is no corresponding original bucket). + defp build_results_list(%__MODULE__{main_query: query, main_results: main_results} = runner) do + case {query.include.compare, query.dimensions} do + {nil, _dimensions} -> + struct!(runner, + results: main_results, + comparison_results: nil + ) + + {_non_nil_compare, ["time:" <> _]} -> + struct!(runner, + results: main_results, + comparison_results: build_comparison_results(runner) + ) + + {_non_nil_compare, _dimensions} -> + struct!(runner, + results: merge_with_comparison_results(main_results, runner), + comparison_results: nil + ) + end + end + + defp build_comparison_results(%__MODULE__{main_query: query} = runner) do + main_map = index_by_dimensions(runner.main_results) + + comp_label_to_main_label = + Enum.zip(Time.time_labels(runner.comparison_query), Time.time_labels(query)) + |> Map.new() + + Enum.map(runner.comparison_results, fn %{dimensions: [comp_label]} = comp_row -> + main_label = Map.get(comp_label_to_main_label, comp_label) + main_metrics = main_label && Map.get(main_map, [main_label]) + change = calculate_metric_changes(query, main_metrics, comp_row.metrics) + + Map.put(comp_row, :change, change) + end) + end + + defp execute_query(query, site) do + query + |> SQL.QueryBuilder.build(site) + |> ClickhouseRepo.all(query: query) + end + + defp build_from_ch(ch_results, query) do + ch_results + |> Enum.map(fn entry -> + dimension_labels = Enum.map(query.dimensions, &dimension_label(&1, entry, query)) + + %{ + dimensions: dimension_labels, + metrics: Enum.map(query.metrics, &get_metric(entry, &1, dimension_labels, query)) + } + end) + end + + defp dimension_label("event:goal", entry, query) do + get_dimension_goal(entry, query) + |> Plausible.Goal.display_name() + end + + defp dimension_label("time:" <> _ = time_dimension, entry, query) do + datetime = Map.get(entry, Util.shortname(query, time_dimension)) + + Time.format_datetime(datetime) + end + + defp dimension_label(dimension, entry, query) do + Map.get(entry, Util.shortname(query, dimension)) + end + + on_ee do + defp get_metric(entry, metric, dimensions, query) + when metric in [:average_revenue, :total_revenue] do + value = Map.get(entry, metric) + + Plausible.Stats.Goal.Revenue.format_revenue_metric(value, query, dimensions) + end + end + + defp get_metric(entry, :events, _dimensions, query) do + cond do + "event:goal" in query.dimensions -> + goal = get_dimension_goal(entry, query) + + if Plausible.Goal.type(goal) != :scroll do + Map.get(entry, :events) + else + nil + end + + # Cannot show aggregate when there are at least some scroll goal filters + Plausible.Stats.Goals.toplevel_scroll_goal_filters?(query) -> + nil + + true -> + Map.get(entry, :events) + end + end + + defp get_metric(entry, metric, _dimensions, _query), do: Map.get(entry, metric) + + defp get_dimension_goal(entry, query) do + goal_index = Map.get(entry, Util.shortname(query, "event:goal")) + + query.preloaded_goals.matching_toplevel_filters + |> Enum.at(goal_index - 1) + end + + defp merge_with_comparison_results(results_list, runner) do + comparison_map = index_by_dimensions(runner.comparison_results) + Enum.map(results_list, &add_comparison_results(&1, runner.main_query, comparison_map)) + end + + defp add_comparison_results(row, query, comparison_map) do + comparison_metrics = metrics_for_dimension_group(comparison_map, row.dimensions, query) + + change = + Enum.zip([query.metrics, row.metrics, comparison_metrics]) + |> Enum.map(fn {metric, main_value, comp_value} -> + Compare.calculate_change(metric, comp_value, main_value) + end) + + Map.merge(row, %{ + comparison: %{ + dimensions: row.dimensions, + metrics: comparison_metrics, + change: change + } + }) + end + + defp index_by_dimensions(results_list) do + results_list + |> Map.new(fn entry -> {entry.dimensions, entry.metrics} end) + end + + defp metrics_for_dimension_group(lookup_map, dimensions, query) do + Map.get_lazy(lookup_map, dimensions, fn -> empty_metrics(query, dimensions) end) + end + + defp empty_metrics(query, dimensions) do + query.metrics + |> Enum.map(fn metric -> Metrics.default_value(metric, query, dimensions) end) + end + + defp calculate_metric_changes(query, main_metrics, comparison_metrics) do + if main_metrics do + Enum.zip([query.metrics, main_metrics, comparison_metrics]) + |> Enum.map(fn {metric, main_value, comp_value} -> + Compare.calculate_change(metric, comp_value, main_value) + end) + end + end + + defp total_rows([]), do: 0 + defp total_rows([first_row | _rest]), do: first_row.total_rows +end diff --git a/lib/plausible/stats/sparkline.ex b/lib/plausible/stats/sparkline.ex new file mode 100644 index 0000000000000000000000000000000000000000..b3527230e97e2d81d5d048c36f1934ebe4d83b96 --- /dev/null +++ b/lib/plausible/stats/sparkline.ex @@ -0,0 +1,166 @@ +defmodule Plausible.Stats.Sparkline do + @moduledoc """ + Module used to generate sparkline overview data for sites, including consolidated views. + The sparkline graphs are used in site index and in CRM. + """ + + alias Plausible.{Site, Stats} + alias Plausible.Stats.QueryBuilder + + @spec parallel_overview([Site.t()], NaiveDateTime.t()) :: map() + def parallel_overview(sites, now \\ NaiveDateTime.utc_now()) when is_list(sites) do + Task.async_stream( + sites, + fn site -> + {site.domain, safe_overview_24h(site, now)} + end, + timeout: 5000, + on_timeout: :kill_task, + max_concurrency: 4 + ) + |> Enum.reduce(%{}, fn + {:ok, {domain, {:ok, stats}}}, acc -> + Map.put(acc, domain, stats) + + _, acc -> + acc + end) + end + + @spec overview_24h(Site.t(), NaiveDateTime.t()) :: map() + def overview_24h(view_or_site, now \\ NaiveDateTime.utc_now()) do + stats = query_24h_stats(view_or_site, now) + intervals = query_24h_intervals(view_or_site, now) + + Map.merge(stats, intervals) + end + + @spec safe_overview_24h(Site.t(), NaiveDateTime.t()) :: + {:ok, map()} | {:error, :inaccessible} | {:error, :not_found} + + def safe_overview_24h(site, now \\ NaiveDateTime.utc_now()) + def safe_overview_24h(nil, _now), do: {:error, :not_found} + + def safe_overview_24h(%Site{} = site, now) do + try do + {:ok, overview_24h(site, now)} + catch + kind, reason -> + message = Exception.format(kind, reason, __STACKTRACE__) + + Sentry.capture_message( + "Could not render overview 24h - site will be kept in loading indefinitely", + extra: %{ + error: message, + site: site.domain, + comment: "Probing error volume" + } + ) + + {:error, :inaccessible} + end + end + + def empty_24h_intervals(now \\ NaiveDateTime.utc_now()) do + first = NaiveDateTime.add(now, -24, :hour) + {:ok, time} = Time.new(first.hour, 0, 0) + first = NaiveDateTime.new!(NaiveDateTime.to_date(first), time) + + for offset <- 0..24 do + %{interval: to_string(NaiveDateTime.add(first, offset, :hour)), visitors: 0} + end + end + + defp query_24h_stats(view_or_site, now) do + if Plausible.Sites.regular?(view_or_site) do + query_24h_stats_regular(view_or_site, now) + else + query_24h_stats_consolidated(view_or_site, now) + end + end + + defp query_24h_stats_regular(site, now) do + stats_query = + QueryBuilder.build!(site, + now: DateTime.from_naive!(now, "Etc/UTC"), + input_date_range: :"24h", + metrics: [:visitors], + include: [compare: :previous_period] + ) + + %Stats.QueryResult{ + results: [ + %{ + metrics: [visitors], + comparison: %{ + change: [visitors_change] + } + } + ] + } = Stats.query(site, stats_query) + + %{ + visitors: visitors, + visitors_change: visitors_change + } + end + + defp query_24h_stats_consolidated(view, now) do + stats_query = + QueryBuilder.build!(view, + now: DateTime.from_naive!(now, "Etc/UTC"), + input_date_range: :"24h", + metrics: [:visitors, :visits, :pageviews, :views_per_visit], + include: [compare: :previous_period] + ) + + %Stats.QueryResult{ + results: [ + %{ + metrics: [visitors, visits, pageviews, views_per_visit], + comparison: %{ + change: [visitors_change, visits_change, pageviews_change, views_per_visit_change] + } + } + ] + } = Stats.query(view, stats_query) + + %{ + visitors: visitors, + visits: visits, + pageviews: pageviews, + views_per_visit: views_per_visit, + visitors_change: visitors_change, + visits_change: visits_change, + pageviews_change: pageviews_change, + views_per_visit_change: views_per_visit_change + } + end + + defp query_24h_intervals(view_or_site, now) do + graph_query = + QueryBuilder.build!(view_or_site, + now: DateTime.from_naive!(now, "Etc/UTC"), + metrics: [:visitors], + input_date_range: :"24h", + dimensions: ["time:hour"], + order_by: [{"time:hour", :asc}], + include: [time_labels: true] + ) + + %Stats.QueryResult{results: results, meta: %{values: [time_labels: time_labels]}} = + Stats.query(view_or_site, graph_query) + + visitors_by_timestamp = + Map.new(results, fn %{dimensions: [timestamp], metrics: [visitors]} -> + {timestamp, visitors} + end) + + %{ + intervals: + Enum.map(time_labels, fn timestamp -> + %{interval: timestamp, visitors: Map.get(visitors_by_timestamp, timestamp, 0)} + end) + } + end +end diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex new file mode 100644 index 0000000000000000000000000000000000000000..a5c433f7da15f2d8aa2e05fe1f4bf17e6928dd26 --- /dev/null +++ b/lib/plausible/stats/sql/expression.ex @@ -0,0 +1,583 @@ +defmodule Plausible.Stats.SQL.Expression do + @moduledoc """ + This module is responsible for generating SQL/Ecto expressions + for dimensions and metrics used in query SELECT statement. + + Each dimension and metric is tagged with with selected_as for easier + usage down the line. + """ + + use Plausible + use Plausible.Stats.SQL.Fragments + + import Ecto.Query + + alias Plausible.Stats.{Query, Filters, SQL, Time} + + @no_ref "Direct / None" + @no_channel "Direct" + @not_set "(not set)" + + defmacrop field_or_blank_value(q, key, expr, empty_value) do + quote do + select_merge_as(unquote(q), [t], %{ + unquote(key) => + fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)) + }) + end + end + + defmacrop time_slots(query, period_in_seconds, first, last) do + quote do + fragment( + """ + timeSlots( + toTimeZone(greatest(?, ?), ?), + toUInt32(timeDiff(greatest(?, ?), least(?, ?))), + toUInt32(?) + ) + """, + s.start, + ^unquote(first), + ^unquote(query).timezone, + s.start, + ^unquote(first), + s.timestamp, + ^unquote(last), + ^unquote(period_in_seconds) + ) + end + end + + def select_dimension(q, key, "time:month", :sessions, query) do + {_first, last_datetime} = Time.utc_boundaries(query) + + select_merge_as(q, [t], %{ + key => + fragment( + "toStartOfMonth(toTimeZone(least(?, ?), ?))", + t.timestamp, + ^last_datetime, + ^query.timezone + ) + }) + end + + def select_dimension(q, key, "time:month", _table, query) do + select_merge_as(q, [t], %{ + key => fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone) + }) + end + + def select_dimension(q, key, "time:week", :sessions, query) do + {_first, last_datetime} = Time.utc_boundaries(query) + date_range = Query.date_range(query) + + select_merge_as(q, [t], %{ + key => + weekstart_not_before( + to_timezone(fragment("least(?, ?)", t.timestamp, ^last_datetime), ^query.timezone), + ^date_range.first + ) + }) + end + + def select_dimension(q, key, "time:week", _table, query) do + date_range = Query.date_range(query) + + select_merge_as(q, [t], %{ + key => + weekstart_not_before( + to_timezone(t.timestamp, ^query.timezone), + ^date_range.first + ) + }) + end + + def select_dimension(q, key, "time:day", :sessions, query) do + {_first, last_datetime} = Time.utc_boundaries(query) + + select_merge_as(q, [t], %{ + key => + fragment( + "toDate(toTimeZone(least(?, ?), ?))", + t.timestamp, + ^last_datetime, + ^query.timezone + ) + }) + end + + def select_dimension(q, key, "time:day", _table, query) do + select_merge_as(q, [t], %{ + key => fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone) + }) + end + + def select_dimension(q, key, "time:hour", :sessions, query) when query.smear_session_metrics do + # :TRICKY: ClickHouse timeSlots works off of unix epoch and is not + # timezone-aware. This means that for e.g. Asia/Katmandu (GMT+5:45) + # to work, we divide time into 15-minute buckets and later combine these + # via toStartOfHour + {first, last} = Time.utc_boundaries(query) + + q + |> join(:inner, [s], time_slot in time_slots(query, 15 * 60, first, last), + as: :time_slot, + hints: "ARRAY", + on: true + ) + |> select_merge_as([s, time_slot: time_slot], %{ + key => fragment("toStartOfHour(?)", time_slot) + }) + end + + def select_dimension(q, key, "time:hour", _table, query) do + select_merge_as(q, [t], %{ + key => fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone) + }) + end + + # :NOTE: This is not exposed in Query APIv2 + def select_dimension(q, key, "time:minute", :sessions, query) + when query.smear_session_metrics do + {first, last} = Time.utc_boundaries(query) + + q + |> join(:inner, [s], time_slot in time_slots(query, 60, first, last), + as: :time_slot, + hints: "ARRAY", + on: true + ) + |> select_merge_as([s, time_slot: time_slot], %{ + key => fragment("?", time_slot) + }) + end + + # :NOTE: This is not exposed in Query APIv2 + def select_dimension(q, key, "time:minute", _table, query) do + select_merge_as(q, [t], %{ + key => fragment("toStartOfMinute(toTimeZone(?, ?))", t.timestamp, ^query.timezone) + }) + end + + def select_dimension(q, key, "event:name", _table, _query), + do: select_merge_as(q, [t], %{key => t.name}) + + def select_dimension(q, key, "event:page", _table, _query), + do: select_merge_as(q, [t], %{key => t.pathname}) + + def select_dimension(q, key, "event:hostname", _table, _query), + do: select_merge_as(q, [t], %{key => t.hostname}) + + def select_dimension(q, key, "event:props:" <> property_name, _table, _query) do + select_merge_as(q, [t], %{ + key => + fragment( + "if(not empty(?), ?, '(none)')", + get_by_key(t, :meta, ^property_name), + get_by_key(t, :meta, ^property_name) + ) + }) + end + + def select_dimension(q, key, "visit:entry_page", _table, _query), + do: select_merge_as(q, [t], %{key => t.entry_page}) + + def select_dimension(q, key, "visit:entry_page_hostname", _table, _query), + do: select_merge_as(q, [t], %{key => t.hostname}) + + def select_dimension(q, key, "visit:exit_page", _table, _query), + do: select_merge_as(q, [t], %{key => t.exit_page}) + + def select_dimension(q, key, "visit:exit_page_hostname", _table, _query), + do: select_merge_as(q, [t], %{key => t.exit_page_hostname}) + + def select_dimension(q, key, "visit:utm_medium", _table, _query), + do: field_or_blank_value(q, key, t.utm_medium, @not_set) + + def select_dimension(q, key, "visit:utm_source", _table, _query), + do: field_or_blank_value(q, key, t.utm_source, @not_set) + + def select_dimension(q, key, "visit:utm_campaign", _table, _query), + do: field_or_blank_value(q, key, t.utm_campaign, @not_set) + + def select_dimension(q, key, "visit:utm_content", _table, _query), + do: field_or_blank_value(q, key, t.utm_content, @not_set) + + def select_dimension(q, key, "visit:utm_term", _table, _query), + do: field_or_blank_value(q, key, t.utm_term, @not_set) + + def select_dimension(q, key, "visit:source", _table, _query), + do: field_or_blank_value(q, key, t.source, @no_ref) + + def select_dimension(q, key, "visit:channel", _table, _query), + do: field_or_blank_value(q, key, t.acquisition_channel, @no_channel) + + def select_dimension(q, key, "visit:referrer", _table, _query), + do: field_or_blank_value(q, key, t.referrer, @no_ref) + + def select_dimension(q, key, "visit:device", _table, _query), + do: field_or_blank_value(q, key, t.device, @not_set) + + def select_dimension(q, key, "visit:os", _table, _query), + do: field_or_blank_value(q, key, t.os, @not_set) + + def select_dimension(q, key, "visit:os_version", _table, _query), + do: field_or_blank_value(q, key, t.os_version, @not_set) + + def select_dimension(q, key, "visit:browser", _table, _query), + do: field_or_blank_value(q, key, t.browser, @not_set) + + def select_dimension(q, key, "visit:browser_version", _table, _query), + do: field_or_blank_value(q, key, t.browser_version, @not_set) + + def select_dimension(q, key, "visit:country", _table, _query), + do: select_merge_as(q, [t], %{key => t.country}) + + def select_dimension(q, key, "visit:region", _table, _query), + do: select_merge_as(q, [t], %{key => t.region}) + + def select_dimension(q, key, "visit:city", _table, _query), + do: select_merge_as(q, [t], %{key => t.city}) + + def select_dimension(q, key, "visit:country_name", _table, _query), + do: select_merge_as(q, [t], %{key => t.country_name}) + + def select_dimension(q, key, "visit:region_name", _table, _query), + do: select_merge_as(q, [t], %{key => t.region_name}) + + def select_dimension(q, key, "visit:city_name", _table, _query), + do: select_merge_as(q, [t], %{key => t.city_name}) + + def select_dimension_internal(q, "visit:entry_page") do + select_merge_as(q, [t], %{ + entry_page: fragment("any(?)", field(t, :entry_page)) + }) + end + + def select_dimension_internal(q, "visit:entry_page_hostname") do + select_merge_as(q, [t], %{ + entry_page_hostname: fragment("any(?)", field(t, :entry_page_hostname)) + }) + end + + def select_dimension_internal(q, "visit:exit_page") do + # As exit page changes with every pageview event over the lifetime + # of a session, only the most recent value must be considered. + select_merge_as(q, [t], %{ + exit_page: fragment("argMax(?, ?)", field(t, :exit_page), field(t, :events)) + }) + end + + def select_dimension_internal(q, "visit:exit_page_hostname") do + select_merge_as(q, [t], %{ + exit_page_hostname: + fragment("argMax(?, ?)", field(t, :exit_page_hostname), field(t, :events)) + }) + end + + def select_dimension_internal(q, _dimension), do: q + + def select_dimension_from_join(q, key, "visit:entry_page"), + do: select_merge_as(q, [..., t], %{key => t.entry_page}) + + def select_dimension_from_join(q, key, "visit:entry_page_hostname"), + do: select_merge_as(q, [..., t], %{key => t.entry_page_hostaname}) + + def select_dimension_from_join(q, key, "visit:exit_page"), + do: select_merge_as(q, [..., t], %{key => t.exit_page}) + + def select_dimension_from_join(q, key, "visit:exit_page_hostname"), + do: select_merge_as(q, [..., t], %{key => t.exit_page_hostname}) + + def select_dimension_from_join(q, _key, _dimension), do: q + + def event_metric(:pageviews, _query) do + wrap_alias([e], %{ + pageviews: scale_sample(fragment("countIf(? = 'pageview')", e.name)) + }) + end + + def event_metric(:events, _query) do + wrap_alias([e], %{ + events: scale_sample(fragment("countIf(? != 'engagement')", e.name)) + }) + end + + def event_metric(:visitors, _query) do + wrap_alias([e], %{ + visitors: scale_sample(fragment("uniq(?)", e.user_id)) + }) + end + + def event_metric(:visits, _query) do + wrap_alias([e], %{ + visits: scale_sample(fragment("uniq(?)", e.session_id)) + }) + end + + on_ee do + def event_metric(:total_revenue, _query) do + wrap_alias( + [e], + %{ + total_revenue: + fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount) + } + ) + end + + def event_metric(:average_revenue, _query) do + wrap_alias( + [e], + %{ + average_revenue: fragment("toDecimal64(avg(?), 3)", e.revenue_reporting_amount) + } + ) + end + end + + def event_metric(:sample_percent, _query) do + wrap_alias([], %{ + sample_percent: + fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") + }) + end + + def event_metric(:percentage, _query), do: %{} + def event_metric(:conversion_rate, _query), do: %{} + def event_metric(:scroll_depth, _query), do: %{} + def event_metric(:group_conversion_rate, _query), do: %{} + def event_metric(:total_visitors, _query), do: %{} + + def event_metric(:time_on_page, query) do + selected = + case query.time_on_page_data do + %{include_new_metric: false} -> + wrap_alias( + [e], + %{ + # :KLUDGE: We would like to but can't use constant 0 here as it leads to cyclic aliases if there's no other + # metrics selected other than time_on_page. + __internal_total_time_on_page: fragment("sumArray([0])"), + __internal_total_time_on_page_visits: fragment("sumArray([0])") + } + ) + + %{include_new_metric: true, cutoff: nil} -> + wrap_alias( + [e], + %{ + __internal_total_time_on_page: fragment("sum(?) / 1000", e.engagement_time), + __internal_total_time_on_page_visits: + fragment("uniqIf(?, ? = 'engagement')", e.session_id, e.name) + } + ) + + %{include_new_metric: true, cutoff: cutoff} -> + wrap_alias( + [e], + %{ + __internal_total_time_on_page: + fragment( + "sumIf(?, ? >= ?) / 1000", + e.engagement_time, + e.timestamp, + ^cutoff + ), + __internal_total_time_on_page_visits: + fragment( + "uniqIf(?, ? = 'engagement' and ? >= ?)", + e.session_id, + e.name, + e.timestamp, + ^cutoff + ) + } + ) + end + + if query.time_on_page_data.include_legacy_metric do + selected + else + Map.merge( + selected, + wrap_alias([e], %{ + time_on_page: + time_on_page( + selected_as(:__internal_total_time_on_page), + selected_as(:__internal_total_time_on_page_visits) + ) + }) + ) + end + end + + def event_metric(unknown, _query), do: raise("Unknown metric: #{unknown}") + + def session_metric(:bounce_rate, query) do + # :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page. + event_page_filter = Filters.get_toplevel_filter(query, "event:page") + condition = SQL.WhereBuilder.build_condition(:entry_page, event_page_filter) + + wrap_alias([], %{ + bounce_rate: + fragment( + # :TRICKY: Before PR #4493, we could have sessions where `sum(is_bounce * sign)` + # is negative, leading to an underflow and >100% bounce rate. This works around + # that issue. + "toUInt32(greatest(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0), 0))", + ^condition, + ^condition + ), + __internal_visits: fragment("toUInt32(greatest(sum(sign), 0))") + }) + end + + def session_metric(:exit_rate, _query) do + wrap_alias([s], %{ + __internal_visits: fragment("toUInt32(greatest(sum(sign), 0))") + }) + end + + def session_metric(:visits, query) when query.smear_session_metrics do + wrap_alias([s], %{ + visits: scale_sample(fragment("uniq(?)", s.session_id)) + }) + end + + def session_metric(:visits, _query) do + wrap_alias([s], %{ + visits: scale_sample(fragment("greatest(sum(?), 0)", s.sign)) + }) + end + + def session_metric(:pageviews, _query) do + wrap_alias([s], %{ + pageviews: scale_sample(fragment("greatest(sum(? * ?), 0)", s.sign, s.pageviews)) + }) + end + + def session_metric(:events, _query) do + wrap_alias([s], %{ + events: scale_sample(fragment("greatest(sum(? * ?), 0)", s.sign, s.events)) + }) + end + + def session_metric(:visitors, _query) do + wrap_alias([s], %{ + visitors: scale_sample(fragment("uniq(?)", s.user_id)) + }) + end + + def session_metric(:visit_duration, _query) do + wrap_alias([], %{ + visit_duration: + fragment("toUInt32(greatest(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0), 0))"), + __internal_visits: fragment("toUInt32(greatest(sum(sign), 0))") + }) + end + + def session_metric(:views_per_visit, _query) do + wrap_alias([s], %{ + views_per_visit: + fragment( + "greatest(ifNotFinite(round(sum(? * ?) / sum(?), 2), 0), 0)", + s.sign, + s.pageviews, + s.sign + ), + __internal_visits: fragment("toUInt32(greatest(sum(sign), 0))") + }) + end + + def session_metric(:sample_percent, _query) do + wrap_alias([], %{ + sample_percent: + fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") + }) + end + + def session_metric(:percentage, _query), do: %{} + def session_metric(:conversion_rate, _query), do: %{} + def session_metric(:group_conversion_rate, _query), do: %{} + + @doc """ + The fragment matches events to goals by: + 1. Checking if the pathname matches the goal's page regex pattern + 2. Verifying the event name matches the expected name for the goal type + 3. Validating scroll depth is within threshold (for scroll goals) + 4. Ensuring all custom properties match (if any are defined on the goal) + + Returns an array of goal indices that the event matches. + """ + defmacro event_goal_join(goal_join_data) do + quote do + fragment( + """ + arrayIntersect( + multiMatchAllIndices(?, ?), + arrayMap( + (expected_name, threshold, index, custom_props_keys, custom_props_values) -> if( + expected_name = ? and ? between threshold and 100 and + (empty(custom_props_keys) OR arrayAll((k, v) -> ?[indexOf(?, k)] = v, custom_props_keys, custom_props_values)), + index, -1 + ), + ?, + ?, + ?, + ?, + ? + ) + ) + """, + e.pathname, + type(^unquote(goal_join_data).page_regexes, {:array, :string}), + e.name, + e.scroll_depth, + field(e, :"meta.value"), + field(e, :"meta.key"), + type(^unquote(goal_join_data).event_names_by_type, {:array, :string}), + type(^unquote(goal_join_data).scroll_thresholds, {:array, :integer}), + type(^unquote(goal_join_data).indices, {:array, :integer}), + type(^unquote(goal_join_data).custom_props_keys, {:array, {:array, :string}}), + type(^unquote(goal_join_data).custom_props_values, {:array, {:array, :string}}) + ) + end + end + + @doc """ + Optimized variant of `event_goal_join/1` for use when no goals have custom + property filters. Omits all references to `meta.key` and `meta.value`, + preventing ClickHouse from reading those expensive Array(String) columns at + scan time. + """ + defmacro event_goal_join_no_props(goal_join_data) do + quote do + fragment( + """ + arrayIntersect( + multiMatchAllIndices(?, ?), + arrayMap( + (expected_name, threshold, index) -> if( + expected_name = ? and ? between threshold and 100, + index, -1 + ), + ?, + ?, + ? + ) + ) + """, + e.pathname, + type(^unquote(goal_join_data).page_regexes, {:array, :string}), + e.name, + e.scroll_depth, + type(^unquote(goal_join_data).event_names_by_type, {:array, :string}), + type(^unquote(goal_join_data).scroll_thresholds, {:array, :integer}), + type(^unquote(goal_join_data).indices, {:array, :integer}) + ) + end + end +end diff --git a/lib/plausible/stats/sql/fragments.ex b/lib/plausible/stats/sql/fragments.ex new file mode 100644 index 0000000000000000000000000000000000000000..36ce9d79904dc79ff40a69b8b3b8baab8e6e7196 --- /dev/null +++ b/lib/plausible/stats/sql/fragments.ex @@ -0,0 +1,230 @@ +defmodule Plausible.Stats.SQL.Fragments do + @moduledoc """ + Various macros and common SQL fragments used in Stats code. + """ + + defmacro __using__(_) do + quote do + import Plausible.Stats.SQL.Fragments + require Plausible.Stats.SQL.Fragments + end + end + + defmacro scale_sample(sampled_fragment) do + quote do + fragment("toUInt64(round(? * any(_sample_factor)))", unquote(sampled_fragment)) + end + end + + defmacro uniq(user_id) do + quote do + scale_sample(fragment("uniq(?)", unquote(user_id))) + end + end + + defmacro total() do + quote do + scale_sample(fragment("count()")) + end + end + + defmacro sample_percent() do + quote do + fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)") + end + end + + defmacro bounce_rate() do + quote do + fragment( + # :TRICKY: Before PR #4493, we could have sessions where `sum(is_bounce * sign)` + # is negative, leading to an underflow and >100% bounce rate. This works around + # that issue. + "toUInt32(greatest(ifNotFinite(round(sum(is_bounce * sign) / sum(sign) * 100), 0), 0))" + ) + end + end + + defmacro visit_duration() do + quote do + fragment("toUInt32(ifNotFinite(round(avg(duration * sign)), 0))") + end + end + + defmacro coalesce_string(fieldA, fieldB) do + quote do + fragment("if(empty(?), ?, ?)", unquote(fieldA), unquote(fieldB), unquote(fieldA)) + end + end + + @doc """ + Converts time or date and time to the specified timezone. + + Reference: https://clickhouse.com/docs/en/sql-reference/functions/date-time-functions/#totimezone + """ + defmacro to_timezone(date, timezone) do + quote do + fragment("toTimeZone(?, ?)", unquote(date), unquote(timezone)) + end + end + + @doc """ + Returns the weekstart for `date`. If the weekstart is before the `not_before` + boundary, `not_before` is returned. + + ## Examples + + In this pseudo-code example, the fragment returns the weekstart. The + `not_before` boundary is set to the past Saturday, which is before the + weekstart, therefore the cap does not apply. + + ``` + > this_wednesday = ~D[2022-11-09] + > past_saturday = ~D[2022-11-05] + > weekstart_not_before(this_wednesday, past_saturday) + ~D[2022-11-07] + ``` + + In this other example, the fragment returns Tuesday and not the weekstart. + The `not_before` boundary is set to Tuesday, which is past the weekstart, + therefore the cap applies. + + ``` + > this_wednesday = ~D[2022-11-09] + > this_tuesday = ~D[2022-11-08] + > weekstart_not_before(this_wednesday, this_tuesday) + ~D[2022-11-08] + ``` + """ + defmacro weekstart_not_before(date, not_before) do + quote do + fragment( + "if(toMonday(?) < toDate(?), toDate(?), toMonday(?))", + unquote(date), + unquote(not_before), + unquote(not_before), + unquote(date) + ) + end + end + + @doc """ + Returns whether a key (usually property) exists under `meta.key` array or similar. + + This macro is used for operating on custom properties. + + ## Examples + + `has_key(e, :meta, "some_property_name")` expands to SQL `has(meta.key, "some_property_name")` + """ + defmacro has_key(table, meta_column, key) do + quote do + fragment( + "has(?, ?)", + field(unquote(table), ^meta_key_column(unquote(meta_column))), + unquote(key) + ) + end + end + + @doc """ + Returns value of a key (usually property) under `meta.value` array or similar. + + This macro is used for operating on custom properties. + Callsites should also check whether key exists first in SQL via `has_key` macro. + + ## Examples + + `get_by_key(e, :meta, "some_property_name")` expands to SQL `meta.value[indexOf(meta.key, "some_property")]` + """ + defmacro get_by_key(table, meta_column, key) do + quote do + fragment( + "?[indexOf(?, ?)]", + field(unquote(table), ^meta_value_column(unquote(meta_column))), + field(unquote(table), ^meta_key_column(unquote(meta_column))), + unquote(key) + ) + end + end + + def meta_key_column(:meta), do: :"meta.key" + def meta_key_column(:entry_meta), do: :"entry_meta.key" + + def meta_value_column(:meta), do: :"meta.value" + def meta_value_column(:entry_meta), do: :"entry_meta.value" + + defmacro time_on_page(total_time_on_page, total_time_on_page_visits) do + quote do + fragment( + "if(? > 0, toInt32(round((?) / (?))), NULL)", + unquote(total_time_on_page_visits), + unquote(total_time_on_page), + unquote(total_time_on_page_visits) + ) + end + end + + @doc """ + Convenience Ecto macro for wrapping a map passed to select_merge_as such that each + expression gets wrapped in dynamic and set as selected_as. + + ### Examples + + iex> wrap_alias([t], %{ foo: t.column }) |> expand_macro_once + "%{foo: dynamic([t], selected_as(t.column, :foo))}" + """ + defmacro wrap_alias(binding, map_literal) do + update_literal_map_values(map_literal, fn {key, expr} -> + key_expr = + if Macro.quoted_literal?(key) do + key + else + quote(do: ^unquote(key)) + end + + quote(do: dynamic(unquote(binding), selected_as(unquote(expr), unquote(key_expr)))) + end) + end + + @doc """ + Convenience Ecto macro for wrapping select_merge where each value gets in turn passed to selected_as. + + ### Examples + + iex> select_merge_as(q, [t], %{ foo: t.column }) |> expand_macro_once + "select_merge(q, [], ^wrap_alias([t], %{foo: t.column}))" + """ + defmacro select_merge_as(q, binding, map_literal) do + quote do + select_merge(unquote(q), [], ^wrap_alias(unquote(binding), unquote(map_literal))) + end + end + + @doc """ + Macro that helps join two Ecto queries by selecting fields from either one + """ + defmacro select_join_fields(q, query, list, table_name) do + quote do + Enum.reduce(unquote(list), unquote(q), fn metric_or_dimension, q -> + key = shortname(unquote(query), metric_or_dimension) + + select_merge_as(q, [e, s], %{ + key => field(unquote(table_name), ^key) + }) + end) + end + end + + defp update_literal_map_values({:%{}, ctx, keyword_list}, mapper_fn) do + { + :%{}, + ctx, + Enum.map(keyword_list, fn {key, expr} -> + {key, mapper_fn.({key, expr})} + end) + } + end + + defp update_literal_map_values(ast, _), do: ast +end diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.ex new file mode 100644 index 0000000000000000000000000000000000000000..8e5b4132c4e968a26802962df0581cbbeb61a34b --- /dev/null +++ b/lib/plausible/stats/sql/query_builder.ex @@ -0,0 +1,310 @@ +defmodule Plausible.Stats.SQL.QueryBuilder do + @moduledoc false + + use Plausible + use Plausible.Stats.SQL.Fragments + + import Ecto.Query + import Plausible.Stats.Imported + import Plausible.Stats.Util + + alias Plausible.Stats.{Query, QueryOptimizer, TableDecider, SQL} + alias Plausible.Stats.SQL.Expression + alias Plausible.Stats.Legacy.TimeOnPage + + require Plausible.Stats.SQL.Expression + + def build(query, site) do + query + |> QueryOptimizer.split() + |> Enum.map(fn {table_type, table_query} -> + q = build_table_query(table_type, site, table_query) + {table_type, table_query, q} + end) + |> join_query_results(query) + |> build_order_by(query) + |> paginate(query.pagination) + |> select_total_rows(query.include.total_rows) + end + + def build_order_by(q, query) do + Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1)) + end + + defp build_table_query(:events, site, events_query) do + q = + from( + e in "events_v2", + where: ^SQL.WhereBuilder.build(:events, events_query), + where: ^SQL.WhereBuilder.derived_name_filter(events_query), + select: ^select_event_metrics(events_query) + ) + + on_ee do + q = Plausible.Stats.Sampling.add_query_hint(q, events_query) + end + + q + |> join_sessions_if_needed(events_query) + |> build_group_by(:events, events_query) + |> merge_imported(site, events_query) + |> SQL.SpecialMetrics.add(site, events_query) + |> TimeOnPage.merge_legacy_time_on_page(events_query) + end + + defp build_table_query(:sessions, site, sessions_query) do + q = + from( + e in "sessions_v2", + where: ^SQL.WhereBuilder.build(:sessions, sessions_query), + select: ^select_session_metrics(sessions_query) + ) + + on_ee do + q = Plausible.Stats.Sampling.add_query_hint(q, sessions_query) + end + + q + |> join_events_if_needed(sessions_query) + |> build_group_by(:sessions, sessions_query) + |> merge_imported(site, sessions_query) + |> SQL.SpecialMetrics.add(site, sessions_query) + end + + defp join_sessions_if_needed(q, query) do + if TableDecider.events_join_sessions?(query) do + %{session: dimensions} = TableDecider.partition_dimensions(query) + + sessions_q = + from( + s in "sessions_v2", + where: ^SQL.WhereBuilder.build(:sessions, query), + where: s.sign == 1, + select: %{session_id: s.session_id}, + group_by: s.session_id + ) + + # The session-only dimension columns are explicitly selected in joined + # sessions table. This enables combining session-only dimensions (entry + # and exit pages) with event-only metrics, like revenue. + sessions_q = + Enum.reduce(dimensions, sessions_q, fn dimension, acc -> + Plausible.Stats.SQL.Expression.select_dimension_internal(acc, dimension) + end) + + on_ee do + sessions_q = Plausible.Stats.Sampling.add_query_hint(sessions_q, query) + end + + from( + e in q, + join: sq in subquery(sessions_q), + on: e.session_id == sq.session_id + ) + else + q + end + end + + def join_events_if_needed(q, query) do + if TableDecider.sessions_join_events?(query) do + events_q = + from(e in "events_v2", + where: ^SQL.WhereBuilder.build(:events, query), + where: ^SQL.WhereBuilder.derived_name_filter(query), + select: %{ + session_id: fragment("DISTINCT ?", e.session_id), + _sample_factor: fragment("_sample_factor") + } + ) + + on_ee do + events_q = Plausible.Stats.Sampling.add_query_hint(events_q, query) + end + + from(s in q, + join: e in subquery(events_q), + on: s.session_id == e.session_id + ) + else + q + end + end + + defp select_event_metrics(query) do + query.metrics + |> Enum.map(&SQL.Expression.event_metric(&1, query)) + |> Enum.reduce(%{}, &Map.merge/2) + end + + defp select_session_metrics(query) do + query.metrics + |> Enum.map(&SQL.Expression.session_metric(&1, query)) + |> Enum.reduce(%{}, &Map.merge/2) + end + + def build_group_by(q, :events, query) do + # Session-only dimensions are extracted from joined sessions table + %{session: session_only_dimensions} = TableDecider.partition_dimensions(query) + event_dimensions = query.dimensions -- session_only_dimensions + + q = + Enum.reduce(event_dimensions, q, &dimension_group_by(&2, :events, query, &1)) + + Enum.reduce( + session_only_dimensions, + q, + &dimension_group_by_join(&2, query, &1) + ) + end + + def build_group_by(q, :sessions, query) do + Enum.reduce(query.dimensions, q, &dimension_group_by(&2, :sessions, query, &1)) + end + + defp dimension_group_by_join(q, query, dimension) do + key = shortname(query, dimension) + + q + |> Expression.select_dimension_from_join(key, dimension) + |> group_by([], selected_as(^key)) + end + + defp dimension_group_by(q, :events, query, "event:prop_key" = dimension) do + from(e in q, + join: meta in fragment("meta"), + hints: "ARRAY", + on: true, + group_by: meta.key + ) + |> select_merge_as([_e, meta], %{shortname(query, dimension) => meta.key}) + end + + defp dimension_group_by(q, :events, query, "event:goal" = dimension) do + goal_join_data = Plausible.Stats.Goals.goal_join_data(query) + + if Enum.all?(goal_join_data.custom_props_keys, &Enum.empty?/1) do + from(e in q, + join: goal in Expression.event_goal_join_no_props(goal_join_data), + hints: "ARRAY", + on: true, + select_merge: %{ + ^shortname(query, dimension) => fragment("?", goal) + }, + group_by: goal + ) + else + from(e in q, + join: goal in Expression.event_goal_join(goal_join_data), + hints: "ARRAY", + on: true, + select_merge: %{ + ^shortname(query, dimension) => fragment("?", goal) + }, + group_by: goal + ) + end + end + + defp dimension_group_by(q, table, query, dimension) do + key = shortname(query, dimension) + + q + |> Expression.select_dimension(key, dimension, table, query) + |> group_by([], selected_as(^key)) + end + + defp build_order_by(q, query, {metric_or_dimension, order_direction}) do + order_by( + q, + [t], + { + ^order_direction, + selected_as(^shortname(query, metric_or_dimension)) + } + ) + end + + # Only one table is being queried - skip joining! + defp join_query_results([{_table_type, _query, q}], _main_query), do: q + + # Multiple tables: join results based on dimensions, select metrics from each and the appropriate dimensions. + defp join_query_results(queries, main_query) do + queries + |> Enum.reduce(nil, fn + {_table_type, query, q}, nil -> + from(e in subquery(q)) + |> select_join_metrics(query, query.metrics) + + {_table_type, query, q}, acc -> + join(acc, main_query.sql_join_type, [], s in subquery(q), + on: ^build_group_by_join(main_query) + ) + |> select_join_metrics(query, query.metrics -- [:sample_percent]) + end) + |> select_dimensions(main_query, queries) + end + + # NOTE: Old queries do their own pagination + defp paginate(q, nil = _pagination), do: q + + defp paginate(q, pagination) do + q + |> limit(^pagination.limit) + |> offset(^pagination.offset) + end + + defp select_total_rows(q, false = _include_total_rows), do: q + + defp select_total_rows(q, true = _include_total_rows) do + q + |> select_merge([], %{total_rows: fragment("count() over ()")}) + end + + def build_group_by_join(%Query{dimensions: []}), do: true + + def build_group_by_join(query) do + query.dimensions + |> Enum.map(fn dim -> + dynamic([a, ..., b], field(a, ^shortname(query, dim)) == field(b, ^shortname(query, dim))) + end) + |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end) + end + + defp select_join_metrics(q, query, metrics) do + Enum.reduce(metrics, q, fn + metric, q -> + select_merge_as(q, [..., x], %{ + shortname(query, metric) => field(x, ^shortname(query, metric)) + }) + end) + end + + defp select_dimensions(q, query, queries) do + Enum.reduce(query.dimensions, q, fn dimension, q -> + case select_from(dimension, query, queries) do + :leftmost_table -> + select_merge_as(q, [x], %{ + shortname(query, dimension) => field(x, ^shortname(query, dimension)) + }) + + :rightmost_table -> + select_merge_as(q, [..., x], %{ + shortname(query, dimension) => field(x, ^shortname(query, dimension)) + }) + end + end) + end + + defp select_from(dimension, query, queries) do + smeared? = Enum.any?(queries, fn {_table_type, query, _q} -> query.smear_session_metrics end) + + cond do + query.sql_join_type == :left -> :leftmost_table + # We generally select dimensions from the left-most table. Only exception is time:minute/time:hour where + # we use sessions table as smeared sessions are considered on-going during the whole period. + dimension in ["time:minute", "time:hour"] and smeared? -> :rightmost_table + true -> :leftmost_table + end + end +end diff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.ex new file mode 100644 index 0000000000000000000000000000000000000000..04b429e6df30608218b7b60d7a7d7dda329feb7b --- /dev/null +++ b/lib/plausible/stats/sql/special_metrics.ex @@ -0,0 +1,295 @@ +defmodule Plausible.Stats.SQL.SpecialMetrics do + @moduledoc """ + This module defines how special metrics like `conversion_rate` and + `percentage` are calculated. + """ + + use Plausible.Stats.SQL.Fragments + + alias Plausible.Stats.{Base, Query, SQL, Filters} + + import Ecto.Query + import Plausible.Stats.Util + + @special_metrics [ + :percentage, + :conversion_rate, + :group_conversion_rate, + :scroll_depth, + :exit_rate + ] + + def add(q, site, query) do + Enum.reduce(@special_metrics, q, fn special_metric, q -> + if special_metric in query.metrics do + add_special_metric(q, special_metric, site, query) + else + q + end + end) + end + + defp add_special_metric(q, :percentage, site, query) do + total_query = + query + |> remove_filters_ignored_in_totals_query() + |> Query.set( + dimensions: [], + include_imported: query.include_imported, + pagination: nil + ) + + q + |> select_merge_as([], total_visitors_subquery(site, total_query, query.include_imported)) + |> select_merge_as([], %{ + percentage: + fragment( + "if(? > 0, round(? / ? * 100, 2), null)", + selected_as(:total_visitors), + selected_as(:visitors), + selected_as(:total_visitors) + ) + }) + end + + # Adds conversion_rate metric to query, calculated as + # X / Y where Y is the same breakdown value without goal or props + # filters. + defp add_special_metric(q, :conversion_rate, site, query) do + total_query = + query + |> Query.remove_top_level_filters(["event:goal", "event:props"]) + |> remove_filters_ignored_in_totals_query() + |> Query.set( + dimensions: [], + include_imported: query.include_imported, + preloaded_goals: Map.put(query.preloaded_goals, :matching_toplevel_filters, []), + pagination: nil + ) + + q + |> select_merge_as( + [], + total_visitors_subquery(site, total_query, query.include_imported) + ) + |> select_merge_as([e], %{ + conversion_rate: + fragment( + "if(? > 0, round(? / ? * 100, 2), 0)", + selected_as(:total_visitors), + selected_as(:visitors), + selected_as(:total_visitors) + ) + }) + end + + # This function injects a group_conversion_rate metric into + # a dimensional query. It is calculated as X / Y, where: + # + # * X is the number of conversions for a set of dimensions + # result (conversion = number of visitors who + # completed the filtered goal with the filtered + # custom properties). + # + # * Y is the number of all visitors for this set of dimensions + # result without the `event:goal` and `event:props:*` + # filters. + defp add_special_metric(q, :group_conversion_rate, site, query) do + group_totals_query = + query + |> Query.remove_top_level_filters(["event:goal", "event:props"]) + |> remove_filters_ignored_in_totals_query() + |> Query.set( + metrics: [:visitors], + order_by: [], + include_imported: query.include_imported, + preloaded_goals: Map.put(query.preloaded_goals, :matching_toplevel_filters, []), + pagination: nil + ) + + from(e in subquery(q), + left_join: c in subquery(SQL.QueryBuilder.build(group_totals_query, site)), + on: ^SQL.QueryBuilder.build_group_by_join(query) + ) + |> select_merge_as([e, c], %{ + total_visitors: c.visitors, + group_conversion_rate: + fragment( + "if(? > 0, round(? / ? * 100, 2), 0)", + c.visitors, + e.visitors, + c.visitors + ) + }) + |> select_join_fields(query, query.dimensions, e) + |> select_join_fields(query, List.delete(query.metrics, :group_conversion_rate), e) + end + + defp add_special_metric(q, :scroll_depth, _site, query) do + max_per_session_q = + Base.base_event_query(query) + |> where([e], e.name == "engagement" and e.scroll_depth <= 100) + |> select([e], %{ + session_id: e.session_id, + max_scroll_depth: max(e.scroll_depth) + }) + |> SQL.QueryBuilder.build_group_by(:events, query) + |> group_by([e], e.session_id) + + dim_shortnames = Enum.map(query.dimensions, fn dim -> shortname(query, dim) end) + + dim_select = + dim_shortnames + |> Enum.map(fn dim -> {dim, dynamic([p], field(p, ^dim))} end) + |> Map.new() + + dim_group_by = + dim_shortnames + |> Enum.map(fn dim -> dynamic([p], field(p, ^dim)) end) + + total_scroll_depth_q = + subquery(max_per_session_q) + |> select([], %{}) + |> select_merge_as([p], %{ + # Note: No need to upscale sample size here since it would end up cancelling out due to the result being an average + total_scroll_depth: fragment("sum(?)", p.max_scroll_depth), + total_scroll_depth_visits: fragment("uniq(?)", p.session_id) + }) + |> select_merge(^dim_select) + |> group_by(^dim_group_by) + + join_on_dim_condition = + if dim_shortnames == [] do + true + else + dim_shortnames + |> Enum.map(fn dim -> dynamic([_e, ..., s], selected_as(^dim) == field(s, ^dim)) end) + # credo:disable-for-next-line Credo.Check.Refactor.Nesting + |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end) + end + + joined_q = + join(q, :left, [e], s in subquery(total_scroll_depth_q), on: ^join_on_dim_condition) + + if query.include_imported do + joined_q + |> select_merge_as([..., s], %{ + scroll_depth: + fragment( + """ + if(? + ? > 0, toInt8(round((? + ?) / (? + ?))), NULL) + """, + s.total_scroll_depth_visits, + selected_as(:__imported_total_scroll_depth_visits), + s.total_scroll_depth, + selected_as(:__imported_total_scroll_depth), + s.total_scroll_depth_visits, + selected_as(:__imported_total_scroll_depth_visits) + ) + }) + else + joined_q + |> select_merge_as([..., s], %{ + scroll_depth: + fragment( + "if(any(?) > 0, toUInt8(round(any(?) / any(?))), NULL)", + s.total_scroll_depth_visits, + s.total_scroll_depth, + s.total_scroll_depth_visits + ) + }) + end + end + + # Selects exit_rate into the query, calculated as X / Y, where X is the + # total number of exits from a page (i.e. the number of sessions with a + # specific exit page), and Y is the total pageviews on that page. + defp add_special_metric(q, :exit_rate, site, query) do + total_pageviews_query = + query + |> Query.remove_top_level_filters(["visit:exit_page"]) + |> remove_filters_ignored_in_totals_query() + |> Query.set( + pagination: nil, + order_by: [], + metrics: [:pageviews], + include_imported: query.include_imported, + dimensions: ["event:page"] + ) + + joined_q = + q + |> join(:left, [], p in subquery(SQL.QueryBuilder.build(total_pageviews_query, site)), + on: + selected_as(^shortname(query, "visit:exit_page")) == + field(p, ^shortname(total_pageviews_query, "event:page")) + ) + + if query.include_imported do + joined_q + |> select_merge_as([..., p], %{ + exit_rate: + fragment( + "if(? > 0, round(? / ? * 100, 1), NULL)", + p.pageviews, + selected_as(:__internal_visits), + p.pageviews + ) + }) + else + joined_q + |> select_merge_as([..., p], %{ + exit_rate: + fragment( + "if(? > 0, round(? / ? * 100, 1), NULL)", + fragment("any(?)", p.pageviews), + selected_as(:__internal_visits), + fragment("any(?)", p.pageviews) + ) + }) + end + end + + # `total_visitors_subquery` returns a subquery which selects `total_visitors` - + # the number used as the denominator in the calculation of `conversion_rate` and + # `percentage` metrics. + + # Usually, when calculating the totals, a new query is passed into this function, + # where certain filters (e.g. goal, props) are removed. That might make the query + # able to include imported data. However, we always want to include imported data + # only if it's included in the base query - otherwise the total will be based on + # a different data set, making the metric inaccurate. This is why we're using an + # explicit `include_imported` argument here. + defp total_visitors_subquery(site, query, include_imported) + + defp total_visitors_subquery(site, query, true = _include_imported) do + wrap_alias([], %{ + total_visitors: + subquery(total_visitors(query)) + + subquery(Plausible.Stats.Imported.total_imported_visitors(site, query)) + }) + end + + defp total_visitors_subquery(_site, query, false = _include_imported) do + wrap_alias([], %{ + total_visitors: subquery(total_visitors(query)) + }) + end + + defp remove_filters_ignored_in_totals_query(query) do + totals_query_filters = + Filters.transform_filters(query.filters, fn + [:ignore_in_totals_query, _] -> [] + filter -> [filter] + end) + + Query.set(query, filters: totals_query_filters) + end + + defp total_visitors(query) do + Base.base_event_query(query) + |> select([e], + total_visitors: scale_sample(fragment("uniq(?)", e.user_id)) + ) + end +end diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.ex new file mode 100644 index 0000000000000000000000000000000000000000..18353d34cba0bd39f203d7253923af77c1cdeead --- /dev/null +++ b/lib/plausible/stats/sql/where_builder.ex @@ -0,0 +1,442 @@ +defmodule Plausible.Stats.SQL.WhereBuilder do + @moduledoc """ + A module for building am ecto where clause of a query out of a query. + """ + + import Ecto.Query + import Plausible.Stats.Time, only: [utc_boundaries: 1] + import Plausible.Stats.Filters.Utils, only: [page_regex: 1] + + use Plausible.Stats.SQL.Fragments + + require Logger + + @sessions_only_visit_fields [ + :entry_page, + :exit_page, + :entry_page_hostname, + :exit_page_hostname + ] + + @doc """ + Builds a WHERE condition to restrict event name, reducing unnecessary reads of + engagement events when they are not needed. + + - Goal filters: no restriction (Goals.add_filter already adds precise name conditions) + - Preloaded goals present: restrict to only the event names relevant to those goals + - Default: exclude engagement events unless time_on_page or scroll_depth metrics are requested + """ + def derived_name_filter(query) do + cond do + Plausible.Stats.Filters.filtering_on_dimension?(query.filters, "event:goal") -> + true + + query.preloaded_goals.matching_toplevel_filters != [] -> + names = goal_event_names(query.preloaded_goals.matching_toplevel_filters) + dynamic([e], e.name in ^names) + + :time_on_page not in query.metrics and :scroll_depth not in query.metrics -> + dynamic([e], e.name != "engagement") + + true -> + true + end + end + + defp goal_event_names(goals) do + goals + |> Enum.map(fn goal -> + case Plausible.Goal.type(goal) do + :event -> goal.event_name + :page -> "pageview" + :scroll -> "engagement" + end + end) + |> Enum.uniq() + end + + @doc "Builds WHERE clause for a given Query against sessions or events table" + def build(table, query) do + base_condition = filter_site_time_range(table, query) + + query.filters + |> Enum.map(&add_filter(table, query, &1)) + |> Enum.reduce(base_condition, fn condition, acc -> dynamic([], ^acc and ^condition) end) + end + + @doc """ + Builds WHERE clause condition based off of a filter and a custom column name + Used for special business logic cases + + Accepts nil as the `filter` parameter, in which case the condition is a no-op (WHERE TRUE). + """ + def build_condition(db_field, filter) do + if filter do + filter_field(db_field, filter) + else + true + end + end + + defp filter_site_time_range(table, query) do + dynamic([], ^filter_site_id(query) and ^filter_time_range(table, query)) + end + + defp filter_site_id(query) do + case query.consolidated_site_ids do + nil -> + dynamic([x], x.site_id == ^query.site_id) + + [_ | _] = ids -> + dynamic([x], fragment("? in ?", x.site_id, ^ids)) + end + end + + defp filter_time_range(:events, query) do + {first_datetime, last_datetime} = utc_boundaries(query) + dynamic([e], e.timestamp >= ^first_datetime and e.timestamp <= ^last_datetime) + end + + defp filter_time_range(:sessions, query) do + {first_datetime, last_datetime} = utc_boundaries(query) + + # Counts each _active_ session in time range even if they started before + dynamic( + [s], + # Currently, the sessions table in ClickHouse only has `start` column + # in its primary key. This means that filtering by `timestamp` is not + # considered when estimating number of returned rows from index + # for sample factor calculation. The redundant lower bound `start` condition + # ensures the lower bound time filter is still present as primary key + # condition and the sample factor estimation has minimal skew. + # + # Without it, the sample factor would be greatly overestimated for large sites, + # as query would be estimated to return _all_ rows matching other conditions + # before `start == last_datetime`. + s.start >= ^NaiveDateTime.add(first_datetime, -7, :day) and + s.timestamp >= ^first_datetime and + s.start <= ^last_datetime + ) + end + + defp add_filter(table, query, [:ignore_in_totals_query, filter]) do + add_filter(table, query, filter) + end + + defp add_filter(table, query, [:not, filter]) do + dynamic([e], not (^add_filter(table, query, filter))) + end + + defp add_filter(table, query, [:and, filters]) do + filters + |> Enum.map(&add_filter(table, query, &1)) + |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end) + end + + defp add_filter(table, query, [:or, filters]) do + filters + |> Enum.map(&add_filter(table, query, &1)) + |> Enum.reduce(fn condition, acc -> dynamic([], ^acc or ^condition) end) + end + + defp add_filter(_table, query, [:has_done, filter]) do + condition = + dynamic([], ^filter_site_time_range(:events, query) and ^add_filter(:events, query, filter)) + + dynamic( + [t], + t.session_id in subquery(from(e in "events_v2", where: ^condition, select: e.session_id)) + ) + end + + defp add_filter(table, query, [:has_not_done, filter]) do + dynamic([], not (^add_filter(table, query, [:has_done, filter]))) + end + + defp add_filter(:events, _query, [:is, "event:name" | _rest] = filter) do + in_clause(col_value(:name), filter) + end + + defp add_filter(:events, query, [_, "event:goal" | _rest] = filter) do + Plausible.Stats.Goals.add_filter(query, filter) + end + + defp add_filter(:events, _query, [_, "event:page" | _rest] = filter) do + filter_field(:pathname, filter) + end + + defp add_filter(:events, _query, [_, "event:hostname" | _rest] = filter) do + filter_field(:hostname, filter) + end + + defp add_filter(:events, _query, [_, "event:props:" <> prop_name | _rest] = filter) do + filter_custom_prop(prop_name, :meta, filter) + end + + defp add_filter(:events, _query, [_, "visit:entry_props:" <> _prop_name | _rest]) do + true + end + + defp add_filter( + :events, + _query, + [_, "visit:" <> key | _rest] = filter + ) do + # Filter events query with visit dimension if possible + field_name = db_field_name(key) + + if Enum.member?(@sessions_only_visit_fields, field_name) do + true + else + filter_field(field_name, filter) + end + end + + defp add_filter(:sessions, _query, [_, "visit:entry_props:" <> prop_name | _rest] = filter) do + filter_custom_prop(prop_name, :entry_meta, filter) + end + + defp add_filter(:sessions, _query, [_, "visit:" <> key | _rest] = filter) do + filter_field(db_field_name(key), filter) + end + + defp add_filter(:sessions, _query, [_, "event:" <> _ | _rest]) do + # Cannot apply sessions filters directly on session query where clause. + true + end + + defp add_filter(table, _query, filter) do + Logger.notice("Unable to process garbage filter. No results are returned", + table: table, + filter: filter + ) + + false + end + + defp filter_custom_prop(prop_name, column_name, [:is, _, clauses | _rest] = filter) do + none_value_included = Enum.member?(clauses, "(none)") + prop_value_expr = custom_prop_value(column_name, prop_name) + + dynamic( + [t], + (has_key(t, column_name, ^prop_name) and ^in_clause(prop_value_expr, filter)) or + (^none_value_included and not has_key(t, column_name, ^prop_name)) + ) + end + + defp filter_custom_prop(prop_name, column_name, [:is_not, _, clauses | _rest] = filter) do + none_value_included = Enum.member?(clauses, "(none)") + prop_value_expr = custom_prop_value(column_name, prop_name) + + dynamic( + [t], + (has_key(t, column_name, ^prop_name) and + not (^in_clause(prop_value_expr, filter))) or + (^none_value_included and + has_key(t, column_name, ^prop_name) and + not (^in_clause(prop_value_expr, filter))) or + (not (^none_value_included) and not has_key(t, column_name, ^prop_name)) + ) + end + + defp filter_custom_prop(prop_name, column_name, [:matches_wildcard, dimension, clauses | rest]) do + regexes = Enum.map(clauses, &page_regex/1) + + filter_custom_prop(prop_name, column_name, [:matches, dimension, regexes | rest]) + end + + defp filter_custom_prop(prop_name, column_name, [ + :matches_wildcard_not, + dimension, + clauses | rest + ]) do + regexes = Enum.map(clauses, &page_regex/1) + + filter_custom_prop(prop_name, column_name, [:matches_not, dimension, regexes | rest]) + end + + defp filter_custom_prop(prop_name, column_name, [:matches, _dimension, clauses | _rest]) do + dynamic( + [t], + has_key(t, column_name, ^prop_name) and + fragment( + "arrayExists(k -> match(?, k), ?)", + get_by_key(t, column_name, ^prop_name), + ^clauses + ) + ) + end + + defp filter_custom_prop(prop_name, column_name, [:matches_not, _dimension, clauses | _rest]) do + dynamic( + [t], + has_key(t, column_name, ^prop_name) and + fragment( + "not(arrayExists(k -> match(?, k), ?))", + get_by_key(t, column_name, ^prop_name), + ^clauses + ) + ) + end + + defp filter_custom_prop(prop_name, column_name, [:contains | _rest] = filter) do + dynamic( + [t], + has_key(t, column_name, ^prop_name) and + ^contains_clause(custom_prop_value(column_name, prop_name), filter) + ) + end + + defp filter_custom_prop(prop_name, column_name, [:contains_not | _] = filter) do + dynamic( + [t], + has_key(t, column_name, ^prop_name) and + not (^contains_clause(custom_prop_value(column_name, prop_name), filter)) + ) + end + + defp filter_field(db_field, [:matches_wildcard, _dimension, glob_exprs | _rest]) do + page_regexes = Enum.map(glob_exprs, &page_regex/1) + + dynamic( + [x], + fragment("multiMatchAny(?, ?)", type(field(x, ^db_field), :string), ^page_regexes) + ) + end + + defp filter_field(db_field, [:matches_wildcard_not | rest]) do + dynamic([], not (^filter_field(db_field, [:matches_wildcard | rest]))) + end + + defp filter_field(db_field, [:contains | _rest] = filter) do + if no_ref_field?(db_field) do + contains_clause_no_ref(col_value_string(db_field), filter) + else + contains_clause(col_value_string(db_field), filter) + end + end + + defp filter_field(db_field, [:contains_not | rest]) do + dynamic([], not (^filter_field(db_field, [:contains | rest]))) + end + + defp filter_field(db_field, [:matches, _dimension, clauses | _rest]) do + dynamic( + [x], + fragment("multiMatchAny(?, ?)", type(field(x, ^db_field), :string), ^clauses) + ) + end + + defp filter_field(db_field, [:matches_not | rest]) do + dynamic([], not (^filter_field(db_field, [:matches | rest]))) + end + + defp filter_field(db_field, [:is, _dimension, clauses | _rest] = filter) do + list = clauses |> Enum.map(&db_field_val(db_field, &1)) + in_clause(col_value(db_field), filter, list) + end + + defp filter_field(db_field, [:is_not | rest]) do + dynamic([], not (^filter_field(db_field, [:is | rest]))) + end + + @no_ref "Direct / None" + @not_set "(not set)" + + defp db_field_name("channel"), do: :acquisition_channel + defp db_field_name(name), do: String.to_existing_atom(name) + + defp no_ref_field?(:source), do: true + defp no_ref_field?(:referrer), do: true + defp no_ref_field?(:utm_medium), do: true + defp no_ref_field?(:utm_source), do: true + defp no_ref_field?(:utm_campaign), do: true + defp no_ref_field?(:utm_content), do: true + defp no_ref_field?(:utm_term), do: true + defp no_ref_field?(_), do: false + + defp db_field_val(:source, @no_ref), do: "" + defp db_field_val(:referrer, @no_ref), do: "" + defp db_field_val(:utm_medium, @no_ref), do: "" + defp db_field_val(:utm_source, @no_ref), do: "" + defp db_field_val(:utm_campaign, @no_ref), do: "" + defp db_field_val(:utm_content, @no_ref), do: "" + defp db_field_val(:utm_term, @no_ref), do: "" + defp db_field_val(_, @not_set), do: "" + defp db_field_val(_, val), do: val + + defp col_value(column_name) do + dynamic([t], field(t, ^column_name)) + end + + # Needed for string functions to work properly + defp col_value_string(column_name) do + dynamic([t], type(field(t, ^column_name), :string)) + end + + defp custom_prop_value(column_name, prop_name) do + dynamic([t], get_by_key(t, column_name, ^prop_name)) + end + + defp in_clause(value_expression, [_, _, clauses | _] = filter, values \\ nil) do + values = values || clauses + + if case_sensitive?(filter) do + dynamic([t], ^value_expression in ^values) + else + values = values |> Enum.map(&String.downcase/1) + dynamic([t], fragment("lower(?)", ^value_expression) in ^values) + end + end + + defp contains_clause(value_expression, [_, _, clauses | _] = filter) do + if case_sensitive?(filter) do + dynamic( + [x], + fragment("multiSearchAny(?, ?)", ^value_expression, ^clauses) + ) + else + dynamic( + [x], + fragment("multiSearchAnyCaseInsensitive(?, ?)", ^value_expression, ^clauses) + ) + end + end + + defp contains_clause_no_ref(value_expression, [_, _, clauses | _] = filter) do + case_sensitive? = case_sensitive?(filter) + + expression = + if case_sensitive? do + dynamic( + [x], + fragment("multiSearchAny(?, ?)", ^value_expression, ^clauses) + ) + else + dynamic( + [x], + fragment("multiSearchAnyCaseInsensitive(?, ?)", ^value_expression, ^clauses) + ) + end + + if Enum.any?(clauses, &matches_no_ref?(&1, case_sensitive?)) do + dynamic([x], ^expression or fragment("? = ?", ^value_expression, "")) + else + expression + end + end + + defp case_sensitive?([_, _, _, %{case_sensitive: false}]), do: false + defp case_sensitive?(_), do: true + + @no_ref_downcase String.downcase(@no_ref) + + defp matches_no_ref?(input, false) do + String.contains?(@no_ref_downcase, String.downcase(input)) + end + + defp matches_no_ref?(input, true) do + String.contains?(@no_ref, input) + end +end diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.ex new file mode 100644 index 0000000000000000000000000000000000000000..41422d1e5bd8267a76373e72f145dd226f6d4258 --- /dev/null +++ b/lib/plausible/stats/table_decider.ex @@ -0,0 +1,201 @@ +defmodule Plausible.Stats.TableDecider do + @moduledoc """ + This module contains logic for deciding which tables need to be queried given a query + and metrics, with the purpose of reducing the number of queries and JOINs needed to perform. + """ + + use Plausible + + import Enum, only: [empty?: 1] + + import Plausible.Stats.Filters, + only: [dimensions_used_in_filters: 1, filtering_on_dimension?: 2] + + alias Plausible.Stats.{Query, QueryError} + + @revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: []) + + def events_join_sessions?(query) do + session_dims_in_filters? = + query.filters + |> dimensions_used_in_filters() + |> Enum.any?(&(dimension_partitioner(query, &1) == :session)) + + session_dims? = + Enum.any?(query.dimensions, &(dimension_partitioner(query, &1) == :session)) + + session_dims? or session_dims_in_filters? + end + + def sessions_join_events?(query) do + query.filters + |> dimensions_used_in_filters() + |> Enum.any?(&(dimension_partitioner(query, &1) == :event)) + end + + @doc """ + Validates whether metrics and dimensions are compatible with each other. + + During query building we split query into two: event and session queries. However dimensions need to be + present in both queries and hence must be compatible. + + Used during query parsing + """ + def validate_no_metrics_dimensions_conflict(query) do + %{event: event_only_metrics, session: session_only_metrics} = + partition(query.metrics, query, &metric_partitioner/2) + + %{event: event_only_dimensions, session: session_only_dimensions} = + partition(query.dimensions, query, &dimension_partitioner/2) + + conflicting_event_metrics = event_only_metrics -- @revenue_metrics + + cond do + # event:page (optionally with event:hostname) is a special case handled in QueryOptimizer.split_sessions_query + "event:page" in event_only_dimensions and + event_only_dimensions -- ["event:page", "event:hostname"] == [] -> + :ok + + not empty?(session_only_metrics) and not empty?(event_only_dimensions) -> + {:error, + %QueryError{ + code: :invalid_metrics, + message: + "Session metric(s) #{i(session_only_metrics)} cannot be queried along with event dimension(s) #{i(event_only_dimensions)}" + }} + + not empty?(conflicting_event_metrics) and not empty?(session_only_dimensions) -> + {:error, + %QueryError{ + code: :invalid_metrics, + message: + "Event metric(s) #{i(conflicting_event_metrics)} cannot be queried along with session dimension(s) #{i(session_only_dimensions)}" + }} + + true -> + :ok + end + end + + def partition_dimensions(query) do + partition(query.dimensions, query, &dimension_partitioner/2) + end + + @type table_type() :: :events | :sessions + @type metric() :: String.t() + + @spec partition_metrics(list(metric()), Query.t()) :: list({table_type(), list(metric())}) + def partition_metrics(requested_metrics, query) do + metrics = partition(requested_metrics, query, &metric_partitioner/2) + + filters = + query.filters + |> dimensions_used_in_filters() + |> partition(query, &dimension_partitioner/2) + + dimensions = partition(query.dimensions, query, &dimension_partitioner/2) + + cond do + # Only one table needs to be queried + empty?(metrics.event) && empty?(filters.event) && empty?(dimensions.event) -> + [sessions: metrics.session ++ metrics.either ++ metrics.sample_percent] + + empty?(metrics.session) && empty?(filters.session) && empty?(dimensions.session) -> + [events: metrics.event ++ metrics.either ++ metrics.sample_percent] + + # Filters and/or dimensions on both events and sessions, but only one kind of metric + empty?(metrics.event) && empty?(dimensions.event) -> + [sessions: metrics.session ++ metrics.either ++ metrics.sample_percent] + + empty?(metrics.session) && empty?(dimensions.session) -> + [events: metrics.event ++ metrics.either ++ metrics.sample_percent] + + # Default: prefer events + true -> + [ + events: metrics.event ++ metrics.either ++ metrics.sample_percent, + sessions: metrics.session ++ metrics.sample_percent + ] + end + |> Enum.flat_map(&smear_session_metrics(&1, query)) + |> Enum.reject(fn {_table_type, metrics} -> empty?(metrics) end) + end + + # :TRICKY: When counting session metrics, we want to count each visit/visitor across + # the length of the session, not just when events occurred or when session started. + # For this reason, we smear the session metrics across the length of the session. + # See `time_slots` usage in `Plausible.Stats.SQL.Expression` to understand how this is done. + @smearable_metrics [:visitors, :visits] + defp smear_session_metrics({:sessions, metrics} = value, query) do + if ("time:minute" in query.dimensions or "time:hour" in query.dimensions) and + not filtering_on_dimension?(query, "event:goal") do + # Split metrics into two groups: one with visitors and visits, and the remaining ones + {smearable_metrics, session_metrics} = Enum.split_with(metrics, &(&1 in @smearable_metrics)) + + [ + {:sessions, session_metrics}, + {:sessions_smeared, smearable_metrics} + ] + else + [value] + end + end + + defp smear_session_metrics(value, _query), do: [value] + + # Note: This is inaccurate when filtering but required for old backwards compatibility + defp metric_partitioner(%Query{legacy_breakdown: true}, :pageviews), do: :either + defp metric_partitioner(%Query{legacy_breakdown: true}, :events), do: :either + + # :TRICKY: For time:minute dimension we prefer sessions over events as there + # might be minutes where no events occurred but the session was active. + defp metric_partitioner(query, metric) when metric in [:visitors, :visits] do + if "time:minute" in query.dimensions and not filtering_on_dimension?(query, "event:goal") do + :session + else + :either + end + end + + defp metric_partitioner(_, :conversion_rate), do: :either + defp metric_partitioner(_, :group_conversion_rate), do: :either + defp metric_partitioner(_, :percentage), do: :either + + defp metric_partitioner(_, :average_revenue), do: :event + defp metric_partitioner(_, :total_revenue), do: :event + defp metric_partitioner(_, :scroll_depth), do: :event + defp metric_partitioner(_, :pageviews), do: :event + defp metric_partitioner(_, :events), do: :event + defp metric_partitioner(_, :bounce_rate), do: :session + defp metric_partitioner(_, :time_on_page), do: :event + defp metric_partitioner(_, :visit_duration), do: :session + defp metric_partitioner(_, :views_per_visit), do: :session + defp metric_partitioner(_, :exit_rate), do: :session + + # Calculated metrics - handled on callsite separately from other metrics. + defp metric_partitioner(_, :total_visitors), do: :other + # Sample percentage is included in both tables if queried. + defp metric_partitioner(_, :sample_percent), do: :sample_percent + + defp dimension_partitioner(_, "event:" <> _), do: :event + defp dimension_partitioner(_, "visit:entry_page"), do: :session + defp dimension_partitioner(_, "visit:entry_page_hostname"), do: :session + defp dimension_partitioner(_, "visit:exit_page"), do: :session + defp dimension_partitioner(_, "visit:exit_page_hostname"), do: :session + + defp dimension_partitioner(_, "visit:" <> _), do: :either + + defp dimension_partitioner(_, _), do: :either + + @default %{event: [], session: [], either: [], other: [], sample_percent: []} + defp partition(values, query, partitioner) do + Enum.reduce(values, @default, fn value, acc -> + key = partitioner.(query, value) + Map.put(acc, key, Map.fetch!(acc, key) ++ [value]) + end) + end + + defp i(list) when is_list(list) do + Enum.map_join(list, ", ", &"`#{&1}`") + end +end diff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.ex new file mode 100644 index 0000000000000000000000000000000000000000..220f9b134f7ffe2132b5fffa8c8de317518ebb86 --- /dev/null +++ b/lib/plausible/stats/time.ex @@ -0,0 +1,233 @@ +defmodule Plausible.Stats.Time do + @moduledoc """ + Collection of functions to work with time in queries. + """ + + alias Plausible.Stats.{Query, DateTimeRange} + + def utc_boundaries(%Query{ + utc_time_range: time_range, + site_native_stats_start_at: native_stats_start_at + }) do + first = + time_range.first + |> DateTime.to_naive() + |> beginning_of_time(native_stats_start_at) + + last = DateTime.to_naive(time_range.last) + + {first, last} + end + + defp beginning_of_time(candidate, native_stats_start_at) do + if NaiveDateTime.after?(native_stats_start_at, candidate) do + native_stats_start_at + else + candidate + end + end + + def format_datetime(%Date{} = date), do: Date.to_string(date) + + def format_datetime(%mod{} = datetime) when mod in [NaiveDateTime, DateTime], + do: Calendar.strftime(datetime, "%Y-%m-%d %H:%M:%S") + + # Realtime graphs return numbers + def format_datetime(other), do: other + + def time_dimension(query) do + Enum.find(query.dimensions, &time_dimension?/1) + end + + def time_dimension?("time" <> _rest), do: true + def time_dimension?(_dimension), do: false + + @doc """ + Returns list of time bucket labels for the given query. + """ + def time_labels(query) do + time_labels_for_dimension(time_dimension(query), query) + end + + defp time_labels_for_dimension("time:month", query) do + date_range = Query.date_range(query) + + n_buckets = + Plausible.Times.diff( + date_range.last, + Date.beginning_of_month(date_range.first), + :month + ) + + Enum.map(n_buckets..0//-1, fn shift -> + date_range.last + |> Date.beginning_of_month() + |> Date.shift(month: -shift) + |> format_datetime() + end) + end + + defp time_labels_for_dimension("time:week", query) do + date_range = Query.date_range(query) + + n_buckets = + Plausible.Times.diff( + date_range.last, + Date.beginning_of_week(date_range.first), + :week + ) + + Enum.map(0..n_buckets, fn shift -> + date_range.first + |> Date.shift(week: shift) + |> date_or_weekstart(date_range) + |> format_datetime() + end) + end + + defp time_labels_for_dimension("time:day", query) do + Query.date_range(query) + |> Enum.into([]) + |> Enum.map(&format_datetime/1) + end + + defp time_labels_for_dimension("time:hour", query) do + time_range = query.utc_time_range |> DateTimeRange.to_timezone(query.timezone) + + from_timestamp = time_range.first |> Map.merge(%{minute: 0, second: 0}) + n_buckets = DateTime.diff(time_range.last, from_timestamp, :hour) + + Enum.map(0..n_buckets, fn step -> + from_timestamp + |> DateTime.to_naive() + |> NaiveDateTime.shift(hour: step) + |> format_datetime() + end) + end + + defp time_labels_for_dimension("time:minute", query) do + time_range = query.utc_time_range |> DateTimeRange.to_timezone(query.timezone) + first_datetime = Map.put(time_range.first, :second, 0) + + first_datetime + |> Stream.iterate(fn datetime -> DateTime.shift(datetime, minute: 1) end) + |> Enum.take_while(fn datetime -> + current_minute = Map.put(query.now, :second, 0) + + DateTime.before?(datetime, time_range.last) && + DateTime.before?(datetime, current_minute) + end) + |> Enum.map(&format_datetime/1) + end + + def partial_time_labels(time_labels, query) do + time_dimension = time_dimension(query) + + range_start = to_naive_in_tz!(query.utc_time_range.first, query.timezone) + range_end = to_naive_in_tz!(query.utc_time_range.last, query.timezone) + now = to_naive_in_tz!(query.now, query.timezone) + + cutoff = if NaiveDateTime.before?(now, range_end), do: now, else: range_end + + first_bucket = List.first(time_labels) + last_bucket = List.last(time_labels) + + first_partial? = + case bucket_start(first_bucket, time_dimension) do + nil -> false + start -> NaiveDateTime.after?(range_start, start) + end + + last_partial? = + case bucket_end(last_bucket, time_dimension) do + nil -> false + bucket_end -> NaiveDateTime.after?(bucket_end, cutoff) + end + + [ + if(first_partial?, do: first_bucket), + if(last_partial?, do: last_bucket) + ] + |> Enum.uniq() + |> Enum.reject(&is_nil/1) + end + + defp bucket_start(label, "time:week") do + case Date.from_iso8601(label) do + {:ok, date} -> NaiveDateTime.new!(Date.beginning_of_week(date), ~T[00:00:00]) + _ -> nil + end + end + + defp bucket_start(label, _time_dimension) do + case Date.from_iso8601(label) do + {:ok, date} -> + NaiveDateTime.new!(date, ~T[00:00:00]) + + _ -> + case NaiveDateTime.from_iso8601(label) do + {:ok, naive_datetime} -> naive_datetime + _ -> nil + end + end + end + + defp bucket_end(label, time_dimension) do + shift_unit = + case time_dimension do + "time:month" -> :month + "time:week" -> :week + "time:day" -> :day + "time:hour" -> :hour + "time:minute" -> :minute + end + + case bucket_start(label, time_dimension) do + nil -> nil + start -> NaiveDateTime.shift(start, [{shift_unit, 1}, {:second, -1}]) + end + end + + defp to_naive_in_tz!(utc_datetime, timezone) do + Plausible.Times.to_naive_datetime!(utc_datetime, timezone) + end + + def present_index(time_labels, query) do + now = DateTime.shift_zone!(query.now, query.timezone) + + current_label = + case time_dimension(query) do + "time:month" -> + DateTime.to_date(now) + |> Date.beginning_of_month() + |> Date.to_string() + + "time:week" -> + DateTime.to_date(now) + |> date_or_weekstart(Query.date_range(query)) + |> Date.to_string() + + "time:day" -> + DateTime.to_date(now) + |> Date.to_string() + + "time:hour" -> + Calendar.strftime(now, "%Y-%m-%d %H:00:00") + + "time:minute" -> + Calendar.strftime(now, "%Y-%m-%d %H:%M:00") + end + + Enum.find_index(time_labels, &(&1 == current_label)) + end + + def date_or_weekstart(date, date_range) do + weekstart = Date.beginning_of_week(date) + + if Enum.member?(date_range, weekstart) do + weekstart + else + date + end + end +end diff --git a/lib/plausible/stats/time_on_page.ex b/lib/plausible/stats/time_on_page.ex new file mode 100644 index 0000000000000000000000000000000000000000..5cf68b97892b877655a6293c74d82768511cb624 --- /dev/null +++ b/lib/plausible/stats/time_on_page.ex @@ -0,0 +1,27 @@ +defmodule Plausible.Stats.TimeOnPage do + @moduledoc """ + Module to check whether the new time on page metric is available. + """ + + def new_time_on_page_visible?(site) do + not is_nil(site.legacy_time_on_page_cutoff) + end + + def legacy_time_on_page_cutoff(site) do + cutoff_datetime(site.legacy_time_on_page_cutoff, site.timezone) + end + + def cutoff_datetime(nil, _timezone), do: nil + + # Workaround for a case where casting unix epoch to DateTime fails for sites that should + # always have the new time-on-page only. Affects CSV imports. + def cutoff_datetime(~D[1970-01-01], _timezone), do: ~U[2000-01-01 00:00:00Z] + + def cutoff_datetime(date, timezone) do + case DateTime.new(date, ~T[00:00:00], timezone) do + {:ok, datetime} -> datetime + {:gap, just_before, _just_after} -> just_before + {:ambiguous, first_datetime, _second_datetime} -> first_datetime + end + end +end diff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.ex new file mode 100644 index 0000000000000000000000000000000000000000..e866fdc3ef38e9a707ebe55e41b1c98d92b5200e --- /dev/null +++ b/lib/plausible/stats/util.ex @@ -0,0 +1,52 @@ +defmodule Plausible.Stats.Util do + @moduledoc """ + Utilities for modifying stat results + """ + + @doc """ + This function adds the `visitors` metric into the list of + given metrics if it's not already there and if it is needed + for any of the other metrics to be calculated. + """ + def maybe_add_visitors_metric(metrics) do + needed? = + Enum.any?( + [:percentage, :conversion_rate, :group_conversion_rate, :time_on_page], + &(&1 in metrics) + ) + + if needed? and :visitors not in metrics do + metrics ++ [:visitors] + else + metrics + end + end + + def shortname(_query, metric) when is_atom(metric), do: metric + def shortname(_query, "time:" <> _), do: :time + + def shortname(query, dimension) do + index = Enum.find_index(query.dimensions, &(&1 == dimension)) + :"dim#{index}" + end + + def percentage(x, y) when is_integer(x) and x > 0 and is_integer(y) and y > 0 do + result = + x + |> Decimal.div(y) + |> Decimal.mult(100) + |> Decimal.round(2) + |> Decimal.to_string() + + case result do + <> -> compact + <> -> compact + <> -> compact + decimal -> decimal + end + end + + def percentage(_x, _y) do + "0" + end +end diff --git a/lib/plausible/teams.ex b/lib/plausible/teams.ex new file mode 100644 index 0000000000000000000000000000000000000000..7cbde3c591fa546d15f117b641627d133591486d --- /dev/null +++ b/lib/plausible/teams.ex @@ -0,0 +1,449 @@ +defmodule Plausible.Teams do + @moduledoc """ + Core context of teams. + """ + + import Ecto.Query + + alias __MODULE__ + alias Plausible.Auth + alias Plausible.Billing + alias Plausible.Repo + use Plausible + + @accept_traffic_until_free ~D[2135-01-01] + + @spec default_name() :: String.t() + def default_name(), do: "My personal sites" + + @spec name(nil | Teams.Team.t()) :: String.t() + def name(nil), do: default_name() + def name(%{setup_complete: false}), do: default_name() + def name(team), do: team.name + + @spec setup?(nil | Teams.Team.t()) :: boolean() + def setup?(nil), do: false + def setup?(%{setup_complete: setup_complete}), do: setup_complete + + @spec get(pos_integer() | binary() | nil) :: Teams.Team.t() | nil + def get(nil), do: nil + + def get(team_id) when is_integer(team_id) do + Repo.get(Teams.Team, team_id) + end + + def get(team_identifier) when is_binary(team_identifier) do + case Ecto.UUID.cast(team_identifier) do + {:ok, uuid} -> + Repo.get_by(Teams.Team, identifier: uuid) + + :error -> + nil + end + end + + @spec get!(pos_integer() | binary()) :: Teams.Team.t() + def get!(team_id) when is_integer(team_id) do + Repo.get!(Teams.Team, team_id) + end + + def get!(team_identifier) when is_binary(team_identifier) do + Repo.get_by!(Teams.Team, identifier: team_identifier) + end + + @spec on_trial?(Teams.Team.t() | nil) :: boolean() + on_ee do + def on_trial?(nil), do: false + def on_trial?(%Teams.Team{trial_expiry_date: nil}), do: false + + def on_trial?(team) do + team = with_subscription(team) + + is_nil(team.subscription) and trial_days_left(team) >= 0 + end + else + def on_trial?(_), do: always(true) + end + + @spec locked?(Teams.Team.t() | nil) :: boolean() + def locked?(nil), do: false + + def locked?(%Teams.Team{locked: locked}), do: locked + + @spec trial_days_left(Teams.Team.t()) :: integer() + def trial_days_left(nil) do + nil + end + + def trial_days_left(team) do + Date.diff(team.trial_expiry_date, Date.utc_today()) + end + + def with_subscription(team) do + Repo.preload(team, subscription: last_subscription_query()) + end + + @spec owned_sites(Teams.Team.t() | nil, pos_integer() | nil) :: [Plausible.Site.t()] + def owned_sites(team, limit \\ nil) + + def owned_sites(nil, _), do: [] + + def owned_sites(team, limit) do + query = + from(s in Plausible.Site.regular(), + where: s.team_id == ^team.id, + order_by: [asc: s.domain] + ) + + if limit do + query + |> limit(^limit) + |> Repo.all() + else + Repo.all(query) + end + end + + @spec owned_sites_ids(Teams.Team.t() | nil) :: [pos_integer()] + def owned_sites_ids(nil) do + [] + end + + def owned_sites_ids(team) do + Repo.all( + from(s in Plausible.Site.regular(), + where: s.team_id == ^team.id, + select: s.id, + order_by: [desc: s.id] + ) + ) + end + + @spec owned_sites_count(Teams.Team.t() | nil) :: non_neg_integer() + def owned_sites_count(nil), do: 0 + + def owned_sites_count(team) do + Repo.aggregate( + from(s in Plausible.Site.regular(), + where: s.team_id == ^team.id + ), + :count + ) + end + + def has_active_sites?(team) do + team + |> owned_sites() + |> Enum.any?(&Plausible.Sites.has_stats?/1) + end + + @doc """ + Get or create user's team. + + If the user has no non-guest membership yet, an implicit "My Personal Sites" team is + created with them as an owner. + + If the user already has an owner membership in an existing team, + that team is returned. + """ + @spec get_or_create(Auth.User.t()) :: + {:ok, Teams.Team.t()} | {:error, :multiple_teams | :permission_denied} + def get_or_create(user) do + with :ok <- check_user_type(user), + {:error, :no_team} <- get_owned_team(user, only_not_setup?: true) do + case create_my_team(user) do + {:ok, team} -> + {:ok, team} + + {:error, :exists_already} -> + get_owned_team(user, only_not_setup?: true) + end + end + end + + @spec force_create_my_team(Auth.User.t()) :: Teams.Team.t() + def force_create_my_team(user) do + # This is going to crash hard for SSO user. This shouldn't happen + # under normal circumstances except in case of a _very_ unlucky timing. + # Manual resolution is necessary anyway. + case check_user_type(user) do + :ok -> :pass + _ -> raise "SSO user tried to force create a personal team" + end + + {:ok, team} = + Repo.transaction(fn -> + clear_autocreated(user) + {:ok, team} = create_my_team(user) + team + end) + + team + end + + @spec complete_setup(Teams.Team.t()) :: Teams.Team.t() + def complete_setup(team) do + if not team.setup_complete do + team = + team + |> Teams.Team.setup_changeset() + |> Repo.update!() + |> Repo.preload(:owners) + + [owner] = team.owners + + clear_autocreated(owner) + + team + else + team + end + end + + @spec delete(Teams.Team.t()) :: {:ok, :deleted} | {:error, :active_subscription} + def delete(team) do + team = Teams.with_subscription(team) + + if Billing.Subscription.Status.active?(team.subscription) do + {:error, :active_subscription} + else + Repo.transaction(fn -> + for site <- Teams.owned_sites(team) do + Plausible.Site.Removal.run(site) + end + + Repo.delete_all(from s in Billing.Subscription, where: s.team_id == ^team.id) + + Repo.delete_all(from ep in Billing.EnterprisePlan, where: ep.team_id == ^team.id) + + Repo.delete!(team) + + :deleted + end) + end + end + + @spec get_by_owner(Auth.User.t(), Keyword.t()) :: + {:ok, Teams.Team.t()} | {:error, :no_team | :multiple_teams} + def get_by_owner(user, opts \\ []) do + get_owned_team(user, opts) + end + + @spec update_accept_traffic_until(Teams.Team.t()) :: Teams.Team.t() + def update_accept_traffic_until(team) do + team + |> Ecto.Changeset.change(accept_traffic_until: accept_traffic_until(team)) + |> Repo.update!() + end + + def start_trial(%Teams.Team{} = team) do + team + |> Teams.Team.start_trial() + |> Repo.update!() + end + + def start_grace_period(team) do + team + |> Teams.GracePeriod.start_changeset() + |> Repo.update!() + end + + def start_manual_lock_grace_period(team) do + team + |> Teams.GracePeriod.start_manual_lock_changeset() + |> Repo.update!() + end + + def end_grace_period(team) do + team + |> Teams.GracePeriod.end_changeset() + |> Repo.update!() + end + + def remove_grace_period(team) do + team + |> Teams.GracePeriod.remove_changeset() + |> Repo.update!() + end + + def maybe_reset_next_upgrade_override(%Teams.Team{} = team) do + if team.allow_next_upgrade_override do + team + |> Ecto.Changeset.change(allow_next_upgrade_override: false) + |> Repo.update!() + else + team + end + end + + @spec accept_traffic_until(Teams.Team.t()) :: Date.t() + on_ee do + def accept_traffic_until(team) do + team = with_subscription(team) + + cond do + on_trial?(team) -> + Date.shift(team.trial_expiry_date, + day: Teams.Team.trial_accept_traffic_until_offset_days() + ) + + team.subscription && team.subscription.paddle_plan_id == "free_10k" -> + @accept_traffic_until_free + + team.subscription && team.subscription.next_bill_date -> + Date.shift(team.subscription.next_bill_date, + day: Teams.Team.subscription_accept_traffic_until_offset_days() + ) + + true -> + raise "This user is neither on trial or has a valid subscription. Manual intervention required." + end + end + else + def accept_traffic_until(_user) do + @accept_traffic_until_free + end + end + + @spec last_subscription_join_query() :: Ecto.Query.t() + def last_subscription_join_query() do + from(subscription in last_subscription_query(), + where: subscription.team_id == parent_as(:team).id + ) + end + + @spec last_subscription_query() :: Ecto.Query.t() + def last_subscription_query() do + from(subscription in Plausible.Billing.Subscription, + order_by: [desc: subscription.inserted_at, desc: subscription.id], + limit: 1 + ) + end + + @spec force_2fa_enabled?(Teams.Team.t() | nil) :: boolean() + def force_2fa_enabled?(nil), do: false + + def force_2fa_enabled?(team) do + team.policy.force_2fa + end + + @spec enable_force_2fa(Teams.Team.t(), Auth.User.t()) :: + {:ok, Teams.Team.t()} | {:error, Ecto.Changeset.t()} + def enable_force_2fa(team, user) do + with {:ok, team} <- set_force_2fa(team, true) do + team + |> Teams.Memberships.all(exclude_guests?: true) + |> Enum.each(fn membership -> + if membership.user.id != user.id and not Auth.TOTP.enabled?(membership.user) do + team + |> PlausibleWeb.Email.force_2fa_enabled(membership.user, user) + |> Plausible.Mailer.deliver_later() + end + end) + + {:ok, team} + end + end + + @spec disable_force_2fa(Teams.Team.t(), Auth.User.t(), String.t()) :: + {:ok, Teams.Team.t()} | {:error, :invalid_password | Ecto.Changeset.t()} + def disable_force_2fa(team, user, password) do + if Auth.Password.match?(password, user.password_hash) do + set_force_2fa(team, false) + else + {:error, :invalid_password} + end + end + + defp set_force_2fa(team, enabled?) do + params = %{policy: %{force_2fa: enabled?}} + + audit_entry_name = if(enabled?, do: "force_2fa_enabled", else: "force_2fa_disabled") + + team + |> Ecto.Changeset.cast(params, []) + |> Ecto.Changeset.cast_embed(:policy, + with: &Teams.Policy.force_2fa_changeset(&1, &2.force_2fa) + ) + |> Repo.update_with_audit(audit_entry_name, %{team_id: team.id}) + end + + # Exposed for use in tests + @doc false + def get_owned_team(user, opts \\ []) do + only_not_setup? = Keyword.get(opts, :only_not_setup?, false) + + query = + from(tm in Teams.Membership, + inner_join: t in assoc(tm, :team), + as: :team, + where: tm.user_id == ^user.id and tm.role == :owner, + select: t, + order_by: t.id + ) + + query = + if only_not_setup? do + where(query, [team: t], t.setup_complete == false) + else + query + end + + result = Repo.all(query) + + case result do + [] -> + {:error, :no_team} + + [team] -> + {:ok, team} + + _teams -> + {:error, :multiple_teams} + end + end + + defp check_user_type(user) do + if Plausible.Users.type(user) == :sso do + {:error, :permission_denied} + else + :ok + end + end + + defp clear_autocreated(user) do + Repo.update_all( + from(tm in Teams.Membership, + where: tm.user_id == ^user.id, + where: tm.is_autocreated == true + ), + set: [is_autocreated: false] + ) + + :ok + end + + defp create_my_team(user) do + team = + %Teams.Team{} + |> Teams.Team.changeset(%{name: default_name()}) + |> Repo.insert!() + + team_membership = + team + |> Teams.Membership.changeset(user, :owner) + |> Ecto.Changeset.put_change(:is_autocreated, true) + |> Repo.insert!( + on_conflict: :nothing, + conflict_target: + {:unsafe_fragment, "(user_id) WHERE role = 'owner' and is_autocreated = true"} + ) + + if team_membership.id do + {:ok, team} + else + Repo.delete!(team) + {:error, :exists_already} + end + end +end diff --git a/lib/plausible/teams/billing.ex b/lib/plausible/teams/billing.ex new file mode 100644 index 0000000000000000000000000000000000000000..2074403fdef41c70c12063643cfa8059aaee116e --- /dev/null +++ b/lib/plausible/teams/billing.ex @@ -0,0 +1,714 @@ +defmodule Plausible.Teams.Billing do + @moduledoc false + + use Plausible + + import Ecto.Query + + alias Plausible.Billing.EnterprisePlan + alias Plausible.Billing.Subscription + alias Plausible.Billing.Subscriptions + alias Plausible.Repo + alias Plausible.Teams + + alias Plausible.Billing.{EnterprisePlan, Feature, Plan, Plans, Quota} + alias Plausible.Billing.Feature.{Goals, Props, SitesAPI, StatsAPI, SharedLinks, SSO} + + require Plausible.Billing.Subscription.Status + + # Features that are always available, regardless of team plan + @free_features [Goals] + + @limit_sites_since ~D[2021-05-05] + + @max_sites_for_usage_breakdown 10 + def max_sites_for_usage_breakdown, do: @max_sites_for_usage_breakdown + + @typep last_30_days_usage() :: %{:last_30_days => Quota.usage_cycle()} + @typep monthly_pageview_usage() :: Quota.cycles_usage() | last_30_days_usage() + + def free_features(), do: @free_features + + def grandfathered_team?(nil), do: false + + def grandfathered_team?(team) do + # timestamps were originally rewritten from owner.inserted_at + Date.before?(team.inserted_at, @limit_sites_since) + end + + def get_subscription(nil), do: nil + + def get_subscription(%Teams.Team{subscription: %Subscription{} = subscription}), + do: subscription + + def get_subscription(%Teams.Team{} = team) do + Teams.with_subscription(team).subscription + end + + def change_plan(team, new_plan_id) do + subscription = active_subscription_for(team) + plan = Plausible.Billing.Plans.find(new_plan_id) + + limit_checking_opts = + if team.allow_next_upgrade_override do + [skip_pageview_limit_check?: true] + else + [] + end + + usage = quota_usage(team) + + with :ok <- + Plausible.Billing.Quota.ensure_within_plan_limits(usage, plan, limit_checking_opts), + do: do_change_plan(subscription, new_plan_id) + end + + defp do_change_plan(subscription, new_plan_id) do + res = + Plausible.Billing.paddle_api().update_subscription(subscription.paddle_subscription_id, %{ + plan_id: new_plan_id + }) + + case res do + {:ok, response} -> + amount = :erlang.float_to_binary(response["next_payment"]["amount"] / 1, decimals: 2) + + Subscription.changeset(subscription, %{ + paddle_plan_id: Integer.to_string(response["plan_id"]), + next_bill_amount: amount, + next_bill_date: response["next_payment"]["date"] + }) + |> Repo.update() + + e -> + e + end + end + + def enterprise_configured?(nil), do: false + + def enterprise_configured?(%Teams.Team{} = team) do + team + |> Ecto.assoc(:enterprise_plan) + |> Repo.exists?() + end + + def latest_enterprise_plan_with_price(team, customer_ip) do + enterprise_plan = + Repo.one!( + from(e in EnterprisePlan, + where: e.team_id == ^team.id, + order_by: [desc: e.inserted_at], + limit: 1 + ) + ) + + {enterprise_plan, Plausible.Billing.Plans.get_price_for(enterprise_plan, customer_ip)} + end + + def has_active_subscription?(nil), do: false + + def has_active_subscription?(team) do + team + |> active_subscription_query() + |> Repo.exists?() + end + + def active_subscription_for(nil), do: nil + + def active_subscription_for(team) do + team + |> active_subscription_query() + |> Repo.one() + end + + @spec check_needs_to_upgrade(Teams.Team.t() | nil, atom()) :: + {:needs_to_upgrade, :no_active_trial_or_subscription | :grace_period_ended} + | :no_upgrade_needed + def check_needs_to_upgrade(team_or_nil, usage_mod \\ Teams.Billing) + + def check_needs_to_upgrade(nil, _usage_mod), + do: {:needs_to_upgrade, :no_active_trial_or_subscription} + + def check_needs_to_upgrade(team, usage_mod) do + team = Teams.with_subscription(team) + + cond do + Plausible.Teams.on_trial?(team) -> + :no_upgrade_needed + + not Subscriptions.active?(team.subscription) -> + {:needs_to_upgrade, :no_active_trial_or_subscription} + + Teams.GracePeriod.expired?(team) -> + revise_pageview_usage(team, usage_mod) + + true -> + :no_upgrade_needed + end + end + + defp revise_pageview_usage(team, usage_mod) do + case Plausible.Workers.CheckUsage.check_pageview_usage_two_cycles(team, usage_mod) do + {:over_limit, _} -> + {:needs_to_upgrade, :grace_period_ended} + + {:below_limit, _} -> + Plausible.Teams.remove_grace_period(team) + :no_upgrade_needed + end + end + + @doc """ + Enterprise plans are always allowed to add more sites (even when + over limit) to avoid service disruption. Their usage is checked + in a background job instead (see `check_usage.ex`). + """ + def ensure_can_add_new_site(nil) do + :ok + end + + def ensure_can_add_new_site(team) do + team = Teams.with_subscription(team) + + case Plans.get_subscription_plan(team.subscription) do + %EnterprisePlan{} -> + :ok + + _ -> + usage = site_usage(team) + limit = site_limit(team) + + if Plausible.Billing.Quota.below_limit?(usage, limit) do + :ok + else + {:error, {:over_limit, limit}} + end + end + end + + on_ee do + @site_limit_for_trials 10 + + def site_limit(nil) do + @site_limit_for_trials + end + + def site_limit(team) do + if grandfathered_team?(team) do + :unlimited + else + get_site_limit_from_plan(team) + end + end + + defp get_site_limit_from_plan(team) do + team = + Teams.with_subscription(team) + + case Plans.get_subscription_plan(team.subscription) do + %{site_limit: site_limit} -> site_limit + :free_10k -> 50 + nil -> @site_limit_for_trials + end + end + else + def site_limit(_team), do: :unlimited + end + + @doc """ + Returns the number of sites the given team owns. + """ + @spec site_usage(Teams.Team.t() | nil) :: non_neg_integer() + def site_usage(nil), do: 0 + + def site_usage(team) do + Teams.owned_sites_count(team) + end + + on_ee do + @team_member_limit_for_trials 10 + + def team_member_limit(nil) do + @team_member_limit_for_trials + end + + def team_member_limit(team) do + team = Teams.with_subscription(team) + + case Plans.get_subscription_plan(team.subscription) do + %{team_member_limit: limit} -> limit + :free_10k -> :unlimited + nil -> @team_member_limit_for_trials + end + end + + def solo?(nil), do: true + + def solo?(team) do + team_member_limit(team) == 0 + end + else + def team_member_limit(_team), do: :unlimited + + def solo?(_team), do: always(false) + end + + @doc """ + Returns a full usage report for the team. + + ### Options + + * `pending_ownership_site_ids` - a list of site IDs from which to count + additional usage. This allows us to look at the total usage from pending + ownerships and owned sites at the same time, which is useful, for example, + when deciding whether to let the team owner upgrade to a plan, or accept a + site ownership. + + * `with_features` - when `true`, the returned map will contain features + usage. Also counts usage from `pending_ownership_site_ids` if that option + is given. + """ + def quota_usage(team, opts \\ []) do + team = Teams.with_subscription(team) + with_features? = Keyword.get(opts, :with_features, false) + pending_site_ids = Keyword.get(opts, :pending_ownership_site_ids, []) + team_site_ids = Teams.owned_sites_ids(team) + all_site_ids = pending_site_ids ++ team_site_ids + + monthly_pageviews = monthly_pageview_usage(team, all_site_ids) + team_member_usage = team_member_usage(team, pending_ownership_site_ids: pending_site_ids) + + basic_usage = %{ + monthly_pageviews: monthly_pageviews, + team_members: team_member_usage, + sites: length(all_site_ids) + } + + if with_features? do + Map.put(basic_usage, :features, features_usage(team, all_site_ids)) + else + basic_usage + end + end + + @monthly_pageview_limit_for_free_10k 10_000 + @monthly_pageview_limit_for_trials :unlimited + + def monthly_pageview_limit(nil) do + @monthly_pageview_limit_for_trials + end + + def monthly_pageview_limit(%Teams.Team{} = team) do + team = Teams.with_subscription(team) + monthly_pageview_limit(team.subscription) + end + + def monthly_pageview_limit(subscription) do + case Plans.get_subscription_plan(subscription) do + %EnterprisePlan{monthly_pageview_limit: limit} -> + limit + + %Plan{monthly_pageview_limit: limit} -> + limit + + :free_10k -> + @monthly_pageview_limit_for_free_10k + + _any -> + Sentry.capture_message("Unknown monthly pageview limit for plan", + extra: %{paddle_plan_id: subscription.paddle_plan_id} + ) + + @monthly_pageview_limit_for_trials + end + end + + @doc """ + Queries the ClickHouse database for the monthly pageview usage. If the given team's + subscription is `active`, `past_due`, or a `deleted` (but not yet expired), a map + with the following structure is returned: + + ```elixir + %{ + current_cycle: usage_cycle(), + last_cycle: usage_cycle(), + penultimate_cycle: usage_cycle() + } + ``` + + In all other cases of the subscription status (or a `free_10k` subscription which + does not have a `last_bill_date` defined) - the following structure is returned: + + ```elixir + %{last_30_days: usage_cycle()} + ``` + + Given only a team as input, the usage is queried from across all the sites that the + team owns. Alternatively, given an optional argument of `site_ids`, the usage from + across all those sites is queried instead. + """ + @spec monthly_pageview_usage(Teams.Team.t() | nil, list() | nil) :: monthly_pageview_usage() + def monthly_pageview_usage(team, site_ids \\ nil) + + def monthly_pageview_usage(team, nil) do + monthly_pageview_usage(team, Teams.owned_sites_ids(team)) + end + + def monthly_pageview_usage(nil, _site_ids) do + %{last_30_days: usage_cycle(nil, :last_30_days, [])} + end + + def monthly_pageview_usage(team, site_ids) do + team = Teams.with_subscription(team) + active_subscription? = Subscriptions.active?(team.subscription) + + if active_subscription? and team.subscription.last_bill_date != nil do + [:current_cycle, :last_cycle, :penultimate_cycle] + |> Task.async_stream(fn cycle -> + {cycle, usage_cycle(team, cycle, site_ids)} + end) + |> Enum.into(%{}, fn {:ok, cycle_usage} -> cycle_usage end) + else + %{last_30_days: usage_cycle(team, :last_30_days, site_ids)} + end + end + + @spec team_member_usage(Teams.Team.t() | nil, Keyword.t()) :: non_neg_integer() + @doc """ + Returns the total count of team members associated with the team's sites. + + * The given team's owner is not counted as a team member. + + * Pending invitations (but not ownership transfers) are counted as team + members even before accepted. + + * Users are counted uniquely - i.e. even if an account is associated with + many sites owned by the given user, they still count as one team member. + + ### Options + + * `exclude_emails` - a list of emails to not count towards the usage. This + allows us to exclude a user from being counted as a team member when + checking whether a site invitation can be created for that same user. + + * `pending_ownership_site_ids` - a list of site IDs from which to count + additional team member usage. Without this option, usage is queried only + across sites owned by the given user. + """ + def team_member_usage(team, opts \\ []) + def team_member_usage(nil, _), do: 0 + + def team_member_usage(team, opts) do + [owner | _] = Repo.preload(team, :owners).owners + exclude_emails = Keyword.get(opts, :exclude_emails, []) ++ [owner.email] + + pending_site_ids = Keyword.get(opts, :pending_ownership_site_ids, []) + + team + |> query_team_member_emails(pending_site_ids, exclude_emails) + |> Repo.aggregate(:count) + end + + def usage_cycle(team, cycle, owned_site_ids \\ nil, today \\ Date.utc_today()) + + def usage_cycle(team, cycle, nil, today) do + owned_site_ids = Teams.owned_sites_ids(team) + usage_cycle(team, cycle, owned_site_ids, today) + end + + def usage_cycle(_team, :last_30_days, owned_site_ids, today) do + date_range = Date.range(Date.shift(today, day: -30), today) + + {pageviews, custom_events} = + Plausible.Stats.Clickhouse.usage_breakdown(owned_site_ids, date_range) + + %{ + date_range: date_range, + pageviews: pageviews, + custom_events: custom_events, + total: pageviews + custom_events, + per_site: per_site_usage(owned_site_ids, date_range) + } + end + + def usage_cycle(team, cycle, owned_site_ids, today) do + team = Teams.with_subscription(team) + last_bill_date = team.subscription.last_bill_date + + normalized_last_bill_date = + Date.shift(last_bill_date, month: Plausible.Times.diff(today, last_bill_date, :month)) + + date_range = + case cycle do + :current_cycle -> + Date.range( + normalized_last_bill_date, + Date.shift(normalized_last_bill_date, month: 1, day: -1) + ) + + :last_cycle -> + Date.range( + Date.shift(normalized_last_bill_date, month: -1), + Date.shift(normalized_last_bill_date, day: -1) + ) + + :penultimate_cycle -> + Date.range( + Date.shift(normalized_last_bill_date, month: -2), + Date.shift(normalized_last_bill_date, day: -1, month: -1) + ) + end + + {pageviews, custom_events} = + Plausible.Stats.Clickhouse.usage_breakdown(owned_site_ids, date_range) + + %{ + date_range: date_range, + pageviews: pageviews, + custom_events: custom_events, + total: pageviews + custom_events, + per_site: per_site_usage(owned_site_ids, date_range) + } + end + + defp per_site_usage(owned_site_ids, _date_range) + when length(owned_site_ids) <= 1 or length(owned_site_ids) > @max_sites_for_usage_breakdown, + do: [] + + defp per_site_usage(owned_site_ids, date_range) do + per_site_usage_breakdown = + Plausible.Stats.Clickhouse.per_site_usage_breakdown(owned_site_ids, date_range) + + active_site_ids = Enum.map(per_site_usage_breakdown, fn {site_id, _, _} -> site_id end) + inactive_site_ids = owned_site_ids -- active_site_ids + + domains = + Map.new( + Repo.all( + from(s in Plausible.Site, + where: s.id in ^owned_site_ids, + select: {s.id, s.domain} + ) + ) + ) + + usage_mapping = + Map.new(per_site_usage_breakdown, fn {site_id, pageviews, custom_events} -> + {site_id, %{pageviews: pageviews, custom_events: custom_events}} + end) + + Enum.map(active_site_ids ++ inactive_site_ids, fn site_id -> + pageviews = usage_mapping[site_id][:pageviews] || 0 + custom_events = usage_mapping[site_id][:custom_events] || 0 + + %{ + domain: Map.get(domains, site_id), + pageviews: pageviews, + custom_events: custom_events, + total: pageviews + custom_events + } + end) + end + + @spec features_usage(Teams.Team.t() | nil, list() | nil) :: [atom()] + @doc """ + Given only a team, this function returns the features used across all the + sites this team owns + StatsAPI if any team user has a configured Stats API key. + + Given a team, and a list of site_ids, returns the features used by those + sites instead + StatsAPI if any user in the team has a configured Stats API key. + + The team can also be passed as `nil`, in which case we will never return + Stats API as a used feature. + """ + def features_usage(team, site_ids \\ nil) + + on_ee do + def features_usage(nil, nil), do: [] + + def features_usage(%Teams.Team{} = team, nil) do + owned_site_ids = Teams.owned_sites_ids(team) + features_usage(team, owned_site_ids) + end + + def features_usage(%Teams.Team{} = team, owned_site_ids) when is_list(owned_site_ids) do + site_scoped_feature_usage = features_usage(nil, owned_site_ids) + + stats_api_used? = + Repo.exists?( + from tm in Plausible.Teams.Membership, + as: :team_membership, + where: tm.team_id == ^team.id, + where: + exists( + from ak in Plausible.Auth.ApiKey, + where: ak.user_id == parent_as(:team_membership).user_id + ) + ) + + site_scoped_feature_usage = + if stats_api_used? do + site_scoped_feature_usage ++ [Feature.StatsAPI] + else + site_scoped_feature_usage + end + + sites_api_used? = + Repo.exists?( + from tm in Plausible.Teams.Membership, + as: :team_membership, + where: tm.team_id == ^team.id, + where: + exists( + from ak in Plausible.Auth.ApiKey, + where: ak.user_id == parent_as(:team_membership).user_id, + where: "sites:provision:*" in ak.scopes + ) + ) + + site_scoped_feature_usage = + if sites_api_used? do + site_scoped_feature_usage ++ [SitesAPI] + else + site_scoped_feature_usage + end + + sso_used? = + case Plausible.Auth.SSO.get_integration_for(team) do + {:ok, _} -> true + _ -> false + end + + if sso_used? do + site_scoped_feature_usage ++ [SSO] + else + site_scoped_feature_usage + end + end + + def features_usage(nil, site_ids) when is_list(site_ids) do + shared_links_usage_q = + from l in Plausible.Site.SharedLink, + where: + l.site_id in ^site_ids and l.name not in ^Plausible.Sites.shared_link_special_names() + + props_usage_q = + from s in Plausible.Site, + where: s.id in ^site_ids and fragment("cardinality(?) > 0", s.allowed_event_props) + + funnels_usage_q = from f in "funnels", where: f.site_id in ^site_ids + + revenue_goals_usage_q = + from g in Plausible.Goal, + where: g.site_id in ^site_ids and not is_nil(g.currency) + + site_segments_usage_q = + from s in Plausible.Segments.Segment, where: s.site_id in ^site_ids and s.type == :site + + site_annotations_usage_q = + from a in Plausible.Annotations.Annotation, + where: a.site_id in ^site_ids and a.type == :site + + [ + {Feature.SharedLinks, shared_links_usage_q}, + {Feature.Props, props_usage_q}, + {Feature.Funnels, funnels_usage_q}, + {Feature.RevenueGoals, revenue_goals_usage_q}, + {Feature.SiteSegments, site_segments_usage_q}, + {Feature.SiteAnnotations, site_annotations_usage_q} + ] + |> Enum.reduce([], fn {feature, query}, acc -> + if Repo.exists?(query), do: acc ++ [feature], else: acc + end) + end + else + def features_usage(_team, _site_ids), do: [] + end + + defp query_team_member_emails(team, pending_ownership_site_ids, exclude_emails) do + pending_owner_memberships_q = + from s in Plausible.Site.regular(), + inner_join: t in assoc(s, :team), + inner_join: tm in assoc(t, :team_memberships), + inner_join: u in assoc(tm, :user), + where: s.id in ^pending_ownership_site_ids, + where: tm.role == :owner, + where: u.email not in ^exclude_emails, + select: %{email: u.email} + + pending_memberships_q = + from tm in Teams.Membership, + inner_join: u in assoc(tm, :user), + left_join: gm in assoc(tm, :guest_memberships), + where: gm.site_id in ^pending_ownership_site_ids, + where: u.email not in ^exclude_emails, + select: %{email: u.email} + + pending_invitations_q = + from ti in Teams.Invitation, + inner_join: gi in assoc(ti, :guest_invitations), + where: gi.site_id in ^pending_ownership_site_ids, + where: ti.email not in ^exclude_emails, + select: %{email: ti.email} + + team_memberships_q = + from tm in Teams.Membership, + inner_join: u in assoc(tm, :user), + where: tm.team_id == ^team.id, + where: u.email not in ^exclude_emails, + select: %{email: u.email} + + team_invitations_q = + from ti in Teams.Invitation, + where: ti.team_id == ^team.id, + where: ti.email not in ^exclude_emails, + select: %{email: ti.email} + + pending_memberships_q + |> union(^pending_owner_memberships_q) + |> union(^pending_invitations_q) + |> union(^team_memberships_q) + |> union(^team_invitations_q) + end + + def allowed_features_for(nil) do + @free_features + end + + def allowed_features_for(team) do + team = Teams.with_subscription(team) + + features = + case Plans.get_subscription_plan(team.subscription) do + %EnterprisePlan{features: features} -> + features + + %Plan{features: features} -> + features + + :free_10k -> + [Props, StatsAPI, SharedLinks] + + nil -> + if Teams.on_trial?(team) do + Feature.list() -- [SitesAPI, SSO] + else + [] + end + end + + features + |> Enum.concat(@free_features) + |> Enum.uniq() + end + + defp active_subscription_query(team) do + from(s in Plausible.Billing.Subscription, + where: + s.team_id == ^team.id and s.status == ^Plausible.Billing.Subscription.Status.active(), + order_by: [desc: s.inserted_at], + limit: 1 + ) + end +end diff --git a/lib/plausible/teams/grace_period.ex b/lib/plausible/teams/grace_period.ex new file mode 100644 index 0000000000000000000000000000000000000000..b773af68adfc6e41cc1531ce67eb03ec17833086 --- /dev/null +++ b/lib/plausible/teams/grace_period.ex @@ -0,0 +1,150 @@ +defmodule Plausible.Teams.GracePeriod do + @moduledoc """ + This embedded schema stores information about the account locking grace + period. + + Teams are given this 7-day grace period to upgrade their account after + outgrowing their subscriptions. The actual account locking happens in + background with `Plausible.Workers.LockSites`. + + The grace period can also be manual, without an end date, being controlled + manually from the CRM, and not by the background site locker job. This is + useful for enterprise subscriptions. + """ + + use Ecto.Schema + alias Plausible.Teams + + @type t() :: %__MODULE__{ + end_date: Date.t() | nil, + is_over: boolean(), + manual_lock: boolean() + } + + embedded_schema do + field :end_date, :date + field :is_over, :boolean + field :manual_lock, :boolean + end + + @spec start_changeset(Teams.Team.t()) :: Ecto.Changeset.t() + @doc """ + Starts a account locking grace period of 7 days by changing the Team struct. + """ + def start_changeset(%Teams.Team{} = team) do + grace_period = %__MODULE__{ + end_date: Date.shift(Date.utc_today(), day: 7), + is_over: false, + manual_lock: false + } + + Ecto.Changeset.change(team, grace_period: grace_period) + end + + @spec start_manual_lock_changeset(Teams.Team.t()) :: Ecto.Changeset.t() + @doc """ + Starts a manual account locking grace period by changing the Team struct. + Manual locking means the grace period can only be removed manually from the + CRM. + """ + def start_manual_lock_changeset(%Teams.Team{} = team) do + grace_period = %__MODULE__{ + end_date: nil, + is_over: false, + manual_lock: true + } + + Ecto.Changeset.change(team, grace_period: grace_period) + end + + @spec end_changeset(Teams.Team.t()) :: Ecto.Changeset.t() + @doc """ + Ends an existing grace period by setting `teams.grace_period.is_over` to true. + This means the grace period has expired. + """ + def end_changeset(%Teams.Team{} = team) do + Ecto.Changeset.change(team, grace_period: %{is_over: true}) + end + + @spec remove_changeset(Teams.Team.t()) :: Ecto.Changeset.t() + @doc """ + Removes the grace period from the Team completely. + """ + def remove_changeset(%Teams.Team{} = team) do + Ecto.Changeset.change(team, grace_period: nil) + end + + @spec active?(Teams.Team.t() | nil) :: boolean() + @doc """ + Returns whether the grace period is still active for a Team. Defaults to + false if the team is nil or there is no grace period. + """ + def active?(team) + + def active?(%{grace_period: %__MODULE__{manual_lock: true}} = team) do + manual_lock_active?(team) + end + + def active?(%{grace_period: %__MODULE__{end_date: %Date{} = end_date}}) do + Date.diff(end_date, Date.utc_today()) >= 0 + end + + def active?(_team), do: false + + @spec manual_lock_active?(Teams.Team.t() | nil) :: boolean() + @doc """ + Returns whether the team has an active manual lock grace period. + Manual locks are used for enterprise customers and have no end date. + """ + def manual_lock_active?(team) + + def manual_lock_active?(%{grace_period: %__MODULE__{manual_lock: true, is_over: false}}), + do: true + + def manual_lock_active?(_team), do: false + + @spec expired?(Teams.Team.t() | nil) :: boolean() + @doc """ + Returns whether the grace period has already expired for a Team. Defaults to + false if the team is nil or there is no grace period. + """ + def expired?(team) do + if team && team.grace_period, do: !active?(team), else: false + end + + @spec expires_in(Teams.Team.t() | nil) :: {non_neg_integer(), :days | :hours} | nil + @doc """ + Returns a tuple representing either the hours (if hours_left < 48) or days left + until the end of a grace period. Switching to hours near the end is to avoid + confusion with timezones. + + Returns `nil` in all the following cases: + + * the given team is `nil` + * the given team does not have a grace period + * the given team has a manual lock grace period + """ + def expires_in(team, now \\ NaiveDateTime.utc_now(:second)) + + def expires_in(%Teams.Team{grace_period: %__MODULE__{end_date: %Date{} = end_date}}, now) do + case full_hours_left(end_date, now) do + hours when hours < 48 -> {hours, :hours} + _ -> {days_left(end_date, now), :days} + end + end + + def expires_in(_, _), do: nil + + defp days_left(%Date{} = end_date, now) do + today = NaiveDateTime.to_date(now) + Date.diff(end_date, today) + end + + defp full_hours_left(%Date{} = end_date, now) do + Date.add(end_date, 1) + |> NaiveDateTime.new!(~T[00:00:00]) + |> NaiveDateTime.diff(now, :second) + |> max(0) + |> div(3600) + end +end diff --git a/lib/plausible/teams/guest_invitation.ex b/lib/plausible/teams/guest_invitation.ex new file mode 100644 index 0000000000000000000000000000000000000000..a8ecc57e294632d47e91cf2ef6ba5441a8006590 --- /dev/null +++ b/lib/plausible/teams/guest_invitation.ex @@ -0,0 +1,29 @@ +defmodule Plausible.Teams.GuestInvitation do + @moduledoc """ + Guest invitation schema + """ + + use Ecto.Schema + + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + schema "guest_invitations" do + field :invitation_id, :string + field :role, Ecto.Enum, values: [:viewer, :editor] + + belongs_to :site, Plausible.Site + belongs_to :team_invitation, Plausible.Teams.Invitation + + timestamps() + end + + def changeset(team_invitation, site, role) do + %__MODULE__{invitation_id: Nanoid.generate()} + |> cast(%{role: role}, [:role]) + |> validate_required(:role) + |> put_assoc(:team_invitation, team_invitation) + |> put_assoc(:site, site) + end +end diff --git a/lib/plausible/teams/guest_membership.ex b/lib/plausible/teams/guest_membership.ex new file mode 100644 index 0000000000000000000000000000000000000000..d045583de298fcf3bcb77f0a2cdb0d4310aa17a9 --- /dev/null +++ b/lib/plausible/teams/guest_membership.ex @@ -0,0 +1,28 @@ +defmodule Plausible.Teams.GuestMembership do + @moduledoc """ + Guest membership schema + """ + + use Ecto.Schema + + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + schema "guest_memberships" do + field :role, Ecto.Enum, values: [:viewer, :editor] + + belongs_to :team_membership, Plausible.Teams.Membership + belongs_to :site, Plausible.Site + + timestamps() + end + + def changeset(team_membership, site, role) do + %__MODULE__{} + |> change() + |> put_change(:role, role) + |> put_assoc(:team_membership, team_membership) + |> put_assoc(:site, site) + end +end diff --git a/lib/plausible/teams/invitation.ex b/lib/plausible/teams/invitation.ex new file mode 100644 index 0000000000000000000000000000000000000000..71d8a977b19c7b9ae00006df16c158d24f09d5f9 --- /dev/null +++ b/lib/plausible/teams/invitation.ex @@ -0,0 +1,40 @@ +defmodule Plausible.Teams.Invitation do + @moduledoc """ + Team invitation schema + """ + + use Ecto.Schema + + import Ecto.Changeset + + @roles Plausible.Teams.Membership.roles() + + @type t() :: %__MODULE__{} + + schema "team_invitations" do + field :invitation_id, :string + field :email, :string + field :role, Ecto.Enum, values: @roles + + belongs_to :inviter, Plausible.Auth.User + belongs_to :team, Plausible.Teams.Team + + has_many :guest_invitations, Plausible.Teams.GuestInvitation, foreign_key: :team_invitation_id + + timestamps() + end + + def roles(), do: @roles + + def changeset(team, opts) do + email = Keyword.fetch!(opts, :email) + role = Keyword.fetch!(opts, :role) + inviter = Keyword.fetch!(opts, :inviter) + + %__MODULE__{invitation_id: Nanoid.generate()} + |> cast(%{email: email, role: role}, [:email, :role]) + |> validate_required([:email, :role]) + |> put_assoc(:team, team) + |> put_assoc(:inviter, inviter) + end +end diff --git a/lib/plausible/teams/invitations.ex b/lib/plausible/teams/invitations.ex new file mode 100644 index 0000000000000000000000000000000000000000..cb849b53266b696287eb20e4cc08fe1a371fd70f --- /dev/null +++ b/lib/plausible/teams/invitations.ex @@ -0,0 +1,871 @@ +defmodule Plausible.Teams.Invitations do + @moduledoc false + + use Plausible + + import Ecto.Query + + alias Plausible.Auth + alias Plausible.Billing + alias Plausible.Repo + alias Plausible.Teams + + def get_team_invitation(team, invitation_id) do + invitation = Repo.get_by(Teams.Invitation, team_id: team.id, invitation_id: invitation_id) + + if invitation do + {:ok, invitation} + else + {:error, :invitation_not_found} + end + end + + @spec find_by_team_identifier(String.t() | nil, Plausible.Auth.User.t()) :: + {:ok, Teams.Invitation.t()} | {:error, :invitation_not_found} + def find_by_team_identifier(nil, _user), do: {:error, :invitation_not_found} + + def find_by_team_identifier(team_identifier, user) do + invitation_query = + from ti in Teams.Invitation, + inner_join: inviter in assoc(ti, :inviter), + inner_join: team in assoc(ti, :team), + where: ti.email == ^user.email, + where: team.identifier == ^team_identifier, + where: ti.role != :guest, + preload: [inviter: inviter, team: team] + + case Repo.one(invitation_query) do + nil -> + {:error, :invitation_not_found} + + invitation -> + {:ok, invitation} + end + end + + def find_for_user(invitation_or_transfer_id, user) do + with {:error, :invitation_not_found} <- + find_team_invitation_for_user(invitation_or_transfer_id, user), + {:error, :invitation_not_found} <- + find_guest_invitation_for_user(invitation_or_transfer_id, user) do + find_transfer_for_user(invitation_or_transfer_id, user) + end + end + + def find_for_site(invitation_or_transfer_id, site) do + with {:error, :invitation_not_found} <- + find_invitation_for_site(invitation_or_transfer_id, site) do + find_transfer_for_site(invitation_or_transfer_id, site) + end + end + + @spec pending_team_invitations_for(Auth.User.t() | Teams.Team.t()) :: [Teams.Invitation.t()] + def pending_team_invitations_for(%Teams.Team{} = team) do + Repo.all( + from ti in Teams.Invitation, + inner_join: inviter in assoc(ti, :inviter), + where: ti.team_id == ^team.id, + preload: [inviter: inviter] + ) + end + + def pending_team_invitations_for(%Plausible.Auth.User{} = user) do + Repo.all( + from ti in Teams.Invitation, + inner_join: inviter in assoc(ti, :inviter), + inner_join: team in assoc(ti, :team), + where: ti.email == ^user.email, + where: ti.role != :guest, + preload: [inviter: inviter, team: team] + ) + end + + @spec pending_guest_invitations_for(Auth.User.t()) :: [Teams.GuestInvitation.t()] + def pending_guest_invitations_for(%Plausible.Auth.User{} = user) do + Repo.all( + from gi in Teams.GuestInvitation, + inner_join: ti in assoc(gi, :team_invitation), + as: :team_invitation, + inner_join: inviter in assoc(ti, :inviter), + inner_join: s in assoc(gi, :site), + as: :site, + where: ti.email == ^user.email, + where: + not exists( + from tm in Teams.Membership, + inner_join: u in assoc(tm, :user), + left_join: gm in assoc(tm, :guest_memberships), + on: gm.site_id == parent_as(:site).id, + where: tm.team_id == parent_as(:team_invitation).team_id, + where: u.email == ^user.email, + where: not is_nil(gm.id) or tm.role != :guest, + select: 1 + ), + preload: [site: s, team_invitation: {ti, inviter: inviter}] + ) + end + + @spec pending_site_transfers_for(Auth.User.t()) :: [Teams.SiteTransfer.t()] + def pending_site_transfers_for(%Plausible.Auth.User{} = user) do + Repo.all( + from st in Teams.SiteTransfer, + inner_join: s in assoc(st, :site), + inner_join: initiator in assoc(st, :initiator), + where: st.email == ^user.email, + preload: [site: s, initiator: initiator] + ) + end + + defp find_team_invitation_for_user(team_invitation_id, user) do + invitation_query = + from ti in Teams.Invitation, + inner_join: inviter in assoc(ti, :inviter), + inner_join: team in assoc(ti, :team), + where: ti.invitation_id == ^team_invitation_id, + where: ti.email == ^user.email, + where: ti.role != :guest, + preload: [inviter: inviter, team: team] + + case Repo.one(invitation_query) do + nil -> + {:error, :invitation_not_found} + + invitation -> + {:ok, invitation} + end + end + + defp find_guest_invitation_for_user(guest_invitation_id, user) do + invitation_query = + from gi in Teams.GuestInvitation, + inner_join: s in assoc(gi, :site), + inner_join: ti in assoc(gi, :team_invitation), + inner_join: t in assoc(ti, :team), + inner_join: inviter in assoc(ti, :inviter), + where: gi.invitation_id == ^guest_invitation_id, + where: ti.email == ^user.email, + preload: [site: s, team_invitation: {ti, team: t, inviter: inviter}] + + case Repo.one(invitation_query) do + nil -> + {:error, :invitation_not_found} + + invitation -> + {:ok, invitation} + end + end + + defp find_transfer_for_user(transfer_id, user) do + transfer = + Teams.SiteTransfer + |> Repo.get_by(transfer_id: transfer_id, email: user.email) + |> Repo.preload([:initiator, site: :team]) + + case transfer do + nil -> + {:error, :invitation_not_found} + + transfer -> + {:ok, transfer} + end + end + + defp find_invitation_for_site(guest_invitation_id, site) do + invitation = + Teams.GuestInvitation + |> Repo.get_by(invitation_id: guest_invitation_id, site_id: site.id) + |> Repo.preload([:site, team_invitation: :inviter]) + + case invitation do + nil -> + {:error, :invitation_not_found} + + invitation -> + {:ok, invitation} + end + end + + defp find_transfer_for_site(transfer_id, site) do + transfer = + Teams.SiteTransfer + |> Repo.get_by(transfer_id: transfer_id, site_id: site.id) + |> Repo.preload([:site, :initiator]) + + case transfer do + nil -> + {:error, :invitation_not_found} + + transfer -> + {:ok, transfer} + end + end + + def invite(%Teams.Team{} = team, invitee_email, role, inviter) do + create_team_invitation(team, invitee_email, inviter, role: role) + end + + def invite(%Plausible.Site{} = site, invitee_email, role, inviter) do + site = Repo.preload(site, :team) + + if role == :owner do + create_site_transfer( + site, + inviter, + invitee_email + ) + else + create_invitation( + site, + invitee_email, + role, + inviter + ) + end + end + + def remove_team_invitation(team_invitation) do + Repo.delete_all( + from ti in Teams.Invitation, + where: ti.id == ^team_invitation.id + ) + + :ok + end + + def remove_guest_invitation(guest_invitation) do + site = Repo.preload(guest_invitation, site: :team).site + + Repo.delete_all( + from gi in Teams.GuestInvitation, + where: gi.id == ^guest_invitation.id + ) + + prune_guest_invitations(site.team) + end + + def remove_site_transfer(site_transfer) do + Repo.delete_all( + from st in Teams.SiteTransfer, + where: st.id == ^site_transfer.id + ) + end + + def accept_site_transfer(site_transfer, team, opts \\ []) do + {:ok, _} = + Repo.transaction(fn -> + :ok = + transfer_site_ownership(site_transfer.site, team, NaiveDateTime.utc_now(:second), opts) + + Repo.delete_all(from st in Teams.SiteTransfer, where: st.id == ^site_transfer.id) + end) + + :ok + end + + def transfer_site(site, team) do + {:ok, _} = + Repo.transaction(fn -> + :ok = transfer_site_ownership(site, team, NaiveDateTime.utc_now(:second)) + end) + + :ok + end + + def accept_guest_invitation(guest_invitation, user) do + guest_invitation = Repo.preload(guest_invitation, :site) + + team_invitation = + guest_invitation.team_invitation + |> Repo.preload([ + :team, + :inviter, + guest_invitations: :site + ]) + + now = NaiveDateTime.utc_now(:second) + + do_accept(team_invitation, user, now, guest_invitations: [guest_invitation]) + end + + def accept_team_invitation(team_invitation, user) do + team_invitation = Repo.preload(team_invitation, [:team, :inviter]) + now = NaiveDateTime.utc_now(:second) + + if Plausible.Users.type(user) == :sso do + {:error, :permission_denied} + else + do_accept(team_invitation, user, now, guest_invitations: []) + end + end + + @doc false + def ensure_transfer_valid(_team, nil, :owner), do: :ok + + def ensure_transfer_valid(team, new_team, :owner) do + if team.id == new_team.id do + {:error, :transfer_to_self} + else + :ok + end + end + + def ensure_transfer_valid(_team, _new_owner, _role), do: :ok + + defp create_site_transfer(site, initiator, invitee_email, now \\ NaiveDateTime.utc_now(:second)) do + result = + Ecto.Multi.new() + |> Ecto.Multi.put( + :site_transfer_changeset, + Teams.SiteTransfer.changeset(site, initiator: initiator, email: invitee_email) + ) + |> Ecto.Multi.run(:ensure_no_invitations, fn _repo, %{site_transfer_changeset: changeset} -> + q = + from ti in Teams.Invitation, + inner_join: gi in assoc(ti, :guest_invitations), + where: ti.email == ^invitee_email, + where: ti.team_id == ^site.team_id, + where: gi.site_id == ^site.id + + if Repo.exists?(q) do + {:error, Ecto.Changeset.add_error(changeset, :invitation, "already sent")} + else + {:ok, :pass} + end + end) + |> Ecto.Multi.insert( + :site_transfer, + fn %{site_transfer_changeset: changeset} -> changeset end, + on_conflict: [set: [updated_at: now]], + conflict_target: [:email, :site_id], + returning: true + ) + |> Repo.transaction() + + case result do + {:ok, success} -> + {:ok, success.site_transfer} + + {:error, _, changeset, _} -> + {:error, changeset} + end + end + + defp do_accept(team_invitation, user, now, opts) do + send_email? = Keyword.get(opts, :send_email?, true) + guest_invitations = Keyword.get(opts, :guest_invitations, team_invitation.guest_invitations) + + Repo.transaction(fn -> + with {:ok, team_membership} <- + create_team_membership(team_invitation.team, team_invitation.role, user, now), + {:ok, guest_memberships} <- + create_guest_memberships(team_membership, guest_invitations, now) do + # Clean up guest invitations after accepting + guest_invitation_ids = Enum.map(guest_invitations, & &1.id) + Repo.delete_all(from gi in Teams.GuestInvitation, where: gi.id in ^guest_invitation_ids) + + if team_membership.role != :guest do + Repo.delete_all(from ti in Teams.Invitation, where: ti.id == ^team_invitation.id) + end + + prune_guest_invitations(team_invitation.team) + + # Prune guest memberships if any exist when team membership role + # is other than guest + maybe_prune_guest_memberships(team_membership) + + if send_email? do + send_invitation_accepted_email(team_invitation, guest_invitations) + end + + %{ + team: team_invitation.team, + team_membership: team_membership, + guest_memberships: guest_memberships + } + else + {:error, changeset} -> Repo.rollback(changeset) + end + end) + end + + defp maybe_prune_guest_memberships(%Teams.Membership{role: :guest}), do: :ok + + defp maybe_prune_guest_memberships(%Teams.Membership{} = team_membership) do + team_membership + |> Ecto.assoc(:guest_memberships) + |> Repo.delete_all() + + :ok + end + + defp transfer_site_ownership(site, team, now, opts \\ []) do + skip_site_members_transfer? = Keyword.get(opts, :skip_site_members_transfer?, false) + + site = + Repo.preload(site, [ + :team, + :owners, + guest_memberships: [team_membership: :user], + guest_invitations: [team_invitation: :inviter] + ]) + + prior_team = site.team + + site + |> Ecto.Changeset.change(team_id: team.id) + |> Repo.update!() + + old_guest_invitations = site.guest_invitations + + if not skip_site_members_transfer? do + Enum.each(old_guest_invitations, fn old_guest_invitation -> + recreate_guest_invitation(site, team, old_guest_invitation) + end) + end + + old_guest_ids = Enum.map(old_guest_invitations, & &1.id) + Repo.delete_all(from gi in Teams.GuestInvitation, where: gi.id in ^old_guest_ids) + :ok = prune_guest_invitations(prior_team) + + old_guest_memberships = site.guest_memberships + + if not skip_site_members_transfer? do + Enum.each(old_guest_memberships, fn old_guest_membership -> + recreate_guest_membership(site, team, old_guest_membership, now) + end) + end + + old_guest_ids = Enum.map(old_guest_memberships, & &1.id) + Repo.delete_all(from gm in Teams.GuestMembership, where: gm.id in ^old_guest_ids) + :ok = Teams.Memberships.prune_guests(prior_team) + + if not skip_site_members_transfer? do + add_prior_owners_as_site_editors(prior_team, team, site, now) + end + + on_ee do + Billing.SiteLocker.update_for(prior_team, send_email?: false) + :unlocked = Billing.SiteLocker.update_for(team, send_email?: false) + Plausible.ConsolidatedView.reset_if_enabled(prior_team) + end + + :ok + end + + defp recreate_guest_invitation(site, new_team, old_guest_invitation) do + old_team_invitation = old_guest_invitation.team_invitation + + {:ok, new_team_invitation} = + create_team_invitation(new_team, old_team_invitation.email, old_team_invitation.inviter) + + {:ok, _new_guest_invitation} = + create_guest_invitation(new_team_invitation, site, old_guest_invitation.role) + end + + defp recreate_guest_membership(site, new_team, old_guest_membership, now) do + old_team_membership = old_guest_membership.team_membership + + {:ok, new_team_membership} = + create_team_membership(new_team, :guest, old_team_membership.user, now) + + if new_team_membership.role == :guest do + {:ok, _} = + new_team_membership + |> Teams.GuestMembership.changeset(site, old_guest_membership.role) + |> Repo.insert( + on_conflict: [set: [updated_at: now, role: old_guest_membership.role]], + conflict_target: [:team_membership_id, :site_id], + returning: true + ) + end + end + + defp add_prior_owners_as_site_editors(prior_team, new_team, site, now) do + prior_owners = Repo.preload(prior_team, :owners).owners + + for prior_owner <- prior_owners do + {:ok, prior_owner_team_membership} = + create_team_membership(new_team, :guest, prior_owner, now) + + if prior_owner_team_membership.role == :guest do + {:ok, _} = + prior_owner_team_membership + |> Teams.GuestMembership.changeset(site, :editor) + |> Repo.insert( + on_conflict: [set: [updated_at: now, role: :editor]], + conflict_target: [:team_membership_id, :site_id], + returning: true + ) + end + end + end + + def prune_guest_invitations(team) do + guest_query = + from( + gi in Teams.GuestInvitation, + where: gi.team_invitation_id == parent_as(:team_invitation).id, + select: true + ) + + Repo.delete_all( + from( + ti in Teams.Invitation, + as: :team_invitation, + where: ti.team_id == ^team.id and ti.role == :guest, + where: not exists(guest_query) + ) + ) + + :ok + end + + def ensure_can_take_ownership(site, team, opts \\ []) + + on_ee do + def ensure_can_take_ownership(_site, nil, _opts), do: {:error, :no_plan} + + def ensure_can_take_ownership(site, team, opts) do + team = Teams.with_subscription(team) + plan = Billing.Plans.get_subscription_plan(team.subscription) + active_subscription? = Billing.Subscriptions.active?(team.subscription) + + if active_subscription? and plan != :free_10k do + team + |> Teams.Billing.quota_usage(pending_ownership_site_ids: [site.id]) + |> Billing.Quota.ensure_within_plan_limits(plan, + skip_site_limit_check?: Teams.Billing.grandfathered_team?(team), + skip_team_member_limit_check?: opts[:skip_site_members_transfer?] == true + ) + else + {:error, :no_plan} + end + end + else + def ensure_can_take_ownership(_site, _team, _opts) do + if always(true) do + :ok + else + {:error, :no_plan} + end + end + end + + def send_transfer_accepted_email(site_transfer, team) do + initiator_as_guest_editor? = + Teams.Memberships.site_role(site_transfer.site, site_transfer.initiator) == + {:ok, {:guest_member, :editor}} + + PlausibleWeb.Email.ownership_transfer_accepted( + site_transfer.email, + site_transfer.initiator.email, + team, + site_transfer.site, + initiator_as_guest_editor? + ) + |> Plausible.Mailer.send() + end + + def send_team_changed_email(site, user, team) do + owners = Repo.preload(team, :owners).owners + + for owner <- owners do + PlausibleWeb.Email.team_changed(owner.email, user, team, site) + |> Plausible.Mailer.send() + end + end + + @spec check_can_transfer_site(Teams.Team.t(), Auth.User.t()) :: + :ok | {:error, :permission_denied} + def check_can_transfer_site(team, user) do + case Teams.Memberships.team_role(team, user) do + {:ok, role} when role in [:owner, :admin] -> + :ok + + _ -> + {:error, :permission_denied} + end + end + + @doc false + def check_invitation_permissions(%Teams.Team{} = team, inviter, invitation_role, opts) do + check_permissions? = Keyword.get(opts, :check_permissions, true) + + if check_permissions? do + case Teams.Memberships.team_role(team, inviter) do + {:ok, :owner} when invitation_role == :owner -> + :ok + + {:ok, inviter_role} + when inviter_role in [:owner, :admin] and invitation_role != :owner -> + :ok + + _ -> + {:error, :permission_denied} + end + else + :ok + end + end + + def check_invitation_permissions(%Plausible.Site{} = site, inviter, _invitation_role, opts) do + check_permissions? = Keyword.get(opts, :check_permissions, true) + + if check_permissions? do + case Teams.Memberships.site_role(site, inviter) do + {:ok, {:team_member, inviter_role}} when inviter_role in [:owner, :admin] -> + :ok + + _ -> + {:error, :permission_denied} + end + else + :ok + end + end + + @doc false + def check_team_member_limit(_team, :owner, _invitee_email), do: :ok + + def check_team_member_limit(team, _role, invitee_email) do + limit = Teams.Billing.team_member_limit(team) + usage = Teams.Billing.team_member_usage(team, exclude_emails: [invitee_email]) + + if Billing.Quota.below_limit?(usage, limit) do + :ok + else + {:error, {:over_limit, limit}} + end + end + + @doc false + def ensure_new_membership(_site_or_team, nil, _role), do: :ok + + def ensure_new_membership(_site_or_team, _invitee, :owner), do: :ok + + def ensure_new_membership(%Teams.Team{} = team, invitee, _role) do + case Teams.Memberships.team_role(team, invitee) do + {:ok, :guest} -> :ok + {:error, :not_a_member} -> :ok + {:ok, _} -> {:error, :already_a_member} + end + end + + def ensure_new_membership(%Plausible.Site{} = site, invitee, _role) do + if Teams.Memberships.site_role(site, invitee) == {:error, :not_a_member} do + :ok + else + {:error, :already_a_member} + end + end + + defp create_invitation(site, invitee_email, role, inviter) do + Repo.transaction(fn -> + with {:ok, team_invitation} <- + create_team_invitation(site.team, invitee_email, inviter, + ensure_no_site_transfers_for: site.id + ), + {:ok, guest_invitation} <- create_guest_invitation(team_invitation, site, role) do + guest_invitation + else + {:error, changeset} -> Repo.rollback(changeset) + end + end) + end + + defp create_team_invitation(team, invitee_email, inviter, opts \\ []) do + now = NaiveDateTime.utc_now(:second) + role = Keyword.get(opts, :role, :guest) + + result = + Ecto.Multi.new() + |> Ecto.Multi.put( + :changeset, + Teams.Invitation.changeset(team, email: invitee_email, role: role, inviter: inviter) + ) + |> Ecto.Multi.run(:ensure_no_site_transfers, fn _repo, %{changeset: changeset} -> + ensure_no_site_transfers(changeset, opts[:ensure_no_site_transfers_for], invitee_email) + end) + |> Ecto.Multi.insert( + :team_invitation, + & &1.changeset, + on_conflict: [set: [updated_at: now, role: role]], + conflict_target: [:team_id, :email], + returning: true + ) + |> Ecto.Multi.run(:prune_guest_entries, fn _repo, %{team_invitation: team_invitation} -> + if team_invitation.role != :guest do + team_invitation + |> Ecto.assoc(:guest_invitations) + |> Repo.delete_all() + end + + {:ok, nil} + end) + |> Repo.transaction() + + case result do + {:ok, success} -> + {:ok, success.team_invitation} + + {:error, _, changeset, _} -> + {:error, changeset} + end + end + + defp create_guest_invitation(team_invitation, site, role) do + now = NaiveDateTime.utc_now(:second) + + team_invitation + |> Teams.GuestInvitation.changeset(site, role) + |> Repo.insert( + on_conflict: [set: [updated_at: now]], + conflict_target: [:team_invitation_id, :site_id], + returning: true + ) + end + + @doc false + def send_invitation_email(%Teams.SiteTransfer{} = transfer, invitee) do + email = + PlausibleWeb.Email.ownership_transfer_request( + transfer.email, + transfer.transfer_id, + transfer.site, + transfer.initiator, + invitee + ) + + Plausible.Mailer.send(email) + end + + def send_invitation_email(%Teams.Invitation{} = team_invitation, invitee) do + email = + if invitee do + PlausibleWeb.Email.existing_user_team_invitation( + team_invitation.email, + team_invitation.team, + team_invitation.inviter + ) + else + PlausibleWeb.Email.new_user_team_invitation( + team_invitation.email, + team_invitation.invitation_id, + team_invitation.team, + team_invitation.inviter + ) + end + + Plausible.Mailer.send(email) + end + + def send_invitation_email(%Teams.GuestInvitation{} = guest_invitation, invitee) do + team_invitation = guest_invitation.team_invitation + + email = + if invitee do + PlausibleWeb.Email.existing_user_invitation( + team_invitation.email, + guest_invitation.site, + team_invitation.inviter + ) + else + PlausibleWeb.Email.new_user_invitation( + team_invitation.email, + guest_invitation.invitation_id, + guest_invitation.site, + team_invitation.inviter + ) + end + + Plausible.Mailer.send(email) + end + + @team_role_type Plausible.Teams.Membership.__schema__(:type, :role) + + def create_team_membership(team, role, user, now) do + conflict_query = + from(tm in Teams.Membership, + update: [ + set: [ + updated_at: ^now, + role: + fragment( + "CASE WHEN ? = 'guest' THEN ? ELSE ? END", + tm.role, + type(^role, ^@team_role_type), + tm.role + ) + ] + ] + ) + + team + |> Teams.Membership.changeset(user, role) + |> Repo.insert( + on_conflict: conflict_query, + conflict_target: [:team_id, :user_id], + returning: true + ) + end + + defp create_guest_memberships(_team_membership, [], _now) do + {:ok, []} + end + + defp create_guest_memberships(%{role: role} = _team_membership, _, _) when role != :guest do + {:ok, []} + end + + defp create_guest_memberships(team_membership, guest_invitations, now) do + Enum.reduce_while(guest_invitations, {:ok, []}, fn guest_invitation, + {:ok, guest_memberships} -> + result = + team_membership + |> Teams.GuestMembership.changeset(guest_invitation.site, guest_invitation.role) + |> Repo.insert( + on_conflict: [set: [updated_at: now]], + conflict_target: [:team_membership_id, :site_id] + ) + + case result do + {:ok, guest_membership} -> {:cont, {:ok, [guest_membership | guest_memberships]}} + {:error, changeset} -> {:halt, {:error, changeset}} + end + end) + end + + defp send_invitation_accepted_email(team_invitation, []) do + team_invitation.inviter.email + |> PlausibleWeb.Email.team_invitation_accepted(team_invitation.email, team_invitation.team) + |> Plausible.Mailer.send() + end + + defp send_invitation_accepted_email(team_invitation, [guest_invitation | _]) do + team_invitation.inviter.email + |> PlausibleWeb.Email.guest_invitation_accepted( + team_invitation.email, + team_invitation.team, + guest_invitation.site + ) + |> Plausible.Mailer.send() + end + + defp ensure_no_site_transfers(_, nil, _) do + {:ok, :skip} + end + + defp ensure_no_site_transfers(changeset, site_id, invitee_email) + when is_integer(site_id) and is_binary(invitee_email) do + q = + from st in Teams.SiteTransfer, + where: st.email == ^invitee_email, + where: st.site_id == ^site_id + + if Repo.exists?(q) do + {:error, Ecto.Changeset.add_error(changeset, :invitation, "already sent")} + else + {:ok, :pass} + end + end +end diff --git a/lib/plausible/teams/invitations/accept.ex b/lib/plausible/teams/invitations/accept.ex new file mode 100644 index 0000000000000000000000000000000000000000..08b5fb9372b34ef6f52cab666d47d3491684c539 --- /dev/null +++ b/lib/plausible/teams/invitations/accept.ex @@ -0,0 +1,60 @@ +defmodule Plausible.Teams.Invitations.Accept do + use Plausible + + @moduledoc """ + Service for accepting invitations, including ownership transfers. + + Accepting invitation accounts for the fact that it's possible + that accepting user has an existing membership for the site and + acts permissively to not unnecessarily disrupt the flow while + also maintaining integrity of site memberships. This also applies + to cases where users update their email address between issuing + the invitation and accepting it. + """ + + alias Plausible.Auth + alias Plausible.Billing + alias Plausible.Teams + + @type accept_error() :: + :invitation_not_found + | Billing.Quota.Limits.over_limits_error() + | Ecto.Changeset.t() + | :no_plan + | :transfer_to_self + | :multiple_teams + | :permission_denied + + @spec accept(String.t(), Auth.User.t(), Teams.Team.t() | nil) :: + {:ok, map()} | {:error, accept_error()} + def accept(invitation_or_transfer_id, user, team \\ nil) do + with {:ok, invitation_or_transfer} <- + Teams.Invitations.find_for_user(invitation_or_transfer_id, user) do + case invitation_or_transfer do + %Teams.SiteTransfer{} = site_transfer -> + Teams.Sites.Transfer.accept(site_transfer, user, team) + + %Teams.Invitation{} = team_invitation -> + Teams.Invitations.accept_team_invitation(team_invitation, user) + + %Teams.GuestInvitation{} = guest_invitation -> + Teams.Invitations.accept_guest_invitation(guest_invitation, user) + end + end + end + + @spec accept_transfer_no_members(String.t(), Auth.User.t(), Teams.Team.t() | nil) :: + {:ok, map()} | {:error, accept_error()} + def accept_transfer_no_members(transfer_id, user, team \\ nil) do + case Teams.Invitations.find_for_user(transfer_id, user) do + {:ok, %Teams.SiteTransfer{} = site_transfer} -> + Teams.Sites.Transfer.accept(site_transfer, user, team, skip_site_members_transfer?: true) + + {:ok, _} -> + {:error, :permission_denied} + + {:error, _} = error -> + error + end + end +end diff --git a/lib/plausible/teams/invitations/invite_to_site.ex b/lib/plausible/teams/invitations/invite_to_site.ex new file mode 100644 index 0000000000000000000000000000000000000000..5218b99509c1f6389691bc1892aa910d72aca97a --- /dev/null +++ b/lib/plausible/teams/invitations/invite_to_site.ex @@ -0,0 +1,67 @@ +defmodule Plausible.Teams.Invitations.InviteToSite do + @moduledoc """ + Service for inviting new or existing users to a sites, including ownershhip + transfers. + """ + + alias Plausible.Auth.User + alias Plausible.Repo + alias Plausible.Site + alias Plausible.Teams + use Plausible + + @type invite_error() :: + Ecto.Changeset.t() + | :already_a_member + | {:over_limit, non_neg_integer()} + | :permission_denied + + @type invitation :: %Teams.GuestInvitation{} | %Teams.SiteTransfer{} + + @doc """ + Invites a new team member to the given site. Returns either + `%Teams.GuestInvitation{}` or `%Teams.SiteTransfer{}` struct + and sends the invitee an email to accept this invitation. + + The inviter must have enough permissions to invite the new team member, + otherwise this function returns `{:error, :permission_denied}`. + + If the new team member role is `:owner`, this function handles the invitation + as an ownership transfer and requires the inviter to be the owner of the site. + """ + @spec invite(Site.t(), User.t(), String.t(), atom()) :: + {:ok, invitation} | {:error, invite_error()} + def invite(site, inviter, invitee_email, role) do + Repo.transaction(fn -> + do_invite(site, inviter, invitee_email, role) + end) + end + + defp do_invite(site, inviter, invitee_email, role, opts \\ []) do + with site <- Repo.preload(site, [:owners, :team]), + :ok <- Teams.Invitations.check_invitation_permissions(site, inviter, role, opts), + :ok <- Teams.Invitations.check_team_member_limit(site.team, role, invitee_email), + invitee = Plausible.Auth.find_user_by(email: invitee_email), + :ok <- Teams.Invitations.ensure_new_membership(site, invitee, role), + {:ok, invitation_or_transfer} <- + Teams.Invitations.invite(site, invitee_email, role, inviter) do + send_invitation_email(invitation_or_transfer, invitee) + + invitation_or_transfer + else + {:error, cause} -> Repo.rollback(cause) + end + end + + defp send_invitation_email(%Teams.GuestInvitation{} = guest_invitation, invitee) do + guest_invitation + |> Repo.preload([:site, team_invitation: :inviter]) + |> Teams.Invitations.send_invitation_email(invitee) + end + + defp send_invitation_email(%Teams.SiteTransfer{} = site_transfer, invitee) do + site_transfer + |> Repo.preload([:site, :initiator]) + |> Teams.Invitations.send_invitation_email(invitee) + end +end diff --git a/lib/plausible/teams/invitations/invite_to_team.ex b/lib/plausible/teams/invitations/invite_to_team.ex new file mode 100644 index 0000000000000000000000000000000000000000..5c38043c75a1657d82eaf83d8cca3e4f3c89753e --- /dev/null +++ b/lib/plausible/teams/invitations/invite_to_team.ex @@ -0,0 +1,38 @@ +defmodule Plausible.Teams.Invitations.InviteToTeam do + @moduledoc """ + Service for inviting new or existing users to team. + """ + + alias Plausible.Repo + alias Plausible.Teams + + @valid_roles Plausible.Teams.Invitation.roles() -- [:guest] + @valid_roles @valid_roles ++ Enum.map(@valid_roles, &to_string/1) + + def invite(team, inviter, invitee_email, role, opts \\ []) + + def invite(team, inviter, invitee_email, role, opts) when role in @valid_roles do + with team <- Repo.preload(team, [:owners]), + :ok <- Teams.Invitations.check_invitation_permissions(team, inviter, role, opts), + :ok <- Teams.Invitations.check_team_member_limit(team, role, invitee_email), + invitee = Plausible.Auth.find_user_by(email: invitee_email), + :ok <- Teams.Invitations.ensure_new_membership(team, invitee, role), + {:ok, invitation} <- Teams.Invitations.invite(team, invitee_email, role, inviter) do + if Keyword.get(opts, :send_email?, true) do + send_invitation_email(invitation, invitee) + end + + {:ok, invitation} + end + end + + def invite(_team, _inviter, _invitee_email, role, _opts) do + raise "Invalid role passed: #{inspect(role)}" + end + + def send_invitation_email(invitation, invitee) do + invitation + |> Repo.preload([:team, :inviter]) + |> Teams.Invitations.send_invitation_email(invitee) + end +end diff --git a/lib/plausible/teams/invitations/reject.ex b/lib/plausible/teams/invitations/reject.ex new file mode 100644 index 0000000000000000000000000000000000000000..eab1b9a11ce67a19610675ad41cb7bddb09235da --- /dev/null +++ b/lib/plausible/teams/invitations/reject.ex @@ -0,0 +1,52 @@ +defmodule Plausible.Teams.Invitations.Reject do + @moduledoc """ + Service for rejecting invitations. + """ + + alias Plausible.Auth + alias Plausible.Teams + + @spec reject(String.t(), Auth.User.t()) :: + {:ok, Teams.GuestInvitation.t() | Teams.SiteTransfer.t()} + | {:error, :invitation_not_found} + def reject(invitation_or_transfer_id, user) do + with {:ok, invitation_or_transfer} <- + Teams.Invitations.find_for_user(invitation_or_transfer_id, user) do + do_reject(invitation_or_transfer) + {:ok, invitation_or_transfer} + end + end + + defp do_reject(%Teams.Invitation{} = team_invitation) do + Teams.Invitations.remove_team_invitation(team_invitation) + + notify_team_invitation_rejected(team_invitation) + end + + defp do_reject(%Teams.GuestInvitation{} = guest_invitation) do + Teams.Invitations.remove_guest_invitation(guest_invitation) + + notify_guest_invitation_rejected(guest_invitation) + end + + defp do_reject(%Teams.SiteTransfer{} = site_transfer) do + Teams.Invitations.remove_site_transfer(site_transfer) + + notify_site_transfer_rejected(site_transfer) + end + + defp notify_site_transfer_rejected(site_transfer) do + PlausibleWeb.Email.ownership_transfer_rejected(site_transfer) + |> Plausible.Mailer.send() + end + + defp notify_guest_invitation_rejected(guest_invitation) do + PlausibleWeb.Email.guest_invitation_rejected(guest_invitation) + |> Plausible.Mailer.send() + end + + defp notify_team_invitation_rejected(team_invitation) do + PlausibleWeb.Email.team_invitation_rejected(team_invitation) + |> Plausible.Mailer.send() + end +end diff --git a/lib/plausible/teams/invitations/remove_from_site.ex b/lib/plausible/teams/invitations/remove_from_site.ex new file mode 100644 index 0000000000000000000000000000000000000000..377b66b055159fa8fe3eb448bb790ca24da1ec77 --- /dev/null +++ b/lib/plausible/teams/invitations/remove_from_site.ex @@ -0,0 +1,27 @@ +defmodule Plausible.Teams.Invitations.RemoveFromSite do + @moduledoc """ + Service for removing invitations. + """ + + alias Plausible.Teams + + @spec remove(String.t(), Plausible.Site.t()) :: + {:ok, Teams.GuestInvitation.t() | Teams.SiteTransfer.t()} + | {:error, :invitation_not_found} + def remove(invitation_or_transfer_id, site) do + with {:ok, invitation_or_transfer} <- + Teams.Invitations.find_for_site(invitation_or_transfer_id, site) do + do_delete(invitation_or_transfer) + + {:ok, invitation_or_transfer} + end + end + + defp do_delete(%Teams.GuestInvitation{} = guest_invitation) do + Teams.Invitations.remove_guest_invitation(guest_invitation) + end + + defp do_delete(%Teams.SiteTransfer{} = site_transfer) do + Teams.Invitations.remove_site_transfer(site_transfer) + end +end diff --git a/lib/plausible/teams/invitations/remove_from_team.ex b/lib/plausible/teams/invitations/remove_from_team.ex new file mode 100644 index 0000000000000000000000000000000000000000..e575e3d8837d6e6181eb9e5b4d9c61ab79d7b121 --- /dev/null +++ b/lib/plausible/teams/invitations/remove_from_team.ex @@ -0,0 +1,31 @@ +defmodule Plausible.Teams.Invitations.RemoveFromTeam do + @moduledoc """ + Service for removing a team invitation. + """ + + alias Plausible.Repo + alias Plausible.Teams.Invitations + alias Plausible.Teams.Memberships + + def remove(nil, _invitation_id, _current_user) do + {:error, :permission_denied} + end + + def remove(team, invitation_id, current_user) do + with {:ok, team_invitation} <- Invitations.get_team_invitation(team, invitation_id), + {:ok, current_user_role} <- Memberships.team_role(team, current_user), + :ok <- check_can_remove_invitation(current_user_role) do + Repo.delete!(team_invitation) + + {:ok, team_invitation} + end + end + + defp check_can_remove_invitation(role) do + if role in [:owner, :admin] do + :ok + else + {:error, :permission_denied} + end + end +end diff --git a/lib/plausible/teams/management/layout.ex b/lib/plausible/teams/management/layout.ex new file mode 100644 index 0000000000000000000000000000000000000000..e67abe71601be3ae83a3d9f83205e2328f537a21 --- /dev/null +++ b/lib/plausible/teams/management/layout.ex @@ -0,0 +1,174 @@ +defmodule Plausible.Teams.Management.Layout do + @moduledoc """ + Abstraction for team membership(s) layout - provides a high level CRUD for + setting up team memberships, including invitations. Persisting the layout, + effectively takes care of delegating the operations to specialized services + and sending out e-mail notifications on success, if need be. + To be used in UIs allowing team memberships adjustments. + """ + alias Plausible.Teams + alias Plausible.Teams.Management.Layout.Entry + alias Plausible.Repo + alias Plausible.Auth.User + + @type t() :: %{String.t() => Entry.t()} + + @spec init(Teams.Team.t()) :: t() + def init(%Teams.Team{} = team) do + invitations_sent = Teams.Invitations.pending_team_invitations_for(team) + all_members = Teams.Memberships.all(team) + build_by_email(invitations_sent ++ all_members) + end + + @spec build_by_email([Teams.Invitation.t() | Teams.Membership.t()]) :: + t() + def build_by_email(entities) do + Enum.reduce(entities, %{}, fn + %Teams.Invitation{} = invitation, acc -> + Map.put(acc, invitation.email, Entry.new(invitation)) + + %Teams.Membership{} = membership, acc -> + Map.put( + acc, + membership.user.email, + Entry.new(membership) + ) + end) + end + + @spec active_count(t()) :: non_neg_integer() + def active_count(layout) do + Enum.count(layout, fn {_, entry} -> entry.queued_op != :delete end) + end + + @spec owners_count(t()) :: non_neg_integer() + def owners_count(layout) do + Enum.count(layout, fn {_, entry} -> entry.queued_op != :delete and entry.role == :owner end) + end + + @spec has_guests?(t()) :: boolean() + def has_guests?(layout) do + not is_nil( + Enum.find( + layout, + fn + {_, entry} -> entry.role == :guest and entry.queued_op != :delete + end + ) + ) + end + + @spec update_role(t(), String.t(), atom()) :: t() + def update_role(layout, email, role) do + entry = Map.fetch!(layout, email) + Map.put(layout, email, Entry.patch(entry, role: role, queued_op: :update)) + end + + @spec schedule_send(t(), String.t(), atom(), Keyword.t()) :: t() + def schedule_send(layout, email, role, entry_attrs \\ []) do + invitation = %Teams.Invitation{email: email, role: role} + Map.put(layout, email, Entry.new(invitation, Keyword.merge(entry_attrs, queued_op: :send))) + end + + @spec schedule_delete(t(), String.t()) :: t() + def schedule_delete(layout, email) do + entry = Map.fetch!(layout, email) + Map.put(layout, email, Entry.patch(entry, queued_op: :delete)) + end + + @spec verify_removable(t(), String.t()) :: :ok | {:error, String.t()} + def verify_removable(layout, email) do + ensure_at_least_one_owner(layout, email) + end + + @spec removable?(t(), String.t()) :: boolean() + def removable?(layout, email) do + verify_removable(layout, email) == :ok + end + + @spec sorted_for_display(t()) :: [{String.t(), Entry.t()}] + def sorted_for_display(layout) do + layout + |> Enum.reject(fn {_, entry} -> entry.queued_op == :delete end) + |> Enum.sort_by(fn {email, entry} -> + primary_criterion = + case entry do + %{role: :guest, type: :invitation_pending} -> 10 + %{role: :guest, type: :invitation_sent} -> 11 + %{role: :guest, type: :membership} -> 12 + %{type: :invitation_pending} -> 0 + %{type: :invitation_sent} -> 1 + %{type: :membership} -> 2 + end + + secondary_criterion = entry.name + tertiary_criterion = email + {primary_criterion, secondary_criterion, tertiary_criterion} + end) + end + + @spec persist(t(), %{current_user: User.t(), current_team: Teams.Team.t()}) :: + {:ok, integer()} | {:error, any()} + def persist(layout, context) do + result = + Repo.transaction(fn -> + Teams.complete_setup(context.current_team) + + layout + |> sorted_for_persistence() + |> Enum.reduce([], fn {_, entry}, acc -> + persist_entry(entry, context, acc) + end) + end) + + case result do + {:ok, persisted} -> + persisted + |> Enum.each(fn + {%Entry{type: :invitation_pending}, invitation} -> + invitee = Plausible.Auth.find_user_by(email: invitation.email) + Teams.Invitations.InviteToTeam.send_invitation_email(invitation, invitee) + + {%Entry{type: :membership, queued_op: :delete}, team_membership} -> + Teams.Memberships.Remove.send_team_member_removed_email(team_membership) + + _ -> + :noop + end) + + {:ok, length(persisted)} + + {:error, _} = error -> + error + end + end + + defp sorted_for_persistence(layout) do + # sort by deletions first, so team member limits are triggered accurately + Enum.sort_by(layout, fn {_email, entry} -> + case entry.queued_op do + :delete -> 0 + _ -> 1 + end + end) + end + + defp ensure_at_least_one_owner(layout, email) do + if Enum.find(layout, fn {_email, entry} -> + entry.email != email and + entry.role == :owner and + entry.type == :membership and + entry.queued_op != :delete + end), + do: :ok, + else: {:error, "The team has to have at least one owner"} + end + + def persist_entry(entry, context, acc) do + case Entry.persist(entry, context) do + {:ok, :ignore} -> acc + {:ok, persist_result} -> [{entry, persist_result} | acc] + {:error, error} -> Repo.rollback(error) + end + end +end diff --git a/lib/plausible/teams/management/layout/entry.ex b/lib/plausible/teams/management/layout/entry.ex new file mode 100644 index 0000000000000000000000000000000000000000..c0637e206820611cebe438e85e161ef338d061c6 --- /dev/null +++ b/lib/plausible/teams/management/layout/entry.ex @@ -0,0 +1,123 @@ +defmodule Plausible.Teams.Management.Layout.Entry do + @moduledoc """ + Module representing a single Team Layout element and all its state + transitions, including persistence options. + """ + alias Plausible.Teams + + defstruct [:email, :name, :role, :type, :meta, :queued_op, :id] + + @type t() :: %__MODULE__{} + + @spec new(Teams.Invitation.t() | Teams.Membership.t(), Keyword.t()) :: t() + def new(object, attrs \\ []) + + def new( + %Teams.Invitation{id: existing} = invitation, + attrs + ) + when is_integer(existing) do + %__MODULE__{ + name: "Invited User", + id: 0, + email: invitation.email, + role: invitation.role, + type: :invitation_sent, + meta: invitation + } + |> Map.merge(Enum.into(attrs, %{})) + end + + def new(%Teams.Invitation{id: nil} = pending, attrs) do + %__MODULE__{ + name: "Invited User", + id: 0, + email: pending.email, + role: pending.role, + type: :invitation_pending, + meta: pending + } + |> Map.merge(Enum.into(attrs, %{})) + end + + def new(%Teams.Membership{} = membership, attrs) do + %__MODULE__{ + name: membership.user.name, + role: membership.role, + id: membership.user.id, + email: membership.user.email, + type: :membership, + meta: membership + } + |> Map.merge(Enum.into(attrs, %{})) + end + + @spec patch(t(), Keyword.t()) :: t() + def patch(%__MODULE__{} = entry, attrs) do + struct!(entry, attrs) + end + + @spec persist(t(), map()) :: + {:ok, :ignore | Teams.Invitation.t() | Teams.Membership.t()} | {:error, any()} + def persist(%__MODULE__{queued_op: nil}, _context) do + {:ok, :ignore} + end + + def persist(%__MODULE__{type: :invitation_pending, queued_op: :delete}, _context) do + {:ok, :ignore} + end + + def persist( + %__MODULE__{email: email, role: role, type: :invitation_pending, queued_op: op}, + context + ) + when op in [:update, :send] do + Teams.Invitations.InviteToTeam.invite(context.current_team, context.current_user, email, role, + send_email?: false + ) + end + + def persist( + %__MODULE__{type: :invitation_sent, email: email, role: role, queued_op: :update}, + context + ) do + Teams.Invitations.InviteToTeam.invite(context.current_team, context.current_user, email, role, + send_email?: false + ) + end + + def persist( + %__MODULE__{type: :invitation_sent, queued_op: :delete, meta: meta}, + context + ) do + Plausible.Teams.Invitations.RemoveFromTeam.remove( + context.current_team, + meta.invitation_id, + context.current_user + ) + end + + def persist( + %__MODULE__{type: :membership, queued_op: :delete, meta: meta}, + context + ) do + Plausible.Teams.Memberships.Remove.remove( + context.current_team, + meta.user.id, + context.current_user, + send_email?: false + ) + end + + def persist( + %__MODULE__{type: :membership, queued_op: :update, role: role, meta: meta}, + context + ) do + Plausible.Teams.Memberships.UpdateRole.update( + context.current_team, + meta.user.id, + "#{role}", + context.current_user + ) + end +end diff --git a/lib/plausible/teams/membership.ex b/lib/plausible/teams/membership.ex new file mode 100644 index 0000000000000000000000000000000000000000..c7fec4593b7058bbb828652be7120ae41ec84223 --- /dev/null +++ b/lib/plausible/teams/membership.ex @@ -0,0 +1,37 @@ +defmodule Plausible.Teams.Membership do + @moduledoc """ + Team membership schema + """ + + use Ecto.Schema + + import Ecto.Changeset + + @roles [:guest, :viewer, :editor, :admin, :owner, :billing] + + @type t() :: %__MODULE__{} + + @type role() :: unquote(Enum.reduce(@roles, &{:|, [], [&1, &2]})) + + schema "team_memberships" do + field :role, Ecto.Enum, values: @roles + field :is_autocreated, :boolean, default: false + + belongs_to :user, Plausible.Auth.User + belongs_to :team, Plausible.Teams.Team + + has_many :guest_memberships, Plausible.Teams.GuestMembership, foreign_key: :team_membership_id + + timestamps() + end + + def roles(), do: @roles + + def changeset(team, user, role) do + %__MODULE__{} + |> change() + |> put_change(:role, role) + |> put_assoc(:team, team) + |> put_assoc(:user, user) + end +end diff --git a/lib/plausible/teams/memberships.ex b/lib/plausible/teams/memberships.ex new file mode 100644 index 0000000000000000000000000000000000000000..b3186422561fb6667b053edebf5543da050876c4 --- /dev/null +++ b/lib/plausible/teams/memberships.ex @@ -0,0 +1,275 @@ +defmodule Plausible.Teams.Memberships do + @moduledoc false + + import Ecto.Query + + alias Plausible.Auth + alias Plausible.Repo + alias Plausible.Teams + + require Teams.Memberships.UserPreference + + @spec all(Teams.Team.t(), Keyword.t()) :: [Teams.Membership.t()] + def all(team, opts \\ []) do + exclude_guests? = Keyword.get(opts, :exclude_guests?, false) + + query = + from tm in Teams.Membership, + inner_join: u in assoc(tm, :user), + where: tm.team_id == ^team.id, + order_by: [asc: u.id], + preload: [user: u] + + query = + if exclude_guests? do + from tm in query, where: tm.role != :guest + else + query + end + + Repo.all(query) + end + + def all_pending_site_transfers(email) do + email + |> pending_site_transfers_query() + |> Repo.all() + end + + def owners_count(team) do + Repo.aggregate( + from(tm in Teams.Membership, where: tm.team_id == ^team.id and tm.role == :owner), + :count + ) + end + + @spec team_role(Teams.Team.t(), Auth.User.t()) :: + {:ok, Teams.Membership.role()} | {:error, :not_a_member} + def team_role(team, user) do + result = + from(u in Auth.User, + inner_join: tm in assoc(u, :team_memberships), + where: tm.team_id == ^team.id and tm.user_id == ^user.id, + select: tm.role + ) + |> Repo.one() + + case result do + nil -> {:error, :not_a_member} + role -> {:ok, role} + end + end + + @spec can_add_site?(Teams.Team.t(), Auth.User.t()) :: boolean() + def can_add_site?(team, user) do + user_type = Plausible.Users.type(user) + + role = + case team_role(team, user) do + {:ok, role} -> role + {:error, _} -> :not_a_member + end + + case {user_type, role, team} do + {:sso, :owner, %{setup_complete: false}} -> false + {_, role, _} when role in [:owner, :admin, :editor] -> true + _ -> false + end + end + + @spec site_role(Plausible.Site.t(), Auth.User.t() | nil) :: + {:ok, {:team_member | :guest_member, Teams.Membership.role()}} | {:error, :not_a_member} + + def site_role(_site, nil), do: {:error, :not_a_member} + + def site_role(site, user) do + result = + from(u in Auth.User, + inner_join: tm in assoc(u, :team_memberships), + left_join: gm in assoc(tm, :guest_memberships), + where: tm.team_id == ^site.team_id and tm.user_id == ^user.id, + where: tm.role != :guest or gm.site_id == ^site.id, + select: {tm.role, gm.role} + ) + |> Repo.one() + + case result do + {:guest, role} -> {:ok, {:guest_member, role}} + {role, _} -> {:ok, {:team_member, role}} + _ -> {:error, :not_a_member} + end + end + + @spec site_member?(Plausible.Site.t(), Auth.User.t() | nil) :: boolean() + def site_member?(site, user) do + case site_role(site, user) do + {:ok, _} -> true + _ -> false + end + end + + @spec team_member?(Teams.Team.t(), Auth.User.t()) :: boolean() + def team_member?(team, user) do + case team_role(team, user) do + {:ok, _} -> true + _ -> false + end + end + + @spec has_editor_access?(Plausible.Site.t(), Auth.User.t() | nil) :: boolean() + def has_editor_access?(site, user) do + case site_role(site, user) do + {:ok, {_, role}} when role in [:editor, :admin, :owner] -> + true + + _ -> + false + end + end + + def update_role(site, user_id, new_role_str, current_user, current_user_role) do + new_role = String.to_existing_atom(new_role_str) + + case get_guest_membership(site.id, user_id) do + {:ok, guest_membership} -> + can_grant_role? = + if guest_membership.team_membership.user_id == current_user.id do + false + else + can_grant_role_to_other?(current_user_role, new_role) + end + + if can_grant_role? do + guest_membership = + guest_membership + |> Ecto.Changeset.change(role: new_role) + |> Repo.update!() + |> Repo.preload(team_membership: :user) + + {:ok, guest_membership} + else + {:error, :not_allowed} + end + + {:error, _} -> + {:error, :no_guest} + end + end + + def remove(site, user) do + case get_guest_membership(site.id, user.id) do + {:ok, guest_membership} -> + guest_membership = + Repo.preload(guest_membership, [:site, team_membership: [:team, :user]]) + + {:ok, _} = + Repo.transaction(fn -> + Repo.delete!(guest_membership) + prune_guests(guest_membership.team_membership.team) + Plausible.Segments.after_user_removed_from_site(site, user) + Plausible.Annotations.after_user_removed_from_site(site, user) + end) + + send_site_member_removed_email(guest_membership) + + {:error, _} -> + :pass + end + end + + defp can_grant_role_to_other?(:owner, :editor), do: true + defp can_grant_role_to_other?(:owner, :viewer), do: true + defp can_grant_role_to_other?(:admin, :editor), do: true + defp can_grant_role_to_other?(:admin, :viewer), do: true + defp can_grant_role_to_other?(_, _), do: false + + defp send_site_member_removed_email(guest_membership) do + guest_membership + |> PlausibleWeb.Email.site_member_removed() + |> Plausible.Mailer.send() + end + + def prune_guests(team) do + guest_query = + from( + gm in Teams.GuestMembership, + where: gm.team_membership_id == parent_as(:team_membership).id, + select: true + ) + + Repo.delete_all( + from( + tm in Teams.Membership, + as: :team_membership, + where: tm.team_id == ^team.id and tm.role == :guest, + where: not exists(guest_query) + ) + ) + + :ok + end + + def get_team_membership(team, %Auth.User{} = user) do + get_team_membership(team, user.id) + end + + def get_team_membership(team, user_id) do + query = + from( + tm in Teams.Membership, + where: tm.team_id == ^team.id and tm.user_id == ^user_id + ) + + case Repo.one(query) do + nil -> {:error, :membership_not_found} + membership -> {:ok, membership} + end + end + + @spec set_preference(Teams.Membership.t(), atom(), any()) :: + Teams.Memberships.UserPreference.t() + def set_preference(team_membership, option, value) + when option in Teams.Memberships.UserPreference.options() do + team_membership + |> Teams.Memberships.UserPreference.changeset(%{option => value}) + |> Repo.insert!( + conflict_target: [:team_membership_id], + on_conflict: {:replace, [option, :updated_at]}, + returning: true + ) + end + + @spec get_preference(Teams.Membership.t(), atom()) :: any() + def get_preference(team_membership, option) + when option in Teams.Memberships.UserPreference.options() do + defaults = %Teams.Memberships.UserPreference{} + + query = + from( + tup in Teams.Memberships.UserPreference, + where: tup.team_membership_id == ^team_membership.id, + select: field(tup, ^option) + ) + + Repo.one(query) || Map.fetch!(defaults, option) + end + + defp get_guest_membership(site_id, user_id) do + query = + from( + gm in Teams.GuestMembership, + inner_join: tm in assoc(gm, :team_membership), + where: gm.site_id == ^site_id and tm.user_id == ^user_id, + preload: [team_membership: tm] + ) + + case Repo.one(query) do + nil -> {:error, :no_guest} + membership -> {:ok, membership} + end + end + + defp pending_site_transfers_query(email) do + from st in Teams.SiteTransfer, where: st.email == ^email, select: st.site_id + end +end diff --git a/lib/plausible/teams/memberships/leave.ex b/lib/plausible/teams/memberships/leave.ex new file mode 100644 index 0000000000000000000000000000000000000000..18436fcf6ba81fcfb4c993bf7d1478964dd588ab --- /dev/null +++ b/lib/plausible/teams/memberships/leave.ex @@ -0,0 +1,67 @@ +defmodule Plausible.Teams.Memberships.Leave do + @moduledoc """ + Service for leaving a team by member. + """ + + alias Plausible.Repo + alias Plausible.Teams + alias Plausible.Teams.Memberships + + @spec leave(Teams.Team.t(), Plausible.Auth.User.t(), Keyword.t()) :: + {:ok, Teams.Membership.t()} | {:error, :only_one_owner | :membership_not_found} + def leave(team, user, opts \\ []) do + with {:ok, team_membership} <- Memberships.get_team_membership(team, user.id), + :ok <- check_owner_can_leave(team, team_membership.role) do + team_membership = Repo.preload(team_membership, [:team, :user]) + + {:ok, _} = + Repo.transaction(fn -> + delete_membership!(team_membership) + + Plausible.Segments.after_user_removed_from_team( + team_membership.team, + team_membership.user + ) + + Plausible.Annotations.after_user_removed_from_team( + team_membership.team, + team_membership.user + ) + end) + + if Keyword.get(opts, :send_email?, true) do + send_team_member_left_email(team_membership) + end + + {:ok, team_membership} + end + end + + defp delete_membership!(team_membership) do + user = team_membership.user + + Repo.delete!(team_membership) + + if Plausible.Users.type(user) == :sso do + {:ok, :deleted} = Plausible.Auth.delete_user(user) + end + + :ok + end + + defp check_owner_can_leave(team, :owner) do + if Memberships.owners_count(team) > 1 do + :ok + else + {:error, :only_one_owner} + end + end + + defp check_owner_can_leave(_team, _role), do: :ok + + def send_team_member_left_email(team_membership) do + team_membership + |> PlausibleWeb.Email.team_member_left() + |> Plausible.Mailer.send() + end +end diff --git a/lib/plausible/teams/memberships/remove.ex b/lib/plausible/teams/memberships/remove.ex new file mode 100644 index 0000000000000000000000000000000000000000..52261f36502768a4211f74fabedc26118a83b102 --- /dev/null +++ b/lib/plausible/teams/memberships/remove.ex @@ -0,0 +1,72 @@ +defmodule Plausible.Teams.Memberships.Remove do + @moduledoc """ + Service for removing a team member. + """ + + alias Plausible.Repo + alias Plausible.Teams.Memberships + + def remove(nil, _, _), do: {:error, :permission_denied} + + def remove(team, user_id, current_user, opts \\ []) do + with {:ok, team_membership} <- Memberships.get_team_membership(team, user_id), + {:ok, current_user_role} <- Memberships.team_role(team, current_user), + :ok <- check_can_remove_membership(current_user_role, team_membership.role), + :ok <- check_owner_can_get_removed(team, team_membership.role) do + team_membership = Repo.preload(team_membership, [:team, :user]) + + {:ok, _} = + Repo.transaction(fn -> + delete_membership!(team_membership) + + Plausible.Segments.after_user_removed_from_team( + team_membership.team, + team_membership.user + ) + + Plausible.Annotations.after_user_removed_from_team( + team_membership.team, + team_membership.user + ) + end) + + if Keyword.get(opts, :send_email?, true) do + send_team_member_removed_email(team_membership) + end + + {:ok, team_membership} + end + end + + defp delete_membership!(team_membership) do + user = team_membership.user + + Repo.delete!(team_membership) + + if Plausible.Users.type(user) == :sso do + {:ok, :deleted} = Plausible.Auth.delete_user(user) + end + + :ok + end + + defp check_can_remove_membership(:owner, _), do: :ok + defp check_can_remove_membership(:admin, role) when role != :owner, do: :ok + defp check_can_remove_membership(_, _), do: {:error, :permission_denied} + + defp check_owner_can_get_removed(team, :owner) do + if Memberships.owners_count(team) > 1 do + :ok + else + {:error, :only_one_owner} + end + end + + defp check_owner_can_get_removed(_team, _role), do: :ok + + def send_team_member_removed_email(team_membership) do + team_membership + |> PlausibleWeb.Email.team_member_removed() + |> Plausible.Mailer.send() + end +end diff --git a/lib/plausible/teams/memberships/update_role.ex b/lib/plausible/teams/memberships/update_role.ex new file mode 100644 index 0000000000000000000000000000000000000000..e1aa9bcacd3cb3e28e20ab95c52cf7e2953ef35a --- /dev/null +++ b/lib/plausible/teams/memberships/update_role.ex @@ -0,0 +1,143 @@ +defmodule Plausible.Teams.Memberships.UpdateRole do + @moduledoc """ + Service for updating role of a team member. + """ + + use Plausible + + alias Plausible.Repo + alias Plausible.Teams + alias Plausible.Teams.Memberships + + def update(nil, _, _, _), do: {:error, :permission_denied} + + def update(team, user_id, new_role_str, current_user) do + new_role = String.to_existing_atom(new_role_str) + + with :ok <- check_valid_role(new_role), + {:ok, team_membership} <- Memberships.get_team_membership(team, user_id), + {:ok, current_user_role} <- Memberships.team_role(team, current_user), + granting_to_self? = team_membership.user_id == current_user.id, + :ok <- + check_can_grant_role( + current_user_role, + team_membership.role, + new_role, + granting_to_self? + ), + :ok <- check_owner_can_get_demoted(team, team_membership.role, new_role), + team_membership = Repo.preload(team_membership, :user), + :ok <- check_can_promote_to_owner(team, team_membership.user, new_role) do + if team_membership.role == :guest and new_role != :guest do + team_membership.user.email + |> PlausibleWeb.Email.guest_to_team_member_promotion( + team, + current_user + ) + |> Plausible.Mailer.send() + end + + team_membership = + team_membership + |> Ecto.Changeset.change(role: new_role) + |> Repo.update!() + + :ok = maybe_prune_guest_memberships(team_membership) + + {:ok, team_membership} + end + end + + on_ee do + defp check_can_promote_to_owner(team, user, :owner) do + if team.policy.force_sso == :all_but_owners and not Plausible.Auth.TOTP.enabled?(user) do + {:error, :disabled_2fa} + else + :ok + end + end + + defp check_can_promote_to_owner(_team, _user, _new_role), do: :ok + else + defp check_can_promote_to_owner(_team, _user, _new_role) do + # The `else` branch is not reachable. + # This a workaround for Elixir 1.18+ compiler + # being too smart. + if :erlang.phash2(1, 1) == 0 do + :ok + else + {:error, :disabled_2fa} + end + end + end + + defp check_valid_role(role) do + if role in (Teams.Membership.roles() -- [:guest]) do + :ok + else + {:error, :invalid_role} + end + end + + defp check_owner_can_get_demoted(team, :owner, new_role) when new_role != :owner do + if Memberships.owners_count(team) > 1 do + :ok + else + {:error, :only_one_owner} + end + end + + defp check_owner_can_get_demoted(_team, _current_role, _new_role), do: :ok + + defp check_can_grant_role(user_role, _from_role, to_role, true) do + if can_grant_role_to_self?(user_role, to_role) do + :ok + else + {:error, :permission_denied} + end + end + + defp check_can_grant_role(user_role, from_role, to_role, false) do + if can_grant_role_to_other?(user_role, from_role, to_role) do + :ok + else + {:error, :permission_denied} + end + end + + defp can_grant_role_to_self?(:owner, :admin), do: true + defp can_grant_role_to_self?(:owner, :editor), do: true + defp can_grant_role_to_self?(:owner, :viewer), do: true + defp can_grant_role_to_self?(:owner, :billing), do: true + defp can_grant_role_to_self?(:admin, :editor), do: true + defp can_grant_role_to_self?(:admin, :viewer), do: true + defp can_grant_role_to_self?(:admin, :billing), do: true + defp can_grant_role_to_self?(_, _), do: false + + defp can_grant_role_to_other?(:owner, _, _), do: true + defp can_grant_role_to_other?(:admin, :admin, :admin), do: true + defp can_grant_role_to_other?(:admin, :admin, :editor), do: true + defp can_grant_role_to_other?(:admin, :admin, :viewer), do: true + defp can_grant_role_to_other?(:admin, :admin, :billing), do: true + defp can_grant_role_to_other?(:admin, :editor, :admin), do: true + defp can_grant_role_to_other?(:admin, :editor, :editor), do: true + defp can_grant_role_to_other?(:admin, :editor, :viewer), do: true + defp can_grant_role_to_other?(:admin, :editor, :billing), do: true + defp can_grant_role_to_other?(:admin, :viewer, :admin), do: true + defp can_grant_role_to_other?(:admin, :viewer, :editor), do: true + defp can_grant_role_to_other?(:admin, :viewer, :viewer), do: true + defp can_grant_role_to_other?(:admin, :viewer, :billing), do: true + defp can_grant_role_to_other?(:admin, :billing, :billing), do: true + defp can_grant_role_to_other?(_, _, _), do: false + + defp maybe_prune_guest_memberships(%Teams.Membership{role: :guest}), + do: :ok + + defp maybe_prune_guest_memberships(%Teams.Membership{} = team_membership) do + team_membership + |> Ecto.assoc(:guest_memberships) + |> Repo.delete_all() + + :ok + end +end diff --git a/lib/plausible/teams/memberships/user_preference.ex b/lib/plausible/teams/memberships/user_preference.ex new file mode 100644 index 0000000000000000000000000000000000000000..18a5b88e07c2fcbba759df586580c7171f388fdb --- /dev/null +++ b/lib/plausible/teams/memberships/user_preference.ex @@ -0,0 +1,30 @@ +defmodule Plausible.Teams.Memberships.UserPreference do + @moduledoc """ + Team-specific user preferences schema + """ + + use Ecto.Schema + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + @options [:consolidated_view_cta_dismissed, :sort_index_options] + + schema "team_membership_user_preferences" do + field :consolidated_view_cta_dismissed, :boolean, default: false + embeds_one :sort_index_options, Plausible.Sites.Index.UserPreference, on_replace: :update + + belongs_to :team_membership, Plausible.Teams.Membership + + timestamps() + end + + defmacro options, do: @options + + def changeset(team_membership, attrs \\ %{}) do + %__MODULE__{} + |> cast(attrs, [:consolidated_view_cta_dismissed]) + |> cast_embed(:sort_index_options) + |> put_assoc(:team_membership, team_membership) + end +end diff --git a/lib/plausible/teams/policy.ex b/lib/plausible/teams/policy.ex new file mode 100644 index 0000000000000000000000000000000000000000..e0e16dc6de1fd005332e6e737f57d38f49156ce1 --- /dev/null +++ b/lib/plausible/teams/policy.ex @@ -0,0 +1,82 @@ +defmodule Plausible.Teams.Policy do + @moduledoc """ + Team-wide policies. + """ + + use Ecto.Schema + use Plausible + + import Ecto.Changeset + + @sso_member_roles Plausible.Teams.Membership.roles() -- [:guest, :owner] + @force_sso_modes [:none, :all_but_owners] + + @update_fields [:sso_default_role, :sso_session_timeout_minutes] + + @default_timeout_minutes 6 * 60 + @min_timeout_minutes 30 + @max_timeout_minutes 12 * 60 + + @type t() :: %__MODULE__{} + + @type sso_member_role() :: unquote(Enum.reduce(@sso_member_roles, &{:|, [], [&1, &2]})) + + @type force_sso_mode() :: unquote(Enum.reduce(@force_sso_modes, &{:|, [], [&1, &2]})) + + on_ee do + @derive {Plausible.Audit.Encoder, + only: [:force_2fa, :force_sso, :sso_default_role, :sso_session_timeout_minutes]} + end + + @primary_key false + embedded_schema do + # SSO options apply to all team's integrations, should there + # ever be more than one allowed at once. + + # SSO enforcement can have one of 2 states: enforced for none + # or enforced for all but owners. + # The first state is useful in the initial phase of SSO setup + # when it's not yet confirmed to be fully operational. + # The second state is a good default for most, leaving + # escape hatch for cases where IdP starts failing. + field :force_sso, Ecto.Enum, values: [:none, :all_but_owners], default: :none + + # Default role for newly provisioned SSO accounts. + field :sso_default_role, Ecto.Enum, values: @sso_member_roles, default: :viewer + + # Default session timeout for SSO-enabled accounts. We might also + # consider accepting session timeout from assertion, if present. + field :sso_session_timeout_minutes, :integer, default: @default_timeout_minutes + + # Enforce enabling 2FA for all users + field :force_2fa, :boolean, default: false + end + + @spec sso_member_roles() :: [sso_member_role()] + def sso_member_roles(), do: @sso_member_roles + + @spec update_changeset(t(), map()) :: Ecto.Changeset.t() + def update_changeset(policy, params) do + policy + |> cast(params, @update_fields) + |> validate_required(@update_fields) + |> validate_number(:sso_session_timeout_minutes, + greater_than_or_equal_to: @min_timeout_minutes, + less_than_or_equal_to: @max_timeout_minutes + ) + end + + @spec force_sso_changeset(t(), force_sso_mode()) :: Ecto.Changeset.t() + def force_sso_changeset(policy, mode) do + policy + |> cast(%{force_sso: mode}, [:force_sso]) + |> validate_required(:force_sso) + end + + @spec force_2fa_changeset(t(), boolean()) :: Ecto.Changeset.t() + def force_2fa_changeset(policy, enabled?) when is_boolean(enabled?) do + policy + |> cast(%{force_2fa: enabled?}, [:force_2fa]) + |> validate_required(:force_2fa) + end +end diff --git a/lib/plausible/teams/site_transfer.ex b/lib/plausible/teams/site_transfer.ex new file mode 100644 index 0000000000000000000000000000000000000000..54b9f91eff32e77185013d9c90cb4c43126fa9be --- /dev/null +++ b/lib/plausible/teams/site_transfer.ex @@ -0,0 +1,37 @@ +defmodule Plausible.Teams.SiteTransfer do + @moduledoc """ + Site transfer schema + """ + + use Ecto.Schema + + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + schema "team_site_transfers" do + field :transfer_id, :string + field :email, :string + field :transfer_guests, :boolean, default: true + + belongs_to :site, Plausible.Site + belongs_to :initiator, Plausible.Auth.User + belongs_to :destination_team, Plausible.Teams.Team + + timestamps() + end + + def changeset(site, opts) do + initiator = Keyword.fetch!(opts, :initiator) + transfer_guests = Keyword.get(opts, :transfer_guests, true) + destination_team = Keyword.get(opts, :destination_team) + email = Keyword.get(opts, :email) + + %__MODULE__{transfer_id: Nanoid.generate()} + |> cast(%{email: email}, [:email]) + |> put_change(:transfer_guests, transfer_guests) + |> put_assoc(:site, site) + |> put_assoc(:destination_team, destination_team) + |> put_assoc(:initiator, initiator) + end +end diff --git a/lib/plausible/teams/sites.ex b/lib/plausible/teams/sites.ex new file mode 100644 index 0000000000000000000000000000000000000000..1f09baa1c322823f4f9a74e9ce6daf00ed42c19c --- /dev/null +++ b/lib/plausible/teams/sites.ex @@ -0,0 +1,94 @@ +defmodule Plausible.Teams.Sites do + @moduledoc false + + import Ecto.Query + + alias Plausible.Auth + alias Plausible.Repo + alias Plausible.Site + alias Plausible.Teams + + @type list_opt() :: {:filter_by_domain, String.t()} | {:team, Teams.Team.t() | nil} + + @role_type Plausible.Teams.Invitation.__schema__(:type, :role) + + @spec accessible_by(Auth.User.t(), Teams.Team.t() | nil) :: Ecto.Query.t() + def accessible_by(user, team) do + if Teams.setup?(team) do + from(tm in Teams.Membership, + inner_join: t in assoc(tm, :team), + inner_join: s in assoc(t, :sites), + where: tm.user_id == ^user.id and tm.role != :guest, + where: tm.team_id == ^team.id, + select: %{site_id: s.id, role: tm.role} + ) + else + my_team_query = + from(tm in Teams.Membership, + inner_join: t in assoc(tm, :team), + inner_join: s in assoc(t, :sites), + where: tm.user_id == ^user.id and tm.role != :guest, + where: tm.is_autocreated == true, + where: t.setup_complete == false, + select: %{site_id: s.id, role: tm.role} + ) + + guest_membership_query = + from(tm in Teams.Membership, + inner_join: gm in assoc(tm, :guest_memberships), + inner_join: s in assoc(gm, :site), + where: tm.user_id == ^user.id and tm.role == :guest, + select: %{ + site_id: s.id, + role: + fragment( + """ + CASE + WHEN ? = 'editor' THEN 'admin' + ELSE ? + END + """, + gm.role, + gm.role + ) + } + ) + + from(s in my_team_query, union_all: ^guest_membership_query) + end + end + + @spec get_for_user_by_ids(Auth.User.t(), [pos_integer()], [list_opt()]) :: [Site.t()] + def get_for_user_by_ids(_user, [], _opts), do: [] + + def get_for_user_by_ids(user, site_ids, opts) do + team = Keyword.get(opts, :team) + + rows = + from(u in subquery(accessible_by(user, team)), + inner_join: s in ^Plausible.Site.regular(), + on: u.site_id == s.id, + left_join: up in Site.UserPreference, + on: up.site_id == s.id and up.user_id == ^user.id, + where: s.id in ^site_ids, + select: + {s.id, + %{ + s + | pinned_at: selected_as(up.pinned_at, :pinned_at), + memberships: [ + %{ + role: type(u.role, ^@role_type), + site_id: s.id, + site: s + } + ] + }} + ) + |> Repo.all() + |> Map.new() + + # Restore the caller-supplied order + Enum.flat_map(site_ids, fn id -> List.wrap(Map.get(rows, id)) end) + end +end diff --git a/lib/plausible/teams/sites/transfer.ex b/lib/plausible/teams/sites/transfer.ex new file mode 100644 index 0000000000000000000000000000000000000000..fe75ee92e23abca03ec465745897baa249ca2843 --- /dev/null +++ b/lib/plausible/teams/sites/transfer.ex @@ -0,0 +1,69 @@ +defmodule Plausible.Teams.Sites.Transfer do + @moduledoc """ + Service for transferring sites. + """ + + alias Plausible.Auth + alias Plausible.Billing + alias Plausible.Repo + alias Plausible.Site + alias Plausible.Teams + + @type transfer_error() :: + Billing.Quota.Limits.over_limits_error() + | Ecto.Changeset.t() + | :transfer_to_self + | :no_plan + | :multiple_teams + | :permission_denied + + @spec change_team(Site.t(), Auth.User.t(), Teams.Team.t()) :: + :ok | {:error, transfer_error()} + def change_team(site, user, new_team) do + with {:ok, _} <- transfer_ownership(site, user, new_team) do + Teams.Invitations.send_team_changed_email(site, user, new_team) + :ok + end + end + + @spec accept(Teams.SiteTransfer.t(), Auth.User.t(), Teams.Team.t() | nil, Keyword.t()) :: + {:ok, %{team: Teams.Team.t(), team_memberships: [Teams.Membership.t()], site: Site.t()}} + | {:error, transfer_error()} + def accept(site_transfer, new_owner, team, opts \\ []) do + site = Repo.preload(site_transfer.site, :team) + + with {:ok, new_team} <- maybe_get_team(new_owner, team), + :ok <- Teams.Invitations.ensure_transfer_valid(site.team, new_team, :owner), + :ok <- Teams.Invitations.check_can_transfer_site(new_team, new_owner), + :ok <- Teams.Invitations.ensure_can_take_ownership(site, new_team, opts), + :ok <- Teams.Invitations.accept_site_transfer(site_transfer, new_team, opts) do + Teams.Invitations.send_transfer_accepted_email(site_transfer, new_team) + + site = site |> Repo.reload!() |> Repo.preload(ownerships: :user) + + {:ok, %{team: new_team, team_memberships: site.ownerships, site: site}} + end + end + + defp transfer_ownership(site, new_owner, team) do + site = Repo.preload(site, :team) + + with {:ok, new_team} <- maybe_get_team(new_owner, team), + :ok <- Teams.Invitations.ensure_transfer_valid(site.team, new_team, :owner), + :ok <- Teams.Invitations.check_can_transfer_site(new_team, new_owner), + :ok <- Teams.Invitations.ensure_can_take_ownership(site, new_team), + :ok <- Teams.Invitations.transfer_site(site, new_team) do + site = site |> Repo.reload!() |> Repo.preload(ownerships: :user) + + {:ok, site.ownerships} + end + end + + defp maybe_get_team(_user, %Teams.Team{} = team) do + {:ok, team} + end + + defp maybe_get_team(user, nil) do + Teams.get_or_create(user) + end +end diff --git a/lib/plausible/teams/team.ex b/lib/plausible/teams/team.ex new file mode 100644 index 0000000000000000000000000000000000000000..bd4872c3df9e59686db6bb9f6235b294db640e63 --- /dev/null +++ b/lib/plausible/teams/team.ex @@ -0,0 +1,171 @@ +defmodule Plausible.Teams.Team do + @moduledoc """ + Team schema + """ + + defimpl FunWithFlags.Actor, for: __MODULE__ do + def id(%{id: id}) do + "team:#{id}" + end + end + + use Ecto.Schema + use Plausible + + alias Plausible.Auth + + import Ecto.Changeset + + @type t() :: %__MODULE__{} + + @trial_accept_traffic_until_offset_days 14 + @subscription_accept_traffic_until_offset_days 30 + + on_ee do + @derive {Plausible.Audit.Encoder, + only: [ + :id, + :identifier, + :name, + :trial_expiry_date, + :accept_traffic_until, + :allow_next_upgrade_override, + :locked, + :setup_complete, + :setup_at, + :hourly_api_request_limit, + :policy + ]} + end + + schema "teams" do + field :identifier, Ecto.UUID + field :name, :string + field :trial_expiry_date, :date + field :accept_traffic_until, :date + field :allow_next_upgrade_override, :boolean, default: false + field :locked, :boolean, default: false + + field :setup_complete, :boolean, default: false + field :setup_at, :naive_datetime + + # Field synced from current subscription plan, if any. The value of this field is treated as the source of truth when out of sync + field :hourly_api_request_limit, :integer, default: Auth.ApiKey.default_hourly_request_limit() + + # Field for purely informational purposes in CRM context + field :notes, :string + + # Embed for storing team-wide policies + embeds_one :policy, Plausible.Teams.Policy, on_replace: :update, defaults_to_struct: true + + embeds_one :grace_period, Plausible.Teams.GracePeriod, on_replace: :update + + has_many :sites, Plausible.Site + has_many :team_memberships, Plausible.Teams.Membership + has_many :team_invitations, Plausible.Teams.Invitation + has_one :subscription, Plausible.Billing.Subscription + has_one :enterprise_plan, Plausible.Billing.EnterprisePlan + + on_ee do + has_one :sso_integration, Plausible.Auth.SSO.Integration + end + + has_many :ownerships, Plausible.Teams.Membership, + where: [role: :owner], + preload_order: [asc: :id] + + has_many :billing_memberships, Plausible.Teams.Membership, + where: [role: :billing], + preload_order: [asc: :id] + + has_many :owners, through: [:ownerships, :user] + has_many :billing_members, through: [:billing_memberships, :user] + + timestamps() + end + + def crm_changeset(team, params) do + team + |> cast(params, [ + :name, + :notes, + :trial_expiry_date, + :allow_next_upgrade_override, + :accept_traffic_until + ]) + |> maybe_bump_accept_traffic_until() + end + + def changeset(team \\ %__MODULE__{}, attrs \\ %{}, today \\ Date.utc_today()) do + team + |> cast(attrs, [:name]) + |> validate_required(:name) + |> start_trial(today) + |> maybe_bump_accept_traffic_until() + |> maybe_set_identifier() + end + + def name_changeset(team, attrs \\ %{}) do + team + |> cast(attrs, [:name]) + |> validate_required(:name) + |> validate_exclusion(:name, [Plausible.Teams.default_name()]) + end + + def setup_changeset(team, now \\ NaiveDateTime.utc_now(:second)) do + team + |> change( + setup_complete: true, + setup_at: now + ) + end + + def start_trial(team, today \\ Date.utc_today()) do + trial_expiry = trial_expiry(today) + + change(team, + trial_expiry_date: trial_expiry, + accept_traffic_until: Date.add(trial_expiry, @trial_accept_traffic_until_offset_days) + ) + end + + def end_trial(team) do + change(team, trial_expiry_date: Date.utc_today() |> Date.shift(day: -1)) + end + + defp maybe_set_identifier(changeset) do + if get_field(changeset, :identifier) do + changeset + else + put_change(changeset, :identifier, Ecto.UUID.generate()) + end + end + + defp maybe_bump_accept_traffic_until(changeset) do + expiry_change = get_change(changeset, :trial_expiry_date) + + if expiry_change do + put_change( + changeset, + :accept_traffic_until, + Date.add(expiry_change, @trial_accept_traffic_until_offset_days) + ) + else + changeset + end + end + + def trial_accept_traffic_until_offset_days(), do: @trial_accept_traffic_until_offset_days + + def subscription_accept_traffic_until_offset_days(), + do: @subscription_accept_traffic_until_offset_days + + @doc false + def trial_expiry(today \\ Date.utc_today()) do + on_ee do + Date.shift(today, day: 30) + else + Date.shift(today, year: 100) + end + end +end diff --git a/lib/plausible/teams/users.ex b/lib/plausible/teams/users.ex new file mode 100644 index 0000000000000000000000000000000000000000..8e92e9ffe776c27d3e0fe158600c1528941c375b --- /dev/null +++ b/lib/plausible/teams/users.ex @@ -0,0 +1,198 @@ +defmodule Plausible.Teams.Users do + @moduledoc """ + Users API accounting for teams. + """ + + import Ecto.Query + + alias Plausible.Repo + alias Plausible.Teams + + def owned_teams(user) do + Repo.all(teams_query(user, roles: :owner)) + end + + def teams(user, opts \\ []) do + user + |> teams_query(Keyword.merge([order_by: :name], opts)) + |> Repo.all() + |> Repo.preload(:owners) + end + + def teams_query(user, opts \\ []) do + order_by = Keyword.get(opts, :order_by, :name) + roles = Keyword.get(opts, :roles) + identifier = Keyword.get(opts, :identifier) + + query = + from( + tm in Teams.Membership, + as: :team_membership, + inner_join: t in assoc(tm, :team), + as: :team, + where: tm.user_id == ^user.id, + where: tm.role != :guest, + select: t + ) + + query = + if roles do + where(query, [team_membership: tm], tm.role in ^List.wrap(roles)) + else + query + end + + query = + if identifier do + where(query, [team: team], team.identifier == ^identifier) + else + query + end + + case order_by do + :name -> + order_by(query, [team: t], [t.name, t.id]) + + :id_desc -> + order_by(query, [team: t], desc: t.id) + + _ -> + query + end + end + + def team_member?(user, opts \\ []) do + excluded_team_ids = Keyword.get(opts, :except, []) + only_setup? = Keyword.get(opts, :only_setup?, false) + + query = + from( + tm in Teams.Membership, + where: tm.user_id == ^user.id, + where: tm.role != :guest, + where: tm.team_id not in ^excluded_team_ids + ) + + query = + if only_setup? do + query + |> join(:inner, [tm], t in assoc(tm, :team), as: :team) + |> where([team: t], t.setup_complete == true) + else + query + end + + Repo.exists?(query) + end + + def has_sites?(user, opts \\ []) do + include_pending? = Keyword.get(opts, :include_pending?, false) + + sites_query = + from( + s in Plausible.Site, + where: s.team_id == parent_as(:site_parent).team_id, + select: 1 + ) + + # NOTE: Provided guest team memberships are consistently pruned, + # mere presence of guest team membership means there are sites + # associated with it, so querying for team membership alone + # should be enough. + team_member_query = + from( + tm in Teams.Membership, + as: :site_parent, + where: exists(sites_query), + where: tm.user_id == ^user.id, + select: 1 + ) + + query = + if include_pending? do + site_transfer_query = + from( + st in Teams.SiteTransfer, + where: st.email == ^user.email, + select: 1 + ) + + # NOTE: The same principle applies to guest team invitations, + # as it's pruned as well when there are no more guest invitations + # associated with it. + member_invitation_query = + from( + ti in Teams.Invitation, + as: :site_parent, + where: exists(sites_query), + where: ti.email == ^user.email, + select: 1 + ) + + team_member_query + |> union_all(^site_transfer_query) + |> union_all(^member_invitation_query) + else + team_member_query + end + + Repo.exists?(query) + end + + def owns_sites?(user, opts \\ []) do + include_pending? = Keyword.get(opts, :include_pending?, false) + only_team = Keyword.get(opts, :only_team) + + sites_query = + from( + s in Plausible.Site, + where: s.team_id == parent_as(:site_parent).team_id, + select: 1 + ) + + owner_query = + from( + tm in Teams.Membership, + as: :site_parent, + where: exists(sites_query), + where: tm.user_id == ^user.id, + where: tm.role == :owner, + select: 1 + ) + + owner_query = + if only_team do + where(owner_query, [tm], tm.team_id == ^only_team.id) + else + owner_query + end + + query = + if include_pending? do + site_transfer_query = + from( + st in Teams.SiteTransfer, + where: st.email == ^user.email, + select: 1 + ) + + owner_invitation_query = + from( + ti in Teams.Invitation, + as: :site_parent, + where: exists(sites_query), + where: ti.email == ^user.email, + where: ti.role == :owner, + select: 1 + ) + + owner_query + |> union_all(^site_transfer_query) + |> union_all(^owner_invitation_query) + else + owner_query + end + + Repo.exists?(query) + end +end diff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.ex new file mode 100644 index 0000000000000000000000000000000000000000..6ed2543ea424af7461db7babc8b261e83be62469 --- /dev/null +++ b/lib/plausible/telemetry/plausible_metrics.ex @@ -0,0 +1,384 @@ +defmodule Plausible.PromEx.Plugins.PlausibleMetrics do + @moduledoc """ + Custom PromEx plugin for instrumenting code within Plausible app. + """ + use Plausible + use PromEx.Plugin + alias Plausible.Site + alias Plausible.Ingestion + alias Plausible.Ingestion.Persistor + + on_ee do + alias Plausible.InstallationSupport + end + + @impl true + def polling_metrics(opts) do + poll_rate = Keyword.get(opts, :poll_rate, 5_000) + + otp_app = Keyword.fetch!(opts, :otp_app) + + metric_prefix = + Keyword.get(opts, :metric_prefix, PromEx.metric_prefix(otp_app, :plausible_metrics)) + + [ + write_buffer_metrics(metric_prefix, poll_rate), + cache_metrics(metric_prefix, poll_rate) + ] + end + + @impl true + def event_metrics(opts) do + otp_app = Keyword.fetch!(opts, :otp_app) + metric_prefix = Keyword.get(opts, :metric_prefix, PromEx.metric_prefix(otp_app, :plausible)) + + Event.build( + :plausible_internal_telemetry, + [ + distribution( + metric_prefix ++ [:cache_warmer, :sites, :refresh, :all], + event_name: Site.Cache.telemetry_event_refresh(:all), + reporter_options: [ + buckets: [500, 1000, 2000, 5000, 10_000] + ], + unit: {:native, :millisecond}, + measurement: :duration + ), + distribution( + metric_prefix ++ [:cache_warmer, :sites, :refresh, :updated_recently], + event_name: Site.Cache.telemetry_event_refresh(:updated_recently), + reporter_options: [ + buckets: [500, 1000, 2000, 5000, 10_000] + ], + unit: {:native, :millisecond}, + measurement: :duration + ), + distribution( + metric_prefix ++ [:cache_warmer, :tracker_script, :refresh, :all], + event_name: + on_ee do + Plausible.Site.TrackerScriptIdCache.telemetry_event_refresh(:all) + else + Plausible.Site.TrackerScriptCache.telemetry_event_refresh(:all) + end, + reporter_options: [ + buckets: [500, 1000, 2000, 5000, 10_000] + ], + unit: {:native, :millisecond}, + measurement: :duration + ), + distribution( + metric_prefix ++ [:cache_warmer, :tracker_script, :refresh, :updated_recently], + event_name: + on_ee do + Plausible.Site.TrackerScriptIdCache.telemetry_event_refresh(:updated_recently) + else + Plausible.Site.TrackerScriptCache.telemetry_event_refresh(:updated_recently) + end, + reporter_options: [ + buckets: [500, 1000, 2000, 5000, 10_000] + ], + unit: {:native, :millisecond}, + measurement: :duration + ), + distribution( + metric_prefix ++ [:ingest, :events, :pipeline, :steps], + event_name: Ingestion.Event.telemetry_pipeline_step_duration(), + reporter_options: [ + buckets: [10, 50, 100, 250, 350, 500, 1000, 5000, 10_000, 100_000, 500_000] + ], + unit: {:native, :microsecond}, + measurement: :duration, + tags: [:step] + ), + distribution( + metric_prefix ++ [:remote_ingest, :events, :pipeline, :steps], + event_name: Ingestion.Persistor.EmbeddedWithRelay.telemetry_pipeline_step_duration(), + reporter_options: [ + buckets: [ + 10, + 50, + 100, + 250, + 350, + 500, + 1000, + 5000, + 7_000, + 10_000, + 15_000, + 20_000, + 35_000, + 50_000, + 100_000, + 500_000 + ] + ], + unit: {:native, :microsecond}, + measurement: :duration, + tags: [:step] + ), + distribution( + metric_prefix ++ [:sessions, :cache, :register, :lock], + event_name: Plausible.Session.CacheStore.lock_telemetry_event(), + reporter_options: [ + buckets: [10, 50, 100, 250, 350, 500, 1000, 5000, 10_000, 100_000, 500_000] + ], + unit: {:native, :microsecond}, + measurement: :duration + ), + counter( + metric_prefix ++ [:ingest, :events, :buffered, :total], + event_name: Ingestion.Event.telemetry_event_buffered() + ), + counter( + metric_prefix ++ [:ingest, :events, :dropped, :total], + event_name: Ingestion.Event.telemetry_event_dropped(), + tags: [:reason] + ), + counter( + metric_prefix ++ [:remote_ingest, :events, :buffered, :total], + event_name: Ingestion.Persistor.EmbeddedWithRelay.telemetry_event_buffered() + ), + counter( + metric_prefix ++ [:remote_ingest, :events, :dropped, :total], + event_name: Ingestion.Persistor.EmbeddedWithRelay.telemetry_event_dropped(), + tags: [:reason] + ), + counter( + metric_prefix ++ [:ingest, :user_agent_parse, :timeout, :total], + event_name: Ingestion.Event.telemetry_ua_parse_timeout() + ), + counter( + metric_prefix ++ [:plausible_cache, :hit], + event_name: ConCache.Operations.telemetry_hit(), + tags: [:name], + tag_values: &%{name: &1.cache.name} + ), + counter( + metric_prefix ++ [:plausible_cache, :miss], + event_name: ConCache.Operations.telemetry_miss(), + tags: [:name], + tag_values: &%{name: &1.cache.name} + ), + distribution( + metric_prefix ++ [:sessions, :transfer, :duration], + event_name: Plausible.Session.Transfer.telemetry_event(), + reporter_options: [ + buckets: [ + 100_000, + 250_000, + 500_000, + 750_000, + 1_000_000, + 2_500_000, + 5_000_000, + 7_500_000, + 10_000_000 + ] + ], + unit: {:native, :microsecond}, + measurement: :duration + ), + counter( + metric_prefix ++ [:tracker_script, :request, :v2], + event_name: PlausibleWeb.TrackerPlug.telemetry_event(:v2), + tags: [:status], + tag_values: &%{status: &1.status} + ), + counter( + metric_prefix ++ [:tracker_script, :request, :legacy], + event_name: PlausibleWeb.TrackerPlug.telemetry_event(:legacy), + tags: [:status], + tag_values: &%{status: &1.status} + ), + on_ee( + do: + counter( + metric_prefix ++ [:detection, :success], + event_name: InstallationSupport.Detection.Checks.telemetry_event_success() + ) + ), + on_ee( + do: + counter( + metric_prefix ++ [:detection, :failure], + event_name: InstallationSupport.Detection.Checks.telemetry_event_failure() + ) + ), + on_ee( + do: + counter( + metric_prefix ++ [:verification, :handled], + event_name: InstallationSupport.Verification.Checks.telemetry_event_handled() + ) + ), + on_ee( + do: + counter( + metric_prefix ++ [:verification, :unhandled], + event_name: InstallationSupport.Verification.Checks.telemetry_event_unhandled() + ) + ) + ] + |> Enum.concat(persistor_metrics(metric_prefix)) + |> Enum.reject(&is_nil/1) + ) + end + + @doc """ + Add telemetry events for Session and Event write buffers + """ + def execute_write_buffer_metrics do + event_write_buffer_pid = GenServer.whereis(Plausible.Event.WriteBuffer) + + {:message_queue_len, events_message_queue_len} = + if is_pid(event_write_buffer_pid), + do: Process.info(event_write_buffer_pid, :message_queue_len), + else: {:message_queue_len, 0} + + session_write_buffer_pid = GenServer.whereis(Plausible.Event.WriteBuffer) + + {:message_queue_len, sessions_message_queue_len} = + if is_pid(session_write_buffer_pid), + do: Process.info(session_write_buffer_pid, :message_queue_len), + else: {:message_queue_len, 0} + + :telemetry.execute([:prom_ex, :plugin, :write_buffer_metrics, :events_message_queue_len], %{ + count: events_message_queue_len + }) + + :telemetry.execute([:prom_ex, :plugin, :write_buffer_metrics, :sessions_message_queue_len], %{ + count: sessions_message_queue_len + }) + end + + @doc """ + Fire telemetry events for various caches + """ + def execute_cache_metrics do + :telemetry.execute([:prom_ex, :plugin, :cache, :user_agents], %{ + count: Plausible.Cache.Adapter.size(:user_agents) + }) + + :telemetry.execute([:prom_ex, :plugin, :cache, :sessions], %{ + count: Plausible.Cache.Adapter.size(:sessions) + }) + + :telemetry.execute([:prom_ex, :plugin, :cache, :sites], %{ + count: Site.Cache.size() + }) + end + + def measure_duration(event, fun, meta \\ %{}) when is_function(fun, 0) do + {duration, result} = time_it(fun) + :telemetry.execute(event, %{duration: duration}, meta) + result + end + + defp persistor_metrics(metric_prefix) do + [ + distribution( + metric_prefix ++ [:persistor, :remote, :request, :total_duration, :millisecond], + event_name: Persistor.Remote.telemetry_request_duration(), + reporter_options: [ + buckets: [1, 3, 5, 7, 10, 13, 15, 20, 25, 50, 75, 100, 500, 1_000, 10_000, 30_000] + ], + unit: {:native, :millisecond}, + measurement: :duration + ), + distribution( + metric_prefix ++ [:persistor, :remote, :request, :duration, :millisecond], + event_name: Persistor.TelemetryHandler.request_event(), + reporter_options: [ + buckets: [1, 3, 5, 7, 10, 13, 15, 20, 25, 50, 75, 100, 500, 1_000, 10_000, 30_000] + ], + unit: {:native, :millisecond}, + measurement: :duration, + tags: [:result, :path] + ), + distribution( + metric_prefix ++ [:persistor, :remote, :connect, :duration, :millisecond], + event_name: Persistor.TelemetryHandler.connect_event(), + reporter_options: [ + buckets: [1, 10, 50, 100, 500, 1_000, 10_000] + ], + unit: {:native, :millisecond}, + measurement: :duration, + tags: [:status] + ), + distribution( + metric_prefix ++ [:persistor, :remote, :send, :duration, :millisecond], + event_name: Persistor.TelemetryHandler.send_event(), + reporter_options: [ + buckets: [1, 10, 50, 100, 500, 1_000, 10_000] + ], + unit: {:native, :millisecond}, + measurement: :duration, + tags: [:status] + ), + distribution( + metric_prefix ++ [:persistor, :remote, :receive, :duration, :millisecond], + event_name: Persistor.TelemetryHandler.receive_event(), + reporter_options: [ + buckets: [1, 10, 25, 50, 75, 100, 250, 350, 500, 750, 1_000, 5_000, 10_000, 30_000] + ], + unit: {:native, :millisecond}, + measurement: :duration, + tags: [:status] + ) + ] + end + + defp time_it(fun) do + start = System.monotonic_time() + result = fun.() + stop = System.monotonic_time() + {stop - start, result} + end + + defp write_buffer_metrics(metric_prefix, poll_rate) do + Polling.build( + :write_buffer_metrics, + poll_rate, + {__MODULE__, :execute_write_buffer_metrics, []}, + [ + last_value( + metric_prefix ++ [:events, :message_queue_len, :count], + event_name: [:prom_ex, :plugin, :write_buffer_metrics, :events_message_queue_len], + measurement: :count + ), + last_value( + metric_prefix ++ [:sessions, :message_queue_len, :count], + event_name: [:prom_ex, :plugin, :write_buffer_metrics, :sessions_message_queue_len], + measurement: :count + ) + ] + ) + end + + defp cache_metrics(metric_prefix, poll_rate) do + Polling.build( + :cache_metrics, + poll_rate, + {__MODULE__, :execute_cache_metrics, []}, + [ + last_value( + metric_prefix ++ [:cache, :sessions, :size], + event_name: [:prom_ex, :plugin, :cache, :sessions], + measurement: :count + ), + last_value( + metric_prefix ++ [:cache, :user_agents, :size], + event_name: [:prom_ex, :plugin, :cache, :user_agents], + measurement: :count + ), + last_value( + metric_prefix ++ [:cache, :sites, :size], + event_name: [:prom_ex, :plugin, :cache, :sites], + measurement: :count + ) + ] + ) + end +end diff --git a/lib/plausible/themes.ex b/lib/plausible/themes.ex new file mode 100644 index 0000000000000000000000000000000000000000..5772bfaa5dcca7c31737c0a625aaebcf7fda9c29 --- /dev/null +++ b/lib/plausible/themes.ex @@ -0,0 +1,11 @@ +defmodule Plausible.Themes do + @options [ + [key: "Follow System Theme", value: "system"], + [key: "Light", value: "light"], + [key: "Dark", value: "dark"] + ] + + def options() do + @options + end +end diff --git a/lib/plausible/times.ex b/lib/plausible/times.ex new file mode 100644 index 0000000000000000000000000000000000000000..ddbf4cdda32fbdc6af44cfdf90182c043faac117 --- /dev/null +++ b/lib/plausible/times.ex @@ -0,0 +1,90 @@ +defmodule Plausible.Times do + @moduledoc """ + API for working with time wrapping around external libraries where necessary. + """ + + @spec today(String.t()) :: Date.t() + def today(tz) do + tz + |> DateTime.now!() + |> DateTime.to_date() + end + + @spec diff(Date.t() | DateTime.t(), Date.t() | DateTime.t(), :month | :week) :: integer() + def diff(a, b, unit) do + unit = + case unit do + :week -> :weeks + :month -> :months + end + + Timex.diff(a, b, unit) + end + + @spec parse!(String.t(), String.t(), :default | :strftime) :: DateTime.t() | NaiveDateTime.t() + def parse!(str, format, tokenizer \\ :default) + + def parse!(str, "{RFC1123}" = format, :default) do + Timex.parse!(str, format) + end + + def parse!(str, format, :strftime) do + Timex.parse!(str, format, :strftime) + end + + @spec beginning_of_week(DateTime.t()) :: DateTime.t() + def beginning_of_week(dt) do + Timex.beginning_of_week(dt) + end + + @spec beginning_of_month(DateTime.t()) :: DateTime.t() + def beginning_of_month(dt) do + Timex.beginning_of_month(dt) + end + + @spec beginning_of_year(Date.t()) :: Date.t() + def beginning_of_year(d) do + Timex.beginning_of_year(d) + end + + @spec end_of_week(DateTime.t()) :: DateTime.t() + def end_of_week(dt) do + Timex.end_of_week(dt) + end + + @spec end_of_month(DateTime.t()) :: DateTime.t() + def end_of_month(dt) do + Timex.end_of_month(dt) + end + + @spec end_of_year(Date.t()) :: Date.t() + def end_of_year(t) do + Timex.end_of_year(t) + end + + @spec to_date(DateTime.t(), String.t()) :: Date.t() + def to_date(%DateTime{} = dt, tz) do + dt + |> DateTime.shift_zone!(tz) + |> DateTime.to_date() + end + + @spec to_naive_datetime!(DateTime.t(), String.t()) :: NaiveDateTime.t() + def to_naive_datetime!(%DateTime{} = dt, tz) do + dt + |> DateTime.shift_zone!(tz) + |> DateTime.to_naive() + end + + @spec humanize(DateTime.t()) :: String.t() + def humanize(%DateTime{} = dt) do + Timex.Format.DateTime.Formatters.Relative.format!(dt, "{relative}") + end + + @spec humanize_seconds(pos_integer()) :: String.t() + def humanize_seconds(seconds) do + seconds + |> Timex.Duration.from_seconds() + |> Timex.format_duration(Timex.Format.Duration.Formatters.Humanized) + end +end diff --git a/lib/plausible/timezones.ex b/lib/plausible/timezones.ex new file mode 100644 index 0000000000000000000000000000000000000000..1998f90a034c805640664526bb2f9ad40d672bfd --- /dev/null +++ b/lib/plausible/timezones.ex @@ -0,0 +1,62 @@ +defmodule Plausible.Timezones do + @moduledoc """ + API for working with timezones wrapping around external libraries where necessary. + """ + + @spec valid?(String.t()) :: boolean() + def valid?(tz) do + Timex.is_valid_timezone?(tz) + end + + @spec zone_list() :: [String.t()] + def zone_list, do: Tzdata.zone_list() + + @spec options(DateTime.t()) :: [{:key, String.t()}, {:value, String.t()}, {:offset, integer()}] + def options(now \\ DateTime.utc_now()) do + Tzdata.zone_list() + |> Enum.reduce([], fn timezone_code, acc -> build_option(timezone_code, acc, now) end) + |> Enum.sort_by(& &1[:offset], :desc) + end + + @spec to_date_in_timezone(Date.t() | NaiveDateTime.t() | DateTime.t(), String.t()) :: Date.t() + def to_date_in_timezone(dt, timezone) do + to_datetime_in_timezone(dt, timezone) |> DateTime.to_date() + end + + @spec to_datetime_in_timezone(Date.t() | NaiveDateTime.t() | DateTime.t(), String.t()) :: + DateTime.t() + def to_datetime_in_timezone(dt, timezone) do + dt |> to_datetime() |> DateTime.shift_zone!(timezone) + end + + defp to_datetime(%NaiveDateTime{} = naive), do: DateTime.from_naive!(naive, "Etc/UTC") + defp to_datetime(%Date{} = date), do: DateTime.new!(date, ~T[00:00:00], "Etc/UTC") + defp to_datetime(%DateTime{} = already_dt), do: already_dt + + defp build_option(timezone_code, acc, now) do + case Timex.Timezone.get(timezone_code, now) do + %Timex.TimezoneInfo{} = timezone_info -> + offset_in_minutes = timezone_info |> Timex.Timezone.total_offset() |> div(-60) + + hhmm_formatted_offset = + timezone_info + |> Timex.TimezoneInfo.format_offset() + |> String.slice(0..-4//1) + + option = [ + key: "(GMT#{hhmm_formatted_offset}) #{timezone_code}", + value: timezone_code, + offset: offset_in_minutes + ] + + [option | acc] + + error -> + Sentry.capture_message("Failed to fetch timezone", + extra: %{code: timezone_code, error: inspect(error)} + ) + + acc + end + end +end diff --git a/lib/plausible/users.ex b/lib/plausible/users.ex new file mode 100644 index 0000000000000000000000000000000000000000..cb3ddc82341b375a1b357b9b8eea5dc7ac9a35fe --- /dev/null +++ b/lib/plausible/users.ex @@ -0,0 +1,55 @@ +defmodule Plausible.Users do + @moduledoc """ + User context + """ + use Plausible + + import Ecto.Query + + alias Plausible.Auth + alias Plausible.Repo + + on_ee do + @spec type(Auth.User.t()) :: :standard | :sso + def type(user) do + user.type + end + else + @spec type(Auth.User.t()) :: :standard + def type(_user) do + # The `else` branch is not reachable. + # This a workaround for Elixir 1.18+ compiler + # being too smart. + if :erlang.phash2(1, 1) == 0 do + :standard + else + :sso + end + end + end + + @spec bump_last_seen(Auth.User.t() | pos_integer(), NaiveDateTime.t()) :: :ok + def bump_last_seen(%Auth.User{id: user_id}, now) do + bump_last_seen(user_id, now) + end + + def bump_last_seen(user_id, now) do + q = from(u in Auth.User, where: u.id == ^user_id) + + Repo.update_all(q, set: [last_seen: now]) + + :ok + end + + @spec remember_last_team(Auth.User.t(), String.t() | nil) :: :ok + def remember_last_team(%Auth.User{id: user_id}, team_identifier) do + q = from(u in Auth.User, where: u.id == ^user_id) + Repo.update_all(q, set: [last_team_identifier: team_identifier]) + :ok + end + + @spec has_email_code?(Auth.User.t()) :: boolean() + def has_email_code?(user) do + Auth.EmailVerification.any?(user) + end +end diff --git a/lib/plausible_release.ex b/lib/plausible_release.ex new file mode 100644 index 0000000000000000000000000000000000000000..4dc21555519cdb38c3c3b5fc2b9e544bf41c3512 --- /dev/null +++ b/lib/plausible_release.ex @@ -0,0 +1,295 @@ +defmodule Plausible.Release do + @moduledoc """ + Utility functions used by release scripts for routine tasks, + like migrations. + """ + + use Plausible + use Plausible.Repo + + @app :plausible + @start_apps [ + :ssl, + :postgrex, + :ch, + :ecto + ] + + def should_be_first_launch? do + on_ee do + always(false) + else + not (_has_users? = Repo.exists?(Plausible.Auth.User)) + end + end + + @doc """ + `interweave_migrate/0` is a migration function that: + + - Lists all pending migrations across multiple repositories. + - Sorts these migrations into a single list. + - Groups consecutive migrations by repository into "streaks". + - Executes the migrations in the correct order by processing each streak sequentially. + + ### Why Use This Approach? + + This function resolves dependencies between migrations that span across different repositories. + The default `migrate/0` function migrates each repository independently, which may result in + migrations running in the wrong order when there are cross-repository dependencies. + + Consider the following example (adapted from reality, not 100% accurate): + + - **Migration 1**: The PostgreSQL (PG) repository creates a table named `site_imports`. + - **Migration 2**: The ClickHouse (CH) repository creates `import_id` columns in `imported_*` tables. + - **Migration 3**: The PG repository runs a data migration that utilizes both PG and CH databases, + reading from the `import_id` column in `imported_*` tables. + + The default `migrate/0` would execute these migrations by repository, resulting in the following order: + + 1. Migration 1 (PG) + 2. Migration 3 (PG) + 3. Migration 2 (CH) + + This sequence would fail at Migration 3, as the `import_id` columns in the CH repository have not been created yet. + + `interweave_migrate/0` addresses this issue by consolidating all pending migrations into a single, ordered queue: + + 1. Migration 1 (PG) + 2. Migration 2 (CH) + 3. Migration 3 (PG) + + This ensures all dependencies are resolved in the correct order. + """ + def interweave_migrate(repos \\ repos()) do + prepare() + + pending = all_pending_migrations(repos) + streaks = migration_streaks(pending) + + Enum.each(streaks, fn {repo, up_to_version} -> + {:ok, _, _} = + Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :up, to: up_to_version), + log: :notice + ) + end) + end + + defp migration_streaks(pending_migrations) do + sorted_migrations = + pending_migrations + |> Enum.map(fn {repo, version, _name} -> {repo, version} end) + |> Enum.sort_by(fn {_repo, version} -> version end, :asc) + + streaks_reversed = + Enum.reduce(sorted_migrations, [], fn {repo, _version} = latest_migration, streaks_acc -> + case streaks_acc do + # start the streak for repo + [] -> [latest_migration] + # extend the streak + [{^repo, _prev_version} | rest] -> [latest_migration | rest] + # end the streak for prev_repo, start the streak for repo + [{_prev_repo, _prev_version} | _rest] -> [latest_migration | streaks_acc] + end + end) + + :lists.reverse(streaks_reversed) + end + + @spec all_pending_migrations([Ecto.Repo.t()]) :: [{Ecto.Repo.t(), integer, String.t()}] + defp all_pending_migrations(repos) do + Enum.flat_map(repos, fn repo -> + # credo:disable-for-lines:6 Credo.Check.Refactor.Nesting + {:ok, pending, _started} = + Ecto.Migrator.with_repo(repo, fn repo -> + Ecto.Migrator.migrations(repo) + |> Enum.filter(fn {status, _version, _name} -> status == :down end) + |> Enum.map(fn {_status, version, name} -> {repo, version, name} end) + end) + + pending + end) + end + + def pending_streaks(repos \\ repos()) do + prepare() + IO.puts("Collecting pending migrations..") + + pending = all_pending_migrations(repos) + + if pending == [] do + IO.puts("No pending migrations!") + else + streaks = migration_streaks(pending) + print_migration_streaks(streaks, pending) + end + end + + defp print_migration_streaks([{repo, up_to_version} | streaks], pending) do + {streak, pending} = + Enum.split_with(pending, fn {pending_repo, version, _name} -> + pending_repo == repo and version <= up_to_version + end) + + IO.puts( + "\n#{inspect(repo)} [#{Path.relative_to_cwd(Ecto.Migrator.migrations_path(repo))}] streak up to version #{up_to_version}:" + ) + + Enum.each(streak, fn {_repo, version, name} -> IO.puts(" * #{version}_#{name}") end) + print_migration_streaks(streaks, pending) + end + + defp print_migration_streaks([], []), do: :ok + + def seed do + prepare() + # Run seed script + Enum.each(repos(), &run_seeds_for/1) + # Signal shutdown + IO.puts("Success!") + end + + def createdb(repos \\ repos()) do + prepare() + + for repo <- repos do + :ok = ensure_repo_created(repo) + end + + IO.puts("Creation of Db successful!") + end + + def rollback do + prepare() + + get_step = + IO.gets("Enter the number of steps: ") + |> String.trim() + |> Integer.parse() + + case get_step do + {int, _trailing} -> + Enum.each(repos(), fn repo -> run_rollbacks_for(repo, int) end) + IO.puts("Rollback successful!") + + :error -> + IO.puts("Invalid integer") + end + end + + def configure_ref_inspector() do + priv_dir = Application.app_dir(:plausible, "priv/ref_inspector") + Application.put_env(:ref_inspector, :database_path, priv_dir) + end + + def configure_ua_inspector() do + priv_dir = Application.app_dir(:plausible, "priv/ua_inspector") + Application.put_env(:ua_inspector, :database_path, priv_dir) + end + + def dump_plans() do + prepare() + + Repo.delete_all("plans") + + plans = + Plausible.Billing.Plans.all() + |> Plausible.Billing.Plans.with_prices() + |> Enum.map(fn plan -> + plan = Map.from_struct(plan) + + monthly_cost = plan.monthly_cost && Money.to_decimal(plan.monthly_cost) + yearly_cost = plan.yearly_cost && Money.to_decimal(plan.yearly_cost) + + features = + Enum.map(plan.features, fn f -> + {:ok, feat} = Plausible.Billing.Ecto.Feature.dump(f) + feat + end) + + {:ok, team_member_limit} = Plausible.Billing.Ecto.Limit.dump(plan.team_member_limit) + + plan + |> Map.drop([:id]) + |> Map.put(:kind, Atom.to_string(plan.kind)) + |> Map.put(:monthly_cost, monthly_cost) + |> Map.put(:yearly_cost, yearly_cost) + |> Map.put(:features, features) + |> Map.put(:team_member_limit, team_member_limit) + end) + + {count, _} = Repo.insert_all("plans", plans) + IO.puts("Inserted #{count} plans") + end + + ############################## + + defp repos do + Application.fetch_env!(@app, :ecto_repos) + end + + defp run_seeds_for(repo) do + # Run the seed script if it exists + seed_script = seeds_path(repo) + + if File.exists?(seed_script) do + IO.puts("Running seed script..") + Code.eval_file(seed_script) + end + end + + defp ensure_repo_created(repo) do + config = repo.config() + adapter = repo.__adapter__() + + case adapter.storage_status(config) do + :up -> + IO.puts("#{inspect(repo)} database already exists") + :ok + + :down -> + IO.puts("Creating #{inspect(repo)} database..") + + case adapter.storage_up(config) do + :ok -> :ok + {:error, :already_up} -> :ok + {:error, _reason} = error -> error + end + + {:error, _reason} = error -> + error + end + end + + defp run_rollbacks_for(repo, step) do + app = Keyword.get(repo.config(), :otp_app) + IO.puts("Running rollbacks for #{app} (STEP=#{step})") + + {:ok, _, _} = + Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, all: false, step: step)) + end + + defp prepare do + IO.puts("Loading #{@app}..") + # Load the code for myapp, but don't start it + :ok = Application.ensure_loaded(@app) + + IO.puts("Starting dependencies..") + # Start apps necessary for executing migrations + Enum.each(@start_apps, &Application.ensure_all_started/1) + + # Start the Repo(s) for myapp + IO.puts("Starting repos..") + Enum.each(repos(), & &1.start_link(pool_size: 2)) + end + + defp seeds_path(repo), do: priv_path_for(repo, "seeds.exs") + + defp priv_path_for(repo, filename) do + app = Keyword.get(repo.config(), :otp_app) + IO.puts("App: #{app}") + repo_underscore = repo |> Module.split() |> List.last() |> Macro.underscore() + Path.join([priv_dir(app), repo_underscore, filename]) + end + + defp priv_dir(app), do: "#{:code.priv_dir(app)}" +end diff --git a/lib/plausible_web/captcha.ex b/lib/plausible_web/captcha.ex new file mode 100644 index 0000000000000000000000000000000000000000..aff593d88ebad2677204c08a4d598cb977e72630 --- /dev/null +++ b/lib/plausible_web/captcha.ex @@ -0,0 +1,41 @@ +defmodule PlausibleWeb.Captcha do + alias Plausible.HTTPClient + + @verify_endpoint "https://hcaptcha.com/siteverify" + + def enabled? do + is_binary(sitekey()) + end + + def sitekey() do + Application.get_env(:plausible, :hcaptcha, [])[:sitekey] + end + + def verify(token) do + if enabled?() do + res = + HTTPClient.impl().post( + @verify_endpoint, + [{"content-type", "application/x-www-form-urlencoded"}], + %{ + response: token, + secret: secret() + } + ) + + case res do + {:ok, %Finch.Response{status: 200, body: %{"success" => success}}} -> + success + + _ -> + false + end + else + true + end + end + + defp secret() do + Application.get_env(:plausible, :hcaptcha, [])[:secret] + end +end diff --git a/lib/plausible_web/components/billing/billing.ex b/lib/plausible_web/components/billing/billing.ex new file mode 100644 index 0000000000000000000000000000000000000000..d2c670ca599f91a8f8b5bafa349c9ce9e25cd46a --- /dev/null +++ b/lib/plausible_web/components/billing/billing.ex @@ -0,0 +1,451 @@ +defmodule PlausibleWeb.Components.Billing do + @moduledoc false + + use PlausibleWeb, :component + use Plausible + + require Plausible.Billing + + import PlausibleWeb.Components.Icons + + alias Plausible.Billing.{Plan, Plans, EnterprisePlan} + + attr :site, Plausible.Site, required: false, default: nil + attr :current_user, Plausible.Auth.User, required: true + attr :current_team, :any, required: true + attr :locked?, :boolean, required: true + attr :link_class, :string, default: "" + slot :inner_block, required: true + + def feature_gate(assigns) do + ~H""" +
+ {render_slot(@inner_block)} +
+
+
+
+ +
+
+

+ Upgrade to unlock +

+ + To access this feature, + <.upgrade_call_to_action + current_user={@current_user} + current_team={@current_team} + link_class={@link_class} + /> + +
+
+
+ """ + end + + attr :usage, :map, required: true + attr :limit, :any, required: true + attr :total_pageview_usage_domain, :string, default: nil + + def render_monthly_pageview_usage(%{usage: usage} = assigns) + when is_map_key(usage, :last_30_days) do + ~H""" + <.monthly_pageview_usage_breakdown + usage={@usage.last_30_days} + limit={@limit} + period={:last_30_days} + expanded={Enum.count(@usage.last_30_days.per_site) <= 1} + total_pageview_usage_domain={@total_pageview_usage_domain} + /> + """ + end + + def render_monthly_pageview_usage(assigns) do + exceeded = + Plausible.Billing.Quota.exceeded_cycles(assigns.usage, assigns.limit, with_margin: false) + + show_all = :last_cycle in exceeded or :current_cycle in exceeded + + assigns = assign(assigns, :show_all, show_all) + + ~H""" +
+ <.monthly_pageview_usage_breakdown + usage={@usage.current_cycle} + limit={@limit} + period={:current_cycle} + expanded={not @show_all and Enum.count(@usage.current_cycle.per_site) <= 1} + total_pageview_usage_domain={@total_pageview_usage_domain} + /> + <%= if @show_all do %> + <.monthly_pageview_usage_breakdown + usage={@usage.last_cycle} + limit={@limit} + period={:last_cycle} + expanded={false} + total_pageview_usage_domain={@total_pageview_usage_domain} + /> + <.monthly_pageview_usage_breakdown + usage={@usage.penultimate_cycle} + limit={@limit} + period={:penultimate_cycle} + expanded={false} + total_pageview_usage_domain={@total_pageview_usage_domain} + /> + <% end %> +
+ """ + end + + attr(:usage, :map, required: true) + attr(:limit, :any, required: true) + attr(:period, :atom, required: true) + attr(:expanded, :boolean, required: true) + attr(:total_pageview_usage_domain, :string, default: nil) + + defp monthly_pageview_usage_breakdown(assigns) do + assigns = + assign( + assigns, + :total_link, + dashboard_url( + assigns.total_pageview_usage_domain, + assigns.usage.date_range + ) + ) + + ~H""" +
+
+

+ {PlausibleWeb.TextHelpers.format_date_range(@usage.date_range)} + {cycle_label(@period)} +

+ <.usage_progress_bar + :if={@limit != :unlimited} + id={"total_pageviews_#{@period}"} + usage={@usage.total} + limit={@limit} + /> +
+ +
+ <.pageview_usage_row + id={"pageviews_#{@period}"} + label={if Enum.empty?(@usage.per_site), do: "Pageviews", else: "Total pageviews"} + value={@usage.pageviews} + /> + <.pageview_usage_row + id={"custom_events_#{@period}"} + label={if Enum.empty?(@usage.per_site), do: "Custom events", else: "Total custom events"} + value={@usage.custom_events} + /> +
+
+
0} class="border-gray-200 dark:border-gray-700" /> +
+ + {site.domain} + <.tooltip centered?={true}> + <:tooltip_content>View billing period in dashboard + <.link + href={dashboard_url(site.domain, @usage.date_range)} + class="shrink-0 text-indigo-500 hover:text-indigo-600" + > + <.external_link_icon class="ml-0.5 size-3.5 [&_path]:stroke-2" /> + + + + {PlausibleWeb.TextHelpers.number_format(site.total)} +
+ <.pageview_usage_row label="Pageviews" value={site.pageviews} /> + <.pageview_usage_row label="Custom events" value={site.custom_events} /> +
+
+
+
+ """ + end + + attr :id, :string, default: nil + attr :label, :string, required: true + attr :value, :integer, required: true + + defp pageview_usage_row(assigns) do + ~H""" +
+ + • + {@label} + + {PlausibleWeb.TextHelpers.number_format(@value)} +
+ """ + end + + defp dashboard_url(nil, _date_range), do: nil + + defp dashboard_url(domain, date_range) do + base = Routes.stats_path(PlausibleWeb.Endpoint, :stats, domain, []) + + base <> + "?period=custom&from=#{Date.to_iso8601(date_range.first)}&to=#{Date.to_iso8601(date_range.last)}" + end + + defp cycle_label(:current_cycle), do: "(current cycle)" + defp cycle_label(:last_30_days), do: "(last 30 days)" + + @doc """ + Renders a color-coded progress bar based on usage percentage. + + Color scheme: + - 0-90%: Green (healthy usage) + - 91-99%: Gradient from green through yellow to orange (approaching limit) + - 100%: Gradient from green through orange to red (at limit) + """ + attr(:usage, :integer, required: true) + attr(:limit, :any, required: true) + attr(:rest, :global) + + def usage_progress_bar(assigns) do + percentage = calculate_percentage(assigns.usage, assigns.limit) + + assigns = + assigns + |> assign(:percentage, percentage) + |> assign(:color_class, progress_bar_color_from_percentage(percentage, assigns.limit)) + + ~H""" +
+
+
+
+ """ + end + + defp calculate_percentage(_usage, :unlimited), do: 0 + defp calculate_percentage(_usage, 0), do: 0 + + defp calculate_percentage(usage, limit) when is_number(limit) do + percentage = usage / limit * 100 + min(percentage, 100.0) |> Float.round(1) + end + + defp progress_bar_color_from_percentage(_percentage, :unlimited), + do: "bg-green-500 dark:bg-green-600" + + defp progress_bar_color_from_percentage(_percentage, 0), do: "bg-gray-200 dark:bg-gray-700" + + defp progress_bar_color_from_percentage(percentage, _limit) when is_number(percentage) do + cond do + percentage >= 100.0 -> + "bg-gradient-to-r from-green-500 via-orange-500 via-[80%] to-red-500 dark:from-green-600 dark:via-orange-600 dark:to-red-600" + + percentage >= 91 -> + "bg-gradient-to-r from-green-500 via-yellow-500 via-[80%] to-orange-500 dark:from-green-600 dark:via-yellow-600 dark:to-orange-600" + + true -> + "bg-green-500 dark:bg-green-600" + end + end + + def present_enterprise_plan(assigns) do + ~H""" +
    +
  • + Up to {present_limit(@plan, :monthly_pageview_limit)} monthly pageviews +
    • +
  • + Up to {present_limit(@plan, :site_limit)} sites +
    • +
  • + Up to {present_limit(@plan, :hourly_api_request_limit)} hourly api requests +
    • +
+ """ + end + + defp present_limit(enterprise_plan, key) do + enterprise_plan + |> Map.fetch!(key) + |> PlausibleWeb.StatsView.large_number_format() + end + + attr :id, :string, required: true + attr :paddle_product_id, :string, required: true + attr :checkout_disabled, :boolean, default: false + attr :user, :map, required: true + attr :team, :map, default: nil + attr :confirm_message, :any, default: nil + slot :inner_block, required: true + + def paddle_button(assigns) do + js_action_expr = + start_paddle_checkout_expr(assigns.paddle_product_id, assigns.team, assigns.user) + + confirmed = + if assigns.confirm_message, do: "confirm(\"#{assigns.confirm_message}\")", else: "true" + + assigns = + assigns + |> assign(:confirmed, confirmed) + |> assign(:js_action_expr, js_action_expr) + + ~H""" + + """ + end + + if Mix.env() == :dev do + def start_paddle_checkout_expr(paddle_product_id, _team, _user) do + "window.location = '#{Routes.dev_subscription_path(PlausibleWeb.Endpoint, :create_form, paddle_product_id)}'" + end + + def paddle_script(assigns), do: ~H"" + else + def start_paddle_checkout_expr(paddle_product_id, team, user) do + passthrough = + if team do + "ee:#{ee?()};user:#{user.id};team:#{team.id}" + else + "ee:#{ee?()};user:#{user.id}" + end + + paddle_checkout_params = + Jason.encode!(%{ + product: paddle_product_id, + email: user.email, + disableLogout: true, + passthrough: passthrough, + success: Routes.billing_path(PlausibleWeb.Endpoint, :upgrade_success), + theme: "none" + }) + + "Paddle.Checkout.open(#{paddle_checkout_params})" + end + + def paddle_script(assigns) do + ~H""" + + + + """ + end + end + + attr :link_class, :string, default: "" + attr :current_team, :any, required: true + attr :current_user, :atom, required: true + + def upgrade_call_to_action(assigns) do + user = assigns.current_user + site = assigns[:site] + team = Plausible.Teams.with_subscription(assigns.current_team) + + current_role = + cond do + not is_nil(site) -> + case Plausible.Teams.Memberships.site_role(site, user) do + {:ok, {_, site_role}} -> site_role + _ -> nil + end + + not is_nil(team) -> + case Plausible.Teams.Memberships.team_role(team, user) do + {:ok, team_role} -> team_role + _ -> nil + end + + true -> + nil + end + + upgrade_assistance_required? = + case Plans.get_subscription_plan(team && team.subscription) do + %Plan{kind: :business} -> true + %EnterprisePlan{} -> true + _ -> false + end + + cond do + not is_nil(current_role) and current_role not in Plausible.Billing.allowed_roles() -> + ~H"ask your team owner to upgrade their subscription." + + upgrade_assistance_required? -> + ~H""" + contact + <.styled_link href="mailto:hello@plausible.io" class={"font-medium " <> @link_class}> + hello@plausible.io + + to upgrade your subscription. + """ + + true -> + ~H""" + <.styled_link + class={"inline-block font-medium " <> @link_class} + href={Routes.billing_path(PlausibleWeb.Endpoint, :choose_plan)} + > + upgrade your subscription. + + """ + end + end +end diff --git a/lib/plausible_web/components/billing/helpers.ex b/lib/plausible_web/components/billing/helpers.ex new file mode 100644 index 0000000000000000000000000000000000000000..f2b2028fe2aab4cae939cb13f293a5bec87a8a1f --- /dev/null +++ b/lib/plausible_web/components/billing/helpers.ex @@ -0,0 +1,63 @@ +defmodule PlausibleWeb.Components.Billing.Helpers do + @moduledoc false + + require Plausible.Billing.Subscription.Status + alias Plausible.Billing.{Plan, Plans, EnterprisePlan, Subscription, Subscriptions} + + def present_plan_name(%Plan{kind: kind}), + do: kind |> to_string() |> String.capitalize() + + def present_plan_name(%EnterprisePlan{}), do: "Enterprise" + def present_plan_name(:free_10k), do: "Free" + def present_plan_name(_), do: "Plan" + + def present_subscription_interval(subscription) do + case Plans.subscription_interval(subscription) do + "monthly" -> "month" + "yearly" -> "year" + interval -> interval + end + end + + @spec present_subscription_status(Subscription.Status.status()) :: String.t() + def present_subscription_status(Subscription.Status.active()), do: "Active" + def present_subscription_status(Subscription.Status.past_due()), do: "Past due" + def present_subscription_status(Subscription.Status.deleted()), do: "Cancelled" + def present_subscription_status(Subscription.Status.paused()), do: "Paused" + def present_subscription_status(status), do: status + + def subscription_pill_color(Subscription.Status.active()), do: :green + def subscription_pill_color(Subscription.Status.past_due()), do: :yellow + def subscription_pill_color(Subscription.Status.paused()), do: :red + def subscription_pill_color(Subscription.Status.deleted()), do: :red + def subscription_pill_color(_), do: :gray + + def trial_button_label(team) do + if Plausible.Teams.Billing.enterprise_configured?(team) do + "Upgrade" + else + "Choose a plan →" + end + end + + def change_plan_button_label(nil), do: "Upgrade" + + def change_plan_button_label(subscription) do + if Subscriptions.resumable?(subscription) && subscription.cancel_url do + "Change plan" + else + "Upgrade" + end + end + + def format_invoices(invoice_list) do + Enum.map(invoice_list, fn invoice -> + %{ + date: invoice["payout_date"] |> Date.from_iso8601!() |> Calendar.strftime("%b %-d, %Y"), + amount: (invoice["amount"] / 1) |> :erlang.float_to_binary(decimals: 2), + currency: invoice["currency"] |> PlausibleWeb.BillingView.present_currency(), + url: invoice["receipt_url"] + } + end) + end +end diff --git a/lib/plausible_web/components/billing/notice.ex b/lib/plausible_web/components/billing/notice.ex new file mode 100644 index 0000000000000000000000000000000000000000..9c6e19c36c9e2dfffabde21c5691125939a4de26 --- /dev/null +++ b/lib/plausible_web/components/billing/notice.ex @@ -0,0 +1,550 @@ +defmodule PlausibleWeb.Components.Billing.Notice do + @moduledoc false + + use PlausibleWeb, :component + + require Plausible.Billing.Subscription.Status + alias Plausible.Billing.{Subscription, Plans, Subscriptions} + + def active_grace_period(assigns) do + if assigns.enterprise? do + ~H""" + + """ + else + ~H""" + + """ + end + end + + def dashboard_locked(assigns) do + ~H""" + + """ + end + + attr(:current_team, :any, required: true) + attr(:current_user, :atom, required: true) + attr(:limit, :integer, required: true) + attr(:resource, :string, required: true) + attr(:rest, :global) + + def limit_exceeded(assigns) do + ~H""" + <.notice + {@rest} + title={"#{account_label(@current_team)} is limited to #{pretty_print_resource_limit(@limit, @resource)}"} + data-test="limit-exceeded-notice" + theme={:gray} + show_icon={false} + > + To add more {@resource}, + + + """ + end + + defp pretty_print_resource_limit(1 = _limit, resource_plural) do + "a single #{String.trim_trailing(resource_plural, "s")}" + end + + defp pretty_print_resource_limit(limit, resource_plural) do + "#{limit} #{resource_plural}" + end + + attr(:subscription, :map, required: true) + attr(:dismissable, :boolean, default: true) + + @doc """ + Given a user with a cancelled subscription, this component renders a cancelled + subscription notice. If the given user does not have a subscription or it has a + different status, this function returns an empty template. + + It also takes a dismissable argument which renders the notice dismissable (with + the help of JavaScript and localStorage). We show a dismissable notice about a + cancelled subscription across the app, but when the user dismisses it, we will + start displaying it in the account settings > subscription section instead. + + So it's either shown across the app, or only on the /settings page. Depending + on whether the localStorage flag to dismiss it has been set or not. + """ + def subscription_cancelled(assigns) + + def subscription_cancelled( + %{ + dismissable: true, + subscription: %Subscription{status: Subscription.Status.deleted()} + } = assigns + ) do + ~H""" + + """ + end + + def subscription_cancelled( + %{ + dismissable: false, + subscription: %Subscription{status: Subscription.Status.deleted()} + } = assigns + ) do + assigns = assign(assigns, :container_id, "local-subscription-cancelled-notice") + + ~H""" + + + """ + end + + def subscription_cancelled(assigns), do: ~H"" + + attr(:class, :string, default: "") + attr(:subscription, :any, default: nil) + + def subscription_past_due( + %{subscription: %Subscription{status: Subscription.Status.past_due()}} = assigns + ) do + ~H""" + + """ + end + + def subscription_past_due(assigns), do: ~H"" + + attr(:class, :string, default: "") + attr(:subscription, :any, default: nil) + + def subscription_paused( + %{subscription: %Subscription{status: Subscription.Status.paused()}} = assigns + ) do + ~H""" + + """ + end + + def subscription_paused(assigns), do: ~H"" + + def upgrade_ineligible(assigns) do + ~H""" + + """ + end + + def pending_site_ownerships_notice(%{pending_ownership_count: count} = assigns) do + if count > 0 do + message = + "Your account has been invited to become the owner of " <> + if(count == 1, do: "a site, which is", else: "#{count} sites, which are") <> + " being counted towards the usage of your account." + + assigns = assign(assigns, message: message) + + ~H""" + + """ + else + ~H"" + end + end + + def growth_grandfathered(assigns) do + ~H""" +
+ Your subscription has been grandfathered in at the same rate and terms as when you first joined. If you don't need the "Business" features, you're welcome to stay on this plan. You can adjust the pageview limit or change the billing frequency of this grandfathered plan. If you're interested in business features, you can upgrade to a "Business" plan. +
+ """ + end + + @doc """ + Renders a usage notification banner based on the notification type. + + This is used on the subscription settings page to show contextual + notifications about approaching or exceeded limits. + """ + attr(:type, :atom, required: true) + attr(:team, :any, required: true) + + def usage_notification(assigns) + + def usage_notification(%{type: :pageview_approaching_limit} = assigns) do + ~H""" + <.notice title="You're close to your monthly pageview limit" theme={:gray} show_icon={false}> +
+

+ No action is required. Occasional traffic spikes are normal, and we'll keep tracking your stats as usual. Upgrading now gives you room to grow if higher traffic continues. +

+
+ <.button_link href={Routes.billing_path(PlausibleWeb.Endpoint, :choose_plan)} mt?={false}> + Upgrade + + <.button_link + href="https://plausible.io/docs/subscription-plans" + theme="secondary" + mt?={false} + > + Learn more + +
+
+ + """ + end + + def usage_notification(%{type: :team_member_limit_reached} = assigns) do + ~H""" + <.notice title="You've reached your current team member limit" theme={:gray} show_icon={false}> +
+

+ Upgrading lets you add more as your team grows. +

+ <.button_link href={Routes.billing_path(PlausibleWeb.Endpoint, :choose_plan)} mt?={false}> + Upgrade + +
+ + """ + end + + def usage_notification(%{type: :site_limit_reached} = assigns) do + ~H""" + <.notice title="You've reached your current site limit" theme={:gray} show_icon={false}> +
+

+ Upgrading lets you add more sites as you grow. +

+ <.button_link href={Routes.billing_path(PlausibleWeb.Endpoint, :choose_plan)} mt?={false}> + Upgrade + +
+ + """ + end + + def usage_notification(%{type: :site_and_team_member_limit_reached} = assigns) do + ~H""" + <.notice + title="You've reached your current limits for team members and sites" + theme={:gray} + show_icon={false} + > +
+

+ Upgrading gives you room to grow. +

+ <.button_link href={Routes.billing_path(PlausibleWeb.Endpoint, :choose_plan)} mt?={false}> + Upgrade + +
+ + """ + end + + def usage_notification(%{type: :traffic_exceeded_current_cycle} = assigns) do + ~H""" + <.notice title="Traffic has exceeded your plan limit this cycle" theme={:gray} show_icon={false}> +
+

+ No action is required. Occasional traffic spikes are normal, but upgrading now gives you room to grow if higher traffic continues. +

+
+ <.button_link href={Routes.billing_path(PlausibleWeb.Endpoint, :choose_plan)} mt?={false}> + Upgrade + + <.button_link + href="https://plausible.io/docs/subscription-plans" + theme="secondary" + mt?={false} + > + Learn more + +
+
+ + """ + end + + def usage_notification(%{type: :traffic_exceeded_last_cycle} = assigns) do + ~H""" + <.notice title="Traffic exceeded your plan limit last cycle" theme={:gray} show_icon={false}> +
+

+ No action is required. Occasional traffic spikes are normal, but upgrading now gives you room to grow if higher traffic continues. +

+
+ <.button_link href={Routes.billing_path(PlausibleWeb.Endpoint, :choose_plan)} mt?={false}> + Upgrade + + <.button_link + href="https://plausible.io/docs/subscription-plans" + theme="secondary" + mt?={false} + > + Learn more + +
+
+ + """ + end + + def usage_notification(%{type: :traffic_exceeded_sustained} = assigns) do + ~H""" + <.notice title="Upgrade required due to sustained higher traffic" theme={:gray} show_icon={false}> +
+

+ To ensure uninterrupted access to your stats, please upgrade to a plan that fits your current usage. +

+
+ <.button_link href={Routes.billing_path(PlausibleWeb.Endpoint, :choose_plan)} mt?={false}> + Upgrade + + <.button_link + href="https://plausible.io/docs/subscription-plans" + theme="secondary" + mt?={false} + > + Learn more + +
+
+ + """ + end + + def usage_notification(%{type: :manual_lock_grace_period_active} = assigns) do + ~H""" + <.notice title="You've outgrown your custom plan" theme={:yellow} show_icon={false}> +

+ We'll contact you by email to discuss an updated custom plan based on your current usage. +

+ + """ + end + + def usage_notification(%{type: :grace_period_active, team: team} = assigns) do + deadline_text = + case Plausible.Teams.GracePeriod.expires_in(team) do + {0, :hours} -> "within the hour" + {1, :hours} -> "within the next hour" + {n, :hours} -> "within the next #{n} hours" + {n, :days} -> "within the next #{n} days" + end + + assigns = assign(assigns, :deadline_text, deadline_text) + + ~H""" + <.notice title="Upgrade required due to sustained higher traffic" theme={:gray} show_icon={false}> +
+

+ To ensure uninterrupted access to your stats, please upgrade to a plan that fits your current usage {@deadline_text}. +

+
+ <.button_link href={Routes.billing_path(PlausibleWeb.Endpoint, :choose_plan)} mt?={false}> + Upgrade + + <.button_link + href="https://plausible.io/docs/subscription-plans" + theme="secondary" + mt?={false} + > + Learn more + +
+
+ + """ + end + + def usage_notification(%{type: :dashboard_locked} = assigns) do + ~H""" + <.notice title="Dashboard access temporarily locked" theme={:gray} show_icon={false}> +
+

+ Your stats are still being tracked, but dashboard access is temporarily locked because your site exceeded your plan's pageview limit for two consecutive billing cycles. Upgrade to restore access. +

+
+ <.button_link href={Routes.billing_path(PlausibleWeb.Endpoint, :choose_plan)} mt?={false}> + Upgrade + + <.button_link + href="https://plausible.io/docs/subscription-plans" + theme="secondary" + mt?={false} + > + Learn more + +
+
+ + """ + end + + def usage_notification(%{type: :trial_ended} = assigns) do + ~H""" + <.notice title="Your free trial has ended" theme={:gray} show_icon={false}> +
+

+ Upgrade to a monthly or yearly plan to continue accessing your sites. +

+ <.button_link + id="upgrade-or-change-plan-link" + href={Routes.billing_path(PlausibleWeb.Endpoint, :choose_plan)} + mt?={false} + > + Choose a plan + +
+ + """ + end + + def usage_notification(assigns), do: ~H"" + + defp subscription_cancelled_notice_body(assigns) do + ~H""" +

+ You have access to your stats until <%= Calendar.strftime(@subscription.next_bill_date, "%b %-d, %Y") %>. + <.link + class="underline inline-block" + href={Routes.billing_path(PlausibleWeb.Endpoint, :choose_plan)} + > + Upgrade your subscription + + to make sure you don't lose access. +

+ <.lose_grandfathering_warning subscription={@subscription} /> + """ + end + + defp lose_grandfathering_warning(%{subscription: subscription} = assigns) do + plan = Plans.get_regular_plan(subscription, only_non_expired: true) + loses_grandfathering? = plan && plan.generation < 5 + + assigns = assign(assigns, :loses_grandfathering?, loses_grandfathering?) + + ~H""" +

+ Please also note that by letting your subscription expire, you lose access to our grandfathered terms. If you want to subscribe again after that, your account will be offered the <.link + href="https://plausible.io/#pricing" + target="_blank" + rel="noopener noreferrer" + class="underline" + >latest pricing. +

+ """ + end + + defp account_label(%Plausible.Teams.Team{setup_complete: true}), do: "This team" + defp account_label(_team), do: "This account" +end diff --git a/lib/plausible_web/components/billing/pageview_slider.ex b/lib/plausible_web/components/billing/pageview_slider.ex new file mode 100644 index 0000000000000000000000000000000000000000..e46524e113ce0004155990d210dea5f99116fe06 --- /dev/null +++ b/lib/plausible_web/components/billing/pageview_slider.ex @@ -0,0 +1,161 @@ +defmodule PlausibleWeb.Components.Billing.PageviewSlider do + @moduledoc false + + use PlausibleWeb, :component + + def render(assigns) do + ~H""" + <.slider_output volume={@selected_volume} available_volumes={@available_volumes} /> + <.slider_input selected_volume={@selected_volume} available_volumes={@available_volumes} /> + <.slider_styles /> + """ + end + + attr :volume, :any + attr :available_volumes, :list + + defp slider_output(assigns) do + ~H""" + + Up to + + {format_volume(@volume, @available_volumes)} + + monthly pageviews + + """ + end + + defp slider_input(assigns) do + slider_labels = + Enum.map( + assigns.available_volumes ++ [:enterprise], + &format_volume(&1, assigns.available_volumes) + ) + + assigns = assign(assigns, :slider_labels, slider_labels) + + ~H""" +
+
+ + {List.first(@slider_labels)} + +
+ + +
+ + {List.last(@slider_labels)} + +
+
+ + + """ + end + + defp format_volume(volume, available_volumes) do + if volume == :enterprise do + available_volumes + |> List.last() + |> PlausibleWeb.StatsView.large_number_format() + |> Kernel.<>("+") + else + PlausibleWeb.StatsView.large_number_format(volume) + end + end + + defp slider_styles(assigns) do + ~H""" + + """ + end +end diff --git a/lib/plausible_web/components/billing/plan_benefits.ex b/lib/plausible_web/components/billing/plan_benefits.ex new file mode 100644 index 0000000000000000000000000000000000000000..a1a18b389cf31471ee4a321ed288bf04afe902ca --- /dev/null +++ b/lib/plausible_web/components/billing/plan_benefits.ex @@ -0,0 +1,54 @@ +defmodule PlausibleWeb.Components.Billing.PlanBenefits do + @moduledoc """ + This module exposes functions for rendering and returning plan + benefits for Growth, Business, and Enterprise plans. + """ + + use Phoenix.Component + + attr :benefits, :list, required: true + attr :class, :string, default: nil + + @doc """ + This function takes a list of benefits returned by either one of: + + * `Plausible.Billing.PlanBenefits.for_starter/1` + * `Plausible.Billing.PlanBenefits.for_growth/2` + * `Plausible.Billing.PlanBenefits.for_business/3` + * `Plausible.Billing.PlanBenefits.for_enterprise/1`. + + and renders them as HTML. + + The benefits in the given list can be either strings or functions + returning a Phoenix component. This allows, for example, to render + links within the plan benefit text. + """ + def render(assigns) do + ~H""" +
    +
  • + + <%= if benefit == "Sites API" do %> + <.sites_api_benefit /> + <% else %> + {benefit} + <% end %> +
    • +
+ """ + end + + defp sites_api_benefit(assigns) do + ~H""" +

+ Sites API access for + <.link + class="text-indigo-500 hover:underline" + href="https://plausible.io/white-label-web-analytics" + > + reselling + +

+ """ + end +end diff --git a/lib/plausible_web/components/billing/plan_box.ex b/lib/plausible_web/components/billing/plan_box.ex new file mode 100644 index 0000000000000000000000000000000000000000..a5e646b30625794bff1940142e4196a020f333ee --- /dev/null +++ b/lib/plausible_web/components/billing/plan_box.ex @@ -0,0 +1,434 @@ +defmodule PlausibleWeb.Components.Billing.PlanBox do + @moduledoc false + + use PlausibleWeb, :component + + require Plausible.Billing.Subscription.Status + alias PlausibleWeb.Components.Billing.{PlanBenefits} + alias Plausible.Billing.{Plan, Quota, Subscription} + + @plan_box_price_container_class "relative h-20 pt-4 max-h-20 whitespace-nowrap overflow-hidden" + + def standard(assigns) do + highlight = + cond do + assigns.owned && assigns.recommended -> "Current" + assigns.recommended -> "Recommended" + true -> nil + end + + assigns = + assigns + |> assign(:highlight, highlight) + |> assign(:price_container_class, @plan_box_price_container_class) + + ~H""" +
+
+

+ {String.capitalize(to_string(@kind))} +

+ <.highlight_pill :if={@highlight} text={@highlight} /> +
+
+
+ <.render_price_info available={@available} {assigns} /> +
+ <%= if @available do %> + <.checkout id={"#{@kind}-checkout"} {assigns} /> + <% else %> + <.contact_button class="bg-indigo-600 hover:bg-indigo-500 text-white" /> + <% end %> +
+ +
+ """ + end + + def enterprise(assigns) do + assigns = assign(assigns, :price_container_class, @plan_box_price_container_class) + + ~H""" +
+
+

+ Enterprise +

+ + Recommended + +
+
+ + Custom + +
+ <.contact_button class="" /> + +
+ """ + end + + defp highlight_pill(assigns) do + ~H""" +
+

+ {@text} +

+
+ """ + end + + defp render_price_info(%{available: false} = assigns) do + ~H""" +

+ + Custom + +

+

+ """ + end + + defp render_price_info(assigns) do + ~H""" + <.price_tag kind={@kind} selected_interval={@selected_interval} plan_to_render={@plan_to_render} /> +
+ + VAT + +
+ """ + end + + defp price_tag(%{selected_interval: :monthly} = assigns) do + monthly_cost = + case assigns.plan_to_render do + %{monthly_cost: nil} -> "N/A" + %{monthly_cost: monthly_cost} -> Plausible.Billing.format_price(monthly_cost) + end + + assigns = assign(assigns, :monthly_cost, monthly_cost) + + ~H""" +

+ + {@monthly_cost} + + + /month + +

+ """ + end + + defp price_tag(%{selected_interval: :yearly} = assigns) do + monthly_cost = + case assigns.plan_to_render do + %{monthly_cost: nil} -> "N/A" + %{monthly_cost: monthly_cost} -> Plausible.Billing.format_price(monthly_cost) + end + + {yearly_cost, monthly_cost_with_discount} = + case assigns.plan_to_render do + %{yearly_cost: nil} -> + {"N/A", "N/A"} + + %{yearly_cost: yearly_cost} -> + { + Plausible.Billing.format_price(yearly_cost), + Plausible.Billing.format_price(Money.div!(yearly_cost, 12)) + } + end + + assigns = + assigns + |> assign(:monthly_cost, monthly_cost) + |> assign(:yearly_cost, yearly_cost) + |> assign(:monthly_cost_with_discount, monthly_cost_with_discount) + + ~H""" +
+ + {@yearly_cost} + + + + /year + + +
+ + {@monthly_cost} + + + {@monthly_cost_with_discount} + +
+ + + /month + +
+ """ + end + + defp checkout(assigns) do + paddle_product_id = get_paddle_product_id(assigns.plan_to_render, assigns.selected_interval) + change_plan_link_text = change_plan_link_text(assigns) + + subscription = + Plausible.Teams.Billing.get_subscription(assigns.current_team) + + billing_details_expired = + Subscription.Status.in?(subscription, [ + Subscription.Status.paused(), + Subscription.Status.past_due() + ]) + + subscription_deleted = Subscription.Status.deleted?(subscription) + usage_check = check_usage_within_plan_limits(assigns) + + {checkout_disabled, disabled_message} = + cond do + not Quota.eligible_for_upgrade?(assigns.usage) -> + {true, nil} + + change_plan_link_text == "Currently on this plan" && not subscription_deleted -> + {true, nil} + + usage_check != :ok -> + {true, "Your usage exceeds this plan"} + + billing_details_expired -> + {true, "Please update your billing details first"} + + true -> + {false, nil} + end + + exceeded_plan_limits = + case usage_check do + {:error, {:over_plan_limits, limits}} -> + limits + + _ -> + [] + end + + feature_usage_check = Quota.ensure_feature_access(assigns.usage, assigns.plan_to_render) + + assigns = + assigns + |> assign(:paddle_product_id, paddle_product_id) + |> assign(:change_plan_link_text, change_plan_link_text) + |> assign(:checkout_disabled, checkout_disabled) + |> assign(:disabled_message, disabled_message) + |> assign(:exceeded_plan_limits, exceeded_plan_limits) + |> assign(:confirm_message, losing_features_message(feature_usage_check)) + + ~H""" + <%= if @owned_plan && Plausible.Billing.Subscriptions.resumable?(@current_team.subscription) do %> + <.change_plan_link {assigns} /> + <% else %> + + Upgrade + + <% end %> +
+ <.tooltip testid="plan-tooltip"> + + <:tooltip_content> + Your usage exceeds the following limit(s):

+

+ {Phoenix.Naming.humanize(limit)}
+

+ + +
+
+ {@disabled_message} +
+ """ + end + + defp check_usage_within_plan_limits(%{available: false}) do + {:error, :plan_unavailable} + end + + defp check_usage_within_plan_limits(%{ + available: true, + usage: usage, + current_team: current_team, + plan_to_render: plan + }) do + # At this point, the user is *not guaranteed* to have a team, + # with ongoing trial. + trial_active_or_ended_recently? = + not is_nil(current_team) and not is_nil(current_team.trial_expiry_date) and + Plausible.Teams.trial_days_left(current_team) >= -10 + + limit_checking_opts = + cond do + current_team && current_team.allow_next_upgrade_override -> + [skip_pageview_limit_check?: true] + + trial_active_or_ended_recently? && plan.volume == "10k" -> + [pageview_allowance_margin: 0.3] + + trial_active_or_ended_recently? -> + [pageview_allowance_margin: 0.15] + + true -> + [] + end + + Quota.ensure_within_plan_limits(usage, plan, limit_checking_opts) + end + + defp get_paddle_product_id(%Plan{monthly_product_id: plan_id}, :monthly), do: plan_id + defp get_paddle_product_id(%Plan{yearly_product_id: plan_id}, :yearly), do: plan_id + + defp change_plan_link_text( + %{ + owned_plan: %Plan{kind: from_kind, monthly_pageview_limit: from_volume}, + plan_to_render: %Plan{kind: to_kind, monthly_pageview_limit: to_volume}, + current_interval: from_interval, + selected_interval: to_interval + } = _assigns + ) do + cond do + from_kind in [:growth, :business] && to_kind == :starter -> + "Downgrade to Starter" + + from_kind == :business && to_kind == :growth -> + "Downgrade to Growth" + + from_kind == :starter && to_kind == :growth -> + "Upgrade to Growth" + + from_kind in [:starter, :growth] && to_kind == :business -> + "Upgrade to Business" + + from_volume == to_volume && from_interval == to_interval -> + "Currently on this plan" + + from_volume == to_volume -> + "Change billing interval" + + from_volume > to_volume -> + "Downgrade" + + true -> + "Upgrade" + end + end + + defp change_plan_link_text(_), do: nil + + defp change_plan_link(assigns) do + confirmed = + if assigns.confirm_message, do: "confirm(\"#{assigns.confirm_message}\")", else: "true" + + assigns = assign(assigns, :confirmed, confirmed) + + ~H""" + + """ + end + + defp losing_features_message(:ok), do: nil + + defp losing_features_message({:error, {:unavailable_features, features}}) do + features_list_str = + features + |> Enum.map(fn feature_mod -> feature_mod.display_name() end) + |> PlausibleWeb.TextHelpers.pretty_join() + + "This plan does not support #{features_list_str}, which you have been using. By subscribing to this plan, you will not have access to #{if length(features) == 1, do: "this feature", else: "these features"}." + end + + defp contact_button(assigns) do + ~H""" + <.link + href="https://plausible.io/contact" + class={[ + "mt-6 block rounded-md py-2 px-3 text-center text-sm font-semibold leading-6 bg-gray-800 hover:bg-gray-700 text-white dark:bg-indigo-600 dark:hover:bg-indigo-500", + @class + ]} + > + Contact us + + """ + end +end diff --git a/lib/plausible_web/components/first_dashboard_launch_banner.ex b/lib/plausible_web/components/first_dashboard_launch_banner.ex new file mode 100644 index 0000000000000000000000000000000000000000..7646368a6fddb248420d4caf39b2a8da6948b3c9 --- /dev/null +++ b/lib/plausible_web/components/first_dashboard_launch_banner.ex @@ -0,0 +1,52 @@ +defmodule PlausibleWeb.Components.FirstDashboardLaunchBanner do + @moduledoc """ + A banner that appears on the first dashboard launch + """ + + use PlausibleWeb, :component + + attr(:site, Plausible.Site, required: true) + + def set(assigns) do + ~H""" + + """ + end + + attr(:site, Plausible.Site, required: true) + + def render(assigns) do + ~H""" + + """ + end + + defp x_data(site) do + "{show: !!sessionStorage.getItem('#{storage_key(site)}')}" + end + + defp x_init(site) do + "setTimeout(() => sessionStorage.removeItem('#{storage_key(site)}'), 3000)" + end + + defp storage_key(site) do + "dashboard_seen_#{site.domain}" + end +end diff --git a/lib/plausible_web/components/flow_progress.ex b/lib/plausible_web/components/flow_progress.ex new file mode 100644 index 0000000000000000000000000000000000000000..53288065feddc013970108144037b9d2a0790562 --- /dev/null +++ b/lib/plausible_web/components/flow_progress.ex @@ -0,0 +1,65 @@ +defmodule PlausibleWeb.Components.FlowProgress do + @moduledoc """ + Component for provisioning/registration flows displaying + progress status. See `PlausibleWeb.Flows` for the list of + flow definitions. + """ + use Phoenix.Component + + attr :flow, :string, required: true, values: PlausibleWeb.Flows.valid_keys() + attr :current_step, :string, required: true, values: PlausibleWeb.Flows.valid_values() + + def render(assigns) do + steps = PlausibleWeb.Flows.steps(assigns.flow) + current_step_idx = Enum.find_index(steps, &(&1 == assigns.current_step)) + + assigns = + assign(assigns, + steps: steps, + current_step_idx: current_step_idx + ) + + ~H""" + + """ + end +end diff --git a/lib/plausible_web/components/generic.ex b/lib/plausible_web/components/generic.ex new file mode 100644 index 0000000000000000000000000000000000000000..ffdf5277c06a6b96fb0d5c609af652a85554c5cb --- /dev/null +++ b/lib/plausible_web/components/generic.ex @@ -0,0 +1,1369 @@ +defmodule PlausibleWeb.Components.Generic do + @moduledoc """ + Generic reusable components + """ + use Phoenix.Component, global_prefixes: ~w(x-) + + import PlausibleWeb.Components.Icons + + alias PlausibleWeb.Router.Helpers, as: Routes + + @notice_themes %{ + gray: %{ + bg: "bg-gray-100 dark:bg-gray-800", + icon: "text-gray-600 dark:text-gray-300", + title_text: "text-sm text-gray-900 dark:text-gray-100", + body_text: "text-sm text-gray-800 dark:text-gray-200 leading-5" + }, + yellow: %{ + bg: "bg-yellow-100/60 dark:bg-yellow-900/40", + icon: "text-yellow-500", + title_text: "text-sm text-gray-900 dark:text-gray-100", + body_text: "text-sm text-gray-600 dark:text-gray-100/60 leading-5" + }, + red: %{ + bg: "bg-red-100 dark:bg-red-900/30", + icon: "text-red-600 dark:text-red-500", + title_text: "text-sm text-gray-900 dark:text-gray-100", + body_text: "text-sm text-gray-600 dark:text-gray-100/60 leading-5" + }, + white: %{ + bg: "bg-white dark:bg-gray-900 shadow-sm dark:shadow-none", + icon: "text-gray-600 dark:text-gray-400", + title_text: "text-sm text-gray-900 dark:text-gray-100", + body_text: "text-sm text-gray-600 dark:text-gray-300 leading-5" + } + } + + # Themes/sizes/base are defined as component classes in + # assets/css/app.css and mirrored by the React `Button` component in + # assets/js/dashboard/components/button.tsx. Update there only. + @button_themes %{ + "primary" => "btn-theme-primary", + "secondary" => "btn-theme-secondary", + "yellow" => "btn-theme-yellow", + "danger" => "btn-theme-danger", + "ghost" => "btn-theme-ghost", + "icon" => "btn-theme-icon" + } + + @button_base_class "btn-base" + + @button_sizes %{ + "sm" => "btn-sm", + "md" => "btn-md" + } + + attr(:class, :string, default: "") + slot(:inner_block, required: true) + + def auth_container(assigns) do + ~H""" +
+ {render_slot(@inner_block)} +
+ """ + end + + attr(:type, :string, default: "button") + attr(:theme, :string, default: "primary") + attr(:size, :string, default: "md") + attr(:class, :string, default: "") + attr(:disabled, :boolean, default: false) + attr(:mt?, :boolean, default: true) + attr(:rest, :global, include: ~w(name)) + + slot(:inner_block) + + def button(assigns) do + assigns = + assign(assigns, + button_base_class: @button_base_class, + theme_class: @button_themes[assigns.theme], + size_class: @button_sizes[assigns.size] + ) + + ~H""" + + """ + end + + attr(:href, :string, required: true) + attr(:class, :string, default: "") + attr(:theme, :string, default: "primary") + attr(:size, :string, default: "md") + attr(:disabled, :boolean, default: false) + attr(:method, :string, default: "get") + attr(:mt?, :boolean, default: true) + attr(:rest, :global) + + slot(:inner_block) + + def button_link(assigns) do + extra = + if assigns.method == "get" do + [] + else + [ + "data-csrf": Phoenix.Controller.get_csrf_token(), + "data-method": assigns.method, + "data-to": assigns.href + ] + end + + assigns = assign(assigns, extra: extra) + + theme_class = + if assigns.disabled do + "bg-gray-400 text-white transition-all duration-150 dark:text-white dark:text-gray-400 dark:bg-gray-700 cursor-not-allowed" + else + @button_themes[assigns.theme] + end + + onclick = + if assigns.disabled do + "return false;" + else + assigns[:onclick] + end + + assigns = + assign(assigns, + onclick: onclick, + button_base_class: @button_base_class, + theme_class: theme_class, + size_class: @button_sizes[assigns.size] + ) + + ~H""" + <.link + href={@href} + onclick={@onclick} + class={[ + @mt? && "mt-6", + @button_base_class, + @size_class, + @theme_class, + @class + ]} + {@extra} + {@rest} + > + {render_slot(@inner_block)} + + """ + end + + attr(:slug, :string, required: true) + attr(:class, :string, default: nil) + + def docs_info(assigns) do + ~H""" +
+ <.tooltip enabled?={true} centered?={true}> + <:tooltip_content> + Learn more + + + + + +
+ """ + end + + attr(:title, :any, default: "") + attr(:class, :string, default: "") + attr(:rest, :global) + slot(:icon, required: true) + slot(:inner_block) + + def upgrade(assigns) do + ~H""" +
+
+
+ {render_slot(@icon)} +
+
+

+ {@title} +

+

+ {render_slot(@inner_block)} +

+
+
+
+ """ + end + + attr(:title, :any, default: nil) + attr(:theme, :atom, default: :yellow) + attr(:dismissable_id, :any, default: nil) + attr(:show_icon, :boolean, default: true) + attr(:class, :string, default: "") + attr(:icon_class, :string, default: "") + attr(:rest, :global) + slot(:inner_block) + slot(:actions) + slot(:icon) + + def notice(assigns) do + assigns = assign(assigns, :theme, Map.fetch!(@notice_themes, assigns.theme)) + + ~H""" +
+
+ +
+
+
+ <%= if @icon != [] do %> + {render_slot(@icon)} + <% else %> + <.exclamation_triangle_icon class={"size-4.5 #{@theme.icon}"} /> + <% end %> +
+
+

+ {@title} +

+
+

+ {render_slot(@inner_block)} +

+
+
+
+
+ {render_slot(@actions)} +
+
+
+
+ + """ + end + + attr(:href, :string, default: "#") + attr(:new_tab, :boolean, default: false) + attr(:class, :string, default: "") + attr(:rest, :global, include: ~w(patch)) + attr(:method, :string, default: "get") + slot(:inner_block) + + def styled_link(assigns) do + ~H""" + <.unstyled_link + new_tab={@new_tab} + href={@href} + method={@method} + class={"text-indigo-600 hover:text-indigo-700 dark:text-indigo-500 dark:hover:text-indigo-400 transition-colors duration-150 " <> @class} + {@rest} + > + {render_slot(@inner_block)} + + """ + end + + attr :class, :string, default: "" + attr :id, :string, default: nil + + slot :button, required: true do + attr(:class, :string) + end + + slot :menu, required: true do + attr(:class, :string) + end + + def dropdown(assigns) do + assigns = assign(assigns, :menu_class, assigns.menu |> List.first() |> Map.get(:class, "")) + + ~H""" +
+ + +
+ """ + end + + attr(:href, :string) + attr(:class, :string, default: "") + attr(:id, :string, default: nil) + attr(:new_tab, :boolean, default: false) + attr(:disabled, :boolean, default: false) + attr(:rest, :global, include: ~w(method)) + slot(:inner_block, required: true) + + @base_class "block rounded-md text-sm/6 text-gray-900 ui-disabled:text-gray-500 dark:text-gray-100 dark:ui-disabled:text-gray-400 px-3 py-1.5" + @clickable_class "hover:bg-gray-100 dark:hover:bg-gray-700/80" + def dropdown_item(assigns) do + assigns = + if assigns[:disabled] do + assign(assigns, :state, "disabled") + else + assign(assigns, :state, "") + end + + if assigns[:href] && !assigns[:disabled] do + assigns = assign(assigns, :class, [assigns[:class], @base_class, @clickable_class]) + + ~H""" + <.unstyled_link + id={@id} + class={@class} + new_tab={@new_tab} + href={@href} + x-on:click="close()" + data-ui-state={@state} + {@rest} + > + {render_slot(@inner_block)} + + """ + else + assigns = assign(assigns, :class, [assigns[:class], @base_class]) + + ~H""" +
+ {render_slot(@inner_block)} +
+ """ + end + end + + def dropdown_divider(assigns) do + ~H""" + + """ + end + + attr(:href, :string, required: true) + attr(:new_tab, :boolean, default: false) + attr(:class, :string, default: "") + attr(:rest, :global) + attr(:method, :string, default: "get") + slot(:inner_block) + + def unstyled_link(assigns) do + extra = + if assigns.method == "get" do + [] + else + [ + "data-csrf": Phoenix.Controller.get_csrf_token(), + "data-method": assigns.method, + "data-to": assigns.href + ] + end + + assigns = assign(assigns, extra: extra) + + if assigns[:new_tab] do + assigns = assign(assigns, :icon_class, icon_class(assigns)) + + ~H""" + <.link + class={[ + "inline-flex items-center gap-x-1", + @class + ]} + href={@href} + target="_blank" + rel="noopener noreferrer" + {@extra} + {@rest} + > + {render_slot(@inner_block)} + <.external_link_icon class={[@icon_class]} /> + + """ + else + ~H""" + <.link class={@class} href={@href} {@extra} {@rest}>{render_slot(@inner_block)} + """ + end + end + + attr(:class, :any, default: "") + attr(:rest, :global) + + def spinner(assigns) do + ~H""" + + + + + + + """ + end + + attr :id, :string, required: true + attr :js_active_var, :string, default: nil + attr :checked, :boolean, default: nil + attr :id_suffix, :string, default: "" + attr :disabled, :boolean, default: false + + attr(:rest, :global) + + @doc """ + Renders toggle input. + + Can be used in two modes: + + 1. Alpine JS mode: Pass `:js_active_var` to control toggle state via Alpine JS. + Set this outside this component with `x-data="{ : }"`. + + 2. Server-side mode: Pass `:checked` boolean and `phx-click` event handler. + + ### Examples - Alpine JS mode + ``` +
+
+ ``` + + ### Examples - Server-side mode + ``` + <.toggle_switch id="my_toggle" checked={@my_toggle} phx-click="toggle-my-setting" phx-target={@myself} /> + ``` + """ + def toggle_switch(assigns) do + server_mode? = not is_nil(assigns.checked) + assigns = assign(assigns, :server_mode?, server_mode?) + + ~H""" + + """ + end + + attr :id, :string, required: true + attr :js_active_var, :string, required: true + attr :id_suffix, :string, default: "" + attr :disabled, :boolean, default: false + attr :label, :string, required: true + attr :help_text, :string, default: nil + attr :show_help_text_only_when_active?, :boolean, default: false + attr :mt?, :boolean, default: true + + attr(:rest, :global) + + @doc """ + Renders toggle input with a label. Clicking the label also toggles the toggle. + Needs `:js_active_var` that controls toggle state. + Set this outside this component with `x-data="{ : }"` + Can be configured to always show a description of the field / help text `:help_text`, + or only show the help text when the toggle is activated `:show_help_text_only_when_active?`. + """ + def toggle_field(assigns) do + ~H""" +
+
+ + {@label} + +

+ {@help_text} +

+
+ +
+ """ + end + + def settings_tiles(assigns) do + ~H""" +
+ {render_slot(@inner_block)} +
+ """ + end + + attr :docs, :string, default: nil + slot :inner_block, required: true + slot :title, required: true + slot :subtitle, required: false + attr :feature_mod, :atom, default: nil + attr :feature_toggle?, :boolean, default: false + attr :current_team, :any, default: nil + attr :current_user, :any, default: nil + attr :site, :any, default: nil + attr :conn, :any, default: nil + attr :show_content?, :boolean, default: true + + def tile(assigns) do + ~H""" +
+
+ <.title> + {render_slot(@title)} + + <.docs_info :if={@docs} slug={@docs} class="absolute top-4 right-4 z-1" /> + +
+ {render_slot(@subtitle)} +
+ + <.live_component + :if={@feature_toggle?} + module={PlausibleWeb.Components.Site.Feature.ToggleLive} + id={"feature-toggle-#{@site.id}-#{@feature_mod}"} + site={@site} + feature_mod={@feature_mod} + current_user={@current_user} + /> +
+
+
+ <%= if @feature_mod do %> + +
+ {render_slot(@inner_block)} +
+ + <% else %> +
+ {render_slot(@inner_block)} +
+ <% end %> +
+
+ """ + end + + attr(:sticky?, :boolean, default: true) + attr(:enabled?, :boolean, default: true) + attr(:centered?, :boolean, default: false) + attr(:testid, :string, default: nil) + slot(:inner_block, required: true) + slot(:tooltip_content, required: true) + + def tooltip(assigns) do + wrapper_data = + if assigns[:sticky?], do: "{sticky: false, hovered: false}", else: "{hovered: false}" + + show_inner = if assigns[:sticky?], do: "hovered || sticky", else: "hovered" + + base_classes = [ + "absolute", + "pb-2", + "top-0", + "-translate-y-full", + "z-[1000]", + "sm:max-w-64", + "w-max" + ] + + tooltip_position_classes = + if assigns.centered? do + base_classes ++ ["left-1/2", "-translate-x-1/2"] + else + base_classes + end + + assigns = + assign(assigns, + wrapper_data: wrapper_data, + show_inner: show_inner, + tooltip_position_classes: tooltip_position_classes + ) + + if assigns.enabled? do + ~H""" +
+
+
+ {render_slot(@tooltip_content)} +
+
+
+ {render_slot(@inner_block)} +
+
+ """ + else + ~H"{render_slot(@inner_block)}" + end + end + + slot :inner_block, required: true + + def accordion_menu(assigns) do + ~H""" +
+ {render_slot(@inner_block)} +
+ """ + end + + attr :id, :string, required: true + attr :title, :string, required: true + attr :open_by_default, :boolean, default: false + attr :title_class, :string, default: "" + slot :inner_block, required: true + + def accordion_item(assigns) do + ~H""" +
+
+ +
+
+ {render_slot(@inner_block)} +
+
+ """ + end + + attr(:rest, :global, include: ~w(fill stroke stroke-width class)) + attr(:name, :atom, required: true) + attr(:outline, :boolean, default: true) + attr(:solid, :boolean, default: false) + attr(:mini, :boolean, default: false) + + def dynamic_icon(assigns) do + case assigns.name do + :tag -> + PlausibleWeb.Components.Icons.tag_icon(%{class: assigns.rest[:class]}) + + :subscription -> + PlausibleWeb.Components.Icons.subscription_icon(%{class: assigns.rest[:class]}) + + :api_keys -> + PlausibleWeb.Components.Icons.key_icon(%{class: assigns.rest[:class]}) + + icon_name -> + apply(Heroicons, icon_name, [assigns]) + end + end + + attr(:width, :integer, default: 100) + attr(:height, :integer, default: 100) + attr(:id, :string, default: "shuttle") + + defp icon_class(link_assigns) do + classes = List.wrap(link_assigns[:class]) |> Enum.join(" ") + + cond do + String.contains?(classes, "text-sm") -> + ["size-3.5"] + + String.contains?(classes, "text-xs") -> + ["size-3"] + + true -> + ["size-4"] + end + end + + slot(:item, required: true) + + def focus_list(assigns) do + ~H""" +
    +
  1. + {render_slot(item)} +
    2. +
+ """ + end + + slot :title + slot :subtitle + slot :inner_block, required: true + slot :footer + attr :padding?, :boolean, default: true + attr :rest, :global + + def focus_box(assigns) do + ~H""" +
+
+ <.title :if={@title != []}> + {render_slot(@title)} + +
+ +
+ {render_slot(@subtitle)} +
+ +
+ {render_slot(@inner_block)} +
+ +
+ {render_slot(@inner_block)} +
+
+
+
+ {render_slot(@footer)} +
+
+
+ """ + end + + attr :rest, :global + attr :width, :string, default: "min-w-full" + attr :rows, :list, default: [] + attr :row_attrs, :any, default: nil + slot :thead, required: false + slot :tbody, required: true + slot :inner_block, required: false + + def table(assigns) do + ~H""" + tbody>tr:first-child>td]:pt-0 [&>tbody>tr:last-child>td]:pb-0", + @width + ]} + {@rest} + > + + + {render_slot(@thead)} + + + + + {render_slot(@tbody, item)} + + {render_slot(@inner_block)} + +
+ """ + end + + slot :inner_block, required: true + attr :truncate, :boolean, default: false + attr :max_width, :string, default: "" + attr :height, :string, default: "" + attr :class, :string, default: "" + attr :actions, :boolean, default: nil + attr :hide_on_mobile, :boolean, default: nil + attr :rest, :global + + def td(assigns) do + max_width = + cond do + assigns.max_width != "" -> assigns.max_width + assigns.truncate -> "max-w-sm" + true -> "" + end + + assigns = assign(assigns, max_width: max_width) + + ~H""" + +
+ {render_slot(@inner_block)} +
+
+ {render_slot(@inner_block)} +
+ + """ + end + + slot :inner_block, required: true + attr :invisible, :boolean, default: false + attr :hide_on_mobile, :boolean, default: nil + + def th(assigns) do + class = + if assigns[:invisible] do + "invisible" + else + "px-6 first:pl-0 last:pr-0 py-3 text-left text-sm font-semibold" + end + + assigns = assign(assigns, class: class) + + ~H""" + + {render_slot(@inner_block)} + + """ + end + + attr :set_to, :boolean, default: false + attr :disabled?, :boolean, default: false + slot :inner_block, required: true + + def toggle_submit(assigns) do + ~H""" +
+ + + + {render_slot(@inner_block)} + +
+ """ + end + + attr :href, :string, required: false + attr :icon_name, :atom, required: false + attr :theme, :string, default: "default" + attr :class, :string, default: "" + attr :icon_class, :string, default: "" + attr :size, :string, default: "4" + attr :rest, :global, include: ~w(method disabled) + + slot :inner_block, required: false + + def icon_button(assigns) do + icon_source = + case {assigns.inner_block, assigns[:icon_name]} do + {[], nil} -> + raise ArgumentError, + "Either `icon_name` attribute or icon inner block must be provided" + + {[_ | _], icon_name} when not is_nil(icon_name) -> + raise ArgumentError, "Only one of `icon_name` and icon inner block must be provided" + + {[_ | _], nil} -> + :inner_block + + {[], icon_name} when not is_nil(icon_name) -> + :icon_name + end + + text = + case assigns.theme do + "default" -> + %{ + light: "text-indigo-700", + light_hover: "text-indigo-600", + dark: "text-indigo-500", + dark_hover: "text-indigo-400" + } + + "danger" -> + %{ + light: "text-red-700", + light_hover: "text-red-500", + dark: "text-red-500", + dark_hover: "text-red-400" + } + + _ -> + raise ArgumentError, "Invalid `theme` provided" + end + + button_class = [ + "group/button", + "w-fit h-fit", + "p-2", + "hover:bg-gray-100", + "dark:hover:bg-gray-800", + "rounded-md", + "transition-colors", + "duration-150", + assigns.class + ] + + icon_class = [ + "size-#{assigns.size}", + text.light, + "group-hover/button:" <> text.light_hover, + "dark:" <> text.dark, + "dark:group-hover/button:" <> text.dark_hover, + "transition-colors", + "duration-150", + "group-disabled/button:opacity-50", + assigns.icon_class + ] + + assigns = + assigns + |> assign(:icon_source, icon_source) + |> assign(:button_class, button_class) + |> assign(:icon_class, icon_class) + + if assigns[:href] do + ~H""" + <.unstyled_link href={@href} class={@button_class} {@rest}> + + {render_slot(@inner_block, @icon_class)} + + <.dynamic_icon + :if={@icon_source == :icon_name} + name={@icon_name} + class={@icon_class} + /> + + """ + else + ~H""" + + """ + end + end + + attr :href, :string, default: nil + attr :class, :string, default: "" + attr :rest, :global, include: ~w(method disabled) + + def edit_button(assigns) do + ~H""" + <.icon_button :let={icon_class} href={@href} class={@class} {@rest}> + + + + + """ + end + + attr :href, :string, default: nil + attr :class, :string, default: "" + attr :icon, :atom, default: :trash + attr :rest, :global, include: ~w(method disabled) + + def delete_button(assigns) do + ~H""" + <.icon_button + icon_name={@icon} + theme="danger" + class={@class} + href={@href} + {@rest} + /> + """ + end + + attr :filter_text, :string, default: "" + attr :placeholder, :string, default: "" + attr :filtering_enabled?, :boolean, default: true + slot :inner_block, required: false + + def filter_bar(assigns) do + ~H""" +
+
+
+
+
+ +
+ + + +
+
+
+ {render_slot(@inner_block)} +
+ """ + end + + slot :inner_block, required: true + attr :class, :any, default: nil + attr :rest, :global + + def h2(assigns) do + ~H""" +

+ {render_slot(@inner_block)} +

+ """ + end + + slot :inner_block, required: true + attr :class, :any, default: nil + + def title(assigns) do + ~H""" + <.h2 class={["text-lg font-medium text-gray-900 dark:text-gray-100 leading-7", @class]}> + {render_slot(@inner_block)} + + """ + end + + alias Phoenix.LiveView.JS + + slot :inner_block, required: true + + def disclosure(assigns) do + ~H""" +
+ {render_slot(@inner_block)} +
+ """ + end + + slot :inner_block, required: true + attr :class, :any, default: nil + + def disclosure_button(assigns) do + ~H""" + + """ + end + + slot :inner_block, required: true + + def disclosure_panel(assigns) do + ~H""" + + """ + end + + slot :inner_block, required: true + + def highlighted(assigns) do + ~H""" + + {render_slot(@inner_block)} + + """ + end + + attr(:class, :string, default: "") + attr(:color, :atom, default: :gray, values: [:gray, :indigo, :yellow, :green, :red]) + attr(:rest, :global) + slot(:inner_block, required: true) + + def pill(assigns) do + assigns = assign(assigns, :color_classes, get_pill_color_classes(assigns.color)) + + ~H""" + + {render_slot(@inner_block)} + + """ + end + + attr(:plan, :string, required: true) + attr(:color, :atom, default: :indigo, values: [:gray, :indigo, :yellow, :green, :red]) + attr(:rest, :global) + + def upgrade_pill(assigns) do + ~H""" + <.link + href={Routes.billing_path(PlausibleWeb.Endpoint, :choose_plan)} + class="inline-block" + {@rest} + > + <.pill color={@color} class="gap-x-1"> + <.diamond_icon class="size-3.5 [&_path]:stroke-2" /> + {@plan} + + + """ + end + + defp get_pill_color_classes(:gray) do + "bg-gray-100 text-gray-800 dark:bg-gray-750 dark:text-gray-200" + end + + defp get_pill_color_classes(:indigo) do + "bg-indigo-100/60 text-indigo-600 dark:bg-indigo-900/50 dark:text-indigo-300" + end + + defp get_pill_color_classes(:yellow) do + "bg-yellow-100/80 text-yellow-800 dark:bg-yellow-900/40 dark:text-yellow-300" + end + + defp get_pill_color_classes(:green) do + "bg-green-100/70 text-green-800 dark:bg-green-900/40 dark:text-green-300" + end + + defp get_pill_color_classes(:red) do + "bg-red-100/60 text-red-700 dark:bg-red-800/40 dark:text-red-300" + end +end diff --git a/lib/plausible_web/components/google.ex b/lib/plausible_web/components/google.ex new file mode 100644 index 0000000000000000000000000000000000000000..bd8767f954c38f51dc67cec3996f4d1dcc6d4ed5 --- /dev/null +++ b/lib/plausible_web/components/google.ex @@ -0,0 +1,68 @@ +defmodule PlausibleWeb.Components.Google do + @moduledoc """ + Google-related components + """ + use PlausibleWeb, :component + + attr(:to, :string, required: true) + attr(:id, :string, required: true) + + def button(assigns) do + ~H""" + <.unstyled_link + id={@id} + href={@to} + class="inline-flex pr-4 items-center border border-gray-100 shadow-sm rounded-md focus:outline-hidden focus:ring-1 focus:ring-offset-1 focus:ring-gray-200 mt-8 hover:bg-gray-50 dark:hover:bg-gray-700" + > + <.logo /> + + Continue with Google + + + """ + end + + def logo(assigns \\ %{}) do + ~H""" + + + + + + + + + + + + + + + + + + """ + end +end diff --git a/lib/plausible_web/components/icons.ex b/lib/plausible_web/components/icons.ex new file mode 100644 index 0000000000000000000000000000000000000000..51bca1baaf4728d3f5acb7a7aa036a24ffbc3b71 --- /dev/null +++ b/lib/plausible_web/components/icons.ex @@ -0,0 +1,178 @@ +defmodule PlausibleWeb.Components.Icons do + @moduledoc """ + Reusable icon components + """ + use Phoenix.Component + + attr(:rest, :global) + attr(:filled, :boolean, default: false) + + def pin_icon(assigns) do + ~H""" + + + + """ + end + + attr :class, :any, default: [] + + def external_link_icon(assigns) do + ~H""" + + + + """ + end + + attr :class, :any, default: [] + + def tag_icon(assigns) do + ~H""" + + + + + """ + end + + attr :class, :any, default: [] + + def diamond_icon(assigns) do + ~H""" + + + + """ + end + + attr :class, :any, default: [] + + def subscription_icon(assigns) do + ~H""" + + + + + """ + end + + attr :class, :any, default: [] + + def key_icon(assigns) do + ~H""" + + + + + + + """ + end + + attr :class, :any, default: [] + + def exclamation_triangle_icon(assigns) do + ~H""" + + + + + + + + """ + end + + attr :class, :any, default: [] + + def envelope_icon(assigns) do + ~H""" + + + + + """ + end +end diff --git a/lib/plausible_web/components/layout.ex b/lib/plausible_web/components/layout.ex new file mode 100644 index 0000000000000000000000000000000000000000..9fc63ab11c5b3f61f6e885c0ef154cd92f340a93 --- /dev/null +++ b/lib/plausible_web/components/layout.ex @@ -0,0 +1,186 @@ +defmodule PlausibleWeb.Components.Layout do + @moduledoc false + + use Phoenix.Component + + attr :class, :string, default: "w-24 sm:w-30" + + def logo(assigns) do + ~H""" + Plausible logo + Plausible logo + """ + end + + defp logo_url(filename), + do: PlausibleWeb.Router.Helpers.static_path(PlausibleWeb.Endpoint, logo_path(filename)) + + def favicon(assigns) do + ~H""" + + + + """ + end + + def theme_script(assigns) do + ~H""" + + """ + end + + attr(:selected_fn, :any, required: true) + attr(:prefix, :string, default: "") + attr(:options, :list, required: true) + + def settings_sidebar(assigns) do + ~H""" +
+ <.settings_top_tab + :for={%{key: key, value: value, icon: icon} = opts <- @options} + selected_fn={@selected_fn} + prefix={@prefix} + icon={icon} + text={key} + badge={opts[:badge]} + value={value} + /> +
+ """ + end + + attr(:selected_fn, :any) + attr(:prefix, :string, default: "") + attr(:icon, :any, default: nil) + attr(:text, :string, required: true) + attr(:badge, :any, default: nil) + attr(:value, :any, default: nil) + + defp settings_top_tab(assigns) do + ~H""" + <%= if is_binary(@value) do %> + <.settings_tab + selected_fn={@selected_fn} + prefix={@prefix} + icon={@icon} + text={@text} + badge={@badge} + value={@value} + /> + <% else %> + <.settings_tab icon={@icon} text={@text} /> + +
+ <.settings_tab + :for={%{key: key, value: value} = opts <- @value} + selected_fn={@selected_fn} + prefix={@prefix} + icon={nil} + text={key} + badge={opts[:badge]} + value={value} + submenu?={true} + /> +
+ <% end %> + """ + end + + attr(:selected_fn, :any, default: nil) + attr(:prefix, :string, default: "") + attr(:value, :any, default: nil) + attr(:icon, :any, default: nil) + attr(:submenu?, :boolean, default: false) + attr(:text, :string, required: true) + attr(:badge, :any, default: nil) + + defp settings_tab(assigns) do + current_tab? = assigns[:selected_fn] != nil and assigns.selected_fn.(assigns[:value]) + assigns = assign(assigns, :current_tab?, current_tab?) + + ~H""" + "/settings/" <> @value} + class={[ + "text-sm flex items-center px-2 py-2 leading-5 font-medium rounded-md outline-none focus:outline-none transition ease-in-out duration-150", + @current_tab? && + "text-gray-900 dark:text-gray-100 bg-gray-150 font-semibold dark:bg-gray-850 hover:text-gray-900 dark:hover:text-gray-100 focus:bg-gray-200 dark:focus:bg-gray-800", + @value && not @current_tab? && + "text-gray-600 dark:text-gray-300 hover:text-gray-900 dark:hover:text-gray-100 hover:bg-gray-100 dark:hover:bg-gray-850 focus:text-gray-900 focus:bg-gray-50 dark:focus:text-gray-100 dark:focus:bg-gray-800", + !@value && "text-gray-600 dark:text-gray-300" + ]} + > + + {@text} + + NEW 🔥 + + + + """ + end + + defp theme_preference(%{theme: theme}) when not is_nil(theme), do: theme + + defp theme_preference(%{current_user: %Plausible.Auth.User{theme: theme}}) + when not is_nil(theme) do + theme + end + + defp theme_preference(_assigns), do: "system" + + defdelegate logo_path(path), to: PlausibleWeb.LayoutView +end diff --git a/lib/plausible_web/components/prima_dropdown.ex b/lib/plausible_web/components/prima_dropdown.ex new file mode 100644 index 0000000000000000000000000000000000000000..47f506be185a96586ee932995238e2a21fdedaee --- /dev/null +++ b/lib/plausible_web/components/prima_dropdown.ex @@ -0,0 +1,92 @@ +defmodule PlausibleWeb.Components.PrimaDropdown do + @moduledoc false + alias Prima.Dropdown + use Phoenix.Component + + @dropdown_item_icon_base_class "text-gray-600 dark:text-gray-400 group-hover/item:text-gray-900 group-data-focus/item:text-gray-900 dark:group-hover/item:text-gray-100 dark:group-data-focus/item:text-gray-100" + + @trigger_button_base_class "whitespace-nowrap truncate inline-flex items-center justify-between gap-x-2 text-sm font-medium rounded-md cursor-pointer disabled:cursor-not-allowed" + + @trigger_button_themes %{ + "primary" => + "border border-indigo-600 bg-indigo-600 text-white hover:bg-indigo-700 focus-visible:outline-indigo-600 disabled:bg-indigo-400/60 disabled:dark:bg-indigo-600/30 disabled:dark:text-white/35", + "secondary" => + "border border-gray-300 dark:border-gray-700 bg-white dark:bg-gray-700 text-gray-800 dark:text-gray-100 hover:border-gray-400/60 hover:text-gray-900 dark:hover:border-gray-500 dark:hover:text-white disabled:text-gray-700/40 dark:disabled:text-gray-500 dark:disabled:bg-gray-800 dark:disabled:border-gray-800", + "ghost" => + "text-gray-700 dark:text-gray-300 hover:text-gray-900 dark:hover:text-gray-100 disabled:text-gray-500 disabled:dark:text-gray-600" + } + + @trigger_button_sizes %{ + "sm" => "px-3 py-2", + "md" => "px-3.5 py-2.5" + } + + defdelegate dropdown(assigns), to: Prima.Dropdown + + attr(:id, :string, required: true) + attr(:theme, :string, default: "secondary") + attr(:size, :string, default: "md") + attr(:class, :string, default: "") + attr(:rest, :global) + slot(:inner_block, required: true) + + def dropdown_trigger(assigns) do + assigns = + assign(assigns, + computed_class: [ + @trigger_button_base_class, + @trigger_button_sizes[assigns.size], + @trigger_button_themes[assigns.theme], + assigns.class + ] + ) + + ~H""" + + {render_slot(@inner_block)} + + """ + end + + attr(:id, :string, required: true) + slot(:inner_block, required: true) + + # placement: bottom-end should probably be default in prima. Feels more natural + # for dropdown menus than bottom-start which is the current default + def dropdown_menu(assigns) do + ~H""" + + {render_slot(@inner_block)} + + """ + end + + attr(:as, :any, default: nil) + attr(:id, :string, required: true) + attr(:disabled, :boolean, default: false) + attr(:rest, :global, include: ~w(navigate patch href)) + slot(:inner_block, required: true) + + def dropdown_item(assigns) do + ~H""" + + {render_slot(@inner_block)} + + """ + end + + def dropdown_item_icon_class(size \\ "size-4") do + "#{size} #{@dropdown_item_icon_base_class}" + end +end diff --git a/lib/plausible_web/components/site/toggle_live.ex b/lib/plausible_web/components/site/toggle_live.ex new file mode 100644 index 0000000000000000000000000000000000000000..8262d5da42ae13e8cf53ccc295cfd09f9e3827c5 --- /dev/null +++ b/lib/plausible_web/components/site/toggle_live.ex @@ -0,0 +1,92 @@ +defmodule PlausibleWeb.Components.Site.Feature.ToggleLive do + @moduledoc """ + LiveComponent for rendering a user-facing feature toggle in LiveView contexts. + Instead of using form submission, this component messages itself to handle toggles. + """ + use PlausibleWeb, :live_component + + def update(assigns, socket) do + site = Plausible.Repo.preload(assigns.site, :team) + team = Plausible.Teams.with_subscription(site.team) + site = %{site | team: team} + current_setting = assigns.feature_mod.enabled?(site) + disabled? = assigns.feature_mod.check_availability(team) !== :ok + + {:ok, + socket + |> assign(assigns) + |> assign(:site, site) + |> assign(:current_setting, current_setting) + |> assign(:disabled?, disabled?)} + end + + attr :site, Plausible.Site, required: true + attr :feature_mod, :atom, required: true + attr :current_user, Plausible.Auth.User, required: true + + def render(assigns) do + ~H""" +
+
+ + + + Show in dashboard + +
+
+ """ + end + + def handle_event("toggle", _params, socket) do + site = socket.assigns.site + feature_mod = socket.assigns.feature_mod + current_user = socket.assigns.current_user + + case feature_mod.toggle(site, current_user) do + {:ok, updated_site} -> + new_setting = Map.fetch!(updated_site, feature_mod.toggle_field()) + + message = + if new_setting do + "#{feature_mod.toggle_display_name()} are now visible again on your dashboard" + else + "#{feature_mod.toggle_display_name()} are now hidden from your dashboard" + end + + send(self(), {:feature_toggled, message, updated_site}) + + socket = + assign(socket, site: updated_site, current_setting: feature_mod.enabled?(updated_site)) + + {:noreply, socket} + + {:error, _} -> + {:noreply, socket} + end + end +end diff --git a/lib/plausible_web/components/team/notice.ex b/lib/plausible_web/components/team/notice.ex new file mode 100644 index 0000000000000000000000000000000000000000..ccfc2d2dea55758f521ce7867544c23dd433a84e --- /dev/null +++ b/lib/plausible_web/components/team/notice.ex @@ -0,0 +1,288 @@ +defmodule PlausibleWeb.Team.Notice do + @moduledoc """ + Components with teams related notices. + """ + use PlausibleWeb, :component + import PlausibleWeb.Components.Icons + + alias Plausible.Teams + alias PlausibleWeb.Components.PrimaDropdown + + def owner_cta_banner(assigns) do + ~H""" + + """ + end + + def guest_cta_banner(assigns) do + ~H""" + + """ + end + + def team_members_notice(assigns) do + ~H""" + + """ + end + + def team_invitations(assigns) do + ~H""" + + """ + end + + def site_ownership_invitations(assigns) do + ~H""" + + """ + end + + defp site_ownership_invitation(assigns) do + {can_accept?, can_accept_without_members?, exceeded_limits} = + case assigns.invitation.ownership_check do + :ok -> + {true, false, nil} + + {:error, {:over_plan_limits, limits}} -> + {false, limits == [:team_member_limit], PlausibleWeb.TextHelpers.pretty_list(limits)} + + _ -> + {false, false, nil} + end + + assigns = + assign(assigns, + can_accept?: can_accept?, + can_accept_without_members?: can_accept_without_members?, + exceeded_limits: exceeded_limits + ) + + ~H""" + <.notice + id={"site-ownership-invitation-#{@invitation.transfer_id}"} + title={"#{@invitation.initiator.name} has invited you to own #{@invitation.site.domain}"} + theme={:white} + icon_class="hidden md:block" + > + <:icon> +
+ <.envelope_icon class="size-4 text-green-600 dark:text-green-400" /> +
+ +

+ On acceptance, you'll be responsible for billing and this site will join "{Teams.name( + @current_team + )}" +

+

+ You don't have an active subscription. Upgrade to accept ownership and take over billing. +

+

+ This exceeds your current {@exceeded_limits} limits. Upgrade to accept ownership. +

+ <:actions> + <.button_link + method="post" + href={ + Routes.invitation_path( + PlausibleWeb.Endpoint, + :reject_invitation, + @invitation.transfer_id + ) + } + theme="ghost" + size="sm" + class="order-3 md:order-1" + mt?={false} + > + Reject + + <.button_link + :if={@can_accept?} + method="post" + href={ + Routes.invitation_path( + PlausibleWeb.Endpoint, + :accept_invitation, + @invitation.transfer_id + ) + } + theme="secondary" + size="sm" + class="order-1 md:order-2" + mt?={false} + > + Accept + + + + Accept + + + + Upgrade to accept + + link(Map.put(a, :method, "post")) end} + id={"ownership-accept-item-members-#{@invitation.transfer_id}"} + href={ + Routes.invitation_path( + PlausibleWeb.Endpoint, + :accept_invitation, + @invitation.transfer_id, + skip_site_members_transfer: "true" + ) + } + > + Accept without members + + + + <.button_link + :if={not @can_accept? and not @can_accept_without_members?} + href={Routes.billing_path(PlausibleWeb.Endpoint, :choose_plan)} + theme="secondary" + size="sm" + class="order-1 md:order-3" + mt?={false} + > + Upgrade to accept + + + + """ + end + + def site_invitations(assigns) do + ~H""" + + """ + end +end diff --git a/lib/plausible_web/components/two_factor.ex b/lib/plausible_web/components/two_factor.ex new file mode 100644 index 0000000000000000000000000000000000000000..abfcbed83ee66bd546060ead17444e50ca17a2d1 --- /dev/null +++ b/lib/plausible_web/components/two_factor.ex @@ -0,0 +1,160 @@ +defmodule PlausibleWeb.Components.TwoFactor do + @moduledoc """ + Reusable components specific to 2FA + """ + use PlausibleWeb, :component + + attr :text, :string, required: true + attr :scale, :integer, default: 4 + + def qr_code(assigns) do + qr_code = + assigns.text + |> EQRCode.encode() + |> EQRCode.svg(%{width: 160}) + + assigns = assign(assigns, :code, qr_code) + + ~H""" + {Phoenix.HTML.raw(@code)} + """ + end + + attr :id, :string, default: "verify-button" + attr :form, :any, required: true + attr :field, :any, required: true + attr :class, :string, default: "" + attr :show_button?, :boolean, default: true + attr :autofocus?, :boolean, default: false + attr :error, :string, default: nil + + def verify_2fa_input(assigns) do + assigns = assign(assigns, :field, assigns[:form][assigns[:field]]) + + ~H""" +
+
+ <.otp_input + field={@field} + length={6} + oninvalid={@show_button? && "document.getElementById('#{@id}').disabled = false"} + autofocus={@autofocus? && "autofocus"} + /> +

+ {@error} +

+
+ <.button + :if={@show_button?} + type="submit" + id={@id} + mt?={false} + class="w-full [&>span.label-enabled]:block [&>span.label-disabled]:hidden [&[disabled]>span.label-enabled]:hidden [&[disabled]>span.label-disabled]:block" + > + + Verify + + + + <.spinner class="inline-block h-5 w-5 mr-2 text-white dark:text-gray-400" /> Verifying... + + +
+ """ + end + + attr :id, :string, required: true + attr :state_param, :string, required: true + attr :form_data, :any, required: true + attr :form_target, :string, required: true + attr :onsubmit, :string, default: nil + attr :title, :string, required: true + + slot :icon, required: true + slot :inner_block, required: true + slot :buttons, required: true + + def modal(assigns) do + ~H""" + + """ + end +end diff --git a/lib/plausible_web/controllers/api/external_controller.ex b/lib/plausible_web/controllers/api/external_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..87faf17df182d8972b0340729ae5876cb9673033 --- /dev/null +++ b/lib/plausible_web/controllers/api/external_controller.ex @@ -0,0 +1,60 @@ +defimpl FunWithFlags.Actor, for: BitString do + def id(str) do + str + end +end + +defmodule PlausibleWeb.Api.ExternalController do + use PlausibleWeb, :controller + require Logger + + alias Plausible.Ingestion + + def event(conn, _params) do + with {:ok, request, conn} <- Ingestion.Request.build(conn), + _ <- Sentry.Context.set_extra_context(%{request: request}) do + case Ingestion.Event.build_and_buffer(request) do + {:ok, %{dropped: [], buffered: _buffered}} -> + conn + |> put_status(202) + |> text("ok") + + {:ok, %{dropped: dropped, buffered: _}} -> + first_invalid_changeset = find_first_invalid_changeset(dropped) + + if first_invalid_changeset do + conn + |> put_resp_header("x-plausible-dropped", "#{Enum.count(dropped)}") + |> put_status(400) + |> json(%{ + errors: Plausible.ChangesetHelpers.traverse_errors(first_invalid_changeset) + }) + else + conn + |> put_resp_header("x-plausible-dropped", "#{Enum.count(dropped)}") + |> put_status(202) + |> text("ok") + end + end + else + {:error, %Ecto.Changeset{} = changeset} -> + conn + |> put_status(400) + |> json(%{errors: Plausible.ChangesetHelpers.traverse_errors(changeset)}) + end + end + + def error(conn, _params) do + Sentry.capture_message("JS snippet error") + send_resp(conn, 200, "") + end + + defp find_first_invalid_changeset(dropped) do + Enum.find_value(dropped, nil, fn dropped_event -> + case dropped_event.drop_reason do + {:error, %Ecto.Changeset{} = changeset} -> changeset + _ -> false + end + end) + end +end diff --git a/lib/plausible_web/controllers/api/external_query_api_controller.ex b/lib/plausible_web/controllers/api/external_query_api_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..eb983a4b56acc0d89d914d293c847a019831d41b --- /dev/null +++ b/lib/plausible_web/controllers/api/external_query_api_controller.ex @@ -0,0 +1,30 @@ +defmodule PlausibleWeb.Api.ExternalQueryApiController do + @moduledoc false + + use PlausibleWeb, :controller + use Plausible.Repo + use PlausibleWeb.Plugs.ErrorHandler + alias Plausible.Stats.{Query, QueryError} + + def query(conn, params) do + site = Repo.preload(conn.assigns.site, :owners) + + case Query.parse_and_build(site, params, + debug_metadata: debug_metadata(conn), + now: conn.private[:now] + ) do + {:ok, query} -> + results = Plausible.Stats.query(site, query) + json(conn, results) + + {:error, %QueryError{message: message}} -> + conn + |> put_status(400) + |> json(%{error: message}) + end + end + + def schema(conn, _params) do + json(conn, Plausible.Stats.JSONSchema.raw_schema()) + end +end diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..50cebddd70dc08383b76e92bb2a586a3245a3987 --- /dev/null +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -0,0 +1,413 @@ +defmodule PlausibleWeb.Api.ExternalStatsController do + use PlausibleWeb, :controller + use Plausible.Repo + use PlausibleWeb.Plugs.ErrorHandler + alias Plausible.Stats.{Query, Metrics, Filters, Interval, Legacy} + + def realtime_visitors(conn, _params) do + site = conn.assigns.site + json(conn, Plausible.Stats.current_visitors(site)) + end + + def aggregate(conn, params) do + site = Repo.preload(conn.assigns.site, :owners) + + params = Map.put(params, "property", nil) + + with :ok <- validate_period(params), + :ok <- validate_date(params), + query <- Query.from(site, params, debug_metadata: debug_metadata(conn)), + :ok <- validate_filters(site, query.filters), + {:ok, metrics} <- parse_and_validate_metrics(params, query), + :ok <- ensure_custom_props_access(site, query) do + %{results: results, meta: meta} = Legacy.Aggregate.aggregate(site, query, metrics) + + payload = maybe_add_warning(%{results: results}, meta) + + json(conn, payload) + else + err_tuple -> send_json_error_response(conn, err_tuple) + end + end + + def breakdown(conn, params) do + site = Repo.preload(conn.assigns.site, :owners) + + with :ok <- validate_period(params), + :ok <- validate_date(params), + :ok <- validate_property(params), + query <- Query.from(site, params, debug_metadata: debug_metadata(conn)), + :ok <- validate_filters(site, query.filters), + {:ok, metrics} <- parse_and_validate_metrics(params, query), + {:ok, limit} <- validate_or_default_limit(params), + :ok <- ensure_custom_props_access(site, query) do + page = String.to_integer(Map.get(params, "page", "1")) + + %{results: results, meta: meta} = + Legacy.Breakdown.breakdown(site, query, metrics, {limit, page}) + + payload = maybe_add_warning(%{results: results}, meta) + + json(conn, payload) + else + err_tuple -> send_json_error_response(conn, err_tuple) + end + end + + defp validate_property(%{"property" => property}) do + cond do + property == "event:hostname" -> + {:error, + "Property 'event:hostname' is currently not supported for breakdowns. Please provide a valid property for the breakdown endpoint: https://plausible.io/docs/stats-api#properties"} + + Plausible.Stats.Legacy.Dimensions.valid?(property) -> + :ok + + true -> + {:error, + "Invalid property '#{property}'. Please provide a valid property for the breakdown endpoint: https://plausible.io/docs/stats-api#properties"} + end + end + + defp validate_property(_) do + {:error, + "The `property` parameter is required. Please provide at least one property to show a breakdown by."} + end + + @max_breakdown_limit 1000 + defp validate_or_default_limit(%{"limit" => limit}) do + case Integer.parse(limit) do + {limit, ""} when limit > 0 and limit <= @max_breakdown_limit -> + {:ok, limit} + + _ -> + {:error, "Please provide limit as a number between 1 and #{@max_breakdown_limit}."} + end + end + + @default_breakdown_limit 100 + defp validate_or_default_limit(_), do: {:ok, @default_breakdown_limit} + + defp parse_and_validate_metrics(params, query) do + metrics = + Map.get(params, "metrics", "visitors") + |> String.split(",") + + case validate_metrics(metrics, query) do + {:error, reason} -> + {:error, reason} + + metrics -> + {:ok, Enum.map(metrics, &Metrics.from_string!/1)} + end + end + + @spec ensure_custom_props_access(Plausible.Site.t(), Query.t()) :: + :ok | {:error, {402, String.t()}} + defp ensure_custom_props_access(site, query) do + allowed_props = Plausible.Props.allowed_for(site, bypass_setup?: true) + prop_filter = Filters.get_toplevel_filter(query, "event:props:") + + query_allowed? = + case {prop_filter, query.dimensions, allowed_props} do + {_, _, :all} -> + true + + {[_, "event:props:" <> prop | _], _property, allowed_props} -> + prop in allowed_props + + {_filter, ["event:props:" <> prop], allowed_props} -> + prop in allowed_props + + _ -> + true + end + + if query_allowed? do + :ok + else + msg = "The owner of this site does not have access to the custom properties feature" + {:error, {402, msg}} + end + end + + defp validate_metrics(metrics, query) do + if length(metrics) == length(Enum.uniq(metrics)) do + validate_each_metric(metrics, query) + else + {:error, "Metrics cannot be queried multiple times."} + end + end + + defp validate_each_metric(metrics, query) do + Enum.reduce_while(metrics, [], fn metric, acc -> + case validate_metric(metric, query) do + {:ok, metric} -> {:cont, acc ++ [metric]} + {:error, reason} -> {:halt, {:error, reason}} + end + end) + end + + defp validate_metric("time_on_page" = metric, query) do + cond do + Filters.filtering_on_dimension?(query, "event:goal") -> + {:error, "Metric `#{metric}` cannot be queried when filtering by `event:goal`"} + + Filters.filtering_on_dimension?(query, "event:name") -> + {:error, "Metric `#{metric}` cannot be queried when filtering by `event:name`"} + + query.dimensions == ["event:page"] -> + {:ok, metric} + + not Enum.empty?(query.dimensions) -> + {:error, + "Metric `#{metric}` is not supported in breakdown queries (except `event:page` breakdown)"} + + Filters.filtering_on_dimension?(query, "event:page") -> + {:ok, metric} + + true -> + {:error, + "Metric `#{metric}` can only be queried in a page breakdown or with a page filter."} + end + end + + defp validate_metric("conversion_rate" = metric, query) do + cond do + query.dimensions == ["event:goal"] -> + {:ok, metric} + + Filters.filtering_on_dimension?(query, "event:goal") -> + {:ok, metric} + + true -> + {:error, + "Metric `#{metric}` can only be queried in a goal breakdown or with a goal filter"} + end + end + + defp validate_metric(metric, _) when metric in ["visitors", "pageviews", "events"] do + {:ok, metric} + end + + defp validate_metric("views_per_visit" = metric, query) do + cond do + Filters.filtering_on_dimension?(query, "event:page") -> + {:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."} + + not Enum.empty?(query.dimensions) -> + {:error, "Metric `#{metric}` is not supported in breakdown queries."} + + true -> + validate_session_metric(metric, query) + end + end + + defp validate_metric(metric, query) + when metric in ["visits", "bounce_rate", "visit_duration"] do + validate_session_metric(metric, query) + end + + defp validate_metric(metric, _) do + {:error, + "The metric `#{metric}` is not recognized. Find valid metrics from the documentation: https://plausible.io/docs/stats-api#metrics"} + end + + defp validate_session_metric(metric, query) do + cond do + length(query.dimensions) == 1 and event_only_property?(hd(query.dimensions)) -> + {:error, + "Session metric `#{metric}` cannot be queried for breakdown by `#{query.dimensions}`."} + + event_only_filter = find_event_only_filter(query) -> + {:error, + "Session metric `#{metric}` cannot be queried when using a filter on `#{event_only_filter}`."} + + true -> + {:ok, metric} + end + end + + defp find_event_only_filter(query) do + query.filters + |> Enum.map(fn [_op, prop | _] -> prop end) + |> Enum.find(&event_only_property?/1) + end + + defp event_only_property?("event:name"), do: true + defp event_only_property?("event:goal"), do: true + defp event_only_property?("event:props:" <> _), do: true + defp event_only_property?(_), do: false + + def timeseries(conn, params) do + site = Repo.preload(conn.assigns.site, :owners) + + params = Map.put(params, "property", nil) + + params = + if Map.get(params, "interval") == "date" do + %{params | "interval" => "day"} + else + params + end + + with :ok <- validate_period(params), + :ok <- validate_date(params), + :ok <- validate_interval(params), + query <- Query.from(site, params, debug_metadata: debug_metadata(conn)), + :ok <- validate_filters(site, query.filters), + {:ok, metrics} <- parse_and_validate_metrics(params, query), + :ok <- ensure_custom_props_access(site, query) do + time_dimension = interval_to_time_dimension(Map.get(params, "interval"), query) + query = Query.set(query, dimensions: [time_dimension]) + {results, meta} = Legacy.Timeseries.timeseries(site, query, metrics) + + payload = + case meta[:imports_warning] do + nil -> %{results: results} + warning -> %{results: results, warning: warning} + end + + json(conn, payload) + else + err_tuple -> send_json_error_response(conn, err_tuple) + end + end + + defp validate_date(%{"period" => "custom"} = params) do + with {:ok, date} <- Map.fetch(params, "date"), + [from, to] <- String.split(date, ","), + {:ok, _from} <- Date.from_iso8601(String.trim(from)), + {:ok, _to} <- Date.from_iso8601(String.trim(to)) do + :ok + else + :error -> + {:error, + "The `date` parameter is required when using a custom period. See https://plausible.io/docs/stats-api#time-periods"} + + _ -> + {:error, + "Invalid format for `date` parameter. When using a custom period, please include two ISO-8601 formatted dates joined by a comma. See https://plausible.io/docs/stats-api#time-periods"} + end + end + + defp validate_date(%{"date" => date}) do + case Date.from_iso8601(date) do + {:ok, _date} -> + :ok + + {:error, msg} -> + {:error, + "Error parsing `date` parameter: #{msg}. Please specify a valid date in ISO-8601 format."} + end + end + + defp validate_date(_), do: :ok + + defp validate_period(%{"period" => period}) do + if period in ["day", "7d", "30d", "month", "6mo", "12mo", "custom"] do + :ok + else + {:error, + "Error parsing `period` parameter: invalid period `#{period}`. Please find accepted values in our docs: https://plausible.io/docs/stats-api#time-periods"} + end + end + + defp validate_period(_), do: :ok + + @valid_intervals ["day", "month"] + @valid_intervals_str Enum.map_join(@valid_intervals, ", ", &("`" <> &1 <> "`")) + + defp validate_interval(%{"interval" => interval}) do + if interval in @valid_intervals do + :ok + else + {:error, + "Error parsing `interval` parameter: invalid interval `#{interval}`. Valid intervals are #{@valid_intervals_str}"} + end + end + + defp validate_interval(_), do: :ok + + @time_dimensions %{ + "minute" => "time:minute", + "hour" => "time:hour", + "day" => "time:day", + "week" => "time:week", + "month" => "time:month" + } + + defp interval_to_time_dimension(interval, query) do + interval = interval || Interval.default_for_query(query) + Map.fetch!(@time_dimensions, interval) + end + + defp validate_filters(site, filters) do + Enum.reduce_while(filters, :ok, fn filter, _ -> + case validate_filter(site, filter) do + :ok -> {:cont, :ok} + {:error, reason} -> {:halt, {:error, reason}} + end + end) + end + + defp validate_filter(site, [_type, "event:goal", goal_filter | _rest]) do + site = Plausible.Repo.preload(site, :team) + props_available? = Plausible.Billing.Feature.Props.check_availability(site.team) == :ok + + configured_goals = + site + |> Plausible.Goals.for_site(include_goals_with_custom_props?: props_available?) + |> Enum.map(& &1.display_name) + + goals_in_filter = List.wrap(goal_filter) + + if found = Enum.find(goals_in_filter, &(&1 not in configured_goals)) do + msg = + goal_not_configured_message(found) <> + "Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals" + + {:error, msg} + else + :ok + end + end + + defp validate_filter(_site, [_, property | _]) do + if Plausible.Stats.Legacy.Dimensions.valid?(property) do + :ok + else + {:error, + "Invalid filter property '#{property}'. Please provide a valid filter property: https://plausible.io/docs/stats-api#properties"} + end + end + + defp goal_not_configured_message("Visit " <> page_path) do + "The pageview goal for the pathname `#{page_path}` is not configured for this site. " + end + + defp goal_not_configured_message(goal) do + "The goal `#{goal}` is not configured for this site. " + end + + @imported_query_unsupported_warning "Imported stats are not included in the results because query parameters are not supported. For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats" + + defp maybe_add_warning(payload, %Jason.OrderedObject{} = meta) do + case meta[:imports_skip_reason] do + :unsupported_query -> Map.put(payload, :warning, @imported_query_unsupported_warning) + _ -> payload + end + end + + defp send_json_error_response(conn, {:error, {status, msg}}) do + conn + |> put_status(status) + |> json(%{error: msg}) + end + + defp send_json_error_response(conn, {:error, msg}) do + conn + |> put_status(400) + |> json(%{error: msg}) + end +end diff --git a/lib/plausible_web/controllers/api/helpers.ex b/lib/plausible_web/controllers/api/helpers.ex new file mode 100644 index 0000000000000000000000000000000000000000..2e8bb0162a59a6a8801a8f718eb0695275f7cd67 --- /dev/null +++ b/lib/plausible_web/controllers/api/helpers.ex @@ -0,0 +1,51 @@ +defmodule PlausibleWeb.Api.Helpers do + @moduledoc """ + Helper functions for API controllers. + """ + + import Plug.Conn + + def unauthorized(conn, msg) do + conn + |> put_status(401) + |> Phoenix.Controller.json(%{error: msg}) + |> halt() + end + + def not_enough_permissions(conn, msg) do + conn + |> put_status(403) + |> Phoenix.Controller.json(%{error: msg}) + |> halt() + end + + def bad_request(conn, msg, extra \\ %{}) do + payload = Map.merge(extra, %{error: msg}) + + conn + |> put_status(400) + |> Phoenix.Controller.json(payload) + |> halt() + end + + def not_found(conn, msg) do + conn + |> put_status(404) + |> Phoenix.Controller.json(%{error: msg}) + |> halt() + end + + def too_many_requests(conn, msg) do + conn + |> put_status(429) + |> Phoenix.Controller.json(%{error: msg}) + |> halt() + end + + def payment_required(conn, msg) do + conn + |> put_status(402) + |> Phoenix.Controller.json(%{error: msg}) + |> halt() + end +end diff --git a/lib/plausible_web/controllers/api/internal/annotations_controller.ex b/lib/plausible_web/controllers/api/internal/annotations_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..e161589252b172639aa778cdf319a380053f507f --- /dev/null +++ b/lib/plausible_web/controllers/api/internal/annotations_controller.ex @@ -0,0 +1,147 @@ +defmodule PlausibleWeb.Api.Internal.AnnotationsController do + @moduledoc """ + Internal API controller for annotations. + """ + use Plausible + use PlausibleWeb, :controller + use PlausibleWeb.Plugs.ErrorHandler + alias Plausible.Annotations + alias Plausible.ChangesetHelpers + alias Plausible.Stats.{ApiQueryParser, QueryPeriod} + alias PlausibleWeb.Api.Helpers, as: H + + def index(conn, params) do + user = conn.assigns[:current_user] + site = conn.assigns.site + site_role = conn.assigns.site_role + + with {:ok, input_date_range} <- parse_input_date_range(params), + {:ok, relative_date} <- parse_relative_date(params) do + now = DateTime.utc_now(:second) + + range_in_site_tz = + QueryPeriod.build_range_for_site(input_date_range, site, relative_date, now) + + case Annotations.get_all_for_site(site, site_role, user, range_in_site_tz) do + {:ok, result} -> + json(conn, result) + + {:error, :not_enough_permissions} -> + json(conn, "Not enough permissions to get annotations") + end + else + {:error, message} -> H.bad_request(conn, message) + end + end + + defp parse_input_date_range(%{ + "date_range_start" => start, + "date_range_end" => end_ + }) + when is_binary(start) and is_binary(end_) do + case ApiQueryParser.parse_input_date_range([start, end_]) do + {:ok, parsed} -> {:ok, parsed} + {:error, %{message: message}} -> {:error, message} + end + end + + defp parse_input_date_range(%{"date_range" => "realtime"}), do: {:ok, :realtime} + defp parse_input_date_range(%{"date_range" => "realtime_30m"}), do: {:ok, :realtime_30m} + + defp parse_input_date_range(%{"date_range" => date_range}) do + case ApiQueryParser.parse_input_date_range(date_range) do + {:ok, parsed} -> {:ok, parsed} + {:error, %{message: message}} -> {:error, message} + end + end + + defp parse_input_date_range(_), do: {:error, "Required 'date_range' parameter missing"} + + defp parse_relative_date(%{"relative_date" => date}) when is_binary(date) do + case Date.from_iso8601(date) do + {:ok, date} -> {:ok, date} + _ -> {:error, "Failed to convert '#{date}' to date"} + end + end + + defp parse_relative_date(_), do: {:ok, nil} + + def create(conn, params) do + user = conn.assigns[:current_user] + site = conn.assigns.site + site_role = conn.assigns.site_role + + case Annotations.insert_one(user, site, site_role, params) do + {:error, :not_enough_permissions} -> + H.not_enough_permissions(conn, "Not enough permissions to create annotation") + + {:error, :annotations_limit_reached} -> + H.not_enough_permissions(conn, "Annotations limit reached") + + {:error, {:invalid_annotation, errors}} when is_list(errors) -> + conn + |> put_status(400) + |> json(%{ + error: ChangesetHelpers.serialize_first_error(errors) + }) + + {:ok, annotation} -> + json(conn, annotation) + end + end + + def update(conn, params) do + user = conn.assigns[:current_user] + site = conn.assigns.site + site_role = conn.assigns.site_role + annotation_id = normalize_annotation_id_param(params["annotation_id"]) + + case Annotations.update_one(user, site, site_role, annotation_id, params) do + {:error, :not_enough_permissions} -> + H.not_enough_permissions(conn, "Not enough permissions to edit annotation") + + {:error, :annotation_not_found} -> + annotation_not_found(conn, params["annotation_id"]) + + {:error, {:invalid_annotation, errors}} when is_list(errors) -> + conn + |> put_status(400) + |> json(%{ + error: ChangesetHelpers.serialize_first_error(errors) + }) + + {:ok, annotation} -> + json(conn, annotation) + end + end + + def delete(conn, params) do + user = conn.assigns[:current_user] + site = conn.assigns.site + site_role = conn.assigns.site_role + annotation_id = normalize_annotation_id_param(params["annotation_id"]) + + case Annotations.delete_one(user, site, site_role, annotation_id) do + {:error, :not_enough_permissions} -> + H.not_enough_permissions(conn, "Not enough permissions to delete annotation") + + {:error, :annotation_not_found} -> + annotation_not_found(conn, params["annotation_id"]) + + {:ok, annotation} -> + json(conn, annotation) + end + end + + @spec normalize_annotation_id_param(any()) :: nil | pos_integer() + defp normalize_annotation_id_param(input) do + case Integer.parse(input) do + {int_value, ""} when int_value > 0 -> int_value + _ -> nil + end + end + + defp annotation_not_found(%Plug.Conn{} = conn, annotation_id_param) do + H.not_found(conn, "Annotation not found with ID #{inspect(annotation_id_param)}") + end +end diff --git a/lib/plausible_web/controllers/api/internal/segments_controller.ex b/lib/plausible_web/controllers/api/internal/segments_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..623ebf8788a11850acb021a90d1ad5809b413f7c --- /dev/null +++ b/lib/plausible_web/controllers/api/internal/segments_controller.ex @@ -0,0 +1,146 @@ +defmodule PlausibleWeb.Api.Internal.SegmentsController do + @moduledoc """ + Internal API controller for segments. + """ + use Plausible + use PlausibleWeb, :controller + use PlausibleWeb.Plugs.ErrorHandler + alias PlausibleWeb.Api.Helpers, as: H + alias Plausible.ChangesetHelpers + alias Plausible.Segments + + def create( + %Plug.Conn{ + assigns: %{ + site: site, + current_user: %{id: user_id}, + site_role: site_role + } + } = conn, + %{} = params + ) do + case Segments.insert_one(user_id, site, site_role, params) do + {:error, :not_enough_permissions} -> + H.not_enough_permissions(conn, "Not enough permissions to create segment") + + {:error, :segment_limit_reached} -> + H.not_enough_permissions(conn, "Segment limit reached") + + {:error, {:invalid_segment, errors}} when is_list(errors) -> + conn + |> put_status(400) + |> json(%{ + error: ChangesetHelpers.serialize_first_error(errors) + }) + + {:ok, segment} -> + json(conn, Segments.to_response_map(segment, site)) + end + end + + def create(%Plug.Conn{} = conn, _params), do: invalid_request(conn) + + def update( + %Plug.Conn{ + assigns: %{ + site: site, + current_user: %{id: user_id}, + site_role: site_role + } + } = + conn, + %{} = params + ) do + segment_id = normalize_segment_id_param(params["segment_id"]) + + case Segments.update_one(user_id, site, site_role, segment_id, params) do + {:error, :not_enough_permissions} -> + H.not_enough_permissions(conn, "Not enough permissions to edit segment") + + {:error, :segment_not_found} -> + segment_not_found(conn, params["segment_id"]) + + {:error, {:invalid_segment, errors}} when is_list(errors) -> + conn + |> put_status(400) + |> json(%{ + error: ChangesetHelpers.serialize_first_error(errors) + }) + + {:ok, segment} -> + json(conn, Segments.to_response_map(segment, site)) + end + end + + def update(%Plug.Conn{} = conn, _params), do: invalid_request(conn) + + def delete( + %Plug.Conn{ + assigns: %{ + site: site, + current_user: %{id: user_id}, + site_role: site_role + } + } = + conn, + %{} = params + ) do + segment_id = normalize_segment_id_param(params["segment_id"]) + + case Segments.delete_one(user_id, site, site_role, segment_id) do + {:error, :not_enough_permissions} -> + H.not_enough_permissions(conn, "Not enough permissions to delete segment") + + {:error, :segment_not_found} -> + segment_not_found(conn, params["segment_id"]) + + {:ok, segment} -> + json(conn, Segments.to_response_map(segment, site)) + end + end + + def delete(%Plug.Conn{} = conn, _params), do: invalid_request(conn) + + def get_related_shared_links( + %Plug.Conn{ + assigns: %{ + site: site, + current_user: %{id: _user_id}, + site_role: site_role + } + } = + conn, + %{} = params + ) do + segment_id = normalize_segment_id_param(params["segment_id"]) + + case Segments.get_related_shared_links(site, site_role, segment_id) do + {:error, :not_enough_permissions} -> + H.not_enough_permissions(conn, "Not enough permissions to get related shared links") + + {:error, :segment_not_found} -> + segment_not_found(conn, params["segment_id"]) + + {:ok, shared_links} -> + json(conn, shared_links |> Enum.map(& &1.name)) + end + end + + def get_related_shared_links(%Plug.Conn{} = conn, _params), do: invalid_request(conn) + + @spec normalize_segment_id_param(any()) :: nil | pos_integer() + defp normalize_segment_id_param(input) do + case Integer.parse(input) do + {int_value, ""} when int_value > 0 -> int_value + _ -> nil + end + end + + defp segment_not_found(%Plug.Conn{} = conn, segment_id_param) do + H.not_found(conn, "Segment not found with ID #{inspect(segment_id_param)}") + end + + defp invalid_request(%Plug.Conn{} = conn) do + H.bad_request(conn, "Invalid request") + end +end diff --git a/lib/plausible_web/controllers/api/internal_controller.ex b/lib/plausible_web/controllers/api/internal_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..821fe58e58b6bd2de8c698f888d96e36838cd48d --- /dev/null +++ b/lib/plausible_web/controllers/api/internal_controller.ex @@ -0,0 +1,80 @@ +defmodule PlausibleWeb.Api.InternalController do + use PlausibleWeb, :controller + use Plausible.Repo + import Ecto.Query + alias Plausible.{Sites, Auth} + alias Plausible.Auth.User + alias Plausible.Teams + + def sites(conn, _params) do + current_user = conn.assigns[:current_user] + current_team = conn.assigns[:current_team] + + if current_user do + sites = sites_for(current_user, current_team) + + json(conn, %{data: sites}) + else + PlausibleWeb.Api.Helpers.unauthorized( + conn, + "You need to be logged in to request a list of sites" + ) + end + end + + @features %{ + "funnels" => Plausible.Billing.Feature.Funnels, + "props" => Plausible.Billing.Feature.Props, + "conversions" => Plausible.Billing.Feature.Goals + } + def disable_feature(conn, %{"domain" => domain, "feature" => feature}) do + with %User{id: user_id} = user <- conn.assigns[:current_user], + site <- Sites.get_by_domain(domain), + true <- + Plausible.Teams.Memberships.has_editor_access?(site, user) || + Auth.super_admin?(user_id), + {:ok, mod} <- Map.fetch(@features, feature), + {:ok, _site} <- mod.toggle(site, user, override: false) do + json(conn, "ok") + else + {:error, :upgrade_required} -> + PlausibleWeb.Api.Helpers.payment_required( + conn, + "This feature is part of the Plausible Business plan. To get access to this feature, please upgrade your account" + ) + + :error -> + PlausibleWeb.Api.Helpers.bad_request( + conn, + "The feature you tried to disable is not valid. Valid features are: #{@features |> Map.keys() |> Enum.join(", ")}" + ) + + _ -> + PlausibleWeb.Api.Helpers.unauthorized( + conn, + "You need to be logged in as the owner or admin account of this site" + ) + end + end + + defp sites_for(user, team) do + from(u in subquery(Teams.Sites.accessible_by(user, team)), + inner_join: s in ^Plausible.Site.regular(), + on: u.site_id == s.id, + left_join: up in Plausible.Site.UserPreference, + on: up.site_id == s.id and up.user_id == ^user.id, + select: %{domain: s.domain}, + order_by: [ + asc: + fragment( + "CASE WHEN ? IS NOT NULL THEN 'pinned_site' ELSE 'site' END", + up.pinned_at + ), + desc: up.pinned_at, + asc: s.domain + ], + limit: 9 + ) + |> Repo.all() + end +end diff --git a/lib/plausible_web/controllers/api/paddle_controller.ex b/lib/plausible_web/controllers/api/paddle_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..132c0df283ede5f03d6bb326c88eb3a618fc9847 --- /dev/null +++ b/lib/plausible_web/controllers/api/paddle_controller.ex @@ -0,0 +1,118 @@ +defmodule PlausibleWeb.Api.PaddleController do + use PlausibleWeb, :controller + use Plausible.Repo + require Logger + + plug :verify_signature when action in [:webhook] + + def webhook(conn, %{"alert_name" => "subscription_created"} = params) do + Plausible.Billing.subscription_created(params) + |> webhook_response(conn, params) + end + + def webhook(conn, %{"alert_name" => "subscription_updated"} = params) do + Plausible.Billing.subscription_updated(params) + |> webhook_response(conn, params) + end + + def webhook(conn, %{"alert_name" => "subscription_cancelled"} = params) do + Plausible.Billing.subscription_cancelled(params) + |> webhook_response(conn, params) + end + + def webhook(conn, %{"alert_name" => "subscription_payment_succeeded"} = params) do + Plausible.Billing.subscription_payment_succeeded(params) + |> webhook_response(conn, params) + end + + def webhook(conn, _params) do + send_resp(conn, 404, "") |> halt + end + + @default_currency_fallback :EUR + + def currency(conn, _params) do + plan_id = get_currency_reference_plan_id() + customer_ip = PlausibleWeb.RemoteIP.get(conn) + + result = + Plausible.Cache.Adapter.fetch(:customer_currency, {plan_id, customer_ip}, fn -> + case Plausible.Billing.PaddleApi.fetch_prices([plan_id], customer_ip) do + {:ok, %{^plan_id => money}} -> + {:ok, money.currency} + + error -> + Sentry.capture_message("Failed to fetch currency reference plan", + extra: %{error: inspect(error)} + ) + + {:error, :fetch_prices_failed} + end + end) + + case result do + {:ok, currency} -> + conn + |> put_status(200) + |> json(%{currency: Cldr.Currency.currency_for_code!(currency).narrow_symbol}) + + {:error, :fetch_prices_failed} -> + conn + |> put_status(200) + |> json(%{ + currency: Cldr.Currency.currency_for_code!(@default_currency_fallback).narrow_symbol + }) + end + end + + def verify_signature(conn, _opts) do + signature = Base.decode64!(conn.params["p_signature"]) + + msg = + Map.delete(conn.params, "p_signature") + |> Enum.map(fn {key, val} -> {key, "#{val}"} end) + |> List.keysort(0) + |> PhpSerializer.serialize() + + [key_entry] = :public_key.pem_decode(get_paddle_key()) + + public_key = :public_key.pem_entry_decode(key_entry) + + if :public_key.verify(msg, :sha, signature, public_key) do + conn + else + send_resp(conn, 400, "") |> halt + end + end + + @paddle_currency_reference_plan_id "857097" + @paddle_sandbox_currency_reference_plan_id "63842" + defp get_currency_reference_plan_id() do + if Application.get_env(:plausible, :environment) == "staging" do + @paddle_sandbox_currency_reference_plan_id + else + @paddle_currency_reference_plan_id + end + end + + @paddle_prod_key File.read!("priv/paddle.pem") + @paddle_sandbox_key File.read!("priv/paddle_sandbox.pem") + + defp get_paddle_key() do + if Application.get_env(:plausible, :environment) == "staging" do + @paddle_sandbox_key + else + @paddle_prod_key + end + end + + defp webhook_response({:ok, _}, conn, _params) do + json(conn, "") + end + + defp webhook_response({:error, details}, conn, _params) do + Logger.error("Error processing Paddle webhook: #{inspect(details)}") + + conn |> send_resp(400, "") |> halt + end +end diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..51b5ebcfcf830f6c0df16b4d0323694f0060f230 --- /dev/null +++ b/lib/plausible_web/controllers/api/stats_controller.ex @@ -0,0 +1,431 @@ +defmodule PlausibleWeb.Api.StatsController do + use Plausible + use PlausibleWeb, :controller + use Plausible.Repo + use PlausibleWeb.Plugs.ErrorHandler + + alias Plausible.Stats + + alias Plausible.Stats.{Query, Dashboard, ParsedQueryParams, QueryBuilder, QueryError} + + alias Plausible.Stats.Dashboard.CsvExport + alias PlausibleWeb.Api.Helpers, as: H + + require Logger + + plug(:date_validation_plug when action not in [:query, :csv_export_v2]) + plug(:validate_required_filters_plug when action not in [:current_visitors]) + + def query(conn, params) do + site = conn.assigns.site + now = conn.private[:now] + + with {:ok, %ParsedQueryParams{} = params} <- Dashboard.QueryParser.parse(params, now: now), + {:ok, %Query{} = query} <- QueryBuilder.build(site, params, debug_metadata(conn)) do + query = + if query.include.time_labels do + Query.set_include(query, :time_label_result_indices, true) + else + query + end + + json(conn, Plausible.Stats.query(site, query)) + else + {:error, %QueryError{message: message}} -> H.bad_request(conn, message) + end + end + + def csv_export(conn, params) do + site = conn.assigns.site + + case CsvExport.get_csvs(site, params, debug_metadata(conn)) do + {:ok, csvs} -> + {:ok, {_, zip_content}} = :zip.create(~c"export.zip", csvs, [:memory]) + + conn + |> put_resp_content_type("application/zip") + |> put_resp_header( + "content-disposition", + Plausible.Exports.content_disposition("export.zip") + ) + |> send_resp(200, zip_content) + + {:error, %QueryError{message: message}} -> + H.bad_request(conn, message) + end + end + + on_ee do + alias Plausible.Stats.Exploration + + if Mix.env() == :e2e_test do + @exploration_hourly_limit 100_000 + @exploration_burst_limit 100_000 + else + @exploration_hourly_limit 600 + @exploration_burst_limit 10 + end + + defp check_exploration_rate_limit(site) do + key = "exploration:#{site.id}" + + with {:allow, _} <- + Plausible.RateLimit.check_rate(key, :timer.hours(1), @exploration_hourly_limit), + {:allow, _} <- + Plausible.RateLimit.check_rate(key, :timer.seconds(10), @exploration_burst_limit) do + :ok + else + {:deny, _} -> {:error, :rate_limit} + end + end + + def exploration_next(conn, %{"journey" => steps} = params) do + site = conn.assigns.site + search_term = params["search_term"] || "" + + with :ok <- check_exploration_rate_limit(site), + {:ok, journey} <- parse_journey(steps), + {:ok, direction} <- parse_exploration_direction(params["direction"]), + query = Query.from(site, params, debug_metadata: debug_metadata(conn)), + {:ok, next_steps} <- + Exploration.next_steps(site, query, journey, + search_term: search_term, + direction: direction + ) do + json(conn, next_steps) + else + {:error, :rate_limit} -> + H.too_many_requests(conn, "Too many exploration requests") + + {:error, :journey_too_long} -> + H.bad_request(conn, "The journey is too long") + end + end + + def exploration_funnel(conn, %{"journey" => steps} = params) do + site = conn.assigns.site + + with :ok <- check_exploration_rate_limit(site), + {:ok, journey} <- parse_journey(steps), + {:ok, direction} <- parse_exploration_direction(params["direction"]), + query = Query.from(site, params, debug_metadata: debug_metadata(conn)), + {:ok, funnel} <- + Exploration.journey_funnel(query, journey, direction) do + json(conn, funnel) + else + {:error, :rate_limit} -> + H.too_many_requests(conn, "Too many exploration requests") + + {:error, :empty_journey} -> + H.bad_request(conn, "We are unable to show funnels when journey is empty") + + {:error, :journey_too_long} -> + H.bad_request(conn, "The journey is too long") + end + end + + @exploration_max_candidates 50 + + def exploration_next_with_funnel(conn, %{"journey" => steps} = params) do + site = conn.assigns.site + search_term = params["search_term"] || "" + include_funnel? = params["include_funnel"] == true + + with :ok <- check_exploration_rate_limit(site), + {:ok, journey} <- parse_journey(steps), + {:ok, direction} <- parse_exploration_direction(params["direction"]), + query = Query.from(site, params, debug_metadata: debug_metadata(conn)), + {:ok, next_steps} <- + Exploration.next_steps(site, query, journey, + search_term: search_term, + direction: direction, + max_candidates: @exploration_max_candidates + ), + funnel <- maybe_include_funnel(include_funnel?, query, journey, direction) do + json(conn, %{next: next_steps, funnel: funnel}) + else + {:error, :rate_limit} -> + H.too_many_requests(conn, "Too many exploration requests") + + _ -> + H.bad_request(conn, "There was an error with your request") + end + end + + defp maybe_include_funnel(true, query, journey, direction) do + case Exploration.journey_funnel(query, journey, direction) do + {:ok, funnel_data} -> funnel_data + {:error, :empty_journey} -> [] + end + end + + defp maybe_include_funnel(false, _, _, _), do: [] + + defp parse_journey(input) when is_binary(input) do + input + |> Jason.decode!() + |> Enum.map(&parse_journey_step/1) + |> then(&{:ok, &1}) + end + + defp parse_journey_step(%{ + "name" => name, + "pathname" => pathname, + "includes_subpaths" => includes_subpaths, + "subpaths_count" => subpaths_count, + "is_goal" => is_goal + }) do + Exploration.Journey.Step.new( + name, + pathname, + includes_subpaths, + subpaths_count, + is_goal + ) + end + + defp parse_exploration_direction("backward"), do: {:ok, :backward} + defp parse_exploration_direction("forward"), do: {:ok, :forward} + defp parse_exploration_direction(_), do: {:ok, :forward} + end + + on_ee do + def funnel(conn, %{"id" => funnel_id} = params) do + site = Plausible.Repo.preload(conn.assigns.site, :team) + + with :ok <- Plausible.Billing.Feature.Funnels.check_availability(site.team), + query <- Query.from(site, params, debug_metadata: debug_metadata(conn)), + :ok <- validate_funnel_query(query), + {funnel_id, ""} <- Integer.parse(funnel_id), + {:ok, funnel} <- Stats.funnel(site, query, funnel_id) do + json(conn, funnel) + else + {:error, {:invalid_funnel_query, due_to}} -> + H.bad_request( + conn, + "We are unable to show funnels when the dashboard is filtered by #{due_to}", + %{ + level: :normal + } + ) + + {:error, :funnel_not_found} -> + conn + |> put_status(404) + |> json(%{error: "Funnel not found"}) + |> halt() + + {:error, :upgrade_required} -> + H.payment_required( + conn, + "#{Plausible.Billing.Feature.Funnels.display_name()} is part of the Plausible Business plan. To get access to this feature, please upgrade your account." + ) + + _ -> + H.bad_request(conn, "There was an error with your request") + end + end + + defp validate_funnel_query(query) do + cond do + toplevel_goal_filter?(query) -> + {:error, {:invalid_funnel_query, "goals"}} + + Plausible.Stats.Filters.filtering_on_dimension?(query, "event:page") -> + {:error, {:invalid_funnel_query, "pages"}} + + query.input_date_range == :realtime -> + {:error, {:invalid_funnel_query, "realtime period"}} + + true -> + :ok + end + end + end + + def google_search_terms(conn, params) do + site = conn.assigns[:site] + + query = Query.from(site, params, debug_metadata: debug_metadata(conn)) + + is_admin = + if current_user = conn.assigns[:current_user] do + Plausible.Teams.Memberships.has_editor_access?(site, current_user) + else + false + end + + pagination = { + to_int(params["limit"], 9), + to_int(params["page"], 0) + } + + search = params["search"] || "" + + not_configured_error_payload = %{error_code: :not_configured, is_admin: is_admin} + + search_terms = google_api().fetch_stats(site, query, pagination, search) + period_too_recent? = DateTime.diff(query.now, query.utc_time_range.first, :hour) < 72 + + case {search_terms, period_too_recent?} do + {{:error, :google_property_not_configured}, _} -> + conn + |> put_status(422) + |> json(not_configured_error_payload) + + {{:error, :unsupported_filters}, _} -> + conn + |> put_status(422) + |> json(%{error_code: :unsupported_filters}) + + {{:ok, []}, _period_too_recent? = true} -> + # We consider this an error case because Google Search Console + # data is usually delayed 1-3 days. + conn + |> put_status(422) + |> json(%{error_code: :period_too_recent}) + + {{:ok, terms}, _} -> + json(conn, %{results: terms}) + + {{:error, error}, _} -> + Logger.error("Plausible.Google.API.fetch_stats failed with error: `#{inspect(error)}`") + + conn + |> put_status(502) + |> json(not_configured_error_payload) + end + end + + def current_visitors(conn, _) do + site = conn.assigns[:site] + json(conn, Stats.current_visitors(site)) + end + + defp google_api(), do: Application.fetch_env!(:plausible, :google_api) + + def filter_suggestions(conn, params) do + site = conn.assigns[:site] + search_query = params["q"] + + if is_nil(search_query) do + conn + |> put_status(:bad_request) + |> json(%{"error" => "Search parameter 'q' is required"}) + else + query = Query.from(site, params, debug_metadata: debug_metadata(conn)) + + json( + conn, + Stats.filter_suggestions(site, query, params["filter_name"], search_query) + ) + end + end + + def custom_prop_value_filter_suggestions(conn, %{"prop_key" => prop_key} = params) do + site = conn.assigns[:site] + + query = Query.from(site, params, debug_metadata: debug_metadata(conn)) + + json( + conn, + Stats.custom_prop_value_filter_suggestions(site, query, prop_key, params["q"]) + ) + end + + defp to_int(string, default) when is_binary(string) do + case Integer.parse(string) do + {i, ""} when is_integer(i) -> + i + + _ -> + default + end + end + + defp to_int(_, default), do: default + + defp date_validation_plug(conn, _opts) do + case parse_date_params(conn.params) do + {:ok, _dates} -> conn + {:error, message} when is_binary(message) -> H.bad_request(conn, message) + end + end + + defp validate_required_filters_plug( + %Plug.Conn{assigns: %{shared_link: %Plausible.Site.SharedLink{segment_id: segment_id}}} = + conn, + _opts + ) + when is_integer(segment_id) do + case conn.params + |> get_filters_param() + |> ensure_expected_segment_filter_present(segment_id) do + :ok -> + conn + + :error -> + H.bad_request( + conn, + "The first filter must be for the segment with id #{segment_id}" + ) + end + end + + defp validate_required_filters_plug(conn, _opts), do: conn + + defp get_filters_param(%{"filters" => filters} = _params) when is_binary(filters) do + JSON.decode!(filters) + end + + defp get_filters_param(%{"filters" => filters} = _params) when is_list(filters) do + filters + end + + defp get_filters_param(_params) do + nil + end + + defp ensure_expected_segment_filter_present( + filters, + expected_segment_id + ) + when is_list(filters) do + case filters do + [["is", "segment", [segment_id]] | _other_filters] when segment_id == expected_segment_id -> + :ok + + _ -> + :error + end + end + + defp ensure_expected_segment_filter_present(_filters, _expected_segment_id) do + :error + end + + defp parse_date_params(params) do + params + |> Map.take(["from", "to", "date"]) + |> Enum.reduce_while({:ok, %{}}, fn {key, value}, {:ok, acc} -> + case Date.from_iso8601(value) do + {:ok, date} -> + {:cont, {:ok, Map.put(acc, key, date)}} + + _ -> + {:halt, + {:error, + "Failed to parse '#{key}' argument. Only ISO 8601 dates are allowed, e.g. `2019-09-07`, `2020-01-01`"}} + end + end) + end + + on_ee do + defp toplevel_goal_filter?(query) do + Plausible.Stats.Filters.filtering_on_dimension?(query, "event:goal", + max_depth: 0, + behavioral_filters: :ignore + ) + end + end +end diff --git a/lib/plausible_web/controllers/api/system_controller.ex b/lib/plausible_web/controllers/api/system_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..db9ff4add1b34b993285bf17e7d7912040eb4c45 --- /dev/null +++ b/lib/plausible_web/controllers/api/system_controller.ex @@ -0,0 +1,135 @@ +defmodule PlausibleWeb.Api.SystemController do + use Plausible + use PlausibleWeb, :controller + require Logger + + def info(conn, _params) do + build = + :plausible + |> Application.get_env(:runtime_metadata) + |> Keyword.take([:version, :commit, :created, :tags]) + |> Map.new() + + geo_database = Plausible.Geo.database_type() || "(not configured)" + + json(conn, %{ + geo_database: geo_database, + build: build + }) + end + + def liveness(conn, _params) do + json(conn, %{ok: true}) + end + + def api_docs(conn, _params) do + json(conn, %{ + endpoints: [ + %{ + method: "GET", + path: "/health", + purpose: "Returns HTTP 200 when the app is ready for Hugging Face Space transition from starting to running." + }, + %{ + method: "GET", + path: "/api-docs", + purpose: "Exposes documentation of all available API endpoints." + }, + %{ + method: "POST", + path: "/api/event", + purpose: "Send analytical events directly via the API." + }, + %{ + method: "GET", + path: "/api/v1/stats/realtime/visitors", + purpose: "Get current number of active visitors on a site." + }, + %{ + method: "GET", + path: "/api/v1/stats/aggregate", + purpose: "Retrieve aggregate metrics over a given period." + }, + %{ + method: "GET", + path: "/api/v1/stats/breakdown", + purpose: "Break down stats by metrics like browser, country, etc." + }, + %{ + method: "GET", + path: "/api/v1/stats/timeseries", + purpose: "Get metrics over time at regular intervals." + } + ] + }) + end + + @task_timeout 15_000 + @critical_caches [ + Plausible.Site.Cache, + Plausible.Shield.IPRuleCache, + on_ee do + Plausible.Site.TrackerScriptIdCache + else + Plausible.Site.TrackerScriptCache + end + ] + + def readiness(conn, _params) do + postgres_health_task = + Task.async(fn -> + Ecto.Adapters.SQL.query(Plausible.Repo, "SELECT 1", []) + end) + + clickhouse_health_task = + Task.async(fn -> + Ecto.Adapters.SQL.query(Plausible.ClickhouseRepo, "SELECT 1", []) + end) + + postgres_health = + case Task.await(postgres_health_task, @task_timeout) do + {:ok, _} -> + "ok" + + e -> + Logger.error("Postgres health check failure: #{inspect(e)}") + "error" + end + + clickhouse_health = + case Task.await(clickhouse_health_task, @task_timeout) do + {:ok, _} -> + "ok" + + e -> + Logger.error("Clickhouse health check failure: #{inspect(e)}") + "error" + end + + cache_health = + if postgres_health == "ok" and Enum.all?(@critical_caches, & &1.ready?()) do + "ok" + end + + sessions_health = + if Plausible.Session.Transfer.attempted?() do + "ok" + else + "waiting" + end + + status = + case {postgres_health, clickhouse_health, cache_health, sessions_health} do + {"ok", "ok", "ok", "ok"} -> 200 + _ -> 500 + end + + put_status(conn, status) + |> json(%{ + postgres: postgres_health, + clickhouse: clickhouse_health, + sites_cache: cache_health, + sessions: sessions_health + }) + end +end diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..13ae747473d439234f6f5147d383ad69298c2892 --- /dev/null +++ b/lib/plausible_web/controllers/auth_controller.ex @@ -0,0 +1,768 @@ +defmodule PlausibleWeb.AuthController do + use PlausibleWeb, :controller + use Plausible.Repo + use Plausible + + alias Plausible.Auth + alias Plausible.Teams + alias PlausibleWeb.TwoFactor + alias PlausibleWeb.UserAuth + alias PlausibleWeb.LoginPreference + + require Logger + + plug( + PlausibleWeb.RequireLoggedOutPlug + when action in [ + :register, + :register_from_invitation, + :login_form, + :login, + :verify_2fa_form, + :verify_2fa, + :verify_2fa_recovery_code_form, + :verify_2fa_recovery_code + ] + ) + + plug( + PlausibleWeb.RequireAccountPlug + when action in [ + :delete_me, + :activate_form, + :activate, + :request_activation_code, + :force_initiate_2fa_setup, + :initiate_2fa_setup, + :verify_2fa_setup_form, + :verify_2fa_setup, + :disable_2fa, + :generate_2fa_recovery_codes, + :switch_team + ] + ) + + plug Plausible.Plugs.RestrictUserType, + [deny: :sso] when action in [:delete_me, :disable_2fa] + + plug( + :clear_2fa_user + when action not in [ + :verify_2fa_form, + :verify_2fa, + :verify_2fa_recovery_code_form, + :verify_2fa_recovery_code + ] + ) + + # Plug purging 2FA user session cookie outsite 2FA flow + defp clear_2fa_user(conn, _opts) do + TwoFactor.Session.clear_2fa_user(conn) + end + + def activate_form(conn, params) do + user = conn.assigns.current_user + flow = params["flow"] || PlausibleWeb.Flows.register() + team_identifier = params["team_identifier"] + has_email_code? = Plausible.Users.has_email_code?(user) + has_any_memberships? = Plausible.Teams.Users.has_sites?(user) + + render_auth_page(conn, "activate.html", + error: nil, + has_email_code?: has_email_code?, + has_any_memberships?: has_any_memberships?, + heading: activate_heading(has_email_code?), + form_submit_url: "/activate?flow=#{flow}", + team_identifier: team_identifier + ) + end + + defp activate_heading(true), do: "Check your email" + defp activate_heading(false), do: "Activate your account" + + def activate(conn, %{"code" => code}) do + user = conn.assigns[:current_user] + + with :ok <- Auth.rate_limit(:activation_ip, conn), + :ok <- Auth.rate_limit(:activation_user, user) do + do_activate(conn, user, code) + else + {:error, {:rate_limit, _}} -> + has_any_memberships? = Plausible.Teams.Users.has_sites?(user, include_pending?: false) + flow = conn.params["flow"] + team_identifier = conn.params["team_identifier"] + + render_auth_page(conn, "activate.html", + error: "Too many attempts. Please wait a few minutes before trying again.", + has_email_code?: true, + has_any_memberships?: has_any_memberships?, + heading: activate_heading(true), + form_submit_url: "/activate?flow=#{flow}", + team_identifier: team_identifier + ) + end + end + + defp do_activate(conn, user, code) do + has_any_invitations? = Plausible.Teams.Users.has_sites?(user, include_pending?: true) + has_any_memberships? = Plausible.Teams.Users.has_sites?(user, include_pending?: false) + + flow = conn.params["flow"] + team_identifier = conn.params["team_identifier"] + + case Auth.EmailVerification.verify_code(user, code) do + :ok -> + cond do + team_identifier not in ["", nil] -> + redirect_path = accept_team_invitation(conn, team_identifier, user, flow: flow) + redirect(conn, to: redirect_path) + + has_any_memberships? -> + handle_email_updated(conn) + + has_any_invitations? -> + redirect_path = accept_team_invitation(conn, team_identifier, user, flow: flow) + redirect(conn, to: redirect_path) + + true -> + redirect(conn, to: Routes.site_path(conn, :new, flow: flow)) + end + + {:error, :incorrect} -> + render_auth_page(conn, "activate.html", + error: "That code didn't work. Please try again.", + has_email_code?: true, + has_any_memberships?: has_any_memberships?, + heading: activate_heading(true), + form_submit_url: "/activate?flow=#{flow}", + team_identifier: team_identifier + ) + + {:error, :expired} -> + render_auth_page(conn, "activate.html", + error: "The code has expired. Please request another one.", + has_email_code?: false, + has_any_memberships?: has_any_memberships?, + heading: activate_heading(false), + form_submit_url: "/activate?flow=#{flow}", + team_identifier: team_identifier + ) + end + end + + def request_activation_code(conn, _params) do + user = conn.assigns.current_user + + with :ok <- Auth.rate_limit(:activation_request_ip, conn), + :ok <- Auth.rate_limit(:activation_request_user, user) do + Auth.EmailVerification.issue_code(user) + + conn + |> put_flash(:success, "Activation code was sent to #{user.email}") + |> redirect(to: Routes.auth_path(conn, :activate_form)) + else + {:error, {:rate_limit, _}} -> + conn + |> put_flash(:error, "Too many code requests. Please wait before requesting another.") + |> redirect(to: Routes.auth_path(conn, :activate_form)) + end + end + + def password_reset_request_form(conn, _) do + render_password_reset_request_form(conn) + end + + def password_reset_request(conn, %{"email" => ""}) do + render_password_reset_request_form(conn, error: "Please enter an email address") + end + + def password_reset_request(conn, %{"email" => email} = params) do + if PlausibleWeb.Captcha.verify(params["h-captcha-response"]) do + case Auth.lookup(email) do + {:ok, _user} -> + token = Auth.Token.sign_password_reset(email) + url = PlausibleWeb.Endpoint.url() <> "/password/reset?token=#{token}" + email_template = PlausibleWeb.Email.password_reset_email(email, url) + Plausible.Mailer.deliver_later(email_template) + + Logger.debug( + "Password reset e-mail sent. In dev environment GET /sent-emails for details." + ) + + render_auth_page(conn, "password_reset_request_success.html", + heading: "Check your email", + email: email + ) + + {:error, _} -> + render_auth_page(conn, "password_reset_request_success.html", + heading: "Check your email", + email: email + ) + end + else + render_password_reset_request_form(conn, + captcha_error: "Please complete the captcha to reset your password" + ) + end + end + + defp render_password_reset_request_form(conn, extra_assigns \\ []) do + render_auth_page( + conn, + "password_reset_request_form.html", + Keyword.merge( + [ + heading: "Reset your password", + subtitle: "Enter your email to receive a password reset link." + ], + extra_assigns + ) + ) + end + + def password_reset_form(conn, params) do + case Auth.Token.verify_password_reset(params["token"]) do + {:ok, %{email: email}} -> + render_auth_page(conn, "password_reset_form.html", + connect_live_socket: true, + heading: "Reset your password", + subtitle: "Choose a new password for your account", + email: email + ) + + {:error, :expired} -> + conn + |> put_status(401) + |> render_auth_page("password_reset_error.html", + heading: "Password reset link expired", + subtitle: "Request a new one to reset your password." + ) + + {:error, _} -> + conn + |> put_status(401) + |> render_auth_page("password_reset_error.html", + heading: "Password reset link invalid", + subtitle: "Request a new one to reset your password." + ) + end + end + + def password_reset(conn, _params) do + conn + |> UserAuth.log_out_user() + |> put_flash(:login_title, "Password updated successfully") + |> put_flash(:login_instructions, "Please sign in with your new credentials") + |> redirect(to: Routes.auth_path(conn, :login_form)) + end + + on_ee do + def login_form(conn, params) do + login_preference = LoginPreference.get(conn) + error = Phoenix.Flash.get(conn.assigns.flash, :login_error) + + case {login_preference, params["prefer"], error} do + {"sso", nil, nil} -> + redirect(conn, to: Routes.sso_path(conn, :login_form, return_to: params["return_to"])) + + _ -> + render_login_form(conn) + end + end + else + def login_form(conn, _params) do + render_login_form(conn) + end + end + + defp render_login_form(conn) do + heading = Phoenix.Flash.get(conn.assigns.flash, :login_title) || "Sign in to your account" + subtitle = Phoenix.Flash.get(conn.assigns.flash, :login_instructions) + + render_auth_page(conn, "login_form.html", heading: heading, subtitle: subtitle) + end + + def login(conn, %{"user" => params}) do + login(conn, params) + end + + def login(conn, %{"email" => email, "password" => password} = params) do + with :ok <- Auth.rate_limit(:login_ip, conn), + {:ok, user} <- Auth.lookup(email), + :ok <- Auth.rate_limit(:login_user, user), + :ok <- Auth.check_password(user, password), + :ok <- check_2fa_verified(conn, user) do + redirect_path = + cond do + not is_nil(params["register_action"]) and not user.email_verified -> + Auth.EmailVerification.issue_code(user) + + flow = + if params["register_action"] == "register_form" do + PlausibleWeb.Flows.register() + else + PlausibleWeb.Flows.invitation() + end + + Routes.auth_path(conn, :activate_form, + flow: flow, + team_identifier: params["team_identifier"] + ) + + params["register_action"] == "register_from_invitation_form" -> + accept_team_invitation(conn, params["team_identifier"], user) + + params["register_action"] == "register_form" -> + Routes.site_path(conn, :new) + + true -> + params["return_to"] + end + + conn + |> LoginPreference.clear() + |> UserAuth.log_in_user(user, redirect_path) + else + {:error, :wrong_password} -> + Auth.log_failed_login_attempt("wrong password for #{email}") + + conn + |> put_flash(:login_error, "Incorrect email or password. Please try again.") + |> render_login_form() + + {:error, :user_not_found} -> + Auth.log_failed_login_attempt("user not found for #{email}") + Plausible.Auth.Password.dummy_calculation() + + conn + |> put_flash(:login_error, "Incorrect email or password. Please try again.") + |> render_login_form() + + {:error, {:rate_limit, _}} -> + Auth.log_failed_login_attempt("too many login attempts for #{email}") + + render_error( + conn, + 429, + "Too many login attempts. Wait a minute before trying again." + ) + + {:error, {:unverified_2fa, user}} -> + query_params = + if params["return_to"] not in [nil, ""], do: [return_to: params["return_to"]], else: [] + + conn + |> TwoFactor.Session.set_2fa_user(user) + |> redirect(to: Routes.auth_path(conn, :verify_2fa, query_params)) + end + end + + def login(conn, _params) do + conn |> send_resp(403, "") |> halt() + end + + defp accept_team_invitation(conn, team_identifier, user, params \\ []) + + defp accept_team_invitation(conn, no_identifier, _user, params) + when no_identifier in ["", nil] do + Routes.site_path(conn, :index, params) + end + + defp accept_team_invitation(conn, team_identifier, user, extra_params) do + params = Keyword.merge([__team: team_identifier], extra_params) + + # We try switching to the team no matter the invitation presence or acceptance outcome. + case Teams.Invitations.find_by_team_identifier(team_identifier, user) do + {:ok, invitation} -> + {_, _} = Teams.Invitations.accept_team_invitation(invitation, user) + Routes.site_path(conn, :index, params) + + {:error, :invitation_not_found} -> + Routes.site_path(conn, :index, params) + end + end + + defp check_2fa_verified(conn, user) do + if Auth.TOTP.enabled?(user) and not TwoFactor.Session.remember_2fa?(conn, user) do + {:error, {:unverified_2fa, user}} + else + :ok + end + end + + def force_initiate_2fa_setup(conn, _params) do + render(conn, "force_initiate_2fa_setup.html") + end + + def initiate_2fa_setup(conn, params) do + case Auth.TOTP.initiate(conn.assigns.current_user) do + {:ok, user, %{totp_uri: totp_uri, secret: secret}} -> + render(conn, "initiate_2fa_setup.html", + user: user, + totp_uri: totp_uri, + secret: secret, + forced?: params["force"] == "true" + ) + + {:error, :already_setup} -> + conn + |> put_flash(:error, "Two-Factor Authentication is already setup for this account.") + |> redirect(to: Routes.settings_path(conn, :security) <> "#update-2fa") + end + end + + def verify_2fa_setup_form(conn, _params) do + if Auth.TOTP.initiated?(conn.assigns.current_user) do + render(conn, "verify_2fa_setup.html") + else + redirect(conn, to: Routes.settings_path(conn, :security) <> "#update-2fa") + end + end + + def verify_2fa_setup(conn, %{"code" => code}) do + user = conn.assigns.current_user + + with :ok <- Auth.rate_limit(:totp_setup_ip, conn), + :ok <- Auth.rate_limit(:totp_setup_user, user), + {:ok, _, %{recovery_codes: codes}} <- Auth.TOTP.enable(user, code) do + conn + |> put_flash(:success, "Two-Factor Authentication is fully enabled") + |> render("generate_2fa_recovery_codes.html", recovery_codes: codes, from_setup: true) + else + {:error, {:rate_limit, _}} -> + render_error( + conn, + 429, + "Too many attempts. Wait a minute before trying again." + ) + + {:error, :invalid_code} -> + conn + |> put_flash(:error, "The provided code is invalid. Please try again") + |> render("verify_2fa_setup.html") + + {:error, :not_initiated} -> + conn + |> put_flash(:error, "Please enable Two-Factor Authentication for this account first.") + |> redirect(to: Routes.settings_path(conn, :security) <> "#update-2fa") + end + end + + def disable_2fa(conn, %{"password" => password}) do + case Auth.TOTP.disable(conn.assigns.current_user, password) do + {:ok, _} -> + conn + |> TwoFactor.Session.clear_remember_2fa() + |> put_flash(:success, "Two-Factor Authentication is disabled") + |> redirect(to: Routes.settings_path(conn, :security) <> "#update-2fa") + + {:error, :invalid_password} -> + conn + |> put_flash(:error, "Incorrect password provided") + |> redirect(to: Routes.settings_path(conn, :security) <> "#update-2fa") + end + end + + def generate_2fa_recovery_codes(conn, %{"password" => password}) do + case Auth.TOTP.generate_recovery_codes(conn.assigns.current_user, password) do + {:ok, codes} -> + conn + |> put_flash(:success, "New Recovery Codes generated") + |> render("generate_2fa_recovery_codes.html", recovery_codes: codes, from_setup: false) + + {:error, :invalid_password} -> + conn + |> put_flash(:error, "Incorrect password provided") + |> redirect(to: Routes.settings_path(conn, :security) <> "#update-2fa") + + {:error, :not_enabled} -> + conn + |> put_flash(:error, "Please enable Two-Factor Authentication for this account first.") + |> redirect(to: Routes.settings_path(conn, :security) <> "#update-2fa") + end + end + + def verify_2fa_form(conn, _params) do + case TwoFactor.Session.get_2fa_user(conn) do + {:ok, user} -> + if Auth.TOTP.enabled?(user) do + render_verify_2fa(conn) + else + redirect_to_login(conn) + end + + {:error, :not_found} -> + redirect_to_login(conn) + end + end + + def verify_2fa(conn, %{"code" => code} = params) do + with {:ok, user} <- get_2fa_user_limited(conn) do + case Auth.TOTP.validate_code(user, code) do + {:ok, user} -> + conn + |> TwoFactor.Session.maybe_set_remember_2fa(user, params["remember_2fa"]) + |> UserAuth.log_in_user(user, params["return_to"]) + + {:error, :invalid_code} -> + Auth.log_failed_login_attempt("wrong 2FA verification code provided for #{user.email}") + + render_verify_2fa(conn, "The provided code is invalid. Please try again") + + {:error, :not_enabled} -> + UserAuth.log_in_user(conn, user, params["return_to"]) + end + end + end + + def verify_2fa_recovery_code_form(conn, _params) do + case TwoFactor.Session.get_2fa_user(conn) do + {:ok, user} -> + if Auth.TOTP.enabled?(user) do + render_verify_2fa_recovery_code(conn) + else + redirect_to_login(conn) + end + + {:error, :not_found} -> + redirect_to_login(conn) + end + end + + def verify_2fa_recovery_code(conn, %{"recovery_code" => recovery_code}) do + with {:ok, user} <- get_2fa_user_limited(conn) do + case Auth.TOTP.use_recovery_code(user, recovery_code) do + :ok -> + UserAuth.log_in_user(conn, user) + + {:error, :invalid_code} -> + Auth.log_failed_login_attempt("wrong 2FA recovery code provided for #{user.email}") + + render_verify_2fa_recovery_code( + conn, + "The provided recovery code is invalid. Please try another one" + ) + + {:error, :not_enabled} -> + UserAuth.log_in_user(conn, user) + end + end + end + + defp render_verify_2fa(conn, error \\ nil) do + render_auth_page(conn, "verify_2fa.html", + error: error, + heading: "Enter your 2FA code", + subtitle: "Open your authenticator app and enter the 6-digit code.", + remember_2fa_days: TwoFactor.Session.remember_2fa_days() + ) + end + + defp render_verify_2fa_recovery_code(conn, error \\ nil) do + render_auth_page(conn, "verify_2fa_recovery_code.html", + error: error, + heading: "Use a recovery code", + subtitle: "Enter one of your saved recovery codes to sign in." + ) + end + + defp get_2fa_user_limited(conn) do + case TwoFactor.Session.get_2fa_user(conn) do + {:ok, user} -> + with :ok <- Auth.rate_limit(:login_ip, conn), + :ok <- Auth.rate_limit(:login_user, user) do + {:ok, user} + else + {:error, {:rate_limit, _}} -> + Auth.log_failed_login_attempt("too many login attempts for #{user.email}") + + conn + |> TwoFactor.Session.clear_2fa_user() + |> render_error( + 429, + "Too many login attempts. Wait a minute before trying again." + ) + end + + {:error, :not_found} -> + conn + |> redirect(to: Routes.auth_path(conn, :login_form)) + end + end + + defp handle_email_updated(conn) do + conn + |> put_flash(:success, "Email updated successfully") + |> redirect(to: Routes.settings_path(conn, :security) <> "#update-email") + end + + def delete_me(conn, params) do + case Plausible.Auth.delete_user(conn.assigns[:current_user]) do + {:ok, :deleted} -> + logout(conn, params) + + {:error, :active_subscription} -> + conn + |> put_flash( + :error, + "You have an active subscription which must be canceled first." + ) + |> redirect(to: Routes.settings_path(conn, :danger_zone)) + + {:error, :is_only_team_owner} -> + conn + |> put_flash( + :error, + "You can't delete your account when you are the only owner on a team." + ) + |> redirect(to: Routes.settings_path(conn, :danger_zone)) + end + end + + def logout(conn, params) do + redirect_to = Map.get(params, "redirect", "/") + + conn + |> UserAuth.log_out_user() + |> redirect(to: redirect_to) + end + + def google_auth_callback(conn, %{"error" => error, "state" => state} = params) do + case Plausible.Google.API.verify_oauth_state(state) do + {:ok, %{site: site, context: context}} -> + redirect_url = + if context == "import" do + Routes.site_path(conn, :settings_imports_exports, site.domain) + else + Routes.site_path(conn, :settings_integrations, site.domain) + end + + cond do + error == "access_denied" -> + conn + |> put_flash( + :error, + "We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again." + ) + |> redirect(to: redirect_url) + + error in ["server_error", "temporarily_unavailable"] -> + conn + |> put_flash( + :error, + "We are unable to authenticate your Google Analytics account because Google's authentication service is temporarily unavailable. Please try again in a few moments." + ) + |> redirect(to: redirect_url) + + true -> + Sentry.capture_message("Google OAuth callback failed. Params: #{inspect(params)}") + generic_oauth_error_response(conn, redirect_url) + end + + {:error, reason} -> + Sentry.capture_message( + "Google OAuth callback failed. Reason: #{inspect(reason)}. Params: #{inspect(params)}" + ) + + generic_oauth_error_response(conn) + end + end + + def google_auth_callback(conn, %{"code" => code, "state" => state} = params) do + current_user = conn.assigns[:current_user] + + with {:ok, %{site: site, context: context}} <- Plausible.Google.API.verify_oauth_state(state), + :ok <- check_callback_site_permission(site, current_user) do + token_data = Plausible.Google.API.fetch_access_token!(code) + expires_in = Map.fetch!(token_data, "expires_in") + expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in) + + if context == "import" do + google_import_callback(conn, site, token_data, expires_at) + else + google_search_console_callback(conn, site, token_data, expires_at) + end + else + {:error, reason} -> + Sentry.capture_message( + "Google OAuth callback failed. Reason: #{inspect(reason)}. Params: #{inspect(params)}" + ) + + generic_oauth_error_response(conn) + end + end + + defp google_import_callback(conn, site, token_data, expires_at) do + redirect(conn, + to: + Routes.google_analytics_path(conn, :property_form, site.domain, + access_token: Map.fetch!(token_data, "access_token"), + refresh_token: Map.fetch!(token_data, "refresh_token"), + expires_at: NaiveDateTime.to_iso8601(expires_at) + ) + ) + end + + defp google_search_console_callback(conn, site, token_data, expires_at) do + current_user = conn.assigns.current_user + + id_token = Map.fetch!(token_data, "id_token") + [_, body, _] = String.split(id_token, ".") + id = body |> Base.decode64!(padding: false) |> Jason.decode!() + email = Map.fetch!(id, "email") + refresh_token = Map.fetch!(token_data, "refresh_token") + access_token = Map.fetch!(token_data, "access_token") + + Plausible.Site.GoogleAuth.changeset(%Plausible.Site.GoogleAuth{}, %{ + email: email, + refresh_token: refresh_token, + access_token: access_token, + expires: expires_at, + user_id: current_user.id + }) + |> Ecto.Changeset.put_assoc(:site, site) + |> Ecto.Changeset.put_assoc(:user, current_user) + |> Repo.insert!( + on_conflict: [ + set: [ + email: email, + refresh_token: refresh_token, + access_token: access_token, + expires: expires_at, + user_id: current_user.id, + updated_at: NaiveDateTime.utc_now(:second), + property: nil + ] + ], + conflict_target: :site_id + ) + + redirect(conn, to: Routes.site_path(conn, :settings_integrations, site.domain)) + end + + defp check_callback_site_permission(site, current_user) do + if Plausible.Sites.get_for_user(current_user, site.domain, + roles: [:owner, :admin, :editor, :super_admin] + ) do + :ok + else + {:error, :permission_denied} + end + end + + defp generic_oauth_error_response(conn, redirect_url \\ nil) do + redirect_url = redirect_url || Routes.auth_path(conn, :login_form) + + conn + |> put_flash( + :error, + "We were unable to authenticate your Google Analytics account. If the problem persists, please contact support for assistance." + ) + |> redirect(to: redirect_url) + end + + defp redirect_to_login(conn) do + redirect(conn, to: Routes.auth_path(conn, :login_form)) + end +end diff --git a/lib/plausible_web/controllers/avatar_controller.ex b/lib/plausible_web/controllers/avatar_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..49603f0200a358bdcf60584111a0d1952cdc0bf6 --- /dev/null +++ b/lib/plausible_web/controllers/avatar_controller.ex @@ -0,0 +1,38 @@ +defmodule PlausibleWeb.AvatarController do + @moduledoc """ + This module proxies requests to BASE_URL/avatar/:hash to www.gravatar.com/avatar/:hash. + + The purpose is to make use of Gravatar's convenient avatar service without exposing information + that could be used for tracking the Plausible user. Compared to requesting the Gravatar directly + from the browser, this proxy module protects the Plausible user from disclosing to Gravatar: + 1. The client IP address + 2. User-Agent + 3. Referer header which can be used to track which site the user is visiting (i.e. plausible.io or self-hosted URL) + + The downside is the added latency from the request having to go through the Plausible server, rather than contacting the + local CDN server operated by Gravatar's service. + """ + use PlausibleWeb, :controller + alias Plausible.HTTPClient + + @gravatar_base_url "https://www.gravatar.com" + def avatar(conn, params) do + url = Path.join(@gravatar_base_url, ["avatar/", params["hash"]]) <> "?s=150&d=identicon" + + case HTTPClient.impl().get(url) do + {:ok, %Finch.Response{status: 200, body: body, headers: headers}} -> + conn + |> forward_headers(headers) + |> send_resp(200, body) + + {:error, _} -> + send_resp(conn, 400, "") + end + end + + @forwarded_headers ["content-type", "cache-control", "expires"] + defp forward_headers(%Plug.Conn{} = conn, headers) do + headers_to_forward = Enum.filter(headers, fn {k, _} -> k in @forwarded_headers end) + %Plug.Conn{conn | resp_headers: headers_to_forward} + end +end diff --git a/lib/plausible_web/controllers/billing_controller.ex b/lib/plausible_web/controllers/billing_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..5d65988af1678bede541746a3cee17e69f0f541c --- /dev/null +++ b/lib/plausible_web/controllers/billing_controller.ex @@ -0,0 +1,162 @@ +defmodule PlausibleWeb.BillingController do + use PlausibleWeb, :controller + use Plausible.Repo + + require Logger + require Plausible.Billing.Subscription.Status + require Plausible.Billing + + alias Plausible.Billing + alias Plausible.Billing.Subscription + + plug PlausibleWeb.RequireAccountPlug + + plug Plausible.Plugs.AuthorizeTeamAccess, Billing.allowed_roles() + + def ping_subscription(%Plug.Conn{} = conn, _params) do + subscribed? = Plausible.Teams.Billing.has_active_subscription?(conn.assigns.current_team) + + json(conn, %{is_subscribed: subscribed?}) + end + + def choose_plan(conn, _params) do + team = conn.assigns.current_team + + if Plausible.Teams.Billing.enterprise_configured?(team) do + redirect(conn, to: Routes.billing_path(conn, :upgrade_to_enterprise_plan)) + else + render(conn, "choose_plan.html", + live_module: PlausibleWeb.Live.ChoosePlan, + hide_header?: true, + disable_global_notices?: true, + skip_plausible_tracking: true, + connect_live_socket: true + ) + end + end + + def upgrade_to_enterprise_plan(conn, _params) do + team = conn.assigns.current_team + subscription = Plausible.Teams.Billing.get_subscription(team) + + {latest_enterprise_plan, price} = + Plausible.Teams.Billing.latest_enterprise_plan_with_price( + team, + PlausibleWeb.RemoteIP.get(conn) + ) + + subscription_resumable? = + Plausible.Billing.Subscriptions.resumable?(subscription) + + subscribed_to_latest? = + subscription_resumable? && + subscription.paddle_plan_id == latest_enterprise_plan.paddle_plan_id + + cond do + Subscription.Status.in?(subscription, [ + Subscription.Status.past_due(), + Subscription.Status.paused() + ]) -> + redirect(conn, to: Routes.settings_path(conn, :subscription)) + + subscribed_to_latest? -> + render(conn, "change_enterprise_plan_contact_us.html", skip_plausible_tracking: true) + + true -> + render(conn, "upgrade_to_enterprise_plan.html", + latest_enterprise_plan: latest_enterprise_plan, + price: price, + subscription_resumable: subscription_resumable?, + contact_link: "https://plausible.io/contact", + skip_plausible_tracking: true + ) + end + end + + def upgrade_success(conn, _params) do + render(conn, "upgrade_success.html", disable_global_notices?: true) + end + + def change_plan_preview(conn, %{"plan_id" => new_plan_id}) do + team = conn.assigns.current_team + current_user = conn.assigns.current_user + subscription = Plausible.Teams.Billing.active_subscription_for(team) + + case preview_subscription(subscription, new_plan_id) do + {:ok, {subscription, preview_info}} -> + render(conn, "change_plan_preview.html", + back_link: Routes.billing_path(conn, :choose_plan), + skip_plausible_tracking: true, + subscription: subscription, + preview_info: preview_info + ) + + _ -> + msg = + "Something went wrong with loading your plan change information. Please try again, or contact us at support@plausible.io if the issue persists." + + Sentry.capture_message("Error loading change plan preview", + extra: %{ + message: msg, + new_plan_id: new_plan_id, + team_id: team.id, + user_id: current_user.id + } + ) + + conn + |> put_flash(:error, msg) + |> redirect(to: Routes.billing_path(conn, :choose_plan)) + end + end + + def change_plan(conn, %{"new_plan_id" => new_plan_id}) do + team = conn.assigns.current_team + + case Plausible.Teams.Billing.change_plan(team, new_plan_id) do + {:ok, _subscription} -> + conn + |> put_flash(:success, "Plan changed successfully") + |> redirect(to: Routes.settings_path(conn, :subscription)) + + {:error, e} -> + msg = + case e do + {:over_plan_limits, exceeded_limits} -> + "Unable to subscribe to this plan because the following limits are exceeded: #{PlausibleWeb.TextHelpers.pretty_list(exceeded_limits)}" + + %{"code" => 147} -> + # https://developer.paddle.com/api-reference/intro/api-error-codes + "We were unable to charge your card. Click 'update billing info' to update your payment details and try again." + + %{"message" => msg} when not is_nil(msg) -> + msg + + _ -> + "Something went wrong. Please try again or contact support at support@plausible.io" + end + + Sentry.capture_message("Error changing plans", + extra: %{ + errors: inspect(e), + message: msg, + new_plan_id: new_plan_id, + team_id: team.id, + user_id: conn.assigns.current_user.id + } + ) + + conn + |> put_flash(:error, msg) + |> redirect(to: Routes.settings_path(conn, :subscription)) + end + end + + defp preview_subscription(nil, _new_plan_id), do: {:error, :no_subscription} + + defp preview_subscription(subscription, new_plan_id) do + with {:ok, preview_info} <- Billing.change_plan_preview(subscription, new_plan_id) do + {:ok, {subscription, preview_info}} + end + end +end diff --git a/lib/plausible_web/controllers/debug_controller.ex b/lib/plausible_web/controllers/debug_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..7b807816e12cdd6b2b65835cfca360f2e9c506bb --- /dev/null +++ b/lib/plausible_web/controllers/debug_controller.ex @@ -0,0 +1,73 @@ +defmodule PlausibleWeb.DebugController do + use PlausibleWeb, :controller + use Plausible.IngestRepo + use Plausible + + import Ecto.Query + + plug(PlausibleWeb.RequireAccountPlug) + plug(PlausibleWeb.SuperAdminOnlyPlug) + + def clickhouse(conn, params) do + cluster? = Plausible.IngestRepo.clustered_table?("events_v2") + on_cluster = if(cluster?, do: "ON CLUSTER '{cluster}'", else: "") + + # Ensure last logs are flushed + IngestRepo.query("SYSTEM FLUSH LOGS #{on_cluster}") + + queries = + from( + l in from_query_log(cluster?), + select: %{ + query: fragment("formatQuery(?)", l.query), + log_comment: l.log_comment, + type: l.type, + event_time: l.event_time, + query_duration_ms: l.query_duration_ms, + query_id: l.query_id, + memory_usage: fragment("formatReadableSize(?)", l.memory_usage), + read_bytes: fragment("formatReadableSize(?)", l.read_bytes), + result_bytes: fragment("formatReadableSize(?)", l.result_bytes), + result_rows: l.result_rows + }, + where: l.type > 1 and fragment("event_time > now() - toIntervalMinute(15)"), + order_by: [desc: l.event_time] + ) + |> filter_by_params(conn, params) + |> IngestRepo.all() + |> Enum.map(fn data -> + Jason.decode!(data.log_comment) + |> Map.merge(data) + |> Map.delete(:log_comment) + end) + + conn + |> render("clickhouse.html", + queries: queries + ) + end + + defp from_query_log(cluster?) do + case cluster? do + true -> from(l in fragment("clusterAllReplicas('{cluster}', system.query_log)")) + false -> from(l in fragment("system.query_log")) + end + end + + defp filter_by_params(q, _conn, %{"user_id" => user_id}), + do: where(q, [l], fragment("JSONExtractInt(?, \'user_id\') = ?", l.log_comment, ^user_id)) + + defp filter_by_params(q, _conn, %{"site_id" => site_id}), + do: where(q, [l], fragment("JSONExtractInt(?, \'site_id\') = ?", l.log_comment, ^site_id)) + + defp filter_by_params(q, _conn, %{"site_domain" => site_domain}), + do: + where( + q, + [l], + fragment("JSONExtractString(?, \'site_domain\') = ?", l.log_comment, ^site_domain) + ) + + defp filter_by_params(q, conn, _), + do: filter_by_params(q, conn, %{"user_id" => conn.assigns.current_user.id}) +end diff --git a/lib/plausible_web/controllers/error_report_controller.ex b/lib/plausible_web/controllers/error_report_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..601fe9fc3e05d03bda89c96c749a9737b672f2e7 --- /dev/null +++ b/lib/plausible_web/controllers/error_report_controller.ex @@ -0,0 +1,30 @@ +defmodule PlausibleWeb.ErrorReportController do + use PlausibleWeb, :controller + + plug PlausibleWeb.RequireAccountPlug + + def submit_error_report(conn, %{ + "error" => %{"trace_id" => trace_id, "user_feedback" => feedback} + }) do + if String.length(String.trim(feedback)) > 5 do + reported_by = "#{conn.assigns.current_user.name} <#{conn.assigns.current_user.email}>" + email_template = PlausibleWeb.Email.error_report(reported_by, trace_id, feedback) + + Plausible.Mailer.deliver_later(email_template) + end + + thanks(conn) + end + + def submit_error_report(conn, _params) do + thanks(conn) + end + + defp thanks(conn) do + conn + |> put_view(PlausibleWeb.ErrorView) + |> render("server_error_report_thanks.html", + layout: {PlausibleWeb.LayoutView, "base_error.html"} + ) + end +end diff --git a/lib/plausible_web/controllers/google_analytics_controller.ex b/lib/plausible_web/controllers/google_analytics_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..7d85452cc0079c85b78e13990b1b45c35e7938ca --- /dev/null +++ b/lib/plausible_web/controllers/google_analytics_controller.ex @@ -0,0 +1,305 @@ +defmodule PlausibleWeb.GoogleAnalyticsController do + use PlausibleWeb, :controller + + alias Plausible.Google + alias Plausible.Imported + use Plausible + + plug(PlausibleWeb.RequireAccountPlug) + + plug(PlausibleWeb.Plugs.AuthorizeSiteAccess, [:owner, :editor, :admin, :super_admin]) + + def property_form( + conn, + %{ + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at + } = params + ) do + site = conn.assigns.site + + redirect_route = Routes.site_path(conn, :settings_imports_exports, site.domain) + + result = Google.API.list_properties(access_token) + + error = + case params["error"] do + "no_data" -> + "No data found. Nothing to import." + + "no_time_window" -> + "Imported data time range is completely overlapping with existing data. Nothing to import." + + _ -> + nil + end + + case result do + {:ok, properties} -> + conn + |> assign(:skip_plausible_tracking, true) + |> render("property_form.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: conn.assigns.site, + properties: properties, + selected_property_error: error + ) + + {:error, :rate_limit_exceeded} -> + conn + |> put_flash( + :error, + "Google Analytics rate limit has been exceeded. Please try again later." + ) + |> redirect(to: redirect_route) + + {:error, {:authentication_failed, message}} -> + default_message = + "We were unable to authenticate your Google Analytics account. Please check that you have granted us permission to 'See and download your Google Analytics data' and try again." + + message = + if ce?() do + message || default_message + else + default_message + end + + conn + |> put_flash(:ttl, :timer.seconds(5)) + |> put_flash(:error, message) + |> redirect(to: redirect_route) + + {:error, :timeout} -> + conn + |> put_flash( + :error, + "Google Analytics API has timed out. Please try again." + ) + |> redirect(to: redirect_route) + + {:error, _any} -> + conn + |> put_flash( + :error, + "We were unable to list your Google Analytics properties. If the problem persists, please contact support for assistance." + ) + |> redirect(to: redirect_route) + end + end + + def property( + conn, + %{ + "property" => property, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at + } = params + ) do + site = conn.assigns.site + + redirect_route = Routes.site_path(conn, :settings_imports_exports, site.domain) + + with {:ok, api_start_date} <- Google.API.get_analytics_start_date(access_token, property), + {:ok, api_end_date} <- Google.API.get_analytics_end_date(access_token, property), + :ok <- ensure_dates(api_start_date, api_end_date), + {:ok, start_date, end_date} <- Imported.clamp_dates(site, api_start_date, api_end_date) do + redirect(conn, + to: + Routes.google_analytics_path(conn, :confirm, site.domain, + property: property, + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + start_date: Date.to_iso8601(start_date), + end_date: Date.to_iso8601(end_date) + ) + ) + else + {:error, error} when error in [:no_data, :no_time_window] -> + params = + params + |> Map.take(["access_token", "refresh_token", "expires_at"]) + |> Map.put("error", Atom.to_string(error)) + + property_form(conn, params) + + {:error, :rate_limit_exceeded} -> + conn + |> put_flash( + :error, + "Google Analytics rate limit has been exceeded. Please try again later." + ) + |> redirect(to: redirect_route) + + {:error, {:authentication_failed, message}} -> + default_message = + "Google Analytics authentication seems to have expired. Please try again." + + message = + if ce?() do + message || default_message + else + default_message + end + + conn + |> put_flash(:ttl, :timer.seconds(5)) + |> put_flash(:error, message) + |> redirect(to: redirect_route) + + {:error, :timeout} -> + conn + |> put_flash( + :error, + "Google Analytics API has timed out. Please try again." + ) + |> redirect(to: redirect_route) + + {:error, _any} -> + conn + |> put_flash( + :error, + "We were unable to retrieve information from Google Analytics. If the problem persists, please contact support for assistance." + ) + |> redirect(to: redirect_route) + end + end + + def confirm(conn, %{ + "property" => property, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at, + "start_date" => start_date, + "end_date" => end_date + }) do + site = conn.assigns.site + + start_date = Date.from_iso8601!(start_date) + end_date = Date.from_iso8601!(end_date) + + redirect_route = Routes.site_path(conn, :settings_imports_exports, site.domain) + + case Google.API.get_property(access_token, property) do + {:ok, %{name: property_name, id: property}} -> + conn + |> assign(:skip_plausible_tracking, true) + |> render("confirm.html", + access_token: access_token, + refresh_token: refresh_token, + expires_at: expires_at, + site: site, + selected_property: property, + selected_property_name: property_name, + start_date: start_date, + end_date: end_date + ) + + {:error, :rate_limit_exceeded} -> + conn + |> put_flash( + :error, + "Google Analytics rate limit has been exceeded. Please try again later." + ) + |> redirect(to: redirect_route) + + {:error, {:authentication_failed, message}} -> + default_message = + "Google Analytics authentication seems to have expired. Please try again." + + message = + if ce?() do + message || default_message + else + default_message + end + + conn + |> put_flash(:ttl, :timer.seconds(5)) + |> put_flash(:error, message) + |> redirect(to: redirect_route) + + {:error, :timeout} -> + conn + |> put_flash( + :error, + "Google Analytics API has timed out. Please try again." + ) + |> redirect(to: redirect_route) + + {:error, :not_found} -> + conn + |> put_flash( + :error, + "Google Analytics property not found. Please try again." + ) + |> redirect(to: redirect_route) + + {:error, _any} -> + conn + |> put_flash( + :error, + "We were unable to retrieve information from Google Analytics. If the problem persists, please contact support for assistance." + ) + |> redirect(to: redirect_route) + end + end + + def import(conn, %{ + "property" => property, + "start_date" => start_date, + "end_date" => end_date, + "access_token" => access_token, + "refresh_token" => refresh_token, + "expires_at" => expires_at + }) do + site = conn.assigns.site + current_user = conn.assigns.current_user + + start_date = Date.from_iso8601!(start_date) + end_date = Date.from_iso8601!(end_date) + + redirect_route = Routes.site_path(conn, :settings_imports_exports, site.domain) + + import_opts = [ + label: property, + property: property, + start_date: start_date, + end_date: end_date, + access_token: access_token, + refresh_token: refresh_token, + token_expires_at: expires_at + ] + + with {:ok, start_date, end_date} <- Imported.clamp_dates(site, start_date, end_date), + import_opts = [{:start_date, start_date}, {:end_date, end_date} | import_opts], + {:ok, _} <- Imported.GoogleAnalytics4.new_import(site, current_user, import_opts) do + conn + |> put_flash(:success, "Import scheduled. An email will be sent when it completes.") + |> redirect(to: redirect_route) + else + {:error, :no_time_window} -> + conn + |> put_flash( + :error, + "Import failed. No data could be imported because date range overlaps with existing data." + ) + |> redirect(to: redirect_route) + + {:error, :import_in_progress} -> + conn + |> put_flash( + :error, + "There's another import still in progress. Please wait until it's completed or cancel it before starting a new one." + ) + |> redirect(to: redirect_route) + end + end + + defp ensure_dates(%Date{}, %Date{}), do: :ok + defp ensure_dates(_, _), do: {:error, :no_data} +end diff --git a/lib/plausible_web/controllers/helpers.ex b/lib/plausible_web/controllers/helpers.ex new file mode 100644 index 0000000000000000000000000000000000000000..c7e1e33d5806876b55c6cce6a6c5469ddcc74120 --- /dev/null +++ b/lib/plausible_web/controllers/helpers.ex @@ -0,0 +1,64 @@ +defmodule PlausibleWeb.ControllerHelpers do + @moduledoc false + + import Plug.Conn + import Phoenix.Controller + + def render_error(conn, status, message) do + conn + |> put_root_layout(false) + |> put_status(status) + |> put_view(PlausibleWeb.ErrorView) + |> render("#{status}.html", message: message, layout: error_layout()) + end + + def render_error(conn, status) do + conn + |> put_root_layout(false) + |> put_status(status) + |> put_view(PlausibleWeb.ErrorView) + |> render("#{status}.html", layout: error_layout()) + end + + defp error_layout, + do: Application.get_env(:plausible, PlausibleWeb.Endpoint)[:render_errors][:layout] + + @auth_layout_assigns [ + layout: {PlausibleWeb.LayoutView, :auth}, + hide_header?: true, + hide_footer?: true, + disable_global_notices?: true + ] + + def render_auth_page(conn, template, extra_assigns \\ []) do + render(conn, template, Keyword.merge(@auth_layout_assigns, extra_assigns)) + end + + def debug_metadata(conn) do + %{ + request_method: conn.method, + request_path: conn.request_path, + params: conn.params, + phoenix_controller: conn.private.phoenix_controller |> to_string(), + phoenix_action: conn.private.phoenix_action |> to_string(), + site_id: conn.assigns.site.id, + site_domain: conn.assigns.site.domain, + user_id: get_user_id(conn, conn.assigns), + team_id: get_team_id(conn, conn.assigns) + } + end + + defp get_user_id(_conn, %{current_user: user}), do: user.id + + defp get_user_id(conn, _assigns) do + case PlausibleWeb.UserAuth.get_user_session(conn) do + {:ok, user_session} -> user_session.user_id + _ -> nil + end + end + + defp get_team_id(_conn, %{current_team: %Plausible.Teams.Team{id: id}}), do: id + defp get_team_id(_conn, %{site_team: %Plausible.Teams.Team{id: id}}), do: id + defp get_team_id(_conn, %{api_context: :site, site: %Plausible.Site{team_id: id}}), do: id + defp get_team_id(_conn, _assigns), do: nil +end diff --git a/lib/plausible_web/controllers/invitation_controller.ex b/lib/plausible_web/controllers/invitation_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..5aea6bca4efea22ee9310f8351b1f0c8ff0700c8 --- /dev/null +++ b/lib/plausible_web/controllers/invitation_controller.ex @@ -0,0 +1,140 @@ +defmodule PlausibleWeb.InvitationController do + use PlausibleWeb, :controller + + alias Plausible.Teams + + plug PlausibleWeb.RequireAccountPlug + + plug PlausibleWeb.Plugs.AuthorizeSiteAccess, + [:owner, :admin] when action in [:remove_invitation] + + def accept_invitation(conn, %{"invitation_id" => invitation_id} = params) do + current_user = conn.assigns.current_user + team = conn.assigns.current_team + + result = + if params["skip_site_members_transfer"] == "true" do + Teams.Invitations.Accept.accept_transfer_no_members(invitation_id, current_user, team) + else + Teams.Invitations.Accept.accept(invitation_id, current_user, team) + end + + case result do + {:ok, result} -> + team = result.team + + site = + case result do + %{guest_memberships: [guest_membership]} -> + Plausible.Repo.preload(guest_membership, :site).site + + %{guest_memberships: []} -> + nil + + %{site: site} -> + site + end + + if site do + conn + |> put_flash(:success, "You now have access to #{site.domain}") + |> redirect(to: Routes.stats_path(conn, :stats, site.domain, [])) + else + conn + |> put_flash(:success, "You now have access to \"#{team.name}\" team") + |> redirect(to: Routes.site_path(conn, :index, __team: team.identifier)) + end + + {:error, :invitation_not_found} -> + conn + |> put_flash(:error, "Invitation missing or already accepted") + |> redirect(to: "/sites") + + {:error, :transfer_to_self} -> + conn + |> put_flash(:error, "The site is already in the current team") + |> redirect(to: "/sites") + + {:error, :permission_denied} -> + conn + |> put_flash(:error, "You can't add sites in the current team") + |> redirect(to: "/sites") + + {:error, :no_plan} -> + conn + |> put_flash(:error, "No existing subscription") + |> redirect(to: "/sites") + + {:error, {:over_plan_limits, limits}} -> + conn + |> put_flash( + :error, + "Plan limits exceeded: #{PlausibleWeb.TextHelpers.pretty_list(limits)}." + ) + |> redirect(to: "/sites") + + {:error, _} -> + conn + |> put_flash(:error, "Something went wrong, please try again") + |> redirect(to: "/sites") + end + end + + def reject_invitation(conn, %{"invitation_id" => invitation_id}) do + case Teams.Invitations.Reject.reject(invitation_id, conn.assigns.current_user) do + {:ok, _invitation} -> + conn + |> put_flash(:success, "You have rejected the invitation") + |> redirect(to: "/sites") + + {:error, :invitation_not_found} -> + conn + |> put_flash(:error, "Invitation missing or already accepted") + |> redirect(to: "/sites") + end + end + + def remove_invitation(conn, %{"invitation_id" => invitation_id}) do + case Teams.Invitations.RemoveFromSite.remove(invitation_id, conn.assigns.site) do + {:ok, invitation_or_transfer} -> + {site, email} = + case invitation_or_transfer do + %Plausible.Teams.GuestInvitation{} = guest_invitation -> + {guest_invitation.site, guest_invitation.team_invitation.email} + + %Plausible.Teams.SiteTransfer{} = site_transfer -> + {site_transfer.site, site_transfer.email} + end + + conn + |> put_flash(:success, "You have removed the invitation for #{email}") + |> redirect(to: Routes.site_path(conn, :settings_people, site.domain)) + + {:error, :invitation_not_found} -> + conn + |> put_flash(:error, "Invitation missing or already removed") + |> redirect(to: Routes.site_path(conn, :settings_people, conn.assigns.site.domain)) + end + end + + def remove_team_invitation(conn, %{"invitation_id" => invitation_id}) do + %{current_team: team, current_user: current_user} = conn.assigns + + case Teams.Invitations.RemoveFromTeam.remove(team, invitation_id, current_user) do + {:ok, invitation} -> + conn + |> put_flash(:success, "You have removed the invitation for #{invitation.email}") + |> redirect(to: Routes.settings_path(conn, :team_general)) + + {:error, :invitation_not_found} -> + conn + |> put_flash(:error, "Invitation missing or already removed") + |> redirect(to: Routes.settings_path(conn, :team_general)) + + {:error, :permission_denied} -> + conn + |> put_flash(:error, "You are not allowed to remove invitations") + |> redirect(to: Routes.settings_path(conn, :team_general)) + end + end +end diff --git a/lib/plausible_web/controllers/page_controller.ex b/lib/plausible_web/controllers/page_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..62d4a339cb901f482cdbcc7ed7d0b45f47c02932 --- /dev/null +++ b/lib/plausible_web/controllers/page_controller.ex @@ -0,0 +1,15 @@ +defmodule PlausibleWeb.PageController do + use PlausibleWeb, :controller + use Plausible.Repo + + plug PlausibleWeb.RequireLoggedOutPlug + + @doc """ + The root path is never accessible in Plausible.Cloud because it is handled by the upstream reverse proxy. + + This controller action is only ever triggered in self-hosted Plausible. + """ + def index(conn, _params) do + render(conn, "index.html") + end +end diff --git a/lib/plausible_web/controllers/settings_controller.ex b/lib/plausible_web/controllers/settings_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..749a2bfe2846c71d034f17344ae55c5e8f09839a --- /dev/null +++ b/lib/plausible_web/controllers/settings_controller.ex @@ -0,0 +1,434 @@ +defmodule PlausibleWeb.SettingsController do + use PlausibleWeb, :controller + use Plausible + use Plausible.Repo + + alias Plausible.Auth + alias Plausible.Teams + + require Logger + + plug Plausible.Plugs.AuthorizeTeamAccess, + [:owner, :admin] + when action in [:update_team_name] + + plug Plausible.Plugs.AuthorizeTeamAccess, + [:owner] + when action in [ + :team_danger_zone, + :delete_team, + :enable_team_force_2fa, + :disable_team_force_2fa + ] + + plug Plausible.Plugs.RestrictUserType, + [deny: :sso] when action in [:update_name, :update_email, :update_password] + + def index(conn, _params) do + redirect(conn, to: Routes.settings_path(conn, :preferences)) + end + + def team_general(conn, _params) do + render_team_general(conn) + end + + def update_team_name(conn, %{"team" => params}) do + changeset = Teams.Team.name_changeset(conn.assigns.current_team, params) + + case Repo.update(changeset) do + {:ok, _user} -> + conn + |> put_flash(:success, "Team name changed") + |> redirect(to: Routes.settings_path(conn, :team_general) <> "#update-name") + + {:error, changeset} -> + render_team_general(conn, team_name_changeset: changeset) + end + end + + defp render_team_general(conn, opts \\ []) do + if Teams.setup?(conn.assigns.current_team) do + name_changeset = + Keyword.get( + opts, + :team_name_changeset, + Teams.Team.name_changeset(conn.assigns.current_team) + ) + + render(conn, :team_general, + team_name_changeset: name_changeset, + force_2fa_enabled?: Teams.force_2fa_enabled?(conn.assigns.current_team), + layout: {PlausibleWeb.LayoutView, :settings}, + connect_live_socket: true + ) + else + conn + |> redirect(to: Routes.site_path(conn, :index)) + end + end + + def enable_team_force_2fa(conn, _params) do + team = conn.assigns.current_team + user = conn.assigns.current_user + + case Teams.enable_force_2fa(team, user) do + {:ok, _} -> + conn + |> put_flash(:success, "2FA is now required for all team members.") + |> redirect(to: Routes.settings_path(conn, :team_general)) + + {:error, _} -> + conn + |> put_flash(:error, "Failed to enforce 2FA for all team members.") + |> redirect(to: Routes.settings_path(conn, :team_general)) + end + end + + def disable_team_force_2fa(conn, %{"password" => password}) do + team = conn.assigns.current_team + user = conn.assigns.current_user + + case Teams.disable_force_2fa(team, user, password) do + {:ok, _} -> + conn + |> put_flash(:success, "2FA is no longer enforced for team members.") + |> redirect(to: Routes.settings_path(conn, :team_general)) + + {:error, :invalid_password} -> + conn + |> put_flash(:error, "Incorrect password provided.") + |> redirect(to: Routes.settings_path(conn, :team_general)) + + {:error, _} -> + conn + |> put_flash(:error, "Failed to disable enforcing 2FA for all team members.") + |> redirect(to: Routes.settings_path(conn, :team_general)) + end + end + + def leave_team(conn, _params) do + case Teams.Memberships.Leave.leave(conn.assigns.current_team, conn.assigns.current_user) do + {:ok, _} -> + conn + |> put_flash(:success, "You have left \"#{Teams.name(conn.assigns.current_team)}\"") + |> redirect(to: Routes.site_path(conn, :index, __team: "none")) + + {:error, :only_one_owner} -> + conn + |> put_flash(:error, "You can't leave as you are the only Owner on the team") + |> redirect(to: Routes.settings_path(conn, :team_general)) + + {:error, :membership_not_found} -> + redirect(conn, to: Routes.site_path(conn, :index, __team: "none")) + end + end + + def preferences(conn, _params) do + render_preferences(conn) + end + + def security(conn, _params) do + render_security(conn) + end + + def redirect_invoices(conn, _params) do + redirect(conn, to: Routes.settings_path(conn, :subscription)) + end + + def api_keys(conn, _params) do + current_user = conn.assigns.current_user + current_team = conn.assigns[:current_team] + + api_keys = Auth.list_api_keys(current_user, current_team) + + render(conn, :api_keys, layout: {PlausibleWeb.LayoutView, :settings}, api_keys: api_keys) + end + + def new_api_key(conn, _params) do + current_team = conn.assigns[:current_team] + + sites_api_enabled? = + Plausible.Billing.Feature.SitesAPI.check_availability(current_team) == :ok + + changeset = Auth.ApiKey.changeset(%Auth.ApiKey{type: "stats_api"}, current_team, %{}) + + render(conn, "new_api_key.html", changeset: changeset, sites_api_enabled?: sites_api_enabled?) + end + + def create_api_key(conn, %{"api_key" => %{"name" => name, "key" => key, "type" => type}}) do + current_user = conn.assigns.current_user + current_team = conn.assigns.current_team + + sites_api_enabled? = + Plausible.Billing.Feature.SitesAPI.check_availability(current_team) == :ok + + api_key_fn = + if type == "sites_api" do + &Auth.create_sites_api_key/4 + else + &Auth.create_stats_api_key/4 + end + + case api_key_fn.(current_user, current_team, name, key) do + {:ok, _api_key} -> + conn + |> put_flash(:success, "API key created successfully") + |> redirect(to: Routes.settings_path(conn, :api_keys) <> "#api-keys") + + {:error, :upgrade_required} -> + conn + |> put_flash(:error, "Your current subscription plan does not include Sites API access") + |> redirect(to: Routes.settings_path(conn, :new_api_key)) + + {:error, changeset} -> + render(conn, "new_api_key.html", + changeset: changeset, + sites_api_enabled?: sites_api_enabled? + ) + end + end + + def delete_api_key(conn, %{"id" => id}) do + case Auth.delete_api_key(conn.assigns.current_user, id) do + :ok -> + conn + |> put_flash(:success, "API key revoked successfully") + |> redirect(to: Routes.settings_path(conn, :api_keys) <> "#api-keys") + + {:error, :not_found} -> + conn + |> put_flash(:error, "Could not find API Key to delete") + |> redirect(to: Routes.settings_path(conn, :api_keys) <> "#api-keys") + end + end + + def danger_zone(conn, _params) do + solely_owned_teams = + conn.assigns.current_user + |> Teams.Users.owned_teams() + |> Enum.filter(& &1.setup_complete) + |> Enum.reject(fn team -> + Teams.Memberships.owners_count(team) > 1 + end) + + render(conn, :danger_zone, + solely_owned_teams: solely_owned_teams, + layout: {PlausibleWeb.LayoutView, :settings} + ) + end + + def team_danger_zone(conn, _params) do + render(conn, :team_danger_zone, layout: {PlausibleWeb.LayoutView, :settings}) + end + + def delete_team(conn, _params) do + team = conn.assigns.current_team + + case Plausible.Teams.delete(team) do + {:ok, :deleted} -> + conn + |> put_flash(:success, ~s|Team "#{Plausible.Teams.name(team)}" deleted|) + |> redirect(to: Routes.site_path(conn, :index, __team: "none")) + + {:error, :active_subscription} -> + conn + |> put_flash( + :error, + "Team has an active subscription. You must cancel it first." + ) + |> redirect(to: Routes.settings_path(conn, :team_danger_zone)) + end + end + + # Preferences actions + + def update_name(conn, %{"user" => params}) do + changeset = Auth.User.name_changeset(conn.assigns.current_user, params) + + case Repo.update(changeset) do + {:ok, _user} -> + conn + |> put_flash(:success, "Name changed") + |> redirect(to: Routes.settings_path(conn, :preferences) <> "#update-name") + + {:error, changeset} -> + render_preferences(conn, name_changeset: changeset) + end + end + + def update_theme(conn, %{"user" => params}) do + changeset = Auth.User.theme_changeset(conn.assigns.current_user, params) + + case Repo.update(changeset) do + {:ok, _user} -> + conn + |> put_flash(:success, "Theme changed") + |> redirect(to: Routes.settings_path(conn, :preferences) <> "#update-theme") + + {:error, changeset} -> + render_preferences(conn, theme_changeset: changeset) + end + end + + defp render_preferences(conn, opts \\ []) do + name_changeset = + Keyword.get(opts, :name_changeset, Auth.User.name_changeset(conn.assigns.current_user)) + + theme_changeset = + Keyword.get(opts, :theme_changeset, Auth.User.theme_changeset(conn.assigns.current_user)) + + render(conn, :preferences, + name_changeset: name_changeset, + theme_changeset: theme_changeset, + layout: {PlausibleWeb.LayoutView, :settings} + ) + end + + # Security actions + + def update_email(conn, %{"user" => params}) do + user = conn.assigns.current_user + + with :ok <- Auth.rate_limit(:email_change_user, user), + changes = Auth.User.email_changeset(user, params), + {:ok, user} <- Repo.update(changes) do + if user.email_verified do + handle_email_updated(conn) + else + Auth.EmailVerification.issue_code(user) + redirect(conn, to: Routes.auth_path(conn, :activate_form)) + end + else + {:error, %Ecto.Changeset{} = changeset} -> + render_security(conn, email_changeset: changeset) + + {:error, {:rate_limit, _}} -> + changeset = + user + |> Auth.User.email_changeset(params) + |> Ecto.Changeset.add_error(:email, "too many requests, try again in an hour") + |> Map.put(:action, :validate) + + render_security(conn, email_changeset: changeset) + end + end + + def cancel_update_email(conn, _params) do + changeset = Auth.User.cancel_email_changeset(conn.assigns.current_user) + + case Repo.update(changeset) do + {:ok, user} -> + conn + |> put_flash(:success, "Email changed back to #{user.email}") + |> redirect(to: Routes.settings_path(conn, :security) <> "#update-email") + + {:error, _} -> + conn + |> put_flash( + :error, + "Could not cancel email update because previous email has already been taken" + ) + |> redirect(to: Routes.auth_path(conn, :activate_form)) + end + end + + def update_password(conn, %{"user" => params}) do + user = conn.assigns.current_user + user_session = conn.assigns.current_user_session + + with :ok <- Auth.rate_limit(:password_change_user, user), + {:ok, user} <- do_update_password(user, params) do + Auth.UserSessions.revoke_all(user, except: user_session) + + conn + |> put_flash(:success, "Your password is now changed") + |> redirect(to: Routes.settings_path(conn, :security) <> "#update-password") + else + {:error, %Ecto.Changeset{} = changeset} -> + render_security(conn, password_changeset: changeset) + + {:error, {:rate_limit, _}} -> + changeset = + user + |> Auth.User.password_changeset(params) + |> Ecto.Changeset.add_error(:password, "too many attempts, try again in 20 minutes") + |> Map.put(:action, :validate) + + render_security(conn, password_changeset: changeset) + end + end + + defp render_security(conn, opts \\ []) do + user_sessions = Auth.UserSessions.list_for_user(conn.assigns.current_user) + + email_changeset = + Keyword.get( + opts, + :email_changeset, + Auth.User.email_changeset(conn.assigns.current_user, %{email: ""}) + ) + + password_changeset = + Keyword.get( + opts, + :password_changeset, + Auth.User.password_changeset(conn.assigns.current_user) + ) + + render(conn, :security, + totp_enabled?: Auth.TOTP.enabled?(conn.assigns.current_user), + user_sessions: user_sessions, + email_changeset: email_changeset, + password_changeset: password_changeset, + layout: {PlausibleWeb.LayoutView, :settings} + ) + end + + def delete_session(conn, %{"id" => session_id}) do + current_user = conn.assigns.current_user + + :ok = Auth.UserSessions.revoke_by_id(current_user, session_id) + + conn + |> put_flash(:success, "Session logged out successfully") + |> redirect(to: Routes.settings_path(conn, :security) <> "#user-sessions") + end + + defp do_update_password(user, params) do + changes = Auth.User.password_changeset(user, params) + + Repo.transaction(fn -> + with {:ok, user} <- Repo.update(changes), + {:ok, user} <- validate_2fa_code(user, params["two_factor_code"]) do + user + else + {:error, :invalid_2fa} -> + changes + |> Ecto.Changeset.add_error(:password, "invalid 2FA code") + |> Map.put(:action, :validate) + |> Repo.rollback() + + {:error, changeset} -> + Repo.rollback(changeset) + end + end) + end + + defp validate_2fa_code(user, code) do + if Auth.TOTP.enabled?(user) do + case Auth.TOTP.validate_code(user, code) do + {:ok, user} -> {:ok, user} + {:error, :not_enabled} -> {:ok, user} + {:error, _} -> {:error, :invalid_2fa} + end + else + {:ok, user} + end + end + + defp handle_email_updated(conn) do + conn + |> put_flash(:success, "Email updated") + |> redirect(to: Routes.settings_path(conn, :security) <> "#update-email") + end +end diff --git a/lib/plausible_web/controllers/site/membership_controller.ex b/lib/plausible_web/controllers/site/membership_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..adf2a2eab1ab64ae69132bbc64dec7374dd030f7 --- /dev/null +++ b/lib/plausible_web/controllers/site/membership_controller.ex @@ -0,0 +1,164 @@ +defmodule PlausibleWeb.Site.MembershipController do + @moduledoc """ + This controller deals with user management via the UI in Site Settings -> People. It's important to enforce permissions in this controller. + + Owner and Admin - Can manage users, can trigger a 'transfer ownership' request + Editor and Viewer - Can not access user management settings + Anyone - Can accept invitations + + Everything else should be explicitly disallowed. + """ + + use PlausibleWeb, :controller + use Plausible.Repo + use Plausible + + alias Plausible.Teams + + plug PlausibleWeb.RequireAccountPlug + + plug PlausibleWeb.Plugs.AuthorizeSiteAccess, [:owner, :admin] + + def invite_member_form(conn, _params) do + site = + conn.assigns.current_user + |> Plausible.Sites.get_for_user!(conn.assigns.site.domain) + |> Plausible.Repo.preload(:owners) + + limit = Teams.Billing.team_member_limit(site.team) + usage = Teams.Billing.team_member_usage(site.team) + below_limit? = Plausible.Billing.Quota.below_limit?(usage, limit) + + render( + conn, + "invite_member_form.html", + site: site, + team_member_limit: limit, + is_at_limit: not below_limit?, + skip_plausible_tracking: true + ) + end + + def invite_member(conn, %{"email" => email, "role" => role}) do + site_domain = conn.assigns.site.domain + + site = + Plausible.Sites.get_for_user!(conn.assigns.current_user, site_domain) + |> Plausible.Repo.preload(:owners) + + case Teams.Invitations.InviteToSite.invite( + site, + conn.assigns.current_user, + email, + role + ) do + {:ok, invitation} -> + conn + |> put_flash( + :success, + "#{email} has been invited to #{site_domain} as #{PlausibleWeb.SiteView.with_indefinite_article("#{invitation.role}")}" + ) + |> redirect(to: Routes.site_path(conn, :settings_people, site.domain)) + + {:error, :already_a_member} -> + render(conn, "invite_member_form.html", + error: "Cannot send invite because #{email} is already a member of #{site.domain}", + site: site, + skip_plausible_tracking: true + ) + + {:error, {:over_limit, limit}} -> + render(conn, "invite_member_form.html", + error: + "Your account is limited to #{limit} team members. You can upgrade your plan to increase this limit.", + site: site, + skip_plausible_tracking: true, + is_at_limit: true, + team_member_limit: limit + ) + + {:error, %Ecto.Changeset{} = changeset} -> + error_msg = + case changeset.errors[:invitation] do + {"already sent", _} -> + "This invitation has been already sent. To send again, remove it from pending invitations first." + + _ -> + "Something went wrong." + end + + conn + |> put_flash(:error, error_msg) + |> redirect(to: Routes.site_path(conn, :settings_people, site.domain)) + end + end + + @doc """ + Updates the role of a user. The user being updated could be the same or different from the user taking + the action. When updating the role, it's important to enforce permissions: + + Owner - Can update anyone's role except for themselves. If they want to change their own role, they have to use the 'transfer ownership' feature. + Admin - Can update anyone's role except for owners. Can downgrade their own access to 'viewer'. Can promote a viewer to admin. + """ + def update_role_by_user(conn, %{"id" => user_id, "new_role" => new_role_str}) do + %{site: site, current_user: current_user, site_role: site_role} = conn.assigns + + case Teams.Memberships.update_role( + site, + user_id, + new_role_str, + current_user, + site_role + ) do + {:ok, guest_membership} -> + redirect_target = + if guest_membership.team_membership.user_id == current_user.id and + guest_membership.role == :viewer do + Routes.stats_path(conn, :stats, site.domain, []) + else + Routes.site_path(conn, :settings_people, site.domain) + end + + conn + |> put_flash( + :success, + "#{guest_membership.team_membership.user.name} is now #{PlausibleWeb.SiteView.with_indefinite_article(to_string(guest_membership.role))}" + ) + |> redirect(to: redirect_target) + + {:error, _} -> + conn + |> put_flash(:error, "You are not allowed to grant the #{new_role_str} role") + |> redirect(to: Routes.site_path(conn, :settings_people, site.domain)) + end + end + + def remove_member_by_user(conn, %{"id" => user_id} = _params) do + site = conn.assigns.site + + if user = Repo.get(Plausible.Auth.User, user_id) do + Teams.Memberships.remove(site, user) + + redirect_target = + if user_id == conn.assigns[:current_user].id do + Routes.site_path(conn, :index) + else + Routes.site_path(conn, :settings_people, site.domain) + end + + conn + |> put_flash( + :success, + "#{user.name} has been removed from #{site.domain}" + ) + |> redirect(to: redirect_target) + else + conn + |> put_flash( + :success, + "User has been removed from #{site.domain}" + ) + |> redirect(to: Routes.site_path(conn, :settings_people, site.domain)) + end + end +end diff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..7413111898eb66c70ceb28927bd48bf894988672 --- /dev/null +++ b/lib/plausible_web/controllers/site_controller.ex @@ -0,0 +1,662 @@ +defmodule PlausibleWeb.SiteController do + use PlausibleWeb, :controller + use Plausible.Repo + use Plausible + + alias Plausible.Sites + + @unrestricted_actions [:new, :create_site] + @destructive_actions [:settings_danger_zone, :reset_stats, :delete_site] + + @special_cased_actions @unrestricted_actions ++ @destructive_actions + + plug(PlausibleWeb.RequireAccountPlug) + + plug( + PlausibleWeb.Plugs.AuthorizeSiteAccess, + [:owner, :admin, :editor, :super_admin] + when action not in @special_cased_actions + ) + + plug( + PlausibleWeb.Plugs.AuthorizeSiteAccess, + [:owner, :admin, :super_admin] when action in @destructive_actions + ) + + def new(conn, params) do + flow = params["flow"] || PlausibleWeb.Flows.register() + team = conn.assigns.current_team + + render(conn, "new.html", + changeset: Plausible.Site.changeset(%Plausible.Site{}), + site_limit: Plausible.Teams.Billing.site_limit(team), + site_limit_exceeded?: Plausible.Teams.Billing.ensure_can_add_new_site(team) != :ok, + form_submit_url: "/sites?flow=#{flow}", + flow: flow + ) + end + + def create_site(conn, %{"site" => site_params}) do + team = conn.assigns.current_team + user = conn.assigns.current_user + first_site? = Plausible.Teams.Billing.site_usage(team) == 0 + flow = conn.params["flow"] + + case Sites.create(user, site_params, team) do + {:ok, %{site: site}} -> + if first_site? do + PlausibleWeb.Email.welcome_email(user) + |> Plausible.Mailer.send() + end + + redirect(conn, + to: + Routes.site_path(conn, :installation, site.domain, + site_created: true, + flow: flow + ) + ) + + {:error, _, :permission_denied, _} -> + conn + |> put_flash(:error, "You are not permitted to add sites in the current team") + |> render("new.html", + changeset: Plausible.Site.changeset(%Plausible.Site{}), + first_site?: first_site?, + site_limit: Plausible.Teams.Billing.site_limit(team), + site_limit_exceeded?: false, + flow: flow, + form_submit_url: "/sites?flow=#{flow}" + ) + + {:error, _, {:over_limit, limit}, _} -> + render(conn, "new.html", + changeset: Plausible.Site.changeset(%Plausible.Site{}), + first_site?: first_site?, + site_limit: limit, + site_limit_exceeded?: true, + flow: flow, + form_submit_url: "/sites?flow=#{flow}" + ) + + {:error, _, changeset, _} -> + case check_can_already_access(changeset, user) do + {:ok, domain} -> + redirect(conn, to: Routes.stats_path(PlausibleWeb.Endpoint, :stats, domain, [])) + + {:error, :no_access} -> + render(conn, "new.html", + changeset: changeset, + first_site?: first_site?, + site_limit: Plausible.Teams.Billing.site_limit(team), + site_limit_exceeded?: false, + flow: flow, + form_submit_url: "/sites?flow=#{flow}" + ) + end + end + end + + def settings(conn, %{"domain" => domain}) do + redirect(conn, to: Routes.site_path(conn, :settings_general, domain)) + end + + def settings_general(conn, _params) do + site = conn.assigns[:site] + + conn + |> render("settings_general.html", + site: site, + changeset: Plausible.Site.changeset(site, %{}), + connect_live_socket: true, + dogfood_page_path: "/:dashboard/settings/general", + layout: {PlausibleWeb.LayoutView, "site_settings.html"} + ) + end + + def settings_people(conn, _params) do + site = Repo.preload(conn.assigns.site, :team) + + %{memberships: memberships, invitations: invitations} = + Sites.list_people(site) + + conn + |> render("settings_people.html", + site: site, + memberships: memberships, + invitations: invitations, + dogfood_page_path: "/:dashboard/settings/people", + layout: {PlausibleWeb.LayoutView, "site_settings.html"} + ) + end + + def settings_visibility(conn, _params) do + site = conn.assigns[:site] + + conn + |> render("settings_visibility.html", + site: site, + dogfood_page_path: "/:dashboard/settings/visibility", + connect_live_socket: true, + layout: {PlausibleWeb.LayoutView, "site_settings.html"} + ) + end + + def settings_goals(conn, _params) do + conn + |> render("settings_goals.html", + dogfood_page_path: "/:dashboard/settings/goals", + connect_live_socket: true, + layout: {PlausibleWeb.LayoutView, "site_settings.html"} + ) + end + + def settings_funnels(conn, _params) do + conn + |> render("settings_funnels.html", + dogfood_page_path: "/:dashboard/settings/funnels", + connect_live_socket: true, + layout: {PlausibleWeb.LayoutView, "site_settings.html"} + ) + end + + def settings_props(conn, _params) do + conn + |> render("settings_props.html", + dogfood_page_path: "/:dashboard/settings/properties", + layout: {PlausibleWeb.LayoutView, "site_settings.html"}, + connect_live_socket: true + ) + end + + def settings_email_reports(conn, _params) do + site = conn.assigns[:site] + + conn + |> render("settings_email_reports.html", + site: site, + weekly_report: Repo.get_by(Plausible.Site.WeeklyReport, site_id: site.id), + monthly_report: Repo.get_by(Plausible.Site.MonthlyReport, site_id: site.id), + spike_notification: + Repo.get_by(Plausible.Site.TrafficChangeNotification, site_id: site.id, type: :spike), + drop_notification: + Repo.get_by(Plausible.Site.TrafficChangeNotification, site_id: site.id, type: :drop), + dogfood_page_path: "/:dashboard/settings/email-reports", + layout: {PlausibleWeb.LayoutView, "site_settings.html"} + ) + end + + def settings_danger_zone(conn, _params) do + site = conn.assigns[:site] + + conn + |> render("settings_danger_zone.html", + site: site, + connect_live_socket: true, + dogfood_page_path: "/:dashboard/settings/danger-zone", + layout: {PlausibleWeb.LayoutView, "site_settings.html"} + ) + end + + def settings_integrations(conn, _params) do + site = + conn.assigns.site + |> Repo.preload([:google_auth]) + + search_console_domains = + if site.google_auth do + Plausible.Google.API.fetch_verified_properties(site.google_auth) + end + + has_plugins_tokens? = Plausible.Plugins.API.Tokens.any?(site) + + conn + |> render("settings_integrations.html", + site: site, + has_plugins_tokens?: has_plugins_tokens?, + search_console_domains: search_console_domains, + dogfood_page_path: "/:dashboard/settings/integrations", + connect_live_socket: true, + layout: {PlausibleWeb.LayoutView, "site_settings.html"} + ) + end + + def settings_shields(conn, %{"shield" => shield}) + when shield in ["ip_addresses", "countries", "pages", "hostnames"] do + site = conn.assigns.site + + conn + |> render("settings_shields.html", + site: site, + shield: shield, + dogfood_page_path: "/:dashboard/settings/shields/#{shield}", + connect_live_socket: true, + layout: {PlausibleWeb.LayoutView, "site_settings.html"} + ) + end + + def settings_imports_exports(conn, _params) do + site = conn.assigns.site + + conn + |> render("settings_imports_exports.html", + site: site, + dogfood_page_path: "/:dashboard/settings/imports-exports", + connect_live_socket: true, + layout: {PlausibleWeb.LayoutView, "site_settings.html"} + ) + end + + def update_google_auth(conn, %{"google_auth" => attrs}) do + site = conn.assigns[:site] |> Repo.preload(:google_auth) + + Plausible.Site.GoogleAuth.set_property(site.google_auth, attrs) + |> Repo.update!() + + conn + |> put_flash(:success, "Google integration saved successfully") + |> redirect(to: Routes.site_path(conn, :settings_integrations, site.domain)) + end + + def delete_google_auth(conn, _params) do + site = + conn.assigns[:site] + |> Repo.preload(:google_auth) + + if site.google_auth do + Repo.delete!(site.google_auth) + end + + put_flash(conn, :success, "Google account unlinked from Plausible") + redirect(conn, to: Routes.site_path(conn, :settings_integrations, site.domain)) + end + + def update_settings(conn, %{"site" => site_params}) do + site = conn.assigns[:site] + changeset = Plausible.Site.update_changeset(site, site_params) + + case Repo.update(changeset) do + {:ok, site} -> + site_session_key = "authorized_site__" <> site.domain + + conn + |> put_session(site_session_key, nil) + |> put_flash(:success, "Your site settings have been saved") + |> redirect(to: Routes.site_path(conn, :settings_general, site.domain)) + + {:error, changeset} -> + conn + |> put_flash(:error, "Could not update your site settings") + |> render("settings_general.html", + site: site, + changeset: changeset, + layout: {PlausibleWeb.LayoutView, "site_settings.html"} + ) + end + end + + def reset_stats(conn, _params) do + site = conn.assigns[:site] + Plausible.Purge.reset!(site) + + conn + |> put_flash(:success, "#{site.domain} stats will be reset in a few minutes") + |> redirect(to: Routes.site_path(conn, :settings_danger_zone, site.domain)) + end + + def delete_site(conn, _params) do + site = conn.assigns[:site] + + Plausible.Site.Removal.run(site) + + conn + |> put_flash(:success, "Your site and page views deletion process has started.") + |> redirect(to: "/sites") + end + + def make_public(conn, _params) do + site = + conn.assigns[:site] + |> Plausible.Site.make_public() + |> Repo.update!() + + conn + |> put_flash(:success, "Stats for #{site.domain} are now public.") + |> redirect(to: Routes.site_path(conn, :settings_visibility, site.domain)) + end + + def make_private(conn, _params) do + site = + conn.assigns[:site] + |> Plausible.Site.make_private() + |> Repo.update!() + + conn + |> put_flash(:success, "Stats for #{site.domain} are now private.") + |> redirect(to: Routes.site_path(conn, :settings_visibility, site.domain)) + end + + def enable_weekly_report(conn, _params) do + site = conn.assigns[:site] + + result = + Plausible.Site.WeeklyReport.changeset(%Plausible.Site.WeeklyReport{}, %{ + site_id: site.id, + recipients: [conn.assigns[:current_user].email] + }) + |> Repo.insert() + + :ok = tolerate_unique_contraint_violation(result, "weekly_reports_site_id_index") + + conn + |> put_flash(:success, "You will receive an email report every Monday going forward") + |> redirect(to: Routes.site_path(conn, :settings_email_reports, site.domain)) + end + + def disable_weekly_report(conn, _params) do + site = conn.assigns[:site] + Repo.delete_all(from(wr in Plausible.Site.WeeklyReport, where: wr.site_id == ^site.id)) + + conn + |> put_flash(:success, "You will not receive weekly email reports going forward") + |> redirect(to: Routes.site_path(conn, :settings_email_reports, site.domain)) + end + + def add_weekly_report_recipient(conn, %{"recipient" => recipient}) do + site = conn.assigns[:site] + + Repo.get_by(Plausible.Site.WeeklyReport, site_id: site.id) + |> Plausible.Site.WeeklyReport.add_recipient(recipient) + |> Repo.update!() + + conn + |> put_flash(:success, "Added #{recipient} as a recipient for the weekly report") + |> redirect(to: Routes.site_path(conn, :settings_email_reports, site.domain)) + end + + def remove_weekly_report_recipient(conn, %{"recipient" => recipient}) do + site = conn.assigns[:site] + + Repo.get_by(Plausible.Site.WeeklyReport, site_id: site.id) + |> Plausible.Site.WeeklyReport.remove_recipient(recipient) + |> Repo.update!() + + conn + |> put_flash( + :success, + "Removed #{recipient} as a recipient for the weekly report" + ) + |> redirect(to: Routes.site_path(conn, :settings_email_reports, site.domain)) + end + + def enable_monthly_report(conn, _params) do + site = conn.assigns[:site] + + result = + %Plausible.Site.MonthlyReport{} + |> Plausible.Site.MonthlyReport.changeset(%{ + site_id: site.id, + recipients: [conn.assigns[:current_user].email] + }) + |> Repo.insert() + + :ok = tolerate_unique_contraint_violation(result, "monthly_reports_site_id_index") + + conn + |> put_flash(:success, "You will receive an email report every month going forward") + |> redirect(to: Routes.site_path(conn, :settings_email_reports, site.domain)) + end + + def disable_monthly_report(conn, _params) do + site = conn.assigns[:site] + Repo.delete_all(from(mr in Plausible.Site.MonthlyReport, where: mr.site_id == ^site.id)) + + conn + |> put_flash(:success, "You will not receive monthly email reports going forward") + |> redirect(to: Routes.site_path(conn, :settings_email_reports, site.domain)) + end + + def add_monthly_report_recipient(conn, %{"recipient" => recipient}) do + site = conn.assigns[:site] + + Repo.get_by(Plausible.Site.MonthlyReport, site_id: site.id) + |> Plausible.Site.MonthlyReport.add_recipient(recipient) + |> Repo.update!() + + conn + |> put_flash(:success, "Added #{recipient} as a recipient for the monthly report") + |> redirect(to: Routes.site_path(conn, :settings_email_reports, site.domain)) + end + + def remove_monthly_report_recipient(conn, %{"recipient" => recipient}) do + site = conn.assigns[:site] + + Repo.get_by(Plausible.Site.MonthlyReport, site_id: site.id) + |> Plausible.Site.MonthlyReport.remove_recipient(recipient) + |> Repo.update!() + + conn + |> put_flash( + :success, + "Removed #{recipient} as a recipient for the monthly report" + ) + |> redirect(to: Routes.site_path(conn, :settings_email_reports, site.domain)) + end + + def enable_traffic_change_notification(conn, %{"type" => type}) do + site = conn.assigns[:site] + + res = + Plausible.Site.TrafficChangeNotification.changeset( + %Plausible.Site.TrafficChangeNotification{}, + %{ + site_id: site.id, + type: type, + threshold: if(type == "spike", do: 10, else: 1), + recipients: [conn.assigns[:current_user].email] + } + ) + |> Repo.insert() + + case res do + {:ok, _} -> + conn + |> put_flash(:success, "Traffic #{type} notifications enabled") + |> redirect(to: Routes.site_path(conn, :settings_email_reports, site.domain)) + + {:error, _} -> + conn + |> put_flash(:error, "Unable to create a #{type} notification") + |> redirect(to: Routes.site_path(conn, :settings_email_reports, site.domain)) + end + end + + def disable_traffic_change_notification(conn, %{"type" => type}) do + site = conn.assigns[:site] + + Repo.delete_all( + from(mr in Plausible.Site.TrafficChangeNotification, + where: mr.site_id == ^site.id and mr.type == ^type + ) + ) + + conn + |> put_flash(:success, "Traffic #{type} notifications disabled") + |> redirect(to: Routes.site_path(conn, :settings_email_reports, site.domain)) + end + + def update_traffic_change_notification(conn, %{ + "traffic_change_notification" => params, + "type" => type + }) do + site = conn.assigns[:site] + + notification = + Repo.get_by(Plausible.Site.TrafficChangeNotification, site_id: site.id, type: type) + + Plausible.Site.TrafficChangeNotification.changeset(notification, params) + |> Repo.update!() + + conn + |> put_flash(:success, "Notification settings updated") + |> redirect(to: Routes.site_path(conn, :settings_email_reports, site.domain)) + end + + def add_traffic_change_notification_recipient(conn, %{"recipient" => recipient, "type" => type}) do + site = conn.assigns[:site] + + Repo.get_by(Plausible.Site.TrafficChangeNotification, site_id: site.id, type: type) + |> Plausible.Site.TrafficChangeNotification.add_recipient(recipient) + |> Repo.update!() + + conn + |> put_flash(:success, "Added #{recipient} as a recipient for the traffic spike notification") + |> redirect(to: Routes.site_path(conn, :settings_email_reports, site.domain)) + end + + def remove_traffic_change_notification_recipient(conn, %{ + "recipient" => recipient, + "type" => type + }) do + site = conn.assigns[:site] + + Repo.get_by(Plausible.Site.TrafficChangeNotification, site_id: site.id, type: type) + |> Plausible.Site.TrafficChangeNotification.remove_recipient(recipient) + |> Repo.update!() + + conn + |> put_flash( + :success, + "Removed #{recipient} as a recipient for the monthly report" + ) + |> redirect(to: Routes.site_path(conn, :settings_email_reports, site.domain)) + end + + def forget_import(conn, %{"import_id" => import_id}) do + site = conn.assigns.site + + if site_import = Plausible.Imported.get_import(site, import_id) do + Oban.cancel_all_jobs( + from(j in Oban.Job, + where: + j.queue == "analytics_imports" and + fragment("(? ->> 'import_id')::int", j.args) == ^site_import.id + ) + ) + + Plausible.Purge.delete_imported_stats!(site_import) + + Plausible.Repo.delete!(site_import) + end + + conn + |> put_flash(:success, "Imported data has been cleared") + |> redirect(to: Routes.site_path(conn, :settings_imports_exports, site.domain)) + end + + def forget_imported(conn, _params) do + site = conn.assigns.site + + import_ids = + site + |> Plausible.Imported.list_all_imports() + |> Enum.map(& &1.id) + + if import_ids != [] do + Oban.cancel_all_jobs( + from(j in Oban.Job, + where: + j.queue == "analytics_imports" and + fragment("(? ->> 'import_id')::int", j.args) in ^import_ids + ) + ) + + Plausible.Purge.delete_imported_stats!(site) + + Plausible.Imported.delete_imports_for_site(site) + end + + conn + |> put_flash(:success, "Imported data has been cleared") + |> redirect(to: Routes.site_path(conn, :settings_integrations, site.domain)) + end + + on_ee do + def download_export(conn, _params) do + %{id: site_id, domain: domain} = conn.assigns.site + + if s3_export = Plausible.Exports.get_s3_export!(site_id) do + s3_bucket = Plausible.S3.exports_bucket() + download_url = Plausible.S3.download_url(s3_bucket, s3_export.path) + redirect(conn, external: download_url) + else + conn + |> put_flash(:error, "Export not found") + |> redirect(to: Routes.site_path(conn, :settings_imports_exports, domain)) + end + end + else + def download_export(conn, _params) do + %{id: site_id, domain: domain, timezone: timezone} = conn.assigns.site + + if local_export = Plausible.Exports.get_local_export(site_id, domain, timezone) do + %{path: export_path, name: name} = local_export + + conn + |> put_resp_content_type("application/zip") + |> put_resp_header("content-disposition", Plausible.Exports.content_disposition(name)) + |> send_file(200, export_path) + else + conn + |> put_flash(:error, "Export not found") + |> redirect(to: Routes.site_path(conn, :settings_imports_exports, domain)) + end + end + end + + def csv_import(conn, _params) do + conn + |> assign(:skip_plausible_tracking, true) + |> render("csv_import.html", + connect_live_socket: true + ) + end + + defp tolerate_unique_contraint_violation(result, name) do + case result do + {:ok, _} -> + :ok + + {:error, + %{ + errors: [ + site_id: {_, [constraint: :unique, constraint_name: ^name]} + ] + }} -> + :ok + + other -> + other + end + end + + defp check_can_already_access(changeset, user) do + domain = Ecto.Changeset.get_change(changeset, :domain) + + with true <- domain_taken_error?(changeset), + site = Sites.get_by_domain(domain), + true <- not is_nil(site), + true <- Plausible.Teams.Memberships.has_editor_access?(site, user) do + {:ok, domain} + else + false -> {:error, :no_access} + end + end + + defp domain_taken_error?(changeset) do + Enum.any?(changeset.errors, fn + {:domain, {_, [constraint: :unique, constraint_name: "sites_domain_index"]}} -> true + _ -> false + end) + end +end diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..38e9169f66f930eda8fca6f47abaa76aab5a49f8 --- /dev/null +++ b/lib/plausible_web/controllers/stats_controller.ex @@ -0,0 +1,460 @@ +defmodule PlausibleWeb.StatsController do + use Plausible + + @moduledoc """ + This controller is responsible for rendering stats dashboards. + + The stats dashboards are currently the only part of the app that uses client-side + rendering. Since the dashboards are heavily interactive, they are built with React + which is an appropriate choice for highly interactive browser UIs. + +
+ sequenceDiagram + Browser->>StatsController: GET /mydomain.com + StatsController-->>Browser: StatsView.render("stats.html") + Note left of Browser: ReactDom.render(Dashboard) + + Browser -) Api.StatsController: GET /api/stats/mydomain.com/top-stats + Api.StatsController --) Browser: {"top_stats": [...]} + Note left of Browser: TopStats.render() + + Browser -) Api.StatsController: GET /api/stats/mydomain.com/sources + Api.StatsController --) Browser: [{"name": "Google", "visitors": 292150}, ...] + Note left of Browser: Sources.render() + + Note over Browser,StatsController: And so on, for all reports in the viewport +
+ + This reasoning for this sequence is as follows: + 1. First paint is fast because it doesn't do any data aggregation yet - good UX + 2. The basic structure of the dashboard is rendered with spinners before reports are ready - good UX + 2. Rendering on the frontend allows for maximum interactivity. Re-rendering and re-fetching can be as granular as needed. + 3. Routing on the frontend allows the user to navigate the dashboard without reloading the page and losing context + 4. Rendering on the frontend allows caching results in the browser to reduce pressure on backends and storage + 3.1 No client-side caching has been implemented yet. This is still theoretical. See https://github.com/plausible/analytics/discussions/1278 + 3.2 This is a big potential opportunity, because analytics data is mostly immutable. Clients can cache all historical data. + 5. Since frontend rendering & navigation is harder to build and maintain than regular server-rendered HTML, we don't use SPA-style rendering anywhere else + .The only place currently where the benefits outweigh the costs is the dashboard. + """ + + use PlausibleWeb, :controller + use Plausible.Repo + + alias Plausible.Sites + alias Plausible.Teams + alias Plausible.Billing.Feature.SharedLinks + + plug(PlausibleWeb.Plugs.AuthorizeSiteAccess when action in [:stats]) + + def stats(%{assigns: %{site: site}} = conn, _params) do + site = Plausible.Repo.preload(site, :owners) + site_role = conn.assigns[:site_role] + current_user = conn.assigns[:current_user] + stats_start_date = Plausible.Sites.stats_start_date(site) + can_see_stats? = not Teams.locked?(site.team) or site_role == :super_admin + demo = site.domain == "plausible.io" + dogfood_page_path = if demo, do: "/#{site.domain}", else: "/:dashboard" + + consolidated_view? = Plausible.Sites.consolidated?(site) + + {exploration_journey_end_event, exploration_max_journey_steps} = + on_ee( + do: + {Plausible.Stats.Exploration.Journey.Step.journey_end_event(), + Plausible.Stats.Exploration.max_steps()}, + else: {"", 0} + ) + + consolidated_view_available? = + on_ee(do: Plausible.ConsolidatedView.ok_to_display?(site.team), else: false) + + team_identifier = site.team.identifier + + skip_to_dashboard? = + conn.params["skip_to_dashboard"] == "true" or consolidated_view? + + {:ok, segments} = Plausible.Segments.get_all_for_site(site, site_role) + segments = Enum.map(segments, &Plausible.Segments.to_response_map(&1, site)) + + cond do + consolidated_view? and not consolidated_view_available? and site_role != :super_admin -> + redirect(conn, to: Routes.site_path(conn, :index)) + + (stats_start_date && can_see_stats?) || (can_see_stats? && skip_to_dashboard?) -> + flags = get_flags(current_user, site) + + conn + |> put_resp_header("x-robots-tag", "noindex, nofollow") + |> render("stats.html", + site: site, + site_role: site_role, + has_goals: Plausible.Sites.has_goals?(site), + revenue_goals: list_revenue_goals(site), + funnels: list_funnels(site), + has_props: Plausible.Props.configured?(site), + stats_start_date: stats_start_date, + native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at), + title: title(conn, site), + demo: demo, + flags: flags, + dbip?: dbip?(), + segments: segments, + load_dashboard_js: true, + hide_footer?: if(ce?() || demo, do: false, else: site_role != :public), + consolidated_view?: consolidated_view?, + consolidated_view_available?: consolidated_view_available?, + exploration_journey_end_event: exploration_journey_end_event, + exploration_max_journey_steps: exploration_max_journey_steps, + team_identifier: team_identifier, + limited_to_segment_id: nil + ) + + !stats_start_date && can_see_stats? -> + redirect(conn, to: Routes.site_path(conn, :verification, site.domain)) + + Teams.locked?(site.team) -> + site = Plausible.Repo.preload(site, :owners) + render(conn, "site_locked.html", site: site, dogfood_page_path: dogfood_page_path) + end + end + + on_ee do + defp list_funnels(site) do + Plausible.Funnels.list(site) + end + + defp list_revenue_goals(site) do + Plausible.Goals.list_revenue_goals(site) + end + else + defp list_funnels(_site), do: [] + defp list_revenue_goals(_site), do: [] + end + + @doc """ + Authorizes and renders a shared link: + 1. Shared link with no password protection: needs to just make sure the shared link entry is still + in our database. This check makes sure shared link access can be revoked by the site admins. If the + shared link exists, render it directly. + + 2. Shared link with password protection: Same checks as without the password, but an extra step is taken to + protect the page with a password. When the user passes the password challenge, a cookie is set with Plausible.Auth.Token.sign_shared_link(). + The cookie allows the user to access the dashboard for 24 hours without entering the password again. + + ### Backwards compatibility + + The URL format for shared links was changed in [this pull request](https://github.com/plausible/analytics/pull/752) in order + to make the URLs easier to bookmark. The old format is supported along with the new in order to not break old links. + + See: https://plausible.io/docs/shared-links + """ + def shared_link(conn, %{"domain" => domain, "auth" => auth}) do + case find_shared_link(domain, auth) do + {:ok, shared_link} -> + if Plausible.Site.SharedLink.password_protected?(shared_link) do + render_password_protected_shared_link(conn, shared_link) + else + render_shared_link(conn, shared_link) + end + + {:error, :not_found} -> + render_error(conn, 404) + end + end + + @old_format_deprecation_date ~N[2022-01-01 00:00:00] + def shared_link(conn, %{"domain" => slug}) do + shared_link = + Repo.one( + from(l in Plausible.Site.SharedLink, + where: l.slug == ^slug and l.inserted_at < ^@old_format_deprecation_date, + preload: :site + ) + ) + + if shared_link do + new_link_format = + Routes.stats_path(conn, :shared_link, shared_link.site.domain, [], auth: slug) + + redirect(conn, to: new_link_format) + else + render_error(conn, 404) + end + end + + def shared_link(conn, _) do + render_error(conn, 400) + end + + def validate_shared_link_password(conn, shared_link) do + with {:ok, token} <- Map.fetch(conn.req_cookies, shared_link_cookie_name(shared_link.slug)), + {:ok, %{slug: token_slug}} <- Plausible.Auth.Token.verify_shared_link(token), + true <- token_slug == shared_link.slug do + {:ok, shared_link} + else + _e -> {:error, :unauthorized} + end + end + + defp render_password_protected_shared_link(conn, shared_link) do + conn = Plug.Conn.fetch_cookies(conn) + + # discard untrustworthy return_to given from query params + trimmed_query_string = conn.query_string |> omit_from_query_string("return_to") + star_path_fragment = serialize_star_path_as_query_string_fragment(conn) + + # set valid return_to if star path is set + query_string = + [trimmed_query_string, star_path_fragment] + |> Enum.filter(fn v -> is_binary(v) and String.length(v) > 0 end) + |> Enum.join("&") + + case validate_shared_link_password(conn, shared_link) do + {:ok, shared_link} -> + render_shared_link(conn, shared_link) + + _ -> + conn + |> render("shared_link_password.html", + link: shared_link, + query_string: query_string, + dogfood_page_path: "/share/:dashboard" + ) + end + end + + defp find_shared_link(domain, auth) do + link_query = + from(link in Plausible.Site.SharedLink, + inner_join: site in assoc(link, :site), + inner_join: team in assoc(site, :team), + where: link.slug == ^auth, + where: site.domain == ^domain, + limit: 1, + preload: [site: {site, team: team}] + ) + + case Repo.one(link_query) do + %Plausible.Site.SharedLink{} = link -> + {:ok, link} + + nil -> + {:error, :not_found} + end + end + + def authenticate_shared_link(conn, %{"slug" => slug, "password" => password}) do + shared_link = + Repo.get_by(Plausible.Site.SharedLink, slug: slug) + |> Repo.preload(:site) + + if shared_link do + if Plausible.Auth.Password.match?(password, shared_link.password_hash) do + token = Plausible.Auth.Token.sign_shared_link(slug) + + star_path = parse_star_path(conn) + + # The filter query params format used by the FE breaks when it passes through Phoenix / Plug.Conn decode/encode. + # This function works around that by using the original query string. + query_string_fragment = + get_rest_of_query_string(conn) + # omitted because return_to param was needed only for this function + |> omit_from_query_string("return_to") + # omitted because `auth: slug` query param is set definitively below + |> omit_from_query_string("auth") + + conn + |> put_resp_cookie(shared_link_cookie_name(slug), token) + |> redirect( + to: + Routes.stats_path( + conn, + :shared_link, + shared_link.site.domain, + star_path, + auth: slug + ) <> + query_string_fragment + ) + else + conn + |> render("shared_link_password.html", + link: shared_link, + error: "Incorrect password. Please try again.", + query_string: conn.query_string, + dogfood_page_path: "/share/:dashboard" + ) + end + else + render_error(conn, 404) + end + end + + defp serialize_star_path_as_query_string_fragment(conn) do + star_path = conn.path_params["path"] + + if length(star_path) > 0 do + # make the path start with a / + # to be able to reject values that don't start with a / + %{"return_to" => "/#{Enum.join(star_path, "/")}"} |> URI.encode_query() + else + nil + end + end + + defp parse_star_path(conn) do + case conn.query_params["return_to"] do + # omit prefix added in `serialize_star_path_as_query_string_fragment` + "/" <> return_to -> + return_to + |> String.split("/") + # disallow constructing links that navigate up + |> Enum.filter(fn part -> part !== ".." end) + + _ -> + [] + end + end + + defp get_rest_of_query_string(conn) when conn.query_string in [nil, ""], do: "" + + defp get_rest_of_query_string(conn), do: "&#{conn.query_string}" + + defp omit_from_query_string(query_string, key) do + query_string + |> String.split("&") + |> Enum.reject(fn key_and_value -> + key_and_value == key || String.starts_with?(key_and_value, "#{key}=") + end) + |> Enum.join("&") + end + + defp render_shared_link(conn, shared_link) do + shared_links_feature_access? = + SharedLinks.check_availability(shared_link.site.team) == :ok or + shared_link.name in Plausible.Sites.shared_link_special_names() + + cond do + Teams.locked?(shared_link.site.team) -> + owners = Plausible.Repo.preload(shared_link.site, :owners) + + render(conn, "site_locked.html", + owners: owners, + site: shared_link.site, + dogfood_page_path: "/share/:dashboard" + ) + + not shared_links_feature_access? -> + owners = Plausible.Repo.preload(shared_link.site, :owners) + + render(conn, "site_locked.html", + only_shared_link_access_missing?: true, + owners: owners, + site: shared_link.site, + dogfood_page_path: "/share/:dashboard" + ) + + not Teams.locked?(shared_link.site.team) -> + current_user = conn.assigns[:current_user] + site_role = get_fallback_site_role(conn) + shared_link = Plausible.Repo.preload(shared_link, :segment, site: [:owners]) + stats_start_date = Plausible.Sites.stats_start_date(shared_link.site) + + flags = get_flags(current_user, shared_link.site) + + {exploration_journey_end_event, exploration_max_journey_steps} = + on_ee( + do: + {Plausible.Stats.Exploration.Journey.Step.journey_end_event(), + Plausible.Stats.Exploration.max_steps()}, + else: {"", 0} + ) + + limited_to_segment_id = + if Plausible.Site.SharedLink.limited_to_segment?(shared_link) do + shared_link.segment.id + else + nil + end + + segments = + if is_nil(limited_to_segment_id) do + {:ok, segments} = Plausible.Segments.get_all_for_site(shared_link.site, site_role) + Enum.map(segments, &Plausible.Segments.to_response_map(&1, shared_link.site)) + else + shared_link.segment + |> Plausible.Segments.to_response_map(shared_link.site) + |> List.wrap() + end + + embedded? = conn.params["embed"] == "true" + + true = Plausible.Sites.regular?(shared_link.site) + + team_identifier = shared_link.site.team.identifier + + conn + |> put_resp_header("x-robots-tag", "noindex, nofollow") + |> delete_resp_header("x-frame-options") + |> render("stats.html", + site: shared_link.site, + site_role: site_role, + has_goals: Sites.has_goals?(shared_link.site), + revenue_goals: list_revenue_goals(shared_link.site), + funnels: list_funnels(shared_link.site), + has_props: Plausible.Props.configured?(shared_link.site), + stats_start_date: stats_start_date, + native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at), + title: title(conn, shared_link.site), + demo: false, + shared_link_auth: shared_link.slug, + embedded: embedded?, + background: conn.params["background"], + theme: conn.params["theme"], + flags: flags, + dbip?: dbip?(), + segments: segments, + load_dashboard_js: true, + hide_footer?: if(ce?(), do: embedded?, else: embedded? || site_role != :public), + # no shared links for consolidated views + consolidated_view?: false, + consolidated_view_available?: false, + exploration_journey_end_event: exploration_journey_end_event, + exploration_max_journey_steps: exploration_max_journey_steps, + team_identifier: team_identifier, + limited_to_segment_id: limited_to_segment_id + ) + end + end + + defp get_fallback_site_role(conn), + do: if(role = conn.assigns[:site_role], do: role, else: :public) + + defp shared_link_cookie_name(slug), do: "shared-link-" <> slug + + defp get_flags(user, site), + do: + [] + |> Enum.map(fn flag -> + {flag, FunWithFlags.enabled?(flag, for: user) || FunWithFlags.enabled?(flag, for: site)} + end) + |> Map.new() + + defp dbip?() do + on_ee do + false + else + Plausible.Geo.database_type() + |> to_string() + |> String.starts_with?("DBIP") + end + end + + defp title(%{path_info: ["plausible.io"]}, _) do + "Plausible Analytics: Live Demo" + end + + defp title(_conn, site) do + "Plausible · " <> site.domain + end +end diff --git a/lib/plausible_web/controllers/unsubscribe_controller.ex b/lib/plausible_web/controllers/unsubscribe_controller.ex new file mode 100644 index 0000000000000000000000000000000000000000..7112cb1250c044f325a1cd448598ec3ab9a998c6 --- /dev/null +++ b/lib/plausible_web/controllers/unsubscribe_controller.ex @@ -0,0 +1,49 @@ +defmodule PlausibleWeb.UnsubscribeController do + use PlausibleWeb, :controller + use Plausible.Repo + alias Plausible.Site.{WeeklyReport, MonthlyReport} + + def weekly_report(conn, %{"domain" => domain, "email" => email}) do + site = Repo.get_by(Plausible.Site, domain: domain) + weekly_report = site && Repo.get_by(WeeklyReport, site_id: site.id) + + if weekly_report do + weekly_report + |> WeeklyReport.remove_recipient(email) + |> Repo.update!() + end + + conn + |> assign(:skip_plausible_tracking, true) + |> render("success.html", + interval: "weekly", + site: site || %{domain: domain} + ) + end + + def weekly_report(conn, _) do + render_error(conn, 400) + end + + def monthly_report(conn, %{"domain" => domain, "email" => email}) do + site = Repo.get_by(Plausible.Site, domain: domain) + monthly_report = site && Repo.get_by(MonthlyReport, site_id: site.id) + + if monthly_report do + monthly_report + |> MonthlyReport.remove_recipient(email) + |> Repo.update!() + end + + conn + |> assign(:skip_plausible_tracking, true) + |> render("success.html", + interval: "monthly", + site: site || %{domain: domain} + ) + end + + def monthly_report(conn, _) do + render_error(conn, 400) + end +end diff --git a/lib/plausible_web/email.ex b/lib/plausible_web/email.ex new file mode 100644 index 0000000000000000000000000000000000000000..b3b6f721d1e180bb4c6335ca796e601fa889ead7 --- /dev/null +++ b/lib/plausible_web/email.ex @@ -0,0 +1,694 @@ +defmodule PlausibleWeb.Email do + @moduledoc """ + Email template rendering functions. + """ + + use Plausible + import Bamboo.Email + import Bamboo.PostmarkHelper + + def mailer_email_from do + Application.get_env(:plausible, :mailer_email) + end + + def activation_email(user, code) do + priority_email() + |> to(user) + |> tag("activation-email") + |> subject("#{code} is your Plausible email verification code") + |> render("activation_email.html", user: user, code: code) + end + + def welcome_email(user) do + base_email() + |> to(user) + |> tag("welcome-email") + |> subject("Welcome to Plausible") + |> render("welcome_email.html", user: user) + end + + def create_site_email(user) do + base_email() + |> to(user) + |> tag("create-site-email") + |> subject("Add your first site to start tracking") + |> render("create_site_email.html", user: user) + end + + def site_setup_help(user, team, site) do + base_email() + |> to(user) + |> tag("help-email") + |> subject("No traffic recorded yet") + |> render("site_setup_help_email.html", + user: user, + site: site, + site_team: team + ) + end + + def site_setup_success(user, site) do + base_email() + |> to(user) + |> tag("setup-success-email") + |> subject("Your first visitor just showed up") + |> render("site_setup_success_email.html", + user: user, + site: site + ) + end + + def check_stats_email(user) do + base_email() + |> to(user) + |> tag("check-stats-email") + |> subject("How Plausible is different") + |> render("check_stats_email.html", user: user) + end + + def password_reset_email(email, reset_link) do + priority_email(%{layout: nil}) + |> to(email) + |> tag("password-reset-email") + |> subject("Plausible password reset") + |> render("password_reset_email.html", reset_link: reset_link) + end + + def two_factor_enabled_email(user) do + priority_email() + |> to(user) + |> tag("two-factor-enabled-email") + |> subject("Plausible Two-Factor Authentication enabled") + |> render("two_factor_enabled_email.html", user: user) + end + + def two_factor_disabled_email(user) do + priority_email() + |> to(user) + |> tag("two-factor-disabled-email") + |> subject("Plausible Two-Factor Authentication disabled") + |> render("two_factor_disabled_email.html", user: user) + end + + def trial_one_week_reminder(user, team) do + base_email() + |> to(user) + |> tag("trial-one-week-reminder") + |> subject("Your Plausible trial ends in one week") + |> render("trial_one_week_reminder.html", user: user, team: team) + end + + def trial_ending_tomorrow_email(user, team, usage, suggested_volume) do + base_email() + |> to(user) + |> tag("trial-ending-tomorrow") + |> subject("Your Plausible trial ends tomorrow") + |> render("trial_ending_tomorrow.html", + user: user, + team: team, + custom_events: usage.custom_events, + usage: usage.total, + suggested_volume: suggested_volume + ) + end + + def trial_ending_today_email(user, team, usage, suggested_volume) do + base_email() + |> to(user) + |> tag("trial-ending-today") + |> subject("Your Plausible trial ends today") + |> render("trial_ending_today.html", + user: user, + team: team, + custom_events: usage.custom_events, + usage: usage.total, + suggested_volume: suggested_volume + ) + end + + def trial_over_email(user, team) do + base_email() + |> to(user) + |> tag("trial-over-email") + |> subject("Your Plausible trial has expired") + |> render("trial_over_email.html", + user: user, + team: team, + extra_offset: Plausible.Teams.Team.trial_accept_traffic_until_offset_days() + ) + end + + def stats_report(email, assigns) do + base_email(%{layout: nil}) + |> to(email) + |> tag("#{assigns.interval}-report") + |> subject("#{assigns.report_name} report for #{assigns.site.domain}") + |> html_body(PlausibleWeb.MJML.StatsReport.render(assigns)) + end + + def spike_notification(email, site, stats, dashboard_link) do + base_email() + |> to(email) + |> tag("spike-notification") + |> subject( + "Traffic spike on #{if Plausible.Sites.consolidated?(site), do: "your sites", else: site.domain}" + ) + |> render("spike_notification.html", %{ + site: site, + current_visitors: stats.current_visitors, + sources: stats.sources, + pages: stats.pages, + link: dashboard_link + }) + end + + def drop_notification(email, site, current_visitors, dashboard_link, installation_link) do + base_email() + |> to(email) + |> tag("drop-notification") + |> subject( + "Traffic drop on #{if Plausible.Sites.consolidated?(site), do: "your sites", else: site.domain}" + ) + |> render("drop_notification.html", %{ + site: site, + current_visitors: current_visitors, + dashboard_link: dashboard_link, + installation_link: installation_link + }) + end + + def over_limit_email(user, team, usage, suggested_volume) do + priority_email() + |> to(user) + |> tag("over-limit") + |> subject("[Action required] You have outgrown your Plausible subscription tier") + |> render("over_limit.html", %{ + user: user, + team: team, + usage: usage, + suggested_volume: suggested_volume + }) + end + + def enterprise_over_limit_internal_email(user, pageview_usage, site_usage, site_allowance) do + base_email(%{layout: nil}) + |> to("enterprise@plausible.io") + |> tag("enterprise-over-limit") + |> subject("#{user.email} has outgrown their enterprise plan") + |> render("enterprise_over_limit_internal.html", %{ + user: user, + pageview_usage: pageview_usage, + site_usage: site_usage, + site_allowance: site_allowance + }) + end + + def dashboard_locked(user, team, usage, suggested_volume) do + priority_email() + |> to(user) + |> tag("dashboard-locked") + |> subject("[Action required] Your Plausible dashboard is now locked") + |> render("dashboard_locked.html", %{ + user: user, + team: team, + usage: usage, + suggested_volume: suggested_volume + }) + end + + def yearly_renewal_notification(team, owner) do + date = Calendar.strftime(team.subscription.next_bill_date, "%B %-d, %Y") + + priority_email() + |> to(owner) + |> tag("yearly-renewal") + |> subject("Your Plausible subscription is up for renewal") + |> render("yearly_renewal_notification.html", %{ + user: owner, + team: team, + date: date, + next_bill_amount: team.subscription.next_bill_amount, + currency: team.subscription.currency_code + }) + end + + def yearly_expiration_notification(team, owner) do + next_bill_date = Calendar.strftime(team.subscription.next_bill_date, "%B %-d, %Y") + + accept_traffic_until = + team + |> Plausible.Teams.accept_traffic_until() + |> Calendar.strftime("%B %-d, %Y") + + priority_email() + |> to(owner) + |> tag("yearly-expiration") + |> subject("Your Plausible subscription is about to expire") + |> render("yearly_expiration_notification.html", %{ + user: owner, + team: team, + next_bill_date: next_bill_date, + accept_traffic_until: accept_traffic_until + }) + end + + def cancellation_email(user) do + base_email() + |> to(user.email) + |> tag("cancelled-email") + |> subject("Mind sharing your thoughts on Plausible?") + |> render("cancellation_email.html", user: user) + end + + def new_user_invitation(email, invitation_id, site, inviter) do + priority_email() + |> to(email) + |> tag("new-user-invitation") + |> subject("[#{Plausible.product_name()}] You've been invited to #{site.domain}") + |> render("new_user_invitation.html", + invitation_id: invitation_id, + site: site, + inviter: inviter + ) + end + + def existing_user_invitation(email, site, inviter) do + priority_email() + |> to(email) + |> tag("existing-user-invitation") + |> subject("[#{Plausible.product_name()}] You've been invited to #{site.domain}") + |> render("existing_user_invitation.html", + site: site, + inviter: inviter + ) + end + + def new_user_team_invitation(email, invitation_id, team, inviter) do + priority_email() + |> to(email) + |> tag("new-user-team-invitation") + |> subject("[#{Plausible.product_name()}] You've been invited to \"#{team.name}\" team") + |> render("new_user_team_invitation.html", + invitation_id: invitation_id, + team: team, + inviter: inviter + ) + end + + def existing_user_team_invitation(email, team, inviter) do + priority_email() + |> to(email) + |> tag("existing-user-team-invitation") + |> subject("[#{Plausible.product_name()}] You've been invited to \"#{team.name}\" team") + |> render("existing_user_team_invitation.html", + team: team, + inviter: inviter + ) + end + + def guest_to_team_member_promotion(email, team, inviter) do + priority_email() + |> to(email) + |> tag("guest-to-team-member-promotion") + |> subject("[#{Plausible.product_name()}] Welcome to \"#{team.name}\" team") + |> render("guest_to_team_member_promotion.html", inviter: inviter, team: team) + end + + def ownership_transfer_request(email, invitation_id, site, inviter, new_owner_account) do + priority_email() + |> to(email) + |> tag("ownership-transfer-request") + |> subject("[#{Plausible.product_name()}] Request to transfer ownership of #{site.domain}") + |> render("ownership_transfer_request.html", + invitation_id: invitation_id, + inviter: inviter, + site: site, + new_owner_account: new_owner_account + ) + end + + def guest_invitation_accepted(inviter_email, invitee_email, team, site) do + priority_email() + |> to(inviter_email) + |> tag("guest-invitation-accepted") + |> subject( + "[#{Plausible.product_name()}] #{invitee_email} accepted your invitation to #{site.domain}" + ) + |> render("guest_invitation_accepted.html", + invitee_email: invitee_email, + team: team, + site: site + ) + end + + def team_invitation_accepted(inviter_email, invitee_email, team) do + priority_email() + |> to(inviter_email) + |> tag("team-invitation-accepted") + |> subject( + "[#{Plausible.product_name()}] #{invitee_email} accepted your invitation to \"#{team.name}\" team" + ) + |> render("team_invitation_accepted.html", + invitee_email: invitee_email, + team: team + ) + end + + def guest_invitation_rejected(guest_invitation) do + priority_email() + |> to(guest_invitation.team_invitation.inviter.email) + |> tag("guest-invitation-rejected") + |> subject( + "[#{Plausible.product_name()}] #{guest_invitation.team_invitation.email} rejected your invitation to #{guest_invitation.site.domain}" + ) + |> render("guest_invitation_rejected.html", + team: guest_invitation.team_invitation.team, + guest_invitation: guest_invitation + ) + end + + def team_invitation_rejected(team_invitation) do + priority_email() + |> to(team_invitation.inviter.email) + |> tag("team-invitation-rejected") + |> subject( + "[#{Plausible.product_name()}] #{team_invitation.email} rejected your invitation to \"#{team_invitation.team.name}\" team" + ) + |> render("team_invitation_rejected.html", + team: team_invitation.team, + team_invitation: team_invitation + ) + end + + def ownership_transfer_accepted( + new_owner_email, + inviter_email, + team, + site, + initiator_as_guest_editor? + ) do + priority_email() + |> to(inviter_email) + |> tag("ownership-transfer-accepted") + |> subject( + "[#{Plausible.product_name()}] #{new_owner_email} accepted the ownership transfer of #{site.domain}" + ) + |> render("ownership_transfer_accepted.html", + new_owner_email: new_owner_email, + team: team, + site: site, + initiator_as_guest_editor?: initiator_as_guest_editor? + ) + end + + def team_changed(owner_email, user, team, site) do + priority_email() + |> to(owner_email) + |> tag("team-changed") + |> subject( + "[#{Plausible.product_name()}] #{user.email} has transferred #{site.domain} to \"#{team.name}\" team" + ) + |> render("team_changed.html", + user: user, + team: team, + site: site + ) + end + + def ownership_transfer_rejected(site_transfer) do + priority_email() + |> to(site_transfer.initiator.email) + |> tag("ownership-transfer-rejected") + |> subject( + "[#{Plausible.product_name()}] #{site_transfer.email} rejected the ownership transfer of #{site_transfer.site.domain}" + ) + |> render("ownership_transfer_rejected.html", + user: site_transfer.initiator, + team: site_transfer.site.team, + site_transfer: site_transfer + ) + end + + def site_member_removed(guest_membership) do + priority_email() + |> to(guest_membership.team_membership.user.email) + |> tag("site-member-removed") + |> subject( + "[#{Plausible.product_name()}] Your access to #{guest_membership.site.domain} has been revoked" + ) + |> render("site_member_removed.html", + user: guest_membership.team_membership.user, + guest_membership: guest_membership + ) + end + + def team_member_removed(team_membership) do + priority_email() + |> to(team_membership.user.email) + |> tag("team-member-removed") + |> subject( + "[#{Plausible.product_name()}] Your access to \"#{team_membership.team.name}\" team has been revoked" + ) + |> render("team_member_removed.html", + user: team_membership.user, + team_membership: team_membership + ) + end + + def team_member_left(team_membership) do + priority_email() + |> to(team_membership.user.email) + |> tag("team-member-left") + |> subject( + "[#{Plausible.product_name()}] You have left \"#{team_membership.team.name}\" team" + ) + |> render("team_member_left.html", + user: team_membership.user, + team_membership: team_membership + ) + end + + def import_success(site_import, user) do + import_api = Plausible.Imported.ImportSources.by_name(site_import.source) + label = import_api.label() + team = site_import.site.team + + priority_email() + |> to(user) + |> tag("import-success-email") + |> subject("#{label} data imported for #{site_import.site.domain}") + |> render(import_api.email_template(), %{ + site_import: site_import, + label: label, + link: + PlausibleWeb.Endpoint.url() <> + "/" <> URI.encode_www_form(site_import.site.domain) <> "?__team=#{team.identifier}", + user: user, + success: true + }) + end + + def import_failure(site_import, user) do + import_api = Plausible.Imported.ImportSources.by_name(site_import.source) + label = import_api.label() + + priority_email() + |> to(user) + |> tag("import-failure-email") + |> subject("#{label} import failed for #{site_import.site.domain}") + |> render(import_api.email_template(), %{ + site_import: site_import, + label: label, + user: user, + success: false + }) + end + + def export_success(user, site, expires_at) do + expires_in = + if expires_at do + Plausible.Times.humanize(expires_at) + end + + download_url = + PlausibleWeb.Router.Helpers.site_url( + PlausibleWeb.Endpoint, + :download_export, + site.domain + ) <> "?__team=#{site.team.identifier}" + + priority_email() + |> to(user) + |> tag("export-success") + |> subject("[#{Plausible.product_name()}] Your export is now ready for download") + |> render("export_success.html", + user: user, + site: site, + download_url: download_url, + expires_in: expires_in + ) + end + + def export_failure(user, site) do + priority_email() + |> to(user) + |> subject("[#{Plausible.product_name()}] Your export has failed") + |> render("export_failure.html", user: user, site: site) + end + + def error_report(reported_by, trace_id, feedback) do + Map.new() + |> Map.put(:layout, nil) + |> base_email() + |> to("bugs@plausible.io") + |> put_param("ReplyTo", reported_by) + |> tag("sentry") + |> subject("Feedback to Sentry Trace #{trace_id}") + |> render("error_report_email.html", %{ + reported_by: reported_by, + feedback: feedback, + trace_id: trace_id + }) + end + + def approaching_accept_traffic_until(notification) do + base_email() + |> to(notification.email) + |> tag("drop-traffic-warning-first") + |> subject("Your stats stop collecting soon") + |> render("approaching_accept_traffic_until.html", + time: "next week", + user: %{email: notification.email, name: notification.name}, + team: notification.team + ) + end + + def approaching_accept_traffic_until_tomorrow(notification) do + base_email() + |> to(notification.email) + |> tag("drop-traffic-warning-final") + |> subject("Your stats stop tomorrow") + |> render("approaching_accept_traffic_until.html", + time: "tomorrow", + user: %{email: notification.email, name: notification.name}, + team: notification.team + ) + end + + on_ee do + def sso_domain_verification_success(domain, user) do + priority_email() + |> to(user.email) + |> subject("Your SSO domain #{domain} is ready!") + |> render("sso_domain_verification_success.html", domain: domain) + end + + def sso_domain_verification_failure(domain, user) do + priority_email() + |> to(user) + |> subject("SSO domain #{domain} verification failure") + |> render("sso_domain_verification_failure.html", domain: domain) + end + end + + def force_2fa_enabled(team, user, enabling_user) do + priority_email() + |> to(user.email) + |> subject("Your team now requires 2FA") + |> render("force_2fa_enabled.html", team: team, enabling_user: enabling_user) + end + + @doc """ + Unlike the default 'base' emails, priority emails cannot be unsubscribed from. This is achieved + by sending them through a dedicated 'priority' message stream in Postmark. + """ + def priority_email(), do: priority_email(%{layout: "priority_email.html"}) + + def priority_email(%{layout: layout}) do + email = base_email(%{layout: layout}) + + if ee?() do + put_param(email, "MessageStream", "priority") + else + email + end + end + + def base_email(), do: base_email(%{layout: "base_email.html"}) + + def base_email(%{layout: layout}) do + new_email() + |> put_param("TrackOpens", false) + |> from(mailer_email_from()) + |> maybe_put_layout(layout) + end + + defp maybe_put_layout(email, nil), do: email + + defp maybe_put_layout(%{assigns: assigns} = email, layout) do + %{email | assigns: Map.put(assigns, :layout, {PlausibleWeb.LayoutView, layout})} + end + + @doc false + def render(email, template, assigns \\ []) do + assigns = Map.merge(email.assigns, Map.new(assigns)) + html = Phoenix.View.render_to_string(PlausibleWeb.EmailView, template, assigns) + email |> html_body(html) |> text_body(textify(html)) + end + + defp textify(html) do + html + |> LazyHTML.from_fragment() + |> LazyHTML.to_tree() + |> traverse_and_textify() + |> LazyHTML.from_tree() + |> LazyHTML.text() + |> collapse_whitespace() + end + + defp traverse_and_textify([head | tail]) do + [traverse_and_textify(head) | traverse_and_textify(tail)] + end + + defp traverse_and_textify(text) when is_binary(text) do + String.replace(text, "\n", "\s") + end + + defp traverse_and_textify({"a" = tag, attrs, children}) do + href = with {"href", href} <- List.keyfind(attrs, "href", 0), do: href + children = traverse_and_textify(children) + + if href do + text = children |> LazyHTML.from_tree() |> LazyHTML.text() + + if text == href do + # avoids rendering "http://localhost:8000 (http://localhost:8000)" in base_email footer + text + else + IO.iodata_to_binary([text, " (", href, ?)]) + end + else + {tag, attrs, children} + end + end + + defp traverse_and_textify({"br", _attrs, _children}) do + "\n" + end + + defp traverse_and_textify({tag, attrs, children}) do + {tag, attrs, traverse_and_textify(children)} + end + + defp traverse_and_textify(other), do: other + + defp collapse_whitespace(text) do + text + |> String.split("\n") + |> Enum.map_join("\n", fn line -> + line + |> String.split(" ", trim: true) + |> Enum.join(" ") + end) + end +end