diff --git a/lib/plausible_web.ex b/lib/plausible_web.ex new file mode 100644 index 0000000000000000000000000000000000000000..67ba7b156589f1c61dae3b5d021c03b5d572ed3a --- /dev/null +++ b/lib/plausible_web.ex @@ -0,0 +1,152 @@ +defmodule PlausibleWeb do + @moduledoc false + + use Plausible + + def live_view(opts \\ []) do + quote do + use Plausible + use Phoenix.LiveView, global_prefixes: ~w(x-) + use PlausibleWeb.Live.Flash + + use PlausibleWeb.Live.AuthContext + + unless :no_sentry_context in unquote(opts) do + use PlausibleWeb.Live.SentryContext + end + + on_ee do + use Plausible.Audit.LiveContext + end + + alias PlausibleWeb.Router.Helpers, as: Routes + alias Phoenix.LiveView.JS + + import PlausibleWeb.Components.Generic + import PlausibleWeb.Live.Components.Form + end + end + + def live_component do + quote do + use Phoenix.LiveComponent, global_prefixes: ~w(x-) + import PlausibleWeb.Components.Generic + import PlausibleWeb.Live.Components.Form + alias Phoenix.LiveView.JS + alias PlausibleWeb.Router.Helpers, as: Routes + end + end + + def component do + quote do + use Phoenix.Component, global_prefixes: ~w(x-) + import PlausibleWeb.Components.Generic + import PlausibleWeb.Live.Components.Form + alias Phoenix.LiveView.JS + alias PlausibleWeb.Router.Helpers, as: Routes + end + end + + def controller do + quote do + use Phoenix.Controller, formats: [html: "View", json: "View"] + + import Plug.Conn + import PlausibleWeb.ControllerHelpers + alias PlausibleWeb.Router.Helpers, as: Routes + end + end + + def view do + quote do + use Phoenix.View, + root: "lib/plausible_web/templates" + + # Import convenience functions from controllers + import Phoenix.Controller, only: [view_module: 1] + + use Phoenix.Component, global_prefixes: ~w(x-) + + import PlausibleWeb.Components.Generic + import PlausibleWeb.Live.Components.Form + alias PlausibleWeb.Router.Helpers, as: Routes + end + end + + on_ee do + def extra_view do + quote do + use Phoenix.View, + root: "extra/lib/plausible_web/templates" + + # Import convenience functions from controllers + import Phoenix.Controller, only: [view_module: 1] + + use Phoenix.Component + + import PlausibleWeb.Components.Generic + import PlausibleWeb.Live.Components.Form + alias PlausibleWeb.Router.Helpers, as: Routes + end + end + end + + def router do + quote do + use Phoenix.Router + import Plug.Conn + import Phoenix.Controller + end + end + + def channel do + quote do + use Phoenix.Channel + end + end + + def plugins_api_controller do + quote do + use Phoenix.Controller, formats: [:json] + import Plug.Conn + import PlausibleWeb.Router.Helpers + + alias PlausibleWeb.Plugins.API.Schemas + alias PlausibleWeb.Plugins.API.Views + alias PlausibleWeb.Plugins.API.Errors + alias Plausible.Plugins.API + + plug(OpenApiSpex.Plug.CastAndValidate, json_render_error_v2: true, replace_params: false) + + use OpenApiSpex.ControllerSpecs + end + end + + def plugins_api_view do + quote do + use Phoenix.View, root: "" + + alias PlausibleWeb.Router.Helpers + import PlausibleWeb.Plugins.API.Views.Pagination, only: [render_metadata_links: 4] + end + end + + def open_api_schema do + quote do + require OpenApiSpex + alias OpenApiSpex.Schema + alias PlausibleWeb.Plugins.API.Schemas + end + end + + @doc """ + When used, dispatch to the appropriate controller/view/etc. + """ + defmacro __using__(which) when is_atom(which) do + apply(__MODULE__, which, []) + end + + defmacro __using__([{which, opts}]) when is_atom(which) do + apply(__MODULE__, which, [List.wrap(opts)]) + end +end diff --git a/lib/plausible_web/views/stats_view.ex b/lib/plausible_web/views/stats_view.ex new file mode 100644 index 0000000000000000000000000000000000000000..4048023c37d3b366040b3ae74d778fa79dd09007 --- /dev/null +++ b/lib/plausible_web/views/stats_view.ex @@ -0,0 +1,80 @@ +defmodule PlausibleWeb.StatsView do + use PlausibleWeb, :view + use Plausible + + require Plausible.Billing + + def plausible_url do + PlausibleWeb.Endpoint.url() + end + + def large_number_format(n, opts \\ []) do + k = if Keyword.get(opts, :capitalize_k?, false), do: "K", else: "k" + + cond do + n >= 1_000 && n < 1_000_000 -> + thousands = trunc(n / 100) / 10 + + if thousands == trunc(thousands) || n >= 100_000 do + "#{trunc(thousands)}" <> k + else + "#{thousands}" <> k + end + + n >= 1_000_000 && n < 1_000_000_000 -> + millions = trunc(n / 100_000) / 10 + + if millions == trunc(millions) || n > 100_000_000 do + "#{trunc(millions)}M" + else + "#{millions}M" + end + + n >= 1_000_000_000 && n < 1_000_000_000_000 -> + billions = trunc(n / 100_000_000) / 10 + + if billions == trunc(billions) || n > 100_000_000_000 do + "#{trunc(billions)}B" + else + "#{billions}B" + end + + is_integer(n) -> + Integer.to_string(n) + end + end + + def stats_container_class(conn) do + cond do + conn.assigns[:embedded] && conn.params["width"] == "manual" -> "px-6" + conn.assigns[:embedded] -> "max-w-screen-xl mx-auto px-6" + !conn.assigns[:embedded] -> "container print:max-w-full" + end + end + + @doc """ + Returns a readable stats URL. + + Native Phoenix router functions percent-encode all diacritics, resulting in + ugly URLs, e.g. `https://plausible.io/café.com` transforms into + `https://plausible.io/caf%C3%A9.com`. + + This function encodes only the slash (`/`) character from the site's domain. + + ## Examples + + iex> PlausibleWeb.StatsView.pretty_stats_url(%Plausible.Site{domain: "user.gittea.io/repo"}) + "http://localhost:8000/user.gittea.io%2Frepo" + + iex> PlausibleWeb.StatsView.pretty_stats_url(%Plausible.Site{domain: "anakin.test"}) + "http://localhost:8000/anakin.test" + + iex> PlausibleWeb.StatsView.pretty_stats_url(%Plausible.Site{domain: "café.test"}) + "http://localhost:8000/café.test" + + """ + def pretty_stats_url(%Plausible.Site{domain: domain}) when is_binary(domain) do + pretty_domain = String.replace(domain, "/", "%2F") + "#{plausible_url()}/#{pretty_domain}" + end +end diff --git a/lib/plausible_web/views/text_helpers.ex b/lib/plausible_web/views/text_helpers.ex new file mode 100644 index 0000000000000000000000000000000000000000..17c346948d482cef092cbce712e800912bcce4b1 --- /dev/null +++ b/lib/plausible_web/views/text_helpers.ex @@ -0,0 +1,53 @@ +defmodule PlausibleWeb.TextHelpers do + @moduledoc false + + @spec pretty_join([String.t()]) :: String.t() + + @doc """ + Turns a list of strings into a string and replaces the last comma + with the word "and". + + ### Examples: + + iex> ["one"] |> PlausibleWeb.TextHelpers.pretty_join() + "one" + + iex> ["one", "two"] |> PlausibleWeb.TextHelpers.pretty_join() + "one and two" + + iex> ["one", "two", "three"] |> PlausibleWeb.TextHelpers.pretty_join() + "one, two and three" + """ + def pretty_join([str]), do: str + + def pretty_join(list) do + [last_string | rest] = Enum.reverse(list) + + rest_string = + rest + |> Enum.reverse() + |> Enum.join(", ") + + "#{rest_string} and #{last_string}" + end + + def pretty_list(list) do + list + |> Enum.map(&String.replace("#{&1}", "_", " ")) + |> pretty_join() + end + + def format_date_range(date_range) do + "#{format_date(date_range.first)} - #{format_date(date_range.last)}" + end + + def format_date(date) do + Calendar.strftime(date, "%b %-d, %Y") + end + + def number_format(number) when is_integer(number) do + Cldr.Number.to_string!(number) + end + + def number_format(other), do: other +end diff --git a/lib/plausible_web/views/unsubscribe_view.ex b/lib/plausible_web/views/unsubscribe_view.ex new file mode 100644 index 0000000000000000000000000000000000000000..ecb1b20eda7e3e78e051e9f42793959a44f2ec43 --- /dev/null +++ b/lib/plausible_web/views/unsubscribe_view.ex @@ -0,0 +1,3 @@ +defmodule PlausibleWeb.UnsubscribeView do + use PlausibleWeb, :view +end diff --git a/lib/sentry_filter.ex b/lib/sentry_filter.ex new file mode 100644 index 0000000000000000000000000000000000000000..1b538e172911b28f6ef448712ea9ff8bfff340b8 --- /dev/null +++ b/lib/sentry_filter.ex @@ -0,0 +1,69 @@ +defmodule Plausible.SentryFilter do + @moduledoc """ + Sentry callbacks for filtering and grouping events + """ + + @spec before_send(Sentry.Event.t()) :: Sentry.Event.t() + def before_send(event) + + def before_send(%{original_exception: %Bamboo.PostmarkAdapter.Error{} = e} = event) do + if Bamboo.PostmarkAdapter.Error.is_hard_bounce(e), do: false, else: event + end + + def before_send(%{original_exception: %Phoenix.NotAcceptableError{}}), do: false + def before_send(%{original_exception: %Plug.CSRFProtection.InvalidCSRFTokenError{}}), do: false + def before_send(%{original_exception: %Plug.Static.InvalidPathError{}}), do: false + + def before_send( + %{ + exception: [%{type: "DBConnection.ConnectionError"}], + original_exception: %{reason: reason} + } = event + ) do + %{event | fingerprint: ["db_connection", reason]} + end + + def before_send( + %{ + exception: [%{type: "Mint.TransportError"}], + original_exception: %{reason: reason} + } = event + ) do + %{event | fingerprint: ["mint_transport", reason]} + end + + def before_send( + %{ + exception: [%{type: "Finch.TransportError"}], + original_exception: %{reason: reason} + } = event + ) do + %{event | fingerprint: ["mint_transport", reason]} + end + + def before_send( + %{source: :logger, message: %{formatted: "Ch.Connection (#PID<" <> rest}} = event + ) do + if String.ends_with?(rest, ")) disconnected: ** (Mint.HTTPError) the connection is closed") do + false + else + event + end + end + + def before_send(%{extra: %{request: %Plausible.Ingestion.Request{}}} = event) do + %{event | fingerprint: ["ingestion_request"]} + end + + def before_send(%{source: :logger, message: %{formatted: "Ranch listener" <> rest}} = event) do + if String.contains?(rest, "had its request process") do + false + else + event + end + end + + def before_send(event) do + event + end +end diff --git a/lib/workers/accept_traffic_until_notification.ex b/lib/workers/accept_traffic_until_notification.ex new file mode 100644 index 0000000000000000000000000000000000000000..698e795736798cf614230202079c354dcdea84c5 --- /dev/null +++ b/lib/workers/accept_traffic_until_notification.ex @@ -0,0 +1,106 @@ +defmodule Plausible.Workers.AcceptTrafficUntil do + @moduledoc """ + A worker meant to be run once a day that sends out e-mail notifications to site + owners assuming: + - their sites still receive traffic (i.e. have stats for yesterday) + - `site.accept_traffic_until` is approaching either tomorrow or exactly in 7 days + + Users having no sites or sites that receive no traffic, won't be notified. + We make a tiny effort here to make sure we send the same notification at most once a day. + """ + use Oban.Worker, queue: :check_accept_traffic_until + import Ecto.Query + + alias Plausible.Repo + alias Plausible.ClickhouseRepo + + def dry_run(date) do + perform(nil, date, true) + end + + @impl Oban.Worker + def perform(_job, today \\ Date.utc_today(), dry_run? \\ false) do + tomorrow = today |> Date.add(+1) + next_week = today |> Date.add(+7) + + # send at most one notification per user, per day + sent_today_query = + from s in "sent_accept_traffic_until_notifications", + where: s.user_id == parent_as(:users).id and s.sent_on == ^today, + select: true + + notifications = + Repo.all( + from t in Plausible.Teams.Team, + inner_join: u in assoc(t, :owners), + as: :users, + inner_join: s in assoc(t, :sites), + where: t.accept_traffic_until == ^tomorrow or t.accept_traffic_until == ^next_week, + where: not exists(sent_today_query), + select: %{ + id: u.id, + email: u.email, + deadline: t.accept_traffic_until, + site_ids: fragment("array_agg(?.id)", s), + name: u.name, + team: t + }, + group_by: [u.id, t.id] + ) + + for notification <- notifications do + case {has_stats?(notification.site_ids, today), notification.deadline} do + {true, ^tomorrow} -> + if dry_run? do + IO.puts("Will send final notification to #{notification.email}") + else + notification + |> store_sent(today) + |> PlausibleWeb.Email.approaching_accept_traffic_until_tomorrow() + |> Plausible.Mailer.send() + end + + {true, ^next_week} -> + if dry_run? do + IO.puts("Will send weekly notification to #{notification.email}") + else + notification + |> store_sent(today) + |> PlausibleWeb.Email.approaching_accept_traffic_until() + |> Plausible.Mailer.send() + end + + _ -> + nil + end + end + + {:ok, Enum.count(notifications)} + end + + defp has_stats?(site_ids, today) do + ago_2d = Date.add(today, -2) + + ClickhouseRepo.exists?( + from e in "events_v2", + where: fragment("toDate(?) >= ?", e.timestamp, ^ago_2d), + where: e.site_id in ^site_ids + ) + end + + defp store_sent(notification, today) do + Repo.insert_all( + "sent_accept_traffic_until_notifications", + [ + %{ + user_id: notification.id, + sent_on: today + } + ], + on_conflict: :nothing, + conflict_target: [:user_id, :sent_on] + ) + + notification + end +end diff --git a/lib/workers/check_usage.ex b/lib/workers/check_usage.ex new file mode 100644 index 0000000000000000000000000000000000000000..c91431c9713b25e494bbcee7e276e7ac61669f58 --- /dev/null +++ b/lib/workers/check_usage.ex @@ -0,0 +1,170 @@ +defmodule Plausible.Workers.CheckUsage do + use Plausible.Repo + use Oban.Worker, queue: :check_usage + require Plausible.Billing.Subscription.Status + alias Plausible.Billing.{Subscription, Quota} + alias Plausible.Teams + + defmacro yesterday() do + quote do + fragment("now() - INTERVAL '1 day'") + end + end + + defmacro last_day_of_month(day) do + quote do + fragment( + "(date_trunc('month', ?::date) + interval '1 month' - interval '1 day')::date", + unquote(day) + ) + end + end + + defmacro day_of_month(date) do + quote do + fragment("EXTRACT(day from ?::date)", unquote(date)) + end + end + + defmacro least(left, right) do + quote do + fragment("least(?, ?)", unquote(left), unquote(right)) + end + end + + @impl Oban.Worker + def perform(_job, usage_mod \\ Teams.Billing, today \\ Date.utc_today()) do + yesterday = today |> Date.shift(day: -1) + + active_subscribers = + Repo.all( + from(t in Teams.Team, + as: :team, + inner_join: o in assoc(t, :owners), + left_join: bm in assoc(t, :billing_members), + inner_lateral_join: s in subquery(Teams.last_subscription_join_query()), + on: true, + left_join: ep in Plausible.Billing.EnterprisePlan, + on: ep.team_id == t.id and ep.paddle_plan_id == s.paddle_plan_id, + where: + s.status in [ + ^Subscription.Status.active(), + ^Subscription.Status.past_due(), + ^Subscription.Status.deleted() + ], + where: not is_nil(s.last_bill_date), + # Accounts for situations like last_bill_date==2021-01-31 AND today==2021-03-01. Since February never reaches the 31st day, the account is checked on 2021-03-01. + where: s.next_bill_date >= ^today, + where: + least(day_of_month(s.last_bill_date), day_of_month(last_day_of_month(^yesterday))) == + day_of_month(^yesterday), + order_by: t.id, + preload: [subscription: s, enterprise_plan: ep, owners: o, billing_members: bm] + ) + ) + + for subscriber <- active_subscribers do + case {subscriber.grace_period, subscriber.enterprise_plan} do + {nil, nil} -> + check_regular_subscriber(subscriber, usage_mod) + + {nil, _} -> + check_enterprise_subscriber(subscriber, usage_mod) + + {_, nil} -> + maybe_remove_grace_period(subscriber, usage_mod) + + _ -> + :skip + end + end + + :ok + end + + defp check_site_usage_for_enterprise(subscriber) do + limit = subscriber.enterprise_plan.site_limit + + usage = Teams.Billing.site_usage(subscriber) + + if Quota.below_limit?(usage, limit) do + {:below_limit, {usage, limit}} + else + {:over_limit, {usage, limit}} + end + end + + def maybe_remove_grace_period(subscriber, usage_mod) do + case check_pageview_usage_last_cycle(subscriber, usage_mod) do + {:below_limit, _} -> + Plausible.Teams.remove_grace_period(subscriber) + :ok + + _ -> + :skip + end + end + + defp check_regular_subscriber(subscriber, usage_mod) do + case check_pageview_usage_two_cycles(subscriber, usage_mod) do + {:over_limit, pageview_usage} -> + suggested_volume = + Plausible.Billing.Plans.suggest_volume(subscriber, pageview_usage.last_cycle.total) + + for owner <- subscriber.owners ++ subscriber.billing_members do + PlausibleWeb.Email.over_limit_email(owner, subscriber, pageview_usage, suggested_volume) + |> Plausible.Mailer.send() + end + + Plausible.Teams.start_grace_period(subscriber) + + _ -> + nil + end + end + + def check_enterprise_subscriber(subscriber, usage_mod) do + pageview_usage = check_pageview_usage_two_cycles(subscriber, usage_mod) + site_usage = check_site_usage_for_enterprise(subscriber) + + case {pageview_usage, site_usage} do + {{:below_limit, _}, {:below_limit, _}} -> + nil + + {{_, pageview_usage}, {_, {site_usage, site_allowance}}} -> + for owner <- subscriber.owners ++ subscriber.billing_members do + PlausibleWeb.Email.enterprise_over_limit_internal_email( + owner, + pageview_usage, + site_usage, + site_allowance + ) + |> Plausible.Mailer.send() + end + + Plausible.Teams.start_manual_lock_grace_period(subscriber) + end + end + + def check_pageview_usage_two_cycles(subscriber, usage_mod) do + usage = usage_mod.monthly_pageview_usage(subscriber) + limit = Teams.Billing.monthly_pageview_limit(subscriber.subscription) + + if Quota.exceeds_last_two_usage_cycles?(usage, limit) do + {:over_limit, usage} + else + {:below_limit, usage} + end + end + + defp check_pageview_usage_last_cycle(subscriber, usage_mod) do + usage = usage_mod.monthly_pageview_usage(subscriber) + limit = Teams.Billing.monthly_pageview_limit(subscriber.subscription) + + if :last_cycle in Quota.exceeded_cycles(usage, limit) do + {:over_limit, usage} + else + {:below_limit, usage} + end + end +end diff --git a/lib/workers/clean_invitations.ex b/lib/workers/clean_invitations.ex new file mode 100644 index 0000000000000000000000000000000000000000..84b9a99ef3e295ed6d94448054d18fbdc4c8add5 --- /dev/null +++ b/lib/workers/clean_invitations.ex @@ -0,0 +1,25 @@ +defmodule Plausible.Workers.CleanInvitations do + use Plausible.Repo + use Oban.Worker, queue: :clean_invitations + + @cutoff Duration.new!(hour: -48) + + @impl Oban.Worker + def perform(_job) do + cutoff_time = + NaiveDateTime.utc_now(:second) + |> NaiveDateTime.shift(@cutoff) + + Repo.delete_all( + from ti in Plausible.Teams.Invitation, + where: ti.inserted_at < ^cutoff_time + ) + + Repo.delete_all( + from ti in Plausible.Teams.SiteTransfer, + where: ti.inserted_at < ^cutoff_time + ) + + :ok + end +end diff --git a/lib/workers/clean_user_sessions.ex b/lib/workers/clean_user_sessions.ex new file mode 100644 index 0000000000000000000000000000000000000000..8b2640e0ad65c44fb8ed737e0993c7fe33e08135 --- /dev/null +++ b/lib/workers/clean_user_sessions.ex @@ -0,0 +1,27 @@ +defmodule Plausible.Workers.CleanUserSessions do + @moduledoc """ + Job removing expired user sessions. A grace period is applied. + """ + + use Plausible.Repo + use Oban.Worker, queue: :clean_user_sessions + + @grace_period Duration.new!(day: -7) + + @spec grace_period_duration() :: Duration.t() + def grace_period_duration(), do: @grace_period + + @impl Oban.Worker + def perform(_job) do + grace_cutoff = + NaiveDateTime.utc_now(:second) + |> NaiveDateTime.shift(@grace_period) + + Repo.delete_all( + from us in Plausible.Auth.UserSession, + where: us.timeout_at < ^grace_cutoff + ) + + :ok + end +end diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.ex new file mode 100644 index 0000000000000000000000000000000000000000..a425fdfa58641f872dccb8a018874f8bfb5c4573 --- /dev/null +++ b/lib/workers/clickhouse_clean_sites.ex @@ -0,0 +1,81 @@ +defmodule Plausible.Workers.ClickhouseCleanSites do + @moduledoc """ + Cleans deleted site data from ClickHouse asynchronously. + + We batch up data deletions from ClickHouse as deleting a single site is + just as expensive as deleting many. + """ + + use Plausible.Repo + use Plausible.ClickhouseRepo + use Plausible.IngestRepo + use Oban.Worker, queue: :clickhouse_clean_sites + + import Ecto.Query + + require Logger + + @tables_to_clear [ + "events_v2", + "sessions_v2", + "ingest_counters", + "imported_browsers", + "imported_devices", + "imported_entry_pages", + "imported_exit_pages", + "imported_locations", + "imported_operating_systems", + "imported_pages", + "imported_custom_events", + "imported_sources", + "imported_visitors" + ] + + @settings if Mix.env() in [:test, :ce_test, :e2e_test], do: [mutations_sync: 2], else: [] + + def perform(_job) do + deleted_sites = get_deleted_sites_with_clickhouse_data() + + if not Enum.empty?(deleted_sites) do + Logger.notice( + "Clearing ClickHouse data for the following #{length(deleted_sites)} sites which have been deleted: #{inspect(deleted_sites)}" + ) + + for table <- @tables_to_clear do + IngestRepo.query!( + "ALTER TABLE {$0:Identifier} DELETE WHERE site_id IN {$1:Array(UInt64)}", + [table, deleted_sites], + settings: @settings + ) + end + end + + :ok + end + + def get_deleted_sites_with_clickhouse_data() do + pg_sites = + from(s in Plausible.Site.regular(), select: s.id) + |> Plausible.Repo.all() + |> MapSet.new() + + {:ok, ch} = + Plausible.ClickhouseRepo.get_config_without_ch_query_execution_timeout() + |> Ch.start_link() + + %Ch.Result{columns: ["site_id"], rows: rows} = + DBConnection.run( + ch, + fn conn -> + Ch.query!(conn, "FROM events_v2 SELECT site_id GROUP BY site_id", [], + timeout: :infinity + ) + end, + timeout: :infinity + ) + + ch_sites = rows |> MapSet.new(fn [site_id] -> site_id end) + + MapSet.difference(ch_sites, pg_sites) |> MapSet.to_list() + end +end diff --git a/lib/workers/expire_domain_change_transitions.ex b/lib/workers/expire_domain_change_transitions.ex new file mode 100644 index 0000000000000000000000000000000000000000..b6956feb46fb8300f176a6faef92368f03880cd9 --- /dev/null +++ b/lib/workers/expire_domain_change_transitions.ex @@ -0,0 +1,22 @@ +defmodule Plausible.Workers.ExpireDomainChangeTransitions do + @moduledoc """ + Periodic worker that expires domain change transition period. + Old domains are frozen for a given time, so users can still access them + before redeploying their scripts and integrations. + """ + use Plausible.Repo + use Oban.Worker, queue: :domain_change_transition + + require Logger + + @impl Oban.Worker + def perform(_job) do + {:ok, n} = Plausible.Site.Domain.expire_change_transitions() + + if n > 0 do + Logger.warning("Expired #{n} from the domain change transition period.") + end + + :ok + end +end diff --git a/lib/workers/export_analytics.ex b/lib/workers/export_analytics.ex new file mode 100644 index 0000000000000000000000000000000000000000..bccc9f3a8da8dd406f31c06e7437d2b948279be5 --- /dev/null +++ b/lib/workers/export_analytics.ex @@ -0,0 +1,121 @@ +defmodule Plausible.Workers.ExportAnalytics do + @moduledoc """ + Worker for running CSV export jobs. Supports S3 and local storage. + To avoid blocking the queue, a timeout of 15 minutes is enforced. + """ + + use Oban.Worker, + queue: :analytics_exports, + max_attempts: 3 + + alias Plausible.Exports + + @doc "This base query filters export jobs for a site" + def base_query(site_id) do + import Ecto.Query, only: [from: 2] + + from j in Oban.Job, + where: j.worker == ^Oban.Worker.to_string(__MODULE__), + where: j.args["site_id"] == ^site_id + end + + @impl true + def timeout(_job), do: :timer.minutes(15) + + @impl true + def perform(%Oban.Job{args: args} = job) do + %{ + "storage" => storage, + "site_id" => site_id + } = args + + site = Plausible.Repo.get!(Plausible.Site, site_id) + true = Plausible.Sites.regular?(site) + %Date.Range{} = date_range = Exports.date_range(site.id, site.timezone) + + queries = + Exports.export_queries(site, + date_range: date_range, + timezone: site.timezone, + extname: ".csv" + ) + + # since each worker / `perform` attempt runs in a separate process + # it's ok to use start_link to keep connection lifecycle + # bound to that of the worker + {:ok, ch} = + Plausible.ClickhouseRepo.get_config_without_ch_query_execution_timeout() + |> Ch.start_link() + + try do + case storage do + "s3" -> perform_s3_export(ch, site, queries, args) + "local" -> perform_local_export(ch, queries, args) + end + after + Exports.oban_notify(site_id) + end + + email_success(job.args) + + :ok + catch + class, reason -> + if job.attempt >= job.max_attempts, do: email_failure(job.args) + :erlang.raise(class, reason, __STACKTRACE__) + end + + defp perform_s3_export(ch, site, queries, args) do + %{ + "s3_bucket" => s3_bucket, + "s3_path" => s3_path + } = args + + created_on = Plausible.Timezones.to_date_in_timezone(DateTime.utc_now(), site.timezone) + filename = Exports.archive_filename(site.domain, created_on) + + DBConnection.run( + ch, + fn conn -> + conn + |> Exports.stream_archive(queries, format: "CSVWithNames", timeout: :infinity) + |> Plausible.S3.export_upload_multipart(s3_bucket, s3_path, filename) + end, + timeout: :infinity + ) + end + + defp perform_local_export(ch, queries, args) do + %{"local_path" => local_path} = args + tmp_path = Plug.Upload.random_file!("tmp-plausible-export") + + DBConnection.run( + ch, + fn conn -> + Exports.stream_archive(conn, queries, format: "CSVWithNames") + |> Stream.into(File.stream!(tmp_path)) + |> Stream.run() + end, + timeout: :infinity + ) + + File.mkdir_p!(Path.dirname(local_path)) + if File.exists?(local_path), do: File.rm!(local_path) + Plausible.File.mv!(tmp_path, local_path) + end + + defp email_failure(args) do + args |> Map.put("status", "failure") |> email() + end + + defp email_success(args) do + args |> Map.put("status", "success") |> email() + end + + defp email(args) do + # email delivery can potentially fail and cause already successful + # export to be repeated which is costly, hence email is delivered + # in a separate job + Oban.insert!(Plausible.Workers.NotifyExportedAnalytics.new(args)) + end +end diff --git a/lib/workers/import_analytics.ex b/lib/workers/import_analytics.ex new file mode 100644 index 0000000000000000000000000000000000000000..6bcc0ec6e3dcc7cd88f875bbea435228f0801e4e --- /dev/null +++ b/lib/workers/import_analytics.ex @@ -0,0 +1,96 @@ +defmodule Plausible.Workers.ImportAnalytics do + @moduledoc """ + Worker for running analytics import jobs. + """ + + use Plausible.Repo + + use Oban.Worker, + queue: :analytics_imports, + max_attempts: 3, + unique: [fields: [:args], keys: [:import_id], period: 60] + + alias Plausible.Imported.ImportSources + alias Plausible.Imported.Importer + alias Plausible.Imported.SiteImport + + @impl Oban.Worker + def perform(%Oban.Job{ + args: %{"import_id" => import_id} = args + }) do + site_import = + SiteImport + |> Repo.get!(import_id) + |> Repo.preload(:site) + + import_api = ImportSources.by_name(site_import.source) + + case import_api.run_import(site_import, args) do + {:ok, site_import} -> + import_complete(site_import) + + :ok + + {:error, error, error_opts} -> + Sentry.capture_message("Failed to import from #{site_import.source}", + extra: %{ + import_id: site_import.id, + site: site_import.site.domain, + error: inspect(error) + } + ) + + import_fail(site_import, error_opts) + + {:discard, error} + end + end + + @impl Oban.Worker + def backoff(_job) do + # 5 minutes + 300 + end + + def import_complete(site_import) do + site_import = Repo.preload(site_import, [:imported_by, site: :team]) + + PlausibleWeb.Email.import_success(site_import, site_import.imported_by) + |> Plausible.Mailer.send() + + Plausible.Sites.clear_stats_start_date!(site_import.site) + + Importer.notify(site_import, :complete) + + :ok + end + + def import_fail_transient(site_import) do + Plausible.Purge.delete_imported_stats!(site_import) + + Importer.notify(site_import, :transient_fail) + end + + def import_fail(site_import, opts) do + skip_purge? = Keyword.get(opts, :skip_purge?, false) + skip_mark_failed? = Keyword.get(opts, :skip_mark_failed?, false) + + if not skip_purge? do + Plausible.Purge.delete_imported_stats!(site_import) + end + + if not skip_mark_failed? do + import_api = ImportSources.by_name(site_import.source) + + site_import = + site_import + |> import_api.mark_failed() + |> Repo.preload([:site, :imported_by]) + + Importer.notify(site_import, :fail) + + PlausibleWeb.Email.import_failure(site_import, site_import.imported_by) + |> Plausible.Mailer.send() + end + end +end diff --git a/lib/workers/local_import_analytics_cleaner.ex b/lib/workers/local_import_analytics_cleaner.ex new file mode 100644 index 0000000000000000000000000000000000000000..f7aac2621bf7ade12c5802b2ea6512a7c53bc155 --- /dev/null +++ b/lib/workers/local_import_analytics_cleaner.ex @@ -0,0 +1,32 @@ +defmodule Plausible.Workers.LocalImportAnalyticsCleaner do + @moduledoc """ + Worker for cleaning local files left after analytics import jobs. + """ + + use Oban.Worker, queue: :analytics_imports, unique: [period: 3600] + + @impl Oban.Worker + def perform(%Oban.Job{args: args}) do + %{"import_id" => import_id, "paths" => paths} = args + + if import_in_progress?(import_id) do + {:snooze, _one_hour = 3600} + else + Enum.each(paths, fn path -> + # credo:disable-for-next-line Credo.Check.Refactor.Nesting + if File.exists?(path), do: File.rm!(path) + end) + end + end + + defp import_in_progress?(import_id) do + import Ecto.Query + require Plausible.Imported.SiteImport + alias Plausible.Imported.SiteImport + + SiteImport + |> where(id: ^import_id) + |> where([i], i.status in ^[SiteImport.pending(), SiteImport.importing()]) + |> Plausible.Repo.exists?() + end +end diff --git a/lib/workers/locations_sync.ex b/lib/workers/locations_sync.ex new file mode 100644 index 0000000000000000000000000000000000000000..a261af32613a0240f51c351f095e2c9fa845bd04 --- /dev/null +++ b/lib/workers/locations_sync.ex @@ -0,0 +1,15 @@ +defmodule Plausible.Workers.LocationsSync do + @moduledoc false + + use Plausible.Repo + use Oban.Worker, queue: :locations_sync + + @impl Oban.Worker + def perform(_job) do + if Plausible.DataMigration.LocationsSync.out_of_date?() do + Plausible.DataMigration.LocationsSync.run() + end + + :ok + end +end diff --git a/lib/workers/lock_sites.ex b/lib/workers/lock_sites.ex new file mode 100644 index 0000000000000000000000000000000000000000..6221019923a475bbd20b4a4945a7e44129a2fd93 --- /dev/null +++ b/lib/workers/lock_sites.ex @@ -0,0 +1,24 @@ +defmodule Plausible.Workers.LockSites do + use Plausible.Repo + use Oban.Worker, queue: :lock_sites + + alias Plausible.Teams + + @impl Oban.Worker + def perform(_job) do + teams = + Repo.all( + from t in Teams.Team, + as: :team, + left_lateral_join: s in subquery(Teams.last_subscription_join_query()), + on: true, + preload: [subscription: s] + ) + + for team <- teams do + Plausible.Billing.SiteLocker.update_for(team) + end + + :ok + end +end diff --git a/lib/workers/notify_annual_renewal.ex b/lib/workers/notify_annual_renewal.ex new file mode 100644 index 0000000000000000000000000000000000000000..59b56305ab4a17c47d09816db13ade83eb1d144b --- /dev/null +++ b/lib/workers/notify_annual_renewal.ex @@ -0,0 +1,76 @@ +defmodule Plausible.Workers.NotifyAnnualRenewal do + use Plausible.Repo + use Oban.Worker, queue: :notify_annual_renewal + + require Plausible.Billing.Subscription.Status + + alias Money.Subscription + alias Plausible.Billing.Subscription + alias Plausible.Teams + + @yearly_plans Plausible.Billing.Plans.yearly_product_ids() + + @impl Oban.Worker + @doc """ + Sends a notification at most 7 days and at least 1 day before the renewal of an annual subscription + """ + def perform(_job) do + sent_notification = + from( + s in "sent_renewal_notifications", + where: s.timestamp > fragment("now() - INTERVAL '1 month'") + ) + + teams = + Repo.all( + from t in Teams.Team, + as: :team, + inner_join: o in assoc(t, :owners), + left_join: bm in assoc(t, :billing_members), + inner_lateral_join: s in subquery(Teams.last_subscription_join_query()), + on: true, + left_join: sent in ^sent_notification, + on: o.id == sent.user_id, + where: is_nil(sent.id), + where: s.paddle_plan_id in @yearly_plans, + where: + s.next_bill_date > fragment("now()::date") and + s.next_bill_date <= fragment("now()::date + INTERVAL '7 days'"), + preload: [owners: o, subscription: s, billing_members: bm] + ) + + for team <- teams do + recipients = team.owners ++ team.billing_members + + case team.subscription.status do + Subscription.Status.active() -> + for recipient <- recipients do + template = PlausibleWeb.Email.yearly_renewal_notification(team, recipient) + Plausible.Mailer.send(template) + end + + Subscription.Status.deleted() -> + for recipient <- recipients do + template = PlausibleWeb.Email.yearly_expiration_notification(team, recipient) + Plausible.Mailer.send(template) + end + + _ -> + Sentry.capture_message("Invalid subscription for renewal", + team: team + ) + end + + for recipient <- recipients do + Repo.insert_all("sent_renewal_notifications", [ + %{ + user_id: recipient.id, + timestamp: NaiveDateTime.utc_now() + } + ]) + end + end + + :ok + end +end diff --git a/lib/workers/notify_exported_analytics.ex b/lib/workers/notify_exported_analytics.ex new file mode 100644 index 0000000000000000000000000000000000000000..d4132075689b8cbd92b0c86f549971b7ac373dd0 --- /dev/null +++ b/lib/workers/notify_exported_analytics.ex @@ -0,0 +1,43 @@ +defmodule Plausible.Workers.NotifyExportedAnalytics do + @moduledoc "This worker delivers emails for successful and failed exports" + + use Oban.Worker, + queue: :notify_exported_analytics, + max_attempts: 5 + + @impl true + def perform(%Oban.Job{args: args}) do + %{ + "status" => status, + "storage" => storage, + "email_to" => email_to, + "site_id" => site_id + } = args + + user = Plausible.Repo.get_by!(Plausible.Auth.User, email: email_to) + + site = + Plausible.Site + |> Plausible.Repo.get!(site_id) + |> Plausible.Repo.preload(:team) + + email = + case status do + "success" -> + case storage do + "s3" -> + %{expires_at: expires_at} = Plausible.Exports.get_s3_export!(site_id, _retries = 10) + PlausibleWeb.Email.export_success(user, site, expires_at) + + "local" -> + PlausibleWeb.Email.export_success(user, site, _expires_at = nil) + end + + "failure" -> + PlausibleWeb.Email.export_failure(user, site) + end + + Plausible.Mailer.deliver_now!(email) + :ok + end +end diff --git a/lib/workers/purge_cdn_cache.ex b/lib/workers/purge_cdn_cache.ex new file mode 100644 index 0000000000000000000000000000000000000000..d89d5fb266b2cdd76b2c50636ef0894cff35269c --- /dev/null +++ b/lib/workers/purge_cdn_cache.ex @@ -0,0 +1,71 @@ +defmodule Plausible.Workers.PurgeCDNCache do + @moduledoc """ + Worker for purging CDN cache for tracker scripts on cloud. + + Uses Bunny CDN's API to purge cache by tag. + Docs ref: https://docs.bunny.net/reference/pullzonepublic_purgecachepostbytag + + Note that purging by id "*" is equivalent to purging ALL cache. + """ + + use Oban.Worker, + queue: :purge_cdn_cache, + max_attempts: 5, + # To avoid running into API rate limits, we: + # - Schedule jobs with a delay + # - Bump the scheduled time every time a new one is scheduled with the same args + unique: [ + states: [:scheduled], + fields: [:args] + ] + + require Logger + + @impl Oban.Worker + def perform(%Oban.Job{args: %{"id" => id}}) do + pullzone_id = Application.get_env(:plausible, __MODULE__, [])[:pullzone_id] + api_key = Application.get_env(:plausible, __MODULE__, [])[:api_key] + + purge_cache(id, pullzone_id, api_key) + end + + @impl Oban.Worker + def backoff(%Oban.Job{attempt: attempt}) do + # Exponential backoff starting at 3 minutes + trunc(:math.pow(2, attempt - 1) * 180) + end + + defp purge_cache(id, pullzone_id, api_key) when is_nil(pullzone_id) or is_nil(api_key) do + Logger.warning("Ignoring purge CDN cache for tracker script #{id}: Configuration missing") + {:discard, "Configuration missing"} + end + + defp purge_cache(id, pullzone_id, api_key) do + options = + [ + headers: [ + {"content-type", "application/json"}, + {"AccessKey", api_key} + ], + body: Jason.encode!(%{"CacheTag" => "tracker_script::#{id}"}) + ] + |> Keyword.merge(Application.get_env(:plausible, __MODULE__)[:req_opts] || []) + + case Req.post("https://api.bunny.net/pullzone/#{pullzone_id}/purgeCache", options) do + {:ok, %{status: 204}} -> + Logger.notice("Successfully purged CDN cache for tracker script #{id}") + {:ok, :success} + + {:ok, %{status: status}} -> + Logger.warning( + "Failed to purge CDN cache for tracker script #{id}: Unexpected status: #{status}" + ) + + {:error, "Unexpected status: #{status}"} + + {:error, reason} -> + Logger.warning("Failed to purge CDN cache for tracker script #{id}: #{inspect(reason)}") + {:error, reason} + end + end +end diff --git a/lib/workers/rotate_salts.ex b/lib/workers/rotate_salts.ex new file mode 100644 index 0000000000000000000000000000000000000000..7bc0d1effa946ea7c3c3a292e8f00acb0160a8a9 --- /dev/null +++ b/lib/workers/rotate_salts.ex @@ -0,0 +1,9 @@ +defmodule Plausible.Workers.RotateSalts do + use Plausible.Repo + use Oban.Worker, queue: :rotate_salts + + @impl Oban.Worker + def perform(_job) do + Plausible.Session.Salts.rotate() + end +end diff --git a/lib/workers/schedule_email_reports.ex b/lib/workers/schedule_email_reports.ex new file mode 100644 index 0000000000000000000000000000000000000000..82dbf063deac19643f23f4f756cde06a4359c8b7 --- /dev/null +++ b/lib/workers/schedule_email_reports.ex @@ -0,0 +1,100 @@ +defmodule Plausible.Workers.ScheduleEmailReports do + @moduledoc """ + Job scheduling dispatch of weekly and monthly email reports. + """ + use Plausible.Repo + use Oban.Worker, queue: :schedule_email_reports + alias Plausible.Workers.SendEmailReport + + @impl Oban.Worker + @doc """ + Email reports should be sent on Monday at 9am according to the timezone + of a site. + """ + def perform(_job) do + schedule_weekly_emails() + schedule_monthly_emails() + end + + defp schedule_weekly_emails() do + weekly_jobs = + from( + j in Oban.Job, + where: + j.worker == "Plausible.Workers.SendEmailReport" and + fragment("(? ->> 'interval')", j.args) == "weekly" + ) + + sites = + Repo.all( + from s in Plausible.Site, + inner_join: t in assoc(s, :team), + join: wr in Plausible.Site.WeeklyReport, + on: wr.site_id == s.id, + left_join: job in subquery(weekly_jobs), + on: + fragment("(? -> 'site_id')::int", job.args) == s.id and + job.state not in ["completed", "discarded"], + where: is_nil(job), + where: not t.locked, + preload: [weekly_report: wr] + ) + + for site <- sites do + SendEmailReport.new(%{site_id: site.id, interval: "weekly"}, + scheduled_at: monday_9am(site.timezone) + ) + |> Oban.insert!() + end + + :ok + end + + def monday_9am(timezone) do + DateTime.now!(timezone) + |> DateTime.shift(week: 1) + |> Plausible.Times.beginning_of_week() + |> DateTime.shift(hour: 9) + end + + defp schedule_monthly_emails() do + monthly_jobs = + from( + j in Oban.Job, + where: + j.worker == "Plausible.Workers.SendEmailReport" and + fragment("(? ->> 'interval')", j.args) == "monthly" + ) + + sites = + Repo.all( + from s in Plausible.Site, + inner_join: t in assoc(s, :team), + join: mr in Plausible.Site.MonthlyReport, + on: mr.site_id == s.id, + left_join: job in subquery(monthly_jobs), + on: + fragment("(? -> 'site_id')::int", job.args) == s.id and + job.state not in ["completed", "discarded"], + where: is_nil(job), + where: not t.locked, + preload: [monthly_report: mr] + ) + + for site <- sites do + SendEmailReport.new(%{site_id: site.id, interval: "monthly"}, + scheduled_at: first_of_month_9am(site.timezone) + ) + |> Oban.insert!() + end + + :ok + end + + def first_of_month_9am(timezone) do + DateTime.now!(timezone) + |> DateTime.shift(month: 1) + |> Plausible.Times.beginning_of_month() + |> DateTime.shift(hour: 9) + end +end diff --git a/lib/workers/send_check_stats_emails.ex b/lib/workers/send_check_stats_emails.ex new file mode 100644 index 0000000000000000000000000000000000000000..c0ec65da5fb5d658ad99aa3eae36af314dfae743 --- /dev/null +++ b/lib/workers/send_check_stats_emails.ex @@ -0,0 +1,55 @@ +defmodule Plausible.Workers.SendCheckStatsEmails do + use Plausible.Repo + use Oban.Worker, queue: :check_stats_emails + + @impl Oban.Worker + def perform(_job) do + q = + from(u in Plausible.Auth.User, + left_join: ce in "check_stats_emails", + on: ce.user_id == u.id, + where: is_nil(ce.id), + where: + u.inserted_at > fragment("(now() at time zone 'utc') - '14 days'::interval") and + u.inserted_at < fragment("(now() at time zone 'utc') - '7 days'::interval") and + u.last_seen < fragment("(now() at time zone 'utc') - '7 days'::interval") + ) + + for user <- Repo.all(q) do + if eligible_for_check_stats_email?(user) do + send_check_stats_email(user) + end + end + + :ok + end + + defp eligible_for_check_stats_email?(user) do + sites = + from(tm in Plausible.Teams.Membership, + inner_join: t in assoc(tm, :team), + inner_join: s in assoc(t, :sites), + left_join: gm in assoc(tm, :guest_memberships), + where: tm.user_id == ^user.id, + where: tm.role != :guest or gm.site_id == s.id, + select: s + ) + |> Repo.all() + |> Repo.preload(:weekly_report) + + not Enum.any?(sites, fn site -> site.weekly_report end) and + Enum.any?(sites, &Plausible.Sites.has_stats?/1) + end + + defp send_check_stats_email(user) do + PlausibleWeb.Email.check_stats_email(user) + |> Plausible.Mailer.send() + + Repo.insert_all("check_stats_emails", [ + %{ + user_id: user.id, + timestamp: NaiveDateTime.utc_now() + } + ]) + end +end diff --git a/lib/workers/send_email_report.ex b/lib/workers/send_email_report.ex new file mode 100644 index 0000000000000000000000000000000000000000..89ce4f6e2f1dba4030341a9f55b2ca8c848ba20d --- /dev/null +++ b/lib/workers/send_email_report.ex @@ -0,0 +1,210 @@ +defmodule Plausible.Workers.SendEmailReport do + use Plausible + use Plausible.Repo + use Oban.Worker, queue: :send_email_reports, max_attempts: 1 + + alias Plausible.Stats.{QueryResult, QueryBuilder, ParsedQueryParams, QueryInclude} + + import Ecto.Query, only: [from: 2] + + @weekly "weekly" + @monthly "monthly" + + def perform(%Oban.Job{args: %{"interval" => interval, "site_id" => site_id}}) + when interval in [@weekly, @monthly] do + report_type = report_type(interval) + + site = + from(s in Plausible.Site, + where: s.id == ^site_id, + inner_join: r in assoc(s, ^report_type), + inner_join: t in assoc(s, :team), + preload: [{^report_type, r}, {:team, t}] + ) + |> Repo.one() + + with %Plausible.Site{} <- site, + %{} = report <- Map.fetch!(site, report_type), + true <- ok_to_send?(site) do + date_range = date_range(site, interval) + report_name = report_name(interval, date_range.first) + date_label = Calendar.strftime(date_range.last, "%-d %b %Y") + stats = stats(site, date_range) + + report + |> Map.fetch!(:recipients) + |> Enum.each(fn email -> + assigns = %{ + site: site, + report_name: report_name, + date_label: date_label, + unsubscribe_link: unsubscribe_link(site, email, interval), + site_member?: site_member?(site, email), + interval: interval, + stats: stats + } + + email + |> PlausibleWeb.Email.stats_report(assigns) + |> Plausible.Mailer.send() + end) + else + _ -> :discard + end + end + + defp report_type(@weekly), do: :weekly_report + defp report_type(@monthly), do: :monthly_report + + defp site_member?(site, email) do + user = Plausible.Auth.find_user_by(email: email) + user && Plausible.Teams.Memberships.site_member?(site, user) + end + + defp unsubscribe_link(site, email, interval) do + PlausibleWeb.Endpoint.url() <> + "/sites/#{URI.encode_www_form(site.domain)}/#{interval}-report/unsubscribe?email=#{email}" + end + + defp report_name(@weekly, _), do: "Weekly" + defp report_name(@monthly, first), do: Calendar.strftime(first, "%B") + + defp stats(site, date_range) do + shared_params = %ParsedQueryParams{ + metrics: [:visitors], + input_date_range: {:date_range, date_range.first, date_range.last}, + pagination: %{limit: 5, offset: 0} + } + + stats = stats_aggregates(site, shared_params) + pages = pages(site, shared_params) + sources = sources(site, shared_params) + goals = goals(site, shared_params) + + stats + |> Map.put(:pages, pages) + |> Map.put(:sources, sources) + |> Map.put(:goals, goals) + end + + defp stats_aggregates(site, %ParsedQueryParams{} = shared_params) do + query = + QueryBuilder.build!( + site, + struct!(shared_params, + metrics: [:pageviews, :visitors, :bounce_rate], + include: %QueryInclude{compare: :previous_period} + ) + ) + + %QueryResult{ + results: [ + %{ + metrics: [pageviews, visitors, bounce_rate], + comparison: %{ + change: [pageviews_change, visitors_change, bounce_rate_change] + } + } + ] + } = Plausible.Stats.query(site, query) + + %{ + pageviews: %{value: pageviews, change: pageviews_change}, + visitors: %{value: visitors, change: visitors_change}, + bounce_rate: %{value: bounce_rate, change: bounce_rate_change} + } + end + + defp pages(site, %ParsedQueryParams{} = shared_params) do + query = QueryBuilder.build!(site, struct!(shared_params, dimensions: ["event:page"])) + + site + |> Plausible.Stats.query(query) + |> Map.fetch!(:results) + |> Enum.map(fn %{metrics: [visitors], dimensions: [page]} -> + %{ + page: page, + visitors: visitors + } + end) + end + + defp sources(site, %ParsedQueryParams{} = shared_params) do + query = + QueryBuilder.build!( + site, + struct!(shared_params, + dimensions: ["visit:source"], + filters: [[:is_not, "visit:source", ["Direct / None"]]] + ) + ) + + site + |> Plausible.Stats.query(query) + |> Map.fetch!(:results) + |> Enum.map(fn %{metrics: [visitors], dimensions: [source]} -> + %{ + source: source, + visitors: visitors + } + end) + end + + defp goals(site, %ParsedQueryParams{} = shared_params) do + query = QueryBuilder.build!(site, struct!(shared_params, dimensions: ["event:goal"])) + + site + |> Plausible.Stats.query(query) + |> Map.fetch!(:results) + |> Enum.map(fn %{metrics: [visitors], dimensions: [goal_name]} -> + %{ + goal: goal_name, + visitors: visitors + } + end) + end + + defp date_range(site, @weekly) do + first = + site.timezone + |> DateTime.now!() + |> Date.shift(day: -7) + |> Date.beginning_of_week() + + last = + site.timezone + |> DateTime.now!() + |> DateTime.to_date() + |> Date.shift(day: -7) + |> Date.end_of_week() + + Date.range(first, last) + end + + defp date_range(site, @monthly) do + first = + site.timezone + |> DateTime.now!() + |> Date.shift(month: -1) + |> Date.beginning_of_month() + + last = + site.timezone + |> DateTime.now!() + |> DateTime.shift(month: -1) + |> DateTime.to_date() + |> Date.end_of_month() + + Date.range(first, last) + end + + on_ee do + defp ok_to_send?(site) do + Plausible.Sites.regular?(site) or + (Plausible.Sites.consolidated?(site) and + Plausible.ConsolidatedView.ok_to_display?(site.team)) + end + else + defp ok_to_send?(_site), do: always(true) + end +end diff --git a/lib/workers/send_site_setup_emails.ex b/lib/workers/send_site_setup_emails.ex new file mode 100644 index 0000000000000000000000000000000000000000..c3eef54666ba6c6a2e2dbc798f7f38c81d2f8fc6 --- /dev/null +++ b/lib/workers/send_site_setup_emails.ex @@ -0,0 +1,121 @@ +defmodule Plausible.Workers.SendSiteSetupEmails do + @moduledoc """ + Job sending site setup emails. + """ + + use Plausible.Repo + use Oban.Worker, queue: :site_setup_emails + + @impl Oban.Worker + def perform(_job) do + send_create_site_emails() + send_setup_help_emails() + send_setup_success_emails() + + :ok + end + + defp send_create_site_emails() do + q = + from u in Plausible.Auth.User, + as: :user, + where: + not exists( + from tm in Plausible.Teams.Membership, + where: tm.user_id == parent_as(:user).id, + select: true + ), + where: + not exists( + from se in "create_site_emails", + where: se.user_id == parent_as(:user).id, + select: true + ), + where: + u.inserted_at > fragment("(now() at time zone 'utc') - '72 hours'::interval") and + u.inserted_at < fragment("(now() at time zone 'utc') - '48 hours'::interval") + + for user <- Repo.all(q) do + send_create_site_email(user) + end + end + + defp send_setup_help_emails() do + q = + from(s in Plausible.Site.regular(), + left_join: se in "setup_help_emails", + on: se.site_id == s.id, + where: is_nil(se.id), + where: s.inserted_at > fragment("(now() at time zone 'utc') - '72 hours'::interval"), + preload: [:owners, :team] + ) + + for site <- Repo.all(q) do + owners = site.owners + setup_completed = Plausible.Sites.has_stats?(site) + hours_passed = NaiveDateTime.diff(DateTime.utc_now(), site.inserted_at, :hour) + + if !setup_completed && hours_passed > 47 do + send_setup_help_email(owners, site) + end + end + end + + defp send_setup_success_emails() do + q = + from(s in Plausible.Site.regular(), + left_join: se in "setup_success_emails", + on: se.site_id == s.id, + where: is_nil(se.id), + inner_join: t in assoc(s, :team), + where: s.inserted_at > fragment("(now() at time zone 'utc') - '72 hours'::interval"), + preload: [:owners, team: t] + ) + + for site <- Repo.all(q) do + if Plausible.Sites.has_stats?(site) do + send_setup_success_email(site) + end + end + end + + defp send_create_site_email(user) do + PlausibleWeb.Email.create_site_email(user) + |> Plausible.Mailer.send() + + Repo.insert_all("create_site_emails", [ + %{ + user_id: user.id, + timestamp: NaiveDateTime.utc_now() + } + ]) + end + + defp send_setup_success_email(site) do + for owner <- site.owners do + PlausibleWeb.Email.site_setup_success(owner, site) + |> Plausible.Mailer.send() + end + + Repo.insert_all("setup_success_emails", [ + %{ + site_id: site.id, + timestamp: NaiveDateTime.utc_now() + } + ]) + end + + defp send_setup_help_email(users, site) do + for user <- users do + PlausibleWeb.Email.site_setup_help(user, site.team, site) + |> Plausible.Mailer.send() + end + + Repo.insert_all("setup_help_emails", [ + %{ + site_id: site.id, + timestamp: NaiveDateTime.utc_now() + } + ]) + end +end diff --git a/lib/workers/send_trial_notifications.ex b/lib/workers/send_trial_notifications.ex new file mode 100644 index 0000000000000000000000000000000000000000..8269f749dedfd56454c4fc35de4227c05a3b7850 --- /dev/null +++ b/lib/workers/send_trial_notifications.ex @@ -0,0 +1,85 @@ +defmodule Plausible.Workers.SendTrialNotifications do + @moduledoc """ + Job sending trial end notification emails. + """ + use Plausible.Repo + + use Oban.Worker, + queue: :trial_notification_emails, + max_attempts: 1 + + alias Plausible.Teams + + @impl Oban.Worker + def perform(_job) do + teams = + Repo.all( + from t in Teams.Team, + inner_join: o in assoc(t, :owners), + left_join: bm in assoc(t, :billing_members), + left_join: s in assoc(t, :subscription), + where: not is_nil(t.trial_expiry_date), + where: is_nil(s.id), + order_by: t.inserted_at, + preload: [owners: o, billing_members: bm] + ) + |> Enum.filter(&Teams.has_active_sites?/1) + + for team <- teams do + recipients = team.owners ++ team.billing_members + + case Date.diff(team.trial_expiry_date, Date.utc_today()) do + 7 -> + send_one_week_reminder(recipients, team) + + 1 -> + send_tomorrow_reminder(recipients, team) + + 0 -> + send_today_reminder(recipients, team) + + -1 -> + send_over_reminder(recipients, team) + + _ -> + nil + end + end + + :ok + end + + defp send_one_week_reminder(users, team) do + for user <- users do + PlausibleWeb.Email.trial_one_week_reminder(user, team) + |> Plausible.Mailer.send() + end + end + + defp send_tomorrow_reminder(users, team) do + usage = Plausible.Teams.Billing.usage_cycle(team, :last_30_days) + suggested_volume = Plausible.Billing.Plans.suggest_volume(team, usage.total) + + for user <- users do + PlausibleWeb.Email.trial_ending_tomorrow_email(user, team, usage, suggested_volume) + |> Plausible.Mailer.send() + end + end + + defp send_today_reminder(users, team) do + usage = Plausible.Teams.Billing.usage_cycle(team, :last_30_days) + suggested_volume = Plausible.Billing.Plans.suggest_volume(team, usage.total) + + for user <- users do + PlausibleWeb.Email.trial_ending_today_email(user, team, usage, suggested_volume) + |> Plausible.Mailer.send() + end + end + + defp send_over_reminder(users, team) do + for user <- users do + PlausibleWeb.Email.trial_over_email(user, team) + |> Plausible.Mailer.send() + end + end +end diff --git a/lib/workers/set_legacy_time_on_page_cutoff.ex b/lib/workers/set_legacy_time_on_page_cutoff.ex new file mode 100644 index 0000000000000000000000000000000000000000..391c58d91bd3c2e2e14dc7be130265999ac7985e --- /dev/null +++ b/lib/workers/set_legacy_time_on_page_cutoff.ex @@ -0,0 +1,96 @@ +defmodule Plausible.Workers.SetLegacyTimeOnPageCutoff do + @moduledoc """ + Sets sites `legacy_time_on_page_cutoff` depending on whether they have + sent us engagement data in the past. + """ + + use Plausible.Repo + use Plausible.ClickhouseRepo + use Oban.Worker, queue: :legacy_time_on_page_cutoff + + import Ecto.Query + + require Logger + + @impl Oban.Worker + def perform(%Oban.Job{args: args}) do + cutoff_date = Map.get(args, "cutoff_date", Date.utc_today()) + + {small_sites, large_sites} = + sites_with_engagement_data(cutoff_date) + |> filter_sites_needing_update() + + to_update = small_sites ++ large_sites + + if length(to_update) > 0 do + Logger.notice( + "Setting legacy_time_on_page_cutoff for #{length(to_update)} sites (#{length(small_sites)} small, #{length(large_sites)} large)" + ) + + {count, _} = + Repo.update_all( + from(s in Plausible.Site.regular(), + where: s.id in ^to_update, + where: is_nil(s.legacy_time_on_page_cutoff) + ), + set: [legacy_time_on_page_cutoff: cutoff_date] + ) + + Logger.notice( + "Successfully set legacy_time_on_page_cutoff=#{cutoff_date} for #{count} sites" + ) + else + Logger.debug("No sites legacy_time_on_page_cutoff needs updating") + end + + :ok + end + + defp sites_with_engagement_data(cutoff_date) do + site_info_q = + from( + e in "events_v2", + where: e.timestamp >= fragment("toStartOfHour(? - toIntervalHour(48))", ^cutoff_date), + where: e.timestamp < fragment("toStartOfHour(? - toIntervalHour(24))", ^cutoff_date), + group_by: e.site_id, + select: %{ + site_id: e.site_id, + hours_with_engagement: + fragment( + "uniqIf(toStartOfHour(timestamp), name = 'engagement' AND engagement_time > 0)" + ), + is_small_site: fragment("count() < 2000") + } + ) + + q = + from( + s in subquery(site_info_q), + select: %{ + small_sites: + fragment("groupArrayIf(site_id, (is_small_site and hours_with_engagement > 0))"), + large_sites: + fragment("groupArrayIf(site_id, (not is_small_site and hours_with_engagement = 24))") + } + ) + + result = ClickhouseRepo.one(q) + + {MapSet.new(result.small_sites), MapSet.new(result.large_sites)} + end + + defp filter_sites_needing_update({small_sites, large_sites}) do + needing_update = + from(s in Plausible.Site.regular(), + where: is_nil(s.legacy_time_on_page_cutoff), + select: s.id + ) + |> Repo.all() + |> MapSet.new() + + { + small_sites |> MapSet.intersection(needing_update) |> MapSet.to_list(), + large_sites |> MapSet.intersection(needing_update) |> MapSet.to_list() + } + end +end diff --git a/lib/workers/traffic_change_notifier.ex b/lib/workers/traffic_change_notifier.ex new file mode 100644 index 0000000000000000000000000000000000000000..b026e02eaa15afce5421bb396a5c92cc90aba7d2 --- /dev/null +++ b/lib/workers/traffic_change_notifier.ex @@ -0,0 +1,181 @@ +defmodule Plausible.Workers.TrafficChangeNotifier do + @moduledoc """ + Oban service sending out traffic drop/spike notifications + """ + use Plausible + use Plausible.Repo + alias Plausible.Stats.{Clickhouse, ParsedQueryParams, QueryBuilder} + alias Plausible.Site.TrafficChangeNotification + + alias PlausibleWeb.Router.Helpers, as: Routes + + use Oban.Worker, queue: :spike_notifications + @min_interval_hours 12 + + @impl Oban.Worker + def perform(_job, now \\ NaiveDateTime.utc_now(:second)) do + today = NaiveDateTime.to_date(now) + + notifications = + Repo.all( + from sn in TrafficChangeNotification, + where: + is_nil(sn.last_sent) or + sn.last_sent < ^NaiveDateTime.add(now, -@min_interval_hours, :hour), + inner_join: s in assoc(sn, :site), + inner_join: t in assoc(s, :team), + where: not t.locked, + where: is_nil(t.accept_traffic_until) or t.accept_traffic_until > ^today, + preload: [site: {s, team: t}] + ) + + for notification <- notifications, ok_to_send?(notification.site) do + handle_notification(notification, now) + end + + :ok + end + + defp handle_notification(%TrafficChangeNotification{type: :spike} = notification, now) do + current_visitors = Clickhouse.current_visitors(notification.site) + + if current_visitors >= notification.threshold do + stats = + notification.site + |> get_traffic_spike_stats() + |> Map.put(:current_visitors, current_visitors) + + notify_spike(notification, stats, now) + end + end + + defp handle_notification(%TrafficChangeNotification{type: :drop} = notification, now) do + current_visitors = Clickhouse.current_visitors_12h(notification.site) + + if current_visitors < notification.threshold do + notify_drop(notification, current_visitors, now) + end + end + + defp notify_spike(notification, stats, now) do + for recipient_email <- notification.recipients do + send_spike_notification(recipient_email, notification.site, stats) + end + + notification + |> TrafficChangeNotification.was_sent(now) + |> Repo.update() + end + + defp notify_drop(notification, current_visitors, now) do + for recipient_email <- notification.recipients do + send_drop_notification(recipient_email, notification.site, current_visitors) + end + + notification + |> TrafficChangeNotification.was_sent(now) + |> Repo.update() + end + + defp send_spike_notification(recipient_email, site, stats) do + dashboard_link = + if site_member?(site, recipient_email) do + Routes.stats_url(PlausibleWeb.Endpoint, :stats, site.domain, []) <> + "?__team=#{site.team.identifier}" + end + + template = + PlausibleWeb.Email.spike_notification( + recipient_email, + site, + stats, + dashboard_link + ) + + Plausible.Mailer.send(template) + end + + defp send_drop_notification(recipient_email, site, current_visitors) do + site_member? = site_member?(site, recipient_email) + + dashboard_link = + if site_member? do + Routes.stats_url(PlausibleWeb.Endpoint, :stats, site.domain, []) <> + "?__team=#{site.team.identifier}" + end + + installation_link = + if site_member? and Plausible.Sites.regular?(site) do + Routes.site_url(PlausibleWeb.Endpoint, :installation, site.domain, + flow: PlausibleWeb.Flows.review() + ) <> "&__team=#{site.team.identifier}" + end + + template = + PlausibleWeb.Email.drop_notification( + recipient_email, + site, + current_visitors, + dashboard_link, + installation_link + ) + + Plausible.Mailer.send(template) + end + + defp get_traffic_spike_stats(site) do + %{} + |> put_sources(site) + |> put_pages(site) + end + + @base_query_params %ParsedQueryParams{ + metrics: [:visitors], + pagination: %{limit: 3, offset: 0}, + input_date_range: :realtime + } + + defp put_sources(stats, site) do + query = + QueryBuilder.build!( + site, + struct!(@base_query_params, + dimensions: ["visit:source"], + filters: [[:is_not, "visit:source", ["Direct / None"]]] + ) + ) + + %{results: sources} = Plausible.Stats.query(site, query) + + Map.put(stats, :sources, sources) + end + + defp put_pages(stats, site) do + query = QueryBuilder.build!(site, struct!(@base_query_params, dimensions: ["event:page"])) + + %{results: pages} = Plausible.Stats.query(site, query) + + Map.put(stats, :pages, pages) + end + + defp site_member?(site, recipient_email) do + from(tm in Plausible.Teams.Membership, + inner_join: u in assoc(tm, :user), + left_join: gm in assoc(tm, :guest_memberships), + where: tm.team_id == ^site.team_id, + where: tm.role != :guest or gm.site_id == ^site.id, + where: u.email == ^recipient_email + ) + |> Repo.exists?() + end + + on_ee do + defp ok_to_send?(site) do + Plausible.Sites.regular?(site) or + (Plausible.Sites.consolidated?(site) and + Plausible.ConsolidatedView.ok_to_display?(site.team)) + end + else + defp ok_to_send?(_site), do: always(true) + end +end diff --git a/mix.exs b/mix.exs new file mode 100644 index 0000000000000000000000000000000000000000..052f80b29441b17b1e62a89e8d2d91704cf7e0fe --- /dev/null +++ b/mix.exs @@ -0,0 +1,246 @@ +defmodule Plausible.MixProject do + use Mix.Project + + def project do + [ + name: "Plausible", + source_url: "https://github.com/plausible/analytics", + docs: docs(), + app: :plausible, + version: System.get_env("APP_VERSION", "0.0.1"), + elixir: "~> 1.18", + elixirc_paths: elixirc_paths(Mix.env()), + start_permanent: Mix.env() in [:prod, :ce, :load], + aliases: aliases(), + deps: deps(), + test_coverage: [ + tool: ExCoveralls + ], + listeners: [Phoenix.CodeReloader], + releases: [ + plausible: [ + include_executables_for: [:unix], + config_providers: [ + {Config.Reader, + path: {:system, "RELEASE_ROOT", "/import_extra_config.exs"}, imports: []} + ] + ] + ], + dialyzer: [ + plt_file: {:no_warn, "priv/plts/dialyzer.plt"}, + plt_add_apps: [:mix, :ex_unit] + ] + ] + end + + # Configuration for the OTP application. + # + # Type `mix help compile.app` for more information. + def application do + [ + mod: {Plausible.Application, []}, + extra_applications: + [ + :logger, + :runtime_tools, + :tls_certificate_check, + :opentelemetry_exporter + ] ++ if(Mix.env() in [:dev, :load], do: [:tools, :observer, :wx], else: []) + ] + end + + def cli do + [ + preferred_envs: [ + "test.e2e": :e2e_test, + "test.e2e.ui": :e2e_test + ] + ] + end + + # Specifies which paths to compile per environment. + defp elixirc_paths(env) when env in [:test, :e2e_test, :dev], + do: ["lib", "test/support", "extra/lib"] + + defp elixirc_paths(env) when env in [:ce_test, :ce_dev], + do: ["lib", "test/support"] + + defp elixirc_paths(:ce), do: ["lib"] + defp elixirc_paths(_), do: ["lib", "extra/lib"] + + # Specifies your project dependencies. + # + # Type `mix help deps` for examples and options. + defp deps do + [ + {:bamboo, "~> 2.3", override: true}, + {:bamboo_postmark, git: "https://github.com/plausible/bamboo_postmark.git", branch: "main"}, + {:bamboo_mua, "~> 0.2.0"}, + {:bcrypt_elixir, "~> 3.3"}, + {:bypass, "~> 2.1", only: [:dev, :test, :ce_test, :e2e_test]}, + {:ecto_ch, "~> 0.8.6"}, + {:cloak, "~> 1.1"}, + {:cloak_ecto, "~> 1.2"}, + {:combination, "~> 0.0.3"}, + {:cors_plug, "~> 3.0"}, + {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, + {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}, + {:double, "~> 0.8.0", only: [:dev, :test, :ce_test, :ce_dev, :e2e_test]}, + {:ecto, "~> 3.13.5"}, + {:ecto_sql, "~> 3.13.2"}, + {:envy, "~> 1.1.1"}, + {:eqrcode, "~> 0.2.1"}, + {:ex_machina, "~> 2.3", only: [:dev, :test, :ce_dev, :ce_test, :e2e_test]}, + {:excoveralls, "~> 0.10", only: :test}, + {:finch, "~> 0.23"}, + {:lazy_html, "~> 0.1.8"}, + {:fun_with_flags, "~> 1.13.0"}, + {:fun_with_flags_ui, "~> 1.0"}, + {:locus, "~> 2.3"}, + {:gen_cycle, "~> 1.0.4"}, + {:hackney, "~> 1.8"}, + {:jason, "~> 1.3"}, + {:location, git: "https://github.com/plausible/location.git"}, + {:mox, "~> 1.0", only: [:test, :ce_test, :e2e_test]}, + {:nanoid, "~> 2.1.0"}, + {:nimble_totp, "~> 1.0"}, + {:oban, "~> 2.20.1"}, + {:observer_cli, "~> 1.7"}, + {:opentelemetry, "~> 1.7"}, + {:opentelemetry_api, "~> 1.5"}, + {:opentelemetry_api_experimental, + git: "https://github.com/open-telemetry/opentelemetry-erlang.git", + ref: "b241ebf4feb7558738d009583bae8602b3c49fd0", + sparse: "apps/opentelemetry_api_experimental", + override: true}, + {:opentelemetry_ecto, "~> 1.2"}, + {:opentelemetry_exporter, "~> 1.10"}, + {:opentelemetry_experimental, + git: "https://github.com/open-telemetry/opentelemetry-erlang.git", + ref: "b241ebf4feb7558738d009583bae8602b3c49fd0", + sparse: "apps/opentelemetry_experimental", + override: true}, + {:opentelemetry_phoenix, "~> 2.0.1"}, + {:opentelemetry_oban, "~> 1.1"}, + {:opentelemetry_cowboy, "~> 1.0"}, + # # https://github.com/open-telemetry/opentelemetry-erlang-contrib/issues/428 + {:opentelemetry_semantic_conventions, "~> 1.27", override: true}, + {:phoenix, "~> 1.8.2"}, + {:phoenix_view, "~> 2.0"}, + {:phoenix_ecto, "~> 4.5"}, + {:phoenix_html, "~> 4.1"}, + {:phoenix_live_reload, "~> 1.2", only: [:dev, :ce_dev]}, + {:phoenix_pubsub, "~> 2.0"}, + {:phoenix_live_view, "~> 1.1.17"}, + {:php_serializer, "~> 2.0"}, + {:plug, "~> 1.13", override: true}, + {:prima, "~> 0.2.6"}, + {:plug_cowboy, "~> 2.3"}, + {:polymorphic_embed, "~> 5.0"}, + {:postgrex, "~> 0.22.2"}, + {:prom_ex, "~> 1.8"}, + {:peep, "~> 3.0"}, + {:public_suffix, git: "https://github.com/axelson/publicsuffix-elixir"}, + {:recon, "~> 2.5"}, + {:ref_inspector, "~> 2.0"}, + {:referrer_blocklist, git: "https://github.com/plausible/referrer-blocklist.git"}, + {:sentry, "~> 11.0.4"}, + {:simple_saml, "~> 1.2"}, + {:xml_builder, "~> 2.1"}, + {:siphash, "~> 3.2"}, + {:timex, "~> 3.7"}, + {:ua_inspector, "~> 3.12"}, + {:ex_doc, "~> 0.28", only: :dev, runtime: false}, + {:ex_money, "~> 5.24.2"}, + {:mjml_eex, "~> 0.12.0"}, + {:mjml, "~> 4.0.0"}, + {:heroicons, "~> 0.5.0"}, + {:zxcvbn, git: "https://github.com/techgaun/zxcvbn-elixir.git"}, + {:open_api_spex, "~> 3.22.1"}, + {:joken, "~> 2.5"}, + {:paginator, git: "https://github.com/duffelhq/paginator.git"}, + {:esbuild, "~> 0.7", runtime: Mix.env() in [:dev, :ce_dev]}, + {:tailwind, "~> 0.4.0", runtime: Mix.env() in [:dev, :ce_dev]}, + {:ex_json_logger, "~> 1.4.0"}, + {:ecto_network, "~> 1.6.0"}, + {:ex_aws, "~> 2.5"}, + {:ex_aws_s3, "~> 2.5"}, + {:sweet_xml, "~> 0.7.4"}, + {:zstream, "~> 0.6.4"}, + {:con_cache, + git: "https://github.com/aerosol/con_cache", branch: "ensure-dirty-ops-emit-telemetry"}, + {:req, "~> 0.6"}, + {:opentelemetry_req, "~> 1.0"}, + {:happy_tcp, github: "ruslandoga/happy_tcp", only: [:ce, :ce_dev, :ce_test, :e2e_test]}, + {:ex_json_schema, "~> 0.11.1"}, + {:odgn_json_pointer, "~> 3.1.0"}, + {:phoenix_bakery, "~> 0.1.2", only: [:ce, :ce_dev, :ce_test, :e2e_test]}, + {:site_encrypt, github: "sasa1977/site_encrypt", only: [:ce, :ce_dev, :ce_test, :e2e_test]}, + {:phoenix_html_helpers, "~> 1.0"}, + {:libcluster, "~> 3.5"}, + {:decimal, "~> 3.0", override: true}, + {:logger_backends, "~> 1.0.0"} + ] + end + + defp aliases do + [ + setup: ["deps.get", "ecto.setup", "assets.setup", "assets.build"], + "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"], + "ecto.reset": ["ecto.drop", "ecto.setup"], + test: ["ecto.create --quiet", "ecto.migrate", "test", "clean_clickhouse"], + "e2e.setup": [ + "cmd npm install --prefix ./e2e", + "cmd npm exec playwright install --with-deps chromium --prefix ./e2e" + ], + # accepts Playwright CLI arguments (https://playwright.dev/docs/test-cli), for example + # mix test.e2e --ui + # mix test.e2e --debug segments.spec.ts + "test.e2e": [ + "esbuild default", + "ecto.create --quiet", + "ecto.migrate", + "clean_postgres", + "clean_clickhouse", + "cmd npm run --prefix ./e2e test -- " + ], + "assets.setup": ["tailwind.install --if-missing", "esbuild.install --if-missing"], + "assets.typecheck": ["cmd npm --prefix assets run typecheck"], + "assets.build": [ + "tailwind default", + "esbuild default" + ], + "assets.deploy": [ + "tailwind default --minify", + "esbuild default --minify", + "phx.digest" + ] + ] + end + + defp docs do + [ + main: "readme", + logo: "priv/static/images/ee/favicon-32x32.png", + extras: + Path.wildcard("guides/**/*.md") ++ + [ + "README.md": [filename: "readme", title: "Introduction"], + "CONTRIBUTING.md": [filename: "contributing", title: "Contributing"] + ], + groups_for_extras: [ + Features: Path.wildcard("guides/features/*.md") + ], + before_closing_body_tag: fn + :html -> + """ + + + """ + + _ -> + "" + end + ] + end +end diff --git a/mix.lock b/mix.lock new file mode 100644 index 0000000000000000000000000000000000000000..1ec2693e1faf6e875af4c50fa0b0f2242298f736 --- /dev/null +++ b/mix.lock @@ -0,0 +1,171 @@ +%{ + "acceptor_pool": {:hex, :acceptor_pool, "1.0.0", "43c20d2acae35f0c2bcd64f9d2bde267e459f0f3fd23dab26485bf518c281b21", [:rebar3], [], "hexpm", "0cbcd83fdc8b9ad2eee2067ef8b91a14858a5883cb7cd800e6fcd5803e158788"}, + "attrs": {:hex, :attrs, "0.6.0", "25d738b47829f964a786ef73897d2550b66f3e7d1d7c49a83bc8fd81c71bed93", [:mix], [], "hexpm", "9c30ac15255c2ba8399263db55ba32c2f4e5ec267b654ce23df99168b405c82e"}, + "bamboo": {:hex, :bamboo, "2.5.0", "973f5cb1471a1d2d7d9da7b8e4f6096afb6a133f85394631184fd40be8adb8ab", [:mix], [{:hackney, ">= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.4 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "35c8635ff6677a81ab7258944ff15739280f3254a041b6f0229dddeb9b90ad3d"}, + "bamboo_mua": {:hex, :bamboo_mua, "0.2.4", "1143cb64e0455bd496d4da14132308d1e44a825c2ba5e295c6c2adc33dff3af0", [:mix], [{:bamboo, "~> 2.0", [hex: :bamboo, repo: "hexpm", optional: false]}, {:mail, "~> 0.4.0", [hex: :mail, repo: "hexpm", optional: false]}, {:mua, "~> 0.2.3", [hex: :mua, repo: "hexpm", optional: false]}], "hexpm", "6802c491f8585b047d7f75f0d57f44398b44d105614115a3346710c2de75cc1f"}, + "bamboo_postmark": {:git, "https://github.com/plausible/bamboo_postmark.git", "c6a773d1b7a4e5c9ec802ace64b5bb526504c25a", [branch: "main"]}, + "bcrypt_elixir": {:hex, :bcrypt_elixir, "3.3.2", "d50091e3c9492d73e17fc1e1619a9b09d6a5ef99160eb4d736926fd475a16ca3", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "471be5151874ae7931911057d1467d908955f93554f7a6cd1b7d804cac8cef53"}, + "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, + "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"}, + "castore": {:hex, :castore, "1.0.19", "6903cabdfd9d1af46454126e7c8385186659dd33ecfb74a885cae52221ad6109", [:mix], [], "hexpm", "3669e6cab13f54c2df26b3e6833745d647f35b6e30d8ddd5975df0d5c842ca98"}, + "cc_precompiler": {:hex, :cc_precompiler, "0.1.11", "8c844d0b9fb98a3edea067f94f616b3f6b29b959b6b3bf25fee94ffe34364768", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "3427232caf0835f94680e5bcf082408a70b48ad68a5f5c0b02a3bea9f3a075b9"}, + "certifi": {:hex, :certifi, "2.15.0", "0e6e882fcdaaa0a5a9f2b3db55b1394dba07e8d6d9bcad08318fb604c6839712", [:rebar3], [], "hexpm", "b147ed22ce71d72eafdad94f055165c1c182f61a2ff49df28bcc71d1d5b94a60"}, + "ch": {:hex, :ch, "0.7.1", "116c08094b30d095c3bd6a8fe4ebe19fdaaf3dce84e2413cfdd6af157baf6303", [:mix], [{:db_connection, "~> 2.9.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.13.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "3c1c900291ff9c4c077cd1dc0c265051a3f1d26320d58b37ed9e91b33d41a868"}, + "chatterbox": {:hex, :ts_chatterbox, "0.15.1", "5cac4d15dd7ad61fc3c4415ce4826fc563d4643dee897a558ec4ea0b1c835c9c", [:rebar3], [{:hpack, "~> 0.3.0", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "4f75b91451338bc0da5f52f3480fa6ef6e3a2aeecfc33686d6b3d0a0948f31aa"}, + "cldr_utils": {:hex, :cldr_utils, "2.29.7", "2189bc0117efe91c684558e79174b45eb43135595b7d1fe9b57f53917be195c1", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.5", [hex: :certifi, repo: "hexpm", optional: true]}, {:decimal, "~> 1.9 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "4bddcd597fee34e2d2829ae9ef62bcfef8d97ae5f6b75f0c6ee37a3db31aa73a"}, + "cloak": {:hex, :cloak, "1.1.4", "aba387b22ea4d80d92d38ab1890cc528b06e0e7ef2a4581d71c3fdad59e997e7", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "92b20527b9aba3d939fab0dd32ce592ff86361547cfdc87d74edce6f980eb3d7"}, + "cloak_ecto": {:hex, :cloak_ecto, "1.3.0", "0de127c857d7452ba3c3367f53fb814b0410ff9c680a8d20fbe8b9a3c57a1118", [:mix], [{:cloak, "~> 1.1.1", [hex: :cloak, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "314beb0c123b8a800418ca1d51065b27ba3b15f085977e65c0f7b2adab2de1cc"}, + "combination": {:hex, :combination, "0.0.3", "746aedca63d833293ec6e835aa1f34974868829b1486b1e1cb0685f0b2ae1f41", [:mix], [], "hexpm", "72b099f463df42ef7dc6371d250c7070b57b6c5902853f69deb894f79eda18ca"}, + "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, + "comeonin": {:hex, :comeonin, "5.5.1", "5113e5f3800799787de08a6e0db307133850e635d34e9fab23c70b6501669510", [:mix], [], "hexpm", "65aac8f19938145377cee73973f192c5645873dcf550a8a6b18187d17c13ccdb"}, + "con_cache": {:git, "https://github.com/aerosol/con_cache", "d050e133742e512cf2caa4598607e60dd0161072", [branch: "ensure-dirty-ops-emit-telemetry"]}, + "cors_plug": {:hex, :cors_plug, "3.0.3", "7c3ac52b39624bc616db2e937c282f3f623f25f8d550068b6710e58d04a0e330", [:mix], [{:plug, "~> 1.13", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "3f2d759e8c272ed3835fab2ef11b46bddab8c1ab9528167bd463b6452edf830d"}, + "cowboy": {:hex, :cowboy, "2.16.1", "fa04080b602ff25c40a7700f2dc0152dbc1ba26b42093ae0fa9bb7a337d5a242", [:make, :rebar3], [{:cowlib, ">= 2.16.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "b8ea4dd317a043e3177ec840cfa3bcb47cfb41035d3abb24d954dc7d51def399"}, + "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, + "cowlib": {:hex, :cowlib, "2.18.0", "4a3ef8cb012c21c7cbb7c925f75405e26d7fcd393fd78c46187539dbc9a48310", [:make, :rebar3], [], "hexpm", "c4f89d33a61162d4cfdcb5a1af7e562d80a700b2573cc71020ce6521b152dc1a"}, + "credo": {:hex, :credo, "1.7.19", "cc52129665fc7c15143d47838fda0f9cd6dac9ceced7bf4da6f85fcbfe64b12a", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "2d8bc95d5a7bb99dd2613621d4f08c6a3575c3fd4b62e6a2b48a100352a557b8"}, + "ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"}, + "db_connection": {:hex, :db_connection, "2.9.0", "a6a97c5c958a2d7091a58a9be40caf41ab496b0701d21e1d1abff3fa27a7f371", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "17d502eacaf61829db98facf6f20808ed33da6ccf495354a41e64fe42f9c509c"}, + "decimal": {:hex, :decimal, "3.1.1", "430d87b04011ce6cbd4fd205be758311a81f87d552d40904abd00f015935b1d0", [:mix], [], "hexpm", "c5f25f2ced74a0587d03e6023f595db8e924c9d3922c8c8ffd9edfc4498cf1f6"}, + "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, + "digital_token": {:hex, :digital_token, "2.0.0", "fb3c74411790994895c4a61387ab322b64a682995ef2f54a4627c92d46be5d62", [:mix], [], "hexpm", "cbd2fff52770284a8251540a4b4e529e9738c6fe052d7f3c3428eb5c817385cd"}, + "double": {:hex, :double, "0.8.2", "8e1cfcccdaef76c18846bc08e555555a2a699b806fa207b6468572a60513cc6a", [:mix], [], "hexpm", "90287642b2ec86125e0457aaba2ab0e80f7d7050cc80a0cef733e59bd70aa67c"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, + "ecto": {:hex, :ecto, "3.13.6", "352135b474f91d1ab99a1b502171d207e9db60421c9e3d0ecab4c7ab96b24d14", [:mix], [{:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8afa059bc16cd2c94739ec0a11e3e5df69d828125119109bef35f20a21a76af2"}, + "ecto_ch": {:hex, :ecto_ch, "0.8.6", "f31b507e86690c003f46e75d6e742e6b5d8ce34b6b10a86604b1c3aa785e0b56", [:mix], [{:ch, "~> 0.5.0 or ~> 0.6.0 or ~> 0.7.0", [hex: :ch, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.13.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}], "hexpm", "6ca9f1cf9680452b1925c6a3a7b5e3d8b12e38ee134b03c6a45a8b26434fad97"}, + "ecto_network": {:hex, :ecto_network, "1.6.0", "f3159c0adb280321b72dc711b3fc7bcc48e2772f71db292a1c3529c587603611", [:mix], [{:ecto_sql, ">= 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:phoenix_html, ">= 0.0.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.14.0", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "000411a6d3a585845f6809383f20afd418c090e104f862241124ac9518ac1073"}, + "ecto_sql": {:hex, :ecto_sql, "3.13.4", "b6e9d07557ddba62508a9ce4a484989a5bb5e9a048ae0e695f6d93f095c25d60", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.13.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2b38cf0749ca4d1c5a8bcbff79bbe15446861ca12a61f9fba604486cb6b62a14"}, + "elixir_make": {:hex, :elixir_make, "0.9.0", "6484b3cd8c0cee58f09f05ecaf1a140a8c97670671a6a0e7ab4dc326c3109726", [:mix], [], "hexpm", "db23d4fd8b757462ad02f8aa73431a426fe6671c80b200d9710caf3d1dd0ffdb"}, + "envy": {:hex, :envy, "1.1.1", "0bc9bd654dec24fcdf203f7c5aa1b8f30620f12cfb28c589d5e9c38fe1b07475", [:mix], [], "hexpm", "7061eb1a47415fd757145d8dec10dc0b1e48344960265cb108f194c4252c3a89"}, + "eqrcode": {:hex, :eqrcode, "0.2.1", "d12838813e8fc87b8940cc05f9baadb189031f6009facdc56ff074375ec73b6e", [:mix], [], "hexpm", "d5828a222b904c68360e7dc2a40c3ef33a1328b7c074583898040f389f928025"}, + "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"}, + "esbuild": {:hex, :esbuild, "0.10.0", "b0aa3388a1c23e727c5a3e7427c932d89ee791746b0081bbe56103e9ef3d291f", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "468489cda427b974a7cc9f03ace55368a83e1a7be12fba7e30969af78e5f8c70"}, + "ex_aws": {:hex, :ex_aws, "2.5.1", "7418917974ea42e9e84b25e88b9f3d21a861d5f953ad453e212f48e593d8d39f", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "1b95431f70c446fa1871f0eb9b183043c5a625f75f9948a42d25f43ae2eff12b"}, + "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.3", "422468e5c3e1a4da5298e66c3468b465cfd354b842e512cb1f6fbbe4e2f5bdaf", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "4f09dd372cc386550e484808c5ac5027766c8d0cd8271ccc578b82ee6ef4f3b8"}, + "ex_cldr": {:hex, :ex_cldr, "2.47.4", "64e1efda73d6c5fa0342d5b6554f39b740fa02ed540dc03997facf573101ad9e", [:mix], [{:cldr_utils, "~> 2.29", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:decimal, "~> 1.6 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:gettext, "~> 0.19 or ~> 1.0", [hex: :gettext, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: true]}], "hexpm", "918aabc032955f3eac70abbdf2c5469433132edfaaaccee55451f074ee1ccdba"}, + "ex_cldr_currencies": {:hex, :ex_cldr_currencies, "2.17.2", "4892b5255a205e8133297b702abbe13c9b51d0281fa8c2ea100f51df9657c3c7", [:mix], [{:ex_cldr, "~> 2.38", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "797095c106a2fe6632981531e29cfb1d2f8ee7de626f4d6243f974d6f74a0112"}, + "ex_cldr_numbers": {:hex, :ex_cldr_numbers, "2.38.3", "64a99531df09397174b7f477785c43016f223d01856725b541a19d36b8149523", [:mix], [{:decimal, "~> 1.6 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:digital_token, "~> 1.0 or ~> 2.0", [hex: :digital_token, repo: "hexpm", optional: false]}, {:ex_cldr, "~> 2.45", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:ex_cldr_currencies, "~> 2.17", [hex: :ex_cldr_currencies, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "3a0d87ef2747c66d78ae8967023d415d3d76aa310981047ad601e3287e8fe73c"}, + "ex_doc": {:hex, :ex_doc, "0.35.1", "de804c590d3df2d9d5b8aec77d758b00c814b356119b3d4455e4b8a8687aecaf", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "2121c6402c8d44b05622677b761371a759143b958c6c19f6558ff64d0aed40df"}, + "ex_json_logger": {:hex, :ex_json_logger, "1.4.0", "ad1dcc1cfe6940ee1d9d489b20757c89769626ce34c4957548d6fbe155cd96f1", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "7548a1ecba290746e06214d2b3d8783c76760c779a8903a8e44bfd23a7340444"}, + "ex_json_schema": {:hex, :ex_json_schema, "0.11.1", "b593f92937a095f66054bb318681397dfe7304e7d2b6b1a7534ea3aa40024f8c", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "32d651a575a6ce2fd613f140b0fef8dd0acc7cf8e8bcd29a3a1be5c945700dd5"}, + "ex_machina": {:hex, :ex_machina, "2.7.0", "b792cc3127fd0680fecdb6299235b4727a4944a09ff0fa904cc639272cd92dc7", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "419aa7a39bde11894c87a615c4ecaa52d8f107bbdd81d810465186f783245bf8"}, + "ex_money": {:hex, :ex_money, "5.24.2", "131eda8a404b654dbf787ca71a8b782e11112e64726d85c27540856f94d4aee5", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ex_cldr, "~> 2.46", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:ex_cldr_numbers, "~> 2.38", [hex: :ex_cldr_numbers, repo: "hexpm", optional: false]}, {:ex_cldr_units, "~> 3.19", [hex: :ex_cldr_units, repo: "hexpm", optional: true]}, {:gringotts, "~> 1.1", [hex: :gringotts, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.0 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm", "707bf02c99537d6cd93e5e5fd7f4faedde3ea2a7a932032b48e5ba0173307920"}, + "excoveralls": {:hex, :excoveralls, "0.18.0", "b92497e69465dc51bc37a6422226ee690ab437e4c06877e836f1c18daeb35da9", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "1109bb911f3cb583401760be49c02cbbd16aed66ea9509fc5479335d284da60b"}, + "expo": {:hex, :expo, "1.1.1", "4202e1d2ca6e2b3b63e02f69cfe0a404f77702b041d02b58597c00992b601db5", [:mix], [], "hexpm", "5fb308b9cb359ae200b7e23d37c76978673aa1b06e2b3075d814ce12c5811640"}, + "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, + "finch": {:hex, :finch, "0.23.0", "e3f9287ac25a8832f848b144c2b57346aac65b205e2e0629a52adfe6507fd837", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.8", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "80e58d3f936f57e3fdf404f83a3642897ae6d9fb642934e46da4d8fe761b99d5"}, + "fine": {:hex, :fine, "0.1.4", "b19a89c1476c7c57afb5f9314aed5960b5bc95d5277de4cb5ee8e1d1616ce379", [:mix], [], "hexpm", "be3324cc454a42d80951cf6023b9954e9ff27c6daa255483b3e8d608670303f5"}, + "fun_with_flags": {:hex, :fun_with_flags, "1.13.0", "8e8eddd6b723691211387923b160bc779fa593f59485209a058c90ea4bbb1cd5", [:mix], [{:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: true]}, {:redix, "~> 1.0", [hex: :redix, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dbbbff607e6f096d3857c88530d1a4e178d9eeec5dc00118e2cc1d076578710a"}, + "fun_with_flags_ui": {:hex, :fun_with_flags_ui, "1.1.0", "61a394cd7d80c09943a4b02e07c9ab8899781f3392a157b45527d22731bf871a", [:mix], [{:cowboy, ">= 2.0.0", [hex: :cowboy, repo: "hexpm", optional: true]}, {:fun_with_flags, "~> 1.12", [hex: :fun_with_flags, repo: "hexpm", optional: false]}, {:plug, "~> 1.12", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 2.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "99b578a250141537ff7774abb832427f3406a4faff3cfc4f074788beeef38f5d"}, + "gen_cycle": {:hex, :gen_cycle, "1.0.4", "2315199f1256116328fad6a9d4371c2fb0bc39c7f5111b88d2de582300c931cc", [:rebar3], [], "hexpm", "67817e31b352bb00715f80a5571b3c069e26e994b2ebafa376acf76a2d0f66d8"}, + "gettext": {:hex, :gettext, "0.26.2", "5978aa7b21fada6deabf1f6341ddba50bc69c999e812211903b169799208f2a8", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "aa978504bcf76511efdc22d580ba08e2279caab1066b76bb9aa81c4a1e0a32a5"}, + "gproc": {:hex, :gproc, "0.9.1", "f1df0364423539cf0b80e8201c8b1839e229e5f9b3ccb944c5834626998f5b8c", [:rebar3], [], "hexpm", "905088e32e72127ed9466f0bac0d8e65704ca5e73ee5a62cb073c3117916d507"}, + "grpcbox": {:hex, :grpcbox, "0.17.1", "6e040ab3ef16fe699ffb513b0ef8e2e896da7b18931a1ef817143037c454bcce", [:rebar3], [{:acceptor_pool, "~> 1.0.0", [hex: :acceptor_pool, repo: "hexpm", optional: false]}, {:chatterbox, "~> 0.15.1", [hex: :ts_chatterbox, repo: "hexpm", optional: false]}, {:ctx, "~> 0.6.0", [hex: :ctx, repo: "hexpm", optional: false]}, {:gproc, "~> 0.9.1", [hex: :gproc, repo: "hexpm", optional: false]}], "hexpm", "4a3b5d7111daabc569dc9cbd9b202a3237d81c80bf97212fbc676832cb0ceb17"}, + "hackney": {:hex, :hackney, "1.25.0", "390e9b83f31e5b325b9f43b76e1a785cbdb69b5b6cd4e079aa67835ded046867", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "7209bfd75fd1f42467211ff8f59ea74d6f2a9e81cbcee95a56711ee79fd6b1d4"}, + "happy_tcp": {:git, "https://github.com/ruslandoga/happy_tcp.git", "d639436bf8e7b456abc86bac13b722a2b6d60725", []}, + "heroicons": {:hex, :heroicons, "0.5.6", "95d730e7179c633df32d95c1fdaaecdf81b0da11010b89b737b843ac176a7eb5", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:phoenix_live_view, ">= 0.18.2", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}], "hexpm", "ca267f02a5fa695a4178a737b649fb6644a2e399639d4ba7964c18e8a58c2352"}, + "hpack": {:hex, :hpack_erl, "0.3.0", "2461899cc4ab6a0ef8e970c1661c5fc6a52d3c25580bc6dd204f84ce94669926", [:rebar3], [], "hexpm", "d6137d7079169d8c485c6962dfe261af5b9ef60fbc557344511c1e65e3d95fb0"}, + "hpax": {:hex, :hpax, "1.0.4", "777de5d433b0fbdc7c418159c8055910faa8047ffdb3d6b31098d2a46cd7685c", [:mix], [], "hexpm", "afc7cb142ebcc2d01ce7816190b98ce5dd49e799111b24249f3443d730f377ca"}, + "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, + "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, + "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, + "joken": {:hex, :joken, "2.6.2", "5daaf82259ca603af4f0b065475099ada1b2b849ff140ccd37f4b6828ca6892a", [:mix], [{:jose, "~> 1.11.10", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "5134b5b0a6e37494e46dbf9e4dad53808e5e787904b7c73972651b51cce3d72b"}, + "jose": {:hex, :jose, "1.11.12", "06e62b467b61d3726cbc19e9b5489f7549c37993de846dfb3ee8259f9ed208b3", [:mix, :rebar3], [], "hexpm", "31e92b653e9210b696765cdd885437457de1add2a9011d92f8cf63e4641bab7b"}, + "lazy_html": {:hex, :lazy_html, "0.1.10", "ffe42a0b4e70859cf21a33e12a251e0c76c1dff76391609bd56702a0ef5bc429", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.9.0", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:fine, "~> 0.1.0", [hex: :fine, repo: "hexpm", optional: false]}], "hexpm", "50f67e5faa09d45a99c1ddf3fac004f051997877dc8974c5797bb5ccd8e27058"}, + "libcluster": {:hex, :libcluster, "3.5.0", "5ee4cfde4bdf32b2fef271e33ce3241e89509f4344f6c6a8d4069937484866ba", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ebf6561fcedd765a4cd43b4b8c04b1c87f4177b5fb3cbdfe40a780499d72f743"}, + "location": {:git, "https://github.com/plausible/location.git", "a89bf79985c3c3d0830477ae587001156a646ce8", []}, + "locus": {:hex, :locus, "2.3.12", "aa3992023363a523e5c6d0d104da1cd1026626d16b32a480903083201057f281", [:rebar3], [{:tls_certificate_check, "~> 1.9", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "e9bacd644127c5aaac478461e8b4bcd49479e7bb9f665b48ab4ebac2b7adc1ec"}, + "logger_backends": {:hex, :logger_backends, "1.0.1", "08a9bd09eed271eac41ff35dd7603b6abf9e0f8b6ffe40359846cb0bfd37704f", [:mix], [], "hexpm", "d7903244fe8d84eeb3d3d42f8d7de080fd0409b613040df54c78b3591672d652"}, + "mail": {:hex, :mail, "0.4.4", "2ed92e3b2dedb2013ba8eef9fb7ace3892e0eae1f45c0658738a72c195f7649b", [:mix], [], "hexpm", "bd44bf3e253d8be9c7f2e59b3253aff1efc1c9fa7d8ab4430c96780683faa8e2"}, + "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, + "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.0.1", "c7f58c120b2b5aa5fd80d540a89fdf866ed42f1f3994e4fe189abebeab610839", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "8a89a1eeccc2d798d6ea15496a6e4870b75e014d1af514b1b71fa33134f57814"}, + "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, + "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, + "mimerl": {:hex, :mimerl, "1.4.0", "3882a5ca67fbbe7117ba8947f27643557adec38fa2307490c4c4207624cb213b", [:rebar3], [], "hexpm", "13af15f9f68c65884ecca3a3891d50a7b57d82152792f3e19d88650aa126b144"}, + "mint": {:hex, :mint, "1.9.1", "3bc120b743ed2e99ad920910f2613e9faebabb2257731b0e2ea4d8ccd9eceede", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "831101bd560b086316fab5f7adb21a4f3455717d8e4bc8368b052e09aa9163e0"}, + "mjml": {:hex, :mjml, "4.0.0", "3418c1a975112766f92b8786a17d53640a4a1c8c7dedb621016a08e1e1447d20", [:mix], [{:rustler, ">= 0.0.0", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.7.0", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "0886a411f9a850bc4ac6b878c9c0dbea168669176fde1f6583acf55fa8b628dd"}, + "mjml_eex": {:hex, :mjml_eex, "0.12.0", "ee7e58a0b2f361812c024ddd2dd57ccd4107e486632016fcf707d4d0eaa04d62", [:mix], [{:erlexec, "~> 2.0.7", [hex: :erlexec, repo: "hexpm", optional: true]}, {:mjml, "~> 4.0", [hex: :mjml, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.2 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6ca41f6252669b709d31d8b7c74db00c8bc3784dea1913b406a68c12763c5bcb"}, + "mox": {:hex, :mox, "1.2.0", "a2cd96b4b80a3883e3100a221e8adc1b98e4c3a332a8fc434c39526babafd5b3", [:mix], [{:nimble_ownership, "~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}], "hexpm", "c7b92b3cc69ee24a7eeeaf944cd7be22013c52fcb580c1f33f50845ec821089a"}, + "mua": {:hex, :mua, "0.2.5", "e99aa9646964a0109a2efcc8e684c6f8d90c60fb0191f52e1784cea296584daf", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "0e2b18024d0db8943a68e84fb5e2253d3225c8f61d8387cbfc581d66e34d8493"}, + "nanoid": {:hex, :nanoid, "2.1.0", "d192a5bf1d774258bc49762b480fca0e3128178fa6d35a464af2a738526607fd", [:mix], [], "hexpm", "ebc7a342d02d213534a7f93a091d569b9fea7f26fcd3a638dc655060fc1f76ac"}, + "nimble_csv": {:hex, :nimble_csv, "1.3.0", "b7f998dc62b222bce9596e46f028c7a5af04cb5dde6df2ea197c583227c54971", [:mix], [], "hexpm", "41ccdc18f7c8f8bb06e84164fc51635321e80d5a3b450761c4997d620925d619"}, + "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, + "nimble_ownership": {:hex, :nimble_ownership, "1.0.2", "fa8a6f2d8c592ad4d79b2ca617473c6aefd5869abfa02563a77682038bf916cf", [:mix], [], "hexpm", "098af64e1f6f8609c6672127cfe9e9590a5d3fcdd82bc17a377b8692fd81a879"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, + "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, + "nimble_totp": {:hex, :nimble_totp, "1.0.0", "79753bae6ce59fd7cacdb21501a1dbac249e53a51c4cd22b34fa8438ee067283", [:mix], [], "hexpm", "6ce5e4c068feecdb782e85b18237f86f66541523e6bad123e02ee1adbe48eda9"}, + "oban": {:hex, :oban, "2.20.1", "39d0b68787e5cf251541c0d657a698f6142a24d8744e1e40b2cf045d4fa232a6", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.20", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "17a45277dbeb41a455040b41dd8c467163fad685d1366f2f59207def3bcdd1d8"}, + "observer_cli": {:hex, :observer_cli, "1.7.4", "3c1bfb6d91bf68f6a3d15f46ae20da0f7740d363ee5bc041191ce8722a6c4fae", [:mix, :rebar3], [{:recon, "~> 2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "50de6d95d814f447458bd5d72666a74624eddb0ef98bdcee61a0153aae0865ff"}, + "octo_fetch": {:hex, :octo_fetch, "0.4.0", "074b5ecbc08be10b05b27e9db08bc20a3060142769436242702931c418695b19", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "cf8be6f40cd519d7000bb4e84adcf661c32e59369ca2827c4e20042eda7a7fc6"}, + "odgn_json_pointer": {:hex, :odgn_json_pointer, "3.1.0", "be81198ad8c9bce01fc872d6400d88a4a55ce711f7c8838e1166bd21f736d29a", [:mix], [], "hexpm", "85730306bb2e5796c993813f35e9bafeb561526f5e80a517793c33d1635ade94"}, + "open_api_spex": {:hex, :open_api_spex, "3.22.1", "38c99d8bf107dc7ffb112dc669f33e0a287396c6c9fd6bb7eeb27e3fe8dbba0e", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "fa51ecd04ececbad89a8ede55ebd9db7aa9e55cc7ddbb46455522e0f3c098290"}, + "opentelemetry": {:hex, :opentelemetry, "1.7.0", "20d0f12d3d1c398d3670fd44fd1a7c495dd748ab3e5b692a7906662e2fb1a38a", [:rebar3], [{:opentelemetry_api, "~> 1.5.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "a9173b058c4549bf824cbc2f1d2fa2adc5cdedc22aa3f0f826951187bbd53131"}, + "opentelemetry_api": {:hex, :opentelemetry_api, "1.5.0", "1a676f3e3340cab81c763e939a42e11a70c22863f645aa06aafefc689b5550cf", [:mix, :rebar3], [], "hexpm", "f53ec8a1337ae4a487d43ac89da4bd3a3c99ddf576655d071deed8b56a2d5dda"}, + "opentelemetry_api_experimental": {:git, "https://github.com/open-telemetry/opentelemetry-erlang.git", "b241ebf4feb7558738d009583bae8602b3c49fd0", [ref: "b241ebf4feb7558738d009583bae8602b3c49fd0", sparse: "apps/opentelemetry_api_experimental"]}, + "opentelemetry_cowboy": {:hex, :opentelemetry_cowboy, "1.0.0", "786c7cde66a2493323c79d2c94e679ff501d459a9b403d8b60b9bef116333117", [:rebar3], [{:cowboy_telemetry, "~> 0.4", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.4", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:opentelemetry_telemetry, "~> 1.1", [hex: :opentelemetry_telemetry, repo: "hexpm", optional: false]}, {:otel_http, "~> 0.2", [hex: :otel_http, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.1", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7575716eaccacd0eddc3e7e61403aecb5d0a6397183987d6049094aeb0b87a7c"}, + "opentelemetry_ecto": {:hex, :opentelemetry_ecto, "1.2.0", "2382cb47ddc231f953d3b8263ed029d87fbf217915a1da82f49159d122b64865", [:mix], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.2", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "70dfa2e79932e86f209df00e36c980b17a32f82d175f0068bf7ef9a96cf080cf"}, + "opentelemetry_experimental": {:git, "https://github.com/open-telemetry/opentelemetry-erlang.git", "b241ebf4feb7558738d009583bae8602b3c49fd0", [ref: "b241ebf4feb7558738d009583bae8602b3c49fd0", sparse: "apps/opentelemetry_experimental"]}, + "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.10.0", "972e142392dbfa679ec959914664adefea38399e4f56ceba5c473e1cabdbad79", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.7.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.5.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "33a116ed7304cb91783f779dec02478f887c87988077bfd72840f760b8d4b952"}, + "opentelemetry_oban": {:hex, :opentelemetry_oban, "1.1.1", "519e9ba60d3dc3483ad2df3fade131d47056e0dae74f0724c8a40b9718f089d1", [:mix], [{:oban, "~> 2.0", [hex: :oban, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.2", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:opentelemetry_telemetry, "~> 1.1", [hex: :opentelemetry_telemetry, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ae6aed431626a94a4bb6bf5b268247ced687ec8f99eced6887e3754f9d3a2089"}, + "opentelemetry_phoenix": {:hex, :opentelemetry_phoenix, "2.0.1", "c664cdef205738cffcd409b33599439a4ffb2035ef6e21a77927ac1da90463cb", [:mix], [{:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.4", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.3", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:opentelemetry_telemetry, "~> 1.1", [hex: :opentelemetry_telemetry, repo: "hexpm", optional: false]}, {:otel_http, "~> 0.2", [hex: :otel_http, repo: "hexpm", optional: false]}, {:plug, ">= 1.11.0", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a24fdccdfa6b890c8892c6366beab4a15a27ec0c692b0f77ec2a862e7b235f6e"}, + "opentelemetry_process_propagator": {:hex, :opentelemetry_process_propagator, "0.3.0", "ef5b2059403a1e2b2d2c65914e6962e56371570b8c3ab5323d7a8d3444fb7f84", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "7243cb6de1523c473cba5b1aefa3f85e1ff8cc75d08f367104c1e11919c8c029"}, + "opentelemetry_req": {:hex, :opentelemetry_req, "1.0.0", "f8ef479a05311bdc030925a07be869ede45e8158cc2c80d9855e35a7d1efc89c", [:mix], [{:jason, "~> 1.3", [hex: :jason, repo: "hexpm", optional: false]}, {:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.4", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 1.27", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}, {:otel_http, "~> 0.2", [hex: :otel_http, repo: "hexpm", optional: false]}, {:req, ">= 0.3.5", [hex: :req, repo: "hexpm", optional: false]}], "hexpm", "6404ae802d694f7c3e5b43bf41af99364ecf20c3f5fea144fff8b7f873b22bf7"}, + "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "1.27.0", "acd0194a94a1e57d63da982ee9f4a9f88834ae0b31b0bd850815fe9be4bbb45f", [:mix, :rebar3], [], "hexpm", "9681ccaa24fd3d810b4461581717661fd85ff7019b082c2dff89c7d5b1fc2864"}, + "opentelemetry_telemetry": {:hex, :opentelemetry_telemetry, "1.1.2", "410ab4d76b0921f42dbccbe5a7c831b8125282850be649ee1f70050d3961118a", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.3", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.1", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "641ab469deb181957ac6d59bce6e1321d5fe2a56df444fc9c19afcad623ab253"}, + "otel_http": {:hex, :otel_http, "0.2.0", "b17385986c7f1b862f5d577f72614ecaa29de40392b7618869999326b9a61d8a", [:rebar3], [], "hexpm", "f2beadf922c8cfeb0965488dd736c95cc6ea8b9efce89466b3904d317d7cc717"}, + "paginator": {:git, "https://github.com/duffelhq/paginator.git", "3508d6ad77a95ac1faf15d5fd7f959fab3e17da2", []}, + "parent": {:hex, :parent, "0.12.1", "495c4386f06de0df492e0a7a7199c10323a55e9e933b27222060dd86dccd6d62", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2ab589ef1f37bfcedbfb5ecfbab93354972fb7391201b8907a866dadd20b39d1"}, + "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, + "peep": {:hex, :peep, "3.4.2", "49d4ca116d994779351959dfee971fb2c7d6506f1821374f3cc4fd39a3d3fadb", [:mix], [{:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:plug, "~> 1.16", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "cae224b09e224bf5584f5375108becf71e2288572a099a122e66735289cd33f4"}, + "phoenix": {:hex, :phoenix, "1.8.9", "a63ed0962ed5b903b146dab0ae8eb8387fe478f8171a5e26d56a165f35996fe1", [:mix], [{:bandit, "~> 1.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "3477e2dd5a4f61820341169031bdfe21275f659923bea9c5c0ea2aa1c3fcc046"}, + "phoenix_bakery": {:hex, :phoenix_bakery, "0.1.2", "ca57673caea1a98f1cc763f94032796a015774d27eaa3ce5feef172195470452", [:mix], [{:brotli, "~> 0.3.0", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "45cc8cecc5c3002b922447c16389761718c07c360432328b04680034e893ea5b"}, + "phoenix_ecto": {:hex, :phoenix_ecto, "4.7.0", "75c4b9dfb3efdc42aec2bd5f8bccd978aca0651dbcbc7a3f362ea5d9d43153c6", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "1d75011e4254cb4ddf823e81823a9629559a1be93b4321a6a5f11a5306fbf4cc"}, + "phoenix_html": {:hex, :phoenix_html, "4.3.0", "d3577a5df4b6954cd7890c84d955c470b5310bb49647f0a114a6eeecc850f7ad", [:mix], [], "hexpm", "3eaa290a78bab0f075f791a46a981bbe769d94bc776869f4f3063a14f30497ad"}, + "phoenix_html_helpers": {:hex, :phoenix_html_helpers, "1.0.1", "7eed85c52eff80a179391036931791ee5d2f713d76a81d0d2c6ebafe1e11e5ec", [:mix], [{:phoenix_html, "~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cffd2385d1fa4f78b04432df69ab8da63dc5cf63e07b713a4dcf36a3740e3090"}, + "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.6.1", "05df733a09887a005ed0d69a7fc619d376aea2730bf64ce52ac51ce716cc1ef0", [:mix], [{:file_system, "~> 0.2.10 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "74273843d5a6e4fef0bbc17599f33e3ec63f08e69215623a0cd91eea4288e5a0"}, + "phoenix_live_view": {:hex, :phoenix_live_view, "1.1.27", "9afcab28b0c82afdc51044e661bcd5b8de53d242593d34c964a37710b40a42af", [:mix], [{:igniter, ">= 0.6.16 and < 1.0.0-0", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:lazy_html, "~> 0.1.0", [hex: :lazy_html, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0 or ~> 1.8.0-rc", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "415735d0b2c612c9104108b35654e977626a0cb346711e1e4f1ed16e3c827ede"}, + "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.2.0", "ff3a5616e1bed6804de7773b92cbccfc0b0f473faf1f63d7daf1206c7aeaaa6f", [:mix], [], "hexpm", "adc313a5bf7136039f63cfd9668fde73bba0765e0614cba80c06ac9460ff3e96"}, + "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, + "phoenix_view": {:hex, :phoenix_view, "2.0.4", "b45c9d9cf15b3a1af5fb555c674b525391b6a1fe975f040fb4d913397b31abf4", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "4e992022ce14f31fe57335db27a28154afcc94e9983266835bb3040243eb620b"}, + "php_serializer": {:hex, :php_serializer, "2.0.0", "b43f31aca22ed7321f32da2b94fe2ddf9b6739a965cb51541969119e572e821d", [:mix], [], "hexpm", "61e402e99d9062c0225a3f4fcf7e43b4cba1b8654944c0e7c139c3ca9de481da"}, + "plug": {:hex, :plug, "1.20.2", "adbee2441232412e37fbb357fd5e4cd533fdd253b29f2e1992262b0f1fb01462", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b16baf55877d60891002ffc1ce0b3ff7d6f30a38a23e02e4d4293c4ac266f136"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.8.1", "5aa391a5e8d1ac3192e36a3bcaff12b5fd6ef6c7e29b53a38e63a860783e77d0", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "4c200288673d5bc86a0ab7dc6a2a069176a74e5d573ef62740a1c517458a5f26"}, + "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"}, + "polymorphic_embed": {:hex, :polymorphic_embed, "5.0.3", "37444e0af941026a2c29b0539b6471bdd6737a6492a19264bf2bb0118e3ac242", [:mix], [{:attrs, "~> 0.6", [hex: :attrs, repo: "hexpm", optional: false]}, {:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_html_helpers, "~> 1.0", [hex: :phoenix_html_helpers, repo: "hexpm", optional: true]}, {:phoenix_live_view, "~> 0.20 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}], "hexpm", "2fed44f57abf0a0fc7642e0eb0807a55b65de1562712cc0620772cbbb80e49c1"}, + "postgrex": {:hex, :postgrex, "0.22.2", "4aec14df2a72722aee92492566edbeeb44e233ecb86b1915d03136297ef1385d", [:mix], [{:db_connection, "~> 2.9", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "8946382ddb06294f56026ac4278b3cc212bac8a2c82ed68b4087819ed1abc53b"}, + "prima": {:hex, :prima, "0.2.6", "a40993d108760a711c2da183cff20234497e977f6155aad1ee5e17a3b00a46b6", [:mix], [{:esbuild, "~> 0.7", [hex: :esbuild, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 4.2", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 1.1", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}], "hexpm", "00ec378bd7b0c58d58043e59581f9aa5ba35a0d5031e011e36c449d3a0b7418c"}, + "prom_ex": {:hex, :prom_ex, "1.11.0", "1f6d67f2dead92224cb4f59beb3e4d319257c5728d9638b4a5e8ceb51a4f9c7e", [:mix], [{:absinthe, ">= 1.7.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.1.0", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.11.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.18", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.10.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.4", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:peep, "~> 3.0", [hex: :peep, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.7.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.20.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.16.0", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, ">= 2.6.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.2", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.1", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "76b074bc3730f0802978a7eb5c7091a65473eaaf07e99ec9e933138dcc327805"}, + "public_suffix": {:git, "https://github.com/axelson/publicsuffix-elixir", "fa40c243d4b5d8598b90cff268bc4e33f3bb63f1", []}, + "ranch": {:hex, :ranch, "1.8.1", "208169e65292ac5d333d6cdbad49388c1ae198136e4697ae2f474697140f201c", [:make, :rebar3], [], "hexpm", "aed58910f4e21deea992a67bf51632b6d60114895eb03bb392bb733064594dd0"}, + "recon": {:hex, :recon, "2.5.4", "05dd52a119ee4059fa9daa1ab7ce81bc7a8161a2f12e9d42e9d551ffd2ba901c", [:mix, :rebar3], [], "hexpm", "e9ab01ac7fc8572e41eb59385efeb3fb0ff5bf02103816535bacaedf327d0263"}, + "ref_inspector": {:hex, :ref_inspector, "2.0.0", "f3e97e51d9782de4c792f56eed26c80903bc39174c878285392ce76d5e67fe98", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:yamerl, "~> 0.7", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "bf62f3f1a87d6b8b30f457a480668f965373e64f184611282b5e89d8dd81fd33"}, + "referrer_blocklist": {:git, "https://github.com/plausible/referrer-blocklist.git", "d6f52c225cccb4f04b80e3a5d588868ec234139d", []}, + "req": {:hex, :req, "0.6.2", "b9b2024f35bcf60a92cc8cad2eaaf9d4e7aace463ff74be1afe5986830184413", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.21", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cc9cd30a2ddd04989929b887178e1610c940456d962c6c3a52df6146d2eef9bf"}, + "rustler_precompiled": {:hex, :rustler_precompiled, "0.7.3", "42cb9449785cd86c87453e39afdd27a0bdfa5c77a4ec5dc5ce45112e06b9f89b", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "cbc4b3777682e5f6f43ed39b0e0b4a42dccde8053aba91b4514e8f5ff9a5ac6d"}, + "saxy": {:hex, :saxy, "1.6.0", "02cb4e9bd045f25ac0c70fae8164754878327ee393c338a090288210b02317ee", [:mix], [], "hexpm", "ef42eb4ac983ca77d650fbdb68368b26570f6cc5895f0faa04d34a6f384abad3"}, + "sentry": {:hex, :sentry, "11.0.4", "60371c96cefd247e0fc98840bba2648f64f19aa0b8db8e938f5a98421f55b619", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_ownership, "~> 0.3.0 or ~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}, {:opentelemetry, ">= 0.0.0", [hex: :opentelemetry, repo: "hexpm", optional: true]}, {:opentelemetry_api, ">= 0.0.0", [hex: :opentelemetry_api, repo: "hexpm", optional: true]}, {:opentelemetry_exporter, ">= 0.0.0", [hex: :opentelemetry_exporter, repo: "hexpm", optional: true]}, {:opentelemetry_semantic_conventions, ">= 0.0.0", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, "~> 0.20 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "feaafc284dc204c82aadaddc884227aeaa3480decb274d30e184b9d41a700c66"}, + "simple_saml": {:hex, :simple_saml, "1.2.0", "d875939e21cd73cdb3f747e701e4d2d5f0f4e35610d37df50629087454fa43f9", [:mix], [{:simple_xml, "~> 1.3", [hex: :simple_xml, repo: "hexpm", optional: false]}, {:x509, "~> 0.9.0", [hex: :x509, repo: "hexpm", optional: false]}], "hexpm", "24663665e53a91bfc38359ed9f26f575a389bde9fc94ec369f9f2259046feaf0"}, + "simple_xml": {:hex, :simple_xml, "1.3.2", "84b892527b5183748694583a658c729c84bb89fc8ac650d7025ebc2c49e74018", [:mix], [{:saxy, "~> 1.5", [hex: :saxy, repo: "hexpm", optional: false]}, {:x509, "~> 0.9.0", [hex: :x509, repo: "hexpm", optional: false]}], "hexpm", "8c15fa49955a8c82b50f3053617e8a4cd771fcd0a502d660e56993c38ac86ca5"}, + "siphash": {:hex, :siphash, "3.2.0", "ec03fd4066259218c85e2a4b8eec4bb9663bc02b127ea8a0836db376ba73f2ed", [:make, :mix], [], "hexpm", "ba3810701c6e95637a745e186e8a4899087c3b079ba88fb8f33df054c3b0b7c3"}, + "site_encrypt": {:git, "https://github.com/sasa1977/site_encrypt.git", "8d47801c51eeea11ac385cb73eef4ed9a6565367", []}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, + "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, + "tailwind": {:hex, :tailwind, "0.4.1", "e7bcc222fe96a1e55f948e76d13dd84a1a7653fb051d2a167135db3b4b08d3e9", [:mix], [], "hexpm", "6249d4f9819052911120dbdbe9e532e6bd64ea23476056adb7f730aa25c220d1"}, + "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, + "telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"}, + "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.2.1", "c9755987d7b959b557084e6990990cb96a50d6482c683fb9622a63837f3cd3d8", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "5e2c599da4983c4f88a33e9571f1458bf98b0cf6ba930f1dc3a6e8cf45d5afb6"}, + "telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"}, + "timex": {:hex, :timex, "3.7.13", "0688ce11950f5b65e154e42b47bf67b15d3bc0e0c3def62199991b8a8079a1e2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.26", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "09588e0522669328e973b8b4fd8741246321b3f0d32735b589f78b136e6d4c54"}, + "tls_certificate_check": {:hex, :tls_certificate_check, "1.30.0", "ef9bdfcb5b551b747cad231a65ebd449623628bb72471c1d2aefcbbc5730683d", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "dfbada8fddb80ae19b8aeaac34d95cfce98c91393b0db9c3ee079b488fd68d81"}, + "tzdata": {:hex, :tzdata, "1.1.3", "b1cef7bb6de1de90d4ddc25d33892b32830f907e7fc2fccd1e7e22778ab7dfbc", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "d4ca85575a064d29d4e94253ee95912edfb165938743dbf002acdf0dcecb0c28"}, + "ua_inspector": {:hex, :ua_inspector, "3.12.0", "c8c5cfb0ce19b5cd609efb577bf75e3ed60deef8ed26bc0577aa2ff60ea49bcd", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:yamerl, "~> 0.7", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "81fa0e28effb50ae9fd74ffe51622af3846149aee32c11d60b16ceab62b45682"}, + "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"}, + "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, + "websock_adapter": {:hex, :websock_adapter, "0.5.9", "43dc3ba6d89ef5dec5b1d0a39698436a1e856d000d84bf31a3149862b01a287f", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "5534d5c9adad3c18a0f58a9371220d75a803bf0b9a3d87e6fe072faaeed76a08"}, + "x509": {:hex, :x509, "0.9.2", "a75aa605348abd905990f3d2dc1b155fcde4e030fa2f90c4a91534405dce0f6e", [:mix], [], "hexpm", "4c5ede75697e565d4b0f5be04c3b71bb1fd3a090ea243af4bd7dae144e48cfc7"}, + "xml_builder": {:hex, :xml_builder, "2.4.0", "b20d23077266c81f593360dc037ea398461dddb6638a329743da6c73afa56725", [:mix], [], "hexpm", "833e325bb997f032b5a1b740d2fd6feed3c18ca74627f9f5f30513a9ae1a232d"}, + "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, + "zstream": {:hex, :zstream, "0.6.7", "ae74c7544f4e0563ffbe71324bf1c9bbc0ab33bb580a0ae718da511fb8a5c9d6", [:mix], [], "hexpm", "48c43ae0f00cfcda1ccb69c1d044755663d43b2ee8a0a65763648bf2078d634d"}, + "zxcvbn": {:git, "https://github.com/techgaun/zxcvbn-elixir.git", "5c9684590f4ecf00af2d391f90db9ce93bd4e996", []}, +} diff --git a/priv/custom_sources.json b/priv/custom_sources.json new file mode 100644 index 0000000000000000000000000000000000000000..c73c7a9dc35178153a74c4c18b4d40f19b69bc44 --- /dev/null +++ b/priv/custom_sources.json @@ -0,0 +1,84 @@ +{ + "bsky.app":"Bluesky", + "go.bsky.app":"Bluesky", + "android-app://com.reddit.frontpage":"Reddit", + "baidu.com":"Baidu", + "chatgpt.com":"ChatGPT", + "chat.openai.com":"ChatGPT", + "claude.ai":"Claude", + "copilot.microsoft.com":"Microsoft Copilot", + "copilot.com":"Microsoft Copilot", + "copilot":"Microsoft Copilot", + "deepseek.com":"DeepSeek", + "gemini.google.com":"Google Gemini", + "grok.com":"Grok", + "x.ai":"Grok", + "phind.com":"Phind", + "chaos.social":"Mastodon", + "fosstodon.org":"Mastodon", + "hachyderm.io":"Mastodon", + "indieweb.social":"Mastodon", + "infosec.exchange":"Mastodon", + "mas.to":"Mastodon", + "mastodon.online":"Mastodon", + "mastodon.social":"Mastodon", + "mastodon.world":"Mastodon", + "mathstodon.xyz":"Mastodon", + "mstdn.jp":"Mastodon", + "mstdn.social":"Mastodon", + "scholar.social":"Mastodon", + "sigmoid.social":"Mastodon", + "social.tchncs.de":"Mastodon", + "android-app://org.joinmastodon.android": "Mastodon", + "discord.com":"Discord", + "discordapp.com":"Discord", + "linktr.ee":"Linktree", + "m.sogou.com":"Sogou", + "kagi.com":"Kagi", + "l.threads.com":"Threads", + "ntp.msn.com":"Bing", + "perplexity.ai":"Perplexity", + "pplx.ai":"Perplexity", + "ptb.discord.com":"Discord", + "search.brave.com":"Brave", + "sogou.com":"Sogou", + "statics.teams.cdn.office.net":"Microsoft Teams", + "teams.microsoft.com":"Microsoft Teams", + "t.me":"Telegram", + "wap.sogou.com":"Sogou", + "ya.ru":"Yandex", + "yandex.com.tr":"Yandex", + "yandex.eu":"Yandex", + "yandex.fr":"Yandex", + "yandex.kz":"Yandex", + "yandex.tm":"Yandex", + "yandex.uz":"Yandex", + "fb": "Facebook", + "fb-ads": "Facebook", + "fbads": "Facebook", + "fbad": "Facebook", + "facebook-ads": "Facebook", + "facebook_ads": "Facebook", + "fcb": "Facebook", + "facebook_ad": "Facebook", + "facebook_feed_ad": "Facebook", + "ig": "Instagram", + "yt": "Youtube", + "yt-ads": "Youtube", + "reddit-ads": "Reddit", + "google_ads": "Google", + "google-ads": "Google", + "googleads": "Google", + "gads": "Google", + "google ads": "Google", + "adwords": "Google", + "x.com":"X (Twitter)", + "mobile.twitter.com": "X (Twitter)", + "twitter-ads": "X (Twitter)", + "twitter": "X (Twitter)", + "tiktokads": "TikTok", + "tik.tok": "TikTok", + "wikipedia": "Wikipedia", + "canary.discord.com":"Discord", + "webk.telegram.org":"Telegram" +} diff --git a/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_add_materialized_column.sql.eex b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_add_materialized_column.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..47688db5fd2742d7b745e7b6c7caa46e2cc2216d --- /dev/null +++ b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_add_materialized_column.sql.eex @@ -0,0 +1,4 @@ +ALTER TABLE <%= @table %> +<%= @on_cluster_statement %> +ADD COLUMN IF NOT EXISTS acquisition_channel LowCardinality(String) +MATERIALIZED acquisition_channel(referrer_source, utm_medium, utm_campaign, utm_source, click_id_param) diff --git a/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_backfill_materialized_column.sql.eex b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_backfill_materialized_column.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..2b087cdc9ee8fd8bf99c6efeae87932b92d9ee41 --- /dev/null +++ b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_backfill_materialized_column.sql.eex @@ -0,0 +1,2 @@ +ALTER TABLE <%= @table %> +MATERIALIZE COLUMN acquisition_channel diff --git a/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..4de5145210d276cb3fb07c2c411f09350ca93b4b --- /dev/null +++ b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex @@ -0,0 +1,235 @@ +CREATE TABLE IF NOT EXISTS acquisition_channel_source_category +<%= @on_cluster_statement %> +( + referrer_source String, + category LowCardinality(String) +) +<%= if @on_cluster_statement != "" do %> +ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_prod/acquisition_channel_source_category', '{replica}') +<% else %> +ENGINE = MergeTree() +<% end %> +ORDER BY referrer_source +<%= @table_settings %>; + +TRUNCATE TABLE acquisition_channel_source_category SETTINGS alter_sync=2; + +INSERT INTO acquisition_channel_source_category(referrer_source, category) +SELECT t.1 AS referrer_source, t.2 AS category +FROM ( + SELECT arrayJoin({source_categories:Array(Tuple(String, String))}) AS t +) +SETTINGS insert_quorum = <%= @insert_quorum %>; + +CREATE OR REPLACE DICTIONARY acquisition_channel_source_category_dict +<%= @on_cluster_statement %> +( + `referrer_source` String, + `category` String +) +PRIMARY KEY referrer_source +SOURCE(CLICKHOUSE(TABLE acquisition_channel_source_category <%= @dictionary_connection_params %>)) +LIFETIME(0) +LAYOUT(hashed()); + +CREATE TABLE IF NOT EXISTS acquisition_channel_paid_sources +<%= @on_cluster_statement %> +( + referrer_source String +) +<%= if @on_cluster_statement != "" do %> +ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_prod/acquisition_channel_paid_sources', '{replica}') +<% else %> +ENGINE = MergeTree() +<% end %> +ORDER BY referrer_source +<%= @table_settings %>; + +TRUNCATE TABLE acquisition_channel_paid_sources SETTINGS alter_sync=2; + +INSERT INTO acquisition_channel_paid_sources(referrer_source) +SELECT arrayJoin({paid_sources:Array(String)}) AS referrer_source +SETTINGS insert_quorum = <%= @insert_quorum %>; + +CREATE OR REPLACE DICTIONARY acquisition_channel_paid_sources_dict +<%= @on_cluster_statement %> +( + `referrer_source` String +) +PRIMARY KEY referrer_source +SOURCE(CLICKHOUSE(TABLE acquisition_channel_paid_sources <%= @dictionary_connection_params %>)) +LIFETIME(0) +LAYOUT(hashed()); + +CREATE OR REPLACE FUNCTION acquisition_channel_has_category_shopping <%= @on_cluster_statement %> AS +(referrer_source) -> + dictGet('acquisition_channel_source_category_dict', 'category', referrer_source) = 'SOURCE_CATEGORY_SHOPPING'; + +CREATE OR REPLACE FUNCTION acquisition_channel_has_category_social <%= @on_cluster_statement %> AS +(referrer_source) -> + dictGet('acquisition_channel_source_category_dict', 'category', referrer_source) = 'SOURCE_CATEGORY_SOCIAL'; + +CREATE OR REPLACE FUNCTION acquisition_channel_has_category_video <%= @on_cluster_statement %> AS +(referrer_source) -> + dictGet('acquisition_channel_source_category_dict', 'category', referrer_source) = 'SOURCE_CATEGORY_VIDEO'; + +CREATE OR REPLACE FUNCTION acquisition_channel_has_category_search <%= @on_cluster_statement %> AS +(referrer_source) -> + dictGet('acquisition_channel_source_category_dict', 'category', referrer_source) = 'SOURCE_CATEGORY_SEARCH'; + +CREATE OR REPLACE FUNCTION acquisition_channel_has_category_email <%= @on_cluster_statement %> AS +(referrer_source) -> + dictGet('acquisition_channel_source_category_dict', 'category', referrer_source) = 'SOURCE_CATEGORY_EMAIL'; + +CREATE OR REPLACE FUNCTION acquisition_channel_has_category_ai_assistants <%= @on_cluster_statement %> AS +(referrer_source) -> + dictGet('acquisition_channel_source_category_dict', 'category', referrer_source) = 'SOURCE_CATEGORY_AI_ASSISTANTS'; + +CREATE OR REPLACE FUNCTION acquisition_channel_paid_utm_source <%= @on_cluster_statement %> AS +(referrer_source) -> + dictHas('acquisition_channel_paid_sources_dict', referrer_source); + +CREATE OR REPLACE FUNCTION acquisition_channel_cross_network <%= @on_cluster_statement %> AS +(utm_campaign) -> + position(utm_campaign, 'cross-network') > 0; + +CREATE OR REPLACE FUNCTION acquisition_channel_paid_shopping <%= @on_cluster_statement %> AS +(referrer_source, utm_medium, utm_campaign) -> + acquisition_channel_paid_medium(utm_medium) AND + ( + acquisition_channel_has_category_shopping(referrer_source) + OR acquisition_channel_shopping_campaign(utm_campaign) + ); + +CREATE OR REPLACE FUNCTION acquisition_channel_paid_search <%= @on_cluster_statement %> AS +(referrer_source, utm_medium, utm_source, click_id_param) -> + ( + acquisition_channel_has_category_search(referrer_source) + AND ( + acquisition_channel_paid_medium(utm_medium) + OR acquisition_channel_paid_utm_source(utm_source) + ) + ) OR ( + referrer_source == 'google' + AND click_id_param == 'gclid' + ) OR ( + referrer_source == 'bing' + AND click_id_param == 'msclkid' + ); + +CREATE OR REPLACE FUNCTION acquisition_channel_paid_social <%= @on_cluster_statement %> AS +(referrer_source, utm_medium, utm_source) -> + acquisition_channel_has_category_social(referrer_source) + AND ( + acquisition_channel_paid_medium(utm_medium) + OR acquisition_channel_paid_utm_source(utm_source) + ); + +CREATE OR REPLACE FUNCTION acquisition_channel_paid_video <%= @on_cluster_statement %> AS +(referrer_source, utm_medium, utm_source) -> + acquisition_channel_has_category_video(referrer_source) + AND ( + acquisition_channel_paid_medium(utm_medium) + OR acquisition_channel_paid_utm_source(utm_source) + ); + +CREATE OR REPLACE FUNCTION acquisition_channel_display <%= @on_cluster_statement %> AS +(utm_medium) -> + utm_medium IN ('display', 'banner', 'expandable', 'interstitial', 'cpm'); + +CREATE OR REPLACE FUNCTION acquisition_channel_paid_medium <%= @on_cluster_statement %> AS +(utm_medium) -> + match(utm_medium, '^(.*cp.*|ppc|retargeting|paid.*)$'); + +CREATE OR REPLACE FUNCTION acquisition_channel_shopping_campaign <%= @on_cluster_statement %> AS +(utm_campaign) -> + match(utm_campaign, '^(.*(([^a-df-z]|^)shop|shopping).*)$'); + +CREATE OR REPLACE FUNCTION acquisition_channel_organic_shopping <%= @on_cluster_statement %> AS +(referrer_source, utm_campaign) -> + acquisition_channel_has_category_shopping(referrer_source) + OR acquisition_channel_shopping_campaign(utm_campaign); + +CREATE OR REPLACE FUNCTION acquisition_channel_organic_social <%= @on_cluster_statement %> AS +(referrer_source, utm_medium) -> + acquisition_channel_has_category_social(referrer_source) + OR utm_medium IN ( + 'social', + 'social-network', + 'social-media', + 'sm', + 'social network', + 'social media' + ); + +CREATE OR REPLACE FUNCTION acquisition_channel_organic_video <%= @on_cluster_statement %> AS +(referrer_source, utm_medium) -> + acquisition_channel_has_category_video(referrer_source) OR position(utm_medium, 'video') > 0; + +CREATE OR REPLACE FUNCTION acquisition_channel_email <%= @on_cluster_statement %> AS +(referrer_source, utm_source, utm_medium) -> + acquisition_channel_has_category_email(referrer_source) + OR acquisition_channel_contains_email(utm_source) + OR acquisition_channel_contains_email(utm_medium); + +CREATE OR REPLACE FUNCTION acquisition_channel_affiliates <%= @on_cluster_statement %> AS +(utm_medium) -> + utm_medium == 'affiliate'; + +CREATE OR REPLACE FUNCTION acquisition_channel_audio <%= @on_cluster_statement %> AS +(utm_medium) -> + utm_medium == 'audio'; + +CREATE OR REPLACE FUNCTION acquisition_channel_sms <%= @on_cluster_statement %> AS +(column) -> + column == 'sms'; + +CREATE OR REPLACE FUNCTION acquisition_channel_mobile_push_notifications <%= @on_cluster_statement %> AS +(utm_medium, referrer_source) -> + endsWith(utm_medium, 'push') OR + multiSearchAny(utm_medium, ['mobile', 'notification']) OR + referrer_source == 'firebase'; + +CREATE OR REPLACE FUNCTION acquisition_channel_referral <%= @on_cluster_statement %> AS +(utm_medium, referrer_source) -> + utm_medium IN ('referral', 'app', 'link') OR + not empty(referrer_source); + +CREATE OR REPLACE FUNCTION acquisition_channel_contains_email <%= @on_cluster_statement %> AS +(column) -> + match(column, 'e[-_ ]?mail|newsletter'); + +CREATE OR REPLACE FUNCTION acquisition_channel <%= @on_cluster_statement %> AS +(referrer_source, utm_medium, utm_campaign, utm_source, click_id_param) -> + acquisition_channel_lowered( + lower(referrer_source), + lower(utm_medium), + lower(utm_campaign), + lower(utm_source), + click_id_param + ); + +CREATE OR REPLACE FUNCTION acquisition_channel_lowered <%= @on_cluster_statement %> AS +(referrer_source, utm_medium, utm_campaign, utm_source, click_id_param) -> + multiIf( + acquisition_channel_cross_network(utm_campaign), 'Cross-network', + acquisition_channel_display(utm_medium), 'Display', + acquisition_channel_paid_shopping(referrer_source, utm_medium, utm_campaign), 'Paid Shopping', + acquisition_channel_paid_search(referrer_source, utm_medium, utm_source, click_id_param), 'Paid Search', + acquisition_channel_paid_social(referrer_source, utm_medium, utm_source), 'Paid Social', + acquisition_channel_paid_video(referrer_source, utm_medium, utm_source), 'Paid Video', + acquisition_channel_paid_medium(utm_medium), 'Paid Other', + acquisition_channel_organic_shopping(referrer_source, utm_campaign), 'Organic Shopping', + acquisition_channel_organic_social(referrer_source, utm_medium), 'Organic Social', + acquisition_channel_organic_video(referrer_source, utm_medium), 'Organic Video', + acquisition_channel_has_category_ai_assistants(referrer_source), 'AI Assistants', + acquisition_channel_has_category_search(referrer_source), 'Organic Search', + acquisition_channel_email(referrer_source, utm_source, utm_medium), 'Email', + acquisition_channel_affiliates(utm_medium), 'Affiliates', + acquisition_channel_audio(utm_medium), 'Audio', + acquisition_channel_sms(utm_source), 'SMS', + acquisition_channel_sms(utm_medium), 'SMS', + acquisition_channel_mobile_push_notifications(utm_medium, referrer_source), 'Mobile Push Notifications', + acquisition_channel_referral(utm_medium, referrer_source), 'Referral', + 'Direct' + ); diff --git a/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_update_materialized_column.sql.eex b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_update_materialized_column.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..1fd8027f69ee242b7420f15c304a4ee150b9d7a1 --- /dev/null +++ b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_update_materialized_column.sql.eex @@ -0,0 +1,4 @@ +ALTER TABLE <%= @table %> +<%= @on_cluster_statement %> +MODIFY COLUMN acquisition_channel LowCardinality(String) +MATERIALIZED acquisition_channel(referrer_source, utm_medium, utm_campaign, utm_source, click_id_param) diff --git a/priv/data_migrations/FunnelDupeGoals/sql/list-funnels-with-dupe-goal-ids.sql.eex b/priv/data_migrations/FunnelDupeGoals/sql/list-funnels-with-dupe-goal-ids.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..4ea2339c1cc9a5e2bf5a07a3513c6bdb0a04013e --- /dev/null +++ b/priv/data_migrations/FunnelDupeGoals/sql/list-funnels-with-dupe-goal-ids.sql.eex @@ -0,0 +1,62 @@ +WITH DuplicateGoals AS ( + SELECT + g.site_id, + CASE + WHEN g.event_name IS NOT NULL THEN 'Event: ' || g.event_name + WHEN g.page_path IS NOT NULL THEN 'Page: ' || g.page_path + END AS goal_name, + ARRAY_AGG(g.id) AS duplicate_ids + FROM + goals g + WHERE + g.site_id IS NOT NULL + GROUP BY + g.site_id, + goal_name + HAVING COUNT(*) > 1 +), +FunnelStepsWithDuplicateGoals AS ( + SELECT + fs.id, + fs.goal_id, + fs.funnel_id, + f.name, + CASE + WHEN g.event_name IS NOT NULL THEN 'Event: ' || g.event_name + WHEN g.page_path IS NOT NULL THEN 'Page: ' || g.page_path + END AS goal_name, + dg.duplicate_ids + FROM + funnel_steps fs + JOIN goals g ON fs.goal_id = g.id + JOIN funnels f ON fs.funnel_id = f.id + JOIN DuplicateGoals dg ON f.site_id = dg.site_id + WHERE + fs.goal_id = ANY(dg.duplicate_ids) -- Ch +) +SELECT + s.domain, + f.site_id, + f.name as funnel_name, + f.id, + fs.id, + fs.goal_id, + fs.goal_name, + fs.duplicate_ids, + array_agg(fs2.goal_id) +FROM + funnels f +JOIN FunnelStepsWithDuplicateGoals fs ON f.id = fs.funnel_id +JOIN funnel_steps fs2 ON f.id = fs2.funnel_id +JOIN sites s ON s.id = f.site_id +GROUP BY + s.domain, + f.site_id, + f.name, + f.id, + fs.id, + fs.goal_id, + fs.goal_name, + fs.duplicate_ids +ORDER BY fs.id ASC +; diff --git a/priv/data_migrations/LocationsSync/sql/add-alias-column.sql.eex b/priv/data_migrations/LocationsSync/sql/add-alias-column.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..36c77dab8f303e7de70f1c9f91f18ea40cc4c041 --- /dev/null +++ b/priv/data_migrations/LocationsSync/sql/add-alias-column.sql.eex @@ -0,0 +1,5 @@ +ALTER TABLE <%= @table %> +<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> +ADD COLUMN IF NOT EXISTS +<%= @column_name %> String +ALIAS dictGet('location_data_dict', 'name', tuple('<%= @type %>', <%= @input_column %>)) diff --git a/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex b/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..f5e5593f7599801bad39a1c130c1519beb3ff154 --- /dev/null +++ b/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS location_data <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> +( + `type` LowCardinality(String), + `id` String, + `name` String +) +<%= if @cluster? do %> +ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_prod/location_data', '{replica}') +<% else %> +ENGINE = MergeTree() +<% end %> +ORDER BY (type, id) +SETTINGS index_granularity = 128 <%= @table_settings %> diff --git a/priv/data_migrations/LocationsSync/sql/get-location-data-table-comment.sql.eex b/priv/data_migrations/LocationsSync/sql/get-location-data-table-comment.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..8d8acc1e4aea209c10e1c34f4b41495ad57c277c --- /dev/null +++ b/priv/data_migrations/LocationsSync/sql/get-location-data-table-comment.sql.eex @@ -0,0 +1 @@ +select comment from system.tables where database = currentDatabase() and table = 'location_data' diff --git a/priv/data_migrations/LocationsSync/sql/truncate-location-data-table.sql.eex b/priv/data_migrations/LocationsSync/sql/truncate-location-data-table.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..8dee71cb48e1387145bc3ed1dcf409eac51b70d7 --- /dev/null +++ b/priv/data_migrations/LocationsSync/sql/truncate-location-data-table.sql.eex @@ -0,0 +1 @@ +TRUNCATE TABLE IF EXISTS location_data SETTINGS alter_sync=2 diff --git a/priv/data_migrations/LocationsSync/sql/update-location-data-dictionary.sql.eex b/priv/data_migrations/LocationsSync/sql/update-location-data-dictionary.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..1b24d16d3000bbf49b5f40142075741dc0afad3e --- /dev/null +++ b/priv/data_migrations/LocationsSync/sql/update-location-data-dictionary.sql.eex @@ -0,0 +1,11 @@ +CREATE OR REPLACE DICTIONARY location_data_dict +<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> +( + `type` String, + `id` String, + `name` String +) +PRIMARY KEY type, id +SOURCE(CLICKHOUSE(TABLE location_data <%= @dictionary_connection_params %>)) +LIFETIME(0) +LAYOUT(complex_key_cache(size_in_cells 500000)) diff --git a/priv/data_migrations/LocationsSync/sql/update-location-data-table-comment.sql.eex b/priv/data_migrations/LocationsSync/sql/update-location-data-table-comment.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..94b3c081afc689e092e448c0d9339f515ca590c5 --- /dev/null +++ b/priv/data_migrations/LocationsSync/sql/update-location-data-table-comment.sql.eex @@ -0,0 +1,3 @@ +ALTER TABLE location_data +<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> +MODIFY COMMENT '<%= @version %>' diff --git a/priv/data_migrations/NumericIDs/sql/attach-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/attach-tmp-events-v2.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..ca274978ff66b2f5cf5f82bc547bc72969d5d2cd --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/attach-tmp-events-v2.sql.eex @@ -0,0 +1 @@ +ALTER TABLE events_v2 ATTACH PARTITION ID '<%= @partition %>' FROM tmp_events_v2 diff --git a/priv/data_migrations/NumericIDs/sql/attach-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/attach-tmp-sessions-v2.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..1b64580f92b35f3e7a2544a4f18f469e68343d8b --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/attach-tmp-sessions-v2.sql.eex @@ -0,0 +1 @@ +ALTER TABLE sessions_v2 ATTACH PARTITION ID '<%= @partition %>' FROM tmp_sessions_v2 diff --git a/priv/data_migrations/NumericIDs/sql/check-replicas.sql.eex b/priv/data_migrations/NumericIDs/sql/check-replicas.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..6d639edcffc89d8dea61d6f7960563528bd9c0db --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/check-replicas.sql.eex @@ -0,0 +1 @@ +SELECT 1 FROM system.replicas WHERE table = 'events' diff --git a/priv/data_migrations/NumericIDs/sql/create-domains-lookup.sql.eex b/priv/data_migrations/NumericIDs/sql/create-domains-lookup.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..b18d7fb5f40eb23db9b5dd5c9f2507b4af485090 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/create-domains-lookup.sql.eex @@ -0,0 +1 @@ +CREATE TABLE IF NOT EXISTS domains_lookup(site_id UInt64, domain String) ENGINE Memory; diff --git a/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..ebe5fc861951ebca79921a2b36455dab1879a63d --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex @@ -0,0 +1,39 @@ +CREATE TABLE IF NOT EXISTS events_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> +( + `timestamp` DateTime CODEC(Delta(4), LZ4), + `name` LowCardinality(String), + `site_id` UInt64, + `user_id` UInt64, + `session_id` UInt64, + `hostname` String, + `pathname` String CODEC(ZSTD(3)), + `referrer` String, + `referrer_source` String, + `country_code` FixedString(2), + `screen_size` LowCardinality(String), + `operating_system` LowCardinality(String), + `browser` LowCardinality(String), + `utm_medium` String, + `utm_source` String, + `utm_campaign` String, + `meta.key` Array(String), + `meta.value` Array(String), + `browser_version` LowCardinality(String), + `operating_system_version` LowCardinality(String), + `subdivision1_code` LowCardinality(String), + `subdivision2_code` LowCardinality(String), + `city_geoname_id` UInt32, + `utm_content` String, + `utm_term` String, + `transferred_from` String +) +<%= if @cluster? do %> +ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/{database}/events_v2', '{replica}') +<% else %> +ENGINE = MergeTree() +<% end %> +PARTITION BY toYYYYMM(timestamp) +PRIMARY KEY (site_id, toDate(timestamp), name, user_id) +ORDER BY (site_id, toDate(timestamp), name, user_id, timestamp) +SAMPLE BY user_id +<%= @table_settings %> diff --git a/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..e03d910762a0b777c61701adc84384e6b8b2eaf5 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex @@ -0,0 +1,45 @@ +CREATE TABLE IF NOT EXISTS sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> +( + `session_id` UInt64, + `sign` Int8, + `site_id` UInt64, + `user_id` UInt64, + `hostname` String, + `timestamp` DateTime CODEC(DoubleDelta, LZ4), + `start` DateTime CODEC(DoubleDelta, LZ4), + `is_bounce` UInt8, + `entry_page` String, + `exit_page` String, + `pageviews` Int32, + `events` Int32, + `duration` UInt32, + `referrer` String, + `referrer_source` String, + `country_code` LowCardinality(FixedString(2)), + `screen_size` LowCardinality(String), + `operating_system` LowCardinality(String), + `browser` LowCardinality(String), + `utm_medium` String, + `utm_source` String, + `utm_campaign` String, + `browser_version` LowCardinality(String), + `operating_system_version` LowCardinality(String), + `subdivision1_code` LowCardinality(String), + `subdivision2_code` LowCardinality(String), + `city_geoname_id` UInt32, + `utm_content` String, + `utm_term` String, + `transferred_from` String, + `entry_meta.key` Array(String), + `entry_meta.value` Array(String) +) +<%= if @cluster? do %> +ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_prod/sessions_v2', '{replica}', sign) +<% else %> +ENGINE = CollapsingMergeTree(sign) +<% end %> +PARTITION BY toYYYYMM(start) +PRIMARY KEY (site_id, toDate(start), user_id, session_id) +ORDER BY (site_id, toDate(start), user_id, session_id) +SAMPLE BY user_id +<%= @table_settings %> diff --git a/priv/data_migrations/NumericIDs/sql/create-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-tmp-events-v2.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..be70d6c44515e95228db9058b2b9ece10bb951a3 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/create-tmp-events-v2.sql.eex @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS + tmp_events_v2 AS events_v2 ENGINE = MergeTree() + ORDER BY (site_id, toDate(timestamp), name, user_id, timestamp) + PRIMARY KEY (site_id, toDate(timestamp), name, user_id) + PARTITION BY toYYYYMM(timestamp) + <%= @table_settings %> diff --git a/priv/data_migrations/NumericIDs/sql/create-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-tmp-sessions-v2.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..bc93a129b5f95a29df06526d3bacfcd960204f7e --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/create-tmp-sessions-v2.sql.eex @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS + tmp_sessions_v2 AS sessions_v2 ENGINE = CollapsingMergeTree(sign) + PARTITION BY toYYYYMM(start) + PRIMARY KEY (site_id, toDate(start), user_id, session_id) + ORDER BY (site_id, toDate(start), user_id, session_id) + <%= @table_settings %> diff --git a/priv/data_migrations/NumericIDs/sql/drop-domains-lookup.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-domains-lookup.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..6ca833f126640c80fa65d360ef6e93391b33503a --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/drop-domains-lookup.sql.eex @@ -0,0 +1 @@ +DROP TABLE IF EXISTS domains_lookup SYNC; diff --git a/priv/data_migrations/NumericIDs/sql/drop-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-events-v2.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..ae3927e49ddf272b7f265dacd09aea55dc518aba --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/drop-events-v2.sql.eex @@ -0,0 +1 @@ +DROP TABLE IF EXISTS events_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> SYNC diff --git a/priv/data_migrations/NumericIDs/sql/drop-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-sessions-v2.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..eee7668420d3eea0823931572c793211808e6ada --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/drop-sessions-v2.sql.eex @@ -0,0 +1 @@ +DROP TABLE IF EXISTS sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> SYNC diff --git a/priv/data_migrations/NumericIDs/sql/drop-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-tmp-events-v2.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..9ce47fbae7e1696b41e414ed8ff386b998e24d60 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/drop-tmp-events-v2.sql.eex @@ -0,0 +1 @@ +DROP TABLE IF EXISTS tmp_events_v2 SYNC diff --git a/priv/data_migrations/NumericIDs/sql/drop-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-tmp-sessions-v2.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..ea3666c511592a52a3ed07133e943fcdf4bbebf6 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/drop-tmp-sessions-v2.sql.eex @@ -0,0 +1 @@ +DROP TABLE IF EXISTS tmp_sessions_v2 SYNC diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..26500f99fbd4181c864d83954adb0c007c6eba88 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex @@ -0,0 +1,42 @@ +INSERT INTO tmp_events_v2 ( + site_id, timestamp, name, user_id, + session_id, hostname, pathname, referrer, + referrer_source, country_code, screen_size, + operating_system, browser, utm_medium, + utm_source, utm_campaign, meta.key, + meta.value, browser_version, operating_system_version, + subdivision1_code, subdivision2_code, + city_geoname_id, utm_content, utm_term, + transferred_from +) +SELECT + site_id, + timestamp, + name, + user_id, + session_id, + hostname, + pathname, + referrer, + referrer_source, + country_code, + screen_size, + operating_system, + browser, + utm_medium, + utm_source, + utm_campaign, + meta.key, + meta.value, + browser_version, + operating_system_version, + subdivision1_code, + subdivision2_code, + city_geoname_id, + utm_content, + utm_term, + transferred_from +FROM + events +INNER JOIN domains_lookup ON domains_lookup.domain = events.domain +WHERE (_partition_id = '<%= @partition %>') diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..e2222b144fe7d7633d954a83ed6ce233f8db0262 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex @@ -0,0 +1,18 @@ +INSERT INTO tmp_sessions_v2 ( +site_id, session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page, +exit_page, pageviews, events, duration, referrer, referrer_source, country_code, +screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version, +operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id, utm_content, +utm_term, transferred_from, entry_meta.key, entry_meta.value +) +SELECT + site_id, + session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page, + exit_page, pageviews, events, duration, referrer, referrer_source, country_code, + screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version, + operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id, utm_content, + utm_term, transferred_from, entry_meta.key, entry_meta.value +FROM + sessions +INNER JOIN domains_lookup ON domains_lookup.domain = sessions.domain +WHERE (_partition_id = '<%= @partition %>') diff --git a/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex b/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..9522395a30a4041207460dde0738305a58067937 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex @@ -0,0 +1,6 @@ +SELECT distinct(partition) +FROM system.parts +WHERE table = 'events' +<%= if @start_from do %>AND partition >= '<%= @start_from %>'<% end %> +<%= if @stop_at do %>AND partition <= '<%= @stop_at %>'<% end %> +ORDER BY partition diff --git a/priv/data_migrations/NumericIDs/sql/truncate-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/truncate-tmp-events-v2.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..18e6fef98c095d776ba35c2fbf29f8cb40e1dfd0 --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/truncate-tmp-events-v2.sql.eex @@ -0,0 +1 @@ +TRUNCATE TABLE tmp_events_v2 diff --git a/priv/data_migrations/NumericIDs/sql/truncate-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/truncate-tmp-sessions-v2.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..22b8bbddf9bb7a9738c3bb5ba983af6f9c09245c --- /dev/null +++ b/priv/data_migrations/NumericIDs/sql/truncate-tmp-sessions-v2.sql.eex @@ -0,0 +1 @@ +TRUNCATE TABLE tmp_sessions_v2 diff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/create-sessions-dictionary.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/create-sessions-dictionary.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..b5bdac37f24108228531f5f1d78ba553dda20d71 --- /dev/null +++ b/priv/data_migrations/PopulateEventSessionColumns/sql/create-sessions-dictionary.sql.eex @@ -0,0 +1,32 @@ +CREATE OR REPLACE DICTIONARY sessions_dict +<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> +( + site_id UInt64, + session_id UInt64, + referrer String, + referrer_source String, + utm_medium String, + utm_source String, + utm_campaign String, + utm_content String, + utm_term String, + country_code String, + subdivision1_code String, + subdivision2_code String, + city_geoname_id UInt32, + screen_size String, + operating_system String, + operating_system_version String, + browser String, + browser_version String +) +PRIMARY KEY site_id, session_id +SOURCE(CLICKHOUSE(TABLE sessions_v2 <%= @dictionary_connection_params %>)) +LIFETIME(<%= @dictionary_config.lifetime %>) +LAYOUT( + complex_key_cache( + size_in_cells <%= @dictionary_config.size_in_cells %> + max_threads_for_updates <%= @dictionary_config.max_threads_for_updates %> + allow_read_expired_keys 1 + ) +) diff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/drop-sessions-dictionary.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/drop-sessions-dictionary.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..69826e863c9a2c025741ade1d2d8473ff55c87ba --- /dev/null +++ b/priv/data_migrations/PopulateEventSessionColumns/sql/drop-sessions-dictionary.sql.eex @@ -0,0 +1,2 @@ +DROP DICTIONARY IF EXISTS sessions_dict +<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> diff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/get-disks.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/get-disks.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..aa678c4777ed9e5d83fd3c87fbda50d1ae7ffea6 --- /dev/null +++ b/priv/data_migrations/PopulateEventSessionColumns/sql/get-disks.sql.eex @@ -0,0 +1,7 @@ +SELECT + name, + path, + formatReadableSize(total_space - free_space) AS full_space, + formatReadableSize(total_space), + round((total_space - free_space) / total_space * 100, 2) +FROM system.disks diff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/get-merges-progress.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/get-merges-progress.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..c9589f83cab3ee45d99dfe3743c50a1e221a66c9 --- /dev/null +++ b/priv/data_migrations/PopulateEventSessionColumns/sql/get-merges-progress.sql.eex @@ -0,0 +1,4 @@ +SELECT count() +FROM system.merges +WHERE table = 'events_v2' + AND is_mutation diff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/get-mutations-progress.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/get-mutations-progress.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..13efb4fc2c4fba3374f38a1c9fa8dd273866bf6c --- /dev/null +++ b/priv/data_migrations/PopulateEventSessionColumns/sql/get-mutations-progress.sql.eex @@ -0,0 +1,29 @@ +SELECT + total, + length(parts_to_do_names), + running_for_seconds, + formatReadableSize(total_bytes), + formatReadableSize(todo_bytes), + round(100 - todo_bytes / total_bytes * 100, 2), + latest_fail_reason, + sum(part.bytes_on_disk) as total_bytes, + sumIf(part.bytes_on_disk, has(parts_to_do_names, part.name)) AS todo_bytes +FROM ( + SELECT count() AS total, + groupArrayArray(parts_to_do_names) AS parts_to_do_names, + now() - min(create_time) AS running_for_seconds, + anyIf(latest_fail_reason, latest_fail_reason != '') AS latest_fail_reason + FROM system.mutations + WHERE not is_done + AND table = 'events_v2' + AND command ILIKE '%sessions_dict%' +) AS mut +CROSS JOIN ( + SELECT name, bytes_on_disk + FROM system.parts + WHERE table = 'events_v2' + AND active + AND partition >= '<%= @min_partition %>' + AND partition <= '<%= @max_partition %>' +) AS part +GROUP BY mut.total, mut.parts_to_do_names, mut.running_for_seconds, mut.latest_fail_reason diff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/kill-running-mutations.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/kill-running-mutations.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..e37c6149ce3570d5e010e4698089472c579a5a69 --- /dev/null +++ b/priv/data_migrations/PopulateEventSessionColumns/sql/kill-running-mutations.sql.eex @@ -0,0 +1,2 @@ +KILL MUTATION <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> +WHERE command ILIKE '%sessions_dict%' diff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/list-partitions.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/list-partitions.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..0bbc9b5c6439c1b6276c49224ad1641631ac8cd4 --- /dev/null +++ b/priv/data_migrations/PopulateEventSessionColumns/sql/list-partitions.sql.eex @@ -0,0 +1,7 @@ +SELECT arraySort(groupUniqArray(partition)), arraySort(groupArray(name)) +FROM system.parts +WHERE active + AND database = currentDatabase() + AND table = 'events_v2' + AND partition >= '<%= @min_partition %>' + AND partition <= '<%= @max_partition %>' diff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/update-table.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/update-table.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..f9990c98b2279d6efd570d26d05cc856b193c857 --- /dev/null +++ b/priv/data_migrations/PopulateEventSessionColumns/sql/update-table.sql.eex @@ -0,0 +1,21 @@ +ALTER TABLE events_v2 +<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> +UPDATE + referrer = dictGet('sessions_dict', 'referrer', tuple(site_id, session_id)), + referrer_source = dictGet('sessions_dict', 'referrer_source', tuple(site_id, session_id)), + utm_medium = dictGet('sessions_dict', 'utm_medium', tuple(site_id, session_id)), + utm_source = dictGet('sessions_dict', 'utm_source', tuple(site_id, session_id)), + utm_campaign = dictGet('sessions_dict', 'utm_campaign', tuple(site_id, session_id)), + utm_content = dictGet('sessions_dict', 'utm_content', tuple(site_id, session_id)), + utm_term = dictGet('sessions_dict', 'utm_term', tuple(site_id, session_id)), + country_code = dictGet('sessions_dict', 'country_code', tuple(site_id, session_id)), + subdivision1_code = dictGet('sessions_dict', 'subdivision1_code', tuple(site_id, session_id)), + subdivision2_code = dictGet('sessions_dict', 'subdivision2_code', tuple(site_id, session_id)), + city_geoname_id = dictGet('sessions_dict', 'city_geoname_id', tuple(site_id, session_id)), + screen_size = dictGet('sessions_dict', 'screen_size', tuple(site_id, session_id)), + operating_system = dictGet('sessions_dict', 'operating_system', tuple(site_id, session_id)), + operating_system_version = dictGet('sessions_dict', 'operating_system_version', tuple(site_id, session_id)), + browser = dictGet('sessions_dict', 'browser', tuple(site_id, session_id)), + browser_version = dictGet('sessions_dict', 'browser_version', tuple(site_id, session_id)) +IN PARTITION '<%= @partition %>' +WHERE 1=1 diff --git a/priv/data_migrations/VersionedSessions/sql/attach-partition.sql.eex b/priv/data_migrations/VersionedSessions/sql/attach-partition.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..c1d8cfb5f82e55e5298e2e5a1716d094d92239a1 --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/attach-partition.sql.eex @@ -0,0 +1 @@ +ALTER TABLE sessions_v2_tmp_versioned ATTACH PARTITION '<%= @partition %>' FROM sessions_v2 diff --git a/priv/data_migrations/VersionedSessions/sql/check-replicas.sql.eex b/priv/data_migrations/VersionedSessions/sql/check-replicas.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..367a574df485ef8d65d59830ae337d08788ba3bb --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/check-replicas.sql.eex @@ -0,0 +1 @@ +SELECT 1 FROM system.replicas WHERE table = 'sessions_v2' diff --git a/priv/data_migrations/VersionedSessions/sql/create-sessions-tmp-table.sql.eex b/priv/data_migrations/VersionedSessions/sql/create-sessions-tmp-table.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..dc579c85876c98e9fa15fe2f22e63bc2ab3f6a33 --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/create-sessions-tmp-table.sql.eex @@ -0,0 +1,13 @@ +CREATE TABLE sessions_v2_tmp_versioned +<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> +AS sessions_v2 +<%= if @cluster? do %> +ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_prod/sessions_v2_<%= @unique_suffix %>', '{replica}', sign, events) +<% else %> +ENGINE = VersionedCollapsingMergeTree(sign, events) +<% end %> +PARTITION BY toYYYYMM(start) +PRIMARY KEY (site_id, toDate(start), user_id, session_id) +ORDER BY (site_id, toDate(start), user_id, session_id) +SAMPLE BY user_id +<%= @table_settings %> diff --git a/priv/data_migrations/VersionedSessions/sql/drop-sessions-tmp-table.sql.eex b/priv/data_migrations/VersionedSessions/sql/drop-sessions-tmp-table.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..7a967981a15cceb040436d0101903fe70f9df993 --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/drop-sessions-tmp-table.sql.eex @@ -0,0 +1 @@ +DROP TABLE IF EXISTS sessions_v2_tmp_versioned <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> diff --git a/priv/data_migrations/VersionedSessions/sql/exchange-sessions-tables.sql.eex b/priv/data_migrations/VersionedSessions/sql/exchange-sessions-tables.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..416da043ed9b5644b82947e2ebd6aa65624a3481 --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/exchange-sessions-tables.sql.eex @@ -0,0 +1 @@ +EXCHANGE TABLES sessions_v2_tmp_versioned AND sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> diff --git a/priv/data_migrations/VersionedSessions/sql/get-sessions-table-settings.sql.eex b/priv/data_migrations/VersionedSessions/sql/get-sessions-table-settings.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..ffe35fb43b4de063d22416d66d7f417fdef26fde --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/get-sessions-table-settings.sql.eex @@ -0,0 +1,4 @@ +SELECT engine, extract(engine_full, 'SETTINGS .+') +FROM system.tables +WHERE database = currentDatabase() + AND name = 'sessions_v2' diff --git a/priv/data_migrations/VersionedSessions/sql/list-partitions.sql.eex b/priv/data_migrations/VersionedSessions/sql/list-partitions.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..986822688191cacb6588e8e280b44a57690096af --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/list-partitions.sql.eex @@ -0,0 +1,5 @@ +SELECT distinct(partition) +FROM system.parts +WHERE database = currentDatabase() + AND table = 'sessions_v2' +ORDER BY partition diff --git a/priv/data_migrations/VersionedSessions/sql/rename-table.sql.eex b/priv/data_migrations/VersionedSessions/sql/rename-table.sql.eex new file mode 100644 index 0000000000000000000000000000000000000000..ea07c8d93baf02752bbc31f9e44d3fa3feeb9ae6 --- /dev/null +++ b/priv/data_migrations/VersionedSessions/sql/rename-table.sql.eex @@ -0,0 +1 @@ +RENAME TABLE <%= @from %> TO <%= @to %> <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> diff --git a/priv/ga4-source-categories.csv b/priv/ga4-source-categories.csv new file mode 100644 index 0000000000000000000000000000000000000000..be40ba1549fa143261fa9db105aefef6092d1581 --- /dev/null +++ b/priv/ga4-source-categories.csv @@ -0,0 +1,820 @@ +source,source category +360.cn,SOURCE_CATEGORY_SEARCH +43things,SOURCE_CATEGORY_SOCIAL +43things.com,SOURCE_CATEGORY_SOCIAL +51.com,SOURCE_CATEGORY_SOCIAL +5ch.net,SOURCE_CATEGORY_SOCIAL +Google Shopping,SOURCE_CATEGORY_SHOPPING +Hatena,SOURCE_CATEGORY_SOCIAL +IGShopping,SOURCE_CATEGORY_SHOPPING +ImageShack,SOURCE_CATEGORY_SOCIAL +aax-us-east.amazon-adsystem.com,SOURCE_CATEGORY_SHOPPING +aax.amazon-adsystem.com,SOURCE_CATEGORY_SHOPPING +academia.edu,SOURCE_CATEGORY_SOCIAL +activerain,SOURCE_CATEGORY_SOCIAL +activerain.com,SOURCE_CATEGORY_SOCIAL +activeworlds,SOURCE_CATEGORY_SOCIAL +activeworlds.com,SOURCE_CATEGORY_SOCIAL +addthis,SOURCE_CATEGORY_SOCIAL +addthis.com,SOURCE_CATEGORY_SOCIAL +airg.ca,SOURCE_CATEGORY_SOCIAL +alibaba,SOURCE_CATEGORY_SHOPPING +alibaba.com,SOURCE_CATEGORY_SHOPPING +alice,SOURCE_CATEGORY_SEARCH +allnurses.com,SOURCE_CATEGORY_SOCIAL +allrecipes.com,SOURCE_CATEGORY_SOCIAL +alumniclass,SOURCE_CATEGORY_SOCIAL +alumniclass.com,SOURCE_CATEGORY_SOCIAL +amazon,SOURCE_CATEGORY_SHOPPING +amazon.co.uk,SOURCE_CATEGORY_SHOPPING +amazon.com,SOURCE_CATEGORY_SHOPPING +ameba.jp,SOURCE_CATEGORY_SOCIAL +ameblo.jp,SOURCE_CATEGORY_SOCIAL +americantowns,SOURCE_CATEGORY_SOCIAL +americantowns.com,SOURCE_CATEGORY_SOCIAL +amp.reddit.com,SOURCE_CATEGORY_SOCIAL +ancestry.com,SOURCE_CATEGORY_SOCIAL +anobii,SOURCE_CATEGORY_SOCIAL +anobii.com,SOURCE_CATEGORY_SOCIAL +answerbag,SOURCE_CATEGORY_SOCIAL +answerbag.com,SOURCE_CATEGORY_SOCIAL +answers.yahoo.com,SOURCE_CATEGORY_SOCIAL +aol,SOURCE_CATEGORY_SEARCH +aolanswers,SOURCE_CATEGORY_SOCIAL +aolanswers.com,SOURCE_CATEGORY_SOCIAL +apps.facebook.com,SOURCE_CATEGORY_SOCIAL +apps.shopify.com,SOURCE_CATEGORY_SHOPPING +ar.pinterest.com,SOURCE_CATEGORY_SOCIAL +ar.search.yahoo.com,SOURCE_CATEGORY_SEARCH +artstation.com,SOURCE_CATEGORY_SOCIAL +ask,SOURCE_CATEGORY_SEARCH +askubuntu,SOURCE_CATEGORY_SOCIAL +askubuntu.com,SOURCE_CATEGORY_SOCIAL +asmallworld.com,SOURCE_CATEGORY_SOCIAL +at.search.yahoo.com,SOURCE_CATEGORY_SEARCH +athlinks,SOURCE_CATEGORY_SOCIAL +athlinks.com,SOURCE_CATEGORY_SOCIAL +au.search.yahoo.com,SOURCE_CATEGORY_SEARCH +auone,SOURCE_CATEGORY_SEARCH +avg,SOURCE_CATEGORY_SEARCH +away.vk.com,SOURCE_CATEGORY_SOCIAL +awe.sm,SOURCE_CATEGORY_SOCIAL +b.hatena.ne.jp,SOURCE_CATEGORY_SOCIAL +baby-gaga,SOURCE_CATEGORY_SOCIAL +baby-gaga.com,SOURCE_CATEGORY_SOCIAL +babyblog.ru,SOURCE_CATEGORY_SOCIAL +babylon,SOURCE_CATEGORY_SEARCH +badoo,SOURCE_CATEGORY_SOCIAL +badoo.com,SOURCE_CATEGORY_SOCIAL +baidu,SOURCE_CATEGORY_SEARCH +bebo,SOURCE_CATEGORY_SOCIAL +bebo.com,SOURCE_CATEGORY_SOCIAL +beforeitsnews,SOURCE_CATEGORY_SOCIAL +beforeitsnews.com,SOURCE_CATEGORY_SOCIAL +bharatstudent,SOURCE_CATEGORY_SOCIAL +bharatstudent.com,SOURCE_CATEGORY_SOCIAL +biglobe,SOURCE_CATEGORY_SEARCH +biglobe.co.jp,SOURCE_CATEGORY_SEARCH +biglobe.ne.jp,SOURCE_CATEGORY_SEARCH +biip.no,SOURCE_CATEGORY_SOCIAL +bing,SOURCE_CATEGORY_SEARCH +biswap.org,SOURCE_CATEGORY_SOCIAL +bit.ly,SOURCE_CATEGORY_SOCIAL +blackcareernetwork.com,SOURCE_CATEGORY_SOCIAL +blackplanet,SOURCE_CATEGORY_SOCIAL +blackplanet.com,SOURCE_CATEGORY_SOCIAL +blip.fm,SOURCE_CATEGORY_SOCIAL +blog.com,SOURCE_CATEGORY_SOCIAL +blog.feedspot.com,SOURCE_CATEGORY_SOCIAL +blog.goo.ne.jp,SOURCE_CATEGORY_SOCIAL +blog.naver.com,SOURCE_CATEGORY_SOCIAL +blog.twitch.tv,SOURCE_CATEGORY_VIDEO +blog.yahoo.co.jp,SOURCE_CATEGORY_SOCIAL +blogg.no,SOURCE_CATEGORY_SOCIAL +bloggang.com,SOURCE_CATEGORY_SOCIAL +blogger,SOURCE_CATEGORY_SOCIAL +blogger.com,SOURCE_CATEGORY_SOCIAL +blogher,SOURCE_CATEGORY_SOCIAL +blogher.com,SOURCE_CATEGORY_SOCIAL +bloglines,SOURCE_CATEGORY_SOCIAL +bloglines.com,SOURCE_CATEGORY_SOCIAL +blogs.com,SOURCE_CATEGORY_SOCIAL +blogsome,SOURCE_CATEGORY_SOCIAL +blogsome.com,SOURCE_CATEGORY_SOCIAL +blogspot,SOURCE_CATEGORY_SOCIAL +blogspot.com,SOURCE_CATEGORY_SOCIAL +blogster,SOURCE_CATEGORY_SOCIAL +blogster.com,SOURCE_CATEGORY_SOCIAL +blurtit,SOURCE_CATEGORY_SOCIAL +blurtit.com,SOURCE_CATEGORY_SOCIAL +bookmarks.yahoo.co.jp,SOURCE_CATEGORY_SOCIAL +bookmarks.yahoo.com,SOURCE_CATEGORY_SOCIAL +br.pinterest.com,SOURCE_CATEGORY_SOCIAL +br.search.yahoo.com,SOURCE_CATEGORY_SEARCH +brightkite,SOURCE_CATEGORY_SOCIAL +brightkite.com,SOURCE_CATEGORY_SOCIAL +brizzly,SOURCE_CATEGORY_SOCIAL +brizzly.com,SOURCE_CATEGORY_SOCIAL +business.facebook.com,SOURCE_CATEGORY_SOCIAL +buzzfeed,SOURCE_CATEGORY_SOCIAL +buzzfeed.com,SOURCE_CATEGORY_SOCIAL +buzznet,SOURCE_CATEGORY_SOCIAL +buzznet.com,SOURCE_CATEGORY_SOCIAL +ca.search.yahoo.com,SOURCE_CATEGORY_SEARCH +cafe.naver.com,SOURCE_CATEGORY_SOCIAL +cafemom,SOURCE_CATEGORY_SOCIAL +cafemom.com,SOURCE_CATEGORY_SOCIAL +camospace,SOURCE_CATEGORY_SOCIAL +camospace.com,SOURCE_CATEGORY_SOCIAL +canalblog.com,SOURCE_CATEGORY_SOCIAL +care.com,SOURCE_CATEGORY_SOCIAL +care2,SOURCE_CATEGORY_SOCIAL +care2.com,SOURCE_CATEGORY_SOCIAL +caringbridge.org,SOURCE_CATEGORY_SOCIAL +catster,SOURCE_CATEGORY_SOCIAL +catster.com,SOURCE_CATEGORY_SOCIAL +cbnt.io,SOURCE_CATEGORY_SOCIAL +cellufun,SOURCE_CATEGORY_SOCIAL +cellufun.com,SOURCE_CATEGORY_SOCIAL +centerblog.net,SOURCE_CATEGORY_SOCIAL +centrum.cz,SOURCE_CATEGORY_SEARCH +ch.search.yahoo.com,SOURCE_CATEGORY_SEARCH +chat.zalo.me,SOURCE_CATEGORY_SOCIAL +checkout.shopify.com,SOURCE_CATEGORY_SHOPPING +checkout.stripe.com,SOURCE_CATEGORY_SHOPPING +chegg.com,SOURCE_CATEGORY_SOCIAL +chicagonow,SOURCE_CATEGORY_SOCIAL +chicagonow.com,SOURCE_CATEGORY_SOCIAL +chiebukuro.yahoo.co.jp,SOURCE_CATEGORY_SOCIAL +cl.search.yahoo.com,SOURCE_CATEGORY_SEARCH +classmates,SOURCE_CATEGORY_SOCIAL +classmates.com,SOURCE_CATEGORY_SOCIAL +classquest,SOURCE_CATEGORY_SOCIAL +classquest.com,SOURCE_CATEGORY_SOCIAL +cn.bing.com,SOURCE_CATEGORY_SEARCH +cnn,SOURCE_CATEGORY_SEARCH +co.pinterest.com,SOURCE_CATEGORY_SOCIAL +co.search.yahoo.com,SOURCE_CATEGORY_SEARCH +cocolog-nifty,SOURCE_CATEGORY_SOCIAL +cocolog-nifty.com,SOURCE_CATEGORY_SOCIAL +comcast,SOURCE_CATEGORY_SEARCH +conduit,SOURCE_CATEGORY_SEARCH +copainsdavant.linternaute.com,SOURCE_CATEGORY_SOCIAL +couchsurfing.org,SOURCE_CATEGORY_SOCIAL +cozycot,SOURCE_CATEGORY_SOCIAL +cozycot.com,SOURCE_CATEGORY_SOCIAL +cr.shopping.naver.com,SOURCE_CATEGORY_SHOPPING +cr2.shopping.naver.com,SOURCE_CATEGORY_SHOPPING +crackle,SOURCE_CATEGORY_VIDEO +crackle.com,SOURCE_CATEGORY_VIDEO +cross.tv,SOURCE_CATEGORY_SOCIAL +crunchyroll,SOURCE_CATEGORY_SOCIAL +crunchyroll.com,SOURCE_CATEGORY_SOCIAL +curiositystream,SOURCE_CATEGORY_VIDEO +curiositystream.com,SOURCE_CATEGORY_VIDEO +cyworld,SOURCE_CATEGORY_SOCIAL +cyworld.com,SOURCE_CATEGORY_SOCIAL +cz.pinterest.com,SOURCE_CATEGORY_SOCIAL +d.hatena.ne.jp,SOURCE_CATEGORY_SOCIAL +d.tube,SOURCE_CATEGORY_VIDEO +dailymotion,SOURCE_CATEGORY_VIDEO +dailymotion.com,SOURCE_CATEGORY_VIDEO +dailystrength.org,SOURCE_CATEGORY_SOCIAL +dashboard.twitch.tv,SOURCE_CATEGORY_VIDEO +daum,SOURCE_CATEGORY_SEARCH +daum.net,SOURCE_CATEGORY_SEARCH +de.search.yahoo.com,SOURCE_CATEGORY_SEARCH +deluxe.com,SOURCE_CATEGORY_SOCIAL +deviantart,SOURCE_CATEGORY_SOCIAL +deviantart.com,SOURCE_CATEGORY_SOCIAL +dianping,SOURCE_CATEGORY_SOCIAL +dianping.com,SOURCE_CATEGORY_SOCIAL +digg,SOURCE_CATEGORY_SOCIAL +digg.com,SOURCE_CATEGORY_SOCIAL +diigo,SOURCE_CATEGORY_SOCIAL +diigo.com,SOURCE_CATEGORY_SOCIAL +discover.hubpages.com,SOURCE_CATEGORY_SOCIAL +disneyplus,SOURCE_CATEGORY_VIDEO +disneyplus.com,SOURCE_CATEGORY_VIDEO +disqus,SOURCE_CATEGORY_SOCIAL +disqus.com,SOURCE_CATEGORY_SOCIAL +dk.search.yahoo.com,SOURCE_CATEGORY_SEARCH +dogpile,SOURCE_CATEGORY_SEARCH +dogpile.com,SOURCE_CATEGORY_SEARCH +dogster,SOURCE_CATEGORY_SOCIAL +dogster.com,SOURCE_CATEGORY_SOCIAL +dol2day,SOURCE_CATEGORY_SOCIAL +dol2day.com,SOURCE_CATEGORY_SOCIAL +doostang,SOURCE_CATEGORY_SOCIAL +doostang.com,SOURCE_CATEGORY_SOCIAL +dopplr,SOURCE_CATEGORY_SOCIAL +dopplr.com,SOURCE_CATEGORY_SOCIAL +douban,SOURCE_CATEGORY_SOCIAL +douban.com,SOURCE_CATEGORY_SOCIAL +draft.blogger.com,SOURCE_CATEGORY_SOCIAL +draugiem.lv,SOURCE_CATEGORY_SOCIAL +drugs-forum,SOURCE_CATEGORY_SOCIAL +drugs-forum.com,SOURCE_CATEGORY_SOCIAL +duckduckgo,SOURCE_CATEGORY_SEARCH +dzone,SOURCE_CATEGORY_SOCIAL +dzone.com,SOURCE_CATEGORY_SOCIAL +ebay,SOURCE_CATEGORY_SHOPPING +ebay.co.uk,SOURCE_CATEGORY_SHOPPING +ebay.com,SOURCE_CATEGORY_SHOPPING +ebay.com.au,SOURCE_CATEGORY_SHOPPING +ebay.de,SOURCE_CATEGORY_SHOPPING +ecosia.org,SOURCE_CATEGORY_SEARCH +edublogs.org,SOURCE_CATEGORY_SOCIAL +elftown,SOURCE_CATEGORY_SOCIAL +elftown.com,SOURCE_CATEGORY_SOCIAL +email.seznam.cz,SOURCE_CATEGORY_SEARCH +eniro,SOURCE_CATEGORY_SEARCH +epicurious.com,SOURCE_CATEGORY_SOCIAL +es.search.yahoo.com,SOURCE_CATEGORY_SEARCH +espanol.search.yahoo.com,SOURCE_CATEGORY_SEARCH +etsy,SOURCE_CATEGORY_SHOPPING +etsy.com,SOURCE_CATEGORY_SHOPPING +everforo.com,SOURCE_CATEGORY_SOCIAL +exalead.com,SOURCE_CATEGORY_SEARCH +exblog.jp,SOURCE_CATEGORY_SOCIAL +excite.com,SOURCE_CATEGORY_SEARCH +extole,SOURCE_CATEGORY_SOCIAL +extole.com,SOURCE_CATEGORY_SOCIAL +facebook,SOURCE_CATEGORY_SOCIAL +facebook.com,SOURCE_CATEGORY_SOCIAL +faceparty,SOURCE_CATEGORY_SOCIAL +faceparty.com,SOURCE_CATEGORY_SOCIAL +fandom.com,SOURCE_CATEGORY_SOCIAL +fanpop,SOURCE_CATEGORY_SOCIAL +fanpop.com,SOURCE_CATEGORY_SOCIAL +fark,SOURCE_CATEGORY_SOCIAL +fark.com,SOURCE_CATEGORY_SOCIAL +fast.wistia.net,SOURCE_CATEGORY_VIDEO +fb,SOURCE_CATEGORY_SOCIAL +fb.me,SOURCE_CATEGORY_SOCIAL +fc2,SOURCE_CATEGORY_SOCIAL +fc2.com,SOURCE_CATEGORY_SOCIAL +feedspot,SOURCE_CATEGORY_SOCIAL +feministing,SOURCE_CATEGORY_SOCIAL +feministing.com,SOURCE_CATEGORY_SOCIAL +fi.search.yahoo.com,SOURCE_CATEGORY_SEARCH +filmaffinity,SOURCE_CATEGORY_SOCIAL +filmaffinity.com,SOURCE_CATEGORY_SOCIAL +firmy.cz,SOURCE_CATEGORY_SEARCH +flickr,SOURCE_CATEGORY_SOCIAL +flickr.com,SOURCE_CATEGORY_SOCIAL +flipboard,SOURCE_CATEGORY_SOCIAL +flipboard.com,SOURCE_CATEGORY_SOCIAL +folkdirect,SOURCE_CATEGORY_SOCIAL +folkdirect.com,SOURCE_CATEGORY_SOCIAL +foodservice,SOURCE_CATEGORY_SOCIAL +foodservice.com,SOURCE_CATEGORY_SOCIAL +forums.androidcentral.com,SOURCE_CATEGORY_SOCIAL +forums.crackberry.com,SOURCE_CATEGORY_SOCIAL +forums.imore.com,SOURCE_CATEGORY_SOCIAL +forums.nexopia.com,SOURCE_CATEGORY_SOCIAL +forums.webosnation.com,SOURCE_CATEGORY_SOCIAL +forums.wpcentral.com,SOURCE_CATEGORY_SOCIAL +fotki,SOURCE_CATEGORY_SOCIAL +fotki.com,SOURCE_CATEGORY_SOCIAL +fotolog,SOURCE_CATEGORY_SOCIAL +fotolog.com,SOURCE_CATEGORY_SOCIAL +foursquare,SOURCE_CATEGORY_SOCIAL +foursquare.com,SOURCE_CATEGORY_SOCIAL +fr.search.yahoo.com,SOURCE_CATEGORY_SEARCH +free.facebook.com,SOURCE_CATEGORY_SOCIAL +friendfeed,SOURCE_CATEGORY_SOCIAL +friendfeed.com,SOURCE_CATEGORY_SOCIAL +fruehstueckstreff.org,SOURCE_CATEGORY_SOCIAL +fubar,SOURCE_CATEGORY_SOCIAL +fubar.com,SOURCE_CATEGORY_SOCIAL +gaiaonline,SOURCE_CATEGORY_SOCIAL +gaiaonline.com,SOURCE_CATEGORY_SOCIAL +gamerdna,SOURCE_CATEGORY_SOCIAL +gamerdna.com,SOURCE_CATEGORY_SOCIAL +gather.com,SOURCE_CATEGORY_SOCIAL +geni.com,SOURCE_CATEGORY_SOCIAL +getpocket.com,SOURCE_CATEGORY_SOCIAL +glassboard,SOURCE_CATEGORY_SOCIAL +glassboard.com,SOURCE_CATEGORY_SOCIAL +glassdoor,SOURCE_CATEGORY_SOCIAL +glassdoor.com,SOURCE_CATEGORY_SOCIAL +globo,SOURCE_CATEGORY_SEARCH +go.mail.ru,SOURCE_CATEGORY_SEARCH +godtube,SOURCE_CATEGORY_SOCIAL +godtube.com,SOURCE_CATEGORY_SOCIAL +goldenline.pl,SOURCE_CATEGORY_SOCIAL +goldstar,SOURCE_CATEGORY_SOCIAL +goldstar.com,SOURCE_CATEGORY_SOCIAL +goo.gl,SOURCE_CATEGORY_SOCIAL +gooblog,SOURCE_CATEGORY_SOCIAL +goodreads,SOURCE_CATEGORY_SOCIAL +goodreads.com,SOURCE_CATEGORY_SOCIAL +google,SOURCE_CATEGORY_SEARCH +google+,SOURCE_CATEGORY_SOCIAL +google-play,SOURCE_CATEGORY_SEARCH +googlegroups.com,SOURCE_CATEGORY_SOCIAL +googleplus,SOURCE_CATEGORY_SOCIAL +govloop,SOURCE_CATEGORY_SOCIAL +govloop.com,SOURCE_CATEGORY_SOCIAL +gowalla,SOURCE_CATEGORY_SOCIAL +gowalla.com,SOURCE_CATEGORY_SOCIAL +gree.jp,SOURCE_CATEGORY_SOCIAL +groups.google.com,SOURCE_CATEGORY_SOCIAL +gulli.com,SOURCE_CATEGORY_SOCIAL +gutefrage.net,SOURCE_CATEGORY_SOCIAL +habbo,SOURCE_CATEGORY_SOCIAL +habbo.com,SOURCE_CATEGORY_SOCIAL +help.hulu.com,SOURCE_CATEGORY_VIDEO +help.netflix.com,SOURCE_CATEGORY_VIDEO +hi5,SOURCE_CATEGORY_SOCIAL +hi5.com,SOURCE_CATEGORY_SOCIAL +hk.search.yahoo.com,SOURCE_CATEGORY_SEARCH +hootsuite,SOURCE_CATEGORY_SOCIAL +hootsuite.com,SOURCE_CATEGORY_SOCIAL +houzz,SOURCE_CATEGORY_SOCIAL +houzz.com,SOURCE_CATEGORY_SOCIAL +hoverspot,SOURCE_CATEGORY_SOCIAL +hoverspot.com,SOURCE_CATEGORY_SOCIAL +hr.com,SOURCE_CATEGORY_SOCIAL +hu.pinterest.com,SOURCE_CATEGORY_SOCIAL +hubculture,SOURCE_CATEGORY_SOCIAL +hubculture.com,SOURCE_CATEGORY_SOCIAL +hubpages.com,SOURCE_CATEGORY_SOCIAL +hulu,SOURCE_CATEGORY_VIDEO +hulu.com,SOURCE_CATEGORY_VIDEO +hyves.net,SOURCE_CATEGORY_SOCIAL +hyves.nl,SOURCE_CATEGORY_SOCIAL +ibibo,SOURCE_CATEGORY_SOCIAL +ibibo.com,SOURCE_CATEGORY_SOCIAL +id.pinterest.com,SOURCE_CATEGORY_SOCIAL +id.search.yahoo.com,SOURCE_CATEGORY_SEARCH +id.twitch.tv,SOURCE_CATEGORY_VIDEO +identi.ca,SOURCE_CATEGORY_SOCIAL +ig,SOURCE_CATEGORY_SOCIAL +imageshack.com,SOURCE_CATEGORY_SOCIAL +imageshack.us,SOURCE_CATEGORY_SOCIAL +imvu,SOURCE_CATEGORY_SOCIAL +imvu.com,SOURCE_CATEGORY_SOCIAL +in.pinterest.com,SOURCE_CATEGORY_SOCIAL +in.search.yahoo.com,SOURCE_CATEGORY_SEARCH +incredimail,SOURCE_CATEGORY_SEARCH +insanejournal,SOURCE_CATEGORY_SOCIAL +insanejournal.com,SOURCE_CATEGORY_SOCIAL +instagram,SOURCE_CATEGORY_SOCIAL +instagram.com,SOURCE_CATEGORY_SOCIAL +instapaper,SOURCE_CATEGORY_SOCIAL +instapaper.com,SOURCE_CATEGORY_SOCIAL +internations.org,SOURCE_CATEGORY_SOCIAL +interpals.net,SOURCE_CATEGORY_SOCIAL +intherooms,SOURCE_CATEGORY_SOCIAL +intherooms.com,SOURCE_CATEGORY_SOCIAL +iq.com,SOURCE_CATEGORY_VIDEO +iqiyi,SOURCE_CATEGORY_VIDEO +iqiyi.com,SOURCE_CATEGORY_VIDEO +irc-galleria.net,SOURCE_CATEGORY_SOCIAL +is.gd,SOURCE_CATEGORY_SOCIAL +it.search.yahoo.com,SOURCE_CATEGORY_SEARCH +italki,SOURCE_CATEGORY_SOCIAL +italki.com,SOURCE_CATEGORY_SOCIAL +jammerdirect,SOURCE_CATEGORY_SOCIAL +jammerdirect.com,SOURCE_CATEGORY_SOCIAL +jappy.com,SOURCE_CATEGORY_SOCIAL +jappy.de,SOURCE_CATEGORY_SOCIAL +jobs.netflix.com,SOURCE_CATEGORY_VIDEO +justin.tv,SOURCE_CATEGORY_VIDEO +kaboodle.com,SOURCE_CATEGORY_SOCIAL +kakao,SOURCE_CATEGORY_SOCIAL +kakao.com,SOURCE_CATEGORY_SOCIAL +kakaocorp.com,SOURCE_CATEGORY_SOCIAL +kaneva,SOURCE_CATEGORY_SOCIAL +kaneva.com,SOURCE_CATEGORY_SOCIAL +kin.naver.com,SOURCE_CATEGORY_SOCIAL +kvasir,SOURCE_CATEGORY_SEARCH +l.facebook.com,SOURCE_CATEGORY_SOCIAL +l.instagram.com,SOURCE_CATEGORY_SOCIAL +l.messenger.com,SOURCE_CATEGORY_SOCIAL +last.fm,SOURCE_CATEGORY_SOCIAL +lens.google.com,SOURCE_CATEGORY_SEARCH +librarything,SOURCE_CATEGORY_SOCIAL +librarything.com,SOURCE_CATEGORY_SOCIAL +lifestream.aol.com,SOURCE_CATEGORY_SOCIAL +line,SOURCE_CATEGORY_SOCIAL +line.me,SOURCE_CATEGORY_SOCIAL +linkedin,SOURCE_CATEGORY_SOCIAL +linkedin.com,SOURCE_CATEGORY_SOCIAL +listal,SOURCE_CATEGORY_SOCIAL +listal.com,SOURCE_CATEGORY_SOCIAL +listography,SOURCE_CATEGORY_SOCIAL +listography.com,SOURCE_CATEGORY_SOCIAL +lite.qwant.com,SOURCE_CATEGORY_SEARCH +livedoor.com,SOURCE_CATEGORY_SOCIAL +livedoorblog,SOURCE_CATEGORY_SOCIAL +livejournal,SOURCE_CATEGORY_SOCIAL +livejournal.com,SOURCE_CATEGORY_SOCIAL +lm.facebook.com,SOURCE_CATEGORY_SOCIAL +lnkd.in,SOURCE_CATEGORY_SOCIAL +lycos,SOURCE_CATEGORY_SEARCH +m.alibaba.com,SOURCE_CATEGORY_SHOPPING +m.baidu.com,SOURCE_CATEGORY_SEARCH +m.blog.naver.com,SOURCE_CATEGORY_SOCIAL +m.cafe.naver.com,SOURCE_CATEGORY_SOCIAL +m.facebook.com,SOURCE_CATEGORY_SOCIAL +m.kin.naver.com,SOURCE_CATEGORY_SOCIAL +m.naver.com,SOURCE_CATEGORY_SEARCH +m.search.naver.com,SOURCE_CATEGORY_SEARCH +m.shopping.naver.com,SOURCE_CATEGORY_SHOPPING +m.sogou.com,SOURCE_CATEGORY_SEARCH +m.twitch.tv,SOURCE_CATEGORY_VIDEO +m.vk.com,SOURCE_CATEGORY_SOCIAL +m.yelp.com,SOURCE_CATEGORY_SOCIAL +m.youtube.com,SOURCE_CATEGORY_VIDEO +mail.rambler.ru,SOURCE_CATEGORY_SEARCH +mail.yandex.ru,SOURCE_CATEGORY_SEARCH +malaysia.search.yahoo.com,SOURCE_CATEGORY_SEARCH +mbga.jp,SOURCE_CATEGORY_SOCIAL +medium.com,SOURCE_CATEGORY_SOCIAL +meetin.org,SOURCE_CATEGORY_SOCIAL +meetup,SOURCE_CATEGORY_SOCIAL +meetup.com,SOURCE_CATEGORY_SOCIAL +meinvz.net,SOURCE_CATEGORY_SOCIAL +meneame.net,SOURCE_CATEGORY_SOCIAL +menuism.com,SOURCE_CATEGORY_SOCIAL +mercadolibre,SOURCE_CATEGORY_SHOPPING +mercadolibre.com,SOURCE_CATEGORY_SHOPPING +mercadolibre.com.ar,SOURCE_CATEGORY_SHOPPING +mercadolibre.com.mx,SOURCE_CATEGORY_SHOPPING +message.alibaba.com,SOURCE_CATEGORY_SHOPPING +messages.google.com,SOURCE_CATEGORY_SOCIAL +messages.yahoo.co.jp,SOURCE_CATEGORY_SOCIAL +messenger,SOURCE_CATEGORY_SOCIAL +messenger.com,SOURCE_CATEGORY_SOCIAL +mix.com,SOURCE_CATEGORY_SOCIAL +mixi.jp,SOURCE_CATEGORY_SOCIAL +mobile.facebook.com,SOURCE_CATEGORY_SOCIAL +mocospace,SOURCE_CATEGORY_SOCIAL +mocospace.com,SOURCE_CATEGORY_SOCIAL +mouthshut,SOURCE_CATEGORY_SOCIAL +mouthshut.com,SOURCE_CATEGORY_SOCIAL +movabletype,SOURCE_CATEGORY_SOCIAL +movabletype.com,SOURCE_CATEGORY_SOCIAL +msearch.shopping.naver.com,SOURCE_CATEGORY_SHOPPING +msn,SOURCE_CATEGORY_SEARCH +msn.com,SOURCE_CATEGORY_SEARCH +mubi,SOURCE_CATEGORY_SOCIAL +mubi.com,SOURCE_CATEGORY_SOCIAL +music.youtube.com,SOURCE_CATEGORY_VIDEO +mx.search.yahoo.com,SOURCE_CATEGORY_SEARCH +my.opera.com,SOURCE_CATEGORY_SOCIAL +myanimelist.net,SOURCE_CATEGORY_SOCIAL +myheritage,SOURCE_CATEGORY_SOCIAL +myheritage.com,SOURCE_CATEGORY_SOCIAL +mylife,SOURCE_CATEGORY_SOCIAL +mylife.com,SOURCE_CATEGORY_SOCIAL +mymodernmet,SOURCE_CATEGORY_SOCIAL +mymodernmet.com,SOURCE_CATEGORY_SOCIAL +myspace,SOURCE_CATEGORY_SOCIAL +myspace.com,SOURCE_CATEGORY_SOCIAL +najdi,SOURCE_CATEGORY_SEARCH +naver,SOURCE_CATEGORY_SEARCH +naver.com,SOURCE_CATEGORY_SEARCH +netflix,SOURCE_CATEGORY_VIDEO +netflix.com,SOURCE_CATEGORY_VIDEO +netvibes,SOURCE_CATEGORY_SOCIAL +netvibes.com,SOURCE_CATEGORY_SOCIAL +news.google.com,SOURCE_CATEGORY_SEARCH +news.ycombinator.com,SOURCE_CATEGORY_SOCIAL +newsshowcase,SOURCE_CATEGORY_SOCIAL +nexopia,SOURCE_CATEGORY_SOCIAL +ngopost.org,SOURCE_CATEGORY_SOCIAL +niconico,SOURCE_CATEGORY_SOCIAL +nicovideo.jp,SOURCE_CATEGORY_SOCIAL +nightlifelink,SOURCE_CATEGORY_SOCIAL +nightlifelink.com,SOURCE_CATEGORY_SOCIAL +ning,SOURCE_CATEGORY_SOCIAL +ning.com,SOURCE_CATEGORY_SOCIAL +nl.pinterest.com,SOURCE_CATEGORY_SOCIAL +nl.search.yahoo.com,SOURCE_CATEGORY_SEARCH +nl.shopping.net,SOURCE_CATEGORY_SHOPPING +no.search.yahoo.com,SOURCE_CATEGORY_SEARCH +no.shopping.net,SOURCE_CATEGORY_SHOPPING +ntp.msn.com,SOURCE_CATEGORY_SEARCH +nz.search.yahoo.com,SOURCE_CATEGORY_SEARCH +odnoklassniki.ru,SOURCE_CATEGORY_SOCIAL +odnoklassniki.ua,SOURCE_CATEGORY_SOCIAL +offer.alibaba.com,SOURCE_CATEGORY_SHOPPING +okwave.jp,SOURCE_CATEGORY_SOCIAL +old.reddit.com,SOURCE_CATEGORY_SOCIAL +one.walmart.com,SOURCE_CATEGORY_SHOPPING +onet,SOURCE_CATEGORY_SEARCH +onet.pl,SOURCE_CATEGORY_SEARCH +oneworldgroup.org,SOURCE_CATEGORY_SOCIAL +onstartups,SOURCE_CATEGORY_SOCIAL +onstartups.com,SOURCE_CATEGORY_SOCIAL +opendiary,SOURCE_CATEGORY_SOCIAL +opendiary.com,SOURCE_CATEGORY_SOCIAL +order.shopping.yahoo.co.jp,SOURCE_CATEGORY_SHOPPING +oshiete.goo.ne.jp,SOURCE_CATEGORY_SOCIAL +out.reddit.com,SOURCE_CATEGORY_SOCIAL +over-blog.com,SOURCE_CATEGORY_SOCIAL +overblog.com,SOURCE_CATEGORY_SOCIAL +paper.li,SOURCE_CATEGORY_SOCIAL +partners.shopify.com,SOURCE_CATEGORY_SHOPPING +partyflock.nl,SOURCE_CATEGORY_SOCIAL +pe.search.yahoo.com,SOURCE_CATEGORY_SEARCH +ph.search.yahoo.com,SOURCE_CATEGORY_SEARCH +photobucket,SOURCE_CATEGORY_SOCIAL +photobucket.com,SOURCE_CATEGORY_SOCIAL +pinboard,SOURCE_CATEGORY_SOCIAL +pinboard.in,SOURCE_CATEGORY_SOCIAL +pingsta,SOURCE_CATEGORY_SOCIAL +pingsta.com,SOURCE_CATEGORY_SOCIAL +pinterest,SOURCE_CATEGORY_SOCIAL +pinterest.at,SOURCE_CATEGORY_SOCIAL +pinterest.ca,SOURCE_CATEGORY_SOCIAL +pinterest.ch,SOURCE_CATEGORY_SOCIAL +pinterest.cl,SOURCE_CATEGORY_SOCIAL +pinterest.co.kr,SOURCE_CATEGORY_SOCIAL +pinterest.co.uk,SOURCE_CATEGORY_SOCIAL +pinterest.com,SOURCE_CATEGORY_SOCIAL +pinterest.com.au,SOURCE_CATEGORY_SOCIAL +pinterest.com.mx,SOURCE_CATEGORY_SOCIAL +pinterest.de,SOURCE_CATEGORY_SOCIAL +pinterest.es,SOURCE_CATEGORY_SOCIAL +pinterest.fr,SOURCE_CATEGORY_SOCIAL +pinterest.it,SOURCE_CATEGORY_SOCIAL +pinterest.jp,SOURCE_CATEGORY_SOCIAL +pinterest.nz,SOURCE_CATEGORY_SOCIAL +pinterest.ph,SOURCE_CATEGORY_SOCIAL +pinterest.pt,SOURCE_CATEGORY_SOCIAL +pinterest.ru,SOURCE_CATEGORY_SOCIAL +pinterest.se,SOURCE_CATEGORY_SOCIAL +pixiv.net,SOURCE_CATEGORY_SOCIAL +pl.pinterest.com,SOURCE_CATEGORY_SOCIAL +pl.search.yahoo.com,SOURCE_CATEGORY_SEARCH +play.google.com,SOURCE_CATEGORY_SEARCH +playahead.se,SOURCE_CATEGORY_SOCIAL +player.twitch.tv,SOURCE_CATEGORY_VIDEO +player.vimeo.com,SOURCE_CATEGORY_VIDEO +plurk,SOURCE_CATEGORY_SOCIAL +plurk.com,SOURCE_CATEGORY_SOCIAL +plus.google.com,SOURCE_CATEGORY_SOCIAL +plus.url.google.com,SOURCE_CATEGORY_SOCIAL +pocket.co,SOURCE_CATEGORY_SOCIAL +posterous,SOURCE_CATEGORY_SOCIAL +posterous.com,SOURCE_CATEGORY_SOCIAL +pro.homeadvisor.com,SOURCE_CATEGORY_SOCIAL +pulse.yahoo.com,SOURCE_CATEGORY_SOCIAL +qapacity,SOURCE_CATEGORY_SOCIAL +qapacity.com,SOURCE_CATEGORY_SOCIAL +quechup,SOURCE_CATEGORY_SOCIAL +quechup.com,SOURCE_CATEGORY_SOCIAL +quora,SOURCE_CATEGORY_SOCIAL +quora.com,SOURCE_CATEGORY_SOCIAL +qwant,SOURCE_CATEGORY_SEARCH +qwant.com,SOURCE_CATEGORY_SEARCH +qzone.qq.com,SOURCE_CATEGORY_SOCIAL +rakuten,SOURCE_CATEGORY_SEARCH +rakuten.co.jp,SOURCE_CATEGORY_SEARCH +rambler,SOURCE_CATEGORY_SEARCH +rambler.ru,SOURCE_CATEGORY_SEARCH +ravelry,SOURCE_CATEGORY_SOCIAL +ravelry.com,SOURCE_CATEGORY_SOCIAL +reddit,SOURCE_CATEGORY_SOCIAL +reddit.com,SOURCE_CATEGORY_SOCIAL +redux,SOURCE_CATEGORY_SOCIAL +redux.com,SOURCE_CATEGORY_SOCIAL +renren,SOURCE_CATEGORY_SOCIAL +renren.com,SOURCE_CATEGORY_SOCIAL +researchgate.net,SOURCE_CATEGORY_SOCIAL +reunion,SOURCE_CATEGORY_SOCIAL +reunion.com,SOURCE_CATEGORY_SOCIAL +reverbnation,SOURCE_CATEGORY_SOCIAL +reverbnation.com,SOURCE_CATEGORY_SOCIAL +rtl.de,SOURCE_CATEGORY_SOCIAL +ryze,SOURCE_CATEGORY_SOCIAL +ryze.com,SOURCE_CATEGORY_SOCIAL +s3.amazonaws.com,SOURCE_CATEGORY_SHOPPING +salespider,SOURCE_CATEGORY_SOCIAL +salespider.com,SOURCE_CATEGORY_SOCIAL +scoop.it,SOURCE_CATEGORY_SOCIAL +screenrant,SOURCE_CATEGORY_SOCIAL +screenrant.com,SOURCE_CATEGORY_SOCIAL +scribd,SOURCE_CATEGORY_SOCIAL +scribd.com,SOURCE_CATEGORY_SOCIAL +scvngr,SOURCE_CATEGORY_SOCIAL +scvngr.com,SOURCE_CATEGORY_SOCIAL +se.search.yahoo.com,SOURCE_CATEGORY_SEARCH +se.shopping.net,SOURCE_CATEGORY_SHOPPING +search-results,SOURCE_CATEGORY_SEARCH +search.aol.co.uk,SOURCE_CATEGORY_SEARCH +search.aol.com,SOURCE_CATEGORY_SEARCH +search.google.com,SOURCE_CATEGORY_SEARCH +search.smt.docomo.ne.jp,SOURCE_CATEGORY_SEARCH +search.ukr.net,SOURCE_CATEGORY_SEARCH +secondlife,SOURCE_CATEGORY_SOCIAL +secondlife.com,SOURCE_CATEGORY_SOCIAL +secureurl.ukr.net,SOURCE_CATEGORY_SEARCH +serverfault,SOURCE_CATEGORY_SOCIAL +serverfault.com,SOURCE_CATEGORY_SOCIAL +seznam,SOURCE_CATEGORY_SEARCH +seznam.cz,SOURCE_CATEGORY_SEARCH +sg.search.yahoo.com,SOURCE_CATEGORY_SEARCH +shareit,SOURCE_CATEGORY_SOCIAL +sharethis,SOURCE_CATEGORY_SOCIAL +sharethis.com,SOURCE_CATEGORY_SOCIAL +shop.app,SOURCE_CATEGORY_SHOPPING +shopify,SOURCE_CATEGORY_SHOPPING +shopify.com,SOURCE_CATEGORY_SHOPPING +shopping.naver.com,SOURCE_CATEGORY_SHOPPING +shopping.yahoo.co.jp,SOURCE_CATEGORY_SHOPPING +shopping.yahoo.com,SOURCE_CATEGORY_SHOPPING +shopzilla,SOURCE_CATEGORY_SHOPPING +shopzilla.com,SOURCE_CATEGORY_SHOPPING +shvoong.com,SOURCE_CATEGORY_SOCIAL +simplycodes.com,SOURCE_CATEGORY_SHOPPING +sites.google.com,SOURCE_CATEGORY_SOCIAL +skype,SOURCE_CATEGORY_SOCIAL +skyrock,SOURCE_CATEGORY_SOCIAL +skyrock.com,SOURCE_CATEGORY_SOCIAL +slashdot.org,SOURCE_CATEGORY_SOCIAL +slideshare.net,SOURCE_CATEGORY_SOCIAL +smartnews.com,SOURCE_CATEGORY_SOCIAL +snapchat,SOURCE_CATEGORY_SOCIAL +snapchat.com,SOURCE_CATEGORY_SOCIAL +so.com,SOURCE_CATEGORY_SEARCH +social,SOURCE_CATEGORY_SOCIAL +sociallife.com.br,SOURCE_CATEGORY_SOCIAL +socialvibe,SOURCE_CATEGORY_SOCIAL +socialvibe.com,SOURCE_CATEGORY_SOCIAL +sogou,SOURCE_CATEGORY_SEARCH +sogou.com,SOURCE_CATEGORY_SEARCH +sp-web.search.auone.jp,SOURCE_CATEGORY_SEARCH +spaces.live.com,SOURCE_CATEGORY_SOCIAL +spoke,SOURCE_CATEGORY_SOCIAL +spoke.com,SOURCE_CATEGORY_SOCIAL +spruz,SOURCE_CATEGORY_SOCIAL +spruz.com,SOURCE_CATEGORY_SOCIAL +ssense.com,SOURCE_CATEGORY_SOCIAL +stackapps,SOURCE_CATEGORY_SOCIAL +stackapps.com,SOURCE_CATEGORY_SOCIAL +stackexchange,SOURCE_CATEGORY_SOCIAL +stackexchange.com,SOURCE_CATEGORY_SOCIAL +stackoverflow,SOURCE_CATEGORY_SOCIAL +stackoverflow.com,SOURCE_CATEGORY_SOCIAL +stardoll.com,SOURCE_CATEGORY_SOCIAL +startsiden,SOURCE_CATEGORY_SEARCH +startsiden.no,SOURCE_CATEGORY_SEARCH +stickam,SOURCE_CATEGORY_SOCIAL +stickam.com,SOURCE_CATEGORY_SOCIAL +store.shopping.yahoo.co.jp,SOURCE_CATEGORY_SHOPPING +stripe,SOURCE_CATEGORY_SHOPPING +stripe.com,SOURCE_CATEGORY_SHOPPING +studivz.net,SOURCE_CATEGORY_SOCIAL +suche.aol.de,SOURCE_CATEGORY_SEARCH +suomi24.fi,SOURCE_CATEGORY_SOCIAL +superuser,SOURCE_CATEGORY_SOCIAL +superuser.com,SOURCE_CATEGORY_SOCIAL +sweeva,SOURCE_CATEGORY_SOCIAL +sweeva.com,SOURCE_CATEGORY_SOCIAL +t.co,SOURCE_CATEGORY_SOCIAL +t.me,SOURCE_CATEGORY_SOCIAL +tagged,SOURCE_CATEGORY_SOCIAL +tagged.com,SOURCE_CATEGORY_SOCIAL +taggedmail,SOURCE_CATEGORY_SOCIAL +taggedmail.com,SOURCE_CATEGORY_SOCIAL +talkbiznow,SOURCE_CATEGORY_SOCIAL +talkbiznow.com,SOURCE_CATEGORY_SOCIAL +taringa.net,SOURCE_CATEGORY_SOCIAL +techmeme,SOURCE_CATEGORY_SOCIAL +techmeme.com,SOURCE_CATEGORY_SOCIAL +ted,SOURCE_CATEGORY_VIDEO +ted.com,SOURCE_CATEGORY_VIDEO +tencent,SOURCE_CATEGORY_SOCIAL +tencent.com,SOURCE_CATEGORY_SOCIAL +terra,SOURCE_CATEGORY_SEARCH +th.search.yahoo.com,SOURCE_CATEGORY_SEARCH +tiktok,SOURCE_CATEGORY_SOCIAL +tiktok.com,SOURCE_CATEGORY_SOCIAL +tinyurl,SOURCE_CATEGORY_SOCIAL +tinyurl.com,SOURCE_CATEGORY_SOCIAL +toolbox,SOURCE_CATEGORY_SOCIAL +toolbox.com,SOURCE_CATEGORY_SOCIAL +touch.facebook.com,SOURCE_CATEGORY_SOCIAL +tr.pinterest.com,SOURCE_CATEGORY_SOCIAL +tr.search.yahoo.com,SOURCE_CATEGORY_SEARCH +travellerspoint,SOURCE_CATEGORY_SOCIAL +travellerspoint.com,SOURCE_CATEGORY_SOCIAL +tripadvisor,SOURCE_CATEGORY_SOCIAL +tripadvisor.com,SOURCE_CATEGORY_SOCIAL +trombi,SOURCE_CATEGORY_SOCIAL +trombi.com,SOURCE_CATEGORY_SOCIAL +trustpilot,SOURCE_CATEGORY_SOCIAL +tudou,SOURCE_CATEGORY_SOCIAL +tudou.com,SOURCE_CATEGORY_SOCIAL +tuenti,SOURCE_CATEGORY_SOCIAL +tuenti.com,SOURCE_CATEGORY_SOCIAL +tumblr,SOURCE_CATEGORY_SOCIAL +tumblr.com,SOURCE_CATEGORY_SOCIAL +tut.by,SOURCE_CATEGORY_SEARCH +tw.search.yahoo.com,SOURCE_CATEGORY_SEARCH +tweetdeck,SOURCE_CATEGORY_SOCIAL +tweetdeck.com,SOURCE_CATEGORY_SOCIAL +twitch,SOURCE_CATEGORY_VIDEO +twitch.tv,SOURCE_CATEGORY_VIDEO +twitter,SOURCE_CATEGORY_SOCIAL +twitter.com,SOURCE_CATEGORY_SOCIAL +twoo.com,SOURCE_CATEGORY_SOCIAL +typepad,SOURCE_CATEGORY_SOCIAL +typepad.com,SOURCE_CATEGORY_SOCIAL +uk.search.yahoo.com,SOURCE_CATEGORY_SEARCH +uk.shopping.net,SOURCE_CATEGORY_SHOPPING +ukr,SOURCE_CATEGORY_SEARCH +unblog.fr,SOURCE_CATEGORY_SOCIAL +urbanspoon.com,SOURCE_CATEGORY_SOCIAL +us.search.yahoo.com,SOURCE_CATEGORY_SEARCH +ushareit.com,SOURCE_CATEGORY_SOCIAL +ushi.cn,SOURCE_CATEGORY_SOCIAL +utreon,SOURCE_CATEGORY_VIDEO +utreon.com,SOURCE_CATEGORY_VIDEO +vampirefreaks,SOURCE_CATEGORY_SOCIAL +vampirefreaks.com,SOURCE_CATEGORY_SOCIAL +vampirerave,SOURCE_CATEGORY_SOCIAL +vampirerave.com,SOURCE_CATEGORY_SOCIAL +veoh,SOURCE_CATEGORY_VIDEO +veoh.com,SOURCE_CATEGORY_VIDEO +vg.no,SOURCE_CATEGORY_SOCIAL +viadeo.journaldunet.com,SOURCE_CATEGORY_VIDEO +video.ibm.com,SOURCE_CATEGORY_SOCIAL +vimeo,SOURCE_CATEGORY_VIDEO +vimeo.com,SOURCE_CATEGORY_VIDEO +virgilio,SOURCE_CATEGORY_SEARCH +vk.com,SOURCE_CATEGORY_SOCIAL +vkontakte.ru,SOURCE_CATEGORY_SOCIAL +vn.search.yahoo.com,SOURCE_CATEGORY_SEARCH +wakoopa,SOURCE_CATEGORY_SOCIAL +wakoopa.com,SOURCE_CATEGORY_SOCIAL +walmart,SOURCE_CATEGORY_SHOPPING +walmart.com,SOURCE_CATEGORY_SHOPPING +wap.sogou.com,SOURCE_CATEGORY_SEARCH +wattpad,SOURCE_CATEGORY_SOCIAL +wattpad.com,SOURCE_CATEGORY_SOCIAL +web.facebook.com,SOURCE_CATEGORY_SOCIAL +web.skype.com,SOURCE_CATEGORY_SOCIAL +webmaster.yandex.ru,SOURCE_CATEGORY_SEARCH +websearch.rakuten.co.jp,SOURCE_CATEGORY_SEARCH +webshots,SOURCE_CATEGORY_SOCIAL +webshots.com,SOURCE_CATEGORY_SOCIAL +wechat,SOURCE_CATEGORY_SOCIAL +wechat.com,SOURCE_CATEGORY_SOCIAL +weebly,SOURCE_CATEGORY_SOCIAL +weebly.com,SOURCE_CATEGORY_SOCIAL +weibo,SOURCE_CATEGORY_SOCIAL +weibo.com,SOURCE_CATEGORY_SOCIAL +wer-weiss-was.de,SOURCE_CATEGORY_SOCIAL +weread,SOURCE_CATEGORY_SOCIAL +weread.com,SOURCE_CATEGORY_SOCIAL +whatsapp,SOURCE_CATEGORY_SOCIAL +whatsapp.com,SOURCE_CATEGORY_SOCIAL +wiki.answers.com,SOURCE_CATEGORY_SOCIAL +wikihow.com,SOURCE_CATEGORY_SOCIAL +wikitravel.org,SOURCE_CATEGORY_SOCIAL +wistia,SOURCE_CATEGORY_VIDEO +wistia.com,SOURCE_CATEGORY_VIDEO +woot.com,SOURCE_CATEGORY_SOCIAL +wordpress,SOURCE_CATEGORY_SOCIAL +wordpress.com,SOURCE_CATEGORY_SOCIAL +wordpress.org,SOURCE_CATEGORY_SOCIAL +xanga,SOURCE_CATEGORY_SOCIAL +xanga.com,SOURCE_CATEGORY_SOCIAL +xing,SOURCE_CATEGORY_SOCIAL +xing.com,SOURCE_CATEGORY_SOCIAL +yahoo,SOURCE_CATEGORY_SEARCH +yahoo-mbga.jp,SOURCE_CATEGORY_SOCIAL +yahoo.co.jp,SOURCE_CATEGORY_SEARCH +yahoo.com,SOURCE_CATEGORY_SEARCH +yammer,SOURCE_CATEGORY_SOCIAL +yammer.com,SOURCE_CATEGORY_SOCIAL +yandex,SOURCE_CATEGORY_SEARCH +yandex.by,SOURCE_CATEGORY_SEARCH +yandex.com,SOURCE_CATEGORY_SEARCH +yandex.com.tr,SOURCE_CATEGORY_SEARCH +yandex.fr,SOURCE_CATEGORY_SEARCH +yandex.kz,SOURCE_CATEGORY_SEARCH +yandex.ru,SOURCE_CATEGORY_SEARCH +yandex.ua,SOURCE_CATEGORY_SEARCH +yandex.uz,SOURCE_CATEGORY_SEARCH +yelp,SOURCE_CATEGORY_SOCIAL +yelp.co.uk,SOURCE_CATEGORY_SOCIAL +yelp.com,SOURCE_CATEGORY_SOCIAL +youku,SOURCE_CATEGORY_VIDEO +youku.com,SOURCE_CATEGORY_VIDEO +youroom.in,SOURCE_CATEGORY_SOCIAL +youtube,SOURCE_CATEGORY_VIDEO +youtube.com,SOURCE_CATEGORY_VIDEO +za.pinterest.com,SOURCE_CATEGORY_SOCIAL +zalo,SOURCE_CATEGORY_SOCIAL +zen.yandex.ru,SOURCE_CATEGORY_SEARCH +zoo.gr,SOURCE_CATEGORY_SOCIAL +zooppa,SOURCE_CATEGORY_SOCIAL +zooppa.com,SOURCE_CATEGORY_SOCIAL diff --git a/priv/globe_favicon.svg b/priv/globe_favicon.svg new file mode 100644 index 0000000000000000000000000000000000000000..7508112411a2d715334c5d03cfd1249994b47472 --- /dev/null +++ b/priv/globe_favicon.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/priv/ingest_repo/migrations/.formatter.exs b/priv/ingest_repo/migrations/.formatter.exs new file mode 100644 index 0000000000000000000000000000000000000000..49f9151ed22c4d59136609119e01e3a8f75ab6d6 --- /dev/null +++ b/priv/ingest_repo/migrations/.formatter.exs @@ -0,0 +1,4 @@ +[ + import_deps: [:ecto_sql], + inputs: ["*.exs"] +] diff --git a/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs new file mode 100644 index 0000000000000000000000000000000000000000..527612827a7ecd8456d61ee8b0678b55a6e7f546 --- /dev/null +++ b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs @@ -0,0 +1,68 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions do + use Ecto.Migration + + def up do + create_events() + create_sessions() + end + + defp create_events() do + create_if_not_exists table(:events, + primary_key: false, + engine: "MergeTree", + options: """ + PARTITION BY toYYYYMM(timestamp) + ORDER BY (domain, toDate(timestamp), user_id) + #{Plausible.MigrationUtils.table_settings_expr()} + """ + ) do + add(:name, :string) + add(:domain, :string) + add(:user_id, :UInt64) + add(:session_id, :UInt64) + add(:hostname, :string) + add(:pathname, :string) + add(:referrer, :string) + add(:referrer_source, :string) + add(:country_code, :"LowCardinality(FixedString(2))") + add(:screen_size, :"LowCardinality(String)") + add(:operating_system, :"LowCardinality(String)") + add(:browser, :"LowCardinality(String)") + + add(:timestamp, :naive_datetime) + end + end + + defp create_sessions() do + create_if_not_exists table(:sessions, + primary_key: false, + engine: "CollapsingMergeTree(sign)", + options: """ + PARTITION BY toYYYYMM(start) + ORDER BY (domain, toDate(start), user_id, session_id) + #{Plausible.MigrationUtils.table_settings_expr()} + """ + ) do + add(:session_id, :UInt64) + add(:sign, :Int8) + add(:domain, :string) + add(:user_id, :UInt64) + add(:hostname, :string) + add(:is_bounce, :UInt8) + add(:entry_page, :string) + add(:exit_page, :string) + add(:pageviews, :integer) + add(:events, :integer) + add(:duration, :UInt32) + add(:referrer, :string) + add(:referrer_source, :string) + add(:country_code, :"LowCardinality(FixedString(2))") + add(:screen_size, :"LowCardinality(String)") + add(:operating_system, :"LowCardinality(String)") + add(:browser, :"LowCardinality(String)") + + add(:start, :naive_datetime) + add(:timestamp, :naive_datetime) + end + end +end diff --git a/priv/ingest_repo/migrations/20200918075025_add_utm_tags.exs b/priv/ingest_repo/migrations/20200918075025_add_utm_tags.exs new file mode 100644 index 0000000000000000000000000000000000000000..eedd84d1068dffaacfd3a976154410ad4f7dacd0 --- /dev/null +++ b/priv/ingest_repo/migrations/20200918075025_add_utm_tags.exs @@ -0,0 +1,17 @@ +defmodule Plausible.ClickhouseRepo.Migrations.AddUtmTags do + use Ecto.Migration + + def change do + alter table(:events) do + add :utm_medium, :string + add :utm_source, :string + add :utm_campaign, :string + end + + alter table(:sessions) do + add :utm_medium, :string + add :utm_source, :string + add :utm_campaign, :string + end + end +end diff --git a/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs b/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs new file mode 100644 index 0000000000000000000000000000000000000000..4c1b6d44ada4de7492a8ac1a5779bf00537215be --- /dev/null +++ b/priv/ingest_repo/migrations/20201020083739_add_event_metadata.exs @@ -0,0 +1,9 @@ +defmodule Plausible.ClickhouseRepo.Migrations.AddEventMetadata do + use Ecto.Migration + + def change do + alter table(:events) do + add :meta, :"Nested(key String, value String)" + end + end +end diff --git a/priv/ingest_repo/migrations/20201106125234_add_browser_version_and_os_version.exs b/priv/ingest_repo/migrations/20201106125234_add_browser_version_and_os_version.exs new file mode 100644 index 0000000000000000000000000000000000000000..b9ff82081f14b4e1f4f80e467ae9df88c4df0853 --- /dev/null +++ b/priv/ingest_repo/migrations/20201106125234_add_browser_version_and_os_version.exs @@ -0,0 +1,15 @@ +defmodule Plausible.ClickhouseRepo.Migrations.AddBrowserVersionAndOsVersion do + use Ecto.Migration + + def change do + alter table(:events) do + add :browser_version, :"LowCardinality(String)" + add :operating_system_version, :"LowCardinality(String)" + end + + alter table(:sessions) do + add :browser_version, :"LowCardinality(String)" + add :operating_system_version, :"LowCardinality(String)" + end + end +end diff --git a/priv/ingest_repo/migrations/20210323130440_add_sample_by.exs b/priv/ingest_repo/migrations/20210323130440_add_sample_by.exs new file mode 100644 index 0000000000000000000000000000000000000000..1a547ad9545573a811652907902acd298178d100 --- /dev/null +++ b/priv/ingest_repo/migrations/20210323130440_add_sample_by.exs @@ -0,0 +1,8 @@ +defmodule Plausible.ClickhouseRepo.Migrations.AddSampleBy do + use Ecto.Migration + + def change do + execute "ALTER TABLE events MODIFY SAMPLE BY user_id" + execute "ALTER TABLE sessions MODIFY SAMPLE BY user_id" + end +end diff --git a/priv/ingest_repo/migrations/20210712214034_add_more_location_details.exs b/priv/ingest_repo/migrations/20210712214034_add_more_location_details.exs new file mode 100644 index 0000000000000000000000000000000000000000..1127c5ca2975724f6cf22fbf0ab8664eba8907fb --- /dev/null +++ b/priv/ingest_repo/migrations/20210712214034_add_more_location_details.exs @@ -0,0 +1,17 @@ +defmodule Plausible.ClickhouseRepo.Migrations.AddMoreLocationDetails do + use Ecto.Migration + + def change do + alter table(:events) do + add(:subdivision1_code, :"LowCardinality(String)") + add(:subdivision2_code, :"LowCardinality(String)") + add(:city_geoname_id, :UInt32) + end + + alter table(:sessions) do + add(:subdivision1_code, :"LowCardinality(String)") + add(:subdivision2_code, :"LowCardinality(String)") + add(:city_geoname_id, :UInt32) + end + end +end diff --git a/priv/ingest_repo/migrations/20211017093035_add_utm_content_and_term.exs b/priv/ingest_repo/migrations/20211017093035_add_utm_content_and_term.exs new file mode 100644 index 0000000000000000000000000000000000000000..bd7f0715838de4d31aecb8a0676b51a8f4979886 --- /dev/null +++ b/priv/ingest_repo/migrations/20211017093035_add_utm_content_and_term.exs @@ -0,0 +1,15 @@ +defmodule Plausible.ClickhouseRepo.Migrations.AddUtmContentAndTerm do + use Ecto.Migration + + def change do + alter table(:events) do + add :utm_content, :string + add :utm_term, :string + end + + alter table(:sessions) do + add :utm_content, :string + add :utm_term, :string + end + end +end diff --git a/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs new file mode 100644 index 0000000000000000000000000000000000000000..cd1d827db5987d82f7494f48b919ca20c5b58f45 --- /dev/null +++ b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs @@ -0,0 +1,163 @@ +defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do + use Ecto.Migration + + def change do + create_if_not_exists table(:imported_visitors, + primary_key: false, + engine: "MergeTree", + options: """ + ORDER BY (site_id, date) + #{Plausible.MigrationUtils.table_settings_expr()} + """ + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:visitors, :UInt64) + add(:pageviews, :UInt64) + add(:bounces, :UInt64) + add(:visits, :UInt64) + add(:visit_duration, :UInt64) + end + + create_if_not_exists table(:imported_sources, + primary_key: false, + engine: "MergeTree", + options: """ + ORDER BY (site_id, date, source) + #{Plausible.MigrationUtils.table_settings_expr()} + """ + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:source, :string) + add(:utm_medium, :string) + add(:utm_campaign, :string) + add(:utm_content, :string) + add(:utm_term, :string) + add(:visitors, :UInt64) + add(:visits, :UInt64) + add(:visit_duration, :UInt64) + add(:bounces, :UInt32) + end + + create_if_not_exists table(:imported_pages, + primary_key: false, + engine: "MergeTree", + options: """ + ORDER BY (site_id, date, hostname, page) + #{Plausible.MigrationUtils.table_settings_expr()} + """ + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:hostname, :string) + add(:page, :string) + add(:visitors, :UInt64) + add(:pageviews, :UInt64) + add(:exits, :UInt64) + add(:time_on_page, :UInt64) + end + + create_if_not_exists table(:imported_entry_pages, + primary_key: false, + engine: "MergeTree", + options: """ + ORDER BY (site_id, date, entry_page) + #{Plausible.MigrationUtils.table_settings_expr()} + """ + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:entry_page, :string) + add(:visitors, :UInt64) + add(:entrances, :UInt64) + add(:visit_duration, :UInt64) + add(:bounces, :UInt32) + end + + create_if_not_exists table(:imported_exit_pages, + primary_key: false, + engine: "MergeTree", + options: """ + ORDER BY (site_id, date, exit_page) + #{Plausible.MigrationUtils.table_settings_expr()} + """ + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:exit_page, :string) + add(:visitors, :UInt64) + add(:exits, :UInt64) + end + + create_if_not_exists table(:imported_locations, + primary_key: false, + engine: "MergeTree", + options: """ + ORDER BY (site_id, date, country, region, city) + #{Plausible.MigrationUtils.table_settings_expr()} + """ + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:country, :string) + add(:region, :string) + add(:city, :UInt64) + add(:visitors, :UInt64) + add(:visits, :UInt64) + add(:visit_duration, :UInt64) + add(:bounces, :UInt32) + end + + create_if_not_exists table(:imported_devices, + primary_key: false, + engine: "MergeTree", + options: """ + ORDER BY (site_id, date, device) + #{Plausible.MigrationUtils.table_settings_expr()} + """ + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:device, :string) + add(:visitors, :UInt64) + add(:visits, :UInt64) + add(:visit_duration, :UInt64) + add(:bounces, :UInt32) + end + + create_if_not_exists table(:imported_browsers, + primary_key: false, + engine: "MergeTree", + options: """ + ORDER BY (site_id, date, browser) + #{Plausible.MigrationUtils.table_settings_expr()} + """ + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:browser, :string) + add(:visitors, :UInt64) + add(:visits, :UInt64) + add(:visit_duration, :UInt64) + add(:bounces, :UInt32) + end + + create_if_not_exists table(:imported_operating_systems, + primary_key: false, + engine: "MergeTree", + options: """ + ORDER BY (site_id, date, operating_system) + #{Plausible.MigrationUtils.table_settings_expr()} + """ + ) do + add(:site_id, :UInt64) + add(:date, :date) + add(:operating_system, :string) + add(:visitors, :UInt64) + add(:visits, :UInt64) + add(:visit_duration, :UInt64) + add(:bounces, :UInt32) + end + end +end diff --git a/priv/ingest_repo/migrations/20220310104931_add_transferred_from.exs b/priv/ingest_repo/migrations/20220310104931_add_transferred_from.exs new file mode 100644 index 0000000000000000000000000000000000000000..0e72288c7ee9891a3b2fa2ab93a8f861d029a6cd --- /dev/null +++ b/priv/ingest_repo/migrations/20220310104931_add_transferred_from.exs @@ -0,0 +1,13 @@ +defmodule Plausible.ClickhouseRepo.Migrations.AddTransferredFrom do + use Ecto.Migration + + def change do + alter table(:events) do + add(:transferred_from, :string) + end + + alter table(:sessions) do + add(:transferred_from, :string) + end + end +end diff --git a/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs b/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs new file mode 100644 index 0000000000000000000000000000000000000000..2c80ad8d684005b8196db82f99291c1933c1c037 --- /dev/null +++ b/priv/ingest_repo/migrations/20220404123000_add_entry_props_to_session.exs @@ -0,0 +1,9 @@ +defmodule Plausible.ClickhouseRepo.Migrations.AddEntryPropsToSession do + use Ecto.Migration + + def change do + alter table(:sessions) do + add :"entry.meta", :"Nested(key String, value String)" + end + end +end diff --git a/priv/ingest_repo/migrations/20220421161259_remove_entry_props.exs b/priv/ingest_repo/migrations/20220421161259_remove_entry_props.exs new file mode 100644 index 0000000000000000000000000000000000000000..26456213aada38f1844dd5e0527c4e15c0ffa606 --- /dev/null +++ b/priv/ingest_repo/migrations/20220421161259_remove_entry_props.exs @@ -0,0 +1,9 @@ +defmodule Plausible.ClickhouseRepo.Migrations.RemoveEntryProps do + use Ecto.Migration + + def change do + alter table(:sessions) do + remove(:"entry.meta") + end + end +end diff --git a/priv/ingest_repo/migrations/20220422075510_add_entry_props.exs b/priv/ingest_repo/migrations/20220422075510_add_entry_props.exs new file mode 100644 index 0000000000000000000000000000000000000000..ce45f6e3e6da2b1a51fd4809a4bac58883acf99e --- /dev/null +++ b/priv/ingest_repo/migrations/20220422075510_add_entry_props.exs @@ -0,0 +1,9 @@ +defmodule Plausible.ClickhouseRepo.Migrations.AddEntryProps do + use Ecto.Migration + + def change do + alter table(:sessions) do + add(:entry_meta, :"Nested(key String, value String)") + end + end +end diff --git a/priv/ingest_repo/migrations/20230124140348_add_city_name_to_imported_locations.exs b/priv/ingest_repo/migrations/20230124140348_add_city_name_to_imported_locations.exs new file mode 100644 index 0000000000000000000000000000000000000000..58a649a7eda88d84ce0a97f62d0d5af83613fd98 --- /dev/null +++ b/priv/ingest_repo/migrations/20230124140348_add_city_name_to_imported_locations.exs @@ -0,0 +1,9 @@ +defmodule Plausible.ClickhouseRepo.Migrations.AddCityNameToImportedLocations do + use Ecto.Migration + + def change do + alter table(:imported_locations) do + add :city_name, :string + end + end +end diff --git a/priv/ingest_repo/migrations/20230210140348_remove_city_name_to_imported_locations.exs b/priv/ingest_repo/migrations/20230210140348_remove_city_name_to_imported_locations.exs new file mode 100644 index 0000000000000000000000000000000000000000..f2ab335d75650ce961bcd9ad28a4939faaeae8cb --- /dev/null +++ b/priv/ingest_repo/migrations/20230210140348_remove_city_name_to_imported_locations.exs @@ -0,0 +1,9 @@ +defmodule Plausible.ClickhouseRepo.Migrations.RemoveCityNameToImportedLocations do + use Ecto.Migration + + def change do + alter table(:imported_locations) do + remove :city_name + end + end +end diff --git a/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs b/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs new file mode 100644 index 0000000000000000000000000000000000000000..d1e2b4a9f19cecbc24a07960fc7988dacef82517 --- /dev/null +++ b/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs @@ -0,0 +1,20 @@ +defmodule Plausible.IngestRepo.Migrations.CreateIngestCountersTable do + use Ecto.Migration + + def change do + create_if_not_exists table(:ingest_counters, + primary_key: false, + engine: "SummingMergeTree(value)", + options: """ + ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket)) + #{Plausible.MigrationUtils.table_settings_expr()} + """ + ) do + add(:event_timebucket, :utc_datetime) + add(:domain, :"LowCardinality(String)") + add(:site_id, :"Nullable(UInt64)") + add(:metric, :"LowCardinality(String)") + add(:value, :UInt64) + end + end +end diff --git a/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs new file mode 100644 index 0000000000000000000000000000000000000000..00de3b0c3d734dde5049576041b7bbcc141400fb --- /dev/null +++ b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs @@ -0,0 +1,28 @@ +defmodule Plausible.IngestRepo.Migrations.CreateV2Schemas do + @moduledoc """ + Normally, for live environments the migration will be done via + `DataMigration.NumericIDs` module. + + For tests, and entirely new small, self-hosted instances however, + we want to keep the ability of preparing the database without enforcing + any _data_ migration. + """ + + use Ecto.Migration + + use Plausible.DataMigration, dir: "NumericIDs" + + @cluster? false + + def up do + table_settings = Plausible.MigrationUtils.table_settings_expr() + + execute unwrap("create-events-v2", table_settings: table_settings, cluster?: @cluster?) + execute unwrap("create-sessions-v2", table_settings: table_settings, cluster?: @cluster?) + end + + def down do + execute unwrap("drop-events-v2", cluster?: @cluster?) + execute unwrap("drop-sessions-v2", cluster?: @cluster?) + end +end diff --git a/priv/ingest_repo/migrations/20230417104025_add_revenue_to_events.exs b/priv/ingest_repo/migrations/20230417104025_add_revenue_to_events.exs new file mode 100644 index 0000000000000000000000000000000000000000..b329714aa2287a97aeeee5073274174d0967681e --- /dev/null +++ b/priv/ingest_repo/migrations/20230417104025_add_revenue_to_events.exs @@ -0,0 +1,13 @@ +defmodule Plausible.IngestRepo.Migrations.AddRevenueToEvents do + use Ecto.Migration + + def change do + alter table(:events_v2) do + add :revenue_reporting_amount, :"Nullable(Decimal64(3))" + add :revenue_reporting_currency, :"FixedString(3)" + + add :revenue_source_amount, :"Nullable(Decimal64(3))" + add :revenue_source_currency, :"FixedString(3)" + end + end +end diff --git a/priv/ingest_repo/migrations/20230509124919_clean_up_old_tables_after_v2_migration.exs b/priv/ingest_repo/migrations/20230509124919_clean_up_old_tables_after_v2_migration.exs new file mode 100644 index 0000000000000000000000000000000000000000..7b2456030ef50239c77f4411d87aeee37960e121 --- /dev/null +++ b/priv/ingest_repo/migrations/20230509124919_clean_up_old_tables_after_v2_migration.exs @@ -0,0 +1,12 @@ +defmodule Plausible.IngestRepo.Migrations.CleanUpOldTablesAfterV2Migration do + use Ecto.Migration + + import Plausible.MigrationUtils + + def change do + unless community_edition?() do + drop_if_exists table(:events) + drop_if_exists table(:sessions) + end + end +end diff --git a/priv/ingest_repo/migrations/20231017073642_disable_deduplication_window_for_imports.exs b/priv/ingest_repo/migrations/20231017073642_disable_deduplication_window_for_imports.exs new file mode 100644 index 0000000000000000000000000000000000000000..853cd4988f76c492325f910832af096b1d9ea836 --- /dev/null +++ b/priv/ingest_repo/migrations/20231017073642_disable_deduplication_window_for_imports.exs @@ -0,0 +1,29 @@ +defmodule Plausible.IngestRepo.Migrations.DisableDeduplicationWindowForImports do + use Ecto.Migration + + @import_tables ~w( + imported_visitors + imported_sources + imported_pages + imported_entry_pages + imported_exit_pages + imported_locations + imported_devices + imported_browsers + imported_operating_systems + ) + + def up do + on_cluster = Plausible.MigrationUtils.on_cluster_statement("imported_visitors") + + for table <- @import_tables do + execute """ + ALTER TABLE #{table} #{on_cluster} MODIFY SETTING replicated_deduplication_window = 0 + """ + end + end + + def down do + raise "Irreversible" + end +end diff --git a/priv/ingest_repo/migrations/20240123142959_add_import_id_to_imported_tables.exs b/priv/ingest_repo/migrations/20240123142959_add_import_id_to_imported_tables.exs new file mode 100644 index 0000000000000000000000000000000000000000..f1a696f6e5d8b1720021f90e0cbd6a17a6b62e58 --- /dev/null +++ b/priv/ingest_repo/migrations/20240123142959_add_import_id_to_imported_tables.exs @@ -0,0 +1,23 @@ +defmodule Plausible.IngestRepo.Migrations.AddImportIdToImportedTables do + use Ecto.Migration + + @imported_tables [ + :imported_browsers, + :imported_devices, + :imported_entry_pages, + :imported_exit_pages, + :imported_locations, + :imported_operating_systems, + :imported_pages, + :imported_sources, + :imported_visitors + ] + + def change do + for table <- @imported_tables do + alter table(table) do + add(:import_id, :UInt64) + end + end + end +end diff --git a/priv/ingest_repo/migrations/20240209085338_minmax_index_session_timestamp.exs b/priv/ingest_repo/migrations/20240209085338_minmax_index_session_timestamp.exs new file mode 100644 index 0000000000000000000000000000000000000000..f7c2a8d4a7fead51c9dac32a2049d710764075fa --- /dev/null +++ b/priv/ingest_repo/migrations/20240209085338_minmax_index_session_timestamp.exs @@ -0,0 +1,25 @@ +defmodule Plausible.IngestRepo.Migrations.MinmaxIndexSessionTimestamp do + use Ecto.Migration + + def up do + execute """ + ALTER TABLE sessions_v2 + #{Plausible.MigrationUtils.on_cluster_statement("sessions_v2")} + ADD INDEX IF NOT EXISTS minmax_timestamp timestamp + TYPE minmax GRANULARITY 1 + """ + + execute """ + ALTER TABLE sessions_v2 + MATERIALIZE INDEX minmax_timestamp + """ + end + + def down do + execute """ + ALTER TABLE sessions_v2 + #{Plausible.MigrationUtils.on_cluster_statement("sessions_v2")} + DROP INDEX IF EXISTS minmax_timestamp + """ + end +end diff --git a/priv/ingest_repo/migrations/20240220123656_create_sessions_events_compression_options.exs b/priv/ingest_repo/migrations/20240220123656_create_sessions_events_compression_options.exs new file mode 100644 index 0000000000000000000000000000000000000000..8f192df78ac25b3f366ba00240ae5fb7ad2ed241 --- /dev/null +++ b/priv/ingest_repo/migrations/20240220123656_create_sessions_events_compression_options.exs @@ -0,0 +1,45 @@ +defmodule Plausible.IngestRepo.Migrations.CreateSessionsEventsCompressionOptions do + @moduledoc """ + Improves compression options for events_v2 and sessions_v2 tables. + + It's suggested to run OPTIMIZE both tables after this to get full compression wins afterwards + """ + + use Ecto.Migration + + @sessions_codecs %{ + timestamp: "Codec(Delta(4), LZ4)", + start: "Codec(Delta(4), LZ4)", + entry_page: "Codec(ZSTD(3))", + exit_page: "Codec(ZSTD(3))", + hostname: "Codec(ZSTD(3))", + referrer: "Codec(ZSTD(3))", + referrer_source: "Codec(ZSTD(3))", + utm_medium: "Codec(ZSTD(3))", + utm_source: "Codec(ZSTD(3))", + utm_campaign: "Codec(ZSTD(3))", + utm_content: "Codec(ZSTD(3))", + utm_term: "Codec(ZSTD(3))", + "entry_meta.key": "Codec(ZSTD(3))", + "entry_meta.value": "Codec(ZSTD(3))" + } + + @events_codecs %{ + hostname: "Codec(ZSTD(3))", + "meta.key": "Codec(ZSTD(3))", + "meta.value": "Codec(ZSTD(3))" + } + + def up do + for {column, codec} <- @sessions_codecs do + execute "ALTER TABLE sessions_v2 MODIFY COLUMN #{column} #{codec}" + end + + for {column, codec} <- @events_codecs do + execute "ALTER TABLE events_v2 MODIFY COLUMN #{column} #{codec}" + end + end + + # No need to explicitly revert + def down, do: nil +end diff --git a/priv/ingest_repo/migrations/20240222082911_sessions_v2_versioned_collapsing_merge_tree.exs b/priv/ingest_repo/migrations/20240222082911_sessions_v2_versioned_collapsing_merge_tree.exs new file mode 100644 index 0000000000000000000000000000000000000000..6047eff09195070180f54ad55f535626459fedcd --- /dev/null +++ b/priv/ingest_repo/migrations/20240222082911_sessions_v2_versioned_collapsing_merge_tree.exs @@ -0,0 +1,13 @@ +defmodule Plausible.IngestRepo.Migrations.SessionsV2VersionedCollapsingMergeTree do + use Ecto.Migration + + def up do + Plausible.DataMigration.VersionedSessions.run(run_exchange?: true) + + # After this migration a `sessions_v2_tmp_versioned` backup table is left behind, to be cleaned up manually + end + + def down do + raise "Irreversible" + end +end diff --git a/priv/ingest_repo/migrations/20240305085310_events_sessions_columns_improved.exs b/priv/ingest_repo/migrations/20240305085310_events_sessions_columns_improved.exs new file mode 100644 index 0000000000000000000000000000000000000000..3ddc9318da57194cbf1841c3565aed9470849604 --- /dev/null +++ b/priv/ingest_repo/migrations/20240305085310_events_sessions_columns_improved.exs @@ -0,0 +1,25 @@ +defmodule Plausible.IngestRepo.Migrations.EventsSessionsColumnsImproved do + use Ecto.Migration + + @zstd_columns [ + :referrer, + :referrer_source, + :utm_medium, + :utm_source, + :utm_campaign, + :utm_content, + :utm_term + ] + + def up do + for column <- @zstd_columns do + execute "ALTER TABLE events_v2 MODIFY COLUMN #{column} Codec(ZSTD(3))" + end + + execute "ALTER TABLE events_v2 DROP COLUMN IF EXISTS transferred_from" + end + + def down do + raise "Irreversible" + end +end diff --git a/priv/ingest_repo/migrations/20240326134840_add_metrics_to_imported_tables.exs b/priv/ingest_repo/migrations/20240326134840_add_metrics_to_imported_tables.exs new file mode 100644 index 0000000000000000000000000000000000000000..bb076dae4dd9093620a356be3d3253a19165d732 --- /dev/null +++ b/priv/ingest_repo/migrations/20240326134840_add_metrics_to_imported_tables.exs @@ -0,0 +1,35 @@ +defmodule Plausible.IngestRepo.Migrations.AddMetricsToImportedTables do + use Ecto.Migration + + @add_pageviews_for [ + "imported_browsers", + "imported_devices", + "imported_entry_pages", + "imported_exit_pages", + "imported_locations", + "imported_operating_systems", + "imported_sources" + ] + + def up do + for table <- @add_pageviews_for do + add_column(table, "pageviews", "UInt64") + end + + add_column("imported_browsers", "browser_version", "String") + add_column("imported_exit_pages", "bounces", "UInt32") + add_column("imported_exit_pages", "visit_duration", "UInt64") + add_column("imported_operating_systems", "operating_system_version", "String") + add_column("imported_pages", "visits", "UInt64") + add_column("imported_sources", "referrer", "String") + add_column("imported_sources", "utm_source", "String") + end + + defp add_column(table, column, type) do + execute """ + ALTER TABLE #{table} + #{Plausible.MigrationUtils.on_cluster_statement(table)} + ADD COLUMN #{column} #{type} + """ + end +end diff --git a/priv/ingest_repo/migrations/20240327085855_hostnames_in_sessions.exs b/priv/ingest_repo/migrations/20240327085855_hostnames_in_sessions.exs new file mode 100644 index 0000000000000000000000000000000000000000..48c4ab28e5555ce672ffd3080cf95f720451d185 --- /dev/null +++ b/priv/ingest_repo/migrations/20240327085855_hostnames_in_sessions.exs @@ -0,0 +1,19 @@ +defmodule Plausible.ClickhouseRepo.Migrations.HostnamesInSessions do + use Ecto.Migration + + def up do + execute """ + ALTER TABLE sessions_v2 + #{Plausible.MigrationUtils.on_cluster_statement("sessions_v2")} + ADD COLUMN exit_page_hostname String CODEC(ZSTD(3)) + """ + end + + def down do + execute """ + ALTER TABLE sessions_v2 + #{Plausible.MigrationUtils.on_cluster_statement("sessions_v2")} + DROP COLUMN exit_page_hostname + """ + end +end diff --git a/priv/ingest_repo/migrations/20240419133926_add_active_visitors_to_imported_pages.exs b/priv/ingest_repo/migrations/20240419133926_add_active_visitors_to_imported_pages.exs new file mode 100644 index 0000000000000000000000000000000000000000..c4899bc82255720bf9d8082350594e1b7bc7ef85 --- /dev/null +++ b/priv/ingest_repo/migrations/20240419133926_add_active_visitors_to_imported_pages.exs @@ -0,0 +1,19 @@ +defmodule Plausible.IngestRepo.Migrations.AddActiveVisitorsToImportedPages do + use Ecto.Migration + + def up do + execute """ + ALTER TABLE imported_pages + #{Plausible.MigrationUtils.on_cluster_statement("imported_pages")} + ADD COLUMN active_visitors UInt64 + """ + end + + def down do + execute """ + ALTER TABLE imported_pages + #{Plausible.MigrationUtils.on_cluster_statement("imported_pages")} + DROP COLUMN active_visitors + """ + end +end diff --git a/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs b/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs new file mode 100644 index 0000000000000000000000000000000000000000..549f86de4f7471461ed11da78453fd56b5a3efc1 --- /dev/null +++ b/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs @@ -0,0 +1,42 @@ +defmodule Plausible.IngestRepo.Migrations.AddImportedCustomEvents do + use Ecto.Migration + + def change do + # NOTE: Using another table for determining cluster presence + on_cluster = Plausible.MigrationUtils.on_cluster_statement("imported_pages") + table_settings = Plausible.MigrationUtils.table_settings_expr(:suffix) + + settings = + if Plausible.IngestRepo.clustered_table?("imported_pages") do + """ + ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/{database}/imported_custom_events', '{replica}') + ORDER BY (site_id, import_id, date, name) + SETTINGS replicated_deduplication_window = 0 #{table_settings} + """ + else + """ + ENGINE = MergeTree() + ORDER BY (site_id, import_id, date, name) + SETTINGS replicated_deduplication_window = 0 #{table_settings} + """ + end + + execute """ + CREATE TABLE IF NOT EXISTS imported_custom_events #{on_cluster} + ( + `site_id` UInt64, + `import_id` UInt64, + `date` Date, + `name` String CODEC(ZSTD(3)), + `link_url` String CODEC(ZSTD(3)), + `path` String CODEC(ZSTD(3)), + `visitors` UInt64, + `events` UInt64 + ) + #{settings} + """, + """ + DROP TABLE IF EXISTS imported_custom_events #{on_cluster} SYNC + """ + end +end diff --git a/priv/ingest_repo/migrations/20240502115822_alias_api_prop_names.exs b/priv/ingest_repo/migrations/20240502115822_alias_api_prop_names.exs new file mode 100644 index 0000000000000000000000000000000000000000..bf4d1733d1d536dda774b4c075ebb0e88524fbb7 --- /dev/null +++ b/priv/ingest_repo/migrations/20240502115822_alias_api_prop_names.exs @@ -0,0 +1,81 @@ +defmodule Plausible.IngestRepo.Migrations.AliasApiPropNames do + @moduledoc """ + Migration adds a ALIAS columns needed to keep DB schema and api + property naming in sync to reduce overhead in code. + """ + + use Ecto.Migration + + @sessions_prop_names %{ + "source" => "referrer_source", + "device" => "screen_size", + "screen" => "screen_size", + "os" => "operating_system", + "os_version" => "operating_system_version", + "country" => "country_code", + "region" => "subdivision1_code", + "city" => "city_geoname_id", + "entry_page_hostname" => "hostname" + } + + @events_prop_names %{ + "source" => "referrer_source", + "device" => "screen_size", + "screen" => "screen_size", + "os" => "operating_system", + "os_version" => "operating_system_version", + "country" => "country_code", + "region" => "subdivision1_code", + "city" => "city_geoname_id" + } + + def up do + column_types = get_column_types() + on_cluster = Plausible.MigrationUtils.on_cluster_statement("sessions_v2") + + for {alias_name, column_name} <- @sessions_prop_names do + execute """ + ALTER TABLE sessions_v2 + #{on_cluster} + ADD COLUMN #{alias_name} #{Map.fetch!(column_types, column_name)} ALIAS #{column_name} + """ + end + + for {alias_name, column_name} <- @events_prop_names do + execute """ + ALTER TABLE events_v2 + #{on_cluster} + ADD COLUMN #{alias_name} #{Map.fetch!(column_types, column_name)} ALIAS #{column_name} + """ + end + end + + def down do + on_cluster = Plausible.MigrationUtils.on_cluster_statement("sessions_v2") + + for {alias_name, _column_name} <- @sessions_prop_names do + execute """ + ALTER TABLE sessions_v2 + #{on_cluster} + DROP COLUMN #{alias_name} + """ + end + + for {alias_name, _column_name} <- @events_prop_names do + execute """ + ALTER TABLE events_v2 + #{on_cluster} + DROP COLUMN #{alias_name} + """ + end + end + + def get_column_types() do + %{rows: rows} = + Plausible.IngestRepo.query!(""" + SELECT name, type FROM system.columns WHERE table = 'sessions_v2' + """) + + Map.new(rows, fn [name, type] -> {name, type} end) + end +end diff --git a/priv/ingest_repo/migrations/20240709181437_populate_location_data.exs b/priv/ingest_repo/migrations/20240709181437_populate_location_data.exs new file mode 100644 index 0000000000000000000000000000000000000000..8bb0fb126a9c10371585ba562672d0e0add48ee0 --- /dev/null +++ b/priv/ingest_repo/migrations/20240709181437_populate_location_data.exs @@ -0,0 +1,18 @@ +defmodule Plausible.IngestRepo.Migrations.PopulateLocationData do + use Ecto.Migration + + def up do + try do + Location.load_all() + rescue + # Already loaded + ArgumentError -> nil + end + + Plausible.DataMigration.LocationsSync.run() + end + + def down do + raise "Irreversible" + end +end diff --git a/priv/ingest_repo/migrations/20240801091615_capitalize_known_sources.exs b/priv/ingest_repo/migrations/20240801091615_capitalize_known_sources.exs new file mode 100644 index 0000000000000000000000000000000000000000..4bb98b508bf24abbad4b14c0a5ade5764f912d3a --- /dev/null +++ b/priv/ingest_repo/migrations/20240801091615_capitalize_known_sources.exs @@ -0,0 +1,42 @@ +defmodule Plausible.IngestRepo.Migrations.CapitalizeKnownSources do + use Ecto.Migration + + def up() do + {:ok, _} = Application.ensure_all_started(:ref_inspector) + [{"referers.yml", map}] = RefInspector.Database.list(:default) + + {downcased_sources, sources} = + Enum.flat_map(map, fn {_, entries} -> + Enum.map(entries, fn {_, _, _, _, _, _, name} -> name end) + end) + |> Enum.flat_map(fn name -> + downcased_name = String.downcase(name) + + if name != downcased_name do + [{downcased_name, name}] + else + [] + end + end) + |> Enum.unzip() + + events_sql = """ + ALTER TABLE events_v2 + UPDATE referrer_source = transform(referrer_source, {$0:Array(String)}, {$1:Array(String)}) + WHERE ascii(referrer_source) BETWEEN 97 AND 122 + """ + + sessions_sql = """ + ALTER TABLE sessions_v2 + UPDATE referrer_source = transform(referrer_source, {$0:Array(String)}, {$1:Array(String)}) + WHERE ascii(referrer_source) BETWEEN 97 AND 122 + """ + + execute(fn -> repo().query!(events_sql, [downcased_sources, sources]) end) + execute(fn -> repo().query!(sessions_sql, [downcased_sources, sources]) end) + end + + def down do + raise "irreversible" + end +end diff --git a/priv/ingest_repo/migrations/20240829092858_add_acquisition_channel.exs b/priv/ingest_repo/migrations/20240829092858_add_acquisition_channel.exs new file mode 100644 index 0000000000000000000000000000000000000000..69e2d84d04350766be0144f6d3103a67b7aea536 --- /dev/null +++ b/priv/ingest_repo/migrations/20240829092858_add_acquisition_channel.exs @@ -0,0 +1,13 @@ +defmodule Plausible.IngestRepo.Migrations.AddAcquisitionChannel do + use Ecto.Migration + + def change do + alter table(:events_v2) do + add :channel, :"LowCardinality(String)" + end + + alter table(:sessions_v2) do + add :channel, :"LowCardinality(String)" + end + end +end diff --git a/priv/ingest_repo/migrations/20241020114559_add_click_id_param.exs b/priv/ingest_repo/migrations/20241020114559_add_click_id_param.exs new file mode 100644 index 0000000000000000000000000000000000000000..96930c76e09d33a11c09360e776b49f8dd9841dd --- /dev/null +++ b/priv/ingest_repo/migrations/20241020114559_add_click_id_param.exs @@ -0,0 +1,13 @@ +defmodule Plausible.IngestRepo.Migrations.AddClickIdParam do + use Ecto.Migration + + def change do + alter table(:events_v2) do + add :click_id_param, :"LowCardinality(String)" + end + + alter table(:sessions_v2) do + add :click_id_param, :"LowCardinality(String)" + end + end +end diff --git a/priv/ingest_repo/migrations/20241028142653_add_scroll_depth_to_events.exs b/priv/ingest_repo/migrations/20241028142653_add_scroll_depth_to_events.exs new file mode 100644 index 0000000000000000000000000000000000000000..0904db3623b366e760173bba2cb05d05a9865428 --- /dev/null +++ b/priv/ingest_repo/migrations/20241028142653_add_scroll_depth_to_events.exs @@ -0,0 +1,9 @@ +defmodule Plausible.IngestRepo.Migrations.AddScrollDepthToEvents do + use Ecto.Migration + + def change do + alter table(:events_v2) do + add :scroll_depth, :UInt8 + end + end +end diff --git a/priv/ingest_repo/migrations/20241029074741_remap_sources.exs b/priv/ingest_repo/migrations/20241029074741_remap_sources.exs new file mode 100644 index 0000000000000000000000000000000000000000..dda22bcafa55f8cf9c242351f30d0a262c6653ec --- /dev/null +++ b/priv/ingest_repo/migrations/20241029074741_remap_sources.exs @@ -0,0 +1,245 @@ +defmodule Plausible.IngestRepo.Migrations.RemapSources do + use Ecto.Migration + + @mappings %{ + # UTM sources + "fb" => "Facebook", + "fb-ads" => "Facebook", + "fbads" => "Facebook", + "fbad" => "Facebook", + "facebook-ads" => "Facebook", + "facebook_ads" => "Facebook", + "fcb" => "Facebook", + "facebook_ad" => "Facebook", + "facebook_feed_ad" => "Facebook", + "ig" => "Instagram", + "yt" => "Youtube", + "yt-ads" => "Youtube", + "reddit-ads" => "Reddit", + "google_ads" => "Google", + "google-ads" => "Google", + "googleads" => "Google", + "gads" => "Google", + "google ads" => "Google", + "adwords" => "Google", + "twitter-ads" => "Twitter", + "tiktokads" => "TikTok", + "tik.tok" => "TikTok", + "perplexity" => "Perplexity", + "linktree" => "Linktree", + + # Referrers + "android-app://com.reddit.frontpage" => "Reddit", + "perplexity.ai" => "Perplexity", + "search.brave.com" => "Brave", + "yandex.com.tr" => "Yandex", + "yandex.kz" => "Yandex", + "ya.ru" => "Yandex", + "yandex.uz" => "Yandex", + "yandex.fr" => "Yandex", + "yandex.eu" => "Yandex", + "yandex.tm" => "Yandex", + "discord.com" => "Discord", + "t.me" => "Telegram", + "webk.telegram.org" => "Telegram", + "sogou.com" => "Sogou", + "m.sogou.com" => "Sogou", + "wap.sogou.com" => "Sogou", + "canary.discord.com" => "Discord", + "ptb.discord.com" => "Discord", + "discordapp.com" => "Discord", + "linktr.ee" => "Linktree", + "baidu.com" => "Baidu", + "statics.teams.cdn.office.net" => "Microsoft Teams", + "ntp.msn.com" => "Bing", + "en.wikipedia.org" => "Wikipedia", + "en.m.wikipedia.org" => "Wikipedia", + "de.wikipedia.org" => "Wikipedia", + "de.m.wikipedia.org" => "Wikipedia", + "fr.wikipedia.org" => "Wikipedia", + "ru.wikipedia.org" => "Wikipedia", + "fr.m.wikipedia.org" => "Wikipedia", + "es.wikipedia.org" => "Wikipedia", + "ja.wikipedia.org" => "Wikipedia", + "nl.wikipedia.org" => "Wikipedia", + "ru.m.wikipedia.org" => "Wikipedia", + "da.m.wikipedia.org" => "Wikipedia", + "no.wikipedia.org" => "Wikipedia", + "es.m.wikipedia.org" => "Wikipedia", + "it.wikipedia.org" => "Wikipedia", + "nl.m.wikipedia.org" => "Wikipedia", + "da.wikipedia.org" => "Wikipedia", + "sv.wikipedia.org" => "Wikipedia", + "it.m.wikipedia.org" => "Wikipedia", + "zh.wikipedia.org" => "Wikipedia", + "sv.m.wikipedia.org" => "Wikipedia", + "no.m.wikipedia.org" => "Wikipedia", + "fi.m.wikipedia.org" => "Wikipedia", + "fi.wikipedia.org" => "Wikipedia", + "ca.wikipedia.org" => "Wikipedia", + "ja.m.wikipedia.org" => "Wikipedia", + "pt.wikipedia.org" => "Wikipedia", + "pl.wikipedia.org" => "Wikipedia", + "pt.m.wikipedia.org" => "Wikipedia", + "cs.wikipedia.org" => "Wikipedia", + "zh.m.wikipedia.org" => "Wikipedia", + "ca.m.wikipedia.org" => "Wikipedia", + "pl.m.wikipedia.org" => "Wikipedia", + "cs.m.wikipedia.org" => "Wikipedia", + "is.wikipedia.org" => "Wikipedia", + "ko.wikipedia.org" => "Wikipedia", + "uk.wikipedia.org" => "Wikipedia", + "is.m.wikipedia.org" => "Wikipedia", + "he.m.wikipedia.org" => "Wikipedia", + "tr.wikipedia.org" => "Wikipedia", + "he.wikipedia.org" => "Wikipedia", + "id.m.wikipedia.org" => "Wikipedia", + "tr.m.wikipedia.org" => "Wikipedia", + "et.wikipedia.org" => "Wikipedia", + "fa.m.wikipedia.org" => "Wikipedia", + "uk.m.wikipedia.org" => "Wikipedia", + "simple.wikipedia.org" => "Wikipedia", + "ko.m.wikipedia.org" => "Wikipedia", + "id.wikipedia.org" => "Wikipedia", + "hr.m.wikipedia.org" => "Wikipedia", + "simple.m.wikipedia.org" => "Wikipedia", + "vi.wikipedia.org" => "Wikipedia", + "el.wikipedia.org" => "Wikipedia", + "hr.wikipedia.org" => "Wikipedia", + "sk.wikipedia.org" => "Wikipedia", + "hu.wikipedia.org" => "Wikipedia", + "hu.m.wikipedia.org" => "Wikipedia", + "fa.wikipedia.org" => "Wikipedia", + "el.m.wikipedia.org" => "Wikipedia", + "arz.m.wikipedia.org" => "Wikipedia", + "th.m.wikipedia.org" => "Wikipedia", + "ta.m.wikipedia.org" => "Wikipedia", + "ga.wikipedia.org" => "Wikipedia", + "et.m.wikipedia.org" => "Wikipedia", + "vi.m.wikipedia.org" => "Wikipedia", + "ro.wikipedia.org" => "Wikipedia", + "ro.m.wikipedia.org" => "Wikipedia", + "ms.m.wikipedia.org" => "Wikipedia", + "bs.m.wikipedia.org" => "Wikipedia", + "az.m.wikipedia.org" => "Wikipedia", + "bg.m.wikipedia.org" => "Wikipedia", + "nn.wikipedia.org" => "Wikipedia", + "bg.wikipedia.org" => "Wikipedia", + "ml.m.wikipedia.org" => "Wikipedia", + "bn.m.wikipedia.org" => "Wikipedia", + "sl.wikipedia.org" => "Wikipedia", + "nn.m.wikipedia.org" => "Wikipedia", + "sk.m.wikipedia.org" => "Wikipedia", + "ms.wikipedia.org" => "Wikipedia", + "uz.wikipedia.org" => "Wikipedia", + "th.wikipedia.org" => "Wikipedia", + "sr.m.wikipedia.org" => "Wikipedia", + "hi.m.wikipedia.org" => "Wikipedia", + "eu.wikipedia.org" => "Wikipedia", + "uz.m.wikipedia.org" => "Wikipedia", + "sr.wikipedia.org" => "Wikipedia", + "lv.wikipedia.org" => "Wikipedia", + "la.wikipedia.org" => "Wikipedia", + "sl.m.wikipedia.org" => "Wikipedia", + "arz.wikipedia.org" => "Wikipedia", + "ta.wikipedia.org" => "Wikipedia", + "ka.m.wikipedia.org" => "Wikipedia", + "ga.m.wikipedia.org" => "Wikipedia", + "lt.wikipedia.org" => "Wikipedia", + "lv.m.wikipedia.org" => "Wikipedia", + "kk.m.wikipedia.org" => "Wikipedia", + "lt.m.wikipedia.org" => "Wikipedia", + "ar.wikipedia.org" => "Wikipedia", + "eo.wikipedia.org" => "Wikipedia", + "sw.m.wikipedia.org" => "Wikipedia", + "sh.wikipedia.org" => "Wikipedia", + "bs.wikipedia.org" => "Wikipedia", + "ml.wikipedia.org" => "Wikipedia", + "hy.wikipedia.org" => "Wikipedia", + "ka.wikipedia.org" => "Wikipedia", + "hi.wikipedia.org" => "Wikipedia", + "la.m.wikipedia.org" => "Wikipedia", + "bn.wikipedia.org" => "Wikipedia", + "ur.m.wikipedia.org" => "Wikipedia", + "sh.m.wikipedia.org" => "Wikipedia", + "az.wikipedia.org" => "Wikipedia", + "si.m.wikipedia.org" => "Wikipedia", + "sq.wikipedia.org" => "Wikipedia", + "zh-yue.wikipedia.org" => "Wikipedia", + "ckb.m.wikipedia.org" => "Wikipedia", + "kn.wikipedia.org" => "Wikipedia", + "lij.wikipedia.org" => "Wikipedia", + "fy.wikipedia.org" => "Wikipedia", + "lij.m.wikipedia.org" => "Wikipedia", + "hy.m.wikipedia.org" => "Wikipedia", + "mn.m.wikipedia.org" => "Wikipedia", + "ar.m.wikipedia.org" => "Wikipedia", + "tl.wikipedia.org" => "Wikipedia", + "eu.m.wikipedia.org" => "Wikipedia", + "fo.wikipedia.org" => "Wikipedia", + "mn.wikipedia.org" => "Wikipedia", + "mr.wikipedia.org" => "Wikipedia", + "zh-classical.wikipedia.org" => "Wikipedia", + "cy.wikipedia.org" => "Wikipedia", + "olo.wikipedia.org" => "Wikipedia", + "te.m.wikipedia.org" => "Wikipedia", + "mk.wikipedia.org" => "Wikipedia", + "dz.wikipedia.org" => "Wikipedia", + "as.m.wikipedia.org" => "Wikipedia", + "szl.m.wikipedia.org" => "Wikipedia", + "oc.wikipedia.org" => "Wikipedia", + "rw.m.wikipedia.org" => "Wikipedia", + "tl.m.wikipedia.org" => "Wikipedia", + "si.wikipedia.org" => "Wikipedia", + "nostalgia.wikipedia.org" => "Wikipedia", + "lrc.wikipedia.org" => "Wikipedia", + "eo.m.wikipedia.org" => "Wikipedia", + "ky.wikipedia.org" => "Wikipedia", + "new.wikipedia.org" => "Wikipedia", + "be.wikipedia.org" => "Wikipedia", + "bcl.m.wikipedia.org" => "Wikipedia", + "sq.m.wikipedia.org" => "Wikipedia", + "am.wikipedia.org" => "Wikipedia", + "nds-nl.m.wikipedia.org" => "Wikipedia", + "gu.m.wikipedia.org" => "Wikipedia", + "bjn.wikipedia.org" => "Wikipedia", + "pa.m.wikipedia.org" => "Wikipedia", + "sa.m.wikipedia.org" => "Wikipedia", + "tt.wikipedia.org" => "Wikipedia", + "qu.wikipedia.org" => "Wikipedia", + "be-tarask.wikipedia.org" => "Wikipedia", + "mg.m.wikipedia.org" => "Wikipedia", + "dga.m.wikipedia.org" => "Wikipedia", + "bat-smg.wikipedia.org" => "Wikipedia", + "sw.wikipedia.org" => "Wikipedia", + "wuu.wikipedia.org" => "Wikipedia", + "ne.m.wikipedia.org" => "Wikipedia", + "yi.m.wikipedia.org" => "Wikipedia", + "vec.wikipedia.org" => "Wikipedia", + "be-tarask.m.wikipedia.org" => "Wikipedia", + "crh.wikipedia.org" => "Wikipedia" + } + + def up do + {keys, values} = Enum.unzip(@mappings) + + events_sql = """ + ALTER TABLE events_v2 + UPDATE referrer_source = transform(referrer_source, {$0:Array(String)}, {$1:Array(String)}) + WHERE referrer_source IN {$0:Array(String)} + """ + + sessions_sql = """ + ALTER TABLE sessions_v2 + UPDATE referrer_source = transform(referrer_source, {$0:Array(String)}, {$1:Array(String)}) + WHERE referrer_source IN {$0:Array(String)} + """ + + execute(fn -> repo().query!(events_sql, [keys, values]) end) + execute(fn -> repo().query!(sessions_sql, [keys, values]) end) + end + + def down do + raise "irreversible" + end +end diff --git a/priv/ingest_repo/migrations/20241104082248_remap_sources_v2.exs b/priv/ingest_repo/migrations/20241104082248_remap_sources_v2.exs new file mode 100644 index 0000000000000000000000000000000000000000..3995af1994402e8cdcf76b9781b39b4670d62e0b --- /dev/null +++ b/priv/ingest_repo/migrations/20241104082248_remap_sources_v2.exs @@ -0,0 +1,245 @@ +defmodule Plausible.IngestRepo.Migrations.RemapSourcesV2 do + use Ecto.Migration + + @mappings %{ + # UTM sources + "fb" => "Facebook", + "fb-ads" => "Facebook", + "fbads" => "Facebook", + "fbad" => "Facebook", + "facebook-ads" => "Facebook", + "facebook_ads" => "Facebook", + "fcb" => "Facebook", + "facebook_ad" => "Facebook", + "facebook_feed_ad" => "Facebook", + "ig" => "Instagram", + "yt" => "Youtube", + "yt-ads" => "Youtube", + "reddit-ads" => "Reddit", + "google_ads" => "Google", + "google-ads" => "Google", + "googleads" => "Google", + "gads" => "Google", + "google ads" => "Google", + "adwords" => "Google", + "twitter-ads" => "Twitter", + "tiktokads" => "TikTok", + "tik.tok" => "TikTok", + "perplexity" => "Perplexity", + "linktree" => "Linktree", + + # Referrers + "android-app://com.reddit.frontpage" => "Reddit", + "perplexity.ai" => "Perplexity", + "search.brave.com" => "Brave", + "yandex.com.tr" => "Yandex", + "yandex.kz" => "Yandex", + "ya.ru" => "Yandex", + "yandex.uz" => "Yandex", + "yandex.fr" => "Yandex", + "yandex.eu" => "Yandex", + "yandex.tm" => "Yandex", + "discord.com" => "Discord", + "t.me" => "Telegram", + "webk.telegram.org" => "Telegram", + "sogou.com" => "Sogou", + "m.sogou.com" => "Sogou", + "wap.sogou.com" => "Sogou", + "canary.discord.com" => "Discord", + "ptb.discord.com" => "Discord", + "discordapp.com" => "Discord", + "linktr.ee" => "Linktree", + "baidu.com" => "Baidu", + "statics.teams.cdn.office.net" => "Microsoft Teams", + "ntp.msn.com" => "Bing", + "en.wikipedia.org" => "Wikipedia", + "en.m.wikipedia.org" => "Wikipedia", + "de.wikipedia.org" => "Wikipedia", + "de.m.wikipedia.org" => "Wikipedia", + "fr.wikipedia.org" => "Wikipedia", + "ru.wikipedia.org" => "Wikipedia", + "fr.m.wikipedia.org" => "Wikipedia", + "es.wikipedia.org" => "Wikipedia", + "ja.wikipedia.org" => "Wikipedia", + "nl.wikipedia.org" => "Wikipedia", + "ru.m.wikipedia.org" => "Wikipedia", + "da.m.wikipedia.org" => "Wikipedia", + "no.wikipedia.org" => "Wikipedia", + "es.m.wikipedia.org" => "Wikipedia", + "it.wikipedia.org" => "Wikipedia", + "nl.m.wikipedia.org" => "Wikipedia", + "da.wikipedia.org" => "Wikipedia", + "sv.wikipedia.org" => "Wikipedia", + "it.m.wikipedia.org" => "Wikipedia", + "zh.wikipedia.org" => "Wikipedia", + "sv.m.wikipedia.org" => "Wikipedia", + "no.m.wikipedia.org" => "Wikipedia", + "fi.m.wikipedia.org" => "Wikipedia", + "fi.wikipedia.org" => "Wikipedia", + "ca.wikipedia.org" => "Wikipedia", + "ja.m.wikipedia.org" => "Wikipedia", + "pt.wikipedia.org" => "Wikipedia", + "pl.wikipedia.org" => "Wikipedia", + "pt.m.wikipedia.org" => "Wikipedia", + "cs.wikipedia.org" => "Wikipedia", + "zh.m.wikipedia.org" => "Wikipedia", + "ca.m.wikipedia.org" => "Wikipedia", + "pl.m.wikipedia.org" => "Wikipedia", + "cs.m.wikipedia.org" => "Wikipedia", + "is.wikipedia.org" => "Wikipedia", + "ko.wikipedia.org" => "Wikipedia", + "uk.wikipedia.org" => "Wikipedia", + "is.m.wikipedia.org" => "Wikipedia", + "he.m.wikipedia.org" => "Wikipedia", + "tr.wikipedia.org" => "Wikipedia", + "he.wikipedia.org" => "Wikipedia", + "id.m.wikipedia.org" => "Wikipedia", + "tr.m.wikipedia.org" => "Wikipedia", + "et.wikipedia.org" => "Wikipedia", + "fa.m.wikipedia.org" => "Wikipedia", + "uk.m.wikipedia.org" => "Wikipedia", + "simple.wikipedia.org" => "Wikipedia", + "ko.m.wikipedia.org" => "Wikipedia", + "id.wikipedia.org" => "Wikipedia", + "hr.m.wikipedia.org" => "Wikipedia", + "simple.m.wikipedia.org" => "Wikipedia", + "vi.wikipedia.org" => "Wikipedia", + "el.wikipedia.org" => "Wikipedia", + "hr.wikipedia.org" => "Wikipedia", + "sk.wikipedia.org" => "Wikipedia", + "hu.wikipedia.org" => "Wikipedia", + "hu.m.wikipedia.org" => "Wikipedia", + "fa.wikipedia.org" => "Wikipedia", + "el.m.wikipedia.org" => "Wikipedia", + "arz.m.wikipedia.org" => "Wikipedia", + "th.m.wikipedia.org" => "Wikipedia", + "ta.m.wikipedia.org" => "Wikipedia", + "ga.wikipedia.org" => "Wikipedia", + "et.m.wikipedia.org" => "Wikipedia", + "vi.m.wikipedia.org" => "Wikipedia", + "ro.wikipedia.org" => "Wikipedia", + "ro.m.wikipedia.org" => "Wikipedia", + "ms.m.wikipedia.org" => "Wikipedia", + "bs.m.wikipedia.org" => "Wikipedia", + "az.m.wikipedia.org" => "Wikipedia", + "bg.m.wikipedia.org" => "Wikipedia", + "nn.wikipedia.org" => "Wikipedia", + "bg.wikipedia.org" => "Wikipedia", + "ml.m.wikipedia.org" => "Wikipedia", + "bn.m.wikipedia.org" => "Wikipedia", + "sl.wikipedia.org" => "Wikipedia", + "nn.m.wikipedia.org" => "Wikipedia", + "sk.m.wikipedia.org" => "Wikipedia", + "ms.wikipedia.org" => "Wikipedia", + "uz.wikipedia.org" => "Wikipedia", + "th.wikipedia.org" => "Wikipedia", + "sr.m.wikipedia.org" => "Wikipedia", + "hi.m.wikipedia.org" => "Wikipedia", + "eu.wikipedia.org" => "Wikipedia", + "uz.m.wikipedia.org" => "Wikipedia", + "sr.wikipedia.org" => "Wikipedia", + "lv.wikipedia.org" => "Wikipedia", + "la.wikipedia.org" => "Wikipedia", + "sl.m.wikipedia.org" => "Wikipedia", + "arz.wikipedia.org" => "Wikipedia", + "ta.wikipedia.org" => "Wikipedia", + "ka.m.wikipedia.org" => "Wikipedia", + "ga.m.wikipedia.org" => "Wikipedia", + "lt.wikipedia.org" => "Wikipedia", + "lv.m.wikipedia.org" => "Wikipedia", + "kk.m.wikipedia.org" => "Wikipedia", + "lt.m.wikipedia.org" => "Wikipedia", + "ar.wikipedia.org" => "Wikipedia", + "eo.wikipedia.org" => "Wikipedia", + "sw.m.wikipedia.org" => "Wikipedia", + "sh.wikipedia.org" => "Wikipedia", + "bs.wikipedia.org" => "Wikipedia", + "ml.wikipedia.org" => "Wikipedia", + "hy.wikipedia.org" => "Wikipedia", + "ka.wikipedia.org" => "Wikipedia", + "hi.wikipedia.org" => "Wikipedia", + "la.m.wikipedia.org" => "Wikipedia", + "bn.wikipedia.org" => "Wikipedia", + "ur.m.wikipedia.org" => "Wikipedia", + "sh.m.wikipedia.org" => "Wikipedia", + "az.wikipedia.org" => "Wikipedia", + "si.m.wikipedia.org" => "Wikipedia", + "sq.wikipedia.org" => "Wikipedia", + "zh-yue.wikipedia.org" => "Wikipedia", + "ckb.m.wikipedia.org" => "Wikipedia", + "kn.wikipedia.org" => "Wikipedia", + "lij.wikipedia.org" => "Wikipedia", + "fy.wikipedia.org" => "Wikipedia", + "lij.m.wikipedia.org" => "Wikipedia", + "hy.m.wikipedia.org" => "Wikipedia", + "mn.m.wikipedia.org" => "Wikipedia", + "ar.m.wikipedia.org" => "Wikipedia", + "tl.wikipedia.org" => "Wikipedia", + "eu.m.wikipedia.org" => "Wikipedia", + "fo.wikipedia.org" => "Wikipedia", + "mn.wikipedia.org" => "Wikipedia", + "mr.wikipedia.org" => "Wikipedia", + "zh-classical.wikipedia.org" => "Wikipedia", + "cy.wikipedia.org" => "Wikipedia", + "olo.wikipedia.org" => "Wikipedia", + "te.m.wikipedia.org" => "Wikipedia", + "mk.wikipedia.org" => "Wikipedia", + "dz.wikipedia.org" => "Wikipedia", + "as.m.wikipedia.org" => "Wikipedia", + "szl.m.wikipedia.org" => "Wikipedia", + "oc.wikipedia.org" => "Wikipedia", + "rw.m.wikipedia.org" => "Wikipedia", + "tl.m.wikipedia.org" => "Wikipedia", + "si.wikipedia.org" => "Wikipedia", + "nostalgia.wikipedia.org" => "Wikipedia", + "lrc.wikipedia.org" => "Wikipedia", + "eo.m.wikipedia.org" => "Wikipedia", + "ky.wikipedia.org" => "Wikipedia", + "new.wikipedia.org" => "Wikipedia", + "be.wikipedia.org" => "Wikipedia", + "bcl.m.wikipedia.org" => "Wikipedia", + "sq.m.wikipedia.org" => "Wikipedia", + "am.wikipedia.org" => "Wikipedia", + "nds-nl.m.wikipedia.org" => "Wikipedia", + "gu.m.wikipedia.org" => "Wikipedia", + "bjn.wikipedia.org" => "Wikipedia", + "pa.m.wikipedia.org" => "Wikipedia", + "sa.m.wikipedia.org" => "Wikipedia", + "tt.wikipedia.org" => "Wikipedia", + "qu.wikipedia.org" => "Wikipedia", + "be-tarask.wikipedia.org" => "Wikipedia", + "mg.m.wikipedia.org" => "Wikipedia", + "dga.m.wikipedia.org" => "Wikipedia", + "bat-smg.wikipedia.org" => "Wikipedia", + "sw.wikipedia.org" => "Wikipedia", + "wuu.wikipedia.org" => "Wikipedia", + "ne.m.wikipedia.org" => "Wikipedia", + "yi.m.wikipedia.org" => "Wikipedia", + "vec.wikipedia.org" => "Wikipedia", + "be-tarask.m.wikipedia.org" => "Wikipedia", + "crh.wikipedia.org" => "Wikipedia" + } + + def up do + {keys, values} = Enum.unzip(@mappings) + + events_sql = """ + ALTER TABLE events_v2 + UPDATE referrer_source = transform(lower(referrer_source), {$0:Array(String)}, {$1:Array(String)}) + WHERE lower(referrer_source) IN {$0:Array(String)} + """ + + sessions_sql = """ + ALTER TABLE sessions_v2 + UPDATE referrer_source = transform(lower(referrer_source), {$0:Array(String)}, {$1:Array(String)}) + WHERE lower(referrer_source) IN {$0:Array(String)} + """ + + execute(fn -> repo().query!(events_sql, [keys, values]) end) + execute(fn -> repo().query!(sessions_sql, [keys, values]) end) + end + + def down do + raise "irreversible" + end +end diff --git a/priv/ingest_repo/migrations/20241111084056_create_acquisition_channel_column.exs b/priv/ingest_repo/migrations/20241111084056_create_acquisition_channel_column.exs new file mode 100644 index 0000000000000000000000000000000000000000..4e3bdc017e1e8c85f4bdbbb6784a1ac1df389924 --- /dev/null +++ b/priv/ingest_repo/migrations/20241111084056_create_acquisition_channel_column.exs @@ -0,0 +1,11 @@ +defmodule Plausible.IngestRepo.Migrations.CreateAcquisitionChannelColumn do + use Ecto.Migration + + def up do + Plausible.DataMigration.AcquisitionChannel.run(add_column: true, backfill: false) + end + + def down do + raise "irreversible" + end +end diff --git a/priv/ingest_repo/migrations/20241112222848_add_channel_to_imported_source.exs b/priv/ingest_repo/migrations/20241112222848_add_channel_to_imported_source.exs new file mode 100644 index 0000000000000000000000000000000000000000..5fa9ea5df189b75a106a178b2f42881a6caff781 --- /dev/null +++ b/priv/ingest_repo/migrations/20241112222848_add_channel_to_imported_source.exs @@ -0,0 +1,9 @@ +defmodule Plausible.IngestRepo.Migrations.AddChannelToImportedSource do + use Ecto.Migration + + def change do + alter table(:imported_sources) do + add :channel, :"LowCardinality(String)" + end + end +end diff --git a/priv/ingest_repo/migrations/20241118112238_backfill_utm_medium_click_id_param.exs b/priv/ingest_repo/migrations/20241118112238_backfill_utm_medium_click_id_param.exs new file mode 100644 index 0000000000000000000000000000000000000000..df0c470a2837999d061b7079e3f4faaa7164c1d5 --- /dev/null +++ b/priv/ingest_repo/migrations/20241118112238_backfill_utm_medium_click_id_param.exs @@ -0,0 +1,27 @@ +defmodule Plausible.IngestRepo.Migrations.BackfillUtmMediumClickIdParam do + @moduledoc """ + Backfills utm_medium based on referrer_source and click_id_param + """ + use Ecto.Migration + + def up do + execute(fn -> repo().query!(update_query("events_v2")) end) + execute(fn -> repo().query!(update_query("sessions_v2")) end) + end + + def down do + raise "irreversible" + end + + defp update_query(table) do + """ + ALTER TABLE #{table} + UPDATE utm_medium = multiIf( + referrer_source = 'Google' AND click_id_param = 'gclid', '(gclid)', + referrer_source = 'Bing' AND click_id_param = 'msclkid', '(msclkid)', + utm_medium + ) + WHERE empty(utm_medium) AND NOT empty(click_id_param) + """ + end +end diff --git a/priv/ingest_repo/migrations/20241120064325_create_ingest_counters_site_traffic_projection.exs b/priv/ingest_repo/migrations/20241120064325_create_ingest_counters_site_traffic_projection.exs new file mode 100644 index 0000000000000000000000000000000000000000..4738426a649de1358fa8d83e945b3ae1d31d4959 --- /dev/null +++ b/priv/ingest_repo/migrations/20241120064325_create_ingest_counters_site_traffic_projection.exs @@ -0,0 +1,38 @@ +defmodule Plausible.IngestRepo.Migrations.CreateIngestCountersSiteTrafficProjection do + use Ecto.Migration + + def up do + # ClickHouse 24.8 introduced `deduplicate_merge_projection_mode` which we need to set to create this projection. + # As such, try to set the setting and if it fails, ignore it. + execute(fn -> + try do + repo().query!(""" + ALTER TABLE ingest_counters + #{Plausible.MigrationUtils.on_cluster_statement("ingest_counters")} + MODIFY SETTING deduplicate_merge_projection_mode='rebuild' + """) + rescue + _e -> nil + end + end) + + execute """ + ALTER TABLE ingest_counters + #{Plausible.MigrationUtils.on_cluster_statement("ingest_counters")} + ADD PROJECTION ingest_counters_site_traffic_projection ( + SELECT site_id, toDate(event_timebucket), sumIf(value, metric = 'buffered') + GROUP BY site_id, toDate(event_timebucket) + ) + """ + + execute """ + ALTER TABLE ingest_counters + #{Plausible.MigrationUtils.on_cluster_statement("ingest_counters")} + MATERIALIZE PROJECTION ingest_counters_site_traffic_projection + """ + end + + def down do + raise "irreversible" + end +end diff --git a/priv/ingest_repo/migrations/20241216133031_add_scroll_depth_to_imported_pages.exs b/priv/ingest_repo/migrations/20241216133031_add_scroll_depth_to_imported_pages.exs new file mode 100644 index 0000000000000000000000000000000000000000..bac6c85f9e430126407eb2716b4c1a1abe225a4c --- /dev/null +++ b/priv/ingest_repo/migrations/20241216133031_add_scroll_depth_to_imported_pages.exs @@ -0,0 +1,21 @@ +defmodule Plausible.IngestRepo.Migrations.AddScrollDepthToImportedPages do + use Ecto.Migration + + @on_cluster Plausible.MigrationUtils.on_cluster_statement("imported_pages") + + def up do + execute """ + ALTER TABLE imported_pages + #{@on_cluster} + ADD COLUMN scroll_depth UInt8 DEFAULT 255 + """ + end + + def down do + execute """ + ALTER TABLE imported_pages + #{@on_cluster} + DROP COLUMN IF EXISTS scroll_depth + """ + end +end diff --git a/priv/ingest_repo/migrations/20241218102326_drop_and_add_scroll_depth_to_imported_pages.exs b/priv/ingest_repo/migrations/20241218102326_drop_and_add_scroll_depth_to_imported_pages.exs new file mode 100644 index 0000000000000000000000000000000000000000..49db290219095d9e8a59069914f494da55efeb48 --- /dev/null +++ b/priv/ingest_repo/migrations/20241218102326_drop_and_add_scroll_depth_to_imported_pages.exs @@ -0,0 +1,27 @@ +defmodule Plausible.IngestRepo.Migrations.DropAndAddScrollDepthToImportedPages do + use Ecto.Migration + + @on_cluster Plausible.MigrationUtils.on_cluster_statement("imported_pages") + + def up do + execute """ + ALTER TABLE imported_pages + #{@on_cluster} + DROP COLUMN scroll_depth + """ + + execute """ + ALTER TABLE imported_pages + #{@on_cluster} + ADD COLUMN scroll_depth Nullable(UInt64) + """ + end + + def down do + execute """ + ALTER TABLE imported_pages + #{@on_cluster} + DROP COLUMN scroll_depth + """ + end +end diff --git a/priv/ingest_repo/migrations/20241231083407_add_pageleave_visitors_to_imported_pages.exs b/priv/ingest_repo/migrations/20241231083407_add_pageleave_visitors_to_imported_pages.exs new file mode 100644 index 0000000000000000000000000000000000000000..64687c4322a293908f84cde9ad1b187f82fd1ca5 --- /dev/null +++ b/priv/ingest_repo/migrations/20241231083407_add_pageleave_visitors_to_imported_pages.exs @@ -0,0 +1,21 @@ +defmodule Plausible.IngestRepo.Migrations.AddPageleaveVisitorsToImportedPages do + use Ecto.Migration + + @on_cluster Plausible.MigrationUtils.on_cluster_statement("imported_pages") + + def up do + execute """ + ALTER TABLE imported_pages + #{@on_cluster} + ADD COLUMN pageleave_visitors UInt64 + """ + end + + def down do + execute """ + ALTER TABLE imported_pages + #{@on_cluster} + DROP COLUMN pageleave_visitors + """ + end +end diff --git a/priv/ingest_repo/migrations/20250212100953_imported_pages_new_scroll_depth_columns.exs b/priv/ingest_repo/migrations/20250212100953_imported_pages_new_scroll_depth_columns.exs new file mode 100644 index 0000000000000000000000000000000000000000..f5dae39a324d55de824fae51031fd932d450ac2a --- /dev/null +++ b/priv/ingest_repo/migrations/20250212100953_imported_pages_new_scroll_depth_columns.exs @@ -0,0 +1,33 @@ +defmodule Plausible.IngestRepo.Migrations.ImportedPagesNewScrollDepthColumns do + use Ecto.Migration + + @on_cluster Plausible.MigrationUtils.on_cluster_statement("imported_pages") + + def up do + execute """ + ALTER TABLE imported_pages #{@on_cluster} + ADD COLUMN total_scroll_depth UInt64 + """ + + execute """ + ALTER TABLE imported_pages #{@on_cluster} + ADD COLUMN total_scroll_depth_visits UInt64 + """ + + execute """ + ALTER TABLE imported_pages + #{@on_cluster} + DROP COLUMN scroll_depth + """ + + execute """ + ALTER TABLE imported_pages + #{@on_cluster} + DROP COLUMN pageleave_visitors + """ + end + + def down do + raise "irreversible" + end +end diff --git a/priv/ingest_repo/migrations/20250218094453_create_custom_event_array_function.exs b/priv/ingest_repo/migrations/20250218094453_create_custom_event_array_function.exs new file mode 100644 index 0000000000000000000000000000000000000000..01155568f24df8af33ef6d57c50e556ed0997f4b --- /dev/null +++ b/priv/ingest_repo/migrations/20250218094453_create_custom_event_array_function.exs @@ -0,0 +1,15 @@ +defmodule Plausible.IngestRepo.Migrations.CreateCustomEventArrayFunction do + use Ecto.Migration + + def up do + execute """ + CREATE OR REPLACE FUNCTION createCustomEventArray + #{Plausible.MigrationUtils.on_cluster_statement("events_v2")} + AS (foundIndex, scaleBy) -> (if(foundIndex > 0, [toUInt64(foundIndex + scaleBy)], [])); + """ + end + + def down do + raise "irreversible" + end +end diff --git a/priv/ingest_repo/migrations/20250219093806_events_engagement_time.exs b/priv/ingest_repo/migrations/20250219093806_events_engagement_time.exs new file mode 100644 index 0000000000000000000000000000000000000000..b9d30a42001c9bb8b3bcf88c7c9a8e43e0913350 --- /dev/null +++ b/priv/ingest_repo/migrations/20250219093806_events_engagement_time.exs @@ -0,0 +1,9 @@ +defmodule Plausible.IngestRepo.Migrations.EventsEngagementTime do + use Ecto.Migration + + def change do + alter table(:events_v2) do + add :engagement_time, :UInt32 + end + end +end diff --git a/priv/ingest_repo/migrations/20250221124625_imported_pages_total_time_on_page.exs b/priv/ingest_repo/migrations/20250221124625_imported_pages_total_time_on_page.exs new file mode 100644 index 0000000000000000000000000000000000000000..b88a9432af3d1be38a6ccafb8bd9b3b1e4905f60 --- /dev/null +++ b/priv/ingest_repo/migrations/20250221124625_imported_pages_total_time_on_page.exs @@ -0,0 +1,10 @@ +defmodule Plausible.IngestRepo.Migrations.ImportedPagesTotalTimeOnPage do + use Ecto.Migration + + def change do + alter table(:imported_pages) do + add :total_time_on_page, :UInt64 + add :total_time_on_page_visits, :UInt64 + end + end +end diff --git a/priv/ingest_repo/migrations/20250304074501_populate_imported_pages_total_time_on_page.exs b/priv/ingest_repo/migrations/20250304074501_populate_imported_pages_total_time_on_page.exs new file mode 100644 index 0000000000000000000000000000000000000000..548aa369931ffad464a805621d595ddf2517f5f1 --- /dev/null +++ b/priv/ingest_repo/migrations/20250304074501_populate_imported_pages_total_time_on_page.exs @@ -0,0 +1,19 @@ +defmodule Plausible.IngestRepo.Migrations.PopulateImportedPagesTotalTimeOnPage do + use Ecto.Migration + + def up do + # Note: This only populates the new column for old GA4 imports. + # Other imports didn't populate the `time_on_page` column. + execute """ + ALTER TABLE imported_pages + UPDATE + total_time_on_page = time_on_page, + total_time_on_page_visits = visits + WHERE time_on_page > 0 + """ + end + + def down do + raise "Irreversible" + end +end diff --git a/priv/ingest_repo/migrations/20250312063938_imported_pages_remove_old_time_on_page_columns.exs b/priv/ingest_repo/migrations/20250312063938_imported_pages_remove_old_time_on_page_columns.exs new file mode 100644 index 0000000000000000000000000000000000000000..69b141162105d9a260c889d5fd0e4d1688fe6d38 --- /dev/null +++ b/priv/ingest_repo/migrations/20250312063938_imported_pages_remove_old_time_on_page_columns.exs @@ -0,0 +1,23 @@ +defmodule Plausible.IngestRepo.Migrations.ImportedPagesRemoveOldTimeOnPageColumns do + use Ecto.Migration + + def up do + on_cluster = Plausible.MigrationUtils.on_cluster_statement("imported_pages") + + execute """ + ALTER TABLE imported_pages + #{on_cluster} + DROP COLUMN IF EXISTS time_on_page + """ + + execute """ + ALTER TABLE imported_pages + #{on_cluster} + DROP COLUMN IF EXISTS active_visitors + """ + end + + def down do + raise "Irreversible" + end +end diff --git a/priv/ingest_repo/migrations/20250316182725_ingest_counters_tracker_script_version.exs b/priv/ingest_repo/migrations/20250316182725_ingest_counters_tracker_script_version.exs new file mode 100644 index 0000000000000000000000000000000000000000..7217f2b01c3806811b882ce4bb81ebec90da4730 --- /dev/null +++ b/priv/ingest_repo/migrations/20250316182725_ingest_counters_tracker_script_version.exs @@ -0,0 +1,14 @@ +defmodule Plausible.IngestRepo.Migrations.IngestCountersTrackerScriptVersion do + use Ecto.Migration + + def change do + on_cluster = Plausible.MigrationUtils.on_cluster_statement("ingest_counters") + + execute """ + ALTER TABLE ingest_counters + #{on_cluster} + ADD COLUMN IF NOT EXISTS tracker_script_version UInt16, + MODIFY ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket), tracker_script_version) + """ + end +end diff --git a/priv/ingest_repo/migrations/20251106111224_add_recovery_id_to_events_sessions.exs b/priv/ingest_repo/migrations/20251106111224_add_recovery_id_to_events_sessions.exs new file mode 100644 index 0000000000000000000000000000000000000000..2e8aa5957be7dcd659eb8d4fc13d9019706ddb24 --- /dev/null +++ b/priv/ingest_repo/migrations/20251106111224_add_recovery_id_to_events_sessions.exs @@ -0,0 +1,40 @@ +defmodule Plausible.IngestRepo.Migrations.AddRecoveryIdToEventsSessions do + use Ecto.Migration + + import Plausible.MigrationUtils + + @on_cluster_sessions on_cluster_statement("sessions_v2") + @on_cluster_events on_cluster_statement("events_v2") + + def up do + if enterprise_edition?() do + execute """ + ALTER TABLE sessions_v2 + #{@on_cluster_sessions} + ADD COLUMN recovery_id UInt64 + """ + + execute """ + ALTER TABLE events_v2 + #{@on_cluster_events} + ADD COLUMN recovery_id UInt64 + """ + end + end + + def down do + if enterprise_edition?() do + execute """ + ALTER TABLE sessions_v2 + #{@on_cluster_sessions} + DROP COLUMN IF EXISTS recovery_id + """ + + execute """ + ALTER TABLE events_v2 + #{@on_cluster_events} + DROP COLUMN IF EXISTS recovery_id + """ + end + end +end diff --git a/priv/ingest_repo/migrations/20260601000001_remap_sources_v3.exs b/priv/ingest_repo/migrations/20260601000001_remap_sources_v3.exs new file mode 100644 index 0000000000000000000000000000000000000000..5cb78716a357e1ca10f3a1486e597c707b8d6bef --- /dev/null +++ b/priv/ingest_repo/migrations/20260601000001_remap_sources_v3.exs @@ -0,0 +1,83 @@ +defmodule Plausible.IngestRepo.Migrations.RemapSourcesV3 do + use Ecto.Migration + + @mappings %{ + "twitter" => "X (Twitter)", + "x.com" => "X (Twitter)", + "twitter-ads" => "X (Twitter)", + "mobile.twitter.com" => "X (Twitter)", + "bsky.app" => "Bluesky", + "go.bsky.app" => "Bluesky", + "chatgpt.com" => "ChatGPT", + "chat.openai.com" => "ChatGPT", + "claude.ai" => "Claude", + "phind.com" => "Phind", + "deepseek.com" => "DeepSeek", + "copilot" => "Microsoft Copilot", + "copilot.microsoft.com" => "Microsoft Copilot", + "copilot.com" => "Microsoft Copilot", + "grok.com" => "Grok", + "x.ai" => "Grok", + "gemini.google.com" => "Google Gemini", + "pplx.ai" => "Perplexity", + "kagi.com" => "Kagi", + "l.threads.com" => "Threads", + "mastodon.social" => "Mastodon", + "mastodon.online" => "Mastodon", + "mastodon.world" => "Mastodon", + "fosstodon.org" => "Mastodon", + "hachyderm.io" => "Mastodon", + "infosec.exchange" => "Mastodon", + "mas.to" => "Mastodon", + "sigmoid.social" => "Mastodon", + "mstdn.social" => "Mastodon", + "indieweb.social" => "Mastodon", + "mathstodon.xyz" => "Mastodon", + "scholar.social" => "Mastodon", + "chaos.social" => "Mastodon", + "social.tchncs.de" => "Mastodon", + "mstdn.jp" => "Mastodon" + } + + @suffix_mappings %{ + "officeapps.live.com" => "Microsoft 365", + "wikipedia.org" => "Wikipedia" + } + + def up do + {keys, values} = Enum.unzip(@mappings) + + suffix_match = + Enum.map_join(Enum.sort(@suffix_mappings), " OR ", fn {suffix, _name} -> + suffix_condition(suffix) + end) + + suffix_cases = + Enum.map_join(Enum.sort(@suffix_mappings), "\n ", fn {suffix, name} -> + "WHEN #{suffix_condition(suffix)} THEN '#{name}'" + end) + + for table <- ["events_v2", "sessions_v2"] do + sql = """ + ALTER TABLE #{table} + UPDATE referrer_source = + CASE + #{suffix_cases} + WHEN lower(referrer_source) IN {$0:Array(String)} THEN transform(lower(referrer_source), {$0:Array(String)}, {$1:Array(String)}) + ELSE referrer_source + END + WHERE lower(referrer_source) IN {$0:Array(String)} OR #{suffix_match} + """ + + execute(fn -> repo().query!(sql, [keys, values]) end) + end + end + + defp suffix_condition(suffix) do + "lower(referrer_source) = '#{suffix}' OR endsWith(lower(referrer_source), '.#{suffix}')" + end + + def down do + raise "irreversible" + end +end diff --git a/priv/ingest_repo/migrations/20260601000002_update_acquisition_channel.exs b/priv/ingest_repo/migrations/20260601000002_update_acquisition_channel.exs new file mode 100644 index 0000000000000000000000000000000000000000..679cb4964e89dcd2fd3a34d3f6f44b5c728b3cb0 --- /dev/null +++ b/priv/ingest_repo/migrations/20260601000002_update_acquisition_channel.exs @@ -0,0 +1,17 @@ +defmodule Plausible.IngestRepo.Migrations.UpdateAcquisitionChannel do + use Ecto.Migration + + def up do + # Rebuilds the ClickHouse dictionaries and SQL functions with the updated + # source categories (AI Assistants channel, Bluesky, Mastodon, X (Twitter), + # Google Gemini), then backfills all historical rows. + # + # This must run after 20260601000001_remap_sources_v3.exs so that + # referrer_source values have already been remapped to their canonical names. + Plausible.DataMigration.AcquisitionChannel.run(update_column: true, backfill: true) + end + + def down do + raise "irreversible" + end +end diff --git a/priv/ingest_repo/structure.sql b/priv/ingest_repo/structure.sql new file mode 100644 index 0000000000000000000000000000000000000000..ba41d631b49fb4ca57f134902613fd66a080a6f2 --- /dev/null +++ b/priv/ingest_repo/structure.sql @@ -0,0 +1,366 @@ +CREATE TABLE plausible_events_db.sessions_v2 +( + `session_id` UInt64, + `sign` Int8, + `site_id` UInt64, + `user_id` UInt64, + `hostname` String CODEC(ZSTD(3)), + `timestamp` DateTime CODEC(Delta(4), LZ4), + `start` DateTime CODEC(Delta(4), LZ4), + `is_bounce` UInt8, + `entry_page` String CODEC(ZSTD(3)), + `exit_page` String CODEC(ZSTD(3)), + `pageviews` Int32, + `events` Int32, + `duration` UInt32, + `referrer` String CODEC(ZSTD(3)), + `referrer_source` String CODEC(ZSTD(3)), + `country_code` LowCardinality(FixedString(2)), + `screen_size` LowCardinality(String), + `operating_system` LowCardinality(String), + `browser` LowCardinality(String), + `utm_medium` String CODEC(ZSTD(3)), + `utm_source` String CODEC(ZSTD(3)), + `utm_campaign` String CODEC(ZSTD(3)), + `browser_version` LowCardinality(String), + `operating_system_version` LowCardinality(String), + `subdivision1_code` LowCardinality(String), + `subdivision2_code` LowCardinality(String), + `city_geoname_id` UInt32, + `utm_content` String CODEC(ZSTD(3)), + `utm_term` String CODEC(ZSTD(3)), + `transferred_from` String, + `entry_meta.key` Array(String) CODEC(ZSTD(3)), + `entry_meta.value` Array(String) CODEC(ZSTD(3)), + `exit_page_hostname` String CODEC(ZSTD(3)), + `city` UInt32 ALIAS city_geoname_id, + `country` LowCardinality(FixedString(2)) ALIAS country_code, + `device` LowCardinality(String) ALIAS screen_size, + `entry_page_hostname` String ALIAS hostname, + `os` LowCardinality(String) ALIAS operating_system, + `os_version` LowCardinality(String) ALIAS operating_system_version, + `region` LowCardinality(String) ALIAS subdivision1_code, + `screen` LowCardinality(String) ALIAS screen_size, + `source` String ALIAS referrer_source, + `country_name` String ALIAS dictGet('plausible_events_db.location_data_dict', 'name', ('country', country_code)), + `region_name` String ALIAS dictGet('plausible_events_db.location_data_dict', 'name', ('subdivision', subdivision1_code)), + `city_name` String ALIAS dictGet('plausible_events_db.location_data_dict', 'name', ('city', city_geoname_id)), + `channel` LowCardinality(String), + INDEX minmax_timestamp timestamp TYPE minmax GRANULARITY 1 +) +ENGINE = VersionedCollapsingMergeTree(sign, events) +PARTITION BY toYYYYMM(start) +PRIMARY KEY (site_id, toDate(start), user_id, session_id) +ORDER BY (site_id, toDate(start), user_id, session_id) +SAMPLE BY user_id +SETTINGS index_granularity = 8192; + +CREATE DICTIONARY plausible_events_db.location_data_dict +( + `type` String, + `id` String, + `name` String +) +PRIMARY KEY type, id +SOURCE(CLICKHOUSE(TABLE location_data DB 'plausible_events_db')) +LIFETIME(MIN 0 MAX 0) +LAYOUT(COMPLEX_KEY_CACHE(SIZE_IN_CELLS 500000)); + +CREATE TABLE plausible_events_db.location_data +( + `type` LowCardinality(String), + `id` String, + `name` String +) +ENGINE = MergeTree +ORDER BY (type, id) +SETTINGS index_granularity = 128 +COMMENT '2024-07-09'; + +CREATE TABLE plausible_events_db.ingest_counters +( + `event_timebucket` DateTime, + `domain` LowCardinality(String), + `site_id` Nullable(UInt64), + `metric` LowCardinality(String), + `value` UInt64 +) +ENGINE = SummingMergeTree(value) +ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket)) +SETTINGS index_granularity = 8192; + +CREATE TABLE plausible_events_db.imported_visitors +( + `site_id` UInt64, + `date` Date, + `visitors` UInt64, + `pageviews` UInt64, + `bounces` UInt64, + `visits` UInt64, + `visit_duration` UInt64, + `import_id` UInt64 +) +ENGINE = MergeTree +ORDER BY (site_id, date) +SETTINGS index_granularity = 8192, replicated_deduplication_window = 0; + +CREATE TABLE plausible_events_db.imported_sources +( + `site_id` UInt64, + `date` Date, + `source` String, + `utm_medium` String, + `utm_campaign` String, + `utm_content` String, + `utm_term` String, + `visitors` UInt64, + `visits` UInt64, + `visit_duration` UInt64, + `bounces` UInt32, + `import_id` UInt64, + `pageviews` UInt64, + `referrer` String, + `utm_source` String +) +ENGINE = MergeTree +ORDER BY (site_id, date, source) +SETTINGS index_granularity = 8192, replicated_deduplication_window = 0; + +CREATE TABLE plausible_events_db.imported_pages +( + `site_id` UInt64, + `date` Date, + `hostname` String, + `page` String, + `visitors` UInt64, + `pageviews` UInt64, + `exits` UInt64, + `time_on_page` UInt64, + `import_id` UInt64, + `visits` UInt64, + `active_visitors` UInt64 +) +ENGINE = MergeTree +ORDER BY (site_id, date, hostname, page) +SETTINGS index_granularity = 8192, replicated_deduplication_window = 0; + +CREATE TABLE plausible_events_db.imported_operating_systems +( + `site_id` UInt64, + `date` Date, + `operating_system` String, + `visitors` UInt64, + `visits` UInt64, + `visit_duration` UInt64, + `bounces` UInt32, + `import_id` UInt64, + `pageviews` UInt64, + `operating_system_version` String +) +ENGINE = MergeTree +ORDER BY (site_id, date, operating_system) +SETTINGS index_granularity = 8192, replicated_deduplication_window = 0; + +CREATE TABLE plausible_events_db.imported_locations +( + `site_id` UInt64, + `date` Date, + `country` String, + `region` String, + `city` UInt64, + `visitors` UInt64, + `visits` UInt64, + `visit_duration` UInt64, + `bounces` UInt32, + `import_id` UInt64, + `pageviews` UInt64, + `country_name` String ALIAS dictGet('plausible_events_db.location_data_dict', 'name', ('country', country)), + `region_name` String ALIAS dictGet('plausible_events_db.location_data_dict', 'name', ('subdivision', region)), + `city_name` String ALIAS dictGet('plausible_events_db.location_data_dict', 'name', ('city', city)) +) +ENGINE = MergeTree +ORDER BY (site_id, date, country, region, city) +SETTINGS index_granularity = 8192, replicated_deduplication_window = 0; + +CREATE TABLE plausible_events_db.imported_exit_pages +( + `site_id` UInt64, + `date` Date, + `exit_page` String, + `visitors` UInt64, + `exits` UInt64, + `import_id` UInt64, + `pageviews` UInt64, + `bounces` UInt32, + `visit_duration` UInt64 +) +ENGINE = MergeTree +ORDER BY (site_id, date, exit_page) +SETTINGS index_granularity = 8192, replicated_deduplication_window = 0; + +CREATE TABLE plausible_events_db.imported_entry_pages +( + `site_id` UInt64, + `date` Date, + `entry_page` String, + `visitors` UInt64, + `entrances` UInt64, + `visit_duration` UInt64, + `bounces` UInt32, + `import_id` UInt64, + `pageviews` UInt64 +) +ENGINE = MergeTree +ORDER BY (site_id, date, entry_page) +SETTINGS index_granularity = 8192, replicated_deduplication_window = 0; + +CREATE TABLE plausible_events_db.imported_devices +( + `site_id` UInt64, + `date` Date, + `device` String, + `visitors` UInt64, + `visits` UInt64, + `visit_duration` UInt64, + `bounces` UInt32, + `import_id` UInt64, + `pageviews` UInt64 +) +ENGINE = MergeTree +ORDER BY (site_id, date, device) +SETTINGS index_granularity = 8192, replicated_deduplication_window = 0; + +CREATE TABLE plausible_events_db.imported_custom_events +( + `site_id` UInt64, + `import_id` UInt64, + `date` Date, + `name` String CODEC(ZSTD(3)), + `link_url` String CODEC(ZSTD(3)), + `path` String CODEC(ZSTD(3)), + `visitors` UInt64, + `events` UInt64 +) +ENGINE = MergeTree +ORDER BY (site_id, import_id, date, name) +SETTINGS replicated_deduplication_window = 0, index_granularity = 8192; + +CREATE TABLE plausible_events_db.imported_browsers +( + `site_id` UInt64, + `date` Date, + `browser` String, + `visitors` UInt64, + `visits` UInt64, + `visit_duration` UInt64, + `bounces` UInt32, + `import_id` UInt64, + `pageviews` UInt64, + `browser_version` String +) +ENGINE = MergeTree +ORDER BY (site_id, date, browser) +SETTINGS index_granularity = 8192, replicated_deduplication_window = 0; + +CREATE TABLE plausible_events_db.events_v2 +( + `timestamp` DateTime CODEC(Delta(4), LZ4), + `name` LowCardinality(String), + `site_id` UInt64, + `user_id` UInt64, + `session_id` UInt64, + `hostname` String CODEC(ZSTD(3)), + `pathname` String CODEC(ZSTD(3)), + `referrer` String CODEC(ZSTD(3)), + `referrer_source` String CODEC(ZSTD(3)), + `country_code` FixedString(2), + `screen_size` LowCardinality(String), + `operating_system` LowCardinality(String), + `browser` LowCardinality(String), + `utm_medium` String CODEC(ZSTD(3)), + `utm_source` String CODEC(ZSTD(3)), + `utm_campaign` String CODEC(ZSTD(3)), + `meta.key` Array(String) CODEC(ZSTD(3)), + `meta.value` Array(String) CODEC(ZSTD(3)), + `browser_version` LowCardinality(String), + `operating_system_version` LowCardinality(String), + `subdivision1_code` LowCardinality(String), + `subdivision2_code` LowCardinality(String), + `city_geoname_id` UInt32, + `utm_content` String CODEC(ZSTD(3)), + `utm_term` String CODEC(ZSTD(3)), + `revenue_reporting_amount` Nullable(Decimal(18, 3)), + `revenue_reporting_currency` FixedString(3), + `revenue_source_amount` Nullable(Decimal(18, 3)), + `revenue_source_currency` FixedString(3), + `city` UInt32 ALIAS city_geoname_id, + `country` LowCardinality(FixedString(2)) ALIAS country_code, + `device` LowCardinality(String) ALIAS screen_size, + `os` LowCardinality(String) ALIAS operating_system, + `os_version` LowCardinality(String) ALIAS operating_system_version, + `region` LowCardinality(String) ALIAS subdivision1_code, + `screen` LowCardinality(String) ALIAS screen_size, + `source` String ALIAS referrer_source, + `country_name` String ALIAS dictGet('plausible_events_db.location_data_dict', 'name', ('country', country_code)), + `region_name` String ALIAS dictGet('plausible_events_db.location_data_dict', 'name', ('subdivision', subdivision1_code)), + `city_name` String ALIAS dictGet('plausible_events_db.location_data_dict', 'name', ('city', city_geoname_id)), + `channel` LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(timestamp) +PRIMARY KEY (site_id, toDate(timestamp), name, user_id) +ORDER BY (site_id, toDate(timestamp), name, user_id, timestamp) +SAMPLE BY user_id +SETTINGS index_granularity = 8192; + +CREATE TABLE plausible_events_db.schema_migrations +( + `version` Int64, + `inserted_at` DateTime +) +ENGINE = TinyLog; + +CREATE DICTIONARY plausible_events_db.location_data_dict +( + `type` String, + `id` String, + `name` String +) +PRIMARY KEY type, id +SOURCE(CLICKHOUSE(TABLE location_data DB 'plausible_events_db')) +LIFETIME(MIN 0 MAX 0) +LAYOUT(COMPLEX_KEY_CACHE(SIZE_IN_CELLS 500000)); + +INSERT INTO "plausible_events_db"."schema_migrations" (version, inserted_at) VALUES +(20200915070607,'2024-09-11 09:23:21'), +(20200918075025,'2024-09-11 09:23:21'), +(20201020083739,'2024-09-11 09:23:21'), +(20201106125234,'2024-09-11 09:23:21'), +(20210323130440,'2024-09-11 09:23:21'), +(20210712214034,'2024-09-11 09:23:21'), +(20211017093035,'2024-09-11 09:23:21'), +(20211112130238,'2024-09-11 09:23:21'), +(20220310104931,'2024-09-11 09:23:21'), +(20220404123000,'2024-09-11 09:23:21'), +(20220421161259,'2024-09-11 09:23:21'), +(20220422075510,'2024-09-11 09:23:21'), +(20230124140348,'2024-09-11 09:23:21'), +(20230210140348,'2024-09-11 09:23:21'), +(20230214114402,'2024-09-11 09:23:21'), +(20230320094327,'2024-09-11 09:23:21'), +(20230417104025,'2024-09-11 09:23:21'), +(20230509124919,'2024-09-11 09:23:21'), +(20231017073642,'2024-09-11 09:23:21'), +(20240123142959,'2024-09-11 09:23:21'), +(20240209085338,'2024-09-11 09:23:21'), +(20240220123656,'2024-09-11 09:23:21'), +(20240222082911,'2024-09-11 09:23:21'), +(20240305085310,'2024-09-11 09:23:21'), +(20240326134840,'2024-09-11 09:23:21'), +(20240327085855,'2024-09-11 09:23:21'), +(20240419133926,'2024-09-11 09:23:21'), +(20240423094014,'2024-09-11 09:23:21'), +(20240502115822,'2024-09-11 09:23:21'), +(20240709181437,'2024-09-11 09:23:22'), +(20240801091615,'2024-09-11 09:23:22'), +(20240829092858,'2024-09-11 09:23:22'); diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.json new file mode 100644 index 0000000000000000000000000000000000000000..c96ab95536b6eb014c9a83117b6f21b88b5bcc6a --- /dev/null +++ b/priv/json-schemas/query-api-schema.json @@ -0,0 +1,533 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "site_id": { + "type": "string", + "description": "Domain of site to query" + }, + "metrics": { + "type": "array", + "items": { + "$ref": "#/definitions/metric" + }, + "minItems": 1, + "uniqueItems": true, + "description": "List of metrics to query" + }, + "date_range": { + "oneOf": [ + { "$ref": "#/definitions/date_range_shorthand" }, + { "$ref": "#/definitions/date_time_range" }, + { "$ref": "#/definitions/date_range" } + ], + "description": "Date range to query" + }, + "dimensions": { + "type": "array", + "items": { + "$ref": "#/definitions/dimensions" + }, + "description": "What to group the results by. Same as `property` in Plausible API v1", + "uniqueItems": true + }, + "filters": { + "type": "array", + "items": { "$ref": "#/definitions/filter_tree" }, + "description": "How to drill into your data" + }, + "order_by": { + "type": "array", + "items": { + "$ref": "#/definitions/order_by_entry" + }, + "description": "How to order query results" + }, + "include": { + "type": "object", + "additionalProperties": false, + "properties": { + "time_labels": { + "type": "boolean", + "markdownDescription": "Requires a `time` dimension. If enabled, returns each time bucket under `meta.time_labels`" + }, + "imports": { + "type": "boolean" + }, + "total_rows": { + "type": "boolean", + "default": false, + "description": "If set, returns the total number of result rows rows before pagination under `meta.total_rows`" + }, + "trim_relative_date_range": { + "type": "boolean", + "default": false, + "description": "If set and using `day`, `month` or `year` date_ranges, the query will be trimmed to the current date" + } + } + }, + "pagination": { + "$ref": "#/definitions/pagination" + } + }, + "required": ["site_id", "metrics", "date_range"], + "additionalProperties": false, + "definitions": { + "date_time_range": { + "type": "array", + "additionalItems": false, + "minItems": 2, + "maxItems": 2, + "markdownDescription": "A list of two ISO8601 datetimes.", + "examples": [["2024-01-01T00:00:00+03:00", "2024-01-02T12:00:00+03:00"]], + "items": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string", + "format": "date-time" + } + ] + }, + "date_range": { + "type": "array", + "additionalItems": false, + "minItems": 2, + "maxItems": 2, + "markdownDescription": "A list of two ISO8601 dates.", + "examples": [["2024-09-01", "2024-09-30"]], + "items": [ + { + "type": "string", + "format": "date" + }, + { + "type": "string", + "format": "date" + } + ] + }, + "date_range_shorthand": { + "anyOf": [ + { + "const": "all", + "description": "Since the start of stats in Plausible" + }, + { + "const": "day", + "description": "Last day" + }, + { + "const": "24h", + "description": "Last 24 hours (rolling window from now-24h to now)" + }, + { + "const": "7d", + "description": "Last 7 days relative to today" + }, + { + "const": "28d", + "description": "Last 28 days relative to today" + }, + { + "const": "30d", + "description": "Last 30 days relative to today" + }, + { + "const": "91d", + "description": "Last 91 days relative to today" + }, + { + "const": "month", + "description": "Current calendar month" + }, + { + "const": "6mo", + "description": "Last 6 calendar months relative to this month" + }, + { + "const": "12mo", + "description": "Last 12 calendar months relative to this month" + }, + { + "const": "year", + "description": "Since the start of this year" + }, + { + "type": "string", + "pattern": "^\\d+d$", + "description": "Last n days relative to today" + }, + { + "type": "string", + "pattern": "^\\d+mo$", + "description": "Last n months relative to this month" + } + ] + }, + "comparison_mode": { + "oneOf": [ + { + "const": "previous_period", + "description": "Comparison date range shortcut - shift back the original query date range by its length" + }, + { + "const": "year_over_year", + "description": "Comparison date range shortcut - shift back the original query date range by a year" + } + ] + }, + "metric": { + "oneOf": [ + { + "const": "visitors", + "description": "Metric counting the number of unique visitors" + }, + { + "const": "visits", + "description": "Metric counting the number of visits/sessions" + }, + { + "const": "pageviews", + "markdownDescription": "Metric counting the number of `pageview` events" + }, + { + "const": "views_per_visit", + "description": "Metric for the number of pageviews divided by the number of visits. Returns a floating point number." + }, + { + "const": "bounce_rate", + "description": "Bounce rate percentage" + }, + { + "const": "visit_duration", + "description": "Visit duration in seconds" + }, + { + "const": "events", + "markdownDescription": "The number of events (pageviews + custom events). When filtering by a goal, this metric corresponds to `Total Conversions` in the dashboard." + }, + { + "const": "percentage", + "markdownDescription": "The percentage of visitors of total who fall into this category. Requires `dimensions` not to be empty" + }, + { + "const": "conversion_rate", + "markdownDescription": "The percentage of visitors who completed the goal. Requires: `dimension` list passed, an `event:goal` filter or `event:goal` dimension" + }, + { + "const": "group_conversion_rate", + "markdownDescription": "The percentage of visitors who completed the goal with the same dimension. Requires: dimension list passed, an `event:goal` filter or `event:goal` dimension" + }, + { + "const": "time_on_page", + "markdownDescription": "Average time spent on a given page in a visit in seconds. Requires: `event:page` filter or dimension." + }, + { + "const": "total_revenue", + "markdownDescription": "Total revenue", + "$comment": "only :ee" + }, + { + "const": "average_revenue", + "markdownDescription": "Average revenue", + "$comment": "only :ee" + }, + { + "const": "scroll_depth", + "markdownDescription": "Average page scroll depth. Requires: `event:page` filter or `event:page` dimension" + } + ] + }, + "simple_filter_dimensions": { + "type": "string", + "enum": [ + "event:name", + "event:page", + "event:hostname", + "visit:source", + "visit:channel", + "visit:referrer", + "visit:utm_medium", + "visit:utm_source", + "visit:utm_campaign", + "visit:utm_content", + "visit:utm_term", + "visit:screen", + "visit:device", + "visit:browser", + "visit:browser_version", + "visit:os", + "visit:os_version", + "visit:country", + "visit:region", + "visit:city", + "visit:country_name", + "visit:region_name", + "visit:city_name", + "visit:entry_page", + "visit:exit_page", + "visit:entry_page_hostname", + "visit:exit_page_hostname" + ] + }, + "custom_property_filter_dimensions": { + "type": "string", + "tsType": "`event:props:${string}`", + "pattern": "^event:props:.+", + "markdownDescription": "Custom property. See [documentation](https://plausible.io/docs/custom-props/introduction) for more information", + "examples": ["event:props:url", "event:props:path"] + }, + "goal_dimension": { + "const": "event:goal", + "markdownDescription": "Goal dimension" + }, + "time_dimensions": { + "oneOf": [ + { + "type": "string", + "enum": ["time", "time:month", "time:week", "time:day", "time:hour"] + } + ] + }, + "dimensions": { + "oneOf": [ + { "$ref": "#/definitions/simple_filter_dimensions" }, + { "$ref": "#/definitions/custom_property_filter_dimensions" }, + { "$ref": "#/definitions/goal_dimension" }, + { "$ref": "#/definitions/time_dimensions" } + ] + }, + "clauses": { + "type": "array", + "minItems": 1, + "items": { + "type": ["string", "integer"] + } + }, + "filter_operation_wildcard": { + "type": "string", + "enum": ["matches_wildcard", "matches_wildcard_not"], + "description": "filter operation" + }, + "filter_operation_regex": { + "type": "string", + "enum": ["matches", "matches_not"], + "description": "filter operation" + }, + "filter_operation_without_goals": { + "type": "string", + "enum": ["is_not", "contains_not"], + "description": "filter operation" + }, + "filter_with_pattern": { + "type": "array", + "additionalItems": false, + "minItems": 3, + "maxItems": 3, + "items": [ + { "$ref": "#/definitions/filter_operation_regex" }, + { + "oneOf": [ + { "$ref": "#/definitions/simple_filter_dimensions" }, + { "$ref": "#/definitions/custom_property_filter_dimensions" } + ] + }, + { "$ref": "#/definitions/clauses" } + ] + }, + "filter_without_goals": { + "type": "array", + "additionalItems": false, + "minItems": 3, + "maxItems": 4, + "items": [ + { "$ref": "#/definitions/filter_operation_without_goals" }, + { + "oneOf": [ + { "$ref": "#/definitions/simple_filter_dimensions" }, + { "$ref": "#/definitions/custom_property_filter_dimensions" } + ] + }, + { "$ref": "#/definitions/clauses" }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "case_sensitive": { + "type": "boolean", + "default": true + } + } + } + ] + }, + "filter_with_is": { + "type": "array", + "additionalItems": false, + "minItems": 3, + "maxItems": 4, + "items": [ + { + "const": "is", + "description": "filter operation" + }, + { + "oneOf": [ + { "$ref": "#/definitions/goal_dimension" }, + { "$ref": "#/definitions/simple_filter_dimensions" }, + { "$ref": "#/definitions/custom_property_filter_dimensions" }, + { "const": "segment" } + ] + }, + { + "$ref": "#/definitions/clauses" + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "case_sensitive": { + "type": "boolean", + "default": true + } + } + } + ] + }, + "filter_with_contains": { + "type": "array", + "additionalItems": false, + "minItems": 3, + "maxItems": 4, + "items": [ + { + "const": "contains", + "description": "filter operation" + }, + { + "oneOf": [ + { "$ref": "#/definitions/goal_dimension" }, + { "$ref": "#/definitions/simple_filter_dimensions" }, + { "$ref": "#/definitions/custom_property_filter_dimensions" } + ] + }, + { + "$ref": "#/definitions/clauses" + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "case_sensitive": { + "type": "boolean", + "default": true + } + } + } + ] + }, + "filter_entry": { + "oneOf": [ + { "$ref": "#/definitions/filter_without_goals" }, + { "$ref": "#/definitions/filter_with_is" }, + { "$ref": "#/definitions/filter_with_contains" }, + { "$ref": "#/definitions/filter_with_pattern" } + ] + }, + "filter_tree": { + "oneOf": [ + { "$ref": "#/definitions/filter_entry" }, + { "$ref": "#/definitions/filter_and_or" }, + { "$ref": "#/definitions/filter_not" }, + { "$ref": "#/definitions/filter_has_done" } + ] + }, + "filter_not": { + "type": "array", + "additionalItems": false, + "minItems": 2, + "maxItems": 2, + "items": [ + { + "type": "string", + "enum": ["not"] + }, + { "$ref": "#/definitions/filter_tree" } + ] + }, + "filter_and_or": { + "type": "array", + "additionalItems": false, + "minItems": 2, + "maxItems": 2, + "items": [ + { + "type": "string", + "enum": ["and", "or"] + }, + { + "type": "array", + "items": { "$ref": "#/definitions/filter_tree" }, + "minItems": 1 + } + ] + }, + "filter_has_done": { + "type": "array", + "additionalItems": false, + "minItems": 2, + "maxItems": 2, + "items": [ + { + "type": "string", + "enum": ["has_done", "has_not_done"] + }, + { "$ref": "#/definitions/filter_tree" } + ] + }, + "pagination": { + "type": "object", + "additionalProperties": false, + "properties": { + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 10000, + "description": "Number of rows to limit result to." + }, + "offset": { + "type": "integer", + "minimum": 0, + "default": 0, + "description": "Pagination offset." + } + } + }, + "sort_direction": { + "type": "string", + "enum": ["asc", "desc"], + "description": "Sorting order" + }, + "order_by_entry": { + "type": "array", + "additionalItems": false, + "minItems": 2, + "maxItems": 2, + "items": [ + { + "oneOf": [ + { "$ref": "#/definitions/metric" }, + { "$ref": "#/definitions/simple_filter_dimensions" }, + { "$ref": "#/definitions/custom_property_filter_dimensions" }, + { "$ref": "#/definitions/time_dimensions" } + ], + "markdownDescription": "Metric or dimension to order by. Must be listed under `metrics` or `dimensions`" + }, + { "$ref": "#/definitions/sort_direction" } + ] + } + } +} diff --git a/priv/legacy_plans.json b/priv/legacy_plans.json new file mode 100644 index 0000000000000000000000000000000000000000..024807a6fdf6292f0b2ef57f67a267ed8d9219bc --- /dev/null +++ b/priv/legacy_plans.json @@ -0,0 +1,22 @@ +[ + { + "kind":"growth", + "generation": 1, + "monthly_pageview_limit":1000000, + "yearly_product_id":"590753", + "monthly_product_id":"558746", + "site_limit":50, + "team_member_limit":"unlimited", + "features":["goals","props","stats_api"] + }, + { + "kind":"growth", + "generation": 1, + "monthly_pageview_limit":150000000, + "yearly_product_id":"648089", + "monthly_product_id":null, + "site_limit":50, + "team_member_limit":"unlimited", + "features":["goals","props","stats_api"] + } +] diff --git a/priv/link_favicon.ico b/priv/link_favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..35aa65d28acb14ed2f1b2b2467d74e244b00bc3d Binary files /dev/null and b/priv/link_favicon.ico differ diff --git a/priv/link_favicon.svg b/priv/link_favicon.svg new file mode 100644 index 0000000000000000000000000000000000000000..7a9008b2ba18052aa373640c630ee9e4b63b4e41 --- /dev/null +++ b/priv/link_favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/priv/paddle.pem b/priv/paddle.pem new file mode 100644 index 0000000000000000000000000000000000000000..110be6cc1537ff650ae66856ebe3fb5a09379130 --- /dev/null +++ b/priv/paddle.pem @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvn1//X4/nkeKtF8aFPi5 +1ngjAvb68oGnzBmft0dOUxl6t4wSEl93efWI9xC24qPDGxippROPqRlelUne96zO +ZdAj3JZT8K77m0lS/EbtHBF6GSvETSr/UceOcF//pbG8Q7EQDN3+/VfiTlHpzG/s +V9gaOGNYJ28dlaS7gNQZYO+eeoWA92RLLk7X73F/lPQtJ70rxop+kLyAA4fDr08k +TR4gWBAyAkBsyC8W0v9b0zHBqxiiGdS2wQRWzMGrPHti//LGiR92PVxvP3QQd8iW +RELlcXOKLOhPX3LzU9DAWNFlRmtdQ4J3F7t642VIFWaAo2zOdeb4J8cmcoViYNp3 +7tWUQPyVQnm2V8gjfIDNYIG1pMjleikCcsL823rDNuV1LWzsxnZHihzX2KvUhdVY +Js2N89w1YbeOEcvO8d6fFYEkSckOkkrhK9MLGla6OY+nHds0/Clykq0BSD2kbSpr +mZ3qWiBZtbR7PEiEGr6pNbupFWVsV7OyOXR4MmU89Ml7bqnlGOIUy+o9cG7E3FfJ +klALjZIpf7FCZyQH7b6KU6IZlZK+hdXEScx2gL3dyQAzNXNV+hFAyvCKhqns7lQO +u6yrbmj0tDHUN6ing4K3UQ01zmfSv5s83usrqRud27ZSnInYXKeKzcj1mYgDELOS +GNGaLnkTNF9TKBP8qYz++/sCAwEAAQ== +-----END PUBLIC KEY----- diff --git a/priv/paddle_sandbox.pem b/priv/paddle_sandbox.pem new file mode 100644 index 0000000000000000000000000000000000000000..2ccd015491cd973e8dd00dd22c4a5c869550a2f5 --- /dev/null +++ b/priv/paddle_sandbox.pem @@ -0,0 +1,14 @@ +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAwWBTLMN0cdzxAYzzItq1 +vWipCcmkaplQPw2BWDfw4+No/GWpyxi08O/9HZEiT4GxEMcmH9QDHfhAzsHjKDCL +1thBlzX0K1qJnuD+hYA9JoEO0DfR5kMIbEJ1rkNhaFjWuI8p70GvRWqcMpwCLUih +7x51Ksin5+zVqf1VP5k5fjzmjOvgUjWwled/Mpy0ts2UF//MGjd+HlbHzW6c86Ck +QdM+t19aMOOKPZgg1TkK+1yykRekWlrDPU2ktAPgf7M9JaugPw1ZBUrCD7KGRGEP +Q+wLrP4UpY0kB39Flf6aNi44L1zjqGgav1UtS44xOhEy55S3pR/6sx9rPXrIKhK7 +n1ZZmj/5xUY0L0efVzJXl8NKrSkeyaLJb9G1y2dqh1lQCD7EwlD6ZMT9iZ2WleKK +QuBvEoxSoecISSFvnDHHg3iZYOuABysEOBxSIX37oQpf9AqiTSgJLdrNa1q2ezlf +4GW+eFqkmMt+yN3F2453msNefqEtLK33+eFKyhkPBqgjFiK2CwhpB0QSdaaGj3cG +0ssbm7OGY/rJDp6NLxUQ7x8Ie8BNfZoAaH3WKjDjaA489qRrlh7YIiEjKvl5Y+xv +zmDehftRDvWMtSHRDkmmvzw3i5in+sfHGbDek6m5dX1hO2DfYEMVNSN5qk/ppg2M +ZL09XH4lvIY50LHq1FNwa3ECAwEAAQ== +-----END PUBLIC KEY----- diff --git a/priv/plan_prices.json b/priv/plan_prices.json new file mode 100644 index 0000000000000000000000000000000000000000..505ee4fb6cb4e632be466e702d03070269aea1d5 --- /dev/null +++ b/priv/plan_prices.json @@ -0,0 +1,157 @@ +{ + "558018": 6, + "558745": 12, + "558746": 36, + "572810": 48, + "590752": 96, + "590753": 288, + "597309": 69, + "597310": 552, + "597311": 99, + "597312": 792, + "597485": 18, + "597486": 144, + "597487": 27, + "597488": 216, + "597642": 48, + "597643": 384, + "642352": 150, + "642354": 1200, + "642355": 225, + "642356": 1800, + "648089": 4800, + "650652": 330, + "650653": 2640, + "653232": 60, + "653234": 120, + "653236": 200, + "653237": 20, + "653238": 30, + "653239": 300, + "653240": 50, + "653242": 500, + "653253": 70, + "653254": 700, + "653255": 100, + "653256": 1000, + "653257": 1500, + "653258": 2250, + "653259": 3300, + "654177": 6, + "654178": 12, + "654181": 150, + "654182": 225, + "654183": 330, + "749342": 9, + "749343": 90, + "749344": 19, + "749345": 190, + "749346": 29, + "749347": 290, + "749348": 49, + "749349": 490, + "749350": 69, + "749352": 690, + "749353": 89, + "749355": 890, + "749356": 129, + "749357": 1290, + "749358": 169, + "749359": 1690, + "857079": 90, + "857080": 190, + "857081": 290, + "857082": 490, + "857083": 690, + "857084": 890, + "857085": 1290, + "857086": 1690, + "857087": 190, + "857088": 390, + "857089": 590, + "857090": 990, + "857091": 1390, + "857092": 1790, + "857093": 2590, + "857094": 3390, + "857097": 9, + "857098": 19, + "857099": 29, + "857100": 49, + "857101": 69, + "857102": 89, + "857103": 129, + "857104": 169, + "857105": 19, + "857106": 39, + "857107": 59, + "857108": 99, + "857109": 139, + "857110": 179, + "857111": 259, + "857112": 339, + "857481": 19, + "857482": 190, + "857483": 39, + "857484": 390, + "857486": 59, + "857487": 590, + "857490": 99, + "857491": 990, + "857493": 139, + "857494": 1390, + "857495": 179, + "857496": 1790, + "857498": 259, + "857500": 2590, + "857501": 339, + "857502": 3390, + "910413": 9, + "910414": 90, + "910415": 19, + "910416": 190, + "910417": 29, + "910418": 290, + "910419": 49, + "910420": 490, + "910421": 69, + "910422": 690, + "910423": 89, + "910424": 890, + "910425": 129, + "910426": 1290, + "910427": 169, + "910428": 1690, + "910429": 14, + "910430": 140, + "910431": 29, + "910432": 290, + "910433": 44, + "910434": 440, + "910435": 74, + "910436": 740, + "910437": 104, + "910438": 1040, + "910439": 134, + "910440": 1340, + "910441": 194, + "910442": 1940, + "910443": 254, + "910444": 2540, + "910445": 19, + "910446": 190, + "910447": 39, + "910448": 390, + "910449": 59, + "910450": 590, + "910451": 99, + "910452": 990, + "910453": 139, + "910454": 1390, + "910455": 179, + "910456": 1790, + "910457": 259, + "910458": 2590, + "910459": 339, + "910460": 3390 +} \ No newline at end of file diff --git a/priv/plans_v1.json b/priv/plans_v1.json new file mode 100644 index 0000000000000000000000000000000000000000..065eb209d1622b74f03fb339b68f30fa8a127149 --- /dev/null +++ b/priv/plans_v1.json @@ -0,0 +1,152 @@ +[ + { + "kind": "growth", + "generation": 1, + "monthly_pageview_limit": 10000, + "monthly_product_id": "558018", + "yearly_product_id": "572810", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 1, + "monthly_pageview_limit": 100000, + "monthly_product_id": "558745", + "yearly_product_id": "590752", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 1, + "monthly_pageview_limit": 200000, + "monthly_product_id": "597485", + "yearly_product_id": "597486", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 1, + "monthly_pageview_limit": 500000, + "monthly_product_id": "597487", + "yearly_product_id": "597488", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 1, + "monthly_pageview_limit": 1000000, + "monthly_product_id": "597642", + "yearly_product_id": "597643", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 1, + "monthly_pageview_limit": 2000000, + "monthly_product_id": "597309", + "yearly_product_id": "597310", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 1, + "monthly_pageview_limit": 5000000, + "monthly_product_id": "597311", + "yearly_product_id": "597312", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 1, + "monthly_pageview_limit": 10000000, + "monthly_product_id": "642352", + "yearly_product_id": "642354", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 1, + "monthly_pageview_limit": 20000000, + "monthly_product_id": "642355", + "yearly_product_id": "642356", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 1, + "monthly_pageview_limit": 50000000, + "monthly_product_id": "650652", + "yearly_product_id": "650653", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + } +] \ No newline at end of file diff --git a/priv/plans_v2.json b/priv/plans_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..cb24b7d7488cde6f6f895c1a8f6873215acb34dc --- /dev/null +++ b/priv/plans_v2.json @@ -0,0 +1,152 @@ +[ + { + "kind": "growth", + "generation": 2, + "monthly_pageview_limit": 10000, + "monthly_product_id": "654177", + "yearly_product_id": "653232", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 2, + "monthly_pageview_limit": 100000, + "monthly_product_id": "654178", + "yearly_product_id": "653234", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 2, + "monthly_pageview_limit": 200000, + "monthly_product_id": "653237", + "yearly_product_id": "653236", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 2, + "monthly_pageview_limit": 500000, + "monthly_product_id": "653238", + "yearly_product_id": "653239", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 2, + "monthly_pageview_limit": 1000000, + "monthly_product_id": "653240", + "yearly_product_id": "653242", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 2, + "monthly_pageview_limit": 2000000, + "monthly_product_id": "653253", + "yearly_product_id": "653254", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 2, + "monthly_pageview_limit": 5000000, + "monthly_product_id": "653255", + "yearly_product_id": "653256", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 2, + "monthly_pageview_limit": 10000000, + "monthly_product_id": "654181", + "yearly_product_id": "653257", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 2, + "monthly_pageview_limit": 20000000, + "monthly_product_id": "654182", + "yearly_product_id": "653258", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 2, + "monthly_pageview_limit": 50000000, + "monthly_product_id": "654183", + "yearly_product_id": "653259", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + } +] \ No newline at end of file diff --git a/priv/plans_v3.json b/priv/plans_v3.json new file mode 100644 index 0000000000000000000000000000000000000000..0d4310a3f9aa21d7dc6018a9d0fc7d0feafb0cfd --- /dev/null +++ b/priv/plans_v3.json @@ -0,0 +1,282 @@ +[ + { + "kind": "growth", + "generation": 3, + "monthly_pageview_limit": 10000, + "monthly_product_id": "749342", + "yearly_product_id": "749343", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 3, + "monthly_pageview_limit": 100000, + "monthly_product_id": "749344", + "yearly_product_id": "749345", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 3, + "monthly_pageview_limit": 200000, + "monthly_product_id": "749346", + "yearly_product_id": "749347", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 3, + "monthly_pageview_limit": 500000, + "monthly_product_id": "749348", + "yearly_product_id": "749349", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 3, + "monthly_pageview_limit": 1000000, + "monthly_product_id": "749350", + "yearly_product_id": "749352", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 3, + "monthly_pageview_limit": 2000000, + "monthly_product_id": "749353", + "yearly_product_id": "749355", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 3, + "monthly_pageview_limit": 5000000, + "monthly_product_id": "749356", + "yearly_product_id": "749357", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "growth", + "generation": 3, + "monthly_pageview_limit": 10000000, + "monthly_product_id": "749358", + "yearly_product_id": "749359", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "stats_api", + "shared_links" + ] + }, + { + "kind": "business", + "generation": 3, + "monthly_pageview_limit": 10000, + "monthly_product_id": "857481", + "yearly_product_id": "857482", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ] + }, + { + "kind": "business", + "generation": 3, + "monthly_pageview_limit": 100000, + "monthly_product_id": "857483", + "yearly_product_id": "857484", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ] + }, + { + "kind": "business", + "generation": 3, + "monthly_pageview_limit": 200000, + "monthly_product_id": "857486", + "yearly_product_id": "857487", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ] + }, + { + "kind": "business", + "generation": 3, + "monthly_pageview_limit": 500000, + "monthly_product_id": "857490", + "yearly_product_id": "857491", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ] + }, + { + "kind": "business", + "generation": 3, + "monthly_pageview_limit": 1000000, + "monthly_product_id": "857493", + "yearly_product_id": "857494", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ] + }, + { + "kind": "business", + "generation": 3, + "monthly_pageview_limit": 2000000, + "monthly_product_id": "857495", + "yearly_product_id": "857496", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ] + }, + { + "kind": "business", + "generation": 3, + "monthly_pageview_limit": 5000000, + "monthly_product_id": "857498", + "yearly_product_id": "857500", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ] + }, + { + "kind": "business", + "generation": 3, + "monthly_pageview_limit": 10000000, + "monthly_product_id": "857501", + "yearly_product_id": "857502", + "site_limit": 50, + "team_member_limit": "unlimited", + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ] + } +] diff --git a/priv/plans_v4.json b/priv/plans_v4.json new file mode 100644 index 0000000000000000000000000000000000000000..29ed585e95e5ac5bb1f439b4ca588a165740ab1f --- /dev/null +++ b/priv/plans_v4.json @@ -0,0 +1,282 @@ +[ + { + "kind": "growth", + "generation": 4, + "monthly_pageview_limit": 10000, + "monthly_product_id": "857097", + "yearly_product_id": "857079", + "site_limit": 10, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 4, + "monthly_pageview_limit": 100000, + "monthly_product_id": "857098", + "yearly_product_id": "857080", + "site_limit": 10, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 4, + "monthly_pageview_limit": 200000, + "monthly_product_id": "857099", + "yearly_product_id": "857081", + "site_limit": 10, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 4, + "monthly_pageview_limit": 500000, + "monthly_product_id": "857100", + "yearly_product_id": "857082", + "site_limit": 10, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 4, + "monthly_pageview_limit": 1000000, + "monthly_product_id": "857101", + "yearly_product_id": "857083", + "site_limit": 10, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 4, + "monthly_pageview_limit": 2000000, + "monthly_product_id": "857102", + "yearly_product_id": "857084", + "site_limit": 10, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 4, + "monthly_pageview_limit": 5000000, + "monthly_product_id": "857103", + "yearly_product_id": "857085", + "site_limit": 10, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 4, + "monthly_pageview_limit": 10000000, + "monthly_product_id": "857104", + "yearly_product_id": "857086", + "site_limit": 10, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links" + ], + "data_retention_in_years": 3 + }, + { + "kind": "business", + "generation": 4, + "monthly_pageview_limit": 10000, + "monthly_product_id": "857105", + "yearly_product_id": "857087", + "site_limit": 50, + "team_member_limit": 10, + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ], + "data_retention_in_years": 5 + }, + { + "kind": "business", + "generation": 4, + "monthly_pageview_limit": 100000, + "monthly_product_id": "857106", + "yearly_product_id": "857088", + "site_limit": 50, + "team_member_limit": 10, + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ], + "data_retention_in_years": 5 + }, + { + "kind": "business", + "generation": 4, + "monthly_pageview_limit": 200000, + "monthly_product_id": "857107", + "yearly_product_id": "857089", + "site_limit": 50, + "team_member_limit": 10, + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ], + "data_retention_in_years": 5 + }, + { + "kind": "business", + "generation": 4, + "monthly_pageview_limit": 500000, + "monthly_product_id": "857108", + "yearly_product_id": "857090", + "site_limit": 50, + "team_member_limit": 10, + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ], + "data_retention_in_years": 5 + }, + { + "kind": "business", + "generation": 4, + "monthly_pageview_limit": 1000000, + "monthly_product_id": "857109", + "yearly_product_id": "857091", + "site_limit": 50, + "team_member_limit": 10, + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ], + "data_retention_in_years": 5 + }, + { + "kind": "business", + "generation": 4, + "monthly_pageview_limit": 2000000, + "monthly_product_id": "857110", + "yearly_product_id": "857092", + "site_limit": 50, + "team_member_limit": 10, + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ], + "data_retention_in_years": 5 + }, + { + "kind": "business", + "generation": 4, + "monthly_pageview_limit": 5000000, + "monthly_product_id": "857111", + "yearly_product_id": "857093", + "site_limit": 50, + "team_member_limit": 10, + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ], + "data_retention_in_years": 5 + }, + { + "kind": "business", + "generation": 4, + "monthly_pageview_limit": 10000000, + "monthly_product_id": "857112", + "yearly_product_id": "857094", + "site_limit": 50, + "team_member_limit": 10, + "features": [ + "goals", + "props", + "revenue_goals", + "funnels", + "stats_api", + "site_segments", + "site_annotations", + "shared_links", + "consolidated_view" + ], + "data_retention_in_years": 5 + } +] diff --git a/priv/plans_v5.json b/priv/plans_v5.json new file mode 100644 index 0000000000000000000000000000000000000000..ede05fec99af0dde45e066c55a93d7268f42630e --- /dev/null +++ b/priv/plans_v5.json @@ -0,0 +1,402 @@ +[ + { + "kind": "starter", + "generation": 5, + "monthly_pageview_limit": 10000, + "monthly_product_id": "910413", + "yearly_product_id": "910414", + "site_limit": 1, + "team_member_limit": 0, + "features": [ + "goals" + ], + "data_retention_in_years": 3 + }, + { + "kind": "starter", + "generation": 5, + "monthly_pageview_limit": 100000, + "monthly_product_id": "910415", + "yearly_product_id": "910416", + "site_limit": 1, + "team_member_limit": 0, + "features": [ + "goals" + ], + "data_retention_in_years": 3 + }, + { + "kind": "starter", + "generation": 5, + "monthly_pageview_limit": 200000, + "monthly_product_id": "910417", + "yearly_product_id": "910418", + "site_limit": 1, + "team_member_limit": 0, + "features": [ + "goals" + ], + "data_retention_in_years": 3 + }, + { + "kind": "starter", + "generation": 5, + "monthly_pageview_limit": 500000, + "monthly_product_id": "910419", + "yearly_product_id": "910420", + "site_limit": 1, + "team_member_limit": 0, + "features": [ + "goals" + ], + "data_retention_in_years": 3 + }, + { + "kind": "starter", + "generation": 5, + "monthly_pageview_limit": 1000000, + "monthly_product_id": "910421", + "yearly_product_id": "910422", + "site_limit": 1, + "team_member_limit": 0, + "features": [ + "goals" + ], + "data_retention_in_years": 3 + }, + { + "kind": "starter", + "generation": 5, + "monthly_pageview_limit": 2000000, + "monthly_product_id": "910423", + "yearly_product_id": "910424", + "site_limit": 1, + "team_member_limit": 0, + "features": [ + "goals" + ], + "data_retention_in_years": 3 + }, + { + "kind": "starter", + "generation": 5, + "monthly_pageview_limit": 5000000, + "monthly_product_id": "910425", + "yearly_product_id": "910426", + "site_limit": 1, + "team_member_limit": 0, + "features": [ + "goals" + ], + "data_retention_in_years": 3 + }, + { + "kind": "starter", + "generation": 5, + "monthly_pageview_limit": 10000000, + "monthly_product_id": "910427", + "yearly_product_id": "910428", + "site_limit": 1, + "team_member_limit": 0, + "features": [ + "goals" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 5, + "monthly_pageview_limit": 10000, + "monthly_product_id": "910429", + "yearly_product_id": "910430", + "site_limit": 3, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 5, + "monthly_pageview_limit": 100000, + "monthly_product_id": "910431", + "yearly_product_id": "910432", + "site_limit": 3, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 5, + "monthly_pageview_limit": 200000, + "monthly_product_id": "910433", + "yearly_product_id": "910434", + "site_limit": 3, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 5, + "monthly_pageview_limit": 500000, + "monthly_product_id": "910435", + "yearly_product_id": "910436", + "site_limit": 3, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 5, + "monthly_pageview_limit": 1000000, + "monthly_product_id": "910437", + "yearly_product_id": "910438", + "site_limit": 3, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 5, + "monthly_pageview_limit": 2000000, + "monthly_product_id": "910439", + "yearly_product_id": "910440", + "site_limit": 3, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 5, + "monthly_pageview_limit": 5000000, + "monthly_product_id": "910441", + "yearly_product_id": "910442", + "site_limit": 3, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations" + ], + "data_retention_in_years": 3 + }, + { + "kind": "growth", + "generation": 5, + "monthly_pageview_limit": 10000000, + "monthly_product_id": "910443", + "yearly_product_id": "910444", + "site_limit": 3, + "team_member_limit": 3, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations" + ], + "data_retention_in_years": 3 + }, + { + "kind": "business", + "generation": 5, + "monthly_pageview_limit": 10000, + "monthly_product_id": "910445", + "yearly_product_id": "910446", + "site_limit": 10, + "team_member_limit": 10, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations", + "props", + "stats_api", + "revenue_goals", + "funnels", + "consolidated_view" + ], + "data_retention_in_years": 5 + }, + { + "kind": "business", + "generation": 5, + "monthly_pageview_limit": 100000, + "monthly_product_id": "910447", + "yearly_product_id": "910448", + "site_limit": 10, + "team_member_limit": 10, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations", + "props", + "stats_api", + "revenue_goals", + "funnels", + "consolidated_view" + ], + "data_retention_in_years": 5 + }, + { + "kind": "business", + "generation": 5, + "monthly_pageview_limit": 200000, + "monthly_product_id": "910449", + "yearly_product_id": "910450", + "site_limit": 10, + "team_member_limit": 10, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations", + "props", + "stats_api", + "revenue_goals", + "funnels", + "consolidated_view" + ], + "data_retention_in_years": 5 + }, + { + "kind": "business", + "generation": 5, + "monthly_pageview_limit": 500000, + "monthly_product_id": "910451", + "yearly_product_id": "910452", + "site_limit": 10, + "team_member_limit": 10, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations", + "props", + "stats_api", + "revenue_goals", + "funnels", + "consolidated_view" + ], + "data_retention_in_years": 5 + }, + { + "kind": "business", + "generation": 5, + "monthly_pageview_limit": 1000000, + "monthly_product_id": "910453", + "yearly_product_id": "910454", + "site_limit": 10, + "team_member_limit": 10, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations", + "props", + "stats_api", + "revenue_goals", + "funnels", + "consolidated_view" + ], + "data_retention_in_years": 5 + }, + { + "kind": "business", + "generation": 5, + "monthly_pageview_limit": 2000000, + "monthly_product_id": "910455", + "yearly_product_id": "910456", + "site_limit": 10, + "team_member_limit": 10, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations", + "props", + "stats_api", + "revenue_goals", + "funnels", + "consolidated_view" + ], + "data_retention_in_years": 5 + }, + { + "kind": "business", + "generation": 5, + "monthly_pageview_limit": 5000000, + "monthly_product_id": "910457", + "yearly_product_id": "910458", + "site_limit": 10, + "team_member_limit": 10, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations", + "props", + "stats_api", + "revenue_goals", + "funnels", + "consolidated_view" + ], + "data_retention_in_years": 5 + }, + { + "kind": "business", + "generation": 5, + "monthly_pageview_limit": 10000000, + "monthly_product_id": "910459", + "yearly_product_id": "910460", + "site_limit": 10, + "team_member_limit": 10, + "features": [ + "goals", + "shared_links", + "site_segments", + "site_annotations", + "props", + "stats_api", + "revenue_goals", + "funnels", + "consolidated_view" + ], + "data_retention_in_years": 5 + } +] diff --git a/priv/ref_inspector/ref_inspector.readme.md b/priv/ref_inspector/ref_inspector.readme.md new file mode 100644 index 0000000000000000000000000000000000000000..0f0dbee3ce5eadc281cde2d6a55bb426424c68ff --- /dev/null +++ b/priv/ref_inspector/ref_inspector.readme.md @@ -0,0 +1,5 @@ +# RefInspector Parser Database + +The file in this directory is taken from the +[snowplow-referer-parser](https://github.com/snowplow-referer-parser/referer-parser) +project. See there for detailed license information about the data contained. diff --git a/priv/ref_inspector/referers.yml b/priv/ref_inspector/referers.yml new file mode 100644 index 0000000000000000000000000000000000000000..b84042f5006dcab1ab039a98941bc2fe6b9008df --- /dev/null +++ b/priv/ref_inspector/referers.yml @@ -0,0 +1,4206 @@ +# ####################################################################################################### +# +# ALL SUPPORTED REFERERS +# +# Broken down into: +# +# 1. Medium-unknown providers +# 2. Email providers +# 3. Social providers +# 4. Search providers +# 5. Paid media + + +# ####################################################################################################### +# +# MEDIUM-UNKNOWN PROVIDERS +# +# We know the source, but not the medium. +# This section is useful for reducing false positives in the other sections + +unknown: + + Google: + domains: + - support.google.com + - developers.google.com + - maps.google.com + - accounts.google.com + - drive.google.com + - sites.google.com + - groups.google.com + - groups.google.co.uk + + Yahoo!: + domains: + - finance.yahoo.com + - news.yahoo.com + - eurosport.yahoo.com + - sports.yahoo.com + - astrology.yahoo.com + - travel.yahoo.com + - answers.yahoo.com + - screen.yahoo.com + - weather.yahoo.com + - messenger.yahoo.com + - games.yahoo.com + - shopping.yahoo.net + - movies.yahoo.com + - cars.yahoo.com + - lifestyle.yahoo.com + - omg.yahoo.com + - match.yahoo.net + + Yandex Maps: + parameters: + - text + domains: + - maps.yandex.ru + - maps.yandex.ua + - maps.yandex.com + - maps.yandex.by + - n.maps.yandex.ru + +# ####################################################################################################### +# +# EMAIL PROVIDERS + +email: + + 126 Mail: + domains: + - mail.126.com + + 163 Mail: + domains: + - mail.163.com + + 2degrees: + domains: + - webmail.2degreesbroadband.co.nz + + Adam Internet: + domains: + - webmail.adam.com.au + + AOL Mail: + domains: + - mail.aol.com + - com.aol.mobile.aolapp # AOL for Android app + + Beeline: + domains: + - post.ru + + Bigpond: + domains: + - webmail.bigpond.com + - webmail2.bigpond.com + - email.telstra.com + - basic.messaging.bigpond.com + + Commander: + domains: + - webmail.commander.net.au + + Daum Mail: + domains: + - mail2.daum.net + - mail.daum.net + + Dodo: + domains: + - webmail.dodo.com.au + + E1.ru: + domains: + - mail.e1.ru + + earthlink: + domains: + - com.earthlink.myearthlink # myEarthLink for Android app + + Freenet: + domains: + - webmail.freenet.de + + Gmail: + domains: + - mail.google.com + - com.google.android.gm # Gmail for Android app + - inbox.google.com + + iiNet: + domains: + - webmail.iinet.net.au + - mail.iinet.net.au + + Inbox.com: + domains: + - inbox.com + + iPrimus: + domains: + - webmail.iprimus.com.au + + Mail.ru: + domains: + - e.mail.ru + - touch.mail.ru + + Mailchimp: + domains: + - com.mailchimp.mailchimp # Mailchimp app for Android + + Mastermail: + domains: + - mastermail.ru + - m.mastermail.ru + + Mynet Mail: + domains: + - mail.mynet.com + + Naver Mail: + domains: + - mail.naver.com + + Netspace: + domains: + - webmail.netspace.net.au + + Optus Zoo: + domains: + - webmail.optuszoo.com.au + - webmail.optusnet.com.au + + Orange Webmail: + domains: + - orange.fr/webmail + + Outlook.com: + domains: + - mail.live.com + - outlook.live.com + - com.microsoft.office.outlook # Outlook for Android app + + QIP: + domains: + - mail.qip.ru + + QQ Mail: + domains: + - mail.qq.com + - exmail.qq.com + + Rambler: + domains: + - mail.rambler.ru + + Seznam Mail: + domains: + - email.seznam.cz + + Sibmail: + domains: + - sibmail.com + + Ukr.net: + domains: + - mail.ukr.net + + Virgin: + domains: + - webmail.virginbroadband.com.au + + Vodafone: + domains: + - webmail.vodafone.co.nz + + Westnet: + domains: + - webmail.westnet.com.au + + Yandex: + domains: + - mail.yandex.ru + - mail.yandex.com + - mail.yandex.kz + - mail.yandex.ua + - mail.yandex.by + + Yahoo! Mail: + domains: + - mail.yahoo.net + - mail.yahoo.com + - mail.yahoo.co.uk + - mail.yahoo.co.jp + - com.yahoo.mobile.client.android.mail # Yahoo Mail for Android app + + Zoho: + domains: + - mail.zoho.com + +# ####################################################################################################### +# +# SOCIAL PROVIDERS + +social: + + Facebook: + domains: + - facebook.com + - fb.me + - m.facebook.com + - l.facebook.com + - lm.facebook.com + - com.facebook.katana # Facebook for Android app + + Qzone: + domains: + - qzone.qq.com + + Habbo: + domains: + - habbo.com + + Twitter: + domains: + - twitter.com + - t.co + - com.twitter.android # Twitter for Android app + + Instagram: + domains: + - instagram.com + - l.instagram.com + - com.instagram.android # Instagram for Android app + + Threads: + domains: + - threads.net + - l.threads.net + - com.instagram.barcelona # Threads for Android app + + Youtube: + domains: + - youtube.com + - youtu.be + + Vimeo: + domains: + - vimeo.com + + Renren: + domains: + - renren.com + + Windows Live Spaces: + domains: + - login.live.com + + LinkedIn: + domains: + - com.linkedin.android # LinkedIn for Android app + - linkedin.com + - lnkd.in + + Bebo: + domains: + - bebo.com + + Vkontakte: + domains: + - m.vk.com + - vk.com + - away.vk.com + - vkontakte.ru + + Tagged: + domains: + - login.tagged.com + + Orkut: + domains: + - orkut.com + + Myspace: + domains: + - myspace.com + + Friendster: + domains: + - friendster.com + + Badoo: + domains: + - badoo.com + + hi5: + domains: + - hi5.com + + Netlog: + domains: + - netlog.com + + Flixster: + domains: + - flixster.com + + MyLife: + domains: + - mylife.ru + + Paper.li: + domains: + - paper.li + + Classmates: + domains: + - classmates.com + + GitHub: + domains: + - github.com + + Google+: + domains: + - url.google.com + - plus.google.com + + Douban: + domains: + - douban.com + + Odnoklassniki: + domains: + - odnoklassniki.ru + - ok.ru + + Viadeo: + domains: + - viadeo.com + + Flickr: + domains: + - flickr.com + + WeeWorld: + domains: + - weeworld.com + + Last.fm: + domains: + - lastfm.ru + + MyHeritage: + domains: + - myheritage.com + + Xanga: + domains: + - xanga.com + + Mixi: + domains: + - mixi.jp + + Cyworld: + domains: + - global.cyworld.com + + Gaia Online: + domains: + - gaiaonline.com + + Skyrock: + domains: + - skyrock.com + + BlackPlanet: + domains: + - blackplanet.com + + myYearbook: + domains: + - myyearbook.com + + Fotolog: + domains: + - fotolog.com + + Friends Reunited: + domains: + - friendsreunited.com + + LiveJournal: + domains: + - livejournal.ru + + StudiVZ: + domains: + - studivz.net + + StackOverflow: + domains: + - stackoverflow.com + + Sonico.com: + domains: + - sonico.com + + Pinterest: + domains: + - pinterest.ca + - pinterest.cl + - pinterest.co.kr + - pinterest.com + - pinterest.com.au + - pinterest.co.uk + - pinterest.de + - pinterest.dk + - pinterest.es + - pinterest.fr + - pinterest.jp + - pinterest.nz + - pinterest.pt + - pinterest.se + - pinterest.at + - pinterest.ch + - pinterest.com.mx + - pinterest.ie + - pinterest.it + - pinterest.ph + - pinterest.ru + - com.pinterest + + Plaxo: + domains: + - plaxo.com + + Geni: + domains: + - geni.com + + Tuenti: + domains: + - tuenti.com + + XING: + domains: + - xing.com + + Taringa!: + domains: + - taringa.net + + TikTok: + domains: + - tiktok.com + + Tumblr: + domains: + - tumblr.com + - t.umblr.com + + Nasza-klasa.pl: + domains: + - nk.pl + + StumbleUpon: + domains: + - stumbleupon.com + + SourceForge: + domains: + - sourceforge.net + + Hyves: + domains: + - hyves.nl + + WAYN: + domains: + - wayn.com + + Buzznet: + domains: + - buzznet.com + + Multiply: + domains: + - multiply.com + + Foursquare: + domains: + - foursquare.com + + vKruguDruzei.ru: + domains: + - vkrugudruzei.ru + + Mail.ru: + domains: + - my.mail.ru + + MoiKrug.ru: + domains: + - moikrug.ru + + Reddit: + domains: + - reddit.com + - io.syncapps.lemmy_sync + - com.laurencedawson.reddit_sync + - com.laurencedawson.reddit_sync.pro + + Tildes: + domains: + - tildes.net + - com.talklittle.android.tildes # Three Cheers for Tildes for Android app + + Hacker News: + domains: + - news.ycombinator.com + - io.github.hidroh.materialistic # Materialistic for Android app + + Identi.ca: + domains: + - identi.ca + + Weibo: + domains: + - weibo.com + - t.cn + + Delicious: + domains: + - delicious.com + + Pocket: + domains: + - getpocket.com + + ITU Sozluk: + domains: + - itusozluk.com + + Instela: + domains: + - instela.com + + Eksi Sozluk: + domains: + - Sozluk.com + - sourtimes.org + + Uludag Sozluk: + domains: + - uludagsozluk.com + - ulusozluk.com + + Inci Sozluk: + domains: + - inci.sozlukspot.com + - incisozluk.com + - incisozluk.cc + + Hocam.com: + domains: + - hocam.com + + Donanimhaber: + domains: + - donanimhaber.com + + Disqus: + domains: + - redirect.disqus.com + - disq.us + - disqus.com + + Quora: + domains: + - quora.com + + Skype: + domains: + - web.skype.com + + Slack: + domains: + - app.slack.com + - com.slack # Slack for Android app + + Snapchat: + domains: + - com.snapchat.android # Snapchat for Android app + - snapchat.com + + Telegram: + domains: + - web.telegram.org + - org.telegram.messenger # Telegram for Android app + - org.telegram.messenger.web # Telegram for Android app (direct APK) + - org.aka.messenger # aka for Android app + - org.telegram.biftogram # BGram for Android app + - org.telegram.plus # Plus Messenger for Android app + + WhatsApp: + domains: + - web.whatsapp.com + - com.whatsapp + + Whirlpool: + domains: + - forums.whirlpool.net.au + +# ####################################################################################################### +# +# SEARCH PROVIDERS + +search: + + 1.cz: + parameters: + - q + domains: + - 1.cz + + # 123people TODO + + 1&1: + parameters: + - q + domains: + - search.1and1.com + + 1und1: + parameters: + - su + domains: + - search.1und1.de + + 2gis: + domains: + - 2gis.ru + - www.2gis.ru + - link.2gis.ru + - www.link.2gis.ru + + 360.cn: + parameters: + - q + domains: + - so.360.cn + - www.so.com + + Abacho: + parameters: + - q + domains: + - www.abacho.de + - www.abacho.com + - www.abacho.co.uk + - www.se.abacho.com + - www.tr.abacho.com + - www.abacho.at + - www.abacho.fr + - www.abacho.es + - www.abacho.ch + - www.abacho.it + + ABCsøk: + parameters: + - q + domains: + - abcsolk.no + - verden.abcsok.no + + Acoon: + parameters: + - begriff + domains: + - www.acoon.de + + Alexa: + parameters: + - q + domains: + - alexa.com + - search.toolbars.alexa.com + + Alice Adsl: + parameters: + - q + domains: + - rechercher.aliceadsl.fr + + AllTheWeb: + parameters: + - q + domains: + - www.alltheweb.com + + all.by: + parameters: + - query + domains: + - all.by + + Altavista: + parameters: + - q + domains: + - www.altavista.com + - search.altavista.com + - listings.altavista.com + - altavista.de + - altavista.fr + - be-nl.altavista.com + - be-fr.altavista.com + + Amazon: + parameters: + - keywords + domains: + - amazon.com + - www.amazon.com + + AOL: + parameters: + - q + - query + domains: + - search.aol.com + - search.aol.it + - aolsearch.aol.com + - aolsearch.com + - www.aolrecherche.aol.fr + - www.aolrecherches.aol.fr + - www.aolimages.aol.fr + - aim.search.aol.com + - www.recherche.aol.fr + - recherche.aol.fr + - find.web.aol.com + - recherche.aol.ca + - aolsearch.aol.co.uk + - search.aol.co.uk + - aolrecherche.aol.fr + - sucheaol.aol.de + - suche.aol.de + - suche.aolsvc.de + - aolbusqueda.aol.com.mx + - alicesuche.aol.de + - alicesuchet.aol.de + - suchet2.aol.de + - search.hp.my.aol.com.au + - search.hp.my.aol.de + - search.hp.my.aol.it + - search-intl.netscape.com + + Apollo Latvia: + parameters: + - q + domains: + - apollo.lv/portal/search/ + + APOLL07: + parameters: + - query + domains: + - apollo7.de + + Apontador: + parameters: + - q + domains: + - apontador.com.br + - www.apontador.com.br + + Aport: + parameters: + - r + domains: + - sm.aport.ru + + arama: + parameters: + - q + domains: + - arama.com + + Arcor: + parameters: + - Keywords + domains: + - www.arcor.de + + Arianna: + parameters: + - query + domains: + - arianna.libero.it + - www.arianna.com + + Ask: + parameters: + - q + domains: + - ask.com + - www.ask.com + - web.ask.com + - int.ask.com + - mws.ask.com + - uk.ask.com + - images.ask.com + - ask.reference.com + - www.askkids.com + - iwon.ask.com + - www.ask.co.uk + - www.qbyrd.com + - search-results.com + - uk.search-results.com + - www.search-results.com + - int.search-results.com + + Ask Toolbar: + parameters: + - searchfor + domains: + - search.tb.ask.com + + Atlas: + parameters: + - q + domains: + - searchatlas.centrum.cz + + Austronaut: + parameters: + - q + domains: + - www2.austronaut.at + - www1.astronaut.at + + Babylon: + parameters: + - q + domains: + - search.babylon.com + - searchassist.babylon.com + + Baidu: + parameters: + - wd + - word + - kw + - k + domains: + - www.baidu.com + - www1.baidu.com + - zhidao.baidu.com + - tieba.baidu.com + - news.baidu.com + - web.gougou.com + - m.baidu.com + + Biglobe: + parameters: + - q + domains: + - cgi.search.biglobe.ne.jp + + Bing: + parameters: + - q + - Q + domains: + - bing.com + - www.bing.com + - msnbc.msn.com + - dizionario.it.msn.com + - cc.bingj.com + - m.bing.com + + Bing Images: + parameters: + - q + - Q + domains: + - bing.com/images/search + - www.bing.com/images/search + + blekko: + parameters: + - q + domains: + - blekko.com + + Blogdigger: + parameters: + - q + domains: + - www.blogdigger.com + + Blogpulse: + parameters: + - query + domains: + - www.blogpulse.com + + Bluewin: + parameters: + - searchTerm + domains: + - search.bluewin.ch + + British Telecommunications: + parameters: + - p + domains: + - search.bt.com + + canoe.ca: + parameters: + - q + domains: + - web.canoe.ca + + Centrum: + parameters: + - q + domains: + - serach.centrum.cz + - morfeo.centrum.cz + + Certified-Toolbar: + parameters: + - q + domains: + - search.certified-toolbar.com + + Charter: + parameters: + - q + domains: + - www.charter.net + + Clix: + parameters: + - question + domains: + - pesquisa.clix.pt + + Coccoc: + parameters: + - q + domains: + - coccoc.com + + Conduit: + parameters: + - q + domains: + - search.conduit.com + + Comcast: + parameters: + - q + domains: + - serach.comcast.net + + Crawler: + parameters: + - q + domains: + - www.crawler.com + + Compuserve: + parameters: + - query + domains: + - websearch.cs.com + + Cuil: + parameters: + - q + domains: + - www.cuil.com + + Daemon search: + parameters: + - q + domains: + - daemon-search.com + - my.daemon-search.com + + Dalesearch: + parameters: + - q + domains: + - www.dalesearch.com + + DasOertliche: + parameters: + - kw + domains: + - www.dasoertliche.de + + DasTelefonbuch: + parameters: + - kw + domains: + - www1.dastelefonbuch.de + + Daum: + parameters: + - q + domains: + - search.daum.net + + Delfi latvia: + parameters: + - q + domains: + - smart.delfi.lv + + Delfi: + parameters: + - q + domains: + - otsing.delfi.ee + + Digg: + parameters: + - s + domains: + - digg.com + + dmoz: + parameters: + - q + domains: + - dmoz.org + - editors.dmoz.org + + Dodo: + parameters: + - q + domains: + - google.dodo.com.au + + DuckDuckGo: + parameters: + - q + domains: + - duckduckgo.com + + earthlink: + parameters: + - q + domains: + - search.earthlink.net + + Ecosia: + parameters: + - q + domains: + - ecosia.org + + Eniro: + parameters: + - q + - search_word + domains: + - www.eniro.se + + Eurip: + parameters: + - q + domains: + - www.eurip.com + + Euroseek: + parameters: + - string + domains: + - www.euroseek.com + + Everyclick: + parameters: + - keyword + domains: + - www.everyclick.com + + Excite: + parameters: + - q + - search + domains: + - search.excite.it + - search.excite.fr + - search.excite.de + - search.excite.co.uk + - serach.excite.es + - search.excite.nl + - msxml.excite.com + - www.excite.co.jp + + Exalead: + parameters: + - q + domains: + - www.exalead.fr + - www.exalead.com + + eo: + parameters: + - x_query + domains: + - eo.st + + Fast Browser Search: + parameters: + - q + domains: + - www.fastbrowsersearch.com + + Francite: + parameters: + - name + domains: + - recherche.francite.com + + Finderoo: + parameters: + - q + domains: + - www.finderoo.com + + Findwide: + parameters: + - k + domains: + - search.findwide.com + + Fireball: + parameters: + - q + domains: + - www.fireball.de + + Firstfind: + parameters: + - qry + domains: + - www.firstsfind.com + + Fixsuche: + parameters: + - q + domains: + - www.fixsuche.de + + Flix: + parameters: + - keyword + domains: + - www.flix.de + + Forestle: + parameters: + - q + domains: + - forestle.org + - www.forestle.org + - forestle.mobi + + Free: + parameters: + - q + domains: + - search.free.fr + - search1-2.free.fr + - search1-1.free.fr + + Freecause: + parameters: + - p + domains: + - search.freecause.com + + Freenet: + parameters: + - query + - Keywords + domains: + - suche.freenet.de + + Freshweather: + parameters: + - q + domains: + - www.fresh-weather.com + + FriendFeed: + parameters: + - q + domains: + - friendfeed.com + + GAIS: + parameters: + - q + domains: + - gais.cs.ccu.edu.tw + + Geona: + parameters: + - q + domains: + - geona.net + + Genieo: + parameters: + - q + domains: + - search.genieo.com + + Gigablast: + parameters: + - q + domains: + - www.gigablast.com + - dir.gigablast.com + + Globososo: + parameters: + - q + domains: + - searches.globososo.com + - search.globososo.com + + GMX: + parameters: + - su + domains: + - suche.gmx.net + + Gnadenmeer: + parameters: + - keyword + domains: + - www.gnadenmeer.de + + Gomeo: + parameters: + - Keywords + domains: + - www.gomeo.com + + goo: + parameters: + - MT + domains: + - search.goo.ne.jp + - ocnsearch.goo.ne.jp + + Google: + parameters: + - q + - query # For www.cnn.com (powered by Google) + - Keywords # For gooofullsearch.com (powered by Google) + domains: + - www.google.com + - www.google.ac + - www.google.ad + - www.google.com.af + - www.google.com.ag + - www.google.com.ai + - www.google.am + - www.google.it.ao + - www.google.com.ar + - www.google.as + - www.google.at + - www.google.com.au + - www.google.az + - www.google.ba + - www.google.com.bd + - www.google.be + - www.google.bf + - www.google.bg + - www.google.com.bh + - www.google.bi + - www.google.bj + - www.google.com.bn + - www.google.com.bo + - www.google.com.br + - www.google.bs + - www.google.co.bw + - www.google.com.by + - www.google.by + - www.google.com.bz + - www.google.ca + - www.google.com.kh + - www.google.cc + - www.google.cd + - www.google.cf + - www.google.cat + - www.google.cg + - www.google.ch + - www.google.ci + - www.google.co.ck + - www.google.cl + - www.google.cm + - www.google.cn + - www.google.com.co + - www.google.co.cr + - www.google.com.cu + - www.google.cv + - www.google.com.cy + - www.google.cz + - www.google.de + - www.google.dj + - www.google.dk + - www.google.dm + - www.google.com.do + - www.google.dz + - www.google.com.ec + - www.google.ee + - www.google.com.eg + - www.google.es + - www.google.com.et + - www.google.fi + - www.google.com.fj + - www.google.fm + - www.google.fr + - www.google.ga + - www.google.gd + - www.google.ge + - www.google.gf + - www.google.gg + - www.google.com.gh + - www.google.com.gi + - www.google.gl + - www.google.gm + - www.google.gp + - www.google.gr + - www.google.com.gt + - www.google.gy + - www.google.com.hk + - www.google.hn + - www.google.hr + - www.google.ht + - www.google.hu + - www.google.co.id + - www.google.iq + - www.google.ie + - www.google.co.il + - www.google.im + - www.google.co.in + - www.google.io + - www.google.is + - www.google.it + - www.google.je + - www.google.com.jm + - www.google.jo + - www.google.co.jp + - www.google.co.ke + - www.google.ki + - www.google.kg + - www.google.co.kr + - www.google.com.kw + - www.google.kz + - www.google.la + - www.google.com.lb + - www.google.com.lc + - www.google.li + - www.google.lk + - www.google.co.ls + - www.google.lt + - www.google.lu + - www.google.lv + - www.google.com.ly + - www.google.co.ma + - www.google.md + - www.google.me + - www.google.mg + - www.google.mk + - www.google.ml + - www.google.mn + - www.google.ms + - www.google.com.mt + - www.google.mu + - www.google.mv + - www.google.mw + - www.google.com.mx + - www.google.com.my + - www.google.co.mz + - www.google.com.na + - www.google.ne + - www.google.com.nf + - www.google.com.ng + - www.google.com.ni + - www.google.nl + - www.google.no + - www.google.com.np + - www.google.nr + - www.google.nu + - www.google.co.nz + - www.google.com.om + - www.google.com.pa + - www.google.com.pe + - www.google.com.ph + - www.google.com.pk + - www.google.pl + - www.google.pn + - www.google.com.pr + - www.google.ps + - www.google.pt + - www.google.com.py + - www.google.com.qa + - www.google.ro + - www.google.rs + - www.google.ru + - www.google.rw + - www.google.com.sa + - www.google.com.sb + - www.google.sc + - www.google.se + - www.google.com.sg + - www.google.sh + - www.google.si + - www.google.sk + - www.google.com.sl + - www.google.sn + - www.google.sm + - www.google.so + - www.google.st + - www.google.com.sv + - www.google.td + - www.google.tg + - www.google.co.th + - www.google.com.tj + - www.google.tk + - www.google.tl + - www.google.tm + - www.google.to + - www.google.com.tn + - www.google.tn + - www.google.com.tr + - www.google.tt + - www.google.com.tw + - www.google.co.tz + - www.google.com.ua + - www.google.co.ug + - www.google.ae + - www.google.co.uk + - www.google.us + - www.google.com.uy + - www.google.co.uz + - www.google.com.vc + - www.google.co.ve + - www.google.vg + - www.google.co.vi + - www.google.com.vn + - www.google.vu + - www.google.ws + - www.google.co.za + - www.google.co.zm + - www.google.co.zw + - google.com + - google.ac + - google.ad + - google.com.af + - google.com.ag + - google.com.ai + - google.am + - google.it.ao + - google.com.ar + - google.as + - google.at + - google.com.au + - google.az + - google.ba + - google.com.bd + - google.be + - google.bf + - google.bg + - google.com.bh + - google.bi + - google.bj + - google.com.bn + - google.com.bo + - google.com.br + - google.bs + - google.co.bw + - google.com.by + - google.by + - google.com.bz + - google.ca + - google.com.kh + - google.cc + - google.cd + - google.cf + - google.cat + - google.cg + - google.ch + - google.ci + - google.co.ck + - google.cl + - google.cm + - google.cn + - google.com.co + - google.co.cr + - google.com.cu + - google.cv + - google.com.cy + - google.cz + - google.de + - google.dj + - google.dk + - google.dm + - google.com.do + - google.dz + - google.com.ec + - google.ee + - google.com.eg + - google.es + - google.com.et + - google.fi + - google.com.fj + - google.fm + - google.fr + - google.ga + - google.gd + - google.ge + - google.gf + - google.gg + - google.com.gh + - google.com.gi + - google.gl + - google.gm + - google.gp + - google.gr + - google.com.gt + - google.gy + - google.com.hk + - google.hn + - google.hr + - google.ht + - google.hu + - google.co.id + - google.iq + - google.ie + - google.co.il + - google.im + - google.co.in + - google.io + - google.is + - google.it + - google.je + - google.com.jm + - google.jo + - google.co.jp + - google.co.ke + - google.ki + - google.kg + - google.co.kr + - google.com.kw + - google.kz + - google.la + - google.com.lb + - google.com.lc + - google.li + - google.lk + - google.co.ls + - google.lt + - google.lu + - google.lv + - google.com.ly + - google.co.ma + - google.md + - google.me + - google.mg + - google.mk + - google.ml + - google.mn + - google.ms + - google.com.mt + - google.mu + - google.mv + - google.mw + - google.com.mx + - google.com.my + - google.co.mz + - google.com.na + - google.ne + - google.com.nf + - google.com.ng + - google.com.ni + - google.nl + - google.no + - google.com.np + - google.nr + - google.nu + - google.co.nz + - google.com.om + - google.com.pa + - google.com.pe + - google.com.ph + - google.com.pk + - google.pl + - google.pn + - google.com.pr + - google.ps + - google.pt + - google.com.py + - google.com.qa + - google.ro + - google.rs + - google.ru + - google.rw + - google.com.sa + - google.com.sb + - google.sc + - google.se + - google.com.sg + - google.sh + - google.si + - google.sk + - google.com.sl + - google.sn + - google.sm + - google.so + - google.st + - google.com.sv + - google.td + - google.tg + - google.co.th + - google.com.tj + - google.tk + - google.tl + - google.tm + - google.to + - google.com.tn + - google.com.tr + - google.tt + - google.com.tw + - google.co.tz + - google.com.ua + - google.co.ug + - google.ae + - google.co.uk + - google.us + - google.com.uy + - google.co.uz + - google.com.vc + - google.co.ve + - google.vg + - google.co.vi + - google.com.vn + - google.vu + - google.ws + - google.co.za + - google.co.zm + - google.co.zw + - google.tn + # powered by Google + - search.avg.com + - isearch.avg.com + - www.cnn.com + - darkoogle.com + - search.darkoogle.com + - search.foxtab.com + - www.gooofullsearch.com + - search.hiyo.com + - search.incredimail.com + - search1.incredimail.com + - search2.incredimail.com + - search3.incredimail.com + - search4.incredimail.com + - search.incredibar.com + - search.sweetim.com + - www.fastweb.it + - search.juno.com + - find.tdc.dk + - searchresults.verizon.com + - search.walla.co.il + - search.alot.com + # Google Earch + - www.googleearth.de + - www.googleearth.fr + # Google Cache + - webcache.googleusercontent.com + # Google SSL + - encrypted.google.com + # Syndicated search + - googlesyndicatedsearch.com + - com.google.android.googlequicksearchbox # Google for Android app + + Google Blogsearch: + parameters: + - q + domains: + - blogsearch.google.ac + - blogsearch.google.ad + - blogsearch.google.ae + - blogsearch.google.am + - blogsearch.google.as + - blogsearch.google.at + - blogsearch.google.az + - blogsearch.google.ba + - blogsearch.google.be + - blogsearch.google.bf + - blogsearch.google.bg + - blogsearch.google.bi + - blogsearch.google.bj + - blogsearch.google.bs + - blogsearch.google.by + - blogsearch.google.ca + - blogsearch.google.cat + - blogsearch.google.cc + - blogsearch.google.cd + - blogsearch.google.cf + - blogsearch.google.cg + - blogsearch.google.ch + - blogsearch.google.ci + - blogsearch.google.cl + - blogsearch.google.cm + - blogsearch.google.cn + - blogsearch.google.co.bw + - blogsearch.google.co.ck + - blogsearch.google.co.cr + - blogsearch.google.co.id + - blogsearch.google.co.il + - blogsearch.google.co.in + - blogsearch.google.co.jp + - blogsearch.google.co.ke + - blogsearch.google.co.kr + - blogsearch.google.co.ls + - blogsearch.google.co.ma + - blogsearch.google.co.mz + - blogsearch.google.co.nz + - blogsearch.google.co.th + - blogsearch.google.co.tz + - blogsearch.google.co.ug + - blogsearch.google.co.uk + - blogsearch.google.co.uz + - blogsearch.google.co.ve + - blogsearch.google.co.vi + - blogsearch.google.co.za + - blogsearch.google.co.zm + - blogsearch.google.co.zw + - blogsearch.google.com + - blogsearch.google.com.af + - blogsearch.google.com.ag + - blogsearch.google.com.ai + - blogsearch.google.com.ar + - blogsearch.google.com.au + - blogsearch.google.com.bd + - blogsearch.google.com.bh + - blogsearch.google.com.bn + - blogsearch.google.com.bo + - blogsearch.google.com.br + - blogsearch.google.com.by + - blogsearch.google.com.bz + - blogsearch.google.com.co + - blogsearch.google.com.cu + - blogsearch.google.com.cy + - blogsearch.google.com.do + - blogsearch.google.com.ec + - blogsearch.google.com.eg + - blogsearch.google.com.et + - blogsearch.google.com.fj + - blogsearch.google.com.gh + - blogsearch.google.com.gi + - blogsearch.google.com.gt + - blogsearch.google.com.hk + - blogsearch.google.com.jm + - blogsearch.google.com.kh + - blogsearch.google.com.kw + - blogsearch.google.com.lb + - blogsearch.google.com.lc + - blogsearch.google.com.ly + - blogsearch.google.com.mt + - blogsearch.google.com.mx + - blogsearch.google.com.my + - blogsearch.google.com.na + - blogsearch.google.com.nf + - blogsearch.google.com.ng + - blogsearch.google.com.ni + - blogsearch.google.com.np + - blogsearch.google.com.om + - blogsearch.google.com.pa + - blogsearch.google.com.pe + - blogsearch.google.com.ph + - blogsearch.google.com.pk + - blogsearch.google.com.pr + - blogsearch.google.com.py + - blogsearch.google.com.qa + - blogsearch.google.com.sa + - blogsearch.google.com.sb + - blogsearch.google.com.sg + - blogsearch.google.com.sl + - blogsearch.google.com.sv + - blogsearch.google.com.tj + - blogsearch.google.com.tn + - blogsearch.google.com.tr + - blogsearch.google.com.tw + - blogsearch.google.com.ua + - blogsearch.google.com.uy + - blogsearch.google.com.vc + - blogsearch.google.com.vn + - blogsearch.google.cv + - blogsearch.google.cz + - blogsearch.google.de + - blogsearch.google.dj + - blogsearch.google.dk + - blogsearch.google.dm + - blogsearch.google.dz + - blogsearch.google.ee + - blogsearch.google.es + - blogsearch.google.fi + - blogsearch.google.fm + - blogsearch.google.fr + - blogsearch.google.ga + - blogsearch.google.gd + - blogsearch.google.ge + - blogsearch.google.gf + - blogsearch.google.gg + - blogsearch.google.gl + - blogsearch.google.gm + - blogsearch.google.gp + - blogsearch.google.gr + - blogsearch.google.gy + - blogsearch.google.hn + - blogsearch.google.hr + - blogsearch.google.ht + - blogsearch.google.hu + - blogsearch.google.ie + - blogsearch.google.im + - blogsearch.google.io + - blogsearch.google.iq + - blogsearch.google.is + - blogsearch.google.it + - blogsearch.google.it.ao + - blogsearch.google.je + - blogsearch.google.jo + - blogsearch.google.kg + - blogsearch.google.ki + - blogsearch.google.kz + - blogsearch.google.la + - blogsearch.google.li + - blogsearch.google.lk + - blogsearch.google.lt + - blogsearch.google.lu + - blogsearch.google.lv + - blogsearch.google.md + - blogsearch.google.me + - blogsearch.google.mg + - blogsearch.google.mk + - blogsearch.google.ml + - blogsearch.google.mn + - blogsearch.google.ms + - blogsearch.google.mu + - blogsearch.google.mv + - blogsearch.google.mw + - blogsearch.google.ne + - blogsearch.google.nl + - blogsearch.google.no + - blogsearch.google.nr + - blogsearch.google.nu + - blogsearch.google.pl + - blogsearch.google.pn + - blogsearch.google.ps + - blogsearch.google.pt + - blogsearch.google.ro + - blogsearch.google.rs + - blogsearch.google.ru + - blogsearch.google.rw + - blogsearch.google.sc + - blogsearch.google.se + - blogsearch.google.sh + - blogsearch.google.si + - blogsearch.google.sk + - blogsearch.google.sm + - blogsearch.google.sn + - blogsearch.google.so + - blogsearch.google.st + - blogsearch.google.td + - blogsearch.google.tg + - blogsearch.google.tk + - blogsearch.google.tl + - blogsearch.google.tm + - blogsearch.google.to + - blogsearch.google.tt + - blogsearch.google.us + - blogsearch.google.vg + - blogsearch.google.vu + - blogsearch.google.ws + + Google Images: + parameters: + - q + domains: + - google.ac/imgres + - google.ad/imgres + - google.ae/imgres + - google.am/imgres + - google.as/imgres + - google.at/imgres + - google.az/imgres + - google.ba/imgres + - google.be/imgres + - google.bf/imgres + - google.bg/imgres + - google.bi/imgres + - google.bj/imgres + - google.bs/imgres + - google.by/imgres + - google.ca/imgres + - google.cat/imgres + - google.cc/imgres + - google.cd/imgres + - google.cf/imgres + - google.cg/imgres + - google.ch/imgres + - google.ci/imgres + - google.cl/imgres + - google.cm/imgres + - google.cn/imgres + - google.co.bw/imgres + - google.co.ck/imgres + - google.co.cr/imgres + - google.co.id/imgres + - google.co.il/imgres + - google.co.in/imgres + - google.co.jp/imgres + - google.co.ke/imgres + - google.co.kr/imgres + - google.co.ls/imgres + - google.co.ma/imgres + - google.co.mz/imgres + - google.co.nz/imgres + - google.co.th/imgres + - google.co.tz/imgres + - google.co.ug/imgres + - google.co.uk/imgres + - google.co.uz/imgres + - google.co.ve/imgres + - google.co.vi/imgres + - google.co.za/imgres + - google.co.zm/imgres + - google.co.zw/imgres + - google.com/imgres + - google.com.af/imgres + - google.com.ag/imgres + - google.com.ai/imgres + - google.com.ar/imgres + - google.com.au/imgres + - google.com.bd/imgres + - google.com.bh/imgres + - google.com.bn/imgres + - google.com.bo/imgres + - google.com.br/imgres + - google.com.by/imgres + - google.com.bz/imgres + - google.com.co/imgres + - google.com.cu/imgres + - google.com.cy/imgres + - google.com.do/imgres + - google.com.ec/imgres + - google.com.eg/imgres + - google.com.et/imgres + - google.com.fj/imgres + - google.com.gh/imgres + - google.com.gi/imgres + - google.com.gt/imgres + - google.com.hk/imgres + - google.com.jm/imgres + - google.com.kh/imgres + - google.com.kw/imgres + - google.com.lb/imgres + - google.com.lc/imgres + - google.com.ly/imgres + - google.com.mt/imgres + - google.com.mx/imgres + - google.com.my/imgres + - google.com.na/imgres + - google.com.nf/imgres + - google.com.ng/imgres + - google.com.ni/imgres + - google.com.np/imgres + - google.com.om/imgres + - google.com.pa/imgres + - google.com.pe/imgres + - google.com.ph/imgres + - google.com.pk/imgres + - google.com.pr/imgres + - google.com.py/imgres + - google.com.qa/imgres + - google.com.sa/imgres + - google.com.sb/imgres + - google.com.sg/imgres + - google.com.sl/imgres + - google.com.sv/imgres + - google.com.tj/imgres + - google.com.tn/imgres + - google.com.tr/imgres + - google.com.tw/imgres + - google.com.ua/imgres + - google.com.uy/imgres + - google.com.vc/imgres + - google.com.vn/imgres + - google.cv/imgres + - google.cz/imgres + - google.de/imgres + - google.dj/imgres + - google.dk/imgres + - google.dm/imgres + - google.dz/imgres + - google.ee/imgres + - google.es/imgres + - google.fi/imgres + - google.fm/imgres + - google.fr/imgres + - google.ga/imgres + - google.gd/imgres + - google.ge/imgres + - google.gf/imgres + - google.gg/imgres + - google.gl/imgres + - google.gm/imgres + - google.gp/imgres + - google.gr/imgres + - google.gy/imgres + - google.hn/imgres + - google.hr/imgres + - google.ht/imgres + - google.hu/imgres + - google.ie/imgres + - google.im/imgres + - google.io/imgres + - google.iq/imgres + - google.is/imgres + - google.it/imgres + - google.it.ao/imgres + - google.je/imgres + - google.jo/imgres + - google.kg/imgres + - google.ki/imgres + - google.kz/imgres + - google.la/imgres + - google.li/imgres + - google.lk/imgres + - google.lt/imgres + - google.lu/imgres + - google.lv/imgres + - google.md/imgres + - google.me/imgres + - google.mg/imgres + - google.mk/imgres + - google.ml/imgres + - google.mn/imgres + - google.ms/imgres + - google.mu/imgres + - google.mv/imgres + - google.mw/imgres + - google.ne/imgres + - google.nl/imgres + - google.no/imgres + - google.nr/imgres + - google.nu/imgres + - google.pl/imgres + - google.pn/imgres + - google.ps/imgres + - google.pt/imgres + - google.ro/imgres + - google.rs/imgres + - google.ru/imgres + - google.rw/imgres + - google.sc/imgres + - google.se/imgres + - google.sh/imgres + - google.si/imgres + - google.sk/imgres + - google.sm/imgres + - google.sn/imgres + - google.so/imgres + - google.st/imgres + - google.td/imgres + - google.tg/imgres + - google.tk/imgres + - google.tl/imgres + - google.tm/imgres + - google.to/imgres + - google.tt/imgres + - google.us/imgres + - google.vg/imgres + - google.vu/imgres + - images.google.ws + - images.google.ac + - images.google.ad + - images.google.ae + - images.google.am + - images.google.as + - images.google.at + - images.google.az + - images.google.ba + - images.google.be + - images.google.bf + - images.google.bg + - images.google.bi + - images.google.bj + - images.google.bs + - images.google.by + - images.google.ca + - images.google.cat + - images.google.cc + - images.google.cd + - images.google.cf + - images.google.cg + - images.google.ch + - images.google.ci + - images.google.cl + - images.google.cm + - images.google.cn + - images.google.co.bw + - images.google.co.ck + - images.google.co.cr + - images.google.co.id + - images.google.co.il + - images.google.co.in + - images.google.co.jp + - images.google.co.ke + - images.google.co.kr + - images.google.co.ls + - images.google.co.ma + - images.google.co.mz + - images.google.co.nz + - images.google.co.th + - images.google.co.tz + - images.google.co.ug + - images.google.co.uk + - images.google.co.uz + - images.google.co.ve + - images.google.co.vi + - images.google.co.za + - images.google.co.zm + - images.google.co.zw + - images.google.com + - images.google.com.af + - images.google.com.ag + - images.google.com.ai + - images.google.com.ar + - images.google.com.au + - images.google.com.bd + - images.google.com.bh + - images.google.com.bn + - images.google.com.bo + - images.google.com.br + - images.google.com.by + - images.google.com.bz + - images.google.com.co + - images.google.com.cu + - images.google.com.cy + - images.google.com.do + - images.google.com.ec + - images.google.com.eg + - images.google.com.et + - images.google.com.fj + - images.google.com.gh + - images.google.com.gi + - images.google.com.gt + - images.google.com.hk + - images.google.com.jm + - images.google.com.kh + - images.google.com.kw + - images.google.com.lb + - images.google.com.lc + - images.google.com.ly + - images.google.com.mt + - images.google.com.mx + - images.google.com.my + - images.google.com.na + - images.google.com.nf + - images.google.com.ng + - images.google.com.ni + - images.google.com.np + - images.google.com.om + - images.google.com.pa + - images.google.com.pe + - images.google.com.ph + - images.google.com.pk + - images.google.com.pr + - images.google.com.py + - images.google.com.qa + - images.google.com.sa + - images.google.com.sb + - images.google.com.sg + - images.google.com.sl + - images.google.com.sv + - images.google.com.tj + - images.google.com.tn + - images.google.com.tr + - images.google.com.tw + - images.google.com.ua + - images.google.com.uy + - images.google.com.vc + - images.google.com.vn + - images.google.cv + - images.google.cz + - images.google.de + - images.google.dj + - images.google.dk + - images.google.dm + - images.google.dz + - images.google.ee + - images.google.es + - images.google.fi + - images.google.fm + - images.google.fr + - images.google.ga + - images.google.gd + - images.google.ge + - images.google.gf + - images.google.gg + - images.google.gl + - images.google.gm + - images.google.gp + - images.google.gr + - images.google.gy + - images.google.hn + - images.google.hr + - images.google.ht + - images.google.hu + - images.google.ie + - images.google.im + - images.google.io + - images.google.iq + - images.google.is + - images.google.it + - images.google.it.ao + - images.google.je + - images.google.jo + - images.google.kg + - images.google.ki + - images.google.kz + - images.google.la + - images.google.li + - images.google.lk + - images.google.lt + - images.google.lu + - images.google.lv + - images.google.md + - images.google.me + - images.google.mg + - images.google.mk + - images.google.ml + - images.google.mn + - images.google.ms + - images.google.mu + - images.google.mv + - images.google.mw + - images.google.ne + - images.google.nl + - images.google.no + - images.google.nr + - images.google.nu + - images.google.pl + - images.google.pn + - images.google.ps + - images.google.pt + - images.google.ro + - images.google.rs + - images.google.ru + - images.google.rw + - images.google.sc + - images.google.se + - images.google.sh + - images.google.si + - images.google.sk + - images.google.sm + - images.google.sn + - images.google.so + - images.google.st + - images.google.td + - images.google.tg + - images.google.tk + - images.google.tl + - images.google.tm + - images.google.to + - images.google.tt + - images.google.us + - images.google.vg + - images.google.vu + + Google News: + parameters: + - q + domains: + - news.google.ac + - news.google.ad + - news.google.ae + - news.google.am + - news.google.as + - news.google.at + - news.google.az + - news.google.ba + - news.google.be + - news.google.bf + - news.google.bg + - news.google.bi + - news.google.bj + - news.google.bs + - news.google.by + - news.google.ca + - news.google.cat + - news.google.cc + - news.google.cd + - news.google.cf + - news.google.cg + - news.google.ch + - news.google.ci + - news.google.cl + - news.google.cm + - news.google.cn + - news.google.co.bw + - news.google.co.ck + - news.google.co.cr + - news.google.co.id + - news.google.co.il + - news.google.co.in + - news.google.co.jp + - news.google.co.ke + - news.google.co.kr + - news.google.co.ls + - news.google.co.ma + - news.google.co.mz + - news.google.co.nz + - news.google.co.th + - news.google.co.tz + - news.google.co.ug + - news.google.co.uk + - news.google.co.uz + - news.google.co.ve + - news.google.co.vi + - news.google.co.za + - news.google.co.zm + - news.google.co.zw + - news.google.com + - news.google.com.af + - news.google.com.ag + - news.google.com.ai + - news.google.com.ar + - news.google.com.au + - news.google.com.bd + - news.google.com.bh + - news.google.com.bn + - news.google.com.bo + - news.google.com.br + - news.google.com.by + - news.google.com.bz + - news.google.com.co + - news.google.com.cu + - news.google.com.cy + - news.google.com.do + - news.google.com.ec + - news.google.com.eg + - news.google.com.et + - news.google.com.fj + - news.google.com.gh + - news.google.com.gi + - news.google.com.gt + - news.google.com.hk + - news.google.com.jm + - news.google.com.kh + - news.google.com.kw + - news.google.com.lb + - news.google.com.lc + - news.google.com.ly + - news.google.com.mt + - news.google.com.mx + - news.google.com.my + - news.google.com.na + - news.google.com.nf + - news.google.com.ng + - news.google.com.ni + - news.google.com.np + - news.google.com.om + - news.google.com.pa + - news.google.com.pe + - news.google.com.ph + - news.google.com.pk + - news.google.com.pr + - news.google.com.py + - news.google.com.qa + - news.google.com.sa + - news.google.com.sb + - news.google.com.sg + - news.google.com.sl + - news.google.com.sv + - news.google.com.tj + - news.google.com.tn + - news.google.com.tr + - news.google.com.tw + - news.google.com.ua + - news.google.com.uy + - news.google.com.vc + - news.google.com.vn + - news.google.cv + - news.google.cz + - news.google.de + - news.google.dj + - news.google.dk + - news.google.dm + - news.google.dz + - news.google.ee + - news.google.es + - news.google.fi + - news.google.fm + - news.google.fr + - news.google.ga + - news.google.gd + - news.google.ge + - news.google.gf + - news.google.gg + - news.google.gl + - news.google.gm + - news.google.gp + - news.google.gr + - news.google.gy + - news.google.hn + - news.google.hr + - news.google.ht + - news.google.hu + - news.google.ie + - news.google.im + - news.google.io + - news.google.iq + - news.google.is + - news.google.it + - news.google.it.ao + - news.google.je + - news.google.jo + - news.google.kg + - news.google.ki + - news.google.kz + - news.google.la + - news.google.li + - news.google.lk + - news.google.lt + - news.google.lu + - news.google.lv + - news.google.md + - news.google.me + - news.google.mg + - news.google.mk + - news.google.ml + - news.google.mn + - news.google.ms + - news.google.mu + - news.google.mv + - news.google.mw + - news.google.ne + - news.google.nl + - news.google.no + - news.google.nr + - news.google.nu + - news.google.pl + - news.google.pn + - news.google.ps + - news.google.pt + - news.google.ro + - news.google.rs + - news.google.ru + - news.google.rw + - news.google.sc + - news.google.se + - news.google.sh + - news.google.si + - news.google.sk + - news.google.sm + - news.google.sn + - news.google.so + - news.google.st + - news.google.td + - news.google.tg + - news.google.tk + - news.google.tl + - news.google.tm + - news.google.to + - news.google.tt + - news.google.us + - news.google.vg + - news.google.vu + - news.google.ws + + Google Product Search: + parameters: + - q + domains: + - google.ac/products + - google.ad/products + - google.ae/products + - google.am/products + - google.as/products + - google.at/products + - google.az/products + - google.ba/products + - google.be/products + - google.bf/products + - google.bg/products + - google.bi/products + - google.bj/products + - google.bs/products + - google.by/products + - google.ca/products + - google.cat/products + - google.cc/products + - google.cd/products + - google.cf/products + - google.cg/products + - google.ch/products + - google.ci/products + - google.cl/products + - google.cm/products + - google.cn/products + - google.co.bw/products + - google.co.ck/products + - google.co.cr/products + - google.co.id/products + - google.co.il/products + - google.co.in/products + - google.co.jp/products + - google.co.ke/products + - google.co.kr/products + - google.co.ls/products + - google.co.ma/products + - google.co.mz/products + - google.co.nz/products + - google.co.th/products + - google.co.tz/products + - google.co.ug/products + - google.co.uk/products + - google.co.uz/products + - google.co.ve/products + - google.co.vi/products + - google.co.za/products + - google.co.zm/products + - google.co.zw/products + - google.com/products + - google.com.af/products + - google.com.ag/products + - google.com.ai/products + - google.com.ar/products + - google.com.au/products + - google.com.bd/products + - google.com.bh/products + - google.com.bn/products + - google.com.bo/products + - google.com.br/products + - google.com.by/products + - google.com.bz/products + - google.com.co/products + - google.com.cu/products + - google.com.cy/products + - google.com.do/products + - google.com.ec/products + - google.com.eg/products + - google.com.et/products + - google.com.fj/products + - google.com.gh/products + - google.com.gi/products + - google.com.gt/products + - google.com.hk/products + - google.com.jm/products + - google.com.kh/products + - google.com.kw/products + - google.com.lb/products + - google.com.lc/products + - google.com.ly/products + - google.com.mt/products + - google.com.mx/products + - google.com.my/products + - google.com.na/products + - google.com.nf/products + - google.com.ng/products + - google.com.ni/products + - google.com.np/products + - google.com.om/products + - google.com.pa/products + - google.com.pe/products + - google.com.ph/products + - google.com.pk/products + - google.com.pr/products + - google.com.py/products + - google.com.qa/products + - google.com.sa/products + - google.com.sb/products + - google.com.sg/products + - google.com.sl/products + - google.com.sv/products + - google.com.tj/products + - google.com.tn/products + - google.com.tr/products + - google.com.tw/products + - google.com.ua/products + - google.com.uy/products + - google.com.vc/products + - google.com.vn/products + - google.cv/products + - google.cz/products + - google.de/products + - google.dj/products + - google.dk/products + - google.dm/products + - google.dz/products + - google.ee/products + - google.es/products + - google.fi/products + - google.fm/products + - google.fr/products + - google.ga/products + - google.gd/products + - google.ge/products + - google.gf/products + - google.gg/products + - google.gl/products + - google.gm/products + - google.gp/products + - google.gr/products + - google.gy/products + - google.hn/products + - google.hr/products + - google.ht/products + - google.hu/products + - google.ie/products + - google.im/products + - google.io/products + - google.iq/products + - google.is/products + - google.it/products + - google.it.ao/products + - google.je/products + - google.jo/products + - google.kg/products + - google.ki/products + - google.kz/products + - google.la/products + - google.li/products + - google.lk/products + - google.lt/products + - google.lu/products + - google.lv/products + - google.md/products + - google.me/products + - google.mg/products + - google.mk/products + - google.ml/products + - google.mn/products + - google.ms/products + - google.mu/products + - google.mv/products + - google.mw/products + - google.ne/products + - google.nl/products + - google.no/products + - google.nr/products + - google.nu/products + - google.pl/products + - google.pn/products + - google.ps/products + - google.pt/products + - google.ro/products + - google.rs/products + - google.ru/products + - google.rw/products + - google.sc/products + - google.se/products + - google.sh/products + - google.si/products + - google.sk/products + - google.sm/products + - google.sn/products + - google.so/products + - google.st/products + - google.td/products + - google.tg/products + - google.tk/products + - google.tl/products + - google.tm/products + - google.to/products + - google.tt/products + - google.us/products + - google.vg/products + - google.vu/products + - google.ws/products + - www.google.ac/products + - www.google.ad/products + - www.google.ae/products + - www.google.am/products + - www.google.as/products + - www.google.at/products + - www.google.az/products + - www.google.ba/products + - www.google.be/products + - www.google.bf/products + - www.google.bg/products + - www.google.bi/products + - www.google.bj/products + - www.google.bs/products + - www.google.by/products + - www.google.ca/products + - www.google.cat/products + - www.google.cc/products + - www.google.cd/products + - www.google.cf/products + - www.google.cg/products + - www.google.ch/products + - www.google.ci/products + - www.google.cl/products + - www.google.cm/products + - www.google.cn/products + - www.google.co.bw/products + - www.google.co.ck/products + - www.google.co.cr/products + - www.google.co.id/products + - www.google.co.il/products + - www.google.co.in/products + - www.google.co.jp/products + - www.google.co.ke/products + - www.google.co.kr/products + - www.google.co.ls/products + - www.google.co.ma/products + - www.google.co.mz/products + - www.google.co.nz/products + - www.google.co.th/products + - www.google.co.tz/products + - www.google.co.ug/products + - www.google.co.uk/products + - www.google.co.uz/products + - www.google.co.ve/products + - www.google.co.vi/products + - www.google.co.za/products + - www.google.co.zm/products + - www.google.co.zw/products + - www.google.com/products + - www.google.com.af/products + - www.google.com.ag/products + - www.google.com.ai/products + - www.google.com.ar/products + - www.google.com.au/products + - www.google.com.bd/products + - www.google.com.bh/products + - www.google.com.bn/products + - www.google.com.bo/products + - www.google.com.br/products + - www.google.com.by/products + - www.google.com.bz/products + - www.google.com.co/products + - www.google.com.cu/products + - www.google.com.cy/products + - www.google.com.do/products + - www.google.com.ec/products + - www.google.com.eg/products + - www.google.com.et/products + - www.google.com.fj/products + - www.google.com.gh/products + - www.google.com.gi/products + - www.google.com.gt/products + - www.google.com.hk/products + - www.google.com.jm/products + - www.google.com.kh/products + - www.google.com.kw/products + - www.google.com.lb/products + - www.google.com.lc/products + - www.google.com.ly/products + - www.google.com.mt/products + - www.google.com.mx/products + - www.google.com.my/products + - www.google.com.na/products + - www.google.com.nf/products + - www.google.com.ng/products + - www.google.com.ni/products + - www.google.com.np/products + - www.google.com.om/products + - www.google.com.pa/products + - www.google.com.pe/products + - www.google.com.ph/products + - www.google.com.pk/products + - www.google.com.pr/products + - www.google.com.py/products + - www.google.com.qa/products + - www.google.com.sa/products + - www.google.com.sb/products + - www.google.com.sg/products + - www.google.com.sl/products + - www.google.com.sv/products + - www.google.com.tj/products + - www.google.com.tn/products + - www.google.com.tr/products + - www.google.com.tw/products + - www.google.com.ua/products + - www.google.com.uy/products + - www.google.com.vc/products + - www.google.com.vn/products + - www.google.cv/products + - www.google.cz/products + - www.google.de/products + - www.google.dj/products + - www.google.dk/products + - www.google.dm/products + - www.google.dz/products + - www.google.ee/products + - www.google.es/products + - www.google.fi/products + - www.google.fm/products + - www.google.fr/products + - www.google.ga/products + - www.google.gd/products + - www.google.ge/products + - www.google.gf/products + - www.google.gg/products + - www.google.gl/products + - www.google.gm/products + - www.google.gp/products + - www.google.gr/products + - www.google.gy/products + - www.google.hn/products + - www.google.hr/products + - www.google.ht/products + - www.google.hu/products + - www.google.ie/products + - www.google.im/products + - www.google.io/products + - www.google.iq/products + - www.google.is/products + - www.google.it/products + - www.google.it.ao/products + - www.google.je/products + - www.google.jo/products + - www.google.kg/products + - www.google.ki/products + - www.google.kz/products + - www.google.la/products + - www.google.li/products + - www.google.lk/products + - www.google.lt/products + - www.google.lu/products + - www.google.lv/products + - www.google.md/products + - www.google.me/products + - www.google.mg/products + - www.google.mk/products + - www.google.ml/products + - www.google.mn/products + - www.google.ms/products + - www.google.mu/products + - www.google.mv/products + - www.google.mw/products + - www.google.ne/products + - www.google.nl/products + - www.google.no/products + - www.google.nr/products + - www.google.nu/products + - www.google.pl/products + - www.google.pn/products + - www.google.ps/products + - www.google.pt/products + - www.google.ro/products + - www.google.rs/products + - www.google.ru/products + - www.google.rw/products + - www.google.sc/products + - www.google.se/products + - www.google.sh/products + - www.google.si/products + - www.google.sk/products + - www.google.sm/products + - www.google.sn/products + - www.google.so/products + - www.google.st/products + - www.google.td/products + - www.google.tg/products + - www.google.tk/products + - www.google.tl/products + - www.google.tm/products + - www.google.to/products + - www.google.tt/products + - www.google.us/products + - www.google.vg/products + - www.google.vu/products + - www.google.ws/products + + Google Video: + parameters: + - q + domains: + - video.google.com + + Goyellow.de: + parameters: + - MDN + domains: + - www.goyellow.de + + Gule Sider: + parameters: + - q + domains: + - www.gulesider.no + + HighBeam: + parameters: + - q + domains: + - www.highbeam.com + + Hit-Parade: + parameters: + - p7 + domains: + - req.-hit-parade.com + - class.hit-parade.com + - www.hit-parade.com + + Holmes: + parameters: + - q + domains: + - holmes.ge + + Hooseek.com: + parameters: + - recherche + domains: + - www.hooseek.com + + Hotbot: + parameters: + - query + domains: + - www.hotbot.com + + Icerockeet: + parameters: + - q + domains: + - blogs.icerocket.com + + ICQ: + parameters: + - q + domains: + - www.icq.com + - search.icq.com + + Ilse: + parameters: + - search_for + domains: + - www.ilse.nl + + Inbox.com: + parameters: + - q + domains: + - inbox.com/search/ + + Indeed: + domains: + - de.indeed.com + - at.indeed.com + - fr.indeed.com + - it.indeed.com + - ch.indeed.com + - au.indeed.com + + InfoSpace: + parameters: + - q + - s + domains: + - infospace.com + - dogpile.com + - www.dogpile.com + - metacrawler.com + - webfetch.com + - webcrawler.com + - search.kiwee.com + # powered by InfoSpace + - isearch.babylon.com + - start.facemoods.com + - search.magnetic.com + - search.searchcompletion.com + - clusty.com + + Flyingbird: + parameters: + - q + domains: + - inspsearch.com + - viview.inspsearch.com + + Interia: + parameters: + - q + domains: + - www.google.interia.pl + + I-play: + parameters: + - q + domains: + - start.iplay.com + + I.ua: + parameters: + - q + domains: + - search.i.ua + + IXquick: + parameters: + - query + domains: + - ixquick.com + - www.eu.ixquick.com + - ixquick.de + - www.ixquick.de + - us.ixquick.com + - s1.us.ixquick.com + - s2.us.ixquick.com + - s3.us.ixquick.com + - s4.us.ixquick.com + - s5.us.ixquick.com + - eu.ixquick.com + - s8-eu.ixquick.com + - s1-eu.ixquick.de + + Jyxo: + parameters: + - q + domains: + - jyxo.1188.cz + + Jungle Spider: + parameters: + - q + domains: + - www.jungle-spider.de + + Jungle Key: + parameters: + - query + domains: + - junglekey.com + - junglekey.fr + + Kataweb: + parameters: + - q + domains: + - www.kataweb.it + + Kvasir: + parameters: + - q + domains: + - www.kvasir.no + + kununu: + parameters: + - q + domains: + - kununu.com + + Latne: + parameters: + - q + domains: + - www.latne.lv + + La Toile Du Quebec Via Google: + parameters: + - q + domains: + - www.toile.com + - web.toile.com + + Lilo: + parameters: + - q + domains: + - search.lilo.org + + Liveinternet: + parameters: + - q + domains: + - liveinternet.ru + + Looksmart: + parameters: + - key + domains: + - www.looksmart.com + + Lo.st: + parameters: + - x_query + domains: + - lo.st + + Lycos: + parameters: + - query + domains: + - search.lycos.com + - www.lycos.com + - lycos.com + + maailm: + parameters: + - tekst + domains: + - www.maailm.com + + Mail.ru: + parameters: + - q + domains: + - mail.ru + - m.mail.ru + - go.mail.ru + + Mamma: + parameters: + - query + domains: + - www.mamma.com + - mamma75.mamma.com + + Marktplaats: + parameters: + - query + domains: + - www.marktplaats.nl + + Maxwebsearch: + parameters: + - query + domains: + - maxwebsearch.com + + Meta: + parameters: + - q + domains: + - meta.ua + + MetaCrawler.de: + parameters: + - qry + domains: + - s1.metacrawler.de + - s2.metacrawler.de + - s3.metacrawler.de + + Metager: + parameters: + - eingabe + domains: + - meta.rrzn.uni-hannover.de + - www.metager.de + + Metager2: + parameters: + - q + domains: + - metager2.de + + Meinestadt: + parameters: + - words + domains: + - www.meinestadt.de + + Mister Wong: + parameters: + - Keywords + domains: + - www.mister-wong.com + - www.mister-wong.de + + Monster: + parameters: + - q + domains: + - www.monster.be + - www.monster.cz + - www.monster.de + - www.monster.fi + - www.monster.fr + - www.monster.ie + - www.monster.it + - www.monster.lu + - www.monster.ch + - www.monster.co.uk + + Monstercrawler: + parameters: + - qry + domains: + - www.monstercrawler.com + + Mozbot: + parameters: + - q + domains: + - www.mozbot.fr + - www.mozbot.co.uk + - www.mozbot.com + + El Mundo: + parameters: + - q + domains: + - ariadna.elmundo.es + + MySearch: + parameters: + - searchfor + - searchFor + domains: + - mysearch.com + - www.mysearch.com + - ms114.mysearch.com + - ms146.mysearch.com + - kf.mysearch.myway.com + - ki.mysearch.myway.com + - search.myway.com + - search.mywebsearch.com + + Najdi: + parameters: + - q + domains: + - www.najdi.si + + Nate: + parameters: + - q + domains: + - search.nate.com + + Naver: + parameters: + - query + domains: + - search.naver.com + + Naver Images: + parameters: + - query + domains: + - image.search.naver.com + - imagesearch.naver.com + + Needtofind: + parameters: + - searchfor + domains: + - ko.search.need2find.com + + Neti: + parameters: + - query + domains: + - www.neti.ee + + Nifty: + parameters: + - q + domains: + - search.nifty.com + + Nigma: + parameters: + - s + domains: + - nigma.ru + + Onet: + parameters: + - qt + domains: + - szukaj.onet.pl + + Online.no: + parameters: + - q + domains: + - online.no + + Opplysningen 1881: + parameters: + - Query + domains: + - www.1881.no + + Orange: + parameters: + - q + - kw + domains: + - busca.orange.es + - search.orange.co.uk + - lemoteur.orange.fr + + Paperball: + parameters: + - q + domains: + - www.paperball.de + + PeoplePC: + parameters: + - q + domains: + - search.peoplepc.com + + Picsearch: + parameters: + - q + domains: + - www.picsearch.com + + Plazoo: + parameters: + - q + domains: + - www.plazoo.com + + Poisk.ru: + parameters: + - q + domains: + - poisk.ru + + PriceRunner: + parameters: + - q + domains: + - www.pricerunner.co.uk + + qip: + parameters: + - query + domains: + - search.qip.ru + + Qualigo: + parameters: + - q + domains: + - www.qualigo.at + - www.qualigo.ch + - www.qualigo.de + - www.qualigo.nl + + Qwant: + parameters: + - q + domains: + - www.qwant.com + - lite.qwant.com + + Rakuten: + parameters: + - qt + domains: + - websearch.rakuten.co.jp + + Rambler: + parameters: + - query + - words + domains: + - nova.rambler.ru + + RPMFind: + parameters: + - query + domains: + - rpmfind.net + - fr2.rpmfind.net + + Road Runner Search: + parameters: + - q + domains: + - search.rr.com + + Sapo: + parameters: + - q + domains: + - pesquisa.sapo.pt + + # Add Scour.com + + Search This: + parameters: + - q + domains: + - www.searchthis.com + + Search.com: + parameters: + - q + domains: + - www.search.com + + Search.ch: + parameters: + - q + domains: + - www.search.ch + + Searchalot: + parameters: + - q + domains: + - searchalot.com + + SearchCanvas: + parameters: + - q + domains: + - www.searchcanvas.com + + Searchy: + parameters: + - q + domains: + - www.searchy.co.uk + + # Add setooz.com + + Seznam: + parameters: + - q + domains: + - search.seznam.cz + + Sharelook: + parameters: + - keyword + domains: + - www.sharelook.fr + + Skynet: + parameters: + - q + domains: + - www.skynet.be + + The Smart Search: + parameters: + - q + domains: + - thesmartsearch.net + - www.thesmartsearch.net + + Sogou: + parameters: + - query + - w + domains: + - www.sougou.com + - www.soso.com + + Softonic: + parameters: + - q + domains: + - search.softonic.com + + SoSoDesk: + parameters: + - q + domains: + - sosodesktop.com + - search.sosodesktop.com + + Shenma: + parameters: + - q + domains: + - so.m.sm.cn + - yz.m.sm.cn + - m.sm.cn + - quark.sm.cn + - m.sp.sm.cn + - m.yz2.sm.cn + - m.yz.sm.cn + + Snapdo: + parameters: + - q + domains: + - search.snapdo.com + + Startpagina: + parameters: + - q + domains: + - startgoogle.startpagina.nl + + Startsiden: + parameters: + - q + domains: + - www.startsiden.no + + StepStone: + domains: + - www.stepstone.de + - www.stepstone.at + - www.stepstone.be + - www.stepstone.fr + - www.stepstone.nl + - www.stepstone.dk + - www.stepstone.se + + suche.info: + parameters: + - q + domains: + - suche.info + + Suchmaschine.com: + parameters: + - suchstr + domains: + - www.suchmaschine.com + + Suchnase: + parameters: + - q + domains: + - www.suchnase.de + + TalkTalk: + parameters: + - query + domains: + - www.talktalk.co.uk + + Technorati: + parameters: + - q + domains: + - technorati.com + + Telstra: + parameters: + - find + domains: + - search.media.telstra.com.au + + Teoma: + parameters: + - q + domains: + - www.teoma.com + + Terra: + parameters: + - query + domains: + - buscador.terra.es + - buscador.terra.cl + - buscador.terra.com.br + + Tiscali: + parameters: + - q + - key + domains: + - search.tiscali.it + - search-dyn.tiscali.it + - hledani.tiscali.cz + + Tixuma: + parameters: + - sc + domains: + - www.tixuma.de + + T-Online: + parameters: + - q + domains: + - suche.t-online.de + - brisbane.t-online.de + - navigationshilfe.t-online.de + + Toolbarhome: + parameters: + - q + domains: + - www.toolbarhome.com + - vshare.toolbarhome.com + + Trouvez.com: + parameters: + - query + domains: + - www.trouvez.com + + TrovaRapido: + parameters: + - q + domains: + - www.trovarapido.com + + Trusted-Search: + parameters: + - w + domains: + - www.trusted--search.com + + Tut.by: + parameters: + - query + domains: + - search.tut.by + + Twingly: + parameters: + - q + domains: + - www.twingly.com + + UKR.net: + parameters: + - q + domains: + - search.ukr.net + + uol.com.br: + parameters: + - q + domains: + - busca.uol.com.br + + URL.ORGanizier: + parameters: + - q + domains: + - www.url.org + + Vinden: + parameters: + - q + domains: + - www.vinden.nl + + Vindex: + parameters: + - search_for + domains: + - www.vindex.nl + - search.vindex.nl + + Virgilio: + parameters: + - qs + domains: + - ricerca.virgilio.it + - ricercaimmagini.virgilio.it + - ricercavideo.virgilio.it + - ricercanews.virgilio.it + - mobile.virgilio.it + + Voila: + parameters: + - rdata + - kw + domains: + - search.ke.voila.fr + - www.lemoteur.fr + + Volny: + parameters: + - search + domains: + - web.volny.cz + + Walhello : + parameters: + - key + domains: + - www.walhello.info + - www.walhello.com + - www.walhello.de + - www.walhello.nl + + Web.de: + parameters: + - su + domains: + - suche.web.de + + Web.nl: + parameters: + - zoekwoord + domains: + - www.web.nl + + Weborama: + parameters: + - QUERY + domains: + - www.weborama.com + + WebSearch: + parameters: + - qkw + - q + domains: + - www.websearch.com + + Winamp: + parameters: + - q + domains: + - search.winamp.com + + Witch: + parameters: + - search + domains: + - www.witch.de + + Wirtualna Polska: + parameters: + - szukaj + domains: + - szukaj.wp.pl + + WWW: + parameters: + - query + domains: + - search.www.ee + + X-recherche: + parameters: + - MOTS + domains: + - www.x-recherche.com + + Yahoo!: + parameters: + - p + - q + domains: + - search.yahoo.com + - yahoo.com + - ar.search.yahoo.com + - ar.yahoo.com + - au.search.yahoo.com + - au.yahoo.com + - br.search.yahoo.com + - br.yahoo.com + - ca.search.yahoo.com + - ca.yahoo.com + - cade.searchde.yahoo.com + - cade.yahoo.com + - chinese.searchinese.yahoo.com + - chinese.yahoo.com + - cn.search.yahoo.com + - cn.yahoo.com + - de.search.yahoo.com + - de.yahoo.com + - dk.search.yahoo.com + - dk.yahoo.com + - es.search.yahoo.com + - es.yahoo.com + - espanol.searchpanol.yahoo.com + - espanol.yahoo.com + - fr.search.yahoo.com + - fr.yahoo.com + - hk.search.yahoo.com + - hk.yahoo.com + - ie.search.yahoo.com + - ie.yahoo.com + - in.search.yahoo.com + - in.yahoo.com + - it.search.yahoo.com + - it.yahoo.com + - kr.search.yahoo.com + - kr.yahoo.com + - mx.search.yahoo.com + - mx.yahoo.com + - no.search.yahoo.com + - no.yahoo.com + - nz.search.yahoo.com + - nz.yahoo.com + - one.cn.yahoo.com + - one.searchn.yahoo.com + - qc.search.yahoo.com + - qc.yahoo.com + - ru.search.yahoo.com + - ru.yahoo.com + - se.search.yahoo.com + - se.yahoo.com + - search.searcharch.yahoo.com + - tw.search.yahoo.com + - tw.yahoo.com + - uk.search.yahoo.com + - uk.yahoo.com + - us.search.yahoo.com + - us.yahoo.com + - www.yahoo.co.jp + - search.yahoo.co.jp + # powered by Yahoo + - www.cercato.it + - search.offerbox.com + - ys.mirostart.com + + Yahoo! Images: + parameters: + - p + - q + domains: + - image.yahoo.cn + - images.search.yahoo.com + + Yam: + parameters: + - k + domains: + - search.yam.com + + Yandex: + parameters: + - text + domains: + - yandex.ru + - yandex.ua + - yandex.com + - yandex.by + - www.yandex.ru + - www.yandex.ua + - www.yandex.com + - www.yandex.by + - clck.yandex.ru + - clck.yandex.ua + - clck.yandex.com + - clck.yandex.by + + Yandex Images: + parameters: + - text + domains: + - images.yandex.ru + - images.yandex.ua + - images.yandex.com + - images.yandex.by + + Yasni: + parameters: + - query + domains: + - www.yasni.de + - www.yasni.com + - www.yasni.co.uk + - www.yasni.ch + - www.yasni.at + + Yatedo: + parameters: + - q + domains: + - www.yatedo.com + - www.yatedo.fr + + # Add Yellowmap: + + Yippy: + parameters: + - q + - query + domains: + - search.yippy.com + + YouGoo: + parameters: + - q + domains: + - www.yougoo.fr + + Zapmeta: + parameters: + - q + - query + domains: + - www.zapmeta.com + - www.zapmeta.nl + - www.zapmeta.de + - uk.zapmeta.com + + Zoek: + parameters: + - q + domains: + - www3.zoek.nl + + Zhongsou: + parameters: + - w + domains: + - p.zhongsou.com + + Zoeken: + parameters: + - q + domains: + - www.zoeken.nl + + Zoohoo: + parameters: + - q + domains: + - zoohoo.cz + + + +# ####################################################################################################### +# +# PAID MEDIA + +paid: + + Acuity Ads: + domains: + - acuityplatform.com + + Adform: + domains: + - adform.net + + ADFOX: + domains: + - adfox.ru + - www.adfox.ru + - ads.adfox.ru + - www.ads.adfox.ru + + Adition: + domains: + - adition.com + + AdNET: + domains: + - adnet.de + + AdRoll: + domains: + - adroll.com + + AdSpirit: + domains: + - adspirit.de + - rtbcity.com + - plusperformance.com + + AppNexus: + domains: + - ib.adnxs.com + - adnxs.com + - 247realmedia.com + + AudienceScience: + domains: + - wunderloop.net + + BidSwitch: + domains: + - bidswitch.net + + Casale Media: + domains: + - casalemedia.com + + Criteo: + domains: + - cas.jp.as.criteo.com + - cas.criteo.com + + Doubleclick: + domains: + - ad.doubleclick.net + - ad-apac.doubleclick.net + - s0.2mdn.net + - s1.2mdn.net + - dp.g.doubleclick.net + - pubads.g.doubleclick.net + + Eyeota: + domains: + - eyeota.net + + Flashtalking: + domains: + - flashtalking.com + - servedby.flashtalking.com + + Fluct: + domains: + - adingo.jp + + Google: + domains: + - www.googleadservices.com + - partner.googleadservices.com + - googleads.g.doubleclick.net + - tpc.googlesyndication.com + - googleadservices.com + - imasdk.googleapis.com + + LifeStreet: + domains: + - lfstmedia.com + + LowerMyBills: + parameters: + - leadid + domains: + - lowermybills.com + + Jivox: + domains: + - jivox.com + + MicroAd: + domains: + - microad.jp + + Mixpo: + domains: + - mixpo.com + + Mozo: + domains: + - mozo.com.au + - a.mozo.com.au + + Neustar AdAdvisor: + domains: + - adadvisor.net + + ONE by AOL: + domains: + - nexage.com + + OpenX: + domains: + - us-ads.openx.net + - openx.net + - servedbyopenx.com + - openxenterprise.com + + Outbrain: + domains: + - paid.outbrain.com + + Plista: + domains: + - farm.plista.com + + Price.ru: + domains: + - price.ru + - v.price.ru + + PubMatic: + domains: + - sshowads.pubmatic.com + + Rubicon Project: + domains: + - optimized-by.rubiconproject.com + + Sizmek: + domains: + - bs.serving-sys.com + + Sociomantic Labs: + domains: + - sociomantic.com + + Sonobi: + domains: + - sonobi.com + + Sovrn: + domains: + - lijit.com + + SteelHouse: + domains: + - steelhousemedia.com + + StickyADS.tv: + domains: + - stickyadstv.com + - sfx.stickyadstv.com + + Taboola: + domains: + - trc.taboola.com + - api.taboola.com + - taboola.com + + Torg.Mail.ru: + domains: + - torg.mail.ru + + Tribal Fusion: + domains: + - cdnx.tribalfusion.com + + White Pages: + domains: + - www.whitepages.com.au + - mobile.whitepages.com.au + + Yandex.Direct: + domains: + - an.yandex.ru + - yabs.yandex.ru + - yabs.yandex.ua + - yabs.yandex.com + - yabs.yandex.by + + Yandex.Market: + domains: + - market.yandex.ru + - m.market.yandex.ru + + Yieldmo: + domains: + - yieldmo.com + + ZEDO: + domains: + - zedo.com + - z1.zedo.com diff --git a/priv/referer_favicon_domains.json b/priv/referer_favicon_domains.json new file mode 100644 index 0000000000000000000000000000000000000000..6430d7f77344abc4a6d17df39608ed7cc0aa719a --- /dev/null +++ b/priv/referer_favicon_domains.json @@ -0,0 +1 @@ +{"Friends Reunited":"friendsreunited.com","I-play":"start.iplay.com","White Pages":"www.whitepages.com.au","Eniro":"www.eniro.se","Kvasir":"www.kvasir.no","Geona":"geona.net","Neustar AdAdvisor":"adadvisor.net","Wirtualna Polska":"szukaj.wp.pl","Tagged":"login.tagged.com","Liveinternet":"liveinternet.ru","Xanga":"xanga.com","Metager2":"metager2.de","Commander":"webmail.commander.net.au","QIP":"mail.qip.ru","Plazoo":"www.plazoo.com","Globososo":"searches.globososo.com","Zoohoo":"zoohoo.cz","Eurip":"www.eurip.com","Web.de":"suche.web.de","Mailchimp":"com.mailchimp.mailchimp","DuckDuckGo":"duckduckgo.com","Google Blogsearch":"blogsearch.google.ac","Searchy":"www.searchy.co.uk","MySearch":"mysearch.com","AppNexus":"ib.adnxs.com","Outbrain":"paid.outbrain.com","Metager":"meta.rrzn.uni-hannover.de","Search.com":"www.search.com","Westnet":"webmail.westnet.com.au","BlackPlanet":"blackplanet.com","Arcor":"www.arcor.de","Bing":"bing.com","Orange Webmail":"orange.fr/webmail","Nasza-klasa.pl":"nk.pl","URL.ORGanizier":"www.url.org","HighBeam":"www.highbeam.com","Instagram":"instagram.com","Rubicon Project":"optimized-by.rubiconproject.com","Mixi":"mixi.jp","Web.nl":"www.web.nl","X-recherche":"www.x-recherche.com","Freenet":"webmail.freenet.de","Altavista":"www.altavista.com","Teoma":"www.teoma.com","2gis":"2gis.ru","Google Video":"video.google.com","Viadeo":"viadeo.com","DasOertliche":"www.dasoertliche.de","canoe.ca":"web.canoe.ca","Sapo":"pesquisa.sapo.pt","TikTok":"tiktok.com","Voila":"search.ke.voila.fr","Sovrn":"lijit.com","Icerockeet":"blogs.icerocket.com","Babylon":"search.babylon.com","Yandex Images":"images.yandex.ru","Euroseek":"www.euroseek.com","Hacker News":"news.ycombinator.com","Vimeo":"vimeo.com","Shenma":"so.m.sm.cn","AOL":"search.aol.com","Tribal Fusion":"cdnx.tribalfusion.com","Flyingbird":"inspsearch.com","Austronaut":"www2.austronaut.at","Hocam.com":"hocam.com","Yahoo! Mail":"mail.yahoo.net","LinkedIn":"com.linkedin.android","SoSoDesk":"sosodesktop.com","Yippy":"search.yippy.com","PubMatic":"sshowads.pubmatic.com","Fireball":"www.fireball.de","Adition":"adition.com","arama":"arama.com","Hyves":"hyves.nl","Classmates":"classmates.com","Lo.st":"lo.st","Google+":"url.google.com","DasTelefonbuch":"www1.dastelefonbuch.de","E1.ru":"mail.e1.ru","Forestle":"forestle.org","StackOverflow":"stackoverflow.com","AdRoll":"adroll.com","Zoho":"mail.zoho.com","TrovaRapido":"www.trovarapido.com","Finderoo":"www.finderoo.com","Dodo":"webmail.dodo.com.au","Flixster":"flixster.com","GitHub":"github.com","Instela":"instela.com","BidSwitch":"bidswitch.net","ONE by AOL":"nexage.com","Cuil":"www.cuil.com","Donanimhaber":"donanimhaber.com","Threads":"threads.net","Opplysningen 1881":"www.1881.no","Virgin":"webmail.virginbroadband.com.au","Google News":"news.google.ac","Acuity Ads":"acuityplatform.com","myYearbook":"myyearbook.com","SearchCanvas":"www.searchcanvas.com","Goyellow.de":"www.goyellow.de","Taringa!":"taringa.net","Nifty":"search.nifty.com","T-Online":"suche.t-online.de","Startsiden":"www.startsiden.no","Alice Adsl":"rechercher.aliceadsl.fr","MyLife":"mylife.ru","Acoon":"www.acoon.de","Freshweather":"www.fresh-weather.com","Last.fm":"lastfm.ru","Telegram":"web.telegram.org","LiveJournal":"livejournal.ru","Plaxo":"plaxo.com","GAIS":"gais.cs.ccu.edu.tw","Sogou":"www.sougou.com","ZEDO":"zedo.com","WhatsApp":"web.whatsapp.com","Lycos":"search.lycos.com","Paper.li":"paper.li","Criteo":"cas.jp.as.criteo.com","Zhongsou":"p.zhongsou.com","Everyclick":"www.everyclick.com","Adform":"adform.net","Picsearch":"www.picsearch.com","iPrimus":"webmail.iprimus.com.au","SourceForge":"sourceforge.net","Kataweb":"www.kataweb.it","dmoz":"dmoz.org","WWW":"search.www.ee","Sonico.com":"sonico.com","OpenX":"us-ads.openx.net","Vindex":"www.vindex.nl","Alexa":"alexa.com","Arianna":"arianna.libero.it","Fluct":"adingo.jp","AllTheWeb":"www.alltheweb.com","Nigma":"nigma.ru","XING":"xing.com","PeoplePC":"search.peoplepc.com","Online.no":"online.no","Myspace":"myspace.com","Witch":"www.witch.de","Monstercrawler":"www.monstercrawler.com","Meta":"meta.ua","Findwide":"search.findwide.com","Foursquare":"foursquare.com","Cyworld":"global.cyworld.com","British Telecommunications":"search.bt.com","Yahoo!":"finance.yahoo.com","Paperball":"www.paperball.de","AudienceScience":"wunderloop.net","Looksmart":"www.looksmart.com","all.by":"all.by","PriceRunner":"www.pricerunner.co.uk","Mastermail":"mastermail.ru","Excite":"search.excite.it","Blogdigger":"www.blogdigger.com","Apollo Latvia":"apollo.lv/portal/search/","Rakuten":"websearch.rakuten.co.jp","Firstfind":"www.firstsfind.com","Zoek":"www3.zoek.nl","Bebo":"bebo.com","Amazon":"amazon.com","Mail.ru":"e.mail.ru","Quora":"quora.com","Tildes":"tildes.net","Certified-Toolbar":"search.certified-toolbar.com","AdNET":"adnet.de","Hotbot":"www.hotbot.com","Outlook.com":"mail.live.com","Vkontakte":"m.vk.com","Twingly":"www.twingly.com","IXquick":"ixquick.com","WebSearch":"www.websearch.com","Digg":"digg.com","vKruguDruzei.ru":"vkrugudruzei.ru","Identi.ca":"identi.ca","Seznam":"search.seznam.cz","Mixpo":"mixpo.com","Sharelook":"www.sharelook.fr","El Mundo":"ariadna.elmundo.es","Compuserve":"websearch.cs.com","TalkTalk":"www.talktalk.co.uk","Bing Images":"bing.com/images/search","Douban":"douban.com","StumbleUpon":"stumbleupon.com","Poisk.ru":"poisk.ru","Yieldmo":"yieldmo.com","Yandex Maps":"maps.yandex.ru","Beeline":"post.ru","Trouvez.com":"www.trouvez.com","Ukr.net":"mail.ukr.net","Mister Wong":"www.mister-wong.com","Daum":"search.daum.net","Exalead":"www.exalead.fr","Suchmaschine.com":"www.suchmaschine.com","AdSpirit":"adspirit.de","Conduit":"search.conduit.com","ABCsøk":"abcsolk.no","Indeed":"de.indeed.com","WAYN":"wayn.com","ADFOX":"adfox.ru","Search This":"www.searchthis.com","uol.com.br":"busca.uol.com.br","Atlas":"searchatlas.centrum.cz","Biglobe":"cgi.search.biglobe.ne.jp","Seznam Mail":"email.seznam.cz","Uludag Sozluk":"uludagsozluk.com","Optus Zoo":"webmail.optuszoo.com.au","blekko":"blekko.com","Qualigo":"www.qualigo.at","Fast Browser Search":"www.fastbrowsersearch.com","LowerMyBills":"lowermybills.com","Delfi":"otsing.delfi.ee","Weibo":"weibo.com","Gule Sider":"www.gulesider.no","Zoeken":"www.zoeken.nl","Taboola":"trc.taboola.com","Tut.by":"search.tut.by","InfoSpace":"infospace.com","Yam":"search.yam.com","Trusted-Search":"www.trusted--search.com","Needtofind":"ko.search.need2find.com","GMX":"suche.gmx.net","126 Mail":"mail.126.com","ITU Sozluk":"itusozluk.com","Qzone":"qzone.qq.com","MyHeritage":"myheritage.com","Maxwebsearch":"maxwebsearch.com","Torg.Mail.ru":"torg.mail.ru","Apontador":"apontador.com.br","APOLL07":"apollo7.de","Ecosia":"ecosia.org","Dalesearch":"www.dalesearch.com","suche.info":"suche.info","iiNet":"webmail.iinet.net.au","Mamma":"www.mamma.com","Buzznet":"buzznet.com","Road Runner Search":"search.rr.com","WeeWorld":"weeworld.com","Toolbarhome":"www.toolbarhome.com","Odnoklassniki":"odnoklassniki.ru","Inbox.com":"inbox.com","Vodafone":"webmail.vodafone.co.nz","Hooseek.com":"www.hooseek.com","Zapmeta":"www.zapmeta.com","Tuenti":"tuenti.com","Adam Internet":"webmail.adam.com.au","Snapdo":"search.snapdo.com","Startpagina":"startgoogle.startpagina.nl","Aport":"sm.aport.ru","Pocket":"getpocket.com","Jungle Key":"junglekey.com","I.ua":"search.i.ua","Eyeota":"eyeota.net","Eksi Sozluk":"Sozluk.com","Renren":"renren.com","Qwant":"www.qwant.com","Terra":"buscador.terra.es","Naver Mail":"mail.naver.com","Delfi latvia":"smart.delfi.lv","Netlog":"netlog.com","Skynet":"www.skynet.be","Daemon search":"daemon-search.com","Badoo":"badoo.com","StickyADS.tv":"stickyadstv.com","Yandex.Market":"market.yandex.ru","Interia":"www.google.interia.pl","Yasni":"www.yasni.de","Searchalot":"searchalot.com","Hit-Parade":"req.-hit-parade.com","Gnadenmeer":"www.gnadenmeer.de","Rambler":"mail.rambler.ru","Gomeo":"www.gomeo.com","Windows Live Spaces":"login.live.com","Sibmail":"sibmail.com","Disqus":"redirect.disqus.com","Mozo":"mozo.com.au","Snapchat":"com.snapchat.android","Whirlpool":"forums.whirlpool.net.au","AOL Mail":"mail.aol.com","Crawler":"www.crawler.com","Najdi":"www.najdi.si","Jyxo":"jyxo.1188.cz","Orange":"busca.orange.es","Tumblr":"tumblr.com","La Toile Du Quebec Via Google":"www.toile.com","Netspace":"webmail.netspace.net.au","Doubleclick":"ad.doubleclick.net","earthlink":"com.earthlink.myearthlink","Twitter":"twitter.com","Ilse":"www.ilse.nl","Jungle Spider":"www.jungle-spider.de","Softonic":"search.softonic.com","Youtube":"youtube.com","Volny":"web.volny.cz","Habbo":"habbo.com","Coccoc":"coccoc.com","Charter":"www.charter.net","1&1":"search.1and1.com","Gigablast":"www.gigablast.com","Flickr":"flickr.com","Google":"support.google.com","LifeStreet":"lfstmedia.com","Search.ch":"www.search.ch","QQ Mail":"mail.qq.com","Casale Media":"casalemedia.com","Weborama":"www.weborama.com","Neti":"www.neti.ee","Flashtalking":"flashtalking.com","Virgilio":"ricerca.virgilio.it","Yandex":"mail.yandex.ru","163 Mail":"mail.163.com","StepStone":"www.stepstone.de","Onet":"szukaj.onet.pl","Fixsuche":"www.fixsuche.de","Clix":"pesquisa.clix.pt","Baidu":"www.baidu.com","Telstra":"search.media.telstra.com.au","Ask":"ask.com","qip":"search.qip.ru","Vinden":"www.vinden.nl","Plista":"farm.plista.com","SteelHouse":"steelhousemedia.com","Winamp":"search.winamp.com","Sociomantic Labs":"sociomantic.com","Sonobi":"sonobi.com","Jivox":"jivox.com","Mynet Mail":"mail.mynet.com","Inci Sozluk":"inci.sozlukspot.com","YouGoo":"www.yougoo.fr","Centrum":"serach.centrum.cz","Reddit":"reddit.com","Francite":"recherche.francite.com","Google Product Search":"google.ac/products","2degrees":"webmail.2degreesbroadband.co.nz","Ask Toolbar":"search.tb.ask.com","Holmes":"holmes.ge","Gmail":"mail.google.com","eo":"eo.st","Yahoo! Images":"image.yahoo.cn","Fotolog":"fotolog.com","Naver":"search.naver.com","Pinterest":"pinterest.ca","Freecause":"search.freecause.com","Skyrock":"skyrock.com","Delicious":"delicious.com","Nate":"search.nate.com","Tiscali":"search.tiscali.it","Tixuma":"www.tixuma.de","MoiKrug.ru":"moikrug.ru","Sizmek":"bs.serving-sys.com","Multiply":"multiply.com","Orkut":"orkut.com","kununu":"kununu.com","UKR.net":"search.ukr.net","RPMFind":"rpmfind.net","Abacho":"www.abacho.de","Free":"search.free.fr","Friendster":"friendster.com","Meinestadt":"www.meinestadt.de","Lilo":"search.lilo.org","ICQ":"www.icq.com","FriendFeed":"friendfeed.com","Slack":"app.slack.com","Mozbot":"www.mozbot.fr","Genieo":"search.genieo.com","The Smart Search":"thesmartsearch.net","Yandex.Direct":"an.yandex.ru","Daum Mail":"mail2.daum.net","1und1":"search.1und1.de","goo":"search.goo.ne.jp","maailm":"www.maailm.com","1.cz":"1.cz","Suchnase":"www.suchnase.de","Technorati":"technorati.com","Naver Images":"image.search.naver.com","Gaia Online":"gaiaonline.com","MetaCrawler.de":"s1.metacrawler.de","Blogpulse":"www.blogpulse.com","MicroAd":"microad.jp","StudiVZ":"studivz.net","Facebook":"facebook.com","Monster":"www.monster.be","hi5":"hi5.com","Bluewin":"search.bluewin.ch","Yatedo":"www.yatedo.com","Walhello":"www.walhello.info","Flix":"www.flix.de","Google Images":"google.ac/imgres","360.cn":"so.360.cn","Comcast":"serach.comcast.net","Skype":"web.skype.com","Latne":"www.latne.lv","Bigpond":"webmail.bigpond.com","Marktplaats":"www.marktplaats.nl","Geni":"geni.com","Price.ru":"price.ru"} diff --git a/priv/repo/migrations/.formatter.exs b/priv/repo/migrations/.formatter.exs new file mode 100644 index 0000000000000000000000000000000000000000..99706d39eb7928a541fafb6bd4483f42e6addcbe --- /dev/null +++ b/priv/repo/migrations/.formatter.exs @@ -0,0 +1,4 @@ +[ + import_deps: [:ecto_sql, :open_api_spex], + inputs: ["*.exs"] +] diff --git a/priv/repo/migrations/20181201181549_add_pageviews.exs b/priv/repo/migrations/20181201181549_add_pageviews.exs new file mode 100644 index 0000000000000000000000000000000000000000..f728dfbb03d4041e8f2a2c65be6c9add78be2d75 --- /dev/null +++ b/priv/repo/migrations/20181201181549_add_pageviews.exs @@ -0,0 +1,16 @@ +defmodule Plausible.Repo.Migrations.AddPageviews do + use Ecto.Migration + + def change do + create table(:pageviews) do + add :hostname, :text, null: false + add :pathname, :text, null: false + add :referrer, :text + add :user_agent, :text + add :screen_width, :integer + add :screen_height, :integer + + timestamps() + end + end +end diff --git a/priv/repo/migrations/20181214201821_add_new_visitor_to_pageviews.exs b/priv/repo/migrations/20181214201821_add_new_visitor_to_pageviews.exs new file mode 100644 index 0000000000000000000000000000000000000000..7adc3eebac380ad345e7b130aad276b65cd48e2c --- /dev/null +++ b/priv/repo/migrations/20181214201821_add_new_visitor_to_pageviews.exs @@ -0,0 +1,10 @@ +defmodule Plausible.Repo.Migrations.AddNewVisitorToPageviews do + use Ecto.Migration + use Plausible.Repo + + def change do + alter table(:pageviews) do + add :new_visitor, :boolean, null: false + end + end +end diff --git a/priv/repo/migrations/20181215140923_add_session_id_to_pageviews.exs b/priv/repo/migrations/20181215140923_add_session_id_to_pageviews.exs new file mode 100644 index 0000000000000000000000000000000000000000..37144338dfce4529a2decd979c1b4701c77a0bdc --- /dev/null +++ b/priv/repo/migrations/20181215140923_add_session_id_to_pageviews.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AddSessionIdToPageviews do + use Ecto.Migration + + def change do + alter table(:pageviews) do + add :session_id, :string + end + end +end diff --git a/priv/repo/migrations/20190109173917_create_sites.exs b/priv/repo/migrations/20190109173917_create_sites.exs new file mode 100644 index 0000000000000000000000000000000000000000..564ed7a3f5f3c67ff08a3aebe0ed1beaab9072f9 --- /dev/null +++ b/priv/repo/migrations/20190109173917_create_sites.exs @@ -0,0 +1,30 @@ +defmodule Plausible.Repo.Migrations.CreateSites do + use Ecto.Migration + + def change do + create table(:users) do + add :email, :string, null: false + + timestamps() + end + + create unique_index(:users, :email) + + create table(:sites) do + add :domain, :string, null: false + + timestamps() + end + + create unique_index(:sites, :domain) + + create table(:site_memberships) do + add :site_id, references(:sites), null: false + add :user_id, references(:users), null: false + + timestamps() + end + + create unique_index(:site_memberships, [:site_id, :user_id]) + end +end diff --git a/priv/repo/migrations/20190117135714_add_uid_to_pageviews.exs b/priv/repo/migrations/20190117135714_add_uid_to_pageviews.exs new file mode 100644 index 0000000000000000000000000000000000000000..89cc3f8999cb6c456abf1753b6b165970c5c80e3 --- /dev/null +++ b/priv/repo/migrations/20190117135714_add_uid_to_pageviews.exs @@ -0,0 +1,10 @@ +defmodule Plausible.Repo.Migrations.AddUidToPageviews do + use Ecto.Migration + use Plausible.Repo + + def change do + alter table(:pageviews) do + add :user_id, :binary_id, null: false + end + end +end diff --git a/priv/repo/migrations/20190118154210_add_derived_data_to_pageviews.exs b/priv/repo/migrations/20190118154210_add_derived_data_to_pageviews.exs new file mode 100644 index 0000000000000000000000000000000000000000..a372c34060055f016c7ace8b672edc6bc070772a --- /dev/null +++ b/priv/repo/migrations/20190118154210_add_derived_data_to_pageviews.exs @@ -0,0 +1,14 @@ +defmodule Plausible.Repo.Migrations.AddDerivedDataToPageviews do + use Ecto.Migration + use Plausible.Repo + + def change do + alter table(:pageviews) do + add :device_type, :string + add :browser, :string + add :operating_system, :string + add :referrer_source, :string + add :screen_size, :string + end + end +end diff --git a/priv/repo/migrations/20190126135857_add_name_to_users.exs b/priv/repo/migrations/20190126135857_add_name_to_users.exs new file mode 100644 index 0000000000000000000000000000000000000000..92fbc0acbbd2b655ebc1148f8e5aec7740a8ec54 --- /dev/null +++ b/priv/repo/migrations/20190126135857_add_name_to_users.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AddNameToUsers do + use Ecto.Migration + + def change do + alter table(:users) do + add :name, :string + end + end +end diff --git a/priv/repo/migrations/20190127213938_add_tz_to_sites.exs b/priv/repo/migrations/20190127213938_add_tz_to_sites.exs new file mode 100644 index 0000000000000000000000000000000000000000..3649dfb09e4ed2ad8a481e1f5aa1285fb07f5616 --- /dev/null +++ b/priv/repo/migrations/20190127213938_add_tz_to_sites.exs @@ -0,0 +1,18 @@ +defmodule Plausible.Repo.Migrations.AddTzToSites do + use Ecto.Migration + use Plausible.Repo + + def change do + alter table(:sites) do + add :timezone, :string + end + + flush() + + Repo.update_all(Plausible.Site, set: [timezone: "UTC"]) + + alter table(:sites) do + modify :timezone, :string, null: false + end + end +end diff --git a/priv/repo/migrations/20190205165931_add_last_seen_to_users.exs b/priv/repo/migrations/20190205165931_add_last_seen_to_users.exs new file mode 100644 index 0000000000000000000000000000000000000000..24b556acb97798cb92c29a77e90c852e19e15018 --- /dev/null +++ b/priv/repo/migrations/20190205165931_add_last_seen_to_users.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AddLastSeenToUsers do + use Ecto.Migration + + def change do + alter table(:users) do + add :last_seen, :naive_datetime, default: fragment("now()") + end + end +end diff --git a/priv/repo/migrations/20190213224404_add_intro_emails.exs b/priv/repo/migrations/20190213224404_add_intro_emails.exs new file mode 100644 index 0000000000000000000000000000000000000000..ce2789002dd4e17fa0ea4efcdf3034ffaa0eabd8 --- /dev/null +++ b/priv/repo/migrations/20190213224404_add_intro_emails.exs @@ -0,0 +1,10 @@ +defmodule Plausible.Repo.Migrations.AddIntroEmails do + use Ecto.Migration + + def change do + create table(:intro_emails) do + add :user_id, references(:users), null: false + add :timestamp, :naive_datetime + end + end +end diff --git a/priv/repo/migrations/20190219130809_delete_intro_emails_when_user_is_deleted.exs b/priv/repo/migrations/20190219130809_delete_intro_emails_when_user_is_deleted.exs new file mode 100644 index 0000000000000000000000000000000000000000..3fb741dd1550f7328bd0b9bc1d41e09503d9c71a --- /dev/null +++ b/priv/repo/migrations/20190219130809_delete_intro_emails_when_user_is_deleted.exs @@ -0,0 +1,11 @@ +defmodule Plausible.Repo.Migrations.DeleteIntroEmailsWhenUserIsDeleted do + use Ecto.Migration + + def change do + alter table(:intro_emails) do + modify :user_id, references(:users, on_delete: :delete_all), + null: false, + from: references(:users) + end + end +end diff --git a/priv/repo/migrations/20190301122344_add_country_code_to_pageviews.exs b/priv/repo/migrations/20190301122344_add_country_code_to_pageviews.exs new file mode 100644 index 0000000000000000000000000000000000000000..287cfe9c87231bc625fa77006ac0a432f26c5e05 --- /dev/null +++ b/priv/repo/migrations/20190301122344_add_country_code_to_pageviews.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AddCountryCodeToPageviews do + use Ecto.Migration + + def change do + alter table(:pageviews) do + add :country_code, :string, size: 2 + end + end +end diff --git a/priv/repo/migrations/20190324155606_add_password_hash_to_users.exs b/priv/repo/migrations/20190324155606_add_password_hash_to_users.exs new file mode 100644 index 0000000000000000000000000000000000000000..73b844eaf5c254051308e9a9a87a32364fcdfef1 --- /dev/null +++ b/priv/repo/migrations/20190324155606_add_password_hash_to_users.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AddPasswordHashToUsers do + use Ecto.Migration + + def change do + alter table(:users) do + add :password_hash, :string + end + end +end diff --git a/priv/repo/migrations/20190402145007_remove_device_type_from_pageviews.exs b/priv/repo/migrations/20190402145007_remove_device_type_from_pageviews.exs new file mode 100644 index 0000000000000000000000000000000000000000..1cbdde2f5b0d4f28edd951a4b9e4ba273d042ae6 --- /dev/null +++ b/priv/repo/migrations/20190402145007_remove_device_type_from_pageviews.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.RemoveDeviceTypeFromPageviews do + use Ecto.Migration + + def change do + alter table(:pageviews) do + remove :device_type + end + end +end diff --git a/priv/repo/migrations/20190402145357_remove_screen_height_from_pageviews.exs b/priv/repo/migrations/20190402145357_remove_screen_height_from_pageviews.exs new file mode 100644 index 0000000000000000000000000000000000000000..9797e00f4b516ec486ed466b8ca38cc210c5a26e --- /dev/null +++ b/priv/repo/migrations/20190402145357_remove_screen_height_from_pageviews.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.RemoveScreenHeightFromPageviews do + use Ecto.Migration + + def change do + alter table(:pageviews) do + remove :screen_height + end + end +end diff --git a/priv/repo/migrations/20190402172423_add_index_to_pageviews.exs b/priv/repo/migrations/20190402172423_add_index_to_pageviews.exs new file mode 100644 index 0000000000000000000000000000000000000000..3f394fcffb4e0578a33c3e771df44e3c2a4fa5a3 --- /dev/null +++ b/priv/repo/migrations/20190402172423_add_index_to_pageviews.exs @@ -0,0 +1,7 @@ +defmodule Plausible.Repo.Migrations.AddIndexToPageviews do + use Ecto.Migration + + def change do + create index("pageviews", [:hostname]) + end +end diff --git a/priv/repo/migrations/20190410095248_add_feedback_emails.exs b/priv/repo/migrations/20190410095248_add_feedback_emails.exs new file mode 100644 index 0000000000000000000000000000000000000000..bc1f62e455dd21d6e9e66a92a9fae7eb7c4be3f8 --- /dev/null +++ b/priv/repo/migrations/20190410095248_add_feedback_emails.exs @@ -0,0 +1,10 @@ +defmodule Plausible.Repo.Migrations.AddFeedbackEmails do + use Ecto.Migration + + def change do + create table(:feedback_emails) do + add :user_id, references(:users), null: false + add :timestamp, :naive_datetime, null: false + end + end +end diff --git a/priv/repo/migrations/20190424162903_delete_feedback_emails_when_user_is_deleted.exs b/priv/repo/migrations/20190424162903_delete_feedback_emails_when_user_is_deleted.exs new file mode 100644 index 0000000000000000000000000000000000000000..88a0aea43e382e71922939cc34a9428e3f9bbb02 --- /dev/null +++ b/priv/repo/migrations/20190424162903_delete_feedback_emails_when_user_is_deleted.exs @@ -0,0 +1,11 @@ +defmodule Plausible.Repo.Migrations.DeleteFeedbackEmailsWhenUserIsDeleted do + use Ecto.Migration + + def change do + alter table(:feedback_emails) do + modify :user_id, references(:users, on_delete: :delete_all), + null: false, + from: references(:users) + end + end +end diff --git a/priv/repo/migrations/20190430140411_use_citext_for_email.exs b/priv/repo/migrations/20190430140411_use_citext_for_email.exs new file mode 100644 index 0000000000000000000000000000000000000000..ecb6f805e448f5db79ea6f16dc5ccc6a72a6969b --- /dev/null +++ b/priv/repo/migrations/20190430140411_use_citext_for_email.exs @@ -0,0 +1,11 @@ +defmodule Plausible.Repo.Migrations.UseCitextForEmail do + use Ecto.Migration + + def change do + execute "CREATE EXTENSION IF NOT EXISTS citext;" + + alter table(:users) do + modify :email, :citext, null: false + end + end +end diff --git a/priv/repo/migrations/20190430152923_create_subscriptions.exs b/priv/repo/migrations/20190430152923_create_subscriptions.exs new file mode 100644 index 0000000000000000000000000000000000000000..fb44bfacd75be4a8a820318df28d768849b71be9 --- /dev/null +++ b/priv/repo/migrations/20190430152923_create_subscriptions.exs @@ -0,0 +1,20 @@ +defmodule Plausible.Repo.Migrations.CreateSubscriptions do + use Ecto.Migration + + def change do + create table(:subscriptions) do + add :paddle_subscription_id, :string, null: false + add :paddle_plan_id, :string, null: false + add :user_id, references(:users), null: false + add :update_url, :text, null: false + add :cancel_url, :text, null: false + add :status, :string, null: false + add :next_bill_amount, :string, null: false + add :next_bill_date, :date, null: false + + timestamps() + end + + create unique_index(:subscriptions, [:paddle_subscription_id]) + end +end diff --git a/priv/repo/migrations/20190516113517_remove_session_id_from_pageviews.exs b/priv/repo/migrations/20190516113517_remove_session_id_from_pageviews.exs new file mode 100644 index 0000000000000000000000000000000000000000..af88e4b278c4863151197a21e14d42463ea07489 --- /dev/null +++ b/priv/repo/migrations/20190516113517_remove_session_id_from_pageviews.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.RemoveSessionIdFromPageviews do + use Ecto.Migration + + def change do + alter table(:pageviews) do + remove :session_id + end + end +end diff --git a/priv/repo/migrations/20190520144229_change_user_id_to_uuid.exs b/priv/repo/migrations/20190520144229_change_user_id_to_uuid.exs new file mode 100644 index 0000000000000000000000000000000000000000..3e12b54c9ac15c9382922e072d84fc8ec0f1ad44 --- /dev/null +++ b/priv/repo/migrations/20190520144229_change_user_id_to_uuid.exs @@ -0,0 +1,7 @@ +defmodule Plausible.Repo.Migrations.ChangeUserIdToUuid do + use Ecto.Migration + + def change do + execute("ALTER TABLE pageviews ALTER COLUMN user_id TYPE uuid USING user_id::uuid") + end +end diff --git a/priv/repo/migrations/20190523160838_add_raw_referrer.exs b/priv/repo/migrations/20190523160838_add_raw_referrer.exs new file mode 100644 index 0000000000000000000000000000000000000000..46db66257358468722eda791e3b4487b57bf25f6 --- /dev/null +++ b/priv/repo/migrations/20190523160838_add_raw_referrer.exs @@ -0,0 +1,19 @@ +defmodule Plausible.Repo.Migrations.AddRawReferrer do + use Ecto.Migration + + def change do + alter table(:pageviews) do + add :raw_referrer, :text + end + + flush() + + execute "UPDATE pageviews set raw_referrer = referrer" + + flush() + + execute """ + UPDATE pageviews SET referrer = split_part(split_part(regexp_replace(regexp_replace(regexp_replace(raw_referrer, '^https://', ''), '^http://', ''), '^www\.', ''), '?', 1), '#', 1) + """ + end +end diff --git a/priv/repo/migrations/20190523171519_add_indices_to_referrers.exs b/priv/repo/migrations/20190523171519_add_indices_to_referrers.exs new file mode 100644 index 0000000000000000000000000000000000000000..cde0af5d9bbe0b8ac6a2b72666ea897111cd01a4 --- /dev/null +++ b/priv/repo/migrations/20190523171519_add_indices_to_referrers.exs @@ -0,0 +1,8 @@ +defmodule Plausible.Repo.Migrations.AddIndicesToReferrers do + use Ecto.Migration + + def change do + create index(:pageviews, :referrer_source) + create index(:pageviews, :referrer) + end +end diff --git a/priv/repo/migrations/20190618165016_add_public_sites.exs b/priv/repo/migrations/20190618165016_add_public_sites.exs new file mode 100644 index 0000000000000000000000000000000000000000..0950b45ffcffb5e38f40846d2719fba32b2f58da --- /dev/null +++ b/priv/repo/migrations/20190618165016_add_public_sites.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AddPublicSites do + use Ecto.Migration + + def change do + alter table(:sites) do + add :public, :boolean, null: false, default: false + end + end +end diff --git a/priv/repo/migrations/20190718160353_create_google_search_console_integration.exs b/priv/repo/migrations/20190718160353_create_google_search_console_integration.exs new file mode 100644 index 0000000000000000000000000000000000000000..ed869236b92d3284da51164fd38e7614b3e71df4 --- /dev/null +++ b/priv/repo/migrations/20190718160353_create_google_search_console_integration.exs @@ -0,0 +1,17 @@ +defmodule Plausible.Repo.Migrations.CreateGoogleSearchConsoleIntegration do + use Ecto.Migration + + def change do + create table(:google_auth) do + add :user_id, references(:users), null: false + add :email, :string, null: false + add :refresh_token, :string, null: false + add :access_token, :string, null: false + add :expires, :naive_datetime, null: false + + timestamps() + end + + create unique_index(:google_auth, :user_id) + end +end diff --git a/priv/repo/migrations/20190723141824_associate_google_auth_with_site.exs b/priv/repo/migrations/20190723141824_associate_google_auth_with_site.exs new file mode 100644 index 0000000000000000000000000000000000000000..d6ce39af060d59a58e2e7d2eedae05f63ed200b0 --- /dev/null +++ b/priv/repo/migrations/20190723141824_associate_google_auth_with_site.exs @@ -0,0 +1,12 @@ +defmodule Plausible.Repo.Migrations.AssociateGoogleAuthWithSite do + use Ecto.Migration + + def change do + alter table(:google_auth) do + add :site_id, references(:sites), null: false + end + + drop unique_index(:google_auth, :user_id) + create unique_index(:google_auth, :site_id) + end +end diff --git a/priv/repo/migrations/20190730014913_add_monthly_stats.exs b/priv/repo/migrations/20190730014913_add_monthly_stats.exs new file mode 100644 index 0000000000000000000000000000000000000000..ccb53eeaab4c41110b0595796c90d67269bd13bf --- /dev/null +++ b/priv/repo/migrations/20190730014913_add_monthly_stats.exs @@ -0,0 +1,17 @@ +defmodule Plausible.Repo.Migrations.AddMonthlyStats do + use Ecto.Migration + + def change do + create table(:monthly_stats) do + add :month, :date, null: false + add :visitors, :integer, null: false + add :pageviews, :integer, null: false + add :site_id, references(:sites), null: false + + timestamps() + end + + create index(:monthly_stats, :site_id) + create index(:monthly_stats, :month) + end +end diff --git a/priv/repo/migrations/20190730142200_add_weekly_stats.exs b/priv/repo/migrations/20190730142200_add_weekly_stats.exs new file mode 100644 index 0000000000000000000000000000000000000000..05902f11f44f9164e754772a346a014da41c0564 --- /dev/null +++ b/priv/repo/migrations/20190730142200_add_weekly_stats.exs @@ -0,0 +1,17 @@ +defmodule Plausible.Repo.Migrations.AddWeeklyStats do + use Ecto.Migration + + def change do + create table(:weekly_stats) do + add :week, :date, null: false + add :visitors, :integer, null: false + add :pageviews, :integer, null: false + add :site_id, references(:sites), null: false + + timestamps() + end + + create index(:weekly_stats, :site_id) + create index(:weekly_stats, :week) + end +end diff --git a/priv/repo/migrations/20190730144413_add_daily_stats.exs b/priv/repo/migrations/20190730144413_add_daily_stats.exs new file mode 100644 index 0000000000000000000000000000000000000000..3da91cd6cbae9bc9e48e143168b2e93fb7d7e0f7 --- /dev/null +++ b/priv/repo/migrations/20190730144413_add_daily_stats.exs @@ -0,0 +1,17 @@ +defmodule Plausible.Repo.Migrations.AddDailyStats do + use Ecto.Migration + + def change do + create table(:daily_stats) do + add :date, :date, null: false + add :visitors, :integer, null: false + add :pageviews, :integer, null: false + add :site_id, references(:sites), null: false + + timestamps() + end + + create index(:daily_stats, :site_id) + create index(:daily_stats, :date) + end +end diff --git a/priv/repo/migrations/20190809174105_calc_screen_size.exs b/priv/repo/migrations/20190809174105_calc_screen_size.exs new file mode 100644 index 0000000000000000000000000000000000000000..19e7897658f2ca644ed9830197c74787bfcb4568 --- /dev/null +++ b/priv/repo/migrations/20190809174105_calc_screen_size.exs @@ -0,0 +1,17 @@ +defmodule Plausible.Repo.Migrations.CalcScreenSize do + use Ecto.Migration + + def change do + execute """ + UPDATE pageviews SET screen_size= ( + CASE + WHEN screen_width is null THEN null + WHEN screen_width < 576 THEN 'Mobile' + WHEN screen_width < 992 THEN 'Tablet' + WHEN screen_width < 1440 THEN 'Laptop' + ELSE 'Desktop' + END + ); + """ + end +end diff --git a/priv/repo/migrations/20190810145419_remove_unused_indices.exs b/priv/repo/migrations/20190810145419_remove_unused_indices.exs new file mode 100644 index 0000000000000000000000000000000000000000..69ae01f90fa64a09c557ae9ce66e16d1d014f346 --- /dev/null +++ b/priv/repo/migrations/20190810145419_remove_unused_indices.exs @@ -0,0 +1,8 @@ +defmodule Plausible.Repo.Migrations.RemoveUnusedIndices do + use Ecto.Migration + + def change do + drop index(:pageviews, [:referrer]) + drop index(:pageviews, [:referrer_source]) + end +end diff --git a/priv/repo/migrations/20190820140747_remove_rollup_tables.exs b/priv/repo/migrations/20190820140747_remove_rollup_tables.exs new file mode 100644 index 0000000000000000000000000000000000000000..59aa930dfb0537393b1a9e59846692211cde69a6 --- /dev/null +++ b/priv/repo/migrations/20190820140747_remove_rollup_tables.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.RemoveRollupTables do + use Ecto.Migration + + def change do + drop table(:daily_stats) + drop table(:weekly_stats) + drop table(:monthly_stats) + end +end diff --git a/priv/repo/migrations/20190906111810_add_email_reporting.exs b/priv/repo/migrations/20190906111810_add_email_reporting.exs new file mode 100644 index 0000000000000000000000000000000000000000..abfcd3bf58d0171e8e05b889d37b2605e4707f1c --- /dev/null +++ b/priv/repo/migrations/20190906111810_add_email_reporting.exs @@ -0,0 +1,19 @@ +defmodule Plausible.Repo.Migrations.AddEmailReporting do + use Ecto.Migration + + def change do + create table(:email_settings) do + add :site_id, references(:sites, on_delete: :delete_all), null: false + + timestamps() + end + + create table(:sent_email_reports) do + add :site_id, references(:sites, on_delete: :delete_all), null: false + add :year, :integer + add :week, :integer + + add :timestamp, :naive_datetime + end + end +end diff --git a/priv/repo/migrations/20190907134114_add_unique_index_to_email_settings.exs b/priv/repo/migrations/20190907134114_add_unique_index_to_email_settings.exs new file mode 100644 index 0000000000000000000000000000000000000000..f597def79429735713795c746bad9e8b52d2f327 --- /dev/null +++ b/priv/repo/migrations/20190907134114_add_unique_index_to_email_settings.exs @@ -0,0 +1,7 @@ +defmodule Plausible.Repo.Migrations.AddUniqueIndexToEmailSettings do + use Ecto.Migration + + def change do + create unique_index(:email_settings, :site_id) + end +end diff --git a/priv/repo/migrations/20190910120900_add_email_address_to_settings.exs b/priv/repo/migrations/20190910120900_add_email_address_to_settings.exs new file mode 100644 index 0000000000000000000000000000000000000000..3ffea298311f732d2f4ce156bd20362c79851ea7 --- /dev/null +++ b/priv/repo/migrations/20190910120900_add_email_address_to_settings.exs @@ -0,0 +1,16 @@ +defmodule Plausible.Repo.Migrations.AddEmailAddressToSettings do + use Ecto.Migration + use Plausible.Repo + + def change do + alter table(:email_settings) do + add :email, :citext + end + + flush() + + alter table(:email_settings) do + modify :email, :citext, null: false + end + end +end diff --git a/priv/repo/migrations/20190911102027_add_monthly_reports.exs b/priv/repo/migrations/20190911102027_add_monthly_reports.exs new file mode 100644 index 0000000000000000000000000000000000000000..57b95a61a3ac516e162def1ed2c03176509f296a --- /dev/null +++ b/priv/repo/migrations/20190911102027_add_monthly_reports.exs @@ -0,0 +1,78 @@ +defmodule Plausible.Repo.Migrations.AddMonthlyReports do + use Ecto.Migration + use Plausible.Repo + + def up do + drop constraint(:email_settings, "email_settings_site_id_fkey") + drop constraint(:email_settings, "email_settings_pkey") + execute "DROP INDEX email_settings_site_id_index" + + rename table(:email_settings), to: table(:weekly_reports) + + alter table(:weekly_reports) do + modify :id, :bigint, primary_key: true + modify :site_id, references(:sites, on_delete: :delete_all), null: false + end + + execute "ALTER SEQUENCE email_settings_id_seq RENAME TO weekly_reports_id_seq;" + create unique_index(:weekly_reports, :site_id) + + drop constraint(:sent_email_reports, "sent_email_reports_site_id_fkey") + drop constraint(:sent_email_reports, "sent_email_reports_pkey") + + rename table(:sent_email_reports), to: table(:sent_weekly_reports) + + alter table(:sent_weekly_reports) do + modify :id, :bigint, primary_key: true + modify :site_id, references(:sites, on_delete: :delete_all), null: false + end + + execute "ALTER SEQUENCE sent_email_reports_id_seq RENAME TO sent_weekly_reports_id_seq;" + + create table(:monthly_reports) do + add :site_id, references(:sites, on_delete: :delete_all), null: false + add :email, :citext, null: false + + timestamps() + end + + create table(:sent_monthly_reports) do + add :site_id, references(:sites, on_delete: :delete_all), null: false + add :year, :integer, null: false + add :month, :integer, null: false + + add :timestamp, :naive_datetime + end + end + + def down do + drop constraint(:weekly_reports, "weekly_reports_site_id_fkey") + drop constraint(:weekly_reports, "weekly_reports_pkey") + execute "DROP INDEX weekly_reports_site_id_index" + + rename table(:weekly_reports), to: table(:email_settings) + + alter table(:email_settings) do + modify :id, :bigint, primary_key: true + modify :site_id, references(:sites, on_delete: :delete_all), null: false + end + + execute "ALTER SEQUENCE weekly_reports_id_seq RENAME TO email_settings_id_seq;" + create unique_index(:email_settings, :site_id) + + drop constraint(:sent_weekly_reports, "sent_weekly_reports_site_id_fkey") + drop constraint(:sent_weekly_reports, "sent_weekly_reports_pkey") + + rename table(:sent_weekly_reports), to: table(:sent_email_reports) + + alter table(:sent_email_reports) do + modify :id, :bigint, primary_key: true + modify :site_id, references(:sites, on_delete: :delete_all), null: false + end + + execute "ALTER SEQUENCE sent_weekly_reports_id_seq RENAME TO sent_email_reports_id_seq;" + + drop table(:monthly_reports) + drop table(:sent_monthly_reports) + end +end diff --git a/priv/repo/migrations/20191010031425_add_property_to_google_auth.exs b/priv/repo/migrations/20191010031425_add_property_to_google_auth.exs new file mode 100644 index 0000000000000000000000000000000000000000..c9e3ab112676db9da9fb92e4bcc1b68a1cdd5730 --- /dev/null +++ b/priv/repo/migrations/20191010031425_add_property_to_google_auth.exs @@ -0,0 +1,10 @@ +defmodule Plausible.Repo.Migrations.AddPropertyToGoogleAuth do + use Ecto.Migration + use Plausible.Repo + + def change do + alter table(:google_auth) do + add :property, :text + end + end +end diff --git a/priv/repo/migrations/20191015072730_remove_unused_fields.exs b/priv/repo/migrations/20191015072730_remove_unused_fields.exs new file mode 100644 index 0000000000000000000000000000000000000000..a69b3fc69ea7631cf908a6b81e63c7d694cec28e --- /dev/null +++ b/priv/repo/migrations/20191015072730_remove_unused_fields.exs @@ -0,0 +1,11 @@ +defmodule Plausible.Repo.Migrations.RemoveUnusedFields do + use Ecto.Migration + + def change do + alter table(:pageviews) do + remove :raw_referrer + remove :screen_width + remove :user_agent + end + end +end diff --git a/priv/repo/migrations/20191015073507_proper_timestamp_for_pageviews.exs b/priv/repo/migrations/20191015073507_proper_timestamp_for_pageviews.exs new file mode 100644 index 0000000000000000000000000000000000000000..f06a9428e3715a8112ba66462a070ce9d35f3974 --- /dev/null +++ b/priv/repo/migrations/20191015073507_proper_timestamp_for_pageviews.exs @@ -0,0 +1,11 @@ +defmodule Plausible.Repo.Migrations.ProperTimestampForPageviews do + use Ecto.Migration + + def change do + alter table(:pageviews) do + remove :updated_at + end + + rename table(:pageviews), :inserted_at, to: :timestamp + end +end diff --git a/priv/repo/migrations/20191024062200_rename_pageviews_to_events.exs b/priv/repo/migrations/20191024062200_rename_pageviews_to_events.exs new file mode 100644 index 0000000000000000000000000000000000000000..5ed57a6d7452551b5054d86dac27d62dc1a7ea67 --- /dev/null +++ b/priv/repo/migrations/20191024062200_rename_pageviews_to_events.exs @@ -0,0 +1,7 @@ +defmodule Plausible.Repo.Migrations.RenamePageviewsToEvents do + use Ecto.Migration + + def change do + rename table("pageviews"), to: table("events") + end +end diff --git a/priv/repo/migrations/20191025055334_add_name_to_events.exs b/priv/repo/migrations/20191025055334_add_name_to_events.exs new file mode 100644 index 0000000000000000000000000000000000000000..b2ac32ff3d422703cfc5eedbacbdabe6664282d0 --- /dev/null +++ b/priv/repo/migrations/20191025055334_add_name_to_events.exs @@ -0,0 +1,10 @@ +defmodule Plausible.Repo.Migrations.AddNameToEvents do + use Ecto.Migration + use Plausible.Repo + + def change do + alter table(:events) do + add :name, :string, null: false + end + end +end diff --git a/priv/repo/migrations/20191031051340_add_goals.exs b/priv/repo/migrations/20191031051340_add_goals.exs new file mode 100644 index 0000000000000000000000000000000000000000..38942644c3807397c9bc4c6d158b98eafd2c1788 --- /dev/null +++ b/priv/repo/migrations/20191031051340_add_goals.exs @@ -0,0 +1,16 @@ +defmodule Plausible.Repo.Migrations.AddGoals do + use Ecto.Migration + + def change do + create table(:goals) do + add :domain, :text, null: false + add :name, :text, null: false + add :event_name, :text + add :page_path, :text + + timestamps() + end + + create unique_index(:goals, [:domain, :name]) + end +end diff --git a/priv/repo/migrations/20191031063001_remove_goal_name.exs b/priv/repo/migrations/20191031063001_remove_goal_name.exs new file mode 100644 index 0000000000000000000000000000000000000000..b4c57c1f2734832c486e1497753cec42b9b02bc2 --- /dev/null +++ b/priv/repo/migrations/20191031063001_remove_goal_name.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.RemoveGoalName do + use Ecto.Migration + + def change do + alter table(:goals) do + remove :name + end + end +end diff --git a/priv/repo/migrations/20191118075359_allow_free_subscriptions.exs b/priv/repo/migrations/20191118075359_allow_free_subscriptions.exs new file mode 100644 index 0000000000000000000000000000000000000000..3073ba7281f272121efc8081dfd96e5b0026a973 --- /dev/null +++ b/priv/repo/migrations/20191118075359_allow_free_subscriptions.exs @@ -0,0 +1,16 @@ +defmodule Plausible.Repo.Migrations.AllowFreeSubscriptions do + use Ecto.Migration + + def up do + execute """ + ALTER table subscriptions + ALTER paddle_subscription_id DROP NOT NULL, + ALTER update_url DROP NOT NULL, + ALTER cancel_url DROP NOT NULL, + ALTER next_bill_date DROP NOT NULL + """ + end + + def down do + end +end diff --git a/priv/repo/migrations/20191216064647_add_unique_index_to_email_reports.exs b/priv/repo/migrations/20191216064647_add_unique_index_to_email_reports.exs new file mode 100644 index 0000000000000000000000000000000000000000..cbaa056d4b587a63589a658087046ef11a92db55 --- /dev/null +++ b/priv/repo/migrations/20191216064647_add_unique_index_to_email_reports.exs @@ -0,0 +1,8 @@ +defmodule Plausible.Repo.Migrations.AddUniqueIndexToEmailReports do + use Ecto.Migration + use Plausible.Repo + + def change do + create unique_index(:monthly_reports, :site_id) + end +end diff --git a/priv/repo/migrations/20191218082207_add_sessions.exs b/priv/repo/migrations/20191218082207_add_sessions.exs new file mode 100644 index 0000000000000000000000000000000000000000..a8dc68c1175a74e4cb52563b8e9a8dd31081e476 --- /dev/null +++ b/priv/repo/migrations/20191218082207_add_sessions.exs @@ -0,0 +1,23 @@ +defmodule Plausible.Repo.Migrations.AddSessions do + use Ecto.Migration + + def change do + create table(:sessions) do + add :hostname, :text, null: false + add :new_visitor, :boolean, null: false + add :user_id, :binary_id, null: false + + add :is_bounce, :boolean, null: false + add :length, :integer + + add :referrer, :string + add :referrer_source, :string + add :country_code, :string + add :screen_size, :string + add :operating_system, :string + add :browser, :string + + timestamps(inserted_at: :timestamp, updated_at: false) + end + end +end diff --git a/priv/repo/migrations/20191220042658_add_session_start.exs b/priv/repo/migrations/20191220042658_add_session_start.exs new file mode 100644 index 0000000000000000000000000000000000000000..be65ec6a4a98bd518f53f90023bf9e62b207094e --- /dev/null +++ b/priv/repo/migrations/20191220042658_add_session_start.exs @@ -0,0 +1,15 @@ +defmodule Plausible.Repo.Migrations.AddSessionStart do + use Ecto.Migration + + def change do + alter table(:sessions) do + add :start, :naive_datetime + end + + execute "UPDATE sessions set start = timestamp" + + alter table(:sessions) do + modify :start, :naive_datetime, null: false + end + end +end diff --git a/priv/repo/migrations/20200106090739_cascade_google_auth_deletion.exs b/priv/repo/migrations/20200106090739_cascade_google_auth_deletion.exs new file mode 100644 index 0000000000000000000000000000000000000000..20251373e88977ea6953803df36ab7376d7a9ef4 --- /dev/null +++ b/priv/repo/migrations/20200106090739_cascade_google_auth_deletion.exs @@ -0,0 +1,11 @@ +defmodule Plausible.Repo.Migrations.CascadeGoogleAuthDeletion do + use Ecto.Migration + + def change do + drop constraint("google_auth", "google_auth_site_id_fkey") + + alter table(:google_auth) do + modify :site_id, references(:sites, on_delete: :delete_all), null: false + end + end +end diff --git a/priv/repo/migrations/20200107095234_add_entry_page_to_sessions.exs b/priv/repo/migrations/20200107095234_add_entry_page_to_sessions.exs new file mode 100644 index 0000000000000000000000000000000000000000..da0acecc935fd580b968760281e4cfdf90f6942c --- /dev/null +++ b/priv/repo/migrations/20200107095234_add_entry_page_to_sessions.exs @@ -0,0 +1,25 @@ +defmodule Plausible.Repo.Migrations.AddEntryPageToSessions do + use Ecto.Migration + + def change do + alter table(:sessions) do + add :entry_page, :text + end + + execute """ + UPDATE sessions SET entry_page = pathname + FROM events + WHERE events.user_id = sessions.user_id + AND events.name = 'pageview' + AND events.new_visitor + """ + + execute """ + DELETE FROM sessions WHERE entry_page is null + """ + + alter table(:sessions) do + modify :entry_page, :text, null: false + end + end +end diff --git a/priv/repo/migrations/20200113143927_add_exit_page_to_session.exs b/priv/repo/migrations/20200113143927_add_exit_page_to_session.exs new file mode 100644 index 0000000000000000000000000000000000000000..fae3a9c3f926d9d8edd973b5b007347ead67f7ba --- /dev/null +++ b/priv/repo/migrations/20200113143927_add_exit_page_to_session.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AddExitPageToSession do + use Ecto.Migration + + def change do + alter table(:sessions) do + add :exit_page, :text + end + end +end diff --git a/priv/repo/migrations/20200114131538_add_tweets.exs b/priv/repo/migrations/20200114131538_add_tweets.exs new file mode 100644 index 0000000000000000000000000000000000000000..c9cbce9fece4a4d900ee8bb0e9f05cbe50562343 --- /dev/null +++ b/priv/repo/migrations/20200114131538_add_tweets.exs @@ -0,0 +1,20 @@ +defmodule Plausible.Repo.Migrations.AddTweets do + use Ecto.Migration + + def change do + create table(:tweets) do + add :tweet_id, :text, null: false + add :text, :text, null: false + add :author_handle, :text, null: false + add :author_name, :text, null: false + add :author_image, :text, null: false + add :created, :naive_datetime, null: false + add :link, :string, null: false + + timestamps() + end + + create index(:tweets, :link) + create unique_index(:tweets, [:link, :tweet_id]) + end +end diff --git a/priv/repo/migrations/20200120091134_change_session_referrer_to_text.exs b/priv/repo/migrations/20200120091134_change_session_referrer_to_text.exs new file mode 100644 index 0000000000000000000000000000000000000000..d0b2f9a6d287ac019520f11b9e99187b529e5d3e --- /dev/null +++ b/priv/repo/migrations/20200120091134_change_session_referrer_to_text.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.ChangeSessionReferrerToText do + use Ecto.Migration + + def change do + alter table(:sessions) do + modify :referrer, :text + end + end +end diff --git a/priv/repo/migrations/20200121091251_add_recipients.exs b/priv/repo/migrations/20200121091251_add_recipients.exs new file mode 100644 index 0000000000000000000000000000000000000000..d7fa4c02499191a202800048a1fd45843628a121 --- /dev/null +++ b/priv/repo/migrations/20200121091251_add_recipients.exs @@ -0,0 +1,28 @@ +defmodule Plausible.Repo.Migrations.AddRecipients do + use Ecto.Migration + + def up do + alter table(:weekly_reports) do + add :recipients, {:array, :citext}, null: false, default: [] + end + + execute "UPDATE weekly_reports SET recipients = array_append(recipients, email)" + + alter table(:weekly_reports) do + remove :email + end + + alter table(:monthly_reports) do + add :recipients, {:array, :citext}, null: false, default: [] + end + + execute "UPDATE monthly_reports SET recipients = array_append(recipients, email)" + + alter table(:monthly_reports) do + remove :email + end + end + + def down do + end +end diff --git a/priv/repo/migrations/20200122150130_add_shared_links.exs b/priv/repo/migrations/20200122150130_add_shared_links.exs new file mode 100644 index 0000000000000000000000000000000000000000..a63bc6064ead02591571a3a8e16c1001609bea08 --- /dev/null +++ b/priv/repo/migrations/20200122150130_add_shared_links.exs @@ -0,0 +1,13 @@ +defmodule Plausible.Repo.Migrations.AddSharedLinks do + use Ecto.Migration + + def change do + create table(:shared_links) do + add :site_id, references(:sites, on_delete: :delete_all), null: false + add :slug, :string, null: false + add :password_hash, :string + + timestamps() + end + end +end diff --git a/priv/repo/migrations/20200130123049_add_site_id_to_events.exs b/priv/repo/migrations/20200130123049_add_site_id_to_events.exs new file mode 100644 index 0000000000000000000000000000000000000000..f9e199001c46b1feb5b4b2d3c48e9f1648e56597 --- /dev/null +++ b/priv/repo/migrations/20200130123049_add_site_id_to_events.exs @@ -0,0 +1,37 @@ +defmodule Plausible.Repo.Migrations.AddSiteIdToEvents do + use Ecto.Migration + + def up do + alter table(:events) do + add :site_id, :text + end + + alter table(:sessions) do + add :site_id, :text + end + + execute "UPDATE events set site_id=hostname" + execute "UPDATE sessions set site_id=hostname" + + alter table(:events) do + modify :site_id, :text, null: false + end + + alter table(:sessions) do + modify :site_id, :text, null: false + end + + create index(:events, :site_id) + create index(:sessions, :site_id) + end + + def down do + alter table(:events) do + remove :site_id + end + + alter table(:sessions) do + remove :site_id + end + end +end diff --git a/priv/repo/migrations/20200204093801_rename_site_id_to_domain.exs b/priv/repo/migrations/20200204093801_rename_site_id_to_domain.exs new file mode 100644 index 0000000000000000000000000000000000000000..53a313972f34c16aeeafc2dccccd8b2b0d473754 --- /dev/null +++ b/priv/repo/migrations/20200204093801_rename_site_id_to_domain.exs @@ -0,0 +1,8 @@ +defmodule Plausible.Repo.Migrations.RenameSiteIdToDomain do + use Ecto.Migration + + def change do + rename table(:events), :site_id, to: :domain + rename table(:sessions), :site_id, to: :domain + end +end diff --git a/priv/repo/migrations/20200204133522_drop_events_hostname_index.exs b/priv/repo/migrations/20200204133522_drop_events_hostname_index.exs new file mode 100644 index 0000000000000000000000000000000000000000..2f01e62c1fcd9844befc5e6f90c9f8f21c16cea2 --- /dev/null +++ b/priv/repo/migrations/20200204133522_drop_events_hostname_index.exs @@ -0,0 +1,7 @@ +defmodule Plausible.Repo.Migrations.DropEventsHostnameIndex do + use Ecto.Migration + + def change do + drop index("pageviews", :hostname) + end +end diff --git a/priv/repo/migrations/20200210134612_add_fingerprint_to_events.exs b/priv/repo/migrations/20200210134612_add_fingerprint_to_events.exs new file mode 100644 index 0000000000000000000000000000000000000000..aa52987eaed43f4f52745ca9c73c5a0991450292 --- /dev/null +++ b/priv/repo/migrations/20200210134612_add_fingerprint_to_events.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AddFingerprintToEvents do + use Ecto.Migration + + def change do + alter table(:events) do + add :fingerprint, :string, size: 64 + end + end +end diff --git a/priv/repo/migrations/20200211080841_add_raw_fingerprint.exs b/priv/repo/migrations/20200211080841_add_raw_fingerprint.exs new file mode 100644 index 0000000000000000000000000000000000000000..97b8e4d439ebfa0fc25cc2e6e3230db6969a7f90 --- /dev/null +++ b/priv/repo/migrations/20200211080841_add_raw_fingerprint.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AddRawFingerprint do + use Ecto.Migration + + def change do + alter table(:events) do + add :raw_fingerprint, :text + end + end +end diff --git a/priv/repo/migrations/20200211090126_remove_raw_fingerprint.exs b/priv/repo/migrations/20200211090126_remove_raw_fingerprint.exs new file mode 100644 index 0000000000000000000000000000000000000000..f0900902acdb7ba3fb84fcdf4e7201be1533cca2 --- /dev/null +++ b/priv/repo/migrations/20200211090126_remove_raw_fingerprint.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.RemoveRawFingerprint do + use Ecto.Migration + + def change do + alter table(:events) do + remove :raw_fingerprint + end + end +end diff --git a/priv/repo/migrations/20200211133829_add_initial_source_and_referrer_to_events.exs b/priv/repo/migrations/20200211133829_add_initial_source_and_referrer_to_events.exs new file mode 100644 index 0000000000000000000000000000000000000000..65882dbdbe95a9df521bb7032592cb5bfc33650f --- /dev/null +++ b/priv/repo/migrations/20200211133829_add_initial_source_and_referrer_to_events.exs @@ -0,0 +1,13 @@ +defmodule Plausible.Repo.Migrations.AddInitialSourceAndReferrerToEvents do + use Ecto.Migration + + def change do + alter table(:events) do + add :initial_referrer, :text + add :initial_referrer_source, :text + end + + execute "UPDATE events SET initial_referrer=referrer, initial_referrer_source=referrer_source" + execute "UPDATE events SET referrer=null, referrer_source=null WHERE new_visitor=false" + end +end diff --git a/priv/repo/migrations/20200219124314_create_custom_domains.exs b/priv/repo/migrations/20200219124314_create_custom_domains.exs new file mode 100644 index 0000000000000000000000000000000000000000..a4e65a6e0ec04aeb2a3c2e4559ee61ceee66aa96 --- /dev/null +++ b/priv/repo/migrations/20200219124314_create_custom_domains.exs @@ -0,0 +1,15 @@ +defmodule Plausible.Repo.Migrations.CreateCustomDomains do + use Ecto.Migration + + def change do + create table(:custom_domains) do + add :domain, :text, null: false + add :site_id, references(:sites), null: false + add :has_ssl_certificate, :boolean, null: false, default: false + + timestamps() + end + + create unique_index(:custom_domains, :site_id) + end +end diff --git a/priv/repo/migrations/20200227092821_add_fingerprint_sesssions.exs b/priv/repo/migrations/20200227092821_add_fingerprint_sesssions.exs new file mode 100644 index 0000000000000000000000000000000000000000..7bdd02823fcc357ffd01833140f8b4de6146f611 --- /dev/null +++ b/priv/repo/migrations/20200227092821_add_fingerprint_sesssions.exs @@ -0,0 +1,26 @@ +defmodule Plausible.Repo.Migrations.AddFingerprintSesssions do + use Ecto.Migration + + def change do + create table(:fingerprint_sessions) do + add :hostname, :text, null: false + add :domain, :text, null: false + add :fingerprint, :string, size: 64, null: false + + add :is_bounce, :boolean, null: false + add :length, :integer + + add :referrer, :string + add :referrer_source, :string + add :country_code, :string + add :screen_size, :string + add :operating_system, :string + add :browser, :string + add :exit_page, :text + add :entry_page, :text, null: false + add :start, :naive_datetime, null: false + + timestamps(inserted_at: :timestamp, updated_at: false) + end + end +end diff --git a/priv/repo/migrations/20200302105632_flexible_fingerprint_referrer.exs b/priv/repo/migrations/20200302105632_flexible_fingerprint_referrer.exs new file mode 100644 index 0000000000000000000000000000000000000000..1c00b5c7c24bfea939a6156f4ba16cd46ce8fd81 --- /dev/null +++ b/priv/repo/migrations/20200302105632_flexible_fingerprint_referrer.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.FlexibleFingerprintReferrer do + use Ecto.Migration + + def change do + alter table(:fingerprint_sessions) do + modify :referrer, :text + end + end +end diff --git a/priv/repo/migrations/20200317093028_add_trial_expiry_to_users.exs b/priv/repo/migrations/20200317093028_add_trial_expiry_to_users.exs new file mode 100644 index 0000000000000000000000000000000000000000..a8602f72575d7cc99f6f5c0bd3f2baadee55d181 --- /dev/null +++ b/priv/repo/migrations/20200317093028_add_trial_expiry_to_users.exs @@ -0,0 +1,15 @@ +defmodule Plausible.Repo.Migrations.AddTrialExpiryToUsers do + use Ecto.Migration + + def change do + alter table(:users) do + add :trial_expiry_date, :date + end + + execute("UPDATE users SET trial_expiry_date=inserted_at::date + interval '30 days'") + + alter table(:users) do + modify :trial_expiry_date, :date, null: false + end + end +end diff --git a/priv/repo/migrations/20200317142459_backfill_fingerprints.exs b/priv/repo/migrations/20200317142459_backfill_fingerprints.exs new file mode 100644 index 0000000000000000000000000000000000000000..ed22a321ed3aa4047457d87387d542ab26a8acfc --- /dev/null +++ b/priv/repo/migrations/20200317142459_backfill_fingerprints.exs @@ -0,0 +1,14 @@ +defmodule Plausible.Repo.Migrations.BackfillFingerprints do + use Ecto.Migration + + def change do + execute "UPDATE events set fingerprint=user_id where fingerprint is null" + + execute """ + INSERT INTO fingerprint_sessions (hostname, domain, fingerprint, start, length, is_bounce, entry_page, exit_page, referrer, referrer_source, country_code, screen_size, operating_system, browser, timestamp) + SELECT hostname, domain, user_id, start, length, is_bounce, entry_page, exit_page, referrer, referrer_source, country_code, screen_size, operating_system, browser, timestamp + FROM sessions + WHERE sessions.timestamp < '2020-02-27 11:40:55'; + """ + end +end diff --git a/priv/repo/migrations/20200320100803_add_setup_emails.exs b/priv/repo/migrations/20200320100803_add_setup_emails.exs new file mode 100644 index 0000000000000000000000000000000000000000..c9afb0531e5e3a6dd04b8f4857522c7195905aa9 --- /dev/null +++ b/priv/repo/migrations/20200320100803_add_setup_emails.exs @@ -0,0 +1,15 @@ +defmodule Plausible.Repo.Migrations.AddSetupEmails do + use Ecto.Migration + + def change do + create table(:setup_help_emails) do + add :site_id, references(:sites), null: false, on_delete: :delete_all + add :timestamp, :naive_datetime + end + + create table(:setup_success_emails) do + add :site_id, references(:sites), null: false, on_delete: :delete_all + add :timestamp, :naive_datetime + end + end +end diff --git a/priv/repo/migrations/20200323083536_add_create_site_emails.exs b/priv/repo/migrations/20200323083536_add_create_site_emails.exs new file mode 100644 index 0000000000000000000000000000000000000000..f2d2d333dab5d21d3e52ec2021cc42bcace5ce3a --- /dev/null +++ b/priv/repo/migrations/20200323083536_add_create_site_emails.exs @@ -0,0 +1,10 @@ +defmodule Plausible.Repo.Migrations.AddCreateSiteEmails do + use Ecto.Migration + + def change do + create table(:create_site_emails) do + add :user_id, references(:users), null: false, on_delete: :delete_all + add :timestamp, :naive_datetime + end + end +end diff --git a/priv/repo/migrations/20200323084954_add_check_stats_emails.exs b/priv/repo/migrations/20200323084954_add_check_stats_emails.exs new file mode 100644 index 0000000000000000000000000000000000000000..fa03055e747477e7fef0e39aa88459e8b5b7fa24 --- /dev/null +++ b/priv/repo/migrations/20200323084954_add_check_stats_emails.exs @@ -0,0 +1,10 @@ +defmodule Plausible.Repo.Migrations.AddCheckStatsEmails do + use Ecto.Migration + + def change do + create table(:check_stats_emails) do + add :user_id, references(:users), null: false, on_delete: :delete_all + add :timestamp, :naive_datetime + end + end +end diff --git a/priv/repo/migrations/20200324132431_make_cookie_fields_non_required.exs b/priv/repo/migrations/20200324132431_make_cookie_fields_non_required.exs new file mode 100644 index 0000000000000000000000000000000000000000..1ac09de778dd39459986251e3d72dbdd89267ee9 --- /dev/null +++ b/priv/repo/migrations/20200324132431_make_cookie_fields_non_required.exs @@ -0,0 +1,13 @@ +defmodule Plausible.Repo.Migrations.MakeCookieFieldsNonRequired do + use Ecto.Migration + + def up do + alter table(:events) do + modify :new_visitor, :bool, null: true + modify :user_id, :binary_id, null: true + end + end + + def down do + end +end diff --git a/priv/repo/migrations/20200406115153_cascade_custom_domain_deletion.exs b/priv/repo/migrations/20200406115153_cascade_custom_domain_deletion.exs new file mode 100644 index 0000000000000000000000000000000000000000..65c978199f657479b0f7d4abd1f4c65273eb2f3c --- /dev/null +++ b/priv/repo/migrations/20200406115153_cascade_custom_domain_deletion.exs @@ -0,0 +1,11 @@ +defmodule Plausible.Repo.Migrations.CascadeCustomDomainDeletion do + use Ecto.Migration + + def change do + drop constraint("custom_domains", "custom_domains_site_id_fkey") + + alter table(:custom_domains) do + modify :site_id, references(:sites, on_delete: :delete_all), null: false + end + end +end diff --git a/priv/repo/migrations/20200408122329_cascade_setup_emails_deletion.exs b/priv/repo/migrations/20200408122329_cascade_setup_emails_deletion.exs new file mode 100644 index 0000000000000000000000000000000000000000..6a092fd14694adaf55170b11750a50377423a2b4 --- /dev/null +++ b/priv/repo/migrations/20200408122329_cascade_setup_emails_deletion.exs @@ -0,0 +1,16 @@ +defmodule Plausible.Repo.Migrations.CascadeSetupEmailsDeletion do + use Ecto.Migration + + def change do + drop constraint("setup_help_emails", "setup_help_emails_site_id_fkey") + drop constraint("setup_success_emails", "setup_success_emails_site_id_fkey") + + alter table(:setup_help_emails) do + modify :site_id, references(:sites, on_delete: :delete_all), null: false + end + + alter table(:setup_success_emails) do + modify :site_id, references(:sites, on_delete: :delete_all), null: false + end + end +end diff --git a/priv/repo/migrations/20200529071028_add_oban_jobs_table.exs b/priv/repo/migrations/20200529071028_add_oban_jobs_table.exs new file mode 100644 index 0000000000000000000000000000000000000000..e9ae074b218ed060a1b95cccdc926bdeaf8cc720 --- /dev/null +++ b/priv/repo/migrations/20200529071028_add_oban_jobs_table.exs @@ -0,0 +1,11 @@ +defmodule Plausible.Repo.Migrations.AddObanJobsTable do + use Ecto.Migration + + def up do + Oban.Migrations.up() + end + + def down do + Oban.Migrations.down(version: 1) + end +end diff --git a/priv/repo/migrations/20200605134616_remove_events_and_sessions.exs b/priv/repo/migrations/20200605134616_remove_events_and_sessions.exs new file mode 100644 index 0000000000000000000000000000000000000000..ee17a67f14d91591b4d9ed4cf578acea3bd03e2f --- /dev/null +++ b/priv/repo/migrations/20200605134616_remove_events_and_sessions.exs @@ -0,0 +1,8 @@ +defmodule Plausible.Repo.Migrations.RemoveEventsAndSessions do + use Ecto.Migration + + def change do + drop table(:events) + drop table(:sessions) + end +end diff --git a/priv/repo/migrations/20200605142737_remove_fingerprint_sessions_table.exs b/priv/repo/migrations/20200605142737_remove_fingerprint_sessions_table.exs new file mode 100644 index 0000000000000000000000000000000000000000..285df70b2dc9d6986778f7ca5b49396a3641385c --- /dev/null +++ b/priv/repo/migrations/20200605142737_remove_fingerprint_sessions_table.exs @@ -0,0 +1,7 @@ +defmodule Plausible.Repo.Migrations.RemoveFingerprintSessionsTable do + use Ecto.Migration + + def change do + drop table(:fingerprint_sessions) + end +end diff --git a/priv/repo/migrations/20200619071221_create_salts_table.exs b/priv/repo/migrations/20200619071221_create_salts_table.exs new file mode 100644 index 0000000000000000000000000000000000000000..0057bc942ed9ce293513e9b8ce89abb210fe7f91 --- /dev/null +++ b/priv/repo/migrations/20200619071221_create_salts_table.exs @@ -0,0 +1,11 @@ +defmodule Plausible.Repo.Migrations.CreateSaltsTable do + use Ecto.Migration + + def change do + create table(:salts) do + add :salt, :bytea, null: false + + timestamps(updated_at: false) + end + end +end diff --git a/priv/repo/migrations/20201130083829_add_email_verification_codes.exs b/priv/repo/migrations/20201130083829_add_email_verification_codes.exs new file mode 100644 index 0000000000000000000000000000000000000000..405038ea17563a821c30e3dd101d0fb2a58858b9 --- /dev/null +++ b/priv/repo/migrations/20201130083829_add_email_verification_codes.exs @@ -0,0 +1,17 @@ +defmodule Plausible.Repo.Migrations.AddEmailVerificationCodes do + use Ecto.Migration + + def up do + create table(:email_verification_codes, primary_key: false) do + add :code, :integer, null: false + add :user_id, references(:users, on_delete: :delete_all) + add :issued_at, :naive_datetime + end + + execute "INSERT INTO email_verification_codes (code) SELECT code FROM GENERATE_SERIES (1000, 9999) AS s(code) order by random();" + end + + def down do + drop table(:email_verification_codes) + end +end diff --git a/priv/repo/migrations/20201208173543_add_spike_notifications.exs b/priv/repo/migrations/20201208173543_add_spike_notifications.exs new file mode 100644 index 0000000000000000000000000000000000000000..a5386470e468ff689072b4a7db55e166ded87f1e --- /dev/null +++ b/priv/repo/migrations/20201208173543_add_spike_notifications.exs @@ -0,0 +1,14 @@ +defmodule Plausible.Repo.Migrations.AddSpikeNotifications do + use Ecto.Migration + + def change do + create table(:spike_notifications) do + add :site_id, references(:sites), null: false + add :threshold, :integer, null: false + add :last_sent, :naive_datetime + add :recipients, {:array, :citext}, null: false, default: [] + + timestamps() + end + end +end diff --git a/priv/repo/migrations/20201210085345_add_email_verified_to_users.exs b/priv/repo/migrations/20201210085345_add_email_verified_to_users.exs new file mode 100644 index 0000000000000000000000000000000000000000..d170f1952d3a2d824634a8ee3ca7c40f81ab2848 --- /dev/null +++ b/priv/repo/migrations/20201210085345_add_email_verified_to_users.exs @@ -0,0 +1,14 @@ +defmodule Plausible.Repo.Migrations.AddEmailVerifiedToUsers do + use Ecto.Migration + use Plausible.Repo + + def change do + alter table(:users) do + add :email_verified, :boolean, null: false, default: false + end + + flush() + + Repo.update_all("users", set: [email_verified: true]) + end +end diff --git a/priv/repo/migrations/20201214072008_add_theme_pref_to_users.exs b/priv/repo/migrations/20201214072008_add_theme_pref_to_users.exs new file mode 100644 index 0000000000000000000000000000000000000000..3bd588008b3572e465ec8915723287ab11f29b2c --- /dev/null +++ b/priv/repo/migrations/20201214072008_add_theme_pref_to_users.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AddThemePrefToUsers do + use Ecto.Migration + + def change do + alter table(:users) do + add_if_not_exists(:theme, :string, default: "system") + end + end +end diff --git a/priv/repo/migrations/20201230085939_delete_email_records_when_user_is_deleted.exs b/priv/repo/migrations/20201230085939_delete_email_records_when_user_is_deleted.exs new file mode 100644 index 0000000000000000000000000000000000000000..9d081ff6db4a80b5e3e115478fb35160716b2d69 --- /dev/null +++ b/priv/repo/migrations/20201230085939_delete_email_records_when_user_is_deleted.exs @@ -0,0 +1,17 @@ +defmodule Plausible.Repo.Migrations.DeleteEmailRecordsWhenUserIsDeleted do + use Ecto.Migration + + def change do + alter table(:create_site_emails) do + modify :user_id, references(:users, on_delete: :delete_all), + null: false, + from: references(:users) + end + + alter table(:check_stats_emails) do + modify :user_id, references(:users, on_delete: :delete_all), + null: false, + from: references(:users) + end + end +end diff --git a/priv/repo/migrations/20210115092331_cascade_site_deletion_to_spike_notification.exs b/priv/repo/migrations/20210115092331_cascade_site_deletion_to_spike_notification.exs new file mode 100644 index 0000000000000000000000000000000000000000..1abf4bd2d88dd226a76e5ad4912b150a0aa907c7 --- /dev/null +++ b/priv/repo/migrations/20210115092331_cascade_site_deletion_to_spike_notification.exs @@ -0,0 +1,11 @@ +defmodule Plausible.Repo.Migrations.CascadeSiteDeletionToSpikeNotification do + use Ecto.Migration + + def change do + drop constraint("spike_notifications", "spike_notifications_site_id_fkey") + + alter table(:spike_notifications) do + modify :site_id, references(:sites, on_delete: :delete_all), null: false + end + end +end diff --git a/priv/repo/migrations/20210119093337_add_unique_index_to_spike_notification.exs b/priv/repo/migrations/20210119093337_add_unique_index_to_spike_notification.exs new file mode 100644 index 0000000000000000000000000000000000000000..264092eb59c8a2a930849a746e93f562762d497b --- /dev/null +++ b/priv/repo/migrations/20210119093337_add_unique_index_to_spike_notification.exs @@ -0,0 +1,7 @@ +defmodule Plausible.Repo.Migrations.AddUniqueIndexToSpikeNotification do + use Ecto.Migration + + def change do + create unique_index(:spike_notifications, :site_id) + end +end diff --git a/priv/repo/migrations/20210128083453_cascade_site_deletion.exs b/priv/repo/migrations/20210128083453_cascade_site_deletion.exs new file mode 100644 index 0000000000000000000000000000000000000000..b7d2e2eb1357a12bbf439930273b2297802894fe --- /dev/null +++ b/priv/repo/migrations/20210128083453_cascade_site_deletion.exs @@ -0,0 +1,11 @@ +defmodule Plausible.Repo.Migrations.CascadeSiteDeletion do + use Ecto.Migration + + def change do + drop constraint("site_memberships", "site_memberships_site_id_fkey") + + alter table(:site_memberships) do + modify :site_id, references(:sites, on_delete: :delete_all), null: false + end + end +end diff --git a/priv/repo/migrations/20210128084657_create_api_keys.exs b/priv/repo/migrations/20210128084657_create_api_keys.exs new file mode 100644 index 0000000000000000000000000000000000000000..029d54d3332bc19afd2cf337d5b0515d4b7b3778 --- /dev/null +++ b/priv/repo/migrations/20210128084657_create_api_keys.exs @@ -0,0 +1,14 @@ +defmodule Plausible.Repo.Migrations.CreateApiKeys do + use Ecto.Migration + + def change do + create table(:api_keys) do + add :user_id, references(:users, on_delete: :delete_all), null: false + add :name, :string, null: false + add :key_prefix, :string, null: false + add :key_hash, :string, null: false + + timestamps() + end + end +end diff --git a/priv/repo/migrations/20210209095257_add_last_payment_details.exs b/priv/repo/migrations/20210209095257_add_last_payment_details.exs new file mode 100644 index 0000000000000000000000000000000000000000..44e43ab73dd115021b6866ba8426698922f936f6 --- /dev/null +++ b/priv/repo/migrations/20210209095257_add_last_payment_details.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AddLastPaymentDetails do + use Ecto.Migration + + def change do + alter table(:subscriptions) do + add :last_bill_date, :date + end + end +end diff --git a/priv/repo/migrations/20210406073254_add_name_to_shared_links.exs b/priv/repo/migrations/20210406073254_add_name_to_shared_links.exs new file mode 100644 index 0000000000000000000000000000000000000000..108c28f0f4882949ad5edf1beea7bc5d952981f6 --- /dev/null +++ b/priv/repo/migrations/20210406073254_add_name_to_shared_links.exs @@ -0,0 +1,15 @@ +defmodule Plausible.Repo.Migrations.AddNameToSharedLinks do + use Ecto.Migration + + def change do + alter table(:shared_links) do + add :name, :string + end + + execute "UPDATE shared_links SET name=slug" + + alter table(:shared_links) do + modify :name, :string, null: false + end + end +end diff --git a/priv/repo/migrations/20210409074413_add_unique_index_to_shared_link_name.exs b/priv/repo/migrations/20210409074413_add_unique_index_to_shared_link_name.exs new file mode 100644 index 0000000000000000000000000000000000000000..585e79de2f219b37b9ad1ad4145843061a111bc4 --- /dev/null +++ b/priv/repo/migrations/20210409074413_add_unique_index_to_shared_link_name.exs @@ -0,0 +1,7 @@ +defmodule Plausible.Repo.Migrations.AddUniqueIndexToSharedLinkName do + use Ecto.Migration + + def change do + create unique_index(:shared_links, [:site_id, :name], name: :shared_links_site_id_name_index) + end +end diff --git a/priv/repo/migrations/20210409082603_add_api_key_scopes.exs b/priv/repo/migrations/20210409082603_add_api_key_scopes.exs new file mode 100644 index 0000000000000000000000000000000000000000..ab3ea600eba097d35255b9a4854bc27795ccf623 --- /dev/null +++ b/priv/repo/migrations/20210409082603_add_api_key_scopes.exs @@ -0,0 +1,24 @@ +defmodule Plausible.Repo.Migrations.AddApiKeyScopes do + use Ecto.Migration + + def up do + alter table(:api_keys) do + add :scopes, {:array, :text} + end + + execute "UPDATE api_keys SET scopes='{stats:read:*}'" + + alter table(:api_keys) do + modify :scopes, {:array, :text}, null: false + end + + # https://stackoverflow.com/a/4059785 + create index(:api_keys, [:scopes], using: "GIN") + end + + def down do + alter table(:api_keys) do + remove :scopes + end + end +end diff --git a/priv/repo/migrations/20210420075623_add_sent_renewal_notifications.exs b/priv/repo/migrations/20210420075623_add_sent_renewal_notifications.exs new file mode 100644 index 0000000000000000000000000000000000000000..2174a1faacc6a7cff618529ed29fbe9528b923e3 --- /dev/null +++ b/priv/repo/migrations/20210420075623_add_sent_renewal_notifications.exs @@ -0,0 +1,10 @@ +defmodule Plausible.Repo.Migrations.AddSentRenewalNotifications do + use Ecto.Migration + + def change do + create table(:sent_renewal_notifications) do + add :user_id, references(:users), null: false, on_delete: :delete_all + add :timestamp, :naive_datetime + end + end +end diff --git a/priv/repo/migrations/20210426075157_upgrade_oban_jobs_to_v9.exs b/priv/repo/migrations/20210426075157_upgrade_oban_jobs_to_v9.exs new file mode 100644 index 0000000000000000000000000000000000000000..6f2ec10e985d68fafc5f67164f234f1a8768be49 --- /dev/null +++ b/priv/repo/migrations/20210426075157_upgrade_oban_jobs_to_v9.exs @@ -0,0 +1,11 @@ +defmodule Plausible.Repo.Migrations.UpdateObanJobsToV9 do + use Ecto.Migration + + def up do + Oban.Migrations.up(version: 9) + end + + def down do + Oban.Migrations.down(version: 9) + end +end diff --git a/priv/repo/migrations/20210513091653_add_currency_to_subscription.exs b/priv/repo/migrations/20210513091653_add_currency_to_subscription.exs new file mode 100644 index 0000000000000000000000000000000000000000..07393ea1573ad9722886969a6980ede3e6278c13 --- /dev/null +++ b/priv/repo/migrations/20210513091653_add_currency_to_subscription.exs @@ -0,0 +1,15 @@ +defmodule Plausible.Repo.Migrations.AddCurrencyToSubscription do + use Ecto.Migration + + def change do + alter table(:subscriptions) do + add :currency_code, :string + end + + execute "UPDATE subscriptions set currency_code='USD'" + + alter table(:subscriptions) do + modify :currency_code, :string, null: false + end + end +end diff --git a/priv/repo/migrations/20210525085655_add_rate_limit_to_api_keys.exs b/priv/repo/migrations/20210525085655_add_rate_limit_to_api_keys.exs new file mode 100644 index 0000000000000000000000000000000000000000..a0234faf99863ca3ea80d0a209104c28769f9981 --- /dev/null +++ b/priv/repo/migrations/20210525085655_add_rate_limit_to_api_keys.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AddRateLimitToApiKeys do + use Ecto.Migration + + def change do + alter table(:api_keys) do + add :hourly_request_limit, :integer, null: false, default: 1000 + end + end +end diff --git a/priv/repo/migrations/20210531080158_add_role_to_site_memberships.exs b/priv/repo/migrations/20210531080158_add_role_to_site_memberships.exs new file mode 100644 index 0000000000000000000000000000000000000000..a5b8c3b79379d04f2039b3f911d983814b7523b1 --- /dev/null +++ b/priv/repo/migrations/20210531080158_add_role_to_site_memberships.exs @@ -0,0 +1,13 @@ +defmodule Plausible.Repo.Migrations.AddRoleToSiteMemberships do + use Ecto.Migration + + def change do + create_query = "CREATE TYPE site_membership_role AS ENUM ('owner', 'admin', 'viewer')" + drop_query = "DROP TYPE site_membership_role" + execute(create_query, drop_query) + + alter table(:site_memberships) do + add :role, :site_membership_role, null: false, default: "owner" + end + end +end diff --git a/priv/repo/migrations/20210601090924_add_invitations.exs b/priv/repo/migrations/20210601090924_add_invitations.exs new file mode 100644 index 0000000000000000000000000000000000000000..b13cc4b8c97a973ca1a476588a880045967a2f85 --- /dev/null +++ b/priv/repo/migrations/20210601090924_add_invitations.exs @@ -0,0 +1,18 @@ +defmodule Plausible.Repo.Migrations.AddInvitations do + use Ecto.Migration + + def change do + create table(:invitations) do + add :email, :string, null: false + add :site_id, references(:sites), null: false + add :inviter_id, references(:users), null: false + add :role, :site_membership_role, null: false + add :invitation_id, :string + + timestamps() + end + + create unique_index(:invitations, [:site_id, :email]) + create unique_index(:invitations, :invitation_id) + end +end diff --git a/priv/repo/migrations/20210604085943_add_locked_to_sites.exs b/priv/repo/migrations/20210604085943_add_locked_to_sites.exs new file mode 100644 index 0000000000000000000000000000000000000000..7764d9668c25b90fb9d0518bd8f8b6b7dc1c6552 --- /dev/null +++ b/priv/repo/migrations/20210604085943_add_locked_to_sites.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AddLockedToSites do + use Ecto.Migration + + def change do + alter table(:sites) do + add :locked, :boolean, null: false, default: false + end + end +end diff --git a/priv/repo/migrations/20210629124428_cascade_site_deletion_to_invitations.exs b/priv/repo/migrations/20210629124428_cascade_site_deletion_to_invitations.exs new file mode 100644 index 0000000000000000000000000000000000000000..94982953e469d64fb252f7f800059865d00e7d51 --- /dev/null +++ b/priv/repo/migrations/20210629124428_cascade_site_deletion_to_invitations.exs @@ -0,0 +1,11 @@ +defmodule Plausible.Repo.Migrations.CascadeSiteDeletionToInvitations do + use Ecto.Migration + + def change do + drop constraint("invitations", "invitations_site_id_fkey") + + alter table(:invitations) do + modify :site_id, references(:sites, on_delete: :delete_all), null: false + end + end +end diff --git a/priv/repo/migrations/20210726090211_make_invitation_email_case_insensitive.exs b/priv/repo/migrations/20210726090211_make_invitation_email_case_insensitive.exs new file mode 100644 index 0000000000000000000000000000000000000000..024ac42f80fdac3dd5fce3104bc9afb3366cbf6b --- /dev/null +++ b/priv/repo/migrations/20210726090211_make_invitation_email_case_insensitive.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.MakeInvitationEmailCaseInsensitive do + use Ecto.Migration + + def change do + alter table(:invitations) do + modify :email, :citext, null: false + end + end +end diff --git a/priv/repo/migrations/20210906102736_memoize_setup_complete.exs b/priv/repo/migrations/20210906102736_memoize_setup_complete.exs new file mode 100644 index 0000000000000000000000000000000000000000..a1b5ebc0ee601accdd02d3260433e22c97753041 --- /dev/null +++ b/priv/repo/migrations/20210906102736_memoize_setup_complete.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.MemoizeSetupComplete do + use Ecto.Migration + + def change do + alter table(:sites) do + add :has_stats, :boolean, null: false, default: false + end + end +end diff --git a/priv/repo/migrations/20210908081119_allow_trial_expiry_to_be_null.exs b/priv/repo/migrations/20210908081119_allow_trial_expiry_to_be_null.exs new file mode 100644 index 0000000000000000000000000000000000000000..21e935703168f5f81d9d3df18f0728675caad58e --- /dev/null +++ b/priv/repo/migrations/20210908081119_allow_trial_expiry_to_be_null.exs @@ -0,0 +1,9 @@ +defmodule Plausible.Repo.Migrations.AllowTrialExpiryToBeNull do + use Ecto.Migration + + def change do + alter table(:users) do + modify :trial_expiry_date, :date, null: true + end + end +end diff --git a/priv/repo/migrations/20211020093238_add_enterprise_plans.exs b/priv/repo/migrations/20211020093238_add_enterprise_plans.exs new file mode 100644 index 0000000000000000000000000000000000000000..f6b9e23b401113f79f4a8b58502956c432988393 --- /dev/null +++ b/priv/repo/migrations/20211020093238_add_enterprise_plans.exs @@ -0,0 +1,19 @@ +defmodule Plausible.Repo.Migrations.AddEnterprisePlans do + use Ecto.Migration + + def change do + create_query = "CREATE TYPE billing_interval AS ENUM ('monthly', 'yearly')" + drop_query = "DROP TYPE billing_interval" + execute(create_query, drop_query) + + create table(:enterprise_plans) do + add :user_id, references(:users), null: false, unique: true + add :paddle_plan_id, :string, null: false + add :billing_interval, :billing_interval, null: false + add :monthly_pageview_limit, :integer, null: false + add :hourly_api_request_limit, :integer, null: false + + timestamps() + end + end +end