Upload folder using huggingface_hub (part 2)

lib/plausible.ex

@@ -0,0 +1,82 @@
+defmodule Plausible do
+  @moduledoc """
+  Build-related macros
+  """
+
+  @ce_builds [:ce, :ce_test, :ce_dev]
+
+  defmacro __using__(_) do
+    quote do
+      require Plausible
+      import Plausible
+    end
+  end
+
+  defmacro on_ee(clauses) do
+    do_on_ee(clauses)
+  end
+
+  defmacro on_ce(clauses) do
+    do_on_ce(clauses)
+  end
+
+  # :erlang.phash2(1, 1) == 0 tricks dialyzer as per:
+  # https://github.com/elixir-lang/elixir/blob/v1.12.3/lib/elixir/lib/gen_server.ex#L771-L778
+  # and also tricks elixir >1.18 type checker
+
+  defmacro always(term) do
+    quote do
+      case :erlang.phash2(1, 1) do
+        0 ->
+          unquote(term)
+
+        1 ->
+          not unquote(term)
+      end
+    end
+  end
+
+  defmacro ee? do
+    quote do
+      always(unquote(Mix.env() not in @ce_builds))
+    end
+  end
+
+  defmacro ce? do
+    quote do
+      always(unquote(Mix.env() in @ce_builds))
+    end
+  end
+
+  defp do_on_ce(do: block) do
+    do_on_ee(do: nil, else: block)
+  end
+
+  defp do_on_ce(do: do_block, else: else_block) do
+    do_on_ee(do: else_block, else: do_block)
+  end
+
+  defp do_on_ee(do: block) do
+    do_on_ee(do: block, else: nil)
+  end
+
+  defp do_on_ee(do: do_block, else: else_block) do
+    if ee?() do
+      quote do
+        unquote(do_block)
+      end
+    else
+      quote do
+        unquote(else_block)
+      end
+    end
+  end
+
+  def product_name do
+    if ee?() do
+      "Plausible Analytics"
+    else
+      "Plausible CE"
+    end
+  end
+end

lib/plausible/data_migration.ex

@@ -0,0 +1,127 @@
+defmodule Plausible.DataMigration do
+  @moduledoc """
+  Base module for coordinated Clickhouse data migrations
+  run via remote shell or otherwise (TBD).
+  """
+
+  defmacro __using__(opts) do
+    dir = Keyword.fetch!(opts, :dir)
+    repo = Keyword.get(opts, :repo, Plausible.DataMigration.ClickhouseRepo)
+
+    quote bind_quoted: [dir: dir, repo: repo] do
+      @dir dir
+      @repo repo
+
+      def run_sql_confirm(name, assigns \\ [], options \\ []) do
+        query = unwrap_with_io(name, assigns)
+        message = Keyword.get(options, :prompt_message, "Execute?")
+        default_choice = Keyword.get(options, :prompt_default_choice, :yes)
+        confirm(message, fn -> do_run(name, query) end, default_choice)
+      end
+
+      def confirm(message, func, default_choice \\ :yes) do
+        choices =
+          case default_choice do
+            :yes -> " [Y/n]: "
+            :no -> " [y/N]: "
+          end
+
+        prompt = IO.ANSI.white() <> message <> choices <> IO.ANSI.reset()
+        answer = String.downcase(String.trim(IO.gets(prompt)))
+
+        skip = fn ->
+          IO.puts("    #{IO.ANSI.cyan()}Skipped.#{IO.ANSI.reset()}")
+          {:ok, :skip}
+        end
+
+        case answer do
+          "y" ->
+            func.()
+
+          "n" ->
+            skip.()
+
+          _ ->
+            case default_choice do
+              :yes -> func.()
+              :no -> skip.()
+            end
+        end
+      end
+
+      def unwrap(name, assigns \\ []) do
+        :plausible
+        |> :code.priv_dir()
+        |> Path.join("data_migrations")
+        |> Path.join(@dir)
+        |> Path.join("sql")
+        |> Path.join(name <> ".sql.eex")
+        |> EEx.eval_file(assigns: assigns)
+      end
+
+      @doc """
+      Runs a single SQL query in a file.
+
+      Valid options:
+      - `quiet` - reduces output from running the SQL
+      - `params` - List of query parameters.
+      - `query_options` - passed to Repo.query
+      """
+      def run_sql(name, assigns \\ [], options \\ []) do
+        query = unwrap(name, assigns)
+
+        do_run(name, query, options)
+      end
+
+      @doc """
+      Runs multiple SQL queries from a single file.
+
+      Note that each query must be separated by semicolons.
+      """
+      def run_sql_multi(name, assigns \\ [], options \\ []) do
+        unwrap(name, assigns)
+        |> String.trim()
+        |> String.split(";", trim: true)
+        |> Enum.with_index(1)
+        |> Enum.reduce_while(:ok, fn {query, index}, _ ->
+          case do_run("#{name}-#{index}", query, options) do
+            {:ok, _} -> {:cont, :ok}
+            error -> {:halt, error}
+          end
+        end)
+      end
+
+      def do_run(name, query, options \\ []) do
+        params = Keyword.get(options, :params, [])
+        query_options = Keyword.get(options, :query_options, [])
+
+        case @repo.query(query, params, [timeout: :infinity] ++ query_options) do
+          {:ok, res} ->
+            if not Keyword.get(options, :quiet, false) do
+              IO.puts(
+                "    #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n"
+              )
+
+              IO.puts(String.duplicate("-", 78))
+            end
+
+            {:ok, res}
+
+          result ->
+            result
+        end
+      end
+
+      defp unwrap_with_io(name, assigns) do
+        IO.puts("#{IO.ANSI.yellow()}Running #{name}#{IO.ANSI.reset()}")
+        query = unwrap(name, assigns)
+
+        IO.puts("""
+          -> Query: #{IO.ANSI.blue()}#{String.trim(query)}#{IO.ANSI.reset()}
+        """)
+
+        query
+      end
+    end
+  end
+end

lib/plausible/data_migration/postgres_repo.ex

@@ -0,0 +1,22 @@
+defmodule Plausible.DataMigration.PostgresRepo do
+  @moduledoc """
+  Ecto.Repo for Posrtgres data migrations, to be started manually,
+  outside of the main application supervision tree.
+  """
+  use Ecto.Repo,
+    otp_app: :plausible,
+    adapter: Ecto.Adapters.Postgres
+
+  def start(url, opts \\ []) when is_binary(url) do
+    default_config = Plausible.Repo.config()
+
+    start_link(
+      url: url,
+      queue_target: 500,
+      queue_interval: 2000,
+      pool_size: opts[:pool_size] || 1,
+      ssl: opts[:ssl] || default_config[:ssl],
+      ssl_opts: opts[:ssl_opts] || default_config[:ssl_opts]
+    )
+  end
+end

lib/plausible/data_migration/prefix_tracker_script_configuration_id.ex

@@ -0,0 +1,96 @@
+defmodule Plausible.DataMigration.PrefixTrackerScriptConfigurationId do
+  @moduledoc """
+  Migration to update tracker script configuration IDs to use the new prefixed format.
+
+  This migration:
+  1. Processes tracker configurations in batches to avoid memory issues
+  2. Uses batch updates for better performance
+  3. Handles the case where tracker configurations might not exist for all sites
+  4. Provides progress logging
+  """
+
+  import Ecto.Query
+  alias Plausible.Repo
+  alias Plausible.Site.TrackerScriptConfiguration
+
+  def run(batch_size \\ 100) do
+    total_configs = count_total_configurations()
+    IO.puts("Found #{total_configs} total tracker configurations to process")
+
+    process_batch(nil, total_configs, batch_size, 0)
+  end
+
+  defp count_total_configurations do
+    Repo.aggregate(
+      from(config in TrackerScriptConfiguration,
+        where: not like(config.id, "pa-%")
+      ),
+      :count,
+      :id
+    )
+  end
+
+  defp process_batch(last_id, total_configs, batch_size, processed_count) do
+    configs = get_configurations_batch(last_id, batch_size)
+
+    if length(configs) > 0 do
+      batch_num = div(processed_count, batch_size) + 1
+      start_pos = processed_count + 1
+      end_pos = processed_count + length(configs)
+
+      IO.puts("Processing batch #{batch_num} (#{start_pos}-#{end_pos} of #{total_configs})")
+
+      process_configurations_batch(configs, batch_num)
+
+      process_batch(
+        List.last(configs).id,
+        total_configs,
+        batch_size,
+        processed_count + length(configs)
+      )
+    else
+      IO.puts("Migration completed!")
+    end
+  end
+
+  defp get_configurations_batch(last_id, batch_size) do
+    query =
+      from(config in TrackerScriptConfiguration,
+        where: not like(config.id, "pa-%"),
+        order_by: [asc: config.id],
+        limit: ^batch_size,
+        select: %{
+          id: config.id
+        }
+      )
+
+    query =
+      if last_id do
+        from(config in query, where: config.id > ^last_id)
+      else
+        query
+      end
+
+    Repo.all(query)
+  end
+
+  defp process_configurations_batch(configs, batch_num) do
+    config_ids = Enum.map(configs, & &1.id)
+
+    case Repo.query(
+           "UPDATE tracker_script_configuration SET id = 'pa-' || id WHERE id = ANY($1)",
+           [config_ids]
+         ) do
+      {:ok, %{num_rows: updated_count}} ->
+        if updated_count != length(configs) do
+          IO.puts(
+            "Warning: Batch #{batch_num} - Expected to update #{length(configs)} configurations, but updated #{updated_count}"
+          )
+        end
+
+      {:error, error} ->
+        IO.puts("Error updating batch #{batch_num}: #{inspect(error)}")
+        # Continue with next batch instead of crashing
+    end
+  end
+end

lib/plausible/data_migration/site_imports.ex

@@ -0,0 +1,270 @@
+defmodule Plausible.DataMigration.SiteImports do
+  @moduledoc """
+  !!!WARNING!!!: This script is used in migrations. Please take special care
+  when altering it.
+
+  Site imports migration backfilling SiteImport entries for old imports
+  and alters import end dates to match actual end date of respective import stats.
+  """
+
+  import Ecto.Query
+
+  alias Plausible.{Repo, ClickhouseRepo, Site}
+
+  defmodule SiteImportSnapshot do
+    @moduledoc """
+    A snapshot of the Plausible.Imported.SiteImport schema from April 2024.
+    """
+
+    use Ecto.Schema
+
+    schema "site_imports" do
+      field :start_date, :date
+      field :end_date, :date
+      field :label, :string
+      field :source, Ecto.Enum, values: [:universal_analytics, :google_analytics_4, :csv, :noop]
+      field :status, Ecto.Enum, values: [:pending, :importing, :completed, :failed]
+      field :legacy, :boolean, default: false
+
+      belongs_to :site, Plausible.Site
+      belongs_to :imported_by, Plausible.Auth.User
+
+      timestamps()
+    end
+  end
+
+  @imported_tables_april_2024 [
+    "imported_visitors",
+    "imported_sources",
+    "imported_pages",
+    "imported_entry_pages",
+    "imported_exit_pages",
+    "imported_custom_events",
+    "imported_locations",
+    "imported_devices",
+    "imported_browsers",
+    "imported_operating_systems"
+  ]
+
+  def imported_tables_april_2024(), do: @imported_tables_april_2024
+
+  def run(opts \\ []) do
+    dry_run? = Keyword.get(opts, :dry_run?, true)
+
+    site_import_query =
+      from(i in SiteImportSnapshot,
+        where: i.site_id == parent_as(:site).id and i.status == ^:completed,
+        select: 1
+      )
+
+    sites_with_only_legacy_import =
+      from(s in Site,
+        as: :site,
+        select: %{id: s.id, imported_data: s.imported_data},
+        where:
+          not is_nil(s.imported_data) and fragment("?->>'status'", s.imported_data) == "ok" and
+            not exists(site_import_query)
+      )
+      |> Repo.all(log: false)
+
+    site_imports =
+      from(i in SiteImportSnapshot, where: i.status == ^:completed)
+      |> Repo.all(log: false)
+
+    legacy_site_imports = backfill_legacy_site_imports(sites_with_only_legacy_import, dry_run?)
+
+    all_site_imports = Repo.preload(site_imports ++ legacy_site_imports, :site)
+
+    adjust_site_import_end_dates(all_site_imports, dry_run?)
+
+    IO.puts("Finished")
+  end
+
+  defp backfill_legacy_site_imports(sites, dry_run?) do
+    total = length(sites)
+
+    IO.puts("Backfilling legacy site import across #{total} sites (DRY RUN: #{dry_run?})...")
+
+    legacy_site_imports =
+      for {site, idx} <- Enum.with_index(sites) do
+        IO.puts("Creating legacy site import entry for site ID #{site.id} (#{idx + 1}/#{total})")
+
+        params =
+          site.imported_data
+          |> from_legacy()
+          |> Map.put(:site_id, site.id)
+          |> Map.take([:legacy, :start_date, :end_date, :source, :status, :site_id])
+
+        %SiteImportSnapshot{}
+        |> Ecto.Changeset.change(params)
+        |> insert!(dry_run?)
+      end
+
+    IO.puts("Finished backfilling sites.")
+
+    legacy_site_imports
+  end
+
+  defp adjust_site_import_end_dates(site_imports, dry_run?) do
+    total = length(site_imports)
+
+    IO.puts("Adjusting end dates of #{total} site imports (DRY RUN: #{dry_run?})...")
+
+    for {site_import, idx} <- Enum.with_index(site_imports) do
+      IO.puts(
+        "Adjusting end date for site import #{site_import.id} (#{idx + 1}/#{total}) (site ID #{site_import.site_id}, start date: #{site_import.start_date}, end date: #{site_import.end_date})"
+      )
+
+      import_ids =
+        if site_import.legacy do
+          [0, site_import.id]
+        else
+          [site_import.id]
+        end
+
+      end_date = imported_stats_end_date(site_import.site_id, import_ids)
+
+      if !end_date do
+        IO.puts(
+          "Site import #{site_import.id} (site ID #{site_import.site_id}) does not have any recorded stats. Removing it."
+        )
+
+        if site_import.legacy do
+          # sanity check that data is correct
+          "ok" = site_import.site.imported_data.status
+
+          clear_imported_data(site_import.site, dry_run?)
+        end
+
+        delete!(site_import, dry_run?)
+      else
+        case Date.compare(end_date, site_import.end_date) do
+          :lt ->
+            IO.puts(
+              "End date of site import #{site_import.id} (site ID #{site_import.site_id}) is adjusted from #{site_import.end_date} to #{end_date}."
+            )
+
+            site_import
+            |> Ecto.Changeset.change(end_date: end_date)
+            |> update!(dry_run?)
+
+            # credo:disable-for-next-line Credo.Check.Refactor.Nesting
+            if site_import.legacy do
+              # sanity check that data is correct
+              "ok" = site_import.site.imported_data.status
+
+              site_import.site
+              |> Ecto.Changeset.change(imported_data: %{end_date: end_date})
+              |> update!(dry_run?)
+            end
+
+          :eq ->
+            IO.puts(
+              "End date of site import #{site_import.id} (site ID #{site_import.site_id}) is left unadjusted."
+            )
+
+          :gt ->
+            IO.puts(
+              "Site import #{site_import.id} (site ID #{site_import.site_id}) computed end date is later than the current one. Skipping."
+            )
+        end
+      end
+    end
+
+    IO.puts("Finished adjusting end dates of site imports.")
+  end
+
+  # Exposed for testing purposes
+  @doc false
+  def imported_stats_end_date(site_id, import_ids) do
+    [first_table | tables] = @imported_tables_april_2024
+
+    query =
+      Enum.reduce(tables, max_date_query(first_table, site_id, import_ids), fn table, query ->
+        from(s in subquery(union_all(query, ^max_date_query(table, site_id, import_ids))))
+      end)
+
+    dates = ClickhouseRepo.all(from(q in query, select: q.max_date), log: false)
+
+    if dates != [] do
+      case Enum.max(dates, Date) do
+        # no stats for this domain yet
+        ~D[1970-01-01] ->
+          nil
+
+        date ->
+          date
+      end
+    else
+      nil
+    end
+  end
+
+  defp insert!(changeset, false = _dry_run?) do
+    Repo.insert!(changeset)
+  end
+
+  defp insert!(changeset, true = _dry_run?) do
+    if changeset.valid? do
+      changeset
+      |> Ecto.Changeset.change(id: 0)
+      |> Ecto.Changeset.apply_changes()
+    else
+      raise "Invalid insert: #{inspect(changeset)}"
+    end
+  end
+
+  defp clear_imported_data(site, false = _dry_run?) do
+    Repo.update_all(from(s in Site, where: s.id == ^site.id), set: [imported_data: nil])
+  end
+
+  defp clear_imported_data(site, true = _dry_run?) do
+    %{site | imported_data: nil}
+  end
+
+  defp update!(changeset, false = _dry_run?) do
+    Repo.update!(changeset)
+  end
+
+  defp update!(changeset, true = _dry_run?) do
+    if changeset.valid? do
+      Ecto.Changeset.apply_changes(changeset)
+    else
+      raise "Invalid update: #{inspect(changeset)}"
+    end
+  end
+
+  defp delete!(entity, false = _dry_run?) do
+    Repo.delete!(entity)
+  end
+
+  defp delete!(entity, true = _dry_run?) do
+    entity
+  end
+
+  defp max_date_query(table, site_id, import_ids) do
+    from(q in table,
+      where: q.site_id == ^site_id,
+      where: q.import_id in ^import_ids,
+      select: %{max_date: fragment("max(?)", q.date)}
+    )
+  end
+
+  defp from_legacy(%Site.ImportedData{} = data) do
+    status =
+      case data.status do
+        "ok" -> :completed
+        "error" -> :failed
+        _ -> :importing
+      end
+
+    %SiteImportSnapshot{
+      id: 0,
+      legacy: true,
+      start_date: data.start_date,
+      end_date: data.end_date,
+      source: :universal_analytics,
+      status: status
+    }
+  end
+end

lib/plausible/data_migration/versioned_sessions.ex

@@ -0,0 +1,80 @@
+defmodule Plausible.DataMigration.VersionedSessions do
+  @moduledoc """
+  !!!WARNING!!!: This script is used in migrations. Please take special care
+  when altering it.
+
+  Sessions CollapsingMergeTree -> VersionedCollapsingMergeTree migration,
+  SQL files available at:
+
+  priv/data_migrations/VersionedSessions/sql
+  """
+  use Plausible.DataMigration, dir: "VersionedSessions", repo: Plausible.IngestRepo
+
+  @suffix_format "%Y%m%d%H%M%S"
+  @versioned_table_engines [
+    "ReplicatedVersionedCollapsingMergeTree",
+    "VersionedCollapsingMergeTree"
+  ]
+
+  def run(opts \\ []) do
+    run_exchange? = Keyword.get(opts, :run_exchange?, true)
+
+    unique_suffix = DateTime.utc_now() |> Calendar.strftime(@suffix_format)
+
+    cluster? = Plausible.IngestRepo.clustered_table?("sessions_v2")
+
+    {:ok, %{rows: partitions}} = run_sql("list-partitions")
+    partitions = Enum.map(partitions, fn [part] -> part end)
+
+    {:ok, %{rows: [[current_table_engine, table_settings]]}} =
+      run_sql("get-sessions-table-settings")
+
+    if Enum.member?(@versioned_table_engines, current_table_engine) do
+      IO.puts("sessions_v2 table is already versioned, no migration needed")
+    else
+      {:ok, _} = run_sql("drop-sessions-tmp-table", cluster?: cluster?)
+
+      {:ok, _} =
+        run_sql("create-sessions-tmp-table",
+          cluster?: cluster?,
+          table_settings: table_settings,
+          unique_suffix: unique_suffix
+        )
+
+      for partition <- partitions do
+        {:ok, _} = run_sql("attach-partition", partition: partition)
+      end
+
+      if run_exchange? do
+        run_exchange(cluster?)
+      end
+
+      IO.puts("Migration done!")
+    end
+  end
+
+  defp run_exchange(cluster?) do
+    case run_sql("exchange-sessions-tables", cluster?: cluster?) do
+      {:ok, _} ->
+        nil
+
+      # Docker containers don't seem to support EXCHANGE TABLE, hack around this with a non-atomic swap
+      {:error, %Ch.Error{code: code}} when code in [1, 48] ->
+        IO.puts("Exchanging sessions_v2 and sessions_v2_tmp_versioned non-atomically")
+
+        {:ok, _} =
+          run_sql("rename-table",
+            from: "sessions_v2",
+            to: "sessions_v2_backup",
+            cluster?: cluster?
+          )
+
+        {:ok, _} =
+          run_sql("rename-table",
+            from: "sessions_v2_tmp_versioned",
+            to: "sessions_v2",
+            cluster?: cluster?
+          )
+    end
+  end
+end

lib/plausible/dns_lookup.ex

@@ -0,0 +1,31 @@
+defmodule Plausible.DnsLookupInterface do
+  @moduledoc """
+  Behaviour module for DNS lookup operations.
+  """
+
+  @callback lookup(
+              name :: charlist(),
+              class :: atom(),
+              type :: atom(),
+              opts :: list(),
+              timeout :: integer()
+            ) ::
+              list() | []
+end
+
+defmodule Plausible.DnsLookup do
+  @moduledoc """
+  Thin wrapper around `:inet_res.lookup/5`.
+  To use, call `Plausible.DnsLookup.impl().lookup/5`,
+  this allows for mocking DNS lookups in tests.
+  """
+
+  @behaviour Plausible.DnsLookupInterface
+
+  @impl Plausible.DnsLookupInterface
+  def lookup(name, class, type, opts, timeout),
+    do: :inet_res.lookup(name, class, type, opts, timeout)
+
+  @spec impl() :: Plausible.DnsLookup
+  def impl(), do: Application.get_env(:plausible, :dns_lookup_impl, __MODULE__)
+end

lib/plausible/ecto/types/compiled_regex.ex

@@ -0,0 +1,14 @@
+defmodule Plausible.Ecto.Types.CompiledRegex do
+  @moduledoc """
+  Ensures that the regex is compiled on load
+  """
+  use Ecto.Type
+
+  def type, do: :string
+
+  def cast(val) when is_binary(val), do: {:ok, val}
+  def cast(_), do: :error
+
+  def load(val), do: {:ok, Regex.compile!(val, "E")}
+  def dump(val), do: {:ok, val}
+end

lib/plausible/ecto/types/nanoid.ex

@@ -0,0 +1,51 @@
+defmodule Plausible.Ecto.Types.NanoidBase do
+  @moduledoc """
+  Base module for nanoid string types
+  """
+
+  use Ecto.Type
+
+  def type(), do: :string
+
+  def cast(value) when is_binary(value), do: {:ok, value}
+  def cast(_), do: :error
+
+  def load(value), do: {:ok, value}
+
+  def dump(value) when is_binary(value), do: {:ok, value}
+  def dump(_), do: :error
+
+  @callback autogenerate() :: String.t()
+end
+
+defmodule Plausible.Ecto.Types.Nanoid do
+  @moduledoc """
+  Custom column type for nanoid strings
+  """
+
+  @behaviour Plausible.Ecto.Types.NanoidBase
+
+  def type(), do: Plausible.Ecto.Types.NanoidBase.type()
+  def cast(value), do: Plausible.Ecto.Types.NanoidBase.cast(value)
+  def load(value), do: Plausible.Ecto.Types.NanoidBase.load(value)
+  def dump(value), do: Plausible.Ecto.Types.NanoidBase.dump(value)
+
+  @spec autogenerate() :: String.t()
+  def autogenerate(), do: Nanoid.generate()
+end
+
+defmodule Plausible.Ecto.Types.TrackerScriptNanoid do
+  @moduledoc """
+  Custom column type for tracker script configuration nanoid strings with pa- prefix
+  """
+
+  @behaviour Plausible.Ecto.Types.NanoidBase
+
+  def type(), do: Plausible.Ecto.Types.NanoidBase.type()
+  def cast(value), do: Plausible.Ecto.Types.NanoidBase.cast(value)
+  def load(value), do: Plausible.Ecto.Types.NanoidBase.load(value)
+  def dump(value), do: Plausible.Ecto.Types.NanoidBase.dump(value)
+
+  @spec autogenerate() :: String.t()
+  def autogenerate(), do: "pa-#{Nanoid.generate()}"
+end

lib/plausible/event/system_events.ex

@@ -0,0 +1,101 @@
+defmodule Plausible.Event.SystemEvents do
+  @moduledoc """
+  System events are events that require at least one form of special treatment by the application.
+  They are distinguished by event name.
+
+  For some system events, the tracking API may reject events that don't match the expected format
+  (e.g. engagement events without scroll depth or engagement time defined).
+
+  For other system events, it may accept the event (e.g. "Outbound Link: Click" with the url prop not set).
+
+  System events may have corresponding system-managed goals, in which case the goal name and event name will be the same
+  (e.g. "Outbound Link: Click" goal is managed by the application). The system will only be able to manage these goals properly
+  if they are not renamed by the user.
+  """
+  @pageview_event_name "pageview"
+  @engagement_event_name "engagement"
+
+  @outbound_link_click_event_name "Outbound Link: Click"
+  @cloaked_link_click_event_name "Cloaked Link: Click"
+  @file_download_link_click_event_name "File Download"
+
+  @error_404_event_name "404"
+  @wordpress_form_completions_event_name "WP Form Completions"
+  @form_submission_event_name "Form: Submission"
+
+  @all_system_events [
+    @pageview_event_name,
+    @engagement_event_name,
+    @outbound_link_click_event_name,
+    @cloaked_link_click_event_name,
+    @file_download_link_click_event_name,
+    @error_404_event_name,
+    @wordpress_form_completions_event_name,
+    @form_submission_event_name
+  ]
+
+  @interactive_events @all_system_events
+
+  @events_with_url_prop [
+    @outbound_link_click_event_name,
+    @cloaked_link_click_event_name,
+    @file_download_link_click_event_name
+  ]
+
+  @events_with_path_prop [
+    @error_404_event_name,
+    @wordpress_form_completions_event_name,
+    @form_submission_event_name
+  ]
+
+  @events_with_engagement_props [
+    @engagement_event_name
+  ]
+
+  def events() do
+    @all_system_events
+  end
+
+  def events_with_interactive_always_true() do
+    @interactive_events
+  end
+
+  def events_with_url_prop() do
+    @events_with_url_prop
+  end
+
+  def events_with_path_prop() do
+    @events_with_path_prop
+  end
+
+  def events_with_engagement_props() do
+    @events_with_engagement_props
+  end
+
+  @spec special_events_for_prop_key(String.t()) :: [String.t()]
+  def special_events_for_prop_key("url"), do: events_with_url_prop()
+  def special_events_for_prop_key("path"), do: events_with_path_prop()
+
+  @doc """
+  Checks if the event name is for a system event / system goal that should have the event.props.path synced with the event.pathname property.
+
+  ### Examples
+  iex> sync_props_path_with_pathname?("404", [{"path", "/foo"}])
+  false
+
+  Note: Should not override event.props.path if it is set deliberately to nil
+  iex> sync_props_path_with_pathname?("404", [{"path", nil}])
+  false
+
+  iex> sync_props_path_with_pathname?("404", [{"other", "value"}])
+  true
+
+  iex> sync_props_path_with_pathname?("404", [])
+  true
+  """
+  @spec sync_props_path_with_pathname?(String.t(), [{String.t(), String.t()}]) :: boolean()
+  def sync_props_path_with_pathname?(event_name, props_in_request) do
+    event_name in events_with_path_prop() and
+      not Enum.any?(props_in_request, fn {k, _} -> k == "path" end)
+  end
+end

lib/plausible/event/write_buffer.ex

@@ -0,0 +1,38 @@
+defmodule Plausible.Event.WriteBuffer do
+  @moduledoc false
+
+  %{
+    header: header,
+    insert_sql: insert_sql,
+    insert_opts: insert_opts,
+    fields: fields,
+    encoding_types: encoding_types
+  } =
+    Plausible.Ingestion.WriteBuffer.compile_time_prepare(Plausible.ClickhouseEventV2)
+
+  def child_spec(opts) do
+    opts =
+      Keyword.merge(opts,
+        name: __MODULE__,
+        header: unquote(header),
+        insert_sql: unquote(insert_sql),
+        insert_opts: unquote(insert_opts)
+      )
+
+    Plausible.Ingestion.WriteBuffer.child_spec(opts)
+  end
+
+  def insert(event) do
+    row_binary =
+      [Enum.map(unquote(fields), fn field -> Map.fetch!(event, field) end)]
+      |> Ch.RowBinary._encode_rows(unquote(encoding_types))
+      |> IO.iodata_to_binary()
+
+    :ok = Plausible.Ingestion.WriteBuffer.insert(__MODULE__, row_binary)
+    {:ok, event}
+  end
+
+  def flush do
+    Plausible.Ingestion.WriteBuffer.flush(__MODULE__)
+  end
+end

lib/plausible/exports.ex

@@ -0,0 +1,715 @@
+defmodule Plausible.Exports do
+  @moduledoc """
+  Contains functions to export data for events and sessions as Zip archives.
+  """
+
+  use Plausible
+  use Plausible.Stats.SQL.Fragments
+  import Ecto.Query
+
+  @doc "Schedules CSV export job to S3 storage"
+  @spec schedule_s3_export(pos_integer, String.t()) ::
+          {:ok, Oban.Job.t()} | {:error, :no_data}
+  def schedule_s3_export(site_id, email_to) do
+    with :ok <- ensure_has_data(site_id) do
+      args = %{
+        "storage" => "s3",
+        "site_id" => site_id,
+        "email_to" => email_to,
+        "s3_bucket" => Plausible.S3.exports_bucket(),
+        "s3_path" => s3_export_key(site_id)
+      }
+
+      {:ok, Oban.insert!(Plausible.Workers.ExportAnalytics.new(args))}
+    end
+  end
+
+  @doc "Schedules CSV export job to local storage"
+  @spec schedule_local_export(pos_integer, String.t()) :: {:ok, Oban.Job.t()} | {:error, :no_data}
+  def schedule_local_export(site_id, email_to) do
+    with :ok <- ensure_has_data(site_id) do
+      args = %{
+        "storage" => "local",
+        "site_id" => site_id,
+        "email_to" => email_to,
+        "local_path" => local_export_file(site_id)
+      }
+
+      {:ok, Oban.insert!(Plausible.Workers.ExportAnalytics.new(args))}
+    end
+  end
+
+  @spec ensure_has_data(pos_integer) :: :ok | {:error, :no_data}
+  defp ensure_has_data(site_id) do
+    # SELECT true FROM "events_v2" AS e0 WHERE (e0."site_id" = ^site_id) LIMIT 1
+    has_data? = Plausible.ClickhouseRepo.exists?(from "events_v2", where: [site_id: ^site_id])
+    if has_data?, do: :ok, else: {:error, :no_data}
+  end
+
+  @doc "Gets last CSV export job for a site"
+  @spec get_last_export_job(pos_integer) :: Oban.Job.t() | nil
+  def get_last_export_job(site_id) do
+    Plausible.Repo.one(
+      from e in Plausible.Workers.ExportAnalytics.base_query(site_id),
+        order_by: [desc: :id],
+        limit: 1
+    )
+  end
+
+  @doc "Subscribes to CSV export job notifications"
+  def oban_listen, do: Oban.Notifier.listen(__MODULE__)
+  @doc false
+  def oban_notify(site_id), do: Oban.Notifier.notify(__MODULE__, %{"site_id" => site_id})
+
+  @doc """
+  Renders export archive filename.
+
+  Examples:
+
+      iex> archive_filename("plausible.io", _created_on = ~D[2024-12-31])
+      "plausible_io_20241231.zip"
+
+  """
+  def archive_filename(domain, %Date{} = created_on) do
+    String.replace(domain, ".", "_") <> "_" <> Calendar.strftime(created_on, "%Y%m%d") <> ".zip"
+  end
+
+  @doc ~S"""
+  Safely renders content disposition for an arbitrary export filename.
+
+  Examples:
+
+      iex> content_disposition("plausible_io_20241231.zip")
+      "attachment; filename=\"plausible_io_20241231.zip\""
+
+      iex> content_disposition("📊.zip")
+      "attachment; filename=\"plausible-export.zip\"; filename*=utf-8''%F0%9F%93%8A.zip"
+
+  """
+  def content_disposition(filename) do
+    encoded_filename = URI.encode(filename)
+
+    if encoded_filename == filename do
+      ~s[attachment; filename="#{filename}"]
+    else
+      ~s[attachment; filename="plausible-export.zip"; filename*=utf-8''#{encoded_filename}]
+    end
+  end
+
+  @type export :: %{
+          path: Path.t(),
+          name: String.t(),
+          expires_at: DateTime.t() | nil,
+          size: pos_integer
+        }
+
+  @doc "Gets local export for a site"
+  @spec get_local_export(pos_integer, String.t(), String.t()) :: export | nil
+  def get_local_export(site_id, domain, timezone) do
+    path = local_export_file(site_id)
+
+    if File.exists?(path) do
+      %File.Stat{size: size, mtime: mtime} = File.stat!(path, time: :posix)
+      created_at = DateTime.from_unix!(mtime)
+      created_on_in_site_tz = Plausible.Timezones.to_date_in_timezone(created_at, timezone)
+      name = archive_filename(domain, created_on_in_site_tz)
+
+      %{path: path, name: name, expires_at: nil, size: size}
+    end
+  end
+
+  @doc "Deletes local export for a site"
+  @spec delete_local_export(pos_integer) :: :ok
+  def delete_local_export(site_id) do
+    file = local_export_file(site_id)
+
+    if File.exists?(file) do
+      File.rm!(file)
+    end
+
+    :ok
+  end
+
+  @spec local_export_file(pos_integer) :: Path.t()
+  defp local_export_file(site_id) do
+    data_dir = Application.get_env(:plausible, :data_dir)
+    Path.join([data_dir || System.tmp_dir!(), "plausible-exports", Integer.to_string(site_id)])
+  end
+
+  @doc "Gets S3 export for a site. Raises if object storage is unavailable."
+  @spec get_s3_export!(pos_integer, non_neg_integer) :: export | nil
+  def get_s3_export!(site_id, retries \\ 0) do
+    path = s3_export_key(site_id)
+    bucket = Plausible.S3.exports_bucket()
+    head_object_op = ExAws.S3.head_object(bucket, path)
+
+    case ExAws.request(head_object_op, retries: retries) do
+      {:ok, %{status_code: 200, headers: headers}} ->
+        "attachment; filename=" <> filename = :proplists.get_value("content-disposition", headers)
+        name = String.trim(filename, "\"")
+        size = :proplists.get_value("content-length", headers, nil)
+
+        expires_at =
+          if x_amz_expiration = :proplists.get_value("x-amz-expiration", headers, nil) do
+            ["expiry-date=", expiry_date, ", rule-id=", _rule_id] =
+              String.split(x_amz_expiration, "\"", trim: true)
+
+            Plausible.Times.parse!(expiry_date, "{RFC1123}")
+          end
+
+        %{
+          path: path,
+          name: name,
+          expires_at: expires_at,
+          size: String.to_integer(size)
+        }
+
+      {:error, {:http_error, 404, _response}} ->
+        nil
+
+      {:error, %Finch.TransportError{} = e} ->
+        raise e
+    end
+  end
+
+  @doc "Deletes S3 export for a site. Raises if object storage is unavailable."
+  @spec delete_s3_export!(pos_integer) :: :ok
+  def delete_s3_export!(site_id) do
+    if export = get_s3_export!(site_id) do
+      exports_bucket = Plausible.S3.exports_bucket()
+      delete_op = ExAws.S3.delete_object(exports_bucket, export.path)
+      ExAws.request!(delete_op)
+    end
+
+    :ok
+  end
+
+  defp s3_export_key(site_id), do: Integer.to_string(site_id)
+
+  @doc "Returns the date range for the site's events data in site's timezone or `nil` if there is no data"
+  @spec date_range(pos_integer, String.t()) :: Date.Range.t() | nil
+  def date_range(site_id, timezone) do
+    [%Date{} = start_date, %Date{} = end_date] =
+      Plausible.ClickhouseRepo.one(
+        from e in "events_v2",
+          where: [site_id: ^site_id],
+          select: [
+            fragment("toDate(min(?),?)", e.timestamp, ^timezone),
+            fragment("toDate(max(?),?)", e.timestamp, ^timezone)
+          ]
+      )
+
+    unless end_date == ~D[1970-01-01] do
+      Date.range(start_date, end_date)
+    end
+  end
+
+  @doc """
+  Builds Ecto queries to export data from `events_v2` and `sessions_v2`
+  tables into the format of `imported_*` tables for a website.
+  """
+  @spec export_queries(Plausible.Site.t(),
+          extname: String.t(),
+          date_range: Date.Range.t(),
+          timezone: String.t()
+        ) ::
+          %{String.t() => Ecto.Query.t()}
+  def export_queries(site, opts \\ []) do
+    extname = opts[:extname] || ".csv"
+    date_range = opts[:date_range]
+    timezone = opts[:timezone] || "UTC"
+
+    suffix =
+      if date_range do
+        first_date = Calendar.strftime(date_range.first, "%Y%m%d")
+        last_date = Calendar.strftime(date_range.last, "%Y%m%d")
+        "_#{first_date}_#{last_date}" <> extname
+      else
+        extname
+      end
+
+    filename = fn name -> name <> suffix end
+
+    %{
+      filename.("imported_visitors") => export_visitors_q(site, timezone, date_range),
+      filename.("imported_sources") => export_sources_q(site, timezone, date_range),
+      filename.("imported_pages") => export_pages_q(site, timezone, date_range),
+      filename.("imported_entry_pages") => export_entry_pages_q(site, timezone, date_range),
+      filename.("imported_exit_pages") => export_exit_pages_q(site, timezone, date_range),
+      filename.("imported_custom_events") => export_custom_events_q(site, timezone, date_range),
+      filename.("imported_locations") => export_locations_q(site, timezone, date_range),
+      filename.("imported_devices") => export_devices_q(site, timezone, date_range),
+      filename.("imported_browsers") => export_browsers_q(site, timezone, date_range),
+      filename.("imported_operating_systems") =>
+        export_operating_systems_q(site, timezone, date_range),
+      filename.("imported_custom_props") => export_custom_props_q(site, timezone, date_range)
+    }
+  end
+
+  on_ee do
+    defp sampled(table) do
+      Plausible.Stats.Sampling.add_query_hint(from(table))
+    end
+  else
+    defp sampled(table) do
+      table
+    end
+  end
+
+  defp export_filter(site_id, date_range) do
+    filter = dynamic([t], t.site_id == ^site_id)
+
+    if date_range do
+      dynamic(
+        ^filter and
+          selected_as(:date) >= ^date_range.first and
+          selected_as(:date) <= ^date_range.last
+      )
+    else
+      filter
+    end
+  end
+
+  defmacrop date(timestamp, timezone) do
+    quote do
+      selected_as(
+        fragment("toDate(?,?)", unquote(timestamp), unquote(timezone)),
+        :date
+      )
+    end
+  end
+
+  defmacrop visit_duration(t) do
+    quote do
+      selected_as(
+        scale_sample(fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).duration)),
+        :visit_duration
+      )
+    end
+  end
+
+  defmacrop visitors(t) do
+    quote do
+      selected_as(
+        scale_sample(fragment("uniq(?)", unquote(t).user_id)),
+        :visitors
+      )
+    end
+  end
+
+  defmacrop visits(t) do
+    quote do
+      selected_as(
+        scale_sample(fragment("greatest(sum(?),0)", unquote(t).sign)),
+        :visits
+      )
+    end
+  end
+
+  defmacrop bounces(t) do
+    quote do
+      selected_as(
+        scale_sample(
+          fragment(
+            "greatest(sum(?*?),0)",
+            unquote(t).sign,
+            unquote(t).is_bounce
+          )
+        ),
+        :bounces
+      )
+    end
+  end
+
+  defmacrop pageviews(t) do
+    quote do
+      selected_as(
+        scale_sample(
+          fragment(
+            "greatest(sum(?*?),0)",
+            unquote(t).sign,
+            unquote(t).pageviews
+          )
+        ),
+        :pageviews
+      )
+    end
+  end
+
+  defp export_visitors_q(site, timezone, date_range) do
+    visitors_sessions_q =
+      from s in sampled("sessions_v2"),
+        where: ^export_filter(site.id, date_range),
+        group_by: selected_as(:date),
+        select: %{
+          date: date(s.timestamp, ^timezone),
+          bounces: bounces(s),
+          visits: visits(s),
+          visit_duration: visit_duration(s),
+          visitors: visitors(s)
+        }
+
+    visitors_events_q =
+      from e in sampled("events_v2"),
+        where: ^export_filter(site.id, date_range),
+        group_by: selected_as(:date),
+        select: %{
+          date: date(e.timestamp, ^timezone),
+          pageviews:
+            selected_as(
+              scale_sample(fragment("countIf(?='pageview')", e.name)),
+              :pageviews
+            )
+        }
+
+    visitors_q =
+      "e"
+      |> with_cte("e", as: ^visitors_events_q)
+      |> with_cte("s", as: ^visitors_sessions_q)
+
+    from e in visitors_q,
+      full_join: s in "s",
+      on: e.date == s.date,
+      order_by: selected_as(:date),
+      select: [
+        selected_as(fragment("greatest(?,?)", s.date, e.date), :date),
+        s.visitors,
+        e.pageviews,
+        s.bounces,
+        s.visits,
+        s.visit_duration
+      ]
+  end
+
+  defp export_sources_q(site, timezone, date_range) do
+    from s in sampled("sessions_v2"),
+      where: ^export_filter(site.id, date_range),
+      group_by: [
+        selected_as(:date),
+        selected_as(:source),
+        s.referrer,
+        s.utm_source,
+        s.utm_medium,
+        s.utm_campaign,
+        s.utm_content,
+        s.utm_term
+      ],
+      order_by: selected_as(:date),
+      select: [
+        date(s.timestamp, ^timezone),
+        selected_as(s.referrer_source, :source),
+        s.referrer,
+        s.utm_source,
+        s.utm_medium,
+        s.utm_campaign,
+        s.utm_content,
+        s.utm_term,
+        pageviews(s),
+        visitors(s),
+        visits(s),
+        visit_duration(s),
+        bounces(s)
+      ]
+  end
+
+  defp export_pages_q(site, timezone, date_range) do
+    base_q =
+      from(e in sampled("events_v2"),
+        where: ^export_filter(site.id, date_range),
+        where: [name: "pageview"],
+        group_by: [selected_as(:date), selected_as(:page)],
+        order_by: selected_as(:date)
+      )
+
+    max_scroll_depth_per_session_q =
+      from(e in "events_v2",
+        where: ^export_filter(site.id, date_range),
+        where: e.name == "engagement" and e.scroll_depth <= 100,
+        select: %{
+          date: date(e.timestamp, ^timezone),
+          page: selected_as(e.pathname, :page),
+          session_id: e.session_id,
+          max_scroll_depth: max(e.scroll_depth)
+        },
+        group_by: [e.session_id, selected_as(:date), selected_as(:page)]
+      )
+
+    scroll_depth_q =
+      from(p in subquery(max_scroll_depth_per_session_q),
+        select: %{
+          date: p.date,
+          page: p.page,
+          total_scroll_depth: fragment("sum(?)", p.max_scroll_depth),
+          total_scroll_depth_visits: fragment("uniq(?)", p.session_id)
+        },
+        group_by: [:date, :page]
+      )
+
+    from(e in base_q,
+      left_join: s in subquery(scroll_depth_q),
+      on: s.date == selected_as(:date) and s.page == selected_as(:page),
+      select: %{
+        date: date(e.timestamp, ^timezone),
+        hostname: selected_as(fragment("any(?)", e.hostname), :hostname),
+        page: selected_as(e.pathname, :page),
+        visits:
+          selected_as(
+            scale_sample(fragment("uniq(?)", e.session_id)),
+            :visits
+          ),
+        visitors: visitors(e),
+        pageviews: selected_as(scale_sample(fragment("count()")), :pageviews),
+        total_scroll_depth:
+          selected_as(fragment("any(?)", s.total_scroll_depth), :total_scroll_depth),
+        total_scroll_depth_visits:
+          selected_as(fragment("any(?)", s.total_scroll_depth_visits), :total_scroll_depth_visits)
+      }
+    )
+    |> add_time_on_page_columns(site.id, timezone, date_range)
+  end
+
+  defp add_time_on_page_columns(q, site_id, timezone, date_range) do
+    site = Plausible.Repo.get(Plausible.Site, site_id)
+
+    if Plausible.Stats.TimeOnPage.new_time_on_page_visible?(site) do
+      cutoff = Plausible.Stats.TimeOnPage.legacy_time_on_page_cutoff(site)
+
+      engagements_q =
+        from(e in sampled("events_v2"),
+          where: ^export_filter(site_id, date_range),
+          where: e.name == "engagement",
+          group_by: [selected_as(:date), selected_as(:page)],
+          order_by: selected_as(:date),
+          select: %{
+            date: date(e.timestamp, ^timezone),
+            page: selected_as(e.pathname, :page),
+            total_time_on_page:
+              fragment(
+                "toUInt64(round(sumIf(?, ? >= ?) / 1000))",
+                e.engagement_time,
+                e.timestamp,
+                ^cutoff
+              ),
+            total_time_on_page_visits:
+              fragment("uniqIf(?, ? >= ?)", e.session_id, e.timestamp, ^cutoff)
+          }
+        )
+
+      q
+      |> join(:left, [], s in subquery(engagements_q),
+        on: s.date == selected_as(:date) and s.page == selected_as(:page)
+      )
+      |> select_merge_as([..., s], %{
+        total_time_on_page: fragment("any(?)", s.total_time_on_page),
+        total_time_on_page_visits: fragment("any(?)", s.total_time_on_page_visits)
+      })
+    else
+      q
+    end
+  end
+
+  defp export_entry_pages_q(site, timezone, date_range) do
+    from s in sampled("sessions_v2"),
+      where: ^export_filter(site.id, date_range),
+      group_by: [selected_as(:date), s.entry_page],
+      order_by: selected_as(:date),
+      select: [
+        date(s.timestamp, ^timezone),
+        s.entry_page,
+        visitors(s),
+        selected_as(
+          scale_sample(fragment("greatest(sum(?),0)", s.sign)),
+          :entrances
+        ),
+        visit_duration(s),
+        bounces(s),
+        pageviews(s)
+      ]
+  end
+
+  defp export_exit_pages_q(site, timezone, date_range) do
+    from s in sampled("sessions_v2"),
+      where: ^export_filter(site.id, date_range),
+      group_by: [selected_as(:date), s.exit_page],
+      order_by: selected_as(:date),
+      select: [
+        date(s.timestamp, ^timezone),
+        s.exit_page,
+        visitors(s),
+        visit_duration(s),
+        selected_as(
+          scale_sample(fragment("greatest(sum(?),0)", s.sign)),
+          :exits
+        ),
+        bounces(s),
+        pageviews(s)
+      ]
+  end
+
+  defp export_custom_events_q(site, timezone, date_range) do
+    from e in sampled("events_v2"),
+      where: ^export_filter(site.id, date_range),
+      where: e.name != "pageview",
+      group_by: [
+        selected_as(:date),
+        e.name,
+        selected_as(:link_url),
+        selected_as(:path)
+      ],
+      order_by: selected_as(:date),
+      select: [
+        date(e.timestamp, ^timezone),
+        e.name,
+        selected_as(
+          fragment(
+            "if(? in ?, ?, '')",
+            e.name,
+            ^Plausible.Event.SystemEvents.events_with_url_prop(),
+            get_by_key(e, :meta, "url")
+          ),
+          :link_url
+        ),
+        selected_as(
+          fragment(
+            "if(? in ?, ?, '')",
+            e.name,
+            ^Plausible.Event.SystemEvents.events_with_path_prop(),
+            get_by_key(e, :meta, "path")
+          ),
+          :path
+        ),
+        visitors(e),
+        selected_as(scale_sample(fragment("count()")), :events)
+      ]
+  end
+
+  defp export_custom_props_q(site, timezone, date_range) do
+    query =
+      from e in sampled("events_v2"),
+        join: pv in fragment("arrayZip(`meta.key`, `meta.value`)"),
+        on: true,
+        hints: "ARRAY",
+        where: ^export_filter(site.id, date_range),
+        group_by: [
+          selected_as(:date),
+          selected_as(:property),
+          selected_as(:value)
+        ],
+        order_by: selected_as(:date),
+        select: [
+          date(e.timestamp, ^timezone),
+          selected_as(fragment("tupleElement(?, 1)", pv), :property),
+          selected_as(fragment("tupleElement(?, 2)", pv), :value),
+          visitors(e),
+          selected_as(scale_sample(fragment("count()")), :events)
+        ]
+
+    if Plausible.Billing.Feature.Props.enabled?(site) do
+      query
+    else
+      where(query, [], selected_as(:property) in ^Plausible.Props.internal_keys())
+    end
+  end
+
+  defp export_locations_q(site, timezone, date_range) do
+    from s in sampled("sessions_v2"),
+      where: ^export_filter(site.id, date_range),
+      where: s.country_code != "\0\0" and s.country_code != "ZZ",
+      group_by: [selected_as(:date), s.country_code, s.subdivision1_code, s.city_geoname_id],
+      order_by: selected_as(:date),
+      select: [
+        date(s.timestamp, ^timezone),
+        selected_as(s.country_code, :country),
+        selected_as(s.subdivision1_code, :region),
+        selected_as(s.city_geoname_id, :city),
+        visitors(s),
+        visits(s),
+        visit_duration(s),
+        bounces(s),
+        pageviews(s)
+      ]
+  end
+
+  defp export_devices_q(site, timezone, date_range) do
+    from s in sampled("sessions_v2"),
+      where: ^export_filter(site.id, date_range),
+      group_by: [selected_as(:date), s.screen_size],
+      order_by: selected_as(:date),
+      select: [
+        date(s.timestamp, ^timezone),
+        selected_as(s.screen_size, :device),
+        visitors(s),
+        visits(s),
+        visit_duration(s),
+        bounces(s),
+        pageviews(s)
+      ]
+  end
+
+  defp export_browsers_q(site, timezone, date_range) do
+    from s in sampled("sessions_v2"),
+      where: ^export_filter(site.id, date_range),
+      group_by: [selected_as(:date), s.browser, s.browser_version],
+      order_by: selected_as(:date),
+      select: [
+        date(s.timestamp, ^timezone),
+        s.browser,
+        s.browser_version,
+        visitors(s),
+        visits(s),
+        visit_duration(s),
+        bounces(s),
+        pageviews(s)
+      ]
+  end
+
+  defp export_operating_systems_q(site, timezone, date_range) do
+    from s in sampled("sessions_v2"),
+      where: ^export_filter(site.id, date_range),
+      group_by: [selected_as(:date), s.operating_system, s.operating_system_version],
+      order_by: selected_as(:date),
+      select: [
+        date(s.timestamp, ^timezone),
+        s.operating_system,
+        s.operating_system_version,
+        visitors(s),
+        visits(s),
+        visit_duration(s),
+        bounces(s),
+        pageviews(s)
+      ]
+  end
+
+  @doc """
+  Creates a streamable Zip archive from the provided (named) Ecto queries.
+
+  Example usage:
+
+      {:ok, pool} = Ch.start_link(pool_size: 1)
+
+      DBConnection.run(pool, fn conn ->
+        conn
+        |> stream_archive(export_queries(site), format: "CSVWithNames")
+        |> Stream.into(File.stream!("export.zip"))
+        |> Stream.run()
+      end)
+
+  """
+  @spec stream_archive(DBConnection.t(), %{String.t() => Ecto.Query.t()}, [Ch.query_option()]) ::
+          Enumerable.t()
+  def stream_archive(conn, named_queries, opts \\ []) do
+    entries =
+      Enum.map(named_queries, fn {name, query} ->
+        {sql, params} = Plausible.ClickhouseRepo.to_sql(:all, query)
+
+        datastream =
+          conn
+          |> Ch.stream(sql, params, opts)
+          |> Stream.map(fn %Ch.Result{data: data} -> data end)
+
+        Zstream.entry(name, datastream, coder: Zstream.Coder.Stored)
+      end)
+
+    Zstream.zip(entries)
+  end
+end

lib/plausible/file.ex

@@ -0,0 +1,27 @@
+defmodule Plausible.File do
+  @moduledoc """
+  File helpers for Plausible.
+  """
+
+  @doc """
+  Moves a file from one location to another.
+
+  Tries renaming first, and falls back to copying and deleting the original.
+  """
+  @spec mv!(Path.t(), Path.t()) :: :ok
+  def mv!(source, destination) do
+    File.rename!(source, destination)
+  rescue
+    e in File.RenameError ->
+      try do
+        case e.reason do
+          # fallback to cp/rm for cross-device moves
+          # https://github.com/plausible/analytics/issues/4638
+          :exdev -> File.cp!(source, destination)
+          _ -> reraise(e, __STACKTRACE__)
+        end
+      after
+        File.rm(source)
+      end
+  end
+end

lib/plausible/funnel/const.ex

@@ -0,0 +1,26 @@
+defmodule Plausible.Funnel.Const do
+  @moduledoc """
+  Compile-time convenience constants for funnel characteristics.
+  """
+  @min_steps 2
+  @max_steps 8
+
+  defmacro min_steps() do
+    quote do
+      unquote(@min_steps)
+    end
+  end
+
+  defmacro max_steps() do
+    quote do
+      unquote(@max_steps)
+    end
+  end
+
+  defmacro __using__(_opts \\ []) do
+    quote do
+      require Plausible.Funnel.Const
+      alias Plausible.Funnel
+    end
+  end
+end

lib/plausible/geo.ex

@@ -0,0 +1,207 @@
+defmodule Plausible.Geo do
+  @moduledoc """
+  This module provides an API for fetching IP geolocation.
+  """
+
+  require Logger
+
+  @db :geolocation
+
+  @doc """
+  Starts the geodatabase loading process. Two modes are supported: local file
+  and MaxMind license key.
+
+  ## Options
+
+    * `:path` - the path to the .mmdb database local file. When present,
+      `:license_key` and `:edition` are not required.
+
+    * `:license_key` - the [license key](https://support.maxmind.com/hc/en-us/articles/4407111582235-Generate-a-License-Key)
+      from MaxMind to authenticate requests to MaxMind.
+
+    * `:edition` - the name of the MaxMind database to be downloaded from MaxMind
+      servers. Defaults to `GeoLite2-City`.
+
+    * `:cache_dir` - if set, the downloaded .mmdb files are cached there across
+      restarts.
+
+    * `:async` - when used, configures the database loading to run
+      asynchronously.
+
+  ## Examples
+
+    Loading from a local file:
+
+      iex> load_db(path: "/etc/plausible/dbip-city.mmdb")
+      :ok
+
+    Downloading a MaxMind DB (this license key is no longer active):
+
+      iex> load_db(license_key: "LNpsJCCKPis6XvBP", edition: "GeoLite2-City", async: true)
+      :ok
+
+  """
+  def load_db(opts) do
+    cond do
+      license_key = opts[:license_key] ->
+        edition = opts[:edition] || "GeoLite2-City"
+        maxmind_opts = [license_key: license_key]
+
+        loader_opts =
+          if is_binary(opts[:cache_dir]) do
+            [
+              database_cache_file:
+                String.to_charlist(Path.join(opts[:cache_dir], edition <> ".mmdb.gz"))
+            ]
+          else
+            [:no_cache]
+          end
+
+        :ok = :locus.start_loader(@db, {:maxmind, edition}, maxmind_opts ++ loader_opts)
+
+      path = opts[:path] ->
+        :ok = :locus.start_loader(@db, path)
+
+      true ->
+        raise "failed to load geolocation db: need :path or :license_key to be provided"
+    end
+
+    unless opts[:async] do
+      {:ok, _version} = :locus.await_loader(@db)
+    end
+
+    :ok
+  end
+
+  @doc """
+  Waits for the database to start after calling `load_db/1` with the async option.
+  """
+  def await_loader, do: :locus.await_loader(@db)
+
+  @doc """
+  Returns geodatabase type.
+
+  Used for deciding whether to show the DB-IP disclaimer or not.
+
+  ## Examples
+
+    In the case of a DB-IP database:
+
+      iex> database_type()
+      "DBIP-City-Lite"
+
+    In the case of a MaxMind database:
+
+      iex> database_type()
+      "GeoLite2-City"
+
+  """
+  def database_type do
+    case :locus.get_info(@db, :metadata) do
+      {:ok, %{database_type: type}} -> type
+      _other -> nil
+    end
+  end
+
+  @doc """
+  Looks up geo info about an IP address.
+
+  ## Examples
+
+      iex> lookup("8.7.6.5")
+      %{
+        "city" => %{
+          "geoname_id" => 5349755,
+          "names" => %{
+            "de" => "Fontana",
+            "en" => "Fontana",
+            "ja" => "フォンタナ",
+            "ru" => "Фонтана"
+          }
+        },
+        "continent" => %{
+          "code" => "NA",
+          "geoname_id" => 6255149,
+          "names" => %{
+            "de" => "Nordamerika",
+            "en" => "North America",
+            "es" => "Norteamérica",
+            "fr" => "Amérique du Nord",
+            "ja" => "北アメリカ",
+            "pt-BR" => "América do Norte",
+            "ru" => "Северная Америка",
+            "zh-CN" => "北美洲"
+          }
+        },
+        "country" => %{
+          "geoname_id" => 6252001,
+          "iso_code" => "US",
+          "names" => %{
+            "de" => "Vereinigte Staaten",
+            "en" => "United States",
+            "es" => "Estados Unidos",
+            "fr" => "États Unis",
+            "ja" => "アメリカ",
+            "pt-BR" => "EUA",
+            "ru" => "США",
+            "zh-CN" => "美国"
+          }
+        },
+        "location" => %{
+          "accuracy_radius" => 50,
+          "latitude" => 34.1211,
+          "longitude" => -117.4362,
+          "metro_code" => 803,
+          "time_zone" => "America/Los_Angeles"
+        },
+        "postal" => %{"code" => "92336"},
+        "registered_country" => %{
+          "geoname_id" => 6252001,
+          "iso_code" => "US",
+          "names" => %{
+            "de" => "Vereinigte Staaten",
+            "en" => "United States",
+            "es" => "Estados Unidos",
+            "fr" => "États Unis",
+            "ja" => "アメリカ",
+            "pt-BR" => "EUA",
+            "ru" => "США",
+            "zh-CN" => "美国"
+          }
+        },
+        "subdivisions" => [
+          %{
+            "geoname_id" => 5332921,
+            "iso_code" => "CA",
+            "names" => %{
+              "de" => "Kalifornien",
+              "en" => "California",
+              "es" => "California",
+              "fr" => "Californie",
+              "ja" => "カリフォルニア州",
+              "pt-BR" => "Califórnia",
+              "ru" => "Калифорния",
+              "zh-CN" => "加州"
+            }
+          }
+        ]
+      }
+
+  """
+  def lookup(ip_address) do
+    case :locus.lookup(@db, ip_address) do
+      {:ok, entry} ->
+        entry
+
+      :not_found ->
+        nil
+
+      {:error, {:invalid_address, _address}} ->
+        nil
+
+      {:error, reason} ->
+        Logger.error("Failed to lookup IP address. Reason: " <> inspect(reason))
+        nil
+    end
+  end
+end

lib/plausible/goal.ex

@@ -0,0 +1,317 @@
+defmodule Plausible.Goal do
+  @moduledoc """
+  Goal schema.
+  """
+
+  use Plausible
+  use Ecto.Schema
+
+  import Ecto.Changeset
+
+  @type t() :: %__MODULE__{}
+
+  on_ee do
+    @journey_end_event Plausible.Stats.Exploration.Journey.Step.journey_end_event()
+  else
+    @journey_end_event nil
+  end
+
+  schema "goals" do
+    field :event_name, :string
+    field :page_path, :string
+    field :scroll_threshold, :integer, default: -1
+    field :display_name, :string
+
+    on_ee do
+      field :currency, Ecto.Enum, values: Money.Currency.known_current_currencies()
+      many_to_many :funnels, Plausible.Funnel, join_through: Plausible.Funnel.Step
+    else
+      field :currency, :string, virtual: true, default: nil
+      field :funnels, {:array, :map}, virtual: true, default: []
+    end
+
+    field :custom_props, :map, default: %{}
+
+    belongs_to :site, Plausible.Site
+
+    timestamps()
+  end
+
+  @fields [
+            :event_name,
+            :page_path,
+            :scroll_threshold,
+            :display_name,
+            :custom_props
+          ] ++
+            on_ee(do: [:currency], else: [])
+
+  @max_event_name_length 120
+
+  def max_event_name_length(), do: @max_event_name_length
+
+  @max_custom_props_per_goal 3
+
+  def max_custom_props_per_goal(), do: @max_custom_props_per_goal
+
+  @special_goals [
+    "404",
+    "Outbound Link: Click",
+    "Cloaked Link: Click",
+    "File Download",
+    "Form: Submission",
+    "WP Search Queries",
+    "WP Form Completions"
+  ]
+
+  def special_goals(), do: @special_goals
+
+  @spec special_goal?(t() | String.t() | nil) :: boolean()
+  def special_goal?(%__MODULE__{event_name: event_name}), do: special_goal?(event_name)
+  def special_goal?(event_name) when is_binary(event_name), do: event_name in @special_goals
+  def special_goal?(_), do: false
+
+  def changeset(goal, attrs \\ %{}) do
+    goal
+    |> cast(attrs, @fields)
+    |> update_leading_slash()
+    |> validate_event_name_and_page_path()
+    |> validate_page_path_for_scroll_goal()
+    |> maybe_put_display_name()
+    |> validate_change(:custom_props, &validate_custom_props/2)
+    |> unique_constraint(:display_name, name: :goals_display_name_unique)
+    |> unique_constraint(:event_name, name: :goals_event_config_unique)
+    |> unique_constraint([:page_path, :scroll_threshold],
+      name: :goals_pageview_config_unique
+    )
+    |> validate_length(:event_name, max: @max_event_name_length)
+    |> validate_number(:scroll_threshold,
+      greater_than_or_equal_to: -1,
+      less_than_or_equal_to: 100,
+      message: "Should be -1 (missing) or in range [0, 100]"
+    )
+    |> check_constraint(:event_name,
+      name: :check_event_name_or_page_path,
+      message: "cannot co-exist with page_path"
+    )
+    |> maybe_drop_currency()
+    |> prevent_currency_change()
+    |> prevent_special_goal_renames()
+  end
+
+  @spec display_name(t()) :: String.t()
+  def display_name(goal) do
+    goal.display_name
+  end
+
+  @spec type(t()) :: :event | :scroll | :page
+  def type(goal) do
+    cond do
+      is_binary(goal.event_name) -> :event
+      is_binary(goal.page_path) && goal.scroll_threshold > -1 -> :scroll
+      is_binary(goal.page_path) -> :page
+    end
+  end
+
+  @spec has_custom_props?(t()) :: boolean()
+  def has_custom_props?(%__MODULE__{custom_props: custom_props})
+      when map_size(custom_props) > 0 do
+    true
+  end
+
+  def has_custom_props?(_), do: false
+
+  defp update_leading_slash(changeset) do
+    case get_field(changeset, :page_path) do
+      "/" <> _ ->
+        changeset
+
+      page_path when is_binary(page_path) ->
+        put_change(changeset, :page_path, "/" <> page_path)
+
+      _ ->
+        changeset
+    end
+  end
+
+  defp validate_event_name_and_page_path(changeset) do
+    case {validate_page_path(changeset), validate_event_name(changeset)} do
+      {:ok, _} ->
+        update_change(changeset, :page_path, &String.trim/1)
+
+      {_, :ok} ->
+        update_change(changeset, :event_name, &String.trim/1)
+
+      {{:error, page_path_error}, {:error, event_name_error}} ->
+        changeset
+        |> add_error(:event_name, event_name_error)
+        |> add_error(:page_path, page_path_error)
+    end
+  end
+
+  defp validate_page_path_for_scroll_goal(changeset) do
+    scroll_threshold = get_field(changeset, :scroll_threshold)
+    page_path = get_field(changeset, :page_path)
+
+    if scroll_threshold > -1 and is_nil(page_path) do
+      changeset
+      |> add_error(:scroll_threshold, "page_path field missing for page scroll goal")
+    else
+      changeset
+    end
+  end
+
+  defp validate_page_path(changeset) do
+    value = get_field(changeset, :page_path)
+
+    if value && String.match?(value, ~r/^\/.*/) do
+      :ok
+    else
+      {:error, "this field is required and must start with a /"}
+    end
+  end
+
+  defp validate_event_name(changeset) do
+    value = get_field(changeset, :event_name)
+
+    cond do
+      value == "engagement" ->
+        {:error, "The event name 'engagement' is reserved and cannot be used as a goal"}
+
+      journey_end_event?(value) ->
+        {:error,
+         "The event name '#{@journey_end_event}' is reserved and cannot be used as a goal"}
+
+      value && String.match?(value, ~r/^.+/) ->
+        :ok
+
+      true ->
+        {:error, "this field is required and cannot be blank"}
+    end
+  end
+
+  defp maybe_drop_currency(changeset) do
+    if ee?() and get_field(changeset, :page_path) do
+      delete_change(changeset, :currency)
+    else
+      changeset
+    end
+  end
+
+  defp prevent_currency_change(changeset) do
+    if (ee?() and changeset.data.id) && Map.has_key?(changeset.changes, :currency) do
+      add_error(changeset, :currency, "cannot change currency of existing goal")
+    else
+      changeset
+    end
+  end
+
+  defp maybe_put_display_name(changeset) do
+    clause =
+      Enum.map([:display_name, :page_path, :event_name], &get_field(changeset, &1))
+
+    case clause do
+      [nil, path, _] when is_binary(path) ->
+        put_change(changeset, :display_name, "Visit " <> path)
+
+      [nil, _, event_name] when is_binary(event_name) ->
+        put_change(changeset, :display_name, event_name)
+
+      _ ->
+        changeset
+    end
+    |> update_change(:display_name, &String.trim/1)
+    |> validate_required(:display_name)
+  end
+
+  defp validate_custom_props(:custom_props, custom_props) when is_map(custom_props) do
+    cond do
+      map_size(custom_props) > @max_custom_props_per_goal ->
+        [custom_props: "use at most #{@max_custom_props_per_goal} properties per goal"]
+
+      not Enum.all?(custom_props, fn {k, v} ->
+        is_binary(k) and is_binary(v)
+      end) ->
+        [custom_props: "must be a map with string keys and string values"]
+
+      Enum.any?(custom_props, fn {k, _v} ->
+        String.length(k) not in 1..Plausible.Props.max_prop_key_length()
+      end) ->
+        [
+          custom_props: "key length is 1..#{Plausible.Props.max_prop_key_length()} characters"
+        ]
+
+      Enum.any?(custom_props, fn {_k, v} ->
+        String.length(v) not in 1..Plausible.Props.max_prop_value_length()
+      end) ->
+        [
+          custom_props: "value length is 1..#{Plausible.Props.max_prop_value_length()} characters"
+        ]
+
+      true ->
+        []
+    end
+  end
+
+  defp prevent_special_goal_renames(changeset) do
+    if changeset.data.id && special_goal?(changeset.data.event_name) do
+      changeset
+      |> reject_special_goal_field_change(:event_name)
+      |> reject_special_goal_field_change(:display_name)
+    else
+      changeset
+    end
+  end
+
+  defp reject_special_goal_field_change(changeset, :event_name) do
+    if Map.has_key?(changeset.changes, :event_name) do
+      add_error(changeset, :event_name, "cannot be changed for an automated goal")
+    else
+      changeset
+    end
+  end
+
+  defp reject_special_goal_field_change(changeset, :display_name) do
+    new_value = get_change(changeset, :display_name)
+    canonical = changeset.data.event_name
+
+    if new_value && new_value != canonical do
+      add_error(changeset, :display_name, "cannot be changed for an automated goal")
+    else
+      changeset
+    end
+  end
+
+  on_ee do
+    defp journey_end_event?(name) do
+      name == @journey_end_event
+    end
+  else
+    defp journey_end_event?(_name), do: always(false)
+  end
+end
+
+defimpl Jason.Encoder, for: Plausible.Goal do
+  def encode(value, opts) do
+    domain = value.site.domain
+
+    value
+    |> Map.put(:goal_type, Plausible.Goal.type(value))
+    |> Map.take([:id, :goal_type, :event_name, :page_path, :custom_props])
+    |> Map.put(:domain, domain)
+    |> Map.put(:display_name, value.display_name)
+    |> Jason.Encode.map(opts)
+  end
+end
+
+defimpl String.Chars, for: Plausible.Goal do
+  def to_string(goal) do
+    goal.display_name
+  end
+end
+
+defimpl Phoenix.HTML.Safe, for: Plausible.Goal do
+  def to_iodata(data) do
+    data |> to_string() |> Phoenix.HTML.Engine.html_escape()
+  end
+end

lib/plausible/goals/goals.ex

@@ -0,0 +1,490 @@
+defmodule Plausible.Goals do
+  @moduledoc """
+  Goals API.
+  """
+
+  use Plausible
+  use Plausible.Repo
+  use Plausible.Funnel.Const
+
+  import Ecto.Query
+
+  alias Plausible.Goal
+  alias Ecto.Multi
+  alias Ecto.Changeset
+
+  @max_goals_per_site 1_000
+  @spec max_goals_per_site(Keyword.t()) :: pos_integer()
+  def max_goals_per_site(opts \\ []) do
+    override = Keyword.get(opts, :max_goals_per_site)
+
+    if override do
+      override
+    else
+      # see: config/test.exs - you can steer this limit for tests
+      # by providing `max_goals_per_site` option to e.g. create/3
+      Application.get_env(:plausible, :max_goals_per_site, @max_goals_per_site)
+    end
+  end
+
+  @spec get(Plausible.Site.t(), pos_integer()) :: nil | Plausible.Goal.t()
+  def get(site, id) when is_integer(id) do
+    q =
+      from g in Plausible.Goal,
+        where: g.site_id == ^site.id,
+        order_by: [desc: g.id],
+        where: g.id == ^id
+
+    Repo.one(q)
+  end
+
+  @spec create(Plausible.Site.t(), map(), Keyword.t()) ::
+          {:ok, Goal.t()}
+          | {:error, Changeset.t()}
+          | {:error, :upgrade_required}
+          | {:error, :revenue_goals_unavailable}
+  @doc """
+  Creates a Goal for a site.
+
+  If the created goal is a revenue goal, it sets site.updated_at to be
+  refreshed by the sites cache, as revenue goals are used during ingestion.
+
+  Returns `{:ok, goal}` or `{:error, changeset}` when creation fails due to
+  invalid fields. It can also return:
+
+  * `{:error, :upgrade_required}` - Adding a revenue goal is not allowed
+    for team's subscription.
+
+  * `{:error, :revenue_goals_unavailable}` - When the site is a consolidated
+    view and the goal created is a revenue goal. Revenue goal creation is not
+    allowed for consolidated views due to the inability to force a single
+    currency on a goal across all consolidated sites.
+  """
+  def create(site, params, opts \\ []) do
+    Repo.transaction(fn ->
+      case insert_goal(site, params, opts) do
+        {:ok, :insert, goal} ->
+          on_ee do
+            now = Keyword.get(opts, :now, DateTime.utc_now())
+            # credo:disable-for-next-line Credo.Check.Refactor.Nesting
+            if Plausible.Goal.Revenue.revenue?(goal) do
+              Plausible.Site.Cache.touch_site!(site, now)
+            end
+          end
+
+          Repo.preload(goal, :site)
+
+        {:ok, :upsert, goal} ->
+          Repo.preload(goal, :site)
+
+        {:error, cause} ->
+          Repo.rollback(cause)
+      end
+    end)
+  end
+
+  @spec update(Plausible.Goal.t(), map()) ::
+          {:ok, Goal.t()} | {:error, Changeset.t()} | {:error, :upgrade_required}
+  def update(goal, params) do
+    changeset = Goal.changeset(goal, params)
+
+    Repo.transaction(fn ->
+      site = Repo.preload(goal, :site).site
+
+      with :ok <- maybe_check_feature_access(site, changeset),
+           {:ok, updated_goal} <- Repo.update(changeset),
+           :ok <- Plausible.Segments.update_goal_in_segments(site, goal, updated_goal) do
+        on_ee do
+          Repo.preload(updated_goal, :funnels)
+        else
+          updated_goal
+        end
+      else
+        {:error, %Changeset{} = changeset} ->
+          Repo.rollback(changeset)
+
+        {:error, :upgrade_required} ->
+          Repo.rollback(:upgrade_required)
+      end
+    end)
+  end
+
+  def find_or_create(site, params, opts \\ [])
+
+  def find_or_create(
+        site,
+        %{
+          "goal_type" => "event",
+          "event_name" => event_name,
+          "currency" => currency
+        } = params,
+        opts
+      )
+      when is_binary(event_name) and is_binary(currency) do
+    with {:ok, goal} <- create(site, params, do_upsert(opts)) do
+      if to_string(goal.currency) == currency do
+        {:ok, goal}
+      else
+        # we must disallow creation of the same goal name with different currency
+        changeset =
+          goal
+          |> Goal.changeset()
+          |> Changeset.add_error(
+            :event_name,
+            "'#{goal.event_name}' (with currency: #{goal.currency}) has already been taken"
+          )
+
+        {:error, changeset}
+      end
+    end
+  end
+
+  def find_or_create(site, %{"goal_type" => "event", "event_name" => event_name} = params, opts)
+      when is_binary(event_name) do
+    create(site, params, do_upsert(opts))
+  end
+
+  def find_or_create(_, %{"goal_type" => "event"}, _), do: {:missing, "event_name"}
+
+  def find_or_create(site, %{"goal_type" => "page", "page_path" => _} = params, opts) do
+    create(site, params, do_upsert(opts))
+  end
+
+  def find_or_create(_, %{"goal_type" => "page"}, _), do: {:missing, "page_path"}
+
+  def list_revenue_goals(site) do
+    from(g in Plausible.Goal,
+      where: g.site_id == ^site.id and not is_nil(g.currency),
+      select: %{display_name: g.display_name, currency: g.currency},
+      order_by: [desc: g.id],
+      limit: ^max_goals_per_site()
+    )
+    |> Plausible.Repo.all()
+  end
+
+  def for_site(site, opts \\ []) do
+    site
+    |> for_site_query(opts)
+    |> Repo.all()
+    |> Enum.map(&maybe_trim/1)
+  end
+
+  def for_site_query(site \\ nil, opts \\ []) do
+    query =
+      from g in Goal,
+        order_by: [desc: g.id],
+        limit: ^max_goals_per_site(opts)
+
+    query =
+      if site do
+        from g in query,
+          inner_join: assoc(g, :site),
+          where: g.site_id == ^site.id,
+          preload: [:site]
+      else
+        query
+      end
+
+    query =
+      if Keyword.get(opts, :include_goals_with_custom_props?, true) == false do
+        from g in query, where: g.custom_props == ^%{}
+      else
+        query
+      end
+
+    if ee?() and opts[:preload_funnels?] == true do
+      from(g in query,
+        left_join: assoc(g, :funnels),
+        group_by: g.id,
+        preload: [:funnels]
+      )
+    else
+      query
+    end
+  end
+
+  def batch_create_event_goals(names, site, opts \\ []) do
+    Enum.reduce(names, [], fn name, acc ->
+      case insert_goal(site, %{event_name: name}, do_upsert(opts)) do
+        {:ok, _, goal} -> acc ++ [goal]
+        _ -> acc
+      end
+    end)
+  end
+
+  @doc """
+  If a goal belongs to funnel(s), we need to inspect their number of steps.
+
+  If it exceeds the minimum allowed (defined via `Plausible.Funnel.min_steps/0`),
+  the funnel will be reduced (i.e. a step associated with the goal to be deleted
+  is removed), so that the minimum number of steps is preserved. This is done
+  implicitly, by postgres, as per on_delete: :delete_all.
+
+  Otherwise, for associated funnel(s) consisting of minimum number steps only,
+  funnel record(s) are removed completely along with the targeted goal.
+  """
+  def delete(id, %Plausible.Site{id: site_id}) do
+    delete(id, site_id)
+  end
+
+  def delete(id, site_id) do
+    goal_query =
+      from(g in Goal,
+        where: g.id == ^id,
+        where: g.site_id == ^site_id
+      )
+
+    goal_query = on_ee(do: preload(goal_query, funnels: :steps), else: goal_query)
+
+    result =
+      Multi.new()
+      |> Multi.one(
+        :goal,
+        goal_query
+      )
+      |> Multi.run(:funnel_ids_to_wipe, fn
+        _, %{goal: nil} ->
+          {:error, :not_found}
+
+        _, %{goal: %{funnels: []}} ->
+          {:ok, []}
+
+        _, %{goal: %{funnels: funnels}} ->
+          funnels_to_wipe =
+            funnels
+            |> Enum.filter(&(Enum.count(&1.steps) == Funnel.Const.min_steps()))
+            |> Enum.map(& &1.id)
+
+          {:ok, funnels_to_wipe}
+      end)
+      |> Multi.merge(fn
+        %{funnel_ids_to_wipe: []} ->
+          Ecto.Multi.new()
+
+        %{funnel_ids_to_wipe: [_ | _] = funnel_ids} ->
+          Ecto.Multi.new()
+          |> Multi.delete_all(
+            :delete_funnels,
+            from(f in "funnels",
+              where: f.id in ^funnel_ids
+            )
+          )
+      end)
+      |> Multi.delete_all(
+        :delete_goals,
+        fn _ ->
+          from g in Goal,
+            where: g.id == ^id,
+            where: g.site_id == ^site_id
+        end
+      )
+      |> Repo.transaction()
+
+    case result do
+      {:ok, _} -> :ok
+      {:error, _step, reason, _context} -> {:error, reason}
+    end
+  end
+
+  @spec count(Plausible.Site.t()) :: non_neg_integer()
+  def count(site) do
+    Repo.aggregate(
+      from(
+        g in Goal,
+        where: g.site_id == ^site.id
+      ),
+      :count
+    )
+  end
+
+  @spec create_outbound_links(Plausible.Site.t()) :: :ok
+  def create_outbound_links(%Plausible.Site{} = site) do
+    create(site, %{"event_name" => "Outbound Link: Click"}, do_upsert())
+    :ok
+  end
+
+  @spec create_file_downloads(Plausible.Site.t()) :: :ok
+  def create_file_downloads(%Plausible.Site{} = site) do
+    create(site, %{"event_name" => "File Download"}, do_upsert())
+    :ok
+  end
+
+  @spec create_form_submissions(Plausible.Site.t()) :: :ok
+  def create_form_submissions(%Plausible.Site{} = site) do
+    create(site, %{"event_name" => "Form: Submission"}, do_upsert())
+    :ok
+  end
+
+  @spec create_404(Plausible.Site.t()) :: :ok
+  def create_404(%Plausible.Site{} = site) do
+    create(site, %{"event_name" => "404"}, do_upsert())
+    :ok
+  end
+
+  @spec delete_outbound_links(Plausible.Site.t()) :: :ok
+  def delete_outbound_links(%Plausible.Site{} = site) do
+    q =
+      from g in Goal,
+        where: g.site_id == ^site.id,
+        where: g.event_name == "Outbound Link: Click"
+
+    Repo.delete_all(q)
+    :ok
+  end
+
+  @spec delete_file_downloads(Plausible.Site.t()) :: :ok
+  def delete_file_downloads(%Plausible.Site{} = site) do
+    q =
+      from g in Goal,
+        where: g.site_id == ^site.id,
+        where: g.event_name == "File Download"
+
+    Repo.delete_all(q)
+    :ok
+  end
+
+  @spec delete_404(Plausible.Site.t()) :: :ok
+  def delete_404(%Plausible.Site{} = site) do
+    q =
+      from g in Goal,
+        where: g.site_id == ^site.id,
+        where: g.event_name == "404"
+
+    Repo.delete_all(q)
+    :ok
+  end
+
+  @spec delete_form_submissions(Plausible.Site.t()) :: :ok
+  def delete_form_submissions(%Plausible.Site{} = site) do
+    q =
+      from g in Goal,
+        where: g.site_id == ^site.id,
+        where: g.event_name == "Form: Submission"
+
+    Repo.delete_all(q)
+    :ok
+  end
+
+  defp insert_goal(site, params, opts) do
+    params = Map.delete(params, "site_id")
+
+    insert_opts =
+      if upsert?(opts) do
+        [on_conflict: :nothing]
+      else
+        []
+      end
+
+    changeset =
+      %Goal{}
+      |> Goal.changeset(params)
+      |> Ecto.Changeset.put_assoc(:site, site)
+
+    with :ok <- maybe_check_feature_access(site, changeset),
+         :ok <- check_no_currency_if_consolidated(site, changeset),
+         :ok <- check_goals_limit(site, changeset, opts),
+         {:ok, goal} <- Repo.insert(changeset, insert_opts) do
+      # Upsert with `on_conflict: :nothing` strategy
+      # will result in goal struct missing primary key field
+      # which is generated by the database.
+      if goal.id do
+        {:ok, :insert, goal}
+      else
+        get_params =
+          goal
+          |> Map.take([:site_id, :event_name, :page_path])
+          |> Enum.reject(fn {_, value} -> is_nil(value) end)
+
+        {:ok, :upsert, Repo.get_by!(Goal, get_params)}
+      end
+    end
+  end
+
+  defp maybe_check_feature_access(site, changeset) do
+    with :ok <- revenue_goals_access_check(site, changeset) do
+      custom_props_goals_access_check(site, changeset)
+    end
+  end
+
+  defp revenue_goals_access_check(site, changeset) do
+    if Changeset.get_field(changeset, :currency) do
+      site = Plausible.Repo.preload(site, :team)
+      Plausible.Billing.Feature.RevenueGoals.check_availability(site.team)
+    else
+      :ok
+    end
+  end
+
+  defp custom_props_goals_access_check(site, changeset) do
+    if map_size(Changeset.get_field(changeset, :custom_props)) > 0 do
+      site = Plausible.Repo.preload(site, :team)
+      Plausible.Billing.Feature.Props.check_availability(site.team)
+    else
+      :ok
+    end
+  end
+
+  defp check_goals_limit(site, changeset, opts) do
+    if upsert?(opts) and goal_exists_for_upsert?(site, changeset) do
+      :ok
+    else
+      if count(site) >= max_goals_per_site(opts) and changeset.valid? do
+        changeset
+        |> Changeset.add_error(:page_path, "Maximum number of goals reached")
+        |> Changeset.add_error(:event_name, "Maximum number of goals reached")
+        |> Changeset.apply_action(:insert)
+      else
+        :ok
+      end
+    end
+  end
+
+  defp check_no_currency_if_consolidated(site, changeset) do
+    if Plausible.Sites.consolidated?(site) && Changeset.get_field(changeset, :currency) do
+      {:error, :revenue_goals_unavailable}
+    else
+      :ok
+    end
+  end
+
+  defp maybe_trim(%Goal{} = goal) do
+    # we make sure that even if we saved goals erroneously with trailing
+    # space, it's removed during fetch
+    goal
+    |> Map.update!(:event_name, &maybe_trim/1)
+    |> Map.update!(:page_path, &maybe_trim/1)
+  end
+
+  defp maybe_trim(s) when is_binary(s) do
+    String.trim(s)
+  end
+
+  defp maybe_trim(other) do
+    other
+  end
+
+  defp upsert?(opts) do
+    Keyword.get(opts, :upsert?, false)
+  end
+
+  defp do_upsert(opts \\ []) do
+    Keyword.put(opts, :upsert?, true)
+  end
+
+  defp goal_exists_for_upsert?(site, changeset) do
+    event_name = Changeset.get_field(changeset, :event_name)
+    page_path = Changeset.get_field(changeset, :page_path)
+    scroll_threshold = Changeset.get_field(changeset, :scroll_threshold)
+
+    query_params =
+      [site_id: site.id]
+      |> maybe_add_filter(:event_name, event_name)
+      |> maybe_add_filter(:page_path, page_path)
+      |> maybe_add_filter(:scroll_threshold, scroll_threshold)
+
+    Repo.exists?(from(g in Goal, where: ^query_params))
+  end
+
+  defp maybe_add_filter(params, _key, nil), do: params
+  defp maybe_add_filter(params, key, value), do: Keyword.put(params, key, value)
+end

lib/plausible/google/api.ex

@@ -0,0 +1,201 @@
+defmodule Plausible.Google.API do
+  @moduledoc """
+  API to Google services.
+  """
+
+  alias Plausible.Google.HTTP
+  alias Plausible.Google.SearchConsole
+  alias Plausible.Stats.Query
+
+  @search_console_scope URI.encode_www_form(
+                          "email https://www.googleapis.com/auth/webmasters.readonly"
+                        )
+  @import_scope URI.encode_www_form("email https://www.googleapis.com/auth/analytics.readonly")
+
+  @verified_permission_levels ["siteOwner", "siteFullUser", "siteRestrictedUser"]
+
+  @valid_oauth_contexts ["search-console", "import"]
+
+  @oauth_salt "google-oauth-state"
+
+  def search_console_authorize_url(site) do
+    "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@search_console_scope}&state=" <>
+      sign_oauth_state(site, "search-console")
+  end
+
+  def import_authorize_url(site) do
+    "https://accounts.google.com/o/oauth2/v2/auth?client_id=#{client_id()}&redirect_uri=#{redirect_uri()}&prompt=consent&response_type=code&access_type=offline&scope=#{@import_scope}&state=" <>
+      sign_oauth_state(site, "import")
+  end
+
+  @doc false
+  def sign_oauth_state(site, context) do
+    Phoenix.Token.sign(PlausibleWeb.Endpoint, @oauth_salt, [site.id, context])
+  end
+
+  def verify_oauth_state(state) do
+    case Phoenix.Token.verify(PlausibleWeb.Endpoint, @oauth_salt, state, max_age: 3600) do
+      {:ok, [site_id, context]} when context in @valid_oauth_contexts ->
+        if site = Plausible.Repo.get(Plausible.Site, site_id) do
+          {:ok, %{site: site, context: context}}
+        else
+          {:error, {:invalid_oauth_state, :missing_site}}
+        end
+
+      {:error, token_error} ->
+        {:error, {:invalid_oauth_state, token_error}}
+    end
+  end
+
+  def fetch_access_token!(code) do
+    HTTP.fetch_access_token!(code)
+  end
+
+  def list_properties(access_token) do
+    Plausible.Google.GA4.API.list_properties(access_token)
+  end
+
+  def get_property(access_token, property) do
+    Plausible.Google.GA4.API.get_property(access_token, property)
+  end
+
+  def get_analytics_start_date(access_token, property) do
+    Plausible.Google.GA4.API.get_analytics_start_date(access_token, property)
+  end
+
+  def get_analytics_end_date(access_token, property) do
+    Plausible.Google.GA4.API.get_analytics_end_date(access_token, property)
+  end
+
+  def fetch_verified_properties(auth) do
+    with {:ok, access_token} <- maybe_refresh_token(auth),
+         {:ok, sites} <- Plausible.Google.HTTP.list_sites(access_token) do
+      sites
+      |> Map.get("siteEntry", [])
+      |> Enum.filter(fn site -> site["permissionLevel"] in @verified_permission_levels end)
+      |> Enum.map(fn site -> site["siteUrl"] end)
+      |> Enum.map(fn url -> String.trim_trailing(url, "/") end)
+      |> then(&{:ok, &1})
+    end
+  end
+
+  def fetch_stats(site, query, pagination, search) do
+    with {:ok, site} <- ensure_search_console_property(site),
+         {:ok, access_token} <- maybe_refresh_token(site.google_auth),
+         {:ok, gsc_filters} <-
+           SearchConsole.Filters.transform(site.google_auth.property, query.filters, search),
+         {:ok, stats} <-
+           HTTP.list_stats(
+             access_token,
+             site.google_auth.property,
+             Query.date_range(query),
+             pagination,
+             gsc_filters
+           ) do
+      stats
+      |> Map.get("rows", [])
+      |> Enum.map(&search_console_row/1)
+      |> then(&{:ok, &1})
+    else
+      :google_property_not_configured -> {:error, :google_property_not_configured}
+      :unsupported_filters -> {:error, :unsupported_filters}
+      {:error, error} -> {:error, error}
+    end
+  end
+
+  def maybe_refresh_token(%Plausible.Site.GoogleAuth{} = auth) do
+    with true <- needs_to_refresh_token?(auth.expires),
+         {:ok, {new_access_token, expires_at}} <- do_refresh_token(auth.refresh_token),
+         changeset <-
+           Plausible.Site.GoogleAuth.changeset(auth, %{
+             access_token: new_access_token,
+             expires: expires_at
+           }),
+         {:ok, _google_auth} <- Plausible.Repo.update(changeset) do
+      {:ok, new_access_token}
+    else
+      false -> {:ok, auth.access_token}
+      {:error, cause} -> {:error, cause}
+    end
+  end
+
+  def maybe_refresh_token({access_token, refresh_token, expires_at}) do
+    with true <- needs_to_refresh_token?(expires_at),
+         {:ok, {new_access_token, _expires_at}} <- do_refresh_token(refresh_token) do
+      {:ok, new_access_token}
+    else
+      false -> {:ok, access_token}
+      {:error, cause} -> {:error, cause}
+    end
+  end
+
+  def property?(value), do: String.starts_with?(value, "properties/")
+
+  defp do_refresh_token(refresh_token) do
+    case HTTP.refresh_auth_token(refresh_token) do
+      {:ok, %{"access_token" => new_access_token, "expires_in" => expires_in}} ->
+        expires_at = NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in)
+        {:ok, {new_access_token, expires_at}}
+
+      {:error, cause} ->
+        {:error, cause}
+    end
+  end
+
+  defp needs_to_refresh_token?(expires_at) when is_binary(expires_at) do
+    expires_at
+    |> NaiveDateTime.from_iso8601!()
+    |> needs_to_refresh_token?()
+  end
+
+  defp needs_to_refresh_token?(%NaiveDateTime{} = expires_at) do
+    thirty_seconds_ago = DateTime.shift(DateTime.utc_now(), second: 30)
+    NaiveDateTime.before?(expires_at, thirty_seconds_ago)
+  end
+
+  defp ensure_search_console_property(site) do
+    site = Plausible.Repo.preload(site, :google_auth)
+
+    if site.google_auth && site.google_auth.property do
+      {:ok, site}
+    else
+      :google_property_not_configured
+    end
+  end
+
+  defp search_console_row(row) do
+    %{
+      # We always request just one dimension at a time (`query`)
+      name: row["keys"] |> List.first(),
+      visitors: row["clicks"],
+      impressions: row["impressions"],
+      ctr: rounded_ctr(row["ctr"]),
+      position: rounded_position(row["position"])
+    }
+  end
+
+  defp rounded_ctr(ctr) do
+    {:ok, decimal} = Decimal.cast(ctr)
+
+    decimal
+    |> Decimal.mult(100)
+    |> Decimal.round(1)
+    |> Decimal.to_float()
+  end
+
+  defp rounded_position(position) do
+    {:ok, decimal} = Decimal.cast(position)
+
+    decimal
+    |> Decimal.round(1)
+    |> Decimal.to_float()
+  end
+
+  defp client_id() do
+    Keyword.fetch!(Application.get_env(:plausible, :google), :client_id)
+  end
+
+  defp redirect_uri() do
+    PlausibleWeb.Endpoint.url() <> "/auth/google/callback"
+  end
+end

lib/plausible/google/ga4/api.ex

@@ -0,0 +1,215 @@
+defmodule Plausible.Google.GA4.API do
+  @moduledoc """
+  API for Google Analytics 4.
+  """
+
+  alias Plausible.Google
+  alias Plausible.Google.GA4
+
+  require Logger
+
+  @type import_auth :: {
+          access_token :: String.t(),
+          refresh_token :: String.t(),
+          expires_at :: String.t()
+        }
+
+  @per_page 200_000
+  @backoff_factor :timer.seconds(10)
+  @max_attempts 5
+
+  def list_properties(access_token) do
+    case GA4.HTTP.list_accounts_for_user(access_token) do
+      {:ok, %{"accountSummaries" => accounts}} ->
+        accounts =
+          accounts
+          |> Enum.filter(& &1["propertySummaries"])
+          |> Enum.map(fn account ->
+            %{"account" => account_id, "displayName" => account_name} = account
+
+            {"#{account_name} (#{account_id})",
+             Enum.map(account["propertySummaries"], fn property ->
+               %{"displayName" => property_name, "property" => property_id} = property
+
+               {"#{property_name} (#{property_id})", property_id}
+             end)}
+          end)
+
+        {:ok, accounts}
+
+      {:ok, _} ->
+        {:ok, []}
+
+      {:error, cause} ->
+        {:error, cause}
+    end
+  end
+
+  def get_property(access_token, lookup_property) do
+    case GA4.HTTP.get_property(access_token, lookup_property) do
+      {:ok, property} ->
+        %{"displayName" => property_name, "name" => property_id, "account" => account_id} =
+          property
+
+        {:ok,
+         %{
+           id: property_id,
+           name: "#{property_name} (#{property_id})",
+           account_id: account_id
+         }}
+
+      {:error, cause} ->
+        {:error, cause}
+    end
+  end
+
+  def get_analytics_start_date(access_token, property) do
+    GA4.HTTP.get_analytics_start_date(access_token, property)
+  end
+
+  def get_analytics_end_date(access_token, property) do
+    GA4.HTTP.get_analytics_end_date(access_token, property)
+  end
+
+  def import_analytics(date_range, property, auth, opts) do
+    persist_fn = Keyword.fetch!(opts, :persist_fn)
+    fetch_opts = Keyword.get(opts, :fetch_opts, [])
+    resume_opts = Keyword.get(opts, :resume_opts, [])
+
+    Logger.debug(
+      "[#{inspect(__MODULE__)}:#{property}] Starting import from #{date_range.first} to #{date_range.last}"
+    )
+
+    with {:ok, access_token} <- Google.API.maybe_refresh_token(auth) do
+      do_import_analytics(date_range, property, access_token, persist_fn, fetch_opts, resume_opts)
+    end
+  end
+
+  defp do_import_analytics(
+         date_range,
+         property,
+         access_token,
+         persist_fn,
+         fetch_opts,
+         [] = _resume_opts
+       ) do
+    Enum.reduce_while(GA4.ReportRequest.full_report(), :ok, fn report_request, :ok ->
+      Logger.debug(
+        "[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}"
+      )
+
+      report_request = prepare_request(report_request, date_range, property, access_token)
+
+      case fetch_and_persist(report_request, persist_fn: persist_fn, fetch_opts: fetch_opts) do
+        :ok -> {:cont, :ok}
+        {:error, _} = error -> {:halt, error}
+      end
+    end)
+  end
+
+  defp do_import_analytics(
+         date_range,
+         property,
+         access_token,
+         persist_fn,
+         fetch_opts,
+         resume_opts
+       ) do
+    dataset = Keyword.fetch!(resume_opts, :dataset)
+    offset = Keyword.fetch!(resume_opts, :offset)
+
+    GA4.ReportRequest.full_report()
+    |> Enum.drop_while(&(&1.dataset != dataset))
+    |> Enum.reduce_while(:ok, fn report_request, :ok ->
+      Logger.debug(
+        "[#{inspect(__MODULE__)}:#{property}] Starting to import #{report_request.dataset}"
+      )
+
+      request_offset =
+        if report_request.dataset == dataset do
+          offset
+        else
+          0
+        end
+
+      report_request =
+        report_request
+        |> prepare_request(date_range, property, access_token)
+        |> Map.put(:offset, request_offset)
+
+      case fetch_and_persist(report_request, persist_fn: persist_fn, fetch_opts: fetch_opts) do
+        :ok -> {:cont, :ok}
+        {:error, _} = error -> {:halt, error}
+      end
+    end)
+  end
+
+  @spec fetch_and_persist(GA4.ReportRequest.t(), Keyword.t()) ::
+          :ok | {:error, term()}
+  def fetch_and_persist(%GA4.ReportRequest{} = report_request, opts \\ []) do
+    persist_fn = Keyword.fetch!(opts, :persist_fn)
+    attempt = Keyword.get(opts, :attempt, 1)
+    fetch_opts = Keyword.get(opts, :fetch_opts, [])
+    max_attempts = Keyword.get(fetch_opts, :max_attempts, @max_attempts)
+    sleep_time = Keyword.get(fetch_opts, :sleep_time, @backoff_factor)
+
+    case GA4.HTTP.get_report(report_request) do
+      {:ok, {_, 0}} ->
+        Logger.debug(
+          "[#{inspect(__MODULE__)}:#{report_request.property}] Fetched empty response for #{report_request.dataset}"
+        )
+
+        :ok
+
+      {:ok, {rows, row_count}} ->
+        Logger.debug(
+          "[#{inspect(__MODULE__)}:#{report_request.property}] Fetched #{length(rows)} rows of total #{row_count} with offset #{report_request.offset} for #{report_request.dataset}"
+        )
+
+        :ok = persist_fn.(report_request.dataset, rows)
+
+        Logger.debug(
+          "[#{inspect(__MODULE__)}:#{report_request.property}] Persisted #{length(rows)} for #{report_request.dataset}"
+        )
+
+        if report_request.offset + @per_page < row_count do
+          fetch_and_persist(
+            %GA4.ReportRequest{report_request | offset: report_request.offset + @per_page},
+            opts
+          )
+        else
+          :ok
+        end
+
+      {:error, {:rate_limit_exceeded, details}} ->
+        {:error, {:rate_limit_exceeded, details}}
+
+      {:error, cause} ->
+        if attempt >= max_attempts do
+          Logger.debug(
+            "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Terminating."
+          )
+
+          {:error, cause}
+        else
+          Logger.debug(
+            "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}. Will retry."
+          )
+
+          Process.sleep(attempt * sleep_time)
+          fetch_and_persist(report_request, Keyword.merge(opts, attempt: attempt + 1))
+        end
+    end
+  end
+
+  defp prepare_request(%GA4.ReportRequest{} = report_request, date_range, property, access_token) do
+    %GA4.ReportRequest{
+      report_request
+      | date_range: date_range,
+        property: property,
+        access_token: access_token,
+        offset: 0,
+        limit: @per_page
+    }
+  end
+end

lib/plausible/google/ga4/http.ex

@@ -0,0 +1,330 @@
+defmodule Plausible.Google.GA4.HTTP do
+  @moduledoc """
+  HTTP client implementation for Google Analytics 4 API.
+  """
+
+  use Plausible
+
+  alias Plausible.HTTPClient
+
+  require Logger
+
+  @spec get_report(Plausible.Google.GA4.ReportRequest.t()) ::
+          {:ok, {[map()], non_neg_integer()}} | {:error, any()}
+  def get_report(%Plausible.Google.GA4.ReportRequest{} = report_request) do
+    params = %{
+      requests: [
+        %{
+          property: report_request.property,
+          dateRanges: [
+            %{
+              startDate: report_request.date_range.first,
+              endDate: report_request.date_range.last
+            }
+          ],
+          dimensions: Enum.map(report_request.dimensions, &%{name: &1}),
+          metrics: Enum.map(report_request.metrics, &build_metric/1),
+          orderBys: [
+            %{
+              dimension: %{
+                dimensionName: "date",
+                orderType: "ALPHANUMERIC"
+              },
+              desc: true
+            }
+          ],
+          dimensionFilter: report_request.dimension_filter,
+          limit: report_request.limit,
+          offset: report_request.offset
+        }
+      ]
+    }
+
+    url =
+      "#{reporting_api_url()}/v1beta/#{report_request.property}:batchRunReports"
+
+    response =
+      HTTPClient.impl().post(
+        url,
+        [{"Authorization", "Bearer #{report_request.access_token}"}],
+        params,
+        receive_timeout: 80_000
+      )
+
+    with {:ok, %{body: body}} <- response,
+         {:ok, report} <- parse_report_from_response(body),
+         row_count <- Map.get(report, "rowCount", 0),
+         {:ok, report} <- convert_to_maps(report) do
+      {:ok, {report, row_count}}
+    else
+      {:error, %{reason: %{status: 429, body: body}} = error} ->
+        log_ce_error("retrieving report for #{report_request.dataset}", error)
+
+        Logger.debug(
+          "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} due to exceeding rate limit."
+        )
+
+        Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: 429}})
+
+        {:error,
+         {:rate_limit_exceeded, dataset: report_request.dataset, offset: report_request.offset}}
+
+      {:error, %{reason: %{status: status, body: body}} = error} when status >= 500 ->
+        log_ce_error("retrieving report for #{report_request.dataset}", error)
+
+        Logger.warning(
+          "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}"
+        )
+
+        Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
+
+        {:error, {:server_failed, dataset: report_request.dataset, offset: report_request.offset}}
+
+      {:error, %{reason: %{status: status, body: body}} = error} ->
+        log_ce_error("retrieving report for #{report_request.dataset}", error)
+
+        Logger.warning(
+          "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset} with code #{status}: #{inspect(body)}"
+        )
+
+        Sentry.Context.set_extra_context(%{ga_response: %{body: body, status: status}})
+
+        {:error, :request_failed}
+
+      {:error, reason} ->
+        log_ce_error("retrieving report for #{report_request.dataset}", reason)
+
+        Logger.warning(
+          "[#{inspect(__MODULE__)}:#{report_request.property}] Request failed for #{report_request.dataset}: #{inspect(reason)}"
+        )
+
+        Sentry.Context.set_extra_context(%{ga_response: %{body: inspect(reason), status: 0}})
+
+        {:error, {:socket_failed, dataset: report_request.dataset, offset: report_request.offset}}
+    end
+  end
+
+  defp build_metric(expression) do
+    case String.split(expression, " = ") do
+      [name, expression] ->
+        %{
+          name: name,
+          expression: expression
+        }
+
+      [name] ->
+        %{name: name}
+    end
+  end
+
+  defp parse_report_from_response(%{"reports" => [report | _]}) do
+    {:ok, report}
+  end
+
+  defp parse_report_from_response(body) do
+    Sentry.Context.set_extra_context(%{google_analytics4_response: body})
+
+    Logger.error(
+      "Google Analytics 4: Failed to find report in response. Reason: #{inspect(body)}"
+    )
+
+    {:error, {:invalid_response, body}}
+  end
+
+  defp convert_to_maps(%{
+         "rows" => rows,
+         "dimensionHeaders" => dimension_headers,
+         "metricHeaders" => metric_headers
+       })
+       when is_list(rows) do
+    dimension_headers = Enum.map(dimension_headers, & &1["name"])
+    metric_headers = Enum.map(metric_headers, & &1["name"])
+
+    report =
+      Enum.map(rows, fn %{"dimensionValues" => dimensions, "metricValues" => metrics} ->
+        dimension_values = Enum.map(dimensions, & &1["value"])
+        metric_values = Enum.map(metrics, & &1["value"])
+        metrics = Enum.zip(metric_headers, metric_values)
+        dimensions = Enum.zip(dimension_headers, dimension_values)
+        %{metrics: Map.new(metrics), dimensions: Map.new(dimensions)}
+      end)
+
+    {:ok, report}
+  end
+
+  defp convert_to_maps(%{"dimensionHeaders" => _, "metricHeaders" => _}) do
+    {:ok, []}
+  end
+
+  defp convert_to_maps(response) do
+    Logger.error(
+      "Google Analytics 4: Failed to read report in response. Reason: #{inspect(response)}"
+    )
+
+    Sentry.Context.set_extra_context(%{google_analytics4_response: response})
+    {:error, {:invalid_response, response}}
+  end
+
+  def list_accounts_for_user(access_token) do
+    url = "#{admin_api_url()}/v1beta/accountSummaries?pageSize=200"
+
+    headers = [{"Authorization", "Bearer #{access_token}"}]
+
+    case HTTPClient.impl().get(url, headers) do
+      {:ok, %Finch.Response{body: body, status: 200}} ->
+        {:ok, body}
+
+      {:error, %HTTPClient.Non200Error{reason: %{status: 429}} = error} ->
+        log_ce_error("listing accounts for user", error)
+        {:error, :rate_limit_exceeded}
+
+      {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
+        log_ce_error("listing accounts for user", error)
+        {:error, authentication_failed(error)}
+
+      {:error, %{reason: :timeout} = error} ->
+        log_ce_error("listing accounts for user", error)
+        {:error, :timeout}
+
+      {:error, error} ->
+        log_ce_error("listing accounts for user", error)
+        Sentry.capture_message("Error listing GA4 accounts for user", extra: %{error: error})
+        {:error, :unknown}
+    end
+  end
+
+  def get_property(access_token, property) do
+    url = "#{admin_api_url()}/v1beta/#{property}"
+
+    headers = [{"Authorization", "Bearer #{access_token}"}]
+
+    case HTTPClient.impl().get(url, headers) do
+      {:ok, %Finch.Response{body: body, status: 200}} ->
+        {:ok, body}
+
+      {:error, %HTTPClient.Non200Error{reason: %{status: 429}} = error} ->
+        log_ce_error("retrieving property #{property}", error)
+        {:error, :rate_limit_exceeded}
+
+      {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
+        log_ce_error("retrieving property #{property}", error)
+        {:error, authentication_failed(error)}
+
+      {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [404] ->
+        log_ce_error("retrieving property #{property}", error)
+        {:error, :not_found}
+
+      {:error, %{reason: :timeout} = error} ->
+        log_ce_error("retrieving property #{property}", error)
+        {:error, :timeout}
+
+      {:error, error} ->
+        log_ce_error("retrieving property #{property}", error)
+
+        Sentry.capture_message("Error retrieving GA4 property #{property}",
+          extra: %{error: error}
+        )
+
+        {:error, :unknown}
+    end
+  end
+
+  @earliest_valid_date "2015-08-14"
+
+  def get_analytics_start_date(access_token, property) do
+    get_analytics_boundary_date(access_token, property, :start)
+  end
+
+  def get_analytics_end_date(access_token, property) do
+    get_analytics_boundary_date(access_token, property, :end)
+  end
+
+  defp get_analytics_boundary_date(access_token, property, edge) do
+    descending? = edge == :end
+
+    params = %{
+      requests: [
+        %{
+          property: "#{property}",
+          dateRanges: [
+            %{startDate: @earliest_valid_date, endDate: Date.to_iso8601(Date.utc_today())}
+          ],
+          dimensions: [%{name: "date"}],
+          metrics: [%{name: "screenPageViews"}],
+          orderBys: [
+            %{dimension: %{dimensionName: "date", orderType: "ALPHANUMERIC"}, desc: descending?}
+          ],
+          limit: 1
+        }
+      ]
+    }
+
+    url = "#{reporting_api_url()}/v1beta/#{property}:batchRunReports"
+    headers = [{"Authorization", "Bearer #{access_token}"}]
+
+    case HTTPClient.impl().post(url, headers, params) do
+      {:ok, %Finch.Response{body: body, status: 200}} ->
+        report = List.first(body["reports"])
+
+        date =
+          case report["rows"] do
+            [%{"dimensionValues" => [%{"value" => date_str}]}] ->
+              Plausible.Times.parse!(date_str, "%Y%m%d", :strftime) |> NaiveDateTime.to_date()
+
+            _ ->
+              nil
+          end
+
+        {:ok, date}
+
+      {:error, %HTTPClient.Non200Error{reason: %{status: 429}} = error} ->
+        log_ce_error("retrieving #{edge} date", error)
+        {:error, :rate_limit_exceeded}
+
+      {:error, %HTTPClient.Non200Error{} = error} when error.reason.status in [401, 403] ->
+        log_ce_error("retrieving #{edge} date", error)
+        {:error, authentication_failed(error)}
+
+      {:error, %{reason: :timeout} = error} ->
+        log_ce_error("retrieving #{edge} date", error)
+        {:error, :timeout}
+
+      {:error, error} ->
+        log_ce_error("retrieving #{edge} date", error)
+
+        Sentry.capture_message("Error retrieving GA4 #{edge} date",
+          extra: %{error: error}
+        )
+
+        {:error, :unknown}
+    end
+  end
+
+  defp reporting_api_url, do: "https://analyticsdata.googleapis.com"
+  defp admin_api_url, do: "https://analyticsadmin.googleapis.com"
+
+  @spec authentication_failed(HTTPClient.Non200Error.t()) ::
+          {:authentication_failed, String.t() | nil}
+  defp authentication_failed(error) do
+    message =
+      case error.reason.body do
+        %{"error" => %{"message" => message}} when is_binary(message) -> message
+        _ -> nil
+      end
+
+    {:authentication_failed, message}
+  end
+
+  @spec log_ce_error(String.t(), any) :: :ok
+  defp log_ce_error(action, error)
+
+  on_ce do
+    defp log_ce_error(action, error) do
+      Logger.error("Google Analytics 4: Failed when #{action}. Reason: #{inspect(error)}")
+    end
+  end
+
+  on_ee do
+    defp log_ce_error(_action, _error), do: :ok
+  end
+end

lib/plausible/google/ga4/report_request.ex

@@ -0,0 +1,168 @@
+defmodule Plausible.Google.GA4.ReportRequest do
+  @moduledoc """
+  Report request struct for Google Analytics 4 API
+  """
+
+  @excluded_event_names [
+    "page_view",
+    "session_start",
+    "first_visit",
+    "user_engagement"
+  ]
+
+  defstruct [
+    :dataset,
+    :dimensions,
+    :metrics,
+    :date_range,
+    :property,
+    :access_token,
+    :offset,
+    :dimension_filter,
+    :limit
+  ]
+
+  @type t() :: %__MODULE__{
+          dataset: String.t(),
+          dimensions: [String.t()],
+          metrics: [String.t()],
+          date_range: Date.Range.t(),
+          property: term(),
+          access_token: String.t(),
+          offset: non_neg_integer(),
+          limit: non_neg_integer()
+        }
+
+  def full_report do
+    [
+      %__MODULE__{
+        dataset: "imported_visitors",
+        dimensions: ["date"],
+        metrics: [
+          "totalUsers",
+          "screenPageViews",
+          "bounces = sessions - engagedSessions",
+          "sessions",
+          "userEngagementDuration"
+        ]
+      },
+      %__MODULE__{
+        dataset: "imported_sources",
+        dimensions: [
+          "date",
+          "sessionSource",
+          "sessionDefaultChannelGroup",
+          "sessionMedium",
+          "sessionCampaignName",
+          "sessionManualAdContent",
+          "sessionGoogleAdsKeyword"
+        ],
+        metrics: [
+          "screenPageViews",
+          "totalUsers",
+          "sessions",
+          "bounces = sessions - engagedSessions",
+          "userEngagementDuration"
+        ]
+      },
+      %__MODULE__{
+        dataset: "imported_pages",
+        dimensions: ["date", "hostName", "pagePath"],
+        # NOTE: no exits as GA4 DATA API does not provide that metric
+        metrics: [
+          "totalUsers",
+          "activeUsers",
+          "screenPageViews",
+          "sessions",
+          "userEngagementDuration"
+        ]
+      },
+      %__MODULE__{
+        dataset: "imported_entry_pages",
+        dimensions: ["date", "landingPage"],
+        metrics: [
+          "screenPageViews",
+          "totalUsers",
+          "sessions",
+          "userEngagementDuration",
+          "bounces = sessions - engagedSessions"
+        ]
+      },
+      # NOTE: Skipping for now as there's no dimension directly mapping to exit page path
+      # %__MODULE__{
+      #   dataset: "imported_exit_pages",
+      #   dimensions: ["date", "ga:exitPagePath"],
+      #   metrics: [
+      #     "totalUsers",
+      #     "sessions",
+      #     "screenPageViews",
+      #     "userEngagementDuration",
+      #     "bounces = sessions - engagedSessions"
+      #   ]
+      # },
+      %__MODULE__{
+        dataset: "imported_custom_events",
+        dimensions: ["date", "eventName", "linkUrl"],
+        metrics: [
+          "totalUsers",
+          "eventCount"
+        ],
+        dimension_filter: %{
+          "notExpression" => %{
+            "filter" => %{
+              "fieldName" => "eventName",
+              "inListFilter" => %{
+                "values" => @excluded_event_names,
+                "caseSensitive" => true
+              }
+            }
+          }
+        }
+      },
+      %__MODULE__{
+        dataset: "imported_locations",
+        dimensions: ["date", "countryId", "region", "city"],
+        metrics: [
+          "screenPageViews",
+          "totalUsers",
+          "sessions",
+          "bounces = sessions - engagedSessions",
+          "userEngagementDuration"
+        ]
+      },
+      %__MODULE__{
+        dataset: "imported_devices",
+        dimensions: ["date", "deviceCategory"],
+        metrics: [
+          "screenPageViews",
+          "totalUsers",
+          "sessions",
+          "bounces = sessions - engagedSessions",
+          "userEngagementDuration"
+        ]
+      },
+      %__MODULE__{
+        dataset: "imported_browsers",
+        dimensions: ["date", "browser"],
+        metrics: [
+          "screenPageViews",
+          "totalUsers",
+          "sessions",
+          "bounces = sessions - engagedSessions",
+          "userEngagementDuration"
+        ]
+      },
+      %__MODULE__{
+        dataset: "imported_operating_systems",
+        dimensions: ["date", "operatingSystem", "operatingSystemVersion"],
+        metrics: [
+          "screenPageViews",
+          "totalUsers",
+          "sessions",
+          "bounces = sessions - engagedSessions",
+          "userEngagementDuration"
+        ]
+      }
+    ]
+  end
+end

lib/plausible/google/http.ex

@@ -0,0 +1,110 @@
+defmodule Plausible.Google.HTTP do
+  @moduledoc """
+  HTTP client implementation for Google APIs.
+  """
+
+  require Logger
+  alias Plausible.HTTPClient
+
+  def list_sites(access_token) do
+    url = "#{api_url()}/webmasters/v3/sites"
+    headers = [{"Content-Type", "application/json"}, {"Authorization", "Bearer #{access_token}"}]
+
+    case HTTPClient.impl().get(url, headers) do
+      {:ok, %{body: body}} ->
+        {:ok, body}
+
+      {:error, %{reason: %{status: s}}} when s in [401, 403] ->
+        {:error, "google_auth_error"}
+
+      {:error, %{reason: %{body: %{"error" => error}}}} ->
+        {:error, error}
+
+      {:error, reason} ->
+        Logger.error("Google Analytics: failed to list sites: #{inspect(reason)}")
+        {:error, "failed_to_list_sites"}
+    end
+  end
+
+  def fetch_access_token!(code) do
+    url = "#{api_url()}/oauth2/v4/token"
+    headers = [{"Content-Type", "application/x-www-form-urlencoded"}]
+
+    params = %{
+      client_id: client_id(),
+      client_secret: client_secret(),
+      code: code,
+      grant_type: :authorization_code,
+      redirect_uri: redirect_uri()
+    }
+
+    {:ok, response} = HTTPClient.impl().post(url, headers, params)
+
+    response.body
+  end
+
+  def list_stats(access_token, property, date_range, pagination, search_console_filters) do
+    {limit, page} = pagination
+
+    params = %{
+      startDate: Date.to_iso8601(date_range.first),
+      endDate: Date.to_iso8601(date_range.last),
+      dimensions: ["query"],
+      rowLimit: limit,
+      startRow: page * limit,
+      dimensionFilterGroups: search_console_filters
+    }
+
+    url =
+      "#{api_url()}/webmasters/v3/sites/#{URI.encode_www_form(property)}/searchAnalytics/query"
+
+    headers = [{"Authorization", "Bearer #{access_token}"}]
+
+    case HTTPClient.impl().post(url, headers, params) do
+      {:ok, %Finch.Response{body: body, status: 200}} ->
+        {:ok, body}
+
+      {:error, %{reason: %Finch.Response{body: _body, status: status}}}
+      when status in [401, 403] ->
+        {:error, "google_auth_error"}
+
+      {:error, %{reason: %{body: %{"error" => error}}}} ->
+        {:error, error}
+
+      {:error, reason} ->
+        Logger.error("Google Search Console: failed to list stats: #{inspect(reason)}")
+        {:error, "failed_to_list_stats"}
+    end
+  end
+
+  def refresh_auth_token(refresh_token) do
+    url = "#{api_url()}/oauth2/v4/token"
+    headers = [{"content-type", "application/x-www-form-urlencoded"}]
+
+    params = %{
+      client_id: client_id(),
+      client_secret: client_secret(),
+      refresh_token: refresh_token,
+      grant_type: :refresh_token,
+      redirect_uri: redirect_uri()
+    }
+
+    case HTTPClient.impl().post(url, headers, params) do
+      {:ok, %Finch.Response{body: body, status: 200}} ->
+        {:ok, body}
+
+      {:error, %{reason: %Finch.Response{body: %{"error" => error}, status: _non_http_200}}} ->
+        {:error, error}
+
+      {:error, %{reason: _} = e} ->
+        Sentry.capture_message("Error fetching Google queries", extra: %{error: inspect(e)})
+        {:error, :unknown_error}
+    end
+  end
+
+  defp config, do: Application.get_env(:plausible, :google)
+  defp client_id, do: Keyword.fetch!(config(), :client_id)
+  defp client_secret, do: Keyword.fetch!(config(), :client_secret)
+  defp api_url, do: Keyword.fetch!(config(), :api_url)
+  defp redirect_uri, do: PlausibleWeb.Endpoint.url() <> "/auth/google/callback"
+end

lib/plausible/google/search_console/filters.ex

@@ -0,0 +1,77 @@
+defmodule Plausible.Google.SearchConsole.Filters do
+  @moduledoc false
+  import Plausible.Stats.Filters.Utils, only: [page_regex: 1]
+
+  def transform(property, plausible_filters, search) do
+    gsc_filters =
+      Enum.reduce_while(plausible_filters, [], fn plausible_filter, gsc_filters ->
+        case transform_filter(property, plausible_filter) do
+          :unsupported -> {:halt, :unsupported_filters}
+          :ignore -> {:cont, gsc_filters}
+          gsc_filter -> {:cont, [gsc_filter | gsc_filters]}
+        end
+      end)
+      |> maybe_add_search_filter(search)
+
+    case gsc_filters do
+      :unsupported_filters -> :unsupported_filters
+      [] -> {:ok, []}
+      filters when is_list(filters) -> {:ok, [%{filters: filters}]}
+    end
+  end
+
+  defp transform_filter(property, [op, "event:page" | rest]) do
+    transform_filter(property, [op, "visit:entry_page" | rest])
+  end
+
+  # :TODO: Should also work case-insensitive, if not, blacklist.
+  defp transform_filter(property, [:is, "visit:entry_page", pages | _]) when is_list(pages) do
+    expression =
+      Enum.map_join(pages, "|", fn page -> property_url(property, Regex.escape(page)) end)
+
+    %{dimension: "page", operator: "includingRegex", expression: expression}
+  end
+
+  defp transform_filter(property, [:matches_wildcard, "visit:entry_page", pages | _])
+       when is_list(pages) do
+    expression =
+      Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end)
+
+    %{dimension: "page", operator: "includingRegex", expression: expression}
+  end
+
+  defp transform_filter(_property, [:is, "visit:screen", devices | _]) when is_list(devices) do
+    expression = Enum.map_join(devices, "|", &search_console_device/1)
+    %{dimension: "device", operator: "includingRegex", expression: expression}
+  end
+
+  defp transform_filter(_property, [:is, "visit:country", countries | _])
+       when is_list(countries) do
+    expression = Enum.map_join(countries, "|", &search_console_country/1)
+    %{dimension: "country", operator: "includingRegex", expression: expression}
+  end
+
+  defp transform_filter(_, [_, "visit:source" | _rest]), do: :ignore
+
+  defp transform_filter(_, [_, "visit:channel" | _rest]), do: :ignore
+
+  defp transform_filter(_, _filter), do: :unsupported
+
+  defp property_url("sc-domain:" <> domain, page), do: "https://" <> domain <> page
+  defp property_url(url, page), do: url <> page
+
+  defp search_console_device("Desktop"), do: "DESKTOP"
+  defp search_console_device("Mobile"), do: "MOBILE"
+  defp search_console_device("Tablet"), do: "TABLET"
+
+  defp search_console_country(alpha_2) do
+    country = Location.Country.get_country(alpha_2)
+    country.alpha_3
+  end
+
+  defp maybe_add_search_filter(gsc_filters, search) when byte_size(search) > 0 do
+    [%{operator: "includingRegex", expression: search, dimension: "query"} | gsc_filters]
+  end
+
+  defp maybe_add_search_filter(gsc_filters, _search), do: gsc_filters
+end

lib/plausible/helpers/changeset.ex

@@ -0,0 +1,34 @@
+defmodule Plausible.ChangesetHelpers do
+  @moduledoc "Helper function for working with Ecto changesets"
+
+  def traverse_errors(changeset) do
+    Ecto.Changeset.traverse_errors(changeset, fn {msg, opts} ->
+      Regex.replace(~r"%{(\w+)}", msg, fn _, key ->
+        opts
+        |> Keyword.get(String.to_existing_atom(key), key)
+        |> to_string()
+      end)
+    end)
+  end
+
+  @doc """
+  iex> serialize_first_error([{"name", {"should be at most %{count} byte(s)", [count: 255]}}])
+  "name should be at most 255 byte(s)"
+  """
+  def serialize_first_error(errors) do
+    {field, {message, opts}} = List.first(errors)
+
+    formatted_message =
+      Enum.reduce(opts, message, fn {key, value}, acc ->
+        placeholder = "%{#{key}}"
+
+        if String.contains?(acc, placeholder) do
+          String.replace(acc, placeholder, to_string(value))
+        else
+          acc
+        end
+      end)
+
+    "#{field} #{formatted_message}"
+  end
+end

lib/plausible/helpers/config.ex

@@ -0,0 +1,53 @@
+defmodule Plausible.ConfigHelpers do
+  def get_var_from_path_or_env(config_dir, var_name, default \\ nil) do
+    var_path = Path.join(config_dir, var_name)
+
+    if File.exists?(var_path) do
+      File.read!(var_path) |> String.trim()
+    else
+      System.get_env(var_name, default)
+    end
+  end
+
+  def get_int_from_path_or_env(config_dir, var_name, default \\ nil) do
+    var = get_var_from_path_or_env(config_dir, var_name)
+
+    case var do
+      nil ->
+        default
+
+      var ->
+        case Integer.parse(var) do
+          {int, ""} -> int
+          _ -> raise "Config variable #{var_name} must be an integer. Got #{var}"
+        end
+    end
+  end
+
+  def get_bool_from_path_or_env(config_dir, var_name, default \\ nil) do
+    case get_var_from_path_or_env(config_dir, var_name) do
+      nil -> default
+      var -> parse_bool(var)
+    end
+  end
+
+  @var_true ["1", "t", "true", "y", "yes", "on"]
+  @var_false ["0", "f", "false", "n", "no", "off"]
+  @var_bool_message Enum.zip_with(@var_true, @var_false, fn t, f -> [t, f] end)
+                    |> List.flatten()
+                    |> Enum.join(", ")
+
+  defp parse_bool(var) do
+    case String.downcase(var) do
+      t when t in @var_true ->
+        true
+
+      f when f in @var_false ->
+        false
+
+      _ ->
+        raise ArgumentError,
+              "Invalid boolean value: #{inspect(var)}. Expected one of: " <> @var_bool_message
+    end
+  end
+end

lib/plausible/helpers/json.ex

@@ -0,0 +1,15 @@
+defmodule Plausible.Helpers.JSON do
+  @moduledoc """
+  Common helpers for JSON handling
+  """
+
+  def decode_or_fallback(raw) do
+    with raw when is_binary(raw) <- raw,
+         {:ok, %{} = decoded} <- Jason.decode(raw) do
+      decoded
+    else
+      already_a_map when is_map(already_a_map) -> already_a_map
+      _any -> %{}
+    end
+  end
+end

lib/plausible/http_client.ex

@@ -0,0 +1,189 @@
+defmodule Plausible.HTTPClient.Non200Error do
+  @moduledoc false
+
+  defstruct reason: nil
+
+  @type t :: %__MODULE__{reason: Finch.Response.t()}
+
+  @spec new(Finch.Response.t()) :: t()
+  def new(%Finch.Response{status: status} = response)
+      when is_integer(status) and (status < 200 or status >= 300) do
+    %__MODULE__{reason: response}
+  end
+end
+
+defmodule Plausible.HTTPClient.Interface do
+  @moduledoc false
+
+  @type t() :: module()
+
+  @type finch_request_opts() :: Keyword.t()
+  @type url() :: Finch.Request.url()
+  @type headers() :: Finch.Request.headers()
+  @type params() :: Finch.Request.body() | map()
+  @type response() ::
+          {:ok, Finch.Response.t()}
+          | {:error, Finch.error() | Plausible.HTTPClient.Non200Error.t()}
+
+  @callback get(url(), headers(), params()) :: response()
+  @callback get(url(), headers()) :: response()
+  @callback get(url()) :: response()
+  @callback post(url(), headers(), params()) :: response()
+  @callback post(url(), headers(), params(), finch_request_opts()) :: response()
+end
+
+defmodule Plausible.HTTPClient do
+  @moduledoc """
+  HTTP Client built on top of Finch.
+
+  By default, request parameters are json-encoded.
+
+  If a raw binary value is supplied, no encoding is performed.
+  If x-www-form-urlencoded content-type is set in headers,
+  URL encoding is invoked.
+  """
+
+  require OpenTelemetry.Tracer
+  alias Plausible.HTTPClient.Non200Error
+
+  @doc """
+  Make a POST request
+  """
+  @behaviour Plausible.HTTPClient.Interface
+
+  @impl Plausible.HTTPClient.Interface
+  def post(url, headers \\ [], params \\ nil, finch_req_opts \\ []) do
+    call(:post, url, headers, params, finch_req_opts)
+  end
+
+  @doc """
+  Make a GET request
+  """
+  @impl Plausible.HTTPClient.Interface
+  def get(url, headers \\ [], params \\ nil) do
+    call(:get, url, headers, params)
+  end
+
+  @spec impl() :: Plausible.HTTPClient.Interface.t()
+  def impl() do
+    Application.get_env(:plausible, :http_impl, __MODULE__)
+  end
+
+  defp call(method, url, headers, params, finch_req_opts \\ []) do
+    OpenTelemetry.Tracer.with_span "http_client.request" do
+      {params, headers} = maybe_encode_params(params, headers)
+
+      method
+      |> build_request(url, headers, params)
+      |> trace_request()
+      |> do_request(finch_req_opts)
+      |> maybe_decode_body()
+      |> tag_error()
+      |> trace_response()
+    end
+  end
+
+  defp build_request(method, url, headers, params) do
+    Finch.build(method, url, headers, params)
+  end
+
+  defp do_request(request, finch_req_opts) do
+    Finch.request(request, Plausible.Finch, finch_req_opts)
+  end
+
+  defp maybe_encode_params(params, headers) when is_binary(params) or is_nil(params) do
+    {params, headers}
+  end
+
+  defp maybe_encode_params(params, headers) when is_map(params) do
+    content_type =
+      Enum.find_value(headers, "", fn {k, v} ->
+        if String.downcase(k) == "content-type" do
+          v
+        end
+      end)
+
+    case String.downcase(content_type) do
+      "application/x-www-form-urlencoded" ->
+        {URI.encode_query(params), headers}
+
+      "application/json" ->
+        {Jason.encode!(params), headers}
+
+      _ ->
+        {Jason.encode!(params), [{"content-type", "application/json"} | headers]}
+    end
+  end
+
+  defp tag_error({:ok, %Finch.Response{status: status}} = ok)
+       when is_integer(status) and status >= 200 and status < 300 do
+    ok
+  end
+
+  defp tag_error({:ok, %Finch.Response{status: _} = response}) do
+    {:error, Non200Error.new(response)}
+  end
+
+  defp tag_error({:error, _} = error) do
+    error
+  end
+
+  defp maybe_decode_body({:ok, %{headers: headers, body: body} = resp})
+       when is_binary(body) and body != "" do
+    if json?(headers) do
+      {:ok, update_in(resp.body, &Jason.decode!/1)}
+    else
+      {:ok, resp}
+    end
+  end
+
+  defp maybe_decode_body(response), do: response
+
+  defp json?(headers) do
+    found =
+      Enum.find(headers, fn
+        {"content-type", "application/json" <> _} ->
+          true
+
+        _ ->
+          false
+      end)
+
+    is_tuple(found)
+  end
+
+  defp trace_request(%Finch.Request{} = request) do
+    OpenTelemetry.Tracer.set_attributes([
+      {"http_client.request.host", request.host},
+      {"http_client.request.method", request.method},
+      {"http_client.request.path", request.path}
+    ])
+
+    request
+  end
+
+  defp trace_response(response) do
+    case response do
+      {:ok, %{status: status}} ->
+        OpenTelemetry.Tracer.set_attributes([
+          {"http_client.response.status", status}
+        ])
+
+      {:error, %Non200Error{reason: %{status: status}}} ->
+        OpenTelemetry.Tracer.set_attributes([
+          {"http_client.request.failure", "non_200"},
+          {"http_client.response.status", status}
+        ])
+
+      {:error, exception} when is_exception(exception) ->
+        OpenTelemetry.Tracer.set_attributes([
+          {"http_client.request.failure", Exception.message(exception)}
+        ])
+
+      _any ->
+        :skip
+    end
+
+    response
+  end
+end

lib/plausible/import_deletion_repo.ex

@@ -0,0 +1,17 @@
+defmodule Plausible.ImportDeletionRepo do
+  @moduledoc """
+  A dedicated repo for import related mutations
+  """
+
+  use Ecto.Repo,
+    otp_app: :plausible,
+    adapter: Ecto.Adapters.ClickHouse
+
+  defmacro __using__(_) do
+    quote do
+      alias Plausible.ImportDeletionRepo
+      import Ecto
+      import Ecto.Query, only: [from: 1, from: 2]
+    end
+  end
+end

lib/plausible/imported.ex

@@ -0,0 +1,212 @@
+defmodule Plausible.Imported do
+  @moduledoc """
+  Context for managing site statistics imports.
+
+  For list of currently supported import sources see `Plausible.Imported.ImportSources`.
+
+  For more information on implementing importers, see `Plausible.Imported.Importer`.
+  """
+
+  import Ecto.Query
+
+  alias Plausible.{Site, Repo, Imported}
+  alias Plausible.Imported.SiteImport
+  alias Plausible.Stats.Query
+
+  require Plausible.Imported.SiteImport
+
+  @tables [
+    Imported.Visitor,
+    Imported.Source,
+    Imported.Page,
+    Imported.EntryPage,
+    Imported.ExitPage,
+    Imported.CustomEvent,
+    Imported.Location,
+    Imported.Device,
+    Imported.Browser,
+    Imported.OperatingSystem
+  ]
+
+  @table_names Enum.map(@tables, & &1.__schema__(:source))
+  # Maximum number of complete imports to account for when querying stats
+  @max_complete_imports 5
+
+  @spec schemas() :: [module()]
+  def schemas, do: @tables
+
+  @spec tables() :: [String.t()]
+  def tables, do: @table_names
+
+  @spec max_complete_imports() :: non_neg_integer()
+  def max_complete_imports() do
+    @max_complete_imports
+  end
+
+  @spec imported_custom_props() :: [String.t()]
+  def imported_custom_props do
+    # NOTE: Keep up to date with `Plausible.Props.internal_keys/1`,
+    # but _ignore_ unsupported keys. Currently, `search_query` is
+    # not supported in imported queries.
+    Enum.map(~w(url path), &("event:props:" <> &1))
+  end
+
+  @spec any_completed_imports?(Site.t()) :: boolean()
+  def any_completed_imports?(site) do
+    get_completed_imports(site) != []
+  end
+
+  @spec earliest_import_start_date(Site.t()) :: Date.t() | nil
+  def earliest_import_start_date(site) do
+    site
+    |> get_completed_imports()
+    |> Enum.map(& &1.start_date)
+    |> Enum.min(Date, fn -> nil end)
+  end
+
+  @spec complete_import_ids(Site.t()) :: [non_neg_integer()]
+  def complete_import_ids(site) do
+    imports = get_completed_imports(site)
+    has_legacy? = Enum.any?(imports, fn %{legacy: legacy?} -> legacy? end)
+    ids = Enum.map(imports, fn %{id: id} -> id end)
+
+    # account for legacy imports as well
+    if has_legacy? do
+      [0 | ids]
+    else
+      ids
+    end
+  end
+
+  @spec completed_imports_in_query_range(Site.t(), Query.t()) :: [SiteImport.t()]
+  def completed_imports_in_query_range(%Site{} = site, %Query{} = query) do
+    date_range = Query.date_range(query)
+
+    site
+    |> get_completed_imports()
+    |> Enum.reject(fn site_import ->
+      Date.after?(site_import.start_date, date_range.last) or
+        Date.before?(site_import.end_date, date_range.first)
+    end)
+  end
+
+  @spec get_import(Site.t(), non_neg_integer()) :: SiteImport.t() | nil
+  def get_import(site, import_id) do
+    Repo.get_by(SiteImport, id: import_id, site_id: site.id)
+  end
+
+  defdelegate listen(), to: Imported.Importer
+
+  @spec list_all_imports(Site.t(), atom()) :: [SiteImport.t()]
+  def list_all_imports(site, status \\ nil) do
+    from(i in SiteImport, where: i.site_id == ^site.id, order_by: [desc: i.inserted_at])
+    |> maybe_filter_by_status(status)
+    |> Repo.all()
+  end
+
+  @spec other_imports_in_progress?(SiteImport.t()) :: boolean()
+  def other_imports_in_progress?(site_import) do
+    Repo.exists?(
+      from(i in SiteImport,
+        where: i.site_id == ^site_import.site_id and i.id != ^site_import.id,
+        where: i.status in ^[SiteImport.pending(), SiteImport.importing()]
+      )
+    )
+  end
+
+  defp maybe_filter_by_status(query, nil), do: query
+
+  defp maybe_filter_by_status(query, status) do
+    where(query, [i], i.status == ^status)
+  end
+
+  @spec delete_imports_for_site(Site.t()) :: :ok
+  def delete_imports_for_site(site) do
+    Repo.delete_all(from i in SiteImport, where: i.site_id == ^site.id)
+
+    :ok
+  end
+
+  @spec clamp_dates(Site.t(), Date.t(), Date.t()) ::
+          {:ok, Date.t(), Date.t()} | {:error, :no_time_window}
+  def clamp_dates(site, start_date, end_date) do
+    cutoff_date = get_cutoff_date(site)
+    occupied_ranges = get_occupied_date_ranges(site)
+
+    clamp_dates(occupied_ranges, cutoff_date, start_date, end_date)
+  end
+
+  @spec clamp_dates([Date.Range.t()], Date.t(), Date.t(), Date.t()) ::
+          {:ok, Date.t(), Date.t()} | {:error, :no_time_window}
+  def clamp_dates(occupied_ranges, cutoff_date, start_date, end_date) do
+    end_date = Enum.min([end_date, cutoff_date], Date)
+
+    with true <- Date.diff(end_date, start_date) >= 2,
+         [_ | _] = free_ranges <- find_free_ranges(start_date, end_date, occupied_ranges) do
+      longest = Enum.max_by(free_ranges, &Date.diff(&1.last, &1.first))
+      {:ok, longest.first, longest.last}
+    else
+      _ -> {:error, :no_time_window}
+    end
+  end
+
+  @spec get_occupied_date_ranges(Site.t()) :: [Date.Range.t()]
+  def get_occupied_date_ranges(site) do
+    site
+    |> Imported.list_all_imports(Imported.SiteImport.completed())
+    |> Enum.reject(&(Date.diff(&1.end_date, &1.start_date) < 2))
+    |> Enum.map(&Date.range(&1.start_date, &1.end_date))
+    |> Enum.sort_by(& &1.first, Date)
+  end
+
+  @spec get_cutoff_date(Site.t()) :: Date.t()
+  def get_cutoff_date(site) do
+    Plausible.Sites.native_stats_start_date(site) ||
+      DateTime.to_date(DateTime.now!(site.timezone))
+  end
+
+  defp find_free_ranges(start_date, end_date, occupied_ranges) do
+    Date.range(start_date, end_date)
+    |> free_ranges(start_date, occupied_ranges, [])
+  end
+
+  # This function recursively finds open ranges that are not yet occupied
+  # by existing imported data. The idea is that we keep moving a dynamic
+  # date index `d` from start until the end of `imported_range`, hopping
+  # over each occupied range, and capturing the open ranges step-by-step
+  # in the `result` array.
+  defp free_ranges(import_range, d, [occupied_range | rest_of_occupied_ranges], result) do
+    cond do
+      Date.diff(occupied_range.last, d) <= 0 ->
+        free_ranges(import_range, d, rest_of_occupied_ranges, result)
+
+      in_range?(d, occupied_range) || Date.diff(occupied_range.first, d) < 2 ->
+        d = occupied_range.last
+        free_ranges(import_range, d, rest_of_occupied_ranges, result)
+
+      true ->
+        free_range = Date.range(d, occupied_range.first)
+        result = result ++ [free_range]
+        d = occupied_range.last
+        free_ranges(import_range, d, rest_of_occupied_ranges, result)
+    end
+  end
+
+  defp free_ranges(import_range, d, [], result) do
+    if Date.diff(import_range.last, d) < 2 do
+      result
+    else
+      result ++ [Date.range(d, import_range.last)]
+    end
+  end
+
+  defp in_range?(date, range) do
+    Date.before?(range.first, date) && Date.after?(range.last, date)
+  end
+
+  defp get_completed_imports(site) do
+    site
+    |> Repo.preload(:completed_imports)
+    |> Map.fetch!(:completed_imports)
+  end
+end

lib/plausible/imported/browser.ex

@@ -0,0 +1,18 @@
+defmodule Plausible.Imported.Browser do
+  @moduledoc false
+  use Ecto.Schema
+
+  @primary_key false
+  schema "imported_browsers" do
+    field :site_id, Ch, type: "UInt64"
+    field :import_id, Ch, type: "UInt64"
+    field :date, :date
+    field :browser, :string
+    field :browser_version, :string
+    field :visitors, Ch, type: "UInt64"
+    field :visits, Ch, type: "UInt64"
+    field :visit_duration, Ch, type: "UInt64"
+    field :pageviews, Ch, type: "UInt64"
+    field :bounces, Ch, type: "UInt32"
+  end
+end

lib/plausible/imported/buffer.ex

@@ -0,0 +1,115 @@
+defmodule Plausible.Imported.Buffer do
+  @moduledoc """
+  This GenServer inserts records into Clickhouse `imported_*` tables. Multiple buffers are
+  automatically created for each table. Records are flushed when the table buffer reaches the
+  maximum size, defined by `max_buffer_size/0`.
+  """
+
+  use GenServer
+  require Logger
+
+  def start_link(opts \\ []) do
+    GenServer.start_link(__MODULE__, opts)
+  end
+
+  def init(opts) do
+    flush_interval = Keyword.get(opts, :flush_interval_ms, 1000)
+    {:ok, %{flush_interval: flush_interval, buffers: %{}}}
+  end
+
+  @spec insert_many(pid(), term(), [map()]) :: :ok
+  @doc """
+  Puts the given records into the table buffer.
+  """
+  def insert_many(pid, table_name, records) do
+    GenServer.call(pid, {:insert_many, table_name, records})
+  end
+
+  @spec size(pid(), term()) :: non_neg_integer()
+  @doc """
+  Returns the total count of items in the given table buffer.
+  """
+  def size(pid, table_name) do
+    GenServer.call(pid, {:get_size, table_name})
+  end
+
+  @spec flush(pid()) :: :ok
+  @doc """
+  Flushes all table buffers to Clickhouse.
+  """
+  def flush(pid, timeout \\ :infinity) do
+    GenServer.call(pid, :flush_all_buffers, timeout)
+  end
+
+  def stop(pid) do
+    GenServer.stop(pid)
+  end
+
+  def handle_call({:get_size, table_name}, _from, %{buffers: buffers} = state) do
+    size =
+      buffers
+      |> Map.get(table_name, [])
+      |> length()
+
+    {:reply, size, state}
+  end
+
+  def handle_call({:insert_many, table_name, records}, _from, %{buffers: buffers} = state) do
+    Logger.notice("Import: Adding #{length(records)} to #{table_name} buffer")
+
+    new_buffer = Map.get(buffers, table_name, []) ++ records
+    new_state = put_in(state.buffers[table_name], new_buffer)
+
+    if length(new_buffer) >= max_buffer_size() do
+      {:reply, :ok, new_state, {:continue, {:flush, table_name}}}
+    else
+      {:reply, :ok, new_state}
+    end
+  end
+
+  def handle_call(:flush_all_buffers, _from, state) do
+    Enum.each(state.buffers, fn {table_name, records} ->
+      flush_buffer(records, table_name, state.flush_interval)
+    end)
+
+    {:reply, :ok, put_in(state.buffers, %{})}
+  end
+
+  def handle_continue({:flush, table_name}, state) do
+    flush_buffer(state.buffers[table_name], table_name, state.flush_interval)
+    {:noreply, put_in(state.buffers[table_name], [])}
+  end
+
+  defp max_buffer_size do
+    :plausible
+    |> Application.fetch_env!(:imported)
+    |> Keyword.fetch!(:max_buffer_size)
+  end
+
+  defp flush_buffer(records, table_name, flush_interval) do
+    # Clickhouse does not recommend sending more than 1 INSERT operation per second, and this
+    # sleep call slows down the flushing
+    Process.sleep(flush_interval)
+
+    Logger.notice("Import: Flushing #{length(records)} from #{table_name} buffer")
+    insert_all(table_name, records)
+  end
+
+  # used in tests `setup`
+  @doc false
+  def insert_all(table_name, records) do
+    schema = table_schema(table_name)
+    Plausible.IngestRepo.insert_all(schema, records)
+  end
+
+  defp table_schema("imported_visitors"), do: Plausible.Imported.Visitor
+  defp table_schema("imported_sources"), do: Plausible.Imported.Source
+  defp table_schema("imported_pages"), do: Plausible.Imported.Page
+  defp table_schema("imported_entry_pages"), do: Plausible.Imported.EntryPage
+  defp table_schema("imported_exit_pages"), do: Plausible.Imported.ExitPage
+  defp table_schema("imported_custom_events"), do: Plausible.Imported.CustomEvent
+  defp table_schema("imported_locations"), do: Plausible.Imported.Location
+  defp table_schema("imported_devices"), do: Plausible.Imported.Device
+  defp table_schema("imported_browsers"), do: Plausible.Imported.Browser
+  defp table_schema("imported_operating_systems"), do: Plausible.Imported.OperatingSystem
+end

lib/plausible/imported/csv_importer.ex

@@ -0,0 +1,372 @@
+defmodule Plausible.Imported.CSVImporter do
+  @moduledoc """
+  CSV importer from either S3 for which it uses ClickHouse [s3 table function](https://clickhouse.com/docs/en/sql-reference/table-functions/s3)
+  or from local storage for which it uses [input function.](https://clickhouse.com/docs/en/sql-reference/table-functions/input)
+  """
+
+  use Plausible.Imported.Importer
+  import Ecto.Query, only: [from: 2]
+
+  @impl true
+  def name(), do: :csv
+
+  @impl true
+  def label(), do: "CSV"
+
+  @impl true
+  def email_template(), do: "csv_import.html"
+
+  @impl true
+  def parse_args(%{"uploads" => uploads, "storage" => storage}) do
+    uploads = Enum.reject(uploads, &String.starts_with?(&1["filename"], "imported_custom_props_"))
+
+    [uploads: uploads, storage: storage]
+  end
+
+  @impl true
+  def import_data(site_import, opts) do
+    storage = Keyword.fetch!(opts, :storage)
+    uploads = Keyword.fetch!(opts, :uploads)
+
+    if storage == "local" do
+      # we need to remove the imported files from local storage
+      # after the importer has completed or ran out of attempts
+      paths = Enum.map(uploads, &Map.fetch!(&1, "local_path"))
+
+      Oban.insert!(
+        Plausible.Workers.LocalImportAnalyticsCleaner.new(
+          %{"import_id" => site_import.id, "paths" => paths},
+          schedule_in: _one_hour = 3600
+        )
+      )
+    end
+
+    {:ok, ch} =
+      Plausible.IngestRepo.config()
+      |> Keyword.replace!(:pool_size, 1)
+      |> Ch.start_link()
+
+    case storage do
+      "s3" -> import_s3(ch, site_import, uploads)
+      "local" -> import_local(ch, site_import, uploads)
+    end
+  rescue
+    # we are cancelling on any argument or ClickHouse errors, assuming they are permanent
+    e in [ArgumentError, Ch.Error] ->
+      # see Plausible.Imported.Importer for more details on transient vs permanent errors
+      {:error, Exception.message(e)}
+  end
+
+  def on_success(site_import, _extra_data) do
+    has_scroll_depth? =
+      Plausible.ClickhouseRepo.exists?(
+        from(i in "imported_pages",
+          where: i.site_id == ^site_import.site_id,
+          where: i.import_id == ^site_import.id,
+          where: i.total_scroll_depth_visits > 0,
+          select: 1
+        )
+      )
+
+    if has_scroll_depth? do
+      site_import
+      |> Ecto.Changeset.change(%{has_scroll_depth: true})
+      |> Plausible.Repo.update!()
+    end
+
+    :ok
+  end
+
+  defp import_s3(ch, site_import, uploads) do
+    %{
+      id: import_id,
+      site_id: site_id,
+      start_date: start_date,
+      end_date: end_date
+    } = site_import
+
+    %{access_key_id: s3_access_key_id, secret_access_key: s3_secret_access_key} =
+      Plausible.S3.import_clickhouse_credentials()
+
+    Enum.each(uploads, fn upload ->
+      %{"filename" => filename, "s3_url" => s3_url} = upload
+
+      {table, _, _} = parse_filename!(filename)
+      s3_structure = input_structure!(table)
+      s3_columns = input_columns!(table)
+
+      statement =
+        """
+        INSERT INTO {table:Identifier}(site_id,import_id,#{s3_columns}) \
+        SELECT {site_id:UInt64}, {import_id:UInt64}, #{s3_columns} \
+        FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String}) \
+        WHERE date >= {start_date:Date} AND date <= {end_date:Date}\
+        """
+
+      params =
+        %{
+          "table" => table,
+          "site_id" => site_id,
+          "import_id" => import_id,
+          "s3_url" => s3_url,
+          "s3_access_key_id" => s3_access_key_id,
+          "s3_secret_access_key" => s3_secret_access_key,
+          "s3_format" => "CSVWithNames",
+          "s3_structure" => s3_structure,
+          "start_date" => start_date,
+          "end_date" => end_date
+        }
+
+      Ch.query!(ch, statement, params, timeout: :infinity)
+    end)
+  end
+
+  defp import_local(ch, site_import, uploads) do
+    %{
+      id: import_id,
+      site_id: site_id,
+      start_date: start_date,
+      end_date: end_date
+    } = site_import
+
+    DBConnection.run(
+      ch,
+      fn conn ->
+        Enum.each(uploads, fn upload ->
+          %{"filename" => filename, "local_path" => local_path} = upload
+
+          {table, _, _} = parse_filename!(filename)
+          input_structure = input_structure!(table)
+          input_columns = input_columns!(table)
+
+          statement =
+            """
+            INSERT INTO {table:Identifier}(site_id,import_id,#{input_columns}) \
+            SELECT {site_id:UInt64}, {import_id:UInt64}, #{input_columns} \
+            FROM input({input_structure:String}) \
+            WHERE date >= {start_date:Date} AND date <= {end_date:Date} \
+            FORMAT CSVWithNames
+            """
+
+          params = %{
+            "table" => table,
+            "site_id" => site_id,
+            "import_id" => import_id,
+            "input_structure" => input_structure,
+            "start_date" => start_date,
+            "end_date" => end_date
+          }
+
+          # we are reading in 512KB chunks for better performance
+          # the default would've been line by line (not great for a CSV)
+          File.stream!(local_path, 512_000)
+          |> Stream.into(Ch.stream(conn, statement, params))
+          |> Stream.run()
+        end)
+      end,
+      timeout: :infinity
+    )
+  end
+
+  input_structures = %{
+    "imported_browsers" =>
+      "date Date, browser String, browser_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
+    "imported_devices" =>
+      "date Date, device String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
+    "imported_entry_pages" =>
+      "date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
+    "imported_exit_pages" =>
+      "date Date, exit_page String, visitors UInt64, visit_duration UInt64, exits UInt64, bounces UInt32, pageviews UInt64",
+    "imported_custom_events" =>
+      "date Date, name String, link_url String, path String, visitors UInt64, events UInt64",
+    "imported_locations" =>
+      "date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
+    "imported_operating_systems" =>
+      "date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
+    "imported_pages" =>
+      "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, total_scroll_depth UInt64, total_scroll_depth_visits UInt64, total_time_on_page UInt64, total_time_on_page_visits UInt64",
+    "imported_sources" =>
+      "date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
+    "imported_visitors" =>
+      "date Date, visitors UInt64, pageviews UInt64, bounces UInt64, visits UInt64, visit_duration UInt64"
+  }
+
+  @doc """
+  Extracts min/max date range from a list of uploads.
+
+  Examples:
+
+      iex> date_range([
+      ...>   %{"filename" => "imported_devices_20190101_20210101.csv"},
+      ...>   "pages_20200101_20220101.csv"
+      ...> ])
+      Date.range(~D[2019-01-01], ~D[2022-01-01])
+
+      iex> date_range([])
+      nil
+
+  """
+  @spec date_range([String.t() | %{String.t() => String.t()}, ...]) :: Date.Range.t() | nil
+  def date_range([_ | _] = uploads), do: date_range(uploads, _start_date = nil, _end_date = nil)
+  def date_range([]), do: nil
+
+  defp date_range(
+         [%{"filename" => "imported_custom_props_" <> _} | uploads],
+         prev_start_date,
+         prev_end_date
+       ) do
+    date_range(uploads, prev_start_date, prev_end_date)
+  end
+
+  defp date_range([upload | uploads], prev_start_date, prev_end_date) do
+    filename =
+      case upload do
+        %{"filename" => filename} -> filename
+        filename when is_binary(filename) -> filename
+      end
+
+    {_table, start_date, end_date} = parse_filename!(filename)
+
+    start_date =
+      if prev_start_date do
+        Enum.min([start_date, prev_start_date], Date)
+      else
+        start_date
+      end
+
+    end_date =
+      if prev_end_date do
+        Enum.max([end_date, prev_end_date], Date)
+      else
+        end_date
+      end
+
+    date_range(uploads, start_date, end_date)
+  end
+
+  defp date_range([], first, last), do: Date.range(first, last)
+
+  @spec parse_date!(String.t()) :: Date.t()
+  defp parse_date!(<<yyyy::bytes-4, mm::bytes-2, dd::bytes-2>>) do
+    Date.from_iso8601!("#{yyyy}-#{mm}-#{dd}")
+  end
+
+  @doc """
+  Extracts table name and min/max dates from the filename.
+
+  Examples:
+
+      iex> parse_filename!("my_data.csv")
+      ** (ArgumentError) invalid filename
+
+      iex> parse_filename!("imported_devices_00010101_20250101.csv")
+      {"imported_devices", ~D[0001-01-01], ~D[2025-01-01]}
+
+      iex> parse_filename!("devices_00010101_20250101.csv")
+      {"imported_devices", ~D[0001-01-01], ~D[2025-01-01]}
+
+  """
+  @spec parse_filename!(String.t()) ::
+          {table :: String.t(), start_date :: Date.t(), end_date :: Date.t()}
+  def parse_filename!(filename)
+
+  for {table, input_structure} <- input_structures do
+    defp input_structure!(unquote(table)), do: unquote(input_structure)
+
+    input_columns =
+      input_structure
+      |> String.split(",", trim: true)
+      |> Enum.map_join(",", fn kv ->
+        [col, _type] = String.split(kv)
+        String.trim(col)
+      end)
+
+    defp input_columns!(unquote(table)), do: unquote(input_columns)
+
+    def parse_filename!(
+          <<unquote(table)::bytes, ?_, start_date::8-bytes, ?_, end_date::8-bytes, ".csv">>
+        ) do
+      {unquote(table), parse_date!(start_date), parse_date!(end_date)}
+    end
+
+    "imported_" <> name = table
+
+    def parse_filename!(
+          <<unquote(name)::bytes, ?_, start_date::8-bytes, ?_, end_date::8-bytes, ".csv">>
+        ) do
+      {unquote(table), parse_date!(start_date), parse_date!(end_date)}
+    end
+  end
+
+  def parse_filename!(_filename) do
+    raise ArgumentError, "invalid filename"
+  end
+
+  @doc """
+  Checks if the provided filename conforms to the expected format.
+
+  Examples:
+
+      iex> valid_filename?("my_data.csv")
+      false
+
+      iex> valid_filename?("imported_devices_00010101_20250101.csv")
+      true
+
+      iex> valid_filename?("devices_00010101_20250101.csv")
+      true
+
+  """
+  @spec valid_filename?(String.t()) :: boolean
+  def valid_filename?(filename) do
+    try do
+      parse_filename!(filename)
+    else
+      _ -> true
+    rescue
+      _ -> false
+    end
+  end
+
+  @doc """
+  Extracts the table name from the provided filename.
+
+  Raises if the filename doesn't conform to the expected format.
+
+  Examples:
+
+      iex> extract_table("my_data.csv")
+      ** (ArgumentError) invalid filename
+
+      iex> extract_table("imported_devices_00010101_20250101.csv")
+      "imported_devices"
+
+      iex> extract_table("devices_00010101_20250101.csv")
+      "imported_devices"
+
+  """
+  @spec extract_table(String.t()) :: String.t()
+  def extract_table(filename) do
+    {table, _start_date, _end_date} = parse_filename!(filename)
+    table
+  end
+
+  @doc """
+  Returns local directory for CSV imports storage.
+
+  Builds upon `$DATA_DIR`, `$PERSISTENT_CACHE_DIR` or `$DEFAULT_DATA_DIR` (if set) and falls back to /tmp.
+
+  `$DEFAULT_DATA_DIR` is set to `/var/lib/plausible` in container images.
+
+  Examples:
+
+      iex> local_dir = local_dir(_site_id = 37)
+      iex> String.ends_with?(local_dir, "/plausible-imports/37")
+      true
+
+  """
+  def local_dir(site_id) do
+    data_dir = Application.get_env(:plausible, :data_dir)
+    Path.join([data_dir || System.tmp_dir!(), "plausible-imports", Integer.to_string(site_id)])
+  end
+end

lib/plausible/imported/custom_event.ex

@@ -0,0 +1,16 @@
+defmodule Plausible.Imported.CustomEvent do
+  @moduledoc false
+  use Ecto.Schema
+
+  @primary_key false
+  schema "imported_custom_events" do
+    field :site_id, Ch, type: "UInt64"
+    field :import_id, Ch, type: "UInt64"
+    field :date, :date
+    field :name, :string
+    field :link_url, :string
+    field :path, :string
+    field :visitors, Ch, type: "UInt64"
+    field :events, Ch, type: "UInt64"
+  end
+end

lib/plausible/imported/device.ex

@@ -0,0 +1,17 @@
+defmodule Plausible.Imported.Device do
+  @moduledoc false
+  use Ecto.Schema
+
+  @primary_key false
+  schema "imported_devices" do
+    field :site_id, Ch, type: "UInt64"
+    field :import_id, Ch, type: "UInt64"
+    field :date, :date
+    field :device, :string
+    field :visitors, Ch, type: "UInt64"
+    field :visits, Ch, type: "UInt64"
+    field :visit_duration, Ch, type: "UInt64"
+    field :pageviews, Ch, type: "UInt64"
+    field :bounces, Ch, type: "UInt32"
+  end
+end

lib/plausible/imported/entry_page.ex

@@ -0,0 +1,17 @@
+defmodule Plausible.Imported.EntryPage do
+  @moduledoc false
+  use Ecto.Schema
+
+  @primary_key false
+  schema "imported_entry_pages" do
+    field :site_id, Ch, type: "UInt64"
+    field :import_id, Ch, type: "UInt64"
+    field :date, :date
+    field :entry_page, :string
+    field :visitors, Ch, type: "UInt64"
+    field :entrances, Ch, type: "UInt64"
+    field :visit_duration, Ch, type: "UInt64"
+    field :pageviews, Ch, type: "UInt64"
+    field :bounces, Ch, type: "UInt32"
+  end
+end

lib/plausible/imported/exit_page.ex

@@ -0,0 +1,17 @@
+defmodule Plausible.Imported.ExitPage do
+  @moduledoc false
+  use Ecto.Schema
+
+  @primary_key false
+  schema "imported_exit_pages" do
+    field :site_id, Ch, type: "UInt64"
+    field :import_id, Ch, type: "UInt64"
+    field :date, :date
+    field :exit_page, :string
+    field :exits, Ch, type: "UInt64"
+    field :visitors, Ch, type: "UInt64"
+    field :visit_duration, Ch, type: "UInt64"
+    field :pageviews, Ch, type: "UInt64"
+    field :bounces, Ch, type: "UInt32"
+  end
+end

lib/plausible/imported/google_analytics4.ex

@@ -0,0 +1,357 @@
+defmodule Plausible.Imported.GoogleAnalytics4 do
+  @moduledoc """
+  Import implementation for Google Analytics 4.
+  """
+
+  use Plausible.Imported.Importer
+
+  alias Plausible.Imported
+  alias Plausible.Repo
+
+  @recoverable_errors [:rate_limit_exceeded, :socket_failed, :server_failed]
+  @missing_values ["(none)", "(not set)", "(not provided)", "(other)"]
+
+  @impl true
+  def name(), do: :google_analytics_4
+
+  @impl true
+  def label(), do: "Google Analytics 4"
+
+  @impl true
+  def email_template(), do: "google_analytics_import.html"
+
+  @impl true
+  def before_start(site_import, opts) do
+    site_import = Repo.preload(site_import, :site)
+
+    if import_id = Keyword.get(opts, :resume_from_import_id) do
+      if existing_site_import = Imported.get_import(site_import.site, import_id) do
+        Repo.delete!(site_import)
+        {:ok, existing_site_import}
+      else
+        # NOTE: shouldn't happen under normal circumsatnces
+        {:error, {:no_import_to_resume, import_id}}
+      end
+    else
+      {:ok, site_import}
+    end
+  end
+
+  @impl true
+  def parse_args(%{"resume_from_dataset" => dataset, "resume_from_offset" => offset} = args) do
+    args
+    |> Map.drop(["resume_from_dataset", "resume_from_offset"])
+    |> parse_args()
+    |> Keyword.put(:dataset, dataset)
+    |> Keyword.put(:offset, offset)
+  end
+
+  def parse_args(
+        %{"property" => property, "start_date" => start_date, "end_date" => end_date} = args
+      ) do
+    start_date = Date.from_iso8601!(start_date)
+    end_date = Date.from_iso8601!(end_date)
+    date_range = Date.range(start_date, end_date)
+
+    auth = {
+      Map.fetch!(args, "access_token"),
+      Map.fetch!(args, "refresh_token"),
+      Map.fetch!(args, "token_expires_at")
+    }
+
+    [
+      property: property,
+      date_range: date_range,
+      auth: auth
+    ]
+  end
+
+  @doc """
+  Imports stats from a Google Analytics 4 property to a Plausible site.
+
+  This function fetches Google Analytics 4 reports which are then passed in batches
+  to Clickhouse by the `Plausible.Imported.Buffer` process.
+  """
+  @impl true
+  def import_data(site_import, opts) do
+    date_range = Keyword.fetch!(opts, :date_range)
+    property = Keyword.fetch!(opts, :property)
+    auth = Keyword.fetch!(opts, :auth)
+    flush_interval_ms = Keyword.get(opts, :flush_interval_ms, 1000)
+
+    {:ok, buffer} = Plausible.Imported.Buffer.start_link(flush_interval_ms: flush_interval_ms)
+
+    persist_fn = fn table, rows ->
+      records = from_report(rows, site_import.site_id, site_import.id, table)
+      Plausible.Imported.Buffer.insert_many(buffer, table, records)
+    end
+
+    resume_opts = Keyword.take(opts, [:dataset, :offset])
+    fetch_opts = Keyword.get(opts, :fetch_opts, [])
+
+    try do
+      result =
+        Plausible.Google.GA4.API.import_analytics(date_range, property, auth,
+          persist_fn: persist_fn,
+          fetch_opts: fetch_opts,
+          resume_opts: resume_opts
+        )
+
+      case result do
+        {:error, {error, details}} when error in @recoverable_errors ->
+          site_import = Repo.preload(site_import, [:site, :imported_by])
+          dataset = Keyword.fetch!(details, :dataset)
+          offset = Keyword.fetch!(details, :offset)
+          {access_token, refresh_token, token_expires_at} = auth
+
+          resume_import_opts = [
+            property: property,
+            label: property,
+            start_date: date_range.first,
+            end_date: date_range.last,
+            access_token: access_token,
+            refresh_token: refresh_token,
+            token_expires_at: token_expires_at,
+            resume_from_import_id: site_import.id,
+            resume_from_dataset: dataset,
+            resume_from_offset: offset,
+            job_opts: [schedule_in: {65, :minutes}, unique: nil]
+          ]
+
+          new_import(
+            site_import.site,
+            site_import.imported_by,
+            resume_import_opts
+          )
+
+          {:error, error, skip_purge?: true, skip_mark_failed?: true}
+
+        other ->
+          other
+      end
+    after
+      Plausible.Imported.Buffer.flush(buffer)
+      Plausible.Imported.Buffer.stop(buffer)
+    end
+  end
+
+  def from_report(nil, _site_id, _import_id, _metric), do: nil
+
+  def from_report(data, site_id, import_id, table) do
+    Enum.reduce(data, [], fn row, acc ->
+      if Map.get(row.dimensions, "date") in @missing_values do
+        acc
+      else
+        [new_from_report(site_id, import_id, table, row) | acc]
+      end
+    end)
+  end
+
+  defp parse_number(nr) do
+    {float, ""} = Float.parse(nr)
+    round(float)
+  end
+
+  defp maybe_override_event_name("file_download"), do: "File Download"
+  defp maybe_override_event_name("click"), do: "Outbound Link: Click"
+  defp maybe_override_event_name(name), do: name
+
+  defp new_from_report(site_id, import_id, "imported_visitors", row) do
+    %{
+      site_id: site_id,
+      import_id: import_id,
+      date: get_date(row),
+      visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+      pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
+      bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
+      visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+      visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
+    }
+  end
+
+  defp new_from_report(site_id, import_id, "imported_sources", row) do
+    %{
+      site_id: site_id,
+      import_id: import_id,
+      date: get_date(row),
+      source: row.dimensions |> Map.fetch!("sessionSource") |> parse_source(),
+      # GA4 channels map 1-1 to Plausible channels
+      channel: row.dimensions |> Map.fetch!("sessionDefaultChannelGroup"),
+      referrer: nil,
+      # Only `source` exists in GA4 API
+      utm_source: nil,
+      utm_medium: row.dimensions |> Map.fetch!("sessionMedium") |> default_if_missing(),
+      utm_campaign: row.dimensions |> Map.fetch!("sessionCampaignName") |> default_if_missing(),
+      utm_content: row.dimensions |> Map.fetch!("sessionManualAdContent") |> default_if_missing(),
+      utm_term: row.dimensions |> Map.fetch!("sessionGoogleAdsKeyword") |> default_if_missing(),
+      visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+      visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+      pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
+      bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
+      visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
+    }
+  end
+
+  defp new_from_report(site_id, import_id, "imported_pages", row) do
+    %{
+      site_id: site_id,
+      import_id: import_id,
+      date: get_date(row),
+      hostname: row.dimensions |> Map.fetch!("hostName") |> String.replace_prefix("www.", ""),
+      page: row.dimensions |> Map.fetch!("pagePath") |> URI.parse() |> Map.get(:path),
+      visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+      visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+      pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
+      # NOTE: no exits metric in GA4 API currently
+      exits: 0,
+      total_time_on_page: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
+      total_time_on_page_visits: row.metrics |> Map.fetch!("sessions") |> parse_number()
+    }
+  end
+
+  defp new_from_report(site_id, import_id, "imported_entry_pages", row) do
+    %{
+      site_id: site_id,
+      import_id: import_id,
+      date: get_date(row),
+      entry_page: row.dimensions |> Map.fetch!("landingPage"),
+      visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+      entrances: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+      visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
+      pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
+      bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
+    }
+  end
+
+  # NOTE: no exit pages metrics in GA4 API available for now
+  # defp new_from_report(site_id, import_id, "imported_exit_pages", row) do
+  #   %{
+  #     site_id: site_id,
+  #     import_id: import_id,
+  #     date: get_date(row),
+  #     exit_page: Map.fetch!(row.dimensions, "exitPage"),
+  #     visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+  #     exits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+  #     visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number(),
+  #     pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
+  #     bounces: row.metrics |> Map.fetch!("bounces") |> parse_number()
+  #   }
+  # end
+
+  defp new_from_report(site_id, import_id, "imported_custom_events", row) do
+    %{
+      site_id: site_id,
+      import_id: import_id,
+      date: get_date(row),
+      name: row.dimensions |> Map.fetch!("eventName") |> maybe_override_event_name(),
+      link_url: row.dimensions |> Map.fetch!("linkUrl"),
+      visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+      events: row.metrics |> Map.fetch!("eventCount") |> parse_number()
+    }
+  end
+
+  defp new_from_report(site_id, import_id, "imported_locations", row) do
+    country_code = row.dimensions |> Map.fetch!("countryId") |> default_if_missing("")
+    city_name = row.dimensions |> Map.fetch!("city") |> default_if_missing("")
+    city_data = Location.get_city(city_name, country_code)
+
+    %{
+      site_id: site_id,
+      import_id: import_id,
+      date: get_date(row),
+      country: country_code,
+      region: row.dimensions |> Map.fetch!("region") |> default_if_missing(""),
+      city: city_data && city_data.id,
+      visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+      visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+      pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
+      bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
+      visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
+    }
+  end
+
+  defp new_from_report(site_id, import_id, "imported_devices", row) do
+    %{
+      site_id: site_id,
+      import_id: import_id,
+      date: get_date(row),
+      device: row.dimensions |> Map.fetch!("deviceCategory") |> String.capitalize(),
+      visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+      visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+      pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
+      bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
+      visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
+    }
+  end
+
+  @browser_google_to_plausible %{
+    "User-Agent:Opera" => "Opera",
+    "Mozilla Compatible Agent" => "Mobile App",
+    "Android Webview" => "Mobile App",
+    "Android Browser" => "Mobile App",
+    "Safari (in-app)" => "Mobile App",
+    "User-Agent: Mozilla" => "Firefox",
+    "(not set)" => ""
+  }
+
+  defp new_from_report(site_id, import_id, "imported_browsers", row) do
+    browser = Map.fetch!(row.dimensions, "browser")
+
+    %{
+      site_id: site_id,
+      import_id: import_id,
+      date: get_date(row),
+      browser: Map.get(@browser_google_to_plausible, browser, browser),
+      # Does not exist in GA4 API
+      browser_version: nil,
+      visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+      visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+      pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
+      bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
+      visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
+    }
+  end
+
+  @os_google_to_plausible %{
+    "Macintosh" => "Mac",
+    "Linux" => "GNU/Linux",
+    "(not set)" => ""
+  }
+
+  defp new_from_report(site_id, import_id, "imported_operating_systems", row) do
+    os = Map.fetch!(row.dimensions, "operatingSystem")
+
+    %{
+      site_id: site_id,
+      import_id: import_id,
+      date: get_date(row),
+      operating_system: Map.get(@os_google_to_plausible, os, os),
+      operating_system_version: row.dimensions |> Map.fetch!("operatingSystemVersion"),
+      visitors: row.metrics |> Map.fetch!("totalUsers") |> parse_number(),
+      visits: row.metrics |> Map.fetch!("sessions") |> parse_number(),
+      pageviews: row.metrics |> Map.fetch!("screenPageViews") |> parse_number(),
+      bounces: row.metrics |> Map.fetch!("bounces") |> parse_number(),
+      visit_duration: row.metrics |> Map.fetch!("userEngagementDuration") |> parse_number()
+    }
+  end
+
+  defp get_date(%{dimensions: %{"date" => date}}) do
+    date
+    |> Plausible.Times.parse!("%Y%m%d", :strftime)
+    |> NaiveDateTime.to_date()
+  end
+
+  defp default_if_missing(value, default \\ nil)
+  defp default_if_missing(value, default) when value in @missing_values, do: default
+  defp default_if_missing(value, _default), do: value
+
+  defp parse_source(nil), do: nil
+  defp parse_source("(direct)"), do: nil
+  defp parse_source("google"), do: "Google"
+  defp parse_source("bing"), do: "Bing"
+  defp parse_source("duckduckgo"), do: "DuckDuckGo"
+
+  defp parse_source(ref) do
+    Plausible.Ingestion.Source.from_referrer("https://" <> ref)
+  end
+end

lib/plausible/imported/import_sources.ex

@@ -0,0 +1,24 @@
+defmodule Plausible.Imported.ImportSources do
+  @moduledoc """
+  Definitions of import sources.
+  """
+
+  @sources [
+    Plausible.Imported.GoogleAnalytics4,
+    Plausible.Imported.UniversalAnalytics,
+    Plausible.Imported.NoopImporter,
+    Plausible.Imported.CSVImporter
+  ]
+
+  @sources_map Map.new(@sources, &{&1.name(), &1})
+
+  @source_names Enum.map(@sources, & &1.name())
+
+  @spec names() :: [atom()]
+  def names(), do: @source_names
+
+  @spec by_name(atom()) :: module()
+  def by_name(name) do
+    Map.fetch!(@sources_map, name)
+  end
+end

lib/plausible/imported/importer.ex

@@ -0,0 +1,261 @@
+defmodule Plausible.Imported.Importer do
+  @moduledoc """
+  Behaviour that should be implemented for each import source.
+
+  All imports are executed as background jobs run via `Plausible.Workers.ImportAnalytics`
+  Oban worker. Each import source must define a module conforming `Importer` behaviour.
+
+  The callbacks that need to be implemented:
+
+  * `name/0` - Returns import source name as an atom. Example: `:universal_analytics`.
+  * `label/0` - Descriptive, display friendly name of the source.
+    Example: "Google Analytics".
+  * `email_template/0` - Name of the email template to use for notifications in
+    `PlausibleWeb.Email` (`import_success` and `import_failure`). The template
+    should have content customized for a particular source.
+  * `parse_args/1` - Receives Oban job arguments coming from `new_import/3`. Whatever
+    options were passed to `new_import/3` will be present in the input map with string
+    keys and values serialized to primitives. If, for instance `start_date: ~D[2024-01-03]`
+    is passed as an option, `parse_args/1` receives `%{..., "start_date" => "2024-01-03"}`.
+    The expectation is parsing the map values producing a keyword list of options to
+    pass to `import_data/2`.
+  * `import_data/2` - Receives site import struct and options produced by `parse_args/1`.
+    This is where all the import processing is done. The way the import is implemented
+    is entirely arbitrary except the requirement that the process as a whole must
+    by synchronous. The callback is expected to return either `:ok` or `{:ok, %{...}}`
+    on successful import or `{:error, ...}` on failure. The map in success tuple is
+    used for updating site import struct and is passed to `on_success/2` callback.
+    Please note that error tuple should be only returned on errors that can't be
+    recovered from. For transient errors, the import should throw an exception or
+    simply crash. The error tuple has an alternative `{error, reason, opts}` form,
+    where `opts` allow to skip purging imported data so far via `skip_purge?` flag
+    and skip marking the import as failed and notifying the user via `skip_mark_failed?`
+    flag. Both flags are booleans.
+  * `before_start/2` - Optional callback run right before scheduling import job. It's
+    expected to either return `{:ok, site_import}` for the import to proceed
+    or `{:error, ...}` tuple, which will be returned from `new_import/3` call.
+    The `site_import` can be altered or replaced at this stage. The second argument
+    are opts passed to `new_import/3`.
+  * `on_success/2` - Optional callback run once site import is completed. Receives map
+    returned from `import_data/2`. Expected to always return `:ok`.
+  * `on_failure/1` - Optional callback run when import job fails permanently.
+
+  All sources must be added to the list in `Plausible.Imported.ImportSources`.
+
+  In order to schedule a new import job using a given source, respective importer's
+  `new_import/3` function must be called. It accepts site, user who is doing the import
+  and any options necessary to carry out the import.
+
+  There's an expectation that `start_date` and `end_date` are provided either as options
+  passed to `new_import/3` or data in map returned from `import_data/2`. If these parameters
+  are not provided, the import will eventually crash. These parameters define time range
+  of imported data which is in turn used for efficient querying.
+
+  Logic running inside `import_data/2` is expected to populated all `imported_*` tables
+  in ClickHouse with `import_id` column set to site import's ID.
+
+  Managing any configuration or authentication prior to running import is outside of
+  scope of importer logic and is expected to be implemented separately.
+
+  ## Running import fully synchronously
+
+  In case it's necessary to run the whole import job fully synchronously, the
+  `Plausible.Workers.ImportAnalytics` worker sends an `Oban.Notifier` message
+  on completion, failure or transient failure of the import.
+
+  A basic usage scenario looks like this:
+
+  ```elixir
+  {:ok, job} = Plausible.Imported.NoopImporter.new_import(
+    site,
+    user,
+    start_date: ~D[2005-01-01],
+    end_date: Date.utc_today(),
+    # this option is necessary to setup the calling process as listener
+    listen?: true
+  )
+
+  import_id = job.args[:import_id]
+
+  receive do
+    {:notification, :analytics_imports_jobs, %{"event" => "complete", "import_id" => ^import_id}} ->
+      IO.puts("Job completed")
+
+    {:notification, :analytics_imports_jobs, %{"event" => "transient_fail", "import_id" => ^import_id}} ->
+      IO.puts("Job failed transiently")
+
+    {:notification, :analytics_imports_jobs, %{"event" => "fail", "import_id" => ^import_id}} ->
+      IO.puts("Job failed permanently")
+  after
+    15_000 ->
+      IO.puts("Job didn't finish in 15 seconds")
+  end
+  ```
+
+  In a more realistic scenario, job scheduling will be done inside a GenServer process
+  like LiveView, where notifications can be listened for via `handle_info/2`.
+  """
+
+  alias Plausible.Imported
+  alias Plausible.Imported.SiteImport
+  alias Plausible.Repo
+
+  @callback name() :: atom()
+  @callback label() :: String.t()
+  @callback email_template() :: String.t()
+  @callback parse_args(map()) :: Keyword.t()
+  @callback import_data(SiteImport.t(), Keyword.t()) ::
+              :ok | {:error, any()} | {:error, any(), Keyword.t()}
+  @callback before_start(SiteImport.t(), Keyword.t()) :: {:ok, SiteImport.t()} | {:error, any()}
+  @callback on_success(SiteImport.t(), map()) :: :ok
+  @callback on_failure(SiteImport.t()) :: :ok
+
+  defmacro __using__(_opts) do
+    quote do
+      @behaviour Plausible.Imported.Importer
+
+      @spec new_import(Plausible.Site.t(), Plausible.Auth.User.t(), Keyword.t()) ::
+              {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t() | :import_in_progress | any()}
+      def new_import(site, user, opts) do
+        Plausible.Imported.Importer.new_import(name(), site, user, opts, &before_start/2)
+      end
+
+      @doc false
+      @spec run_import(SiteImport.t(), map()) :: {:ok, SiteImport.t()} | {:error, :any}
+      def run_import(site_import, args) do
+        Plausible.Imported.Importer.run_import(
+          site_import,
+          args,
+          &parse_args/1,
+          &import_data/2,
+          &on_success/2
+        )
+      end
+
+      @doc false
+      @spec mark_failed(SiteImport.t()) :: SiteImport.t()
+      def mark_failed(site_import) do
+        site_import =
+          site_import
+          |> SiteImport.fail_changeset()
+          |> Repo.update!()
+
+        :ok = on_failure(site_import)
+
+        site_import
+      end
+
+      @impl true
+      def before_start(site_import, _opts), do: {:ok, site_import}
+
+      @impl true
+      def on_success(_site_import, _extra_data), do: :ok
+
+      @impl true
+      def on_failure(_site_import), do: :ok
+
+      defoverridable before_start: 2, on_success: 2, on_failure: 1
+    end
+  end
+
+  @doc false
+  def new_import(source, site, user, opts, before_start_fun) do
+    import_params =
+      opts
+      |> Keyword.take([:label, :start_date, :end_date, :legacy])
+      |> Keyword.put(:source, source)
+      |> Map.new()
+
+    Repo.transaction(fn ->
+      result =
+        site
+        |> SiteImport.create_changeset(user, import_params)
+        |> Repo.insert()
+
+      with {:ok, site_import} <- result,
+           {:ok, site_import} <- before_start_fun.(site_import, opts),
+           {:ok, job} <- schedule_job(site_import, opts) do
+        job
+      else
+        {:error, error} ->
+          Repo.rollback(error)
+      end
+    end)
+  end
+
+  @doc false
+  def run_import(site_import, args, parse_args_fun, import_fun, on_success_fun) do
+    site_import =
+      site_import
+      |> SiteImport.start_changeset()
+      |> Repo.update!()
+      |> Repo.preload(:site)
+
+    import_opts = parse_args_fun.(args)
+
+    case import_fun.(site_import, import_opts) do
+      :ok ->
+        {:ok, mark_complete(site_import, %{}, on_success_fun)}
+
+      {:ok, extra_data} ->
+        {:ok, mark_complete(site_import, extra_data, on_success_fun)}
+
+      {:error, error} ->
+        {:error, error, []}
+
+      {:error, error, opts} ->
+        {:error, error, opts}
+    end
+  end
+
+  @oban_channel :analytics_imports_jobs
+
+  @doc false
+  def notify(site_import, event) do
+    Oban.Notifier.notify(Oban, @oban_channel, %{
+      "event" => event,
+      "import_id" => site_import.id,
+      "site_id" => site_import.site_id
+    })
+  end
+
+  @doc """
+  Allows to explicitly start listening for importer job notifications.
+
+  Listener must explicitly filter out a subset of imports that apply to the given context.
+  """
+  @spec listen() :: :ok
+  def listen() do
+    :ok = Oban.Notifier.listen([@oban_channel])
+  end
+
+  defp schedule_job(site_import, opts) do
+    {listen?, opts} = Keyword.pop(opts, :listen?, false)
+    {job_opts, opts} = Keyword.pop(opts, :job_opts, [])
+
+    if listen? do
+      :ok = listen()
+    end
+
+    if not Imported.other_imports_in_progress?(site_import) do
+      opts
+      |> Keyword.put(:import_id, site_import.id)
+      |> Map.new()
+      |> Plausible.Workers.ImportAnalytics.new(job_opts)
+      |> Oban.insert()
+    else
+      {:error, :import_in_progress}
+    end
+  end
+
+  defp mark_complete(site_import, extra_data, on_success_fun) do
+    site_import =
+      site_import
+      |> SiteImport.complete_changeset(extra_data)
+      |> Repo.update!()
+
+    :ok = on_success_fun.(site_import, extra_data)
+
+    site_import
+  end
+end

lib/plausible/imported/location.ex

@@ -0,0 +1,19 @@
+defmodule Plausible.Imported.Location do
+  @moduledoc false
+  use Ecto.Schema
+
+  @primary_key false
+  schema "imported_locations" do
+    field :site_id, Ch, type: "UInt64"
+    field :import_id, Ch, type: "UInt64"
+    field :date, :date
+    field :country, :string
+    field :region, :string
+    field :city, Ch, type: "UInt64"
+    field :visitors, Ch, type: "UInt64"
+    field :visits, Ch, type: "UInt64"
+    field :visit_duration, Ch, type: "UInt64"
+    field :pageviews, Ch, type: "UInt64"
+    field :bounces, Ch, type: "UInt32"
+  end
+end

lib/plausible/imported/noop_importer.ex

@@ -0,0 +1,46 @@
+defmodule Plausible.Imported.NoopImporter do
+  @moduledoc """
+  Stub import implementation.
+  """
+
+  use Plausible.Imported.Importer
+
+  @impl true
+  def name(), do: :noop
+
+  @impl true
+  def label(), do: "Noop"
+
+  # reusing existing template from another source
+  @impl true
+  def email_template(), do: "google_analytics_import.html"
+
+  @impl true
+  def parse_args(opts), do: opts
+
+  @impl true
+  def import_data(_site_import, %{"error" => true}), do: {:error, "Something went wrong"}
+  def import_data(_site_import, %{"crash" => true}), do: raise("boom")
+  def import_data(_site_import, _opts), do: :ok
+
+  @impl true
+  def before_start(site_import, _opts) do
+    send(self(), {:before_start, site_import.id})
+
+    {:ok, site_import}
+  end
+
+  @impl true
+  def on_success(site_import, _extra_data) do
+    send(self(), {:on_success, site_import.id})
+
+    :ok
+  end
+
+  @impl true
+  def on_failure(site_import) do
+    send(self(), {:on_failure, site_import.id})
+
+    :ok
+  end
+end

lib/plausible/imported/operating_system.ex

@@ -0,0 +1,18 @@
+defmodule Plausible.Imported.OperatingSystem do
+  @moduledoc false
+  use Ecto.Schema
+
+  @primary_key false
+  schema "imported_operating_systems" do
+    field :site_id, Ch, type: "UInt64"
+    field :import_id, Ch, type: "UInt64"
+    field :date, :date
+    field :operating_system, :string
+    field :operating_system_version, :string
+    field :visitors, Ch, type: "UInt64"
+    field :visits, Ch, type: "UInt64"
+    field :visit_duration, Ch, type: "UInt64"
+    field :pageviews, Ch, type: "UInt64"
+    field :bounces, Ch, type: "UInt32"
+  end
+end

lib/plausible/imported/page.ex

@@ -0,0 +1,21 @@
+defmodule Plausible.Imported.Page do
+  @moduledoc false
+  use Ecto.Schema
+
+  @primary_key false
+  schema "imported_pages" do
+    field :site_id, Ch, type: "UInt64"
+    field :import_id, Ch, type: "UInt64"
+    field :date, :date
+    field :hostname, :string
+    field :page, :string
+    field :visits, Ch, type: "UInt64"
+    field :visitors, Ch, type: "UInt64"
+    field :pageviews, Ch, type: "UInt64"
+    field :exits, Ch, type: "UInt64"
+    field :total_scroll_depth, Ch, type: "UInt64"
+    field :total_scroll_depth_visits, Ch, type: "UInt64"
+    field :total_time_on_page, Ch, type: "UInt64"
+    field :total_time_on_page_visits, Ch, type: "UInt64"
+  end
+end

lib/plausible/imported/site_import.ex

@@ -0,0 +1,73 @@
+defmodule Plausible.Imported.SiteImport do
+  @moduledoc """
+  Site import schema.
+  """
+
+  use Ecto.Schema
+
+  import Ecto.Changeset
+
+  alias Plausible.Auth.User
+  alias Plausible.Imported.ImportSources
+  alias Plausible.Site
+
+  @statuses [:pending, :importing, :completed, :failed]
+
+  @type t() :: %__MODULE__{}
+
+  schema "site_imports" do
+    field :start_date, :date
+    field :end_date, :date
+    field :label, :string
+    field :source, Ecto.Enum, values: ImportSources.names()
+    field :status, Ecto.Enum, values: @statuses
+    field :legacy, :boolean, default: false
+    field :has_scroll_depth, :boolean, default: false
+
+    belongs_to :site, Site
+    belongs_to :imported_by, User
+
+    timestamps()
+  end
+
+  for status <- @statuses do
+    defmacro unquote(status)(), do: unquote(status)
+  end
+
+  @spec label(t()) :: String.t()
+  def label(%{source: source, label: label}) do
+    build_label(ImportSources.by_name(source).label(), label)
+  end
+
+  @spec create_changeset(Site.t(), User.t(), map()) :: Ecto.Changeset.t()
+  def create_changeset(site, user, params) do
+    %__MODULE__{}
+    |> cast(params, [:label, :source, :start_date, :end_date, :legacy])
+    |> validate_required([:source])
+    |> put_assoc(:site, site)
+    |> put_assoc(:imported_by, user)
+    |> put_change(:status, pending())
+  end
+
+  @spec start_changeset(t()) :: Ecto.Changeset.t()
+  def start_changeset(site_import) do
+    site_import
+    |> change(status: importing())
+  end
+
+  @spec complete_changeset(t(), map()) :: Ecto.Changeset.t()
+  def complete_changeset(site_import, params \\ %{}) do
+    site_import
+    |> cast(params, [:start_date, :end_date])
+    |> put_change(:status, completed())
+    |> validate_required([:start_date, :end_date])
+  end
+
+  @spec fail_changeset(t()) :: Ecto.Changeset.t()
+  def fail_changeset(site_import) do
+    change(site_import, status: failed())
+  end
+
+  defp build_label(source, nil), do: source
+  defp build_label(source, label), do: "#{source} (#{label})"
+end

lib/plausible/imported/source.ex

@@ -0,0 +1,24 @@
+defmodule Plausible.Imported.Source do
+  @moduledoc false
+  use Ecto.Schema
+
+  @primary_key false
+  schema "imported_sources" do
+    field :site_id, Ch, type: "UInt64"
+    field :import_id, Ch, type: "UInt64"
+    field :date, :date
+    field :source, :string
+    field :channel, Ch, type: "LowCardinality(String)"
+    field :referrer, :string
+    field :utm_source, :string
+    field :utm_medium, :string
+    field :utm_campaign, :string
+    field :utm_content, :string
+    field :utm_term, :string
+    field :visitors, Ch, type: "UInt64"
+    field :visits, Ch, type: "UInt64"
+    field :visit_duration, Ch, type: "UInt64"
+    field :pageviews, Ch, type: "UInt64"
+    field :bounces, Ch, type: "UInt32"
+  end
+end

lib/plausible/imported/universal_analytics.ex

@@ -0,0 +1,29 @@
+defmodule Plausible.Imported.UniversalAnalytics do
+  @moduledoc """
+  Import implementation for Universal Analytics.
+
+  NOTE: As importing from UA is no longer supported, this module
+  is only used to support rendering existing imports.
+  """
+
+  use Plausible.Imported.Importer
+
+  @impl true
+  def name(), do: :universal_analytics
+
+  @impl true
+  def label(), do: "Google Analytics"
+
+  @impl true
+  def email_template(), do: "google_analytics_import.html"
+
+  @impl true
+  def parse_args(_args) do
+    raise "Importing data not supported"
+  end
+
+  @impl true
+  def import_data(_site_import, _opts) do
+    raise "Importing data not supported"
+  end
+end

lib/plausible/imported/visitor.ex

@@ -0,0 +1,16 @@
+defmodule Plausible.Imported.Visitor do
+  @moduledoc false
+  use Ecto.Schema
+
+  @primary_key false
+  schema "imported_visitors" do
+    field :site_id, Ch, type: "UInt64"
+    field :import_id, Ch, type: "UInt64"
+    field :date, :date
+    field :visitors, Ch, type: "UInt64"
+    field :pageviews, Ch, type: "UInt64"
+    field :bounces, Ch, type: "UInt64"
+    field :visits, Ch, type: "UInt64"
+    field :visit_duration, Ch, type: "UInt64"
+  end
+end

lib/plausible/ingest_repo.ex

@@ -0,0 +1,28 @@
+defmodule Plausible.IngestRepo do
+  @moduledoc """
+  Write-centric Clickhouse access interface
+  """
+
+  use Ecto.Repo,
+    otp_app: :plausible,
+    adapter: Ecto.Adapters.ClickHouse
+
+  defmacro __using__(_) do
+    quote do
+      alias Plausible.IngestRepo
+      import Ecto
+      import Ecto.Query, only: [from: 1, from: 2]
+    end
+  end
+
+  def clustered_table?(table) do
+    replica_count(table) > 1
+  end
+
+  def replica_count(table) do
+    {:ok, %{rows: [[count]]}} =
+      query("SELECT sum(active_replicas) FROM system.replicas WHERE table = '#{table}'")
+
+    count
+  end
+end

lib/plausible/ingestion/acquisition.ex

@@ -0,0 +1,218 @@
+defmodule Plausible.Ingestion.Acquisition do
+  @moduledoc """
+  This module is responsible for figuring out acquisition channel from event referrer_source.
+
+  Acquisition channel is the marketing channel where people come from and convert and help
+  users to understand and improve their marketing flow.
+
+  Note it uses priv/ga4-source-categories.csv as a source, which comes from https://support.google.com/analytics/answer/9756891?hl=en.
+
+  Notable differences from GA4 that have been implemented just for Plausible:
+  1. The @custom_source_categories module attribute contains a list of custom source categories that we have manually
+  added based on our own judgement and user feedback. For example we group AI assistants (ChatGPT, Claude, Gemini, etc.)
+  into their own "AI Assistants" channel.
+  2. Google is in a privileged position to analyze paid traffic from within their own network. The biggest use-case is auto-tagged adwords campaigns.
+  We do our best by categorizing as paid search when source is Google and the url has `gclid` parameter. Same for source Bing and `msclkid` url parameter.
+  3. The @paid_sources module attribute in Plausible.Ingestion.Source contains a list of utm_sources that we will automatically categorize as paid traffic
+  regardless of the medium. Examples are `yt-ads`, `facebook_ad`, `adwords`, etc. See also: Plausible.Ingestion.Source.paid_source?/1
+  """
+
+  @external_resource "priv/ga4-source-categories.csv"
+  @custom_source_categories [
+    {"hacker news", "SOURCE_CATEGORY_SOCIAL"},
+    {"yahoo!", "SOURCE_CATEGORY_SEARCH"},
+    {"gmail", "SOURCE_CATEGORY_EMAIL"},
+    {"telegram", "SOURCE_CATEGORY_SOCIAL"},
+    {"slack", "SOURCE_CATEGORY_SOCIAL"},
+    {"producthunt", "SOURCE_CATEGORY_SOCIAL"},
+    {"github", "SOURCE_CATEGORY_SOCIAL"},
+    {"steamcommunity.com", "SOURCE_CATEGORY_SOCIAL"},
+    {"statics.teams.cdn.office.net", "SOURCE_CATEGORY_SOCIAL"},
+    {"vkontakte", "SOURCE_CATEGORY_SOCIAL"},
+    {"threads", "SOURCE_CATEGORY_SOCIAL"},
+    {"ecosia", "SOURCE_CATEGORY_SEARCH"},
+    {"kagi", "SOURCE_CATEGORY_SEARCH"},
+    {"brave", "SOURCE_CATEGORY_SEARCH"},
+    {"temu.com", "SOURCE_CATEGORY_SHOPPING"},
+    {"discord", "SOURCE_CATEGORY_SOCIAL"},
+    {"sogou", "SOURCE_CATEGORY_SEARCH"},
+    {"microsoft teams", "SOURCE_CATEGORY_SOCIAL"},
+    {"bluesky", "SOURCE_CATEGORY_SOCIAL"},
+    {"mastodon", "SOURCE_CATEGORY_SOCIAL"},
+    {"chatgpt", "SOURCE_CATEGORY_AI_ASSISTANTS"},
+    {"claude", "SOURCE_CATEGORY_AI_ASSISTANTS"},
+    {"phind", "SOURCE_CATEGORY_AI_ASSISTANTS"},
+    {"deepseek", "SOURCE_CATEGORY_AI_ASSISTANTS"},
+    {"microsoft copilot", "SOURCE_CATEGORY_AI_ASSISTANTS"},
+    {"grok", "SOURCE_CATEGORY_AI_ASSISTANTS"},
+    {"google gemini", "SOURCE_CATEGORY_AI_ASSISTANTS"},
+    {"perplexity", "SOURCE_CATEGORY_AI_ASSISTANTS"},
+    {"x (twitter)", "SOURCE_CATEGORY_SOCIAL"}
+  ]
+  @source_categories Application.app_dir(:plausible, "priv/ga4-source-categories.csv")
+                     |> File.read!()
+                     |> NimbleCSV.RFC4180.parse_string(skip_headers: false)
+                     |> Enum.map(fn [source, category] -> {source, category} end)
+                     |> then(&(@custom_source_categories ++ &1))
+                     |> Enum.into(%{})
+
+  def source_categories(), do: @source_categories
+
+  def get_channel(source, utm_medium, utm_campaign, utm_source, click_id_param) do
+    get_channel_lowered(
+      String.downcase(source || ""),
+      String.downcase(utm_medium || ""),
+      String.downcase(utm_campaign || ""),
+      String.downcase(utm_source || ""),
+      click_id_param
+    )
+  end
+
+  defp get_channel_lowered(source, utm_medium, utm_campaign, utm_source, click_id_param) do
+    cond do
+      cross_network?(utm_campaign) -> "Cross-network"
+      paid_shopping?(source, utm_campaign, utm_medium) -> "Paid Shopping"
+      paid_search?(source, utm_medium, utm_source, click_id_param) -> "Paid Search"
+      paid_social?(source, utm_medium, utm_source) -> "Paid Social"
+      paid_video?(source, utm_medium, utm_source) -> "Paid Video"
+      display?(utm_medium) -> "Display"
+      paid_other?(utm_medium) -> "Paid Other"
+      organic_shopping?(source, utm_campaign) -> "Organic Shopping"
+      organic_social?(source, utm_medium) -> "Organic Social"
+      organic_video?(source, utm_medium) -> "Organic Video"
+      ai_assistants?(source) -> "AI Assistants"
+      search_source?(source) -> "Organic Search"
+      email?(source, utm_source, utm_medium) -> "Email"
+      affiliates?(utm_medium) -> "Affiliates"
+      audio?(utm_medium) -> "Audio"
+      sms?(utm_source, utm_medium) -> "SMS"
+      mobile_push_notifications?(source, utm_medium) -> "Mobile Push Notifications"
+      referral?(source, utm_medium) -> "Referral"
+      true -> "Direct"
+    end
+  end
+
+  defp cross_network?(utm_campaign) do
+    String.contains?(utm_campaign, "cross-network")
+  end
+
+  defp paid_shopping?(source, utm_campaign, utm_medium) do
+    (shopping_source?(source) or shopping_campaign?(utm_campaign)) and paid_medium?(utm_medium)
+  end
+
+  defp paid_search?(source, utm_medium, utm_source, click_id_param) do
+    (search_source?(source) and paid_medium?(utm_medium)) or
+      (search_source?(source) and paid_source?(utm_source)) or
+      (source == "google" and click_id_param == "gclid") or
+      (source == "bing" and click_id_param == "msclkid")
+  end
+
+  defp paid_social?(source, utm_medium, utm_source) do
+    (social_source?(source) and paid_medium?(utm_medium)) or
+      (social_source?(source) and paid_source?(utm_source))
+  end
+
+  defp paid_video?(source, utm_medium, utm_source) do
+    (video_source?(source) and paid_medium?(utm_medium)) or
+      (video_source?(source) and paid_source?(utm_source))
+  end
+
+  defp display?(utm_medium) do
+    utm_medium in [
+      "display",
+      "banner",
+      "expandable",
+      "interstitial",
+      "cpm"
+    ]
+  end
+
+  defp paid_other?(utm_medium) do
+    paid_medium?(utm_medium)
+  end
+
+  defp organic_shopping?(source, utm_campaign) do
+    shopping_source?(source) or shopping_campaign?(utm_campaign)
+  end
+
+  defp organic_social?(source, utm_medium) do
+    social_source?(source) or
+      utm_medium in [
+        "social",
+        "social-network",
+        "social-media",
+        "sm",
+        "social network",
+        "social media"
+      ]
+  end
+
+  defp organic_video?(source, utm_medium) do
+    video_source?(source) or String.contains?(utm_medium, "video")
+  end
+
+  defp referral?(source, utm_medium) do
+    utm_medium in ["referral", "app", "link"] or source !== ""
+  end
+
+  @email_tags ["email", "e-mail", "e_mail", "e mail", "newsletter"]
+  defp email?(source, utm_source, utm_medium) do
+    email_source?(source) or
+      String.contains?(utm_source, @email_tags) or
+      String.contains?(utm_medium, @email_tags)
+  end
+
+  defp affiliates?(utm_medium) do
+    utm_medium == "affiliate"
+  end
+
+  defp audio?(utm_medium) do
+    utm_medium == "audio"
+  end
+
+  defp sms?(utm_source, utm_medium) do
+    utm_source == "sms" or utm_medium == "sms"
+  end
+
+  defp mobile_push_notifications?(source, utm_medium) do
+    String.ends_with?(utm_medium, "push") or
+      String.contains?(utm_medium, ["mobile", "notification"]) or
+      source == "firebase"
+  end
+
+  defp shopping_source?(source) do
+    @source_categories[source] == "SOURCE_CATEGORY_SHOPPING"
+  end
+
+  defp search_source?(source) do
+    @source_categories[source] == "SOURCE_CATEGORY_SEARCH"
+  end
+
+  defp social_source?(source) do
+    @source_categories[source] == "SOURCE_CATEGORY_SOCIAL"
+  end
+
+  defp video_source?(source) do
+    @source_categories[source] == "SOURCE_CATEGORY_VIDEO"
+  end
+
+  defp email_source?(source) do
+    @source_categories[source] == "SOURCE_CATEGORY_EMAIL"
+  end
+
+  defp ai_assistants?(source) do
+    @source_categories[source] == "SOURCE_CATEGORY_AI_ASSISTANTS"
+  end
+
+  defp shopping_campaign?(utm_campaign) do
+    Regex.match?(~r/^(.*(([^a-df-z]|^)shop|shopping).*)$/, utm_campaign)
+  end
+
+  defp paid_medium?(utm_medium) do
+    Regex.match?(~r/^(.*cp.*|ppc|retargeting|paid.*)$/, utm_medium)
+  end
+
+  defp paid_source?(utm_source) do
+    Plausible.Ingestion.Source.paid_source?(utm_source)
+  end
+end

lib/plausible/ingestion/city_overrides.ex

@@ -0,0 +1,202 @@
+defmodule Plausible.Ingestion.CityOverrides do
+  @moduledoc false
+
+  @overrides %{
+    # Austria
+    # Gemeindebezirk Floridsdorf -> Vienna
+    2_779_467 => 2_761_369,
+    # Gemeindebezirk Leopoldstadt -> Vienna
+    2_772_614 => 2_761_369,
+    # Gemeindebezirk Landstrasse -> Vienna
+    2_773_040 => 2_761_369,
+    # Gemeindebezirk Donaustadt -> Vienna
+    2_780_851 => 2_761_369,
+    # Gemeindebezirk Favoriten -> Vienna
+    2_779_776 => 2_761_369,
+    # Gemeindebezirk Währing -> Vienna
+    2_762_091 => 2_761_369,
+    # Gemeindebezirk Wieden -> Vienna
+    2_761_393 => 2_761_369,
+    # Gemeindebezirk Innere Stadt -> Vienna
+    2_775_259 => 2_761_369,
+    # Gemeindebezirk Alsergrund -> Vienna
+    2_782_729 => 2_761_369,
+    # Gemeindebezirk Liesing -> Vienna
+    2_772_484 => 2_761_369,
+    # Urfahr -> Linz
+    2_762_518 => 2_772_400,
+
+    # Canada
+    # Old Toronto -> Toronto
+    8_436_019 => 6_167_865,
+    # Etobicoke -> Toronto
+    5_950_267 => 6_167_865,
+    # East York -> Toronto
+    5_946_235 => 6_167_865,
+    # Scarborough -> Toronto
+    6_948_711 => 6_167_865,
+    # North York -> Toronto
+    6_091_104 => 6_167_865,
+
+    # Czech republic
+    # Praha 5 -> Prague
+    11_951_220 => 3_067_696,
+    # Praha 4 -> Prague
+    11_951_218 => 3_067_696,
+    # Praha 11 -> Prague
+    11_951_232 => 3_067_696,
+    # Praha 10 -> Prague
+    11_951_210 => 3_067_696,
+    # Praha 4 -> Prague
+    8_378_772 => 3_067_696,
+
+    # Denmark
+    # København SV -> Copenhagen
+    11_747_123 => 2_618_425,
+    # København NV -> Copenhagen
+    11_746_894 => 2_618_425,
+    # Odense S -> Odense
+    11_746_825 => 2_615_876,
+    # Odense M -> Odense
+    11_746_974 => 2_615_876,
+    # Odense SØ -> Odense
+    11_746_888 => 2_615_876,
+    # Aarhus C -> Aarhus
+    11_746_746 => 2_624_652,
+    # Aarhus N -> Aarhus
+    11_746_890 => 2_624_652,
+
+    # Estonia
+    # Kristiine linnaosa -> Tallinn
+    11_050_530 => 588_409,
+    # Kesklinna linnaosa -> Tallinn
+    11_053_706 => 588_409,
+    # Lasnamäe linnaosa -> Tallinn
+    11_050_526 => 588_409,
+    # Põhja-Tallinna linnaosa -> Tallinn
+    11_049_594 => 588_409,
+    # Mustamäe linnaosa -> Tallinn
+    11_050_531 => 588_409,
+    # Haabersti linnaosa -> Tallinn
+    11_053_707 => 588_409,
+    # Viimsi -> Tallinn
+    587_629 => 588_409,
+
+    # Germany
+    # Bezirk Tempelhof-Schöneberg -> Berlin
+    3_336_297 => 2_950_159,
+    # Bezirk Mitte -> Berlin
+    2_870_912 => 2_950_159,
+    # Bezirk Charlottenburg-Wilmersdorf -> Berlin
+    3_336_294 => 2_950_159,
+    # Bezirk Friedrichshain-Kreuzberg -> Berlin
+    3_336_295 => 2_950_159,
+    # Moosach -> Munich
+    8_351_447 => 2_867_714,
+    # Schwabing-Freimann -> Munich
+    8_351_448 => 2_867_714,
+    # Stadtbezirk 06 -> Düsseldorf
+    6_947_276 => 2_934_246,
+    # Stadtbezirk 04 -> Düsseldorf
+    6_947_274 => 2_934_246,
+    # Köln-Ehrenfeld -> Köln
+    6_947_479 => 2_886_242,
+    # Köln-Lindenthal- -> Köln
+    6_947_481 => 2_886_242,
+    # Beuel -> Bonn
+    2_949_619 => 2_946_447,
+    # Innenstadt I -> Frankfurt am Main
+    6_946_225 => 2_925_533,
+    # Innenstadt II -> Frankfurt am Main
+    6_946_226 => 2_925_533,
+    # Innenstadt III -> Frankfurt am Main
+    6_946_227 => 2_925_533,
+
+    # India
+    # Navi Mumbai -> Mumbai
+    6_619_347 => 1_275_339,
+
+    # Mexico
+    # Miguel Hidalgo Villa Olímpica -> Mexico city
+    11_561_026 => 3_530_597,
+    # Zedec Santa Fe -> Mexico city
+    3_517_471 => 3_530_597,
+    #  Fuentes del Pedregal-> Mexico city
+    11_562_596 => 3_530_597,
+    #  Centro -> Mexico city
+    9_179_691 => 3_530_597,
+    #  Cuauhtémoc-> Mexico city
+    12_266_959 => 3_530_597,
+
+    # Netherlands
+    # Schiphol-Rijk -> Amsterdam
+    10_173_838 => 2_759_794,
+    # Westpoort -> Amsterdam
+    11_525_047 => 2_759_794,
+    # Amsterdam-Zuidoost -> Amsterdam
+    6_544_881 => 2_759_794,
+    # Loosduinen -> The Hague
+    11_525_037 => 2_747_373,
+    # Laak -> The Hague
+    11_525_042 => 2_747_373,
+
+    # Norway
+    # Nordre Aker District -> Oslo
+    6_940_981 => 3_143_244,
+
+    # Romania
+    # Sector 1 -> Bucharest,
+    11_055_041 => 683_506,
+    # Sector 2 -> Bucharest
+    11_055_040 => 683_506,
+    # Sector 3 -> Bucharest
+    11_055_044 => 683_506,
+    # Sector 4 -> Bucharest
+    11_055_042 => 683_506,
+    # Sector 5 -> Bucharest
+    11_055_043 => 683_506,
+    # Sector 6 -> Bucharest
+    11_055_039 => 683_506,
+    # Bucuresti -> Bucharest
+    6_691_781 => 683_506,
+
+    # Slovakia
+    # Bratislava -> Bratislava
+    3_343_955 => 3_060_972,
+
+    # Sweden
+    # Södermalm -> Stockholm
+    2_676_209 => 2_673_730,
+
+    # Switzerland
+    # Vorstädte -> Basel
+    11_789_440 => 2_661_604,
+    # Zürich (Kreis 11) / Oerlikon -> Zürich
+    2_659_310 => 2_657_896,
+    # Zürich (Kreis 3) / Alt-Wiedikon -> Zürich
+    2_658_007 => 2_657_896,
+    # Zürich (Kreis 5) -> Zürich
+    6_295_521 => 2_657_896,
+    # Zürich (Kreis 1) / Hochschulen -> Zürich
+    6_295_489 => 2_657_896,
+
+    # UK
+    # Shadwell -> London
+    6_690_595 => 2_643_743,
+    # City of London -> London
+    2_643_741 => 2_643_743,
+    # South Bank -> London
+    6_545_251 => 2_643_743,
+    # Soho -> London
+    6_545_173 => 2_643_743,
+    # Whitechapel -> London
+    2_634_112 => 2_643_743,
+    # King's Cross -> London
+    6_690_589 => 2_643_743,
+    # Poplar -> London
+    2_640_091 => 2_643_743,
+    # Hackney -> London
+    2_647_694 => 2_643_743
+  }
+  def get(key, default), do: Map.get(@overrides, key, default)
+end