Download lib/plausible/ingestion/write_buffer.ex from Leon4gr45/analytics: direct link, hf CLI and curl.
- Browser
- Download file 4.39 kB
-
https://huggingface.co/spaces/Leon4gr45/analytics/resolve/main/lib/plausible/ingestion/write_buffer.ex
- Command line
-
hf download hf://spaces/Leon4gr45/analytics/lib/plausible/ingestion/write_buffer.ex
-
curl -L -o write_buffer.ex https://huggingface.co/spaces/Leon4gr45/analytics/resolve/main/lib/plausible/ingestion/write_buffer.ex
4.39 kB
| defmodule Plausible.Ingestion.WriteBuffer do | |
| @moduledoc false | |
| use GenServer | |
| require Logger | |
| alias Plausible.IngestRepo | |
| def start_link(opts) do | |
| GenServer.start_link(__MODULE__, opts, name: Keyword.fetch!(opts, :name)) | |
| end | |
| def insert(server, row_binary) do | |
| GenServer.cast(server, {:insert, row_binary}) | |
| end | |
| def flush(server) do | |
| GenServer.call(server, :flush, :infinity) | |
| end | |
| @impl true | |
| def init(opts) do | |
| buffer = opts[:buffer] || [] | |
| max_buffer_size = opts[:max_buffer_size] || default_max_buffer_size() | |
| flush_interval_ms = opts[:flush_interval_ms] || default_flush_interval_ms() | |
| Process.flag(:trap_exit, true) | |
| timer = Process.send_after(self(), :tick, flush_interval_ms) | |
| {:ok, | |
| %{ | |
| buffer: buffer, | |
| timer: timer, | |
| name: Keyword.fetch!(opts, :name), | |
| insert_sql: Keyword.fetch!(opts, :insert_sql), | |
| insert_opts: Keyword.fetch!(opts, :insert_opts), | |
| header: Keyword.fetch!(opts, :header), | |
| buffer_size: IO.iodata_length(buffer), | |
| max_buffer_size: max_buffer_size, | |
| flush_interval_ms: flush_interval_ms | |
| }} | |
| end | |
| @impl true | |
| def handle_cast({:insert, row_binary}, state) do | |
| state = %{ | |
| state | |
| | buffer: [state.buffer | row_binary], | |
| buffer_size: state.buffer_size + IO.iodata_length(row_binary) | |
| } | |
| if state.buffer_size >= state.max_buffer_size do | |
| Logger.notice("#{state.name} buffer full, flushing to ClickHouse") | |
| Process.cancel_timer(state.timer) | |
| do_flush(state) | |
| new_timer = Process.send_after(self(), :tick, state.flush_interval_ms) | |
| {:noreply, %{state | buffer: [], timer: new_timer, buffer_size: 0}} | |
| else | |
| {:noreply, state} | |
| end | |
| end | |
| @impl true | |
| def handle_info(:tick, state) do | |
| do_flush(state) | |
| timer = Process.send_after(self(), :tick, state.flush_interval_ms) | |
| {:noreply, %{state | buffer: [], buffer_size: 0, timer: timer}} | |
| end | |
| def handle_info({:EXIT, _from, reason}, state) do | |
| Logger.warning("#{state.name} received EXIT, keeping buffer alive: #{inspect(reason)}") | |
| {:noreply, state} | |
| end | |
| @impl true | |
| def handle_call(:flush, _from, state) do | |
| %{timer: timer, flush_interval_ms: flush_interval_ms} = state | |
| Process.cancel_timer(timer) | |
| do_flush(state) | |
| new_timer = Process.send_after(self(), :tick, flush_interval_ms) | |
| {:reply, :ok, %{state | buffer: [], buffer_size: 0, timer: new_timer}} | |
| end | |
| @impl true | |
| def terminate(_reason, %{name: name} = state) do | |
| Logger.notice("Flushing #{name} buffer before shutdown...") | |
| do_flush(state) | |
| end | |
| defp do_flush(state) do | |
| %{ | |
| buffer: buffer, | |
| buffer_size: buffer_size, | |
| insert_opts: insert_opts, | |
| insert_sql: insert_sql, | |
| header: header, | |
| name: name | |
| } = state | |
| case buffer do | |
| [] -> | |
| nil | |
| _not_empty -> | |
| Logger.notice("Flushing #{buffer_size} byte(s) RowBinary from #{name}") | |
| IngestRepo.query!(insert_sql, [header | buffer], insert_opts) | |
| end | |
| end | |
| defp default_flush_interval_ms do | |
| Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :flush_interval_ms) | |
| end | |
| defp default_max_buffer_size do | |
| Keyword.fetch!(Application.get_env(:plausible, IngestRepo), :max_buffer_size) | |
| end | |
| @doc false | |
| def compile_time_prepare(schema) do | |
| fields = | |
| schema.__schema__(:fields) | |
| |> Enum.reject(&(&1 in fields_to_ignore())) | |
| types = | |
| Enum.map(fields, fn field -> | |
| type = schema.__schema__(:type, field) || raise "missing type for #{field}" | |
| type | |
| |> Ecto.Type.type() | |
| |> Ecto.Adapters.ClickHouse.Schema.remap_type(schema, field) | |
| end) | |
| encoding_types = Ch.RowBinary.encoding_types(types) | |
| header = | |
| fields | |
| |> Enum.map(&to_string/1) | |
| |> Ch.RowBinary.encode_names_and_types(types) | |
| |> IO.iodata_to_binary() | |
| insert_sql = | |
| "INSERT INTO #{schema.__schema__(:source)} (#{Enum.join(fields, ", ")}) FORMAT RowBinaryWithNamesAndTypes" | |
| %{ | |
| fields: fields, | |
| types: types, | |
| encoding_types: encoding_types, | |
| header: header, | |
| insert_sql: insert_sql, | |
| insert_opts: [ | |
| command: :insert, | |
| encode: false, | |
| source: schema.__schema__(:source), | |
| cast_params: [] | |
| ] | |
| } | |
| end | |
| defp fields_to_ignore(), do: [:acquisition_channel, :interactive?] | |
| end | |