filter_engine.py

# /// script
# requires-python = ">=3.12"
# dependencies = [
#     "marimo>=0.23.8",
#   "opensyndrome @ git+https://github.com/OpenSyndrome/open-syndrome-python.git@main",
#   "plotly>=6.2.0",
#   "polars>=1.38.1",
# ]
# ///
import marimo

__generated_with = "0.21.0"
app = marimo.App(
    width="medium",
    app_title="Open Syndrome Definition - Data Browser",
)


@app.cell
def _():
    import marimo as mo
    import json

    import polars as pl
    import plotly.graph_objects as go

    import yaml
    from opensyndrome.filter import OSDEngine, load_profile
    from opensyndrome.artifacts import get_definition_dir

    return (
        OSDEngine,
        get_definition_dir,
        go,
        json,
        load_profile,
        mo,
        pl,
        yaml,
    )


@app.cell
def _(go, pl):
    def plot_cases(
        _df_filtered, definitions, date_column="date", date_format="%Y-%m-%d %H:%M:%S"
    ):
        _definitions_columns_sum = [
            pl.col(definition).sum().alias(definition) for definition in definitions
        ]
        _agg_df = (
            _df_filtered.with_columns(
                pl.col(date_column)
                .str.to_datetime(format=date_format, strict=False)
                .cast(pl.Date)
                .dt.truncate("1mo")
                .alias("_month")
            )
            .group_by("_month")
            .agg(_definitions_columns_sum)
            .sort("_month")
        )
        _fig = go.Figure()
        for definition in definitions:
            _fig.add_trace(
                go.Scatter(
                    x=_agg_df["_month"].to_list(),
                    y=_agg_df[definition].to_list(),
                    mode="lines+markers",
                    name=definition,
                )
            )
        _fig.update_layout(yaxis=dict(tickformat="d", rangemode="tozero"))
        return _fig

    return (plot_cases,)


@app.cell
def _(go, pl):
    def groupped_bar(_df_filtered, definitions, group_by_column="code", top_n=10):
        _agg_df = (
            _df_filtered.group_by(group_by_column)
            .agg([pl.col(definition).sum() for definition in definitions])
            .sort(group_by_column)
        )

        _agg_df = _agg_df.with_columns(
            (sum([pl.col(definition) for definition in definitions])).alias("total")
        )
        _agg_df = _agg_df.sort("total", descending=True).head(
            top_n
        )  # .sort(group_by_column)
        _fig = go.Figure(
            data=[
                go.Bar(
                    name=definition,
                    x=_agg_df[group_by_column].to_list(),
                    y=_agg_df[definition].to_list(),
                )
                for definition in definitions
            ]
        )
        _fig.update_layout(barmode="group")
        return _fig

    return (groupped_bar,)


@app.cell
def _(mo):
    mo.md(r"""
    # Open Syndrome Definition 👩🏽‍🔬
    """)
    return


@app.cell
def _():
    prompt = """
    Role: Act as an expert in healthcare data engineering and the Open Syndrome Definition (OSD) framework.

    Task: Generate two text files for testing data filtering and syndromic surveillance pipelines.

    File 1: Synthetic Dataset (CSV Format)

    Create a synthetic dataset of roughly 20 ambulatory care records.

    The data must be in [Insert Language, e.g., Brazilian Portuguese, English, German].

    Include the following columns: recording_ts (timestamp), icd_code (ICD-10 code), sex (encoded as [Insert Encoding, e.g., M/F/D]), age (integer), and chief_complaint (string of the symptoms).

    Ensure the clinical presentation logically matches the ICD-10 code and age.

    File 2: OSD Mapping File (YAML Format)

    Create a YAML configuration file that maps the CSV columns to Open Syndrome Definition concepts.

    Define a profile named ambulatory_care.

    Include a value_encodings section that defines the mapping for the sex column.

    Under columns, map each CSV column to its respective OSD concept (e.g., demographic_criteria, diagnosis), attribute (e.g., age, sex), and dtype (integer, string).

    Please output the exact CSV and YAML code in clearly separated code blocks so I can copy them directly into my environment.
    """
    return (prompt,)


@app.cell
def _(mo, prompt):
    mo.callout(
        mo.md(
            f"""
    This is a prototype for filtering your CSV data using definitions from the [Open Syndrome Initiative](https://opensyndrome.org/).

    You can either provide a sample of your own data, up to 10 MB, or generate a toy dataset using your preferred GenAI tool.

    <details>
    <summary>Prompt</summary>
    ```
    {prompt}
    ```
    </details>

    Next, you will need to create a map of your data and the Open Syndrome Definition concepts you want to filter on. Don't worry! We have an example ready for you.

    **Please note that we do not store any data**.
    """
        ),
        kind="neutral",
    )
    return


@app.cell
def _():
    TOY_DATASET_CSV = """recording_ts,icd_code,sex,age,location
2025-04-21 07:27:48,K21,F,5,Frankfurt
2024-04-29 15:47:31,R10,F,51,Bochum
2025-02-13 03:35:35,M54,F,45,Leipzig
2025-05-26 21:38:55,J18,D,17,Frankfurt
2024-05-27 02:19:18,J45,M,79,Hanover
2023-08-05 20:58:57,M54,D,90,Hanover
2024-01-22 12:51:25,I25,M,39,Hamburg
2024-08-05 13:09:02,E11,D,42,Dortmund
2024-07-16 20:38:52,R10,M,45,Düsseldorf
2024-05-14 22:49:06,F32,D,81,Bochum
2025-06-17 15:25:39,J18,D,61,Dortmund
2025-03-05 05:21:49,R10,F,7,Hanover
2025-04-09 23:52:42,R10,M,11,Dresden
2024-02-26 00:27:29,K21,D,100,Bremen
2024-07-09 14:42:37,A90,D,63,Nuremberg
2024-04-08 12:49:23,F32,M,93,Stuttgart
2024-09-30 13:19:42,E11,M,72,Cologne
2025-05-28 00:03:30,J18,M,69,Dortmund
2025-06-09 18:45:01,E11,M,40,Bochum
2023-12-22 02:22:00,R10,F,13,Frankfurt
2024-07-31 10:20:38,R10,F,90,Hamburg
2023-10-21 14:32:57,R10,F,69,Cologne
2024-02-09 08:08:14,I25,D,75,Frankfurt
2023-10-21 23:20:57,N39,M,76,Hanover
2025-02-11 11:30:38,K21,F,73,Cologne
2023-09-17 02:04:21,M54,M,24,Nuremberg
2024-10-27 17:31:57,J45,M,78,Bremen
2025-03-17 08:14:35,M54,F,8,Hamburg
2024-01-16 00:04:32,J45,M,4,Nuremberg
2025-06-11 01:11:54,E11,D,69,Bremen
2024-08-22 02:00:17,K21,D,67,Frankfurt
2025-01-03 03:39:45,R10,M,27,Bochum
2025-03-25 17:50:35,I25,F,74,Frankfurt
2024-10-14 22:34:01,N39,F,84,Bremen
2025-06-05 03:53:29,F32,M,41,Leipzig
2023-10-27 13:51:29,E11,F,75,Bremen
2024-03-02 07:52:08,F32,M,31,Berlin
2023-09-20 03:22:52,M54,M,90,Cologne
2025-01-22 07:20:56,F32,M,42,Düsseldorf
2023-08-31 16:57:27,A90,M,100,Munich
2025-06-02 13:51:33,J18,M,73,Bremen
2025-04-26 00:16:49,R10,D,87,Hamburg
2024-12-17 16:41:01,A90,M,81,Cologne
2024-03-17 21:37:17,I25,D,15,Düsseldorf
2024-07-06 03:40:38,E11,F,14,Berlin
2024-09-12 05:58:57,I25,M,24,Munich
2023-09-21 15:30:20,E11,F,26,Dresden
2024-04-05 05:26:51,A90,D,2,Essen
2024-06-06 15:50:55,K21,D,12,Nuremberg
2024-03-07 00:37:58,M54,M,28,Hamburg
2024-10-21 22:03:27,M54,F,55,Munich
2024-11-13 20:39:23,A90,D,59,Berlin
2025-06-13 14:03:31,I25,M,89,Düsseldorf
2023-12-11 06:02:26,J18,F,45,Bochum
2024-09-06 14:05:13,N39,D,21,Dresden
2024-07-09 22:24:21,J18,M,100,Bremen
2024-06-13 06:42:05,J45,M,43,Essen
2025-05-07 12:50:43,M54,M,76,Bochum
2024-08-13 09:02:07,N39,D,22,Berlin
2024-01-23 18:13:40,N39,D,52,Bochum
2023-08-22 19:16:29,I25,D,39,Bremen
2025-03-21 17:06:19,F32,F,84,Frankfurt
2025-06-17 19:43:32,J45,D,88,Berlin
2024-09-03 14:47:43,N39,D,10,Stuttgart
2025-06-15 21:40:26,A90,D,35,Munich
2023-07-16 08:03:19,J18,F,45,Leipzig
2023-09-15 21:00:45,M54,D,45,Leipzig
2025-01-14 10:34:00,I25,M,91,Frankfurt
2025-04-17 22:44:04,K21,D,53,Nuremberg
2024-05-10 04:54:31,E11,M,76,Cologne
2025-01-07 02:22:57,F32,M,30,Cologne
2025-04-06 02:31:26,N39,F,90,Bochum
2025-03-06 04:39:00,I25,F,4,Düsseldorf
2024-12-05 09:34:39,I25,F,98,Bremen
2024-11-02 02:05:56,A90,M,57,Hamburg
2023-07-06 07:36:10,M54,D,20,Dortmund
2024-10-22 16:53:15,R10,F,71,Leipzig
2024-07-22 19:15:26,A90,M,63,Stuttgart
2025-04-08 14:31:02,M54,F,6,Hanover
2025-05-09 07:35:13,K21,M,70,Bremen
2024-03-20 04:23:17,E11,D,16,Berlin
2024-01-10 21:07:38,K21,D,53,Stuttgart
2025-01-22 15:40:42,A90,M,1,Dresden
2024-03-14 00:32:52,A90,D,67,Leipzig
2025-01-01 23:08:48,E11,M,15,Leipzig
2025-05-11 13:03:32,J18,F,35,Dresden
2023-08-23 08:24:40,J45,M,60,Nuremberg
2023-11-30 20:02:52,K21,D,10,Berlin
2025-06-16 16:46:18,M54,F,14,Nuremberg
2024-09-18 05:27:58,M54,M,83,Essen
2023-12-24 15:09:33,F32,M,19,Frankfurt
2024-01-06 18:25:20,A90,M,5,Cologne
2025-03-20 09:44:10,M54,D,40,Stuttgart
2023-10-24 02:35:13,I25,M,95,Dresden
2025-03-20 06:38:58,I25,D,63,Dresden
2024-08-30 04:21:56,N39,D,55,Stuttgart
2024-06-17 09:03:26,R10,M,26,Düsseldorf
2023-09-14 06:41:16,I25,F,1,Munich
2023-09-05 21:55:04,J45,F,42,Stuttgart
2024-03-10 00:32:57,F32,D,11,Stuttgart
2025-05-15 03:06:30,I25,M,5,Frankfurt
2024-08-02 22:59:56,J45,M,74,Frankfurt
2025-06-28 23:34:55,F32,F,70,Munich
2024-07-19 11:15:57,M54,M,61,Dresden
2023-09-28 03:17:05,J18,M,39,Munich
2024-02-24 13:40:28,R10,D,9,Frankfurt
2024-09-05 14:39:55,K21,F,38,Düsseldorf
2024-02-16 12:37:21,E11,M,71,Bochum
2025-01-04 05:33:30,N39,F,43,Nuremberg
2023-10-10 08:56:22,F32,M,61,Hamburg
2024-09-02 15:20:14,N39,F,4,Frankfurt
2024-03-19 21:13:05,F32,D,87,Bochum
2025-03-29 01:03:42,J45,M,80,Leipzig
2025-06-20 18:16:48,K21,D,11,Hamburg
2024-08-28 01:54:56,E11,M,95,Cologne
2023-12-10 06:15:22,A90,F,1,Hamburg
2025-03-17 08:42:02,K21,D,66,Frankfurt
2024-10-15 22:29:39,N39,F,100,Leipzig
2024-09-02 11:24:27,J18,F,10,Stuttgart
2024-10-03 20:16:30,J18,F,74,Hanover
2024-02-13 20:37:09,J45,F,24,Stuttgart
2024-06-15 01:29:45,E11,M,89,Berlin
2023-09-14 14:26:42,R10,F,96,Bremen
2024-10-10 13:53:30,J18,M,63,Düsseldorf
2025-03-16 04:53:09,M54,M,82,Berlin
2024-04-29 09:22:35,J18,D,18,Hamburg
2024-03-07 16:28:50,J18,F,48,Stuttgart
2024-09-24 13:12:42,R10,M,13,Leipzig
2024-01-09 11:30:13,N39,M,72,Düsseldorf
2024-02-15 22:26:00,K21,D,63,Bremen
2023-11-25 05:51:38,F32,F,63,Bremen
2023-11-29 12:48:33,J18,D,78,Bochum
2025-04-03 11:33:28,J18,M,43,Dresden
2023-11-27 11:11:23,F32,F,4,Essen
2023-07-31 00:56:58,J45,F,77,Hanover
2024-10-23 09:13:08,J45,F,74,Frankfurt
2024-11-26 11:19:02,N39,M,10,Essen
2024-01-23 10:05:53,A90,M,28,Munich
2024-07-09 13:28:50,A90,F,1,Hanover
2023-08-28 02:08:29,N39,F,20,Hanover
2023-09-13 04:44:47,J45,D,58,Stuttgart
2024-12-13 14:37:50,R10,F,67,Essen
2025-02-12 21:03:35,A90,D,11,Bremen
2023-10-14 13:51:31,R10,D,9,Dresden
2025-05-16 10:29:00,K21,M,37,Essen
2023-10-29 14:07:09,I25,F,61,Nuremberg
2025-06-21 15:22:08,K21,D,75,Cologne
2023-08-04 10:37:48,A90,D,0,Dresden
2025-04-19 18:53:45,J45,F,64,Leipzig
2023-09-05 18:14:10,M54,F,8,Dortmund
2025-06-02 01:27:34,M54,F,98,Düsseldorf
2024-12-29 14:37:15,K21,F,7,Munich
2025-06-21 09:48:10,J45,M,36,Dresden
2025-04-09 14:42:22,F32,M,4,Dortmund
2024-08-28 07:48:43,K21,M,62,Bochum
2024-12-17 07:37:57,I25,D,10,Bremen
2024-05-04 08:35:10,J45,F,52,Berlin
2025-04-16 05:14:20,I25,F,49,Dortmund
2025-01-03 05:44:04,A90,M,60,Hanover
2024-08-15 21:03:58,J45,M,4,Leipzig
2024-02-04 12:29:29,I25,M,80,Stuttgart
2024-08-13 17:10:17,E11,D,70,Bremen
2025-03-26 20:09:17,F32,M,49,Hanover
2024-01-04 08:22:10,N39,M,7,Leipzig
2025-04-25 05:51:55,N39,D,80,Bochum
2023-12-19 13:56:31,F32,D,15,Bremen
2025-05-30 15:02:05,I25,F,100,Nuremberg
2024-12-14 04:37:09,J45,F,37,Bochum
2024-10-05 12:59:24,E11,D,100,Cologne
2025-04-06 10:19:19,A90,F,56,Stuttgart
2024-05-26 04:54:54,J18,F,45,Hanover
2024-09-27 16:35:50,E11,M,5,Dortmund
2024-02-04 04:36:02,M54,M,66,Bremen
2024-06-19 09:24:15,I25,D,27,Cologne
2024-07-22 02:03:40,E11,D,99,Essen
2024-11-29 06:09:54,R10,F,64,Frankfurt
2024-11-14 11:04:27,E11,M,54,Bochum
2024-12-30 07:26:56,I25,F,10,Bochum
2025-06-16 15:19:20,E11,F,8,Hamburg
2025-03-12 14:57:02,K21,M,93,Berlin
2024-04-08 00:06:47,N39,F,87,Düsseldorf
2023-09-07 09:50:34,A90,F,41,Dresden
2024-03-13 08:09:08,M54,M,45,Hamburg
2024-10-23 09:24:15,E11,F,88,Berlin
2023-10-03 00:34:27,F32,F,22,Berlin
2024-07-17 06:16:34,J18,F,9,Leipzig
2024-10-03 10:09:53,J45,M,0,Cologne
2024-11-05 21:05:49,E11,D,0,Frankfurt
2024-03-21 20:11:37,F32,D,3,Bochum
2025-04-21 19:41:13,I25,M,18,Munich
2023-11-30 07:13:05,N39,M,61,Stuttgart
2024-01-10 04:48:41,M54,M,3,Cologne
2023-07-18 00:27:26,F32,F,60,Frankfurt
2025-01-23 14:14:47,M54,D,81,Stuttgart
2024-01-31 22:48:46,J45,D,10,Hamburg
2023-10-04 09:50:26,R10,D,39,Munich
2025-01-03 06:39:41,K21,M,16,Hanover
2023-12-23 15:58:53,J18,F,65,Cologne
2024-04-19 10:00:13,J18,M,37,Stuttgart
2025-03-13 14:23:57,K21,D,5,Nuremberg
2023-11-14 20:18:35,J45,D,2,Düsseldorf
2024-03-19 17:13:20,E11,D,9,Munich
2024-05-31 09:57:33,K21,F,70,Düsseldorf
2025-03-30 04:01:18,J45,D,54,Frankfurt
2023-08-22 17:39:02,F32,M,31,Dortmund
2023-09-21 13:45:33,F32,D,67,Berlin
2025-03-04 16:41:10,K21,F,1,Düsseldorf
2023-10-27 02:37:08,F32,F,26,Stuttgart
2025-04-29 18:40:06,M54,F,11,Munich
2024-12-28 07:49:21,E11,F,14,Essen
2023-12-22 08:32:02,I25,D,19,Hanover
2023-08-09 13:31:23,N39,F,23,Hanover
2025-03-31 02:31:32,K21,F,22,Cologne
2024-09-23 03:56:14,N39,F,66,Munich
2024-11-09 00:45:48,F32,F,80,Bremen
2025-06-11 06:13:37,E11,F,96,Cologne
2025-07-01 20:17:31,M54,M,89,Dortmund
2024-08-23 08:28:15,I25,F,0,Cologne
2024-10-15 03:34:14,M54,M,98,Bremen
2024-12-15 01:24:00,M54,D,77,Munich
2023-08-31 21:42:04,K21,D,96,Dortmund
2023-07-26 03:57:48,E11,D,63,Hanover
2023-10-10 02:05:53,J18,D,97,Düsseldorf
2024-11-09 15:33:19,M54,D,2,Hamburg
2023-11-28 20:34:41,M54,D,5,Berlin
2024-10-18 09:15:59,M54,F,67,Bochum
2025-04-10 04:13:34,I25,D,50,Dortmund
2024-09-24 18:50:25,E11,D,32,Stuttgart
2023-10-06 09:35:04,M54,D,25,Leipzig
2024-05-04 06:58:16,E11,M,9,Hanover
2023-09-14 12:34:35,M54,F,68,Stuttgart
2024-04-17 22:12:27,E11,D,31,Bochum
2025-04-13 02:17:09,J45,M,53,Nuremberg
2023-10-03 13:33:21,M54,F,66,Munich
2023-08-02 16:41:45,I25,D,80,Cologne
2024-10-17 17:42:54,R10,M,52,Bremen
2023-07-22 05:37:58,R10,F,94,Hanover
2023-12-03 13:44:14,M54,F,56,Stuttgart
2023-11-26 02:25:33,I25,D,17,Munich
2025-01-13 04:54:06,E11,D,15,Düsseldorf
2023-10-09 15:52:20,K21,D,59,Munich
2024-07-11 15:51:43,R10,D,38,Stuttgart
2023-12-15 23:43:47,N39,D,53,Cologne
2025-01-20 05:27:18,N39,F,88,Essen
2025-03-18 09:17:51,F32,F,83,Berlin
2023-08-21 21:00:14,N39,F,18,Dresden
2025-07-02 04:40:12,N39,M,79,Cologne
2024-12-14 02:14:55,A90,F,60,Düsseldorf
2023-07-23 02:59:49,A90,D,8,Bremen
2024-08-31 03:34:57,E11,D,94,Bochum
2024-05-31 00:24:33,K21,M,46,Berlin
2024-10-29 11:07:16,E11,D,0,Frankfurt
2023-09-22 06:24:01,M54,D,29,Munich
2025-04-14 16:39:14,I25,F,24,Hamburg
2024-01-23 07:33:25,K21,F,91,Stuttgart
2023-08-28 19:16:26,K21,M,42,Düsseldorf
2023-12-23 22:51:18,K21,M,57,Bochum
2023-08-30 07:56:55,J45,D,40,Munich
2024-11-18 21:52:07,J18,M,56,Stuttgart
2024-02-24 06:29:34,K21,F,62,Düsseldorf
2024-04-02 12:15:07,J18,M,56,Cologne
2024-06-08 03:12:28,I25,D,6,Bochum
2023-11-22 11:40:06,K21,M,29,Bremen
2023-08-21 20:12:45,E11,M,46,Berlin
2024-07-30 14:45:50,J45,M,78,Frankfurt
2024-07-02 06:15:23,I25,M,90,Nuremberg
2023-08-05 00:22:17,R10,F,98,Bochum
2024-10-10 17:35:02,F32,F,58,Berlin
2023-09-21 19:36:26,R10,D,83,Essen
2024-12-29 14:37:37,K21,D,58,Dortmund
2024-05-04 08:12:58,M54,D,60,Cologne
2023-12-07 06:33:37,F32,F,5,Berlin
2024-01-23 11:32:04,A90,M,44,Berlin
2024-04-08 22:15:15,M54,D,0,Munich
2025-06-06 21:30:41,E11,F,87,Cologne
2025-01-23 16:16:48,I25,F,71,Bochum
2024-07-04 15:45:49,J18,F,24,Stuttgart
2025-04-16 13:50:20,I25,M,77,Bochum
2025-06-05 06:28:08,E11,F,41,Essen
2025-06-17 20:02:44,N39,F,32,Berlin
2024-09-19 23:33:16,R10,M,24,Stuttgart
2023-11-14 00:36:09,E11,M,67,Stuttgart
2024-12-28 08:18:54,J18,M,32,Bremen
2024-12-08 21:13:41,M54,F,66,Nuremberg
2023-12-10 10:13:25,K21,F,61,Stuttgart
2025-02-20 10:42:32,J18,M,39,Bochum
2023-08-19 12:26:49,R10,M,1,Dortmund
2025-03-08 23:46:07,N39,D,56,Berlin
2024-12-11 15:36:50,K21,F,58,Dortmund
2023-11-05 07:31:18,E11,M,10,Cologne
2025-04-22 02:09:21,E11,M,52,Bochum
2023-11-08 05:20:30,J18,F,78,Hamburg
2024-03-27 07:28:02,K21,D,96,Bochum
2024-08-19 16:24:50,K21,M,23,Cologne
2024-01-16 03:26:59,N39,M,16,Nuremberg
2023-10-04 11:18:17,M54,F,40,Düsseldorf
2024-10-10 23:17:49,E11,D,36,Leipzig
2025-06-20 02:46:58,R10,M,91,Frankfurt
2024-06-12 23:37:49,N39,D,77,Dortmund
2024-12-03 10:05:27,R10,D,33,Frankfurt
2023-11-18 05:10:14,J18,M,15,Leipzig
2023-07-15 16:26:06,E11,M,93,Düsseldorf
2024-07-05 18:02:02,J18,M,36,Bochum
2024-12-15 14:17:29,A90,D,68,Essen
2024-05-18 19:29:22,K21,M,15,Düsseldorf
2025-06-03 13:20:34,M54,M,94,Leipzig
2024-09-19 11:19:46,F32,M,86,Dresden
2025-05-14 21:50:26,R10,D,36,Cologne
2024-08-03 11:17:33,J18,M,66,Frankfurt
2025-06-17 01:56:46,F32,M,47,Bremen
2024-03-03 08:04:24,N39,F,74,Munich
2024-11-20 11:17:46,J45,M,70,Essen
2024-04-07 20:05:20,F32,F,74,Bremen
2024-05-02 07:06:56,A90,M,50,Munich
2025-04-16 01:26:06,J45,D,9,Munich
2024-11-27 05:09:13,J18,F,72,Hanover
2024-01-16 08:01:23,J18,M,93,Munich
2023-11-01 14:07:13,J18,F,45,Leipzig
2024-09-12 20:38:40,I25,M,80,Dortmund
2024-12-23 00:10:07,E11,D,3,Essen
2023-07-15 02:07:44,I25,F,62,Dortmund
2024-06-25 03:11:03,M54,M,28,Essen
2025-01-31 10:14:51,J45,D,63,Hanover
2023-09-04 20:20:29,N39,D,40,Dresden
2024-03-22 08:14:15,E11,F,17,Leipzig
2025-01-29 10:33:29,K21,D,24,Nuremberg
2023-08-09 22:08:36,F32,F,49,Berlin
2024-08-31 05:09:00,J18,M,40,Dresden
2024-02-08 14:20:27,J18,F,56,Bremen
2023-10-18 03:51:15,J18,F,44,Bremen
2024-01-07 22:39:15,J45,F,2,Stuttgart
2025-02-16 10:06:55,I25,D,7,Dresden
2024-07-13 16:12:25,J45,F,2,Dortmund
2024-04-04 00:15:34,A90,M,30,Berlin
2024-10-21 08:03:07,A90,M,83,Stuttgart
2024-02-14 00:43:56,E11,M,44,Bremen
2024-12-09 11:26:58,K21,M,27,Dortmund
2024-04-09 10:17:41,K21,M,45,Frankfurt
2024-10-05 21:52:21,J45,D,42,Dresden
2025-05-28 05:19:54,K21,F,1,Düsseldorf
2024-11-08 16:12:55,M54,D,68,Hanover
2025-04-28 13:48:25,N39,M,72,Bochum
2024-10-29 20:00:38,E11,F,49,Munich
2023-10-16 05:22:31,A90,D,17,Leipzig
2025-01-11 06:24:38,R10,D,89,Munich
2024-03-02 10:39:55,E11,F,30,Nuremberg
2024-05-01 19:27:52,J45,M,2,Hanover
2025-02-05 19:05:14,J45,D,100,Nuremberg
2024-05-22 07:36:37,R10,M,91,Hamburg
2025-04-09 06:10:00,K21,D,13,Leipzig
2024-11-21 22:42:11,N39,D,19,Leipzig
2023-09-06 03:05:13,I25,M,32,Stuttgart
2023-07-06 14:42:45,K21,M,80,Bochum
2024-10-09 06:00:59,A90,F,11,Stuttgart
2023-12-01 19:12:31,M54,D,19,Dortmund
2024-12-29 00:49:27,J18,D,45,Munich
2024-03-21 03:59:54,K21,F,34,Hanover
2023-11-06 18:24:09,N39,F,1,Frankfurt
2024-01-07 09:20:10,R10,F,70,Dortmund
2023-12-25 17:08:38,A90,D,73,Essen
2024-11-11 12:17:06,M54,D,4,Dortmund
2024-06-17 12:08:49,K21,D,15,Düsseldorf
2024-04-24 12:48:38,F32,F,6,Berlin
2025-01-07 04:10:56,M54,D,4,Frankfurt
2025-05-15 09:22:26,I25,D,99,Bochum
2024-12-09 18:22:45,M54,D,97,Cologne
2024-09-18 17:24:33,R10,D,43,Düsseldorf
2025-05-14 16:51:45,M54,M,14,Leipzig
2024-05-22 07:23:39,F32,M,75,Bremen
2023-07-29 22:20:58,R10,D,45,Munich
2024-05-12 21:06:03,J45,F,95,Nuremberg
2025-05-12 11:44:41,A90,D,6,Leipzig
2024-12-31 13:21:05,J45,F,46,Frankfurt
2024-09-18 04:45:01,M54,F,63,Hanover
2024-05-25 06:36:25,K21,D,55,Munich
2025-02-24 04:53:03,A90,M,72,Berlin
2024-11-30 18:31:30,N39,M,43,Bochum
2025-01-06 03:07:46,A90,D,61,Bochum
2023-08-10 18:20:08,M54,D,78,Cologne
2023-11-11 02:42:16,E11,D,54,Frankfurt
2024-06-15 08:15:56,J45,D,21,Hamburg
2023-08-24 01:13:34,J45,D,14,Essen
2024-01-22 18:38:21,R10,D,49,Hanover
2024-12-31 07:30:30,K21,F,39,Frankfurt
2023-11-04 21:47:23,A90,F,99,Nuremberg
2024-06-16 03:08:25,M54,F,68,Essen
2024-08-18 04:58:30,R10,F,43,Cologne
2025-05-22 18:52:44,K21,M,0,Hanover
2024-07-17 21:35:17,R10,M,100,Bremen
2025-06-09 17:23:20,I25,F,46,Dortmund
2024-09-13 11:38:43,A90,D,52,Bremen
2024-10-16 13:15:13,I25,M,2,Stuttgart
2025-02-11 05:33:21,R10,M,87,Cologne
2023-07-05 11:35:18,F32,D,89,Dortmund
2025-05-21 22:55:34,A90,D,94,Cologne
2023-08-04 15:38:15,I25,M,35,Munich
2024-07-23 13:59:18,K21,M,73,Dortmund
2024-03-14 14:20:02,J18,D,91,Dortmund
2024-03-03 19:29:11,R10,D,12,Cologne
2025-04-16 02:50:44,J45,F,8,Bochum
2024-07-02 01:48:04,K21,D,50,Frankfurt
2023-08-13 03:42:03,M54,F,96,Hanover
2025-03-24 20:00:25,F32,D,41,Hamburg
2025-04-04 10:54:36,M54,M,63,Berlin
2025-05-15 01:47:08,M54,M,97,Düsseldorf
2025-03-27 10:48:40,K21,F,99,Bremen
2024-05-22 21:01:17,K21,D,4,Leipzig
2023-11-28 10:53:50,N39,F,72,Munich
2024-09-03 12:14:13,I25,D,35,Stuttgart
2024-11-08 03:55:44,A90,F,60,Bochum
2023-09-08 06:14:15,R10,M,5,Hamburg
2025-04-15 21:28:07,I25,F,46,Berlin
2025-01-18 23:36:45,E11,M,91,Hamburg
2023-10-14 19:24:03,R10,F,5,Stuttgart
2023-07-16 20:52:22,A90,F,50,Bochum
2024-05-12 12:42:27,J45,D,35,Düsseldorf
2024-12-13 10:44:08,J45,D,18,Düsseldorf
2024-05-24 16:47:06,M54,D,7,Munich
2024-03-07 00:16:05,J45,M,61,Dresden
2023-07-08 01:32:44,A90,D,66,Berlin
2023-12-18 06:41:14,R10,D,95,Dresden
2023-07-16 12:42:30,K21,M,44,Hanover
2025-01-12 14:11:25,I25,M,10,Essen
2025-02-10 15:55:48,J45,F,25,Hanover
2023-07-07 06:14:50,M54,F,89,Dresden
2025-04-04 11:07:27,M54,F,66,Bochum
2024-12-11 07:24:50,N39,M,96,Bochum
2024-08-18 19:22:55,N39,D,19,Berlin
2024-10-22 19:57:40,I25,M,82,Essen
2023-09-19 09:07:53,A90,F,9,Cologne
2023-11-30 08:43:46,N39,D,30,Hanover
2023-10-05 18:49:42,N39,D,20,Dresden
2024-10-19 22:56:41,F32,D,17,Dortmund
2024-11-20 17:29:02,R10,M,69,Hamburg
2023-12-06 14:54:45,R10,F,0,Nuremberg
2025-05-14 00:23:12,E11,M,54,Leipzig
2024-02-25 07:44:34,F32,D,57,Stuttgart
2024-02-07 21:33:20,K21,D,46,Nuremberg
2024-12-21 05:36:54,E11,M,40,Bochum
2023-11-23 23:08:51,A90,M,93,Munich
2023-09-21 05:48:09,A90,F,77,Cologne
2024-07-08 20:13:40,A90,F,82,Berlin
2025-04-28 16:51:33,M54,F,6,Leipzig
2025-06-21 03:58:25,J45,F,9,Düsseldorf
2024-05-08 13:15:06,A90,F,45,Leipzig
2024-11-03 22:20:24,I25,F,86,Frankfurt
2023-09-06 14:41:15,I25,F,52,Munich
2023-07-22 09:31:43,A90,F,33,Cologne
2024-02-03 19:20:10,K21,M,45,Hamburg
2025-06-14 06:16:40,E11,M,44,Berlin
2024-06-21 06:25:28,J45,F,78,Dresden
2025-03-17 16:06:34,A90,F,58,Hanover
2025-03-11 20:36:57,R10,D,88,Dortmund
2023-10-03 08:35:00,N39,M,6,Essen
2025-05-04 11:12:41,K21,F,3,Bremen
2024-01-20 15:35:47,R10,M,10,Stuttgart
2023-10-17 01:26:18,F32,M,60,Berlin
2024-11-24 01:40:15,J45,D,80,Hanover
2024-08-12 23:56:10,M54,M,0,Düsseldorf
2025-03-31 05:22:34,F32,M,70,Dresden
2024-11-21 21:46:40,J45,M,22,Hanover
2024-11-17 08:51:52,J45,D,87,Cologne
2024-03-30 23:50:10,I25,F,3,Dortmund
2023-10-02 20:05:34,K21,M,28,Cologne
2025-05-12 15:01:08,M54,F,21,Cologne
2023-11-02 09:16:19,F32,M,20,Bochum
2024-02-06 16:14:03,K21,F,46,Bochum
2025-04-26 14:33:29,I25,M,49,Leipzig
2023-07-19 20:32:59,A90,M,74,Berlin
2023-08-07 06:47:01,K21,D,22,Munich
2023-08-06 00:19:03,A90,M,41,Essen
2023-12-12 05:29:28,A90,M,6,Hanover
2024-12-15 08:59:56,E11,D,78,Munich
2024-01-23 02:53:14,J45,D,100,Düsseldorf
2024-09-11 02:48:16,A90,F,55,Leipzig
2024-03-05 20:58:44,F32,D,31,Bochum
2024-04-15 09:41:42,J45,F,65,Cologne
2025-07-03 08:03:58,R10,F,9,Munich
2024-09-18 04:39:55,K21,D,84,Stuttgart
2025-06-11 03:09:15,E11,F,55,Cologne
2025-04-25 04:58:02,N39,F,28,Bochum
2023-08-07 01:19:13,K21,M,82,Dortmund
2025-04-22 19:35:29,K21,D,8,Nuremberg
2025-02-06 07:11:03,M54,F,67,Stuttgart
2024-12-31 03:56:28,R10,M,77,Leipzig
2024-10-19 08:43:07,N39,M,35,Berlin
2024-11-25 11:07:47,I25,F,50,Bremen
2024-07-23 12:10:48,E11,D,6,Munich
2025-01-08 17:36:29,K21,M,98,Düsseldorf
2024-11-16 18:00:12,K21,F,13,Dresden
2025-06-05 14:02:09,N39,D,59,Munich
2024-06-25 16:25:21,J45,F,60,Düsseldorf
2024-11-13 15:23:45,J45,D,36,Hanover
2024-06-07 15:30:02,R10,M,47,Stuttgart
2024-04-19 17:15:23,J45,D,45,Hanover
2025-05-12 06:34:21,N39,D,66,Hanover
2024-08-26 06:01:06,A90,M,32,Munich
2023-11-20 21:50:03,I25,F,37,Düsseldorf
2023-12-13 20:47:38,A90,F,70,Düsseldorf
2024-04-22 03:35:51,A90,D,52,Frankfurt"""
    return (TOY_DATASET_CSV,)


@app.cell
def _():
    TOY_DATASET_MAPPING = "\n".join(
        [
            "profiles:",
            "- name: ambulatory_care",
            "  value_encodings:",
            "    sex:",
            '      male: "M"',
            '      female: "F"',
            '      other: "O"',
            "  columns:",
            "    age:",
            "      concept: demographic_criteria",
            "      attribute: age",
            "      dtype: integer",
            "    sex:",
            "      concept: demographic_criteria",
            "      attribute: sex",
            "      dtype: string",
            "    icd_code:",
            "      concept: diagnosis",
            "      dtype: string",
            "",
        ]
    )
    return (TOY_DATASET_MAPPING,)


@app.cell
def _(TOY_DATASET_CSV, TOY_DATASET_MAPPING):
    EXAMPLE_DATASETS = {
        "Toy dataset": {
            "csv": TOY_DATASET_CSV,
            "mapping": TOY_DATASET_MAPPING,
            "date_column": "recording_ts",
        },
    }
    return (EXAMPLE_DATASETS,)


@app.cell
def _(EXAMPLE_DATASETS, mo):
    data_source = mo.ui.radio(
        options=["Load example", "Upload your own"],
        value="Load example",
    )
    _datasets = list(EXAMPLE_DATASETS.keys())
    example_picker = mo.ui.dropdown(
        options=_datasets,
        value=_datasets[0] if _datasets else None,
        label="Example dataset",
    )
    sample_file = mo.ui.file(kind="area", filetypes=[".csv"], max_size=10_000_000)
    return data_source, example_picker, sample_file


@app.cell
def _(data_source, example_picker, mo, sample_file):
    mo.vstack(
        [
            mo.md("## Load your data"),
            data_source,
            example_picker if data_source.value == "Load example" else sample_file,
        ]
    )
    return


@app.cell
def _(EXAMPLE_DATASETS, data_source, example_picker, pl, sample_file):
    if data_source.value == "Load example":
        df_selected = (
            pl.read_csv(EXAMPLE_DATASETS[example_picker.value]["csv"].encode())
            if example_picker.value
            else None
        )
    else:
        df_selected = pl.read_csv(sample_file.contents()) if sample_file.value else None
    return (df_selected,)


@app.cell
def _(EXAMPLE_DATASETS, data_source, example_picker):
    _default_yaml = """
    profiles:
    - name: my_dataset
      # value_encodings:  # optional — map OSD canonical values to dataset-specific ones
      #   sex:
      #     male: "M"
      #     female: "F"
      columns:
    # Rename the keys below to match your dataset column names.
    # Available concepts: diagnosis, demographic_criteria, symptom,
    #                     diagnostic_test, epidemiological_history
    my_diagnosis_column:
      concept: diagnosis
      dtype: string
    # my_age_column:
    #   concept: demographic_criteria
    #   attribute: age
    #   dtype: integer
    """
    if data_source.value == "Load example" and example_picker.value:
        _example = EXAMPLE_DATASETS[example_picker.value]
        initial_yaml = _example["mapping"]
        initial_date_column = _example["date_column"]
    else:
        initial_yaml = _default_yaml
        initial_date_column = None
    return initial_date_column, initial_yaml


@app.cell
def _(df_selected, initial_date_column, initial_yaml, mo):
    mo.stop(df_selected is None)

    yaml_editor = mo.ui.code_editor(
        value=initial_yaml,
        language="yaml",
    ).form(label="Data to OSD mapping", show_clear_button=True, bordered=True)

    date_column_picker = mo.ui.dropdown(
        options=df_selected.columns,
        label="Date column",
        value=initial_date_column,
    )

    date_format_input = mo.ui.text(
        value="%Y-%m-%d %H:%M:%S",
        label="Date format<sup>1</sup>",
    )

    date_block = mo.vstack(
        [
            date_format_input,
            mo.md(
                "[^1]: A Python date format code compatible with your data. See other date formats [here](https://strftime.org/)."
            ),
        ]
    )

    _cols_hint = "`, `".join(df_selected.columns)

    mo.vstack(
        [
            mo.md("### Mapping your data to the format"),
            mo.md(
                "Edit the YAML below to map your dataset columns to OSD concepts, "
                "then click **Submit**. "
                "Select the date column separately for the time-series view.\n\n"
                f"Your dataset columns: `{_cols_hint}`"
            ),
            mo.hstack(
                [yaml_editor, mo.vstack([date_column_picker, date_block])],
                widths=[3, 1],
                align="start",
            ),
        ]
    )
    return date_column_picker, date_format_input, yaml_editor


@app.cell
def _(df_selected, initial_yaml, load_profile, mo, yaml, yaml_editor):
    _yaml_to_use = yaml_editor.value if yaml_editor.value is not None else initial_yaml
    mo.stop(_yaml_to_use is None)

    try:
        _parsed = yaml.safe_load(_yaml_to_use)
    except yaml.YAMLError as _e:
        mo.stop(True, mo.callout(mo.md(f"**Invalid YAML:** {_e}"), kind="danger"))

    if not _parsed["profiles"][0]["columns"]:
        mo.stop(
            True,
            mo.callout(mo.md("You need to map **at least one column**"), kind="danger"),
        )

    not_found = []
    for declared_column in _parsed["profiles"][0]["columns"]:
        if declared_column not in df_selected.columns:
            not_found.append(declared_column)

    if not_found:
        mo.stop(
            True,
            mo.callout(
                mo.md(f"**Columns not found:** {', '.join(not_found)}"), kind="danger"
            ),
        )

    try:
        _profile_name = _parsed["profiles"][0]["name"]
        profile = load_profile(_parsed, _profile_name)
    except (KeyError, IndexError, ValueError) as _e:
        mo.stop(True, mo.callout(mo.md(f"**Profile error:** {_e}"), kind="danger"))
    return (profile,)


@app.cell
def _(date_column_picker, mo, profile):
    mo.stop(date_column_picker.value is None)
    date_column = date_column_picker.value
    _diagnosis_cols = [c.col_name for c in profile.columns if c.concept == "diagnosis"]
    code_column = _diagnosis_cols[0] if _diagnosis_cols else None
    return code_column, date_column


@app.cell
def _(get_definition_dir):
    definition_options = {
        filepath.name.replace(".json", ""): filepath
        for filepath in get_definition_dir().glob("**/*.json")
    }
    return (definition_options,)


@app.cell
def _(definition_options, mo):
    definitions_dropdown = mo.ui.multiselect(
        label="Select Syndromic Indicators", options=sorted(definition_options.keys())
    )
    return (definitions_dropdown,)


@app.cell
def _(mo):
    mo.md(r"""
    ### Data sample
    """)
    return


@app.cell
def _(df_selected, mo):
    mo.stop(df_selected is None)
    df_selected.sample(10)
    return


@app.cell
def _(mo):
    mo.md(r"""
    ---
    """)
    return


@app.cell
def _(mo):
    mo.md(r"""
    ## Data & Definitions
    """)
    return


@app.cell
def _(definitions_dropdown, mo):
    mo.hstack([mo.md("**::lucide:filter:: Filters:**"), definitions_dropdown])
    return


@app.cell
def _(
    OSDEngine,
    definition_options,
    definitions_dropdown,
    df_selected,
    json,
    mo,
    profile,
):
    mo.stop(
        df_selected is None or df_selected.is_empty() or not definitions_dropdown.value
    )

    definitions = definitions_dropdown.value

    # skip criteria that can't be evaluated (e.g. professional_judgment)
    engine = OSDEngine(profile, skip_unresolvable=True)

    defs_dict = {
        name: json.loads(definition_options[name].read_text()) for name in definitions
    }
    df_filtered = engine.label(df_selected, defs_dict)
    return definitions, df_filtered


@app.cell
def _(definitions, df_filtered, df_selected, mo):
    mo.stop(definitions is None or df_filtered is None)

    _cards = [
        mo.stat(
            label="Syndromic Indicators",
            value=len(definitions),
            caption=", ".join([definition for definition in definitions]),
            bordered=True,
        ),
        mo.stat(
            label="Rows",
            value=df_selected.shape[0],
        ),
        mo.stat(
            label="Columns",
            value=df_selected.shape[1],
        ),
    ]

    mo.hstack(_cards, widths="equal", align="center")
    return


@app.cell
def _(definition_options, json):
    def load_definition(name: str) -> dict:
        return json.loads(definition_options[name].read_text())

    return (load_definition,)


@app.cell
def _(mo):
    top_n = mo.ui.number(start=1, stop=10, label="Number of top codes", value=3, step=1)
    return (top_n,)


@app.cell
def _(
    code_column,
    date_column,
    date_format_input,
    definitions,
    df_filtered,
    df_selected,
    groupped_bar,
    mo,
    plot_cases,
    top_n,
):
    mo.stop(definitions is None or df_selected is None)

    if code_column:
        diagnosis_chart = [
            mo.md("### Codes comparison per syndromic indicator"),
            top_n.left(),
            groupped_bar(
                df_filtered,
                definitions,
                top_n=top_n.value or 3,
                group_by_column=code_column,
            ),
        ]
    else:
        diagnosis_chart = []

    timeseries = [
        mo.md("### Time series"),
        plot_cases(
            df_filtered,
            definitions,
            date_column=date_column,
            date_format=date_format_input.value,
        ),
    ]

    mo.vstack(
        [
            mo.md("## Findings from the data 📊"),
            *timeseries,
            *diagnosis_chart,
        ]
    )
    return


@app.cell
def _(definitions, load_definition, mo):
    mo.stop(definitions is None)

    mo.vstack(
        [
            mo.md("### Definitions details"),
            mo.md(
                "Here the definitions used to filter the data. See here what criteria were applied. 🔎"
            ),
            mo.accordion(
                {
                    "JSONs": mo.accordion(
                        {
                            definition: mo.json(load_definition(definition))
                            for definition in definitions
                        }
                    ),
                },
            ),
        ]
    )
    return


if __name__ == "__main__":
    app.run()