Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /squirrels-analytics_squirrels /tests /_models_basic_test.py
| import pytest, polars as pl | |
| from pathlib import Path | |
| from squirrels import _models as m, _utils as u, _model_queries as mq | |
| from squirrels._arguments.init_time_args import ParametersArgs | |
| from squirrels._arguments.run_time_args import ContextArgs | |
| from squirrels._schemas.auth_models import GuestUser, CustomUserFields | |
| from squirrels._manifest import DatasetConfig | |
| from squirrels._model_configs import DbviewModelConfig, FederateModelConfig, SeedConfig | |
| from squirrels._env_vars import SquirrelsEnvVars | |
| # Model Type Tests | |
| def test_model_types(): | |
| assert m.ModelType.DBVIEW.value == "dbview" | |
| assert m.ModelType.FEDERATE.value == "federate" | |
| assert m.ModelType.SEED.value == "seed" | |
| # Seed Model Tests | |
| def seed_model() -> m.Seed: | |
| df = pl.LazyFrame({"col1": [1, 2, 3]}) | |
| return m.Seed("seed_model", SeedConfig(), df) | |
| def test_seed_model(seed_model: m.Seed): | |
| assert seed_model.name == "seed_model" | |
| assert seed_model.model_type == m.ModelType.SEED | |
| assert not seed_model.is_target | |
| assert seed_model.result is not None | |
| assert seed_model.result.collect().equals(pl.DataFrame({"col1": [1, 2, 3]})) | |
| # Dbview Model Tests | |
| def dbview_model() -> m.DbviewModel: | |
| config = DbviewModelConfig() | |
| query_file = mq.SqlQueryFile("test.sql", "SELECT * FROM table") | |
| return m.DbviewModel("test_model", config, query_file) | |
| def test_dbview_model(dbview_model: m.DbviewModel): | |
| assert dbview_model.name == "test_model" | |
| assert dbview_model.model_type == m.ModelType.DBVIEW | |
| assert not dbview_model.is_target | |
| assert dbview_model.compiled_query is None | |
| # Federate Model Tests | |
| def federate_model() -> m.FederateModel: | |
| config = FederateModelConfig() | |
| query_file = mq.SqlQueryFile("test.sql", 'SELECT * FROM {{ ref("upstream") }}') | |
| return m.FederateModel("test_model", config, query_file) | |
| def test_federate_model(federate_model: m.FederateModel): | |
| assert federate_model.name == "test_model" | |
| assert federate_model.model_type == m.ModelType.FEDERATE | |
| assert not federate_model.is_target | |
| # DAG Tests | |
| def ctx_args() -> ContextArgs: | |
| param_args = ParametersArgs(project_path="", proj_vars={}, env_vars={}) | |
| user = GuestUser(username="test", custom_fields=CustomUserFields()) | |
| return ContextArgs(**param_args.__dict__, user=user, prms={}, configurables={}, _conn_args=param_args) | |
| def simple_dag() -> m.DAG: | |
| # Create a simple DAG: A -> B -> C | |
| model_c = m.Seed("C", SeedConfig(), pl.LazyFrame({"id": [1, 2, 3]})) | |
| config_b = FederateModelConfig() | |
| query_b = 'SELECT * FROM {{ ref("C") }}' | |
| query_file_b = mq.SqlQueryFile("B.sql", query_b) | |
| model_b = m.FederateModel("B", config_b, query_file_b) | |
| config_a = FederateModelConfig() | |
| query_a = 'SELECT * FROM {{ ref("B") }}' | |
| query_file_a = mq.SqlQueryFile("A.sql", query_a) | |
| model_a = m.FederateModel("A", config_a, query_file_a) | |
| models = {"A": model_a, "B": model_b, "C": model_c} | |
| return m.DAG(DatasetConfig(name="test"), model_a, models) | |
| def test_dag_compilation(simple_dag: m.DAG, ctx_args: ContextArgs): | |
| ctx = {} | |
| simple_dag._compile_models(ctx, ctx_args, True) | |
| model_a = simple_dag.models_dict["A"] | |
| model_b = simple_dag.models_dict["B"] | |
| model_c = simple_dag.models_dict["C"] | |
| assert model_a.upstreams == {"B": model_b} | |
| assert model_b.upstreams == {"C": model_c} | |
| assert model_c.upstreams == {} | |
| def test_dag_terminal_nodes(simple_dag: m.DAG, ctx_args: ContextArgs): | |
| ctx = {} | |
| simple_dag._compile_models(ctx, ctx_args, True) | |
| terminal_nodes = simple_dag._get_terminal_nodes() | |
| assert terminal_nodes == {"C"} | |
| def test_dag_cycle_detection(ctx_args: ContextArgs): | |
| # Create a DAG with cycle: A -> B -> A | |
| config_b = FederateModelConfig() | |
| query_b = 'SELECT * FROM {{ ref("A") }}' | |
| query_file_b = mq.SqlQueryFile("B.sql", query_b) | |
| model_b = m.FederateModel("B", config_b, query_file_b) | |
| config_a = FederateModelConfig() | |
| query_a = 'SELECT * FROM {{ ref("B") }}' | |
| query_file_a = mq.SqlQueryFile("A.sql", query_a) | |
| model_a = m.FederateModel("A", config_a, query_file_a) | |
| models: dict[str, m.DataModel] = {"A": model_a, "B": model_b} | |
| dag = m.DAG(DatasetConfig(name="test"), model_a, models) | |
| ctx = {} | |
| dag._compile_models(ctx, ctx_args, True) | |
| with pytest.raises(u.ConfigurationError, match="Cycle found in model dependency graph"): | |
| dag._get_terminal_nodes() | |
| # ModelsIO Tests | |
| def test_load_files(tmp_path: Path): | |
| # Create temporary model files | |
| builds_path = tmp_path / "models" / "builds" | |
| dbviews_path = tmp_path / "models" / "dbviews" | |
| federates_path = tmp_path / "models" / "federates" | |
| builds_path.mkdir(parents=True) | |
| dbviews_path.mkdir(parents=True) | |
| federates_path.mkdir(parents=True) | |
| # Create a build model | |
| (builds_path / "model0.sql").write_text("SELECT * FROM table0") | |
| # Create a dbview model | |
| (dbviews_path / "model1.sql").write_text("SELECT * FROM table1") | |
| (dbviews_path / "model1.yml").write_text("connection: test_conn") | |
| # Create a federate model | |
| (federates_path / "model2.sql").write_text('SELECT * FROM {{ ref("model1") }}') | |
| logger = u.Logger("") | |
| env_vars = SquirrelsEnvVars(project_path=str(tmp_path)) | |
| build_model_files = m.ModelsIO.load_build_files(logger, env_vars) | |
| dbview_model_files = m.ModelsIO.load_dbview_files(logger, env_vars) | |
| federate_model_files = m.ModelsIO.load_federate_files(logger, env_vars) | |
| assert set(build_model_files.keys()) == {"model0"} | |
| assert set(dbview_model_files.keys()) == {"model1"} | |
| assert set(federate_model_files.keys()) == {"model2"} | |
Xet Storage Details
- Size:
- 5.85 kB
- Xet hash:
- 692fd826d0c390d396acb6c10f21694a4d380e5de900228c2eea9c7fd376668f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.