diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_highlight.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_highlight.py
new file mode 100644
index 0000000000000000000000000000000000000000..98c1f70f08e897bfc7b37d9efe0130ef1708bf2c
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_highlight.py
@@ -0,0 +1,221 @@
+import numpy as np
+import pytest
+
+from pandas import (
+ NA,
+ DataFrame,
+ IndexSlice,
+)
+
+pytest.importorskip("jinja2")
+
+from pandas.io.formats.style import Styler
+
+
+@pytest.fixture(params=[(None, "float64"), (NA, "Int64")])
+def df(request):
+ # GH 45804
+ dtype = request.param[1]
+ item = np.nan if dtype == "float64" else NA
+ return DataFrame(
+ {"A": [0, item, 10], "B": [1, request.param[0], 2]}, dtype=request.param[1]
+ )
+
+
+@pytest.fixture
+def styler(df):
+ return Styler(df, uuid_len=0)
+
+
+def test_highlight_null(styler):
+ result = styler.highlight_null()._compute().ctx
+ expected = {
+ (1, 0): [("background-color", "red")],
+ (1, 1): [("background-color", "red")],
+ }
+ assert result == expected
+
+
+def test_highlight_null_subset(styler):
+ # GH 31345
+ result = (
+ styler.highlight_null(color="red", subset=["A"])
+ .highlight_null(color="green", subset=["B"])
+ ._compute()
+ .ctx
+ )
+ expected = {
+ (1, 0): [("background-color", "red")],
+ (1, 1): [("background-color", "green")],
+ }
+ assert result == expected
+
+
+@pytest.mark.parametrize("f", ["highlight_min", "highlight_max"])
+def test_highlight_minmax_basic(df, f):
+ expected = {
+ (0, 1): [("background-color", "red")],
+ # ignores NaN row,
+ (2, 0): [("background-color", "red")],
+ }
+ if f == "highlight_min":
+ df = -df
+ result = getattr(df.style, f)(axis=1, color="red")._compute().ctx
+ assert result == expected
+
+
+@pytest.mark.parametrize("f", ["highlight_min", "highlight_max"])
+@pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"axis": None, "color": "red"}, # test axis
+ {"axis": 0, "subset": ["A"], "color": "red"}, # test subset and ignores NaN
+ {"axis": None, "props": "background-color: red"}, # test props
+ ],
+)
+def test_highlight_minmax_ext(df, f, kwargs):
+ expected = {(2, 0): [("background-color", "red")]}
+ if f == "highlight_min":
+ df = -df
+ result = getattr(df.style, f)(**kwargs)._compute().ctx
+ assert result == expected
+
+
+@pytest.mark.parametrize("f", ["highlight_min", "highlight_max"])
+@pytest.mark.parametrize("axis", [None, 0, 1])
+def test_highlight_minmax_nulls(f, axis):
+ # GH 42750
+ expected = {
+ (1, 0): [("background-color", "yellow")],
+ (1, 1): [("background-color", "yellow")],
+ }
+ if axis == 1:
+ expected.update({(2, 1): [("background-color", "yellow")]})
+
+ if f == "highlight_max":
+ df = DataFrame({"a": [NA, 1, None], "b": [np.nan, 1, -1]})
+ else:
+ df = DataFrame({"a": [NA, -1, None], "b": [np.nan, -1, 1]})
+
+ result = getattr(df.style, f)(axis=axis)._compute().ctx
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"left": 0, "right": 1}, # test basic range
+ {"left": 0, "right": 1, "props": "background-color: yellow"}, # test props
+ {"left": -100, "right": 100, "subset": IndexSlice[[0, 1], :]}, # test subset
+ {"left": 0, "subset": IndexSlice[[0, 1], :]}, # test no right
+ {"right": 1}, # test no left
+ {"left": [0, 0, 11], "axis": 0}, # test left as sequence
+ {"left": DataFrame({"A": [0, 0, 11], "B": [1, 1, 11]}), "axis": None}, # axis
+ {"left": 0, "right": [0, 1], "axis": 1}, # test sequence right
+ ],
+)
+def test_highlight_between(styler, kwargs):
+ expected = {
+ (0, 0): [("background-color", "yellow")],
+ (0, 1): [("background-color", "yellow")],
+ }
+ result = styler.highlight_between(**kwargs)._compute().ctx
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "arg, map, axis",
+ [
+ ("left", [1, 2], 0), # 0 axis has 3 elements not 2
+ ("left", [1, 2, 3], 1), # 1 axis has 2 elements not 3
+ ("left", np.array([[1, 2], [1, 2]]), None), # df is (2,3) not (2,2)
+ ("right", [1, 2], 0), # same tests as above for 'right' not 'left'
+ ("right", [1, 2, 3], 1), # ..
+ ("right", np.array([[1, 2], [1, 2]]), None), # ..
+ ],
+)
+def test_highlight_between_raises(arg, styler, map, axis):
+ msg = f"supplied '{arg}' is not correct shape"
+ with pytest.raises(ValueError, match=msg):
+ styler.highlight_between(**{arg: map, "axis": axis})._compute()
+
+
+def test_highlight_between_raises2(styler):
+ msg = "values can be 'both', 'left', 'right', or 'neither'"
+ with pytest.raises(ValueError, match=msg):
+ styler.highlight_between(inclusive="badstring")._compute()
+
+ with pytest.raises(ValueError, match=msg):
+ styler.highlight_between(inclusive=1)._compute()
+
+
+@pytest.mark.parametrize(
+ "inclusive, expected",
+ [
+ (
+ "both",
+ {
+ (0, 0): [("background-color", "yellow")],
+ (0, 1): [("background-color", "yellow")],
+ },
+ ),
+ ("neither", {}),
+ ("left", {(0, 0): [("background-color", "yellow")]}),
+ ("right", {(0, 1): [("background-color", "yellow")]}),
+ ],
+)
+def test_highlight_between_inclusive(styler, inclusive, expected):
+ kwargs = {"left": 0, "right": 1, "subset": IndexSlice[[0, 1], :]}
+ result = styler.highlight_between(**kwargs, inclusive=inclusive)._compute()
+ assert result.ctx == expected
+
+
+@pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"q_left": 0.5, "q_right": 1, "axis": 0}, # base case
+ {"q_left": 0.5, "q_right": 1, "axis": None}, # test axis
+ {"q_left": 0, "q_right": 1, "subset": IndexSlice[2, :]}, # test subset
+ {"q_left": 0.5, "axis": 0}, # test no high
+ {"q_right": 1, "subset": IndexSlice[2, :], "axis": 1}, # test no low
+ {"q_left": 0.5, "axis": 0, "props": "background-color: yellow"}, # tst prop
+ ],
+)
+def test_highlight_quantile(styler, kwargs):
+ expected = {
+ (2, 0): [("background-color", "yellow")],
+ (2, 1): [("background-color", "yellow")],
+ }
+ result = styler.highlight_quantile(**kwargs)._compute().ctx
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "f,kwargs",
+ [
+ ("highlight_min", {"axis": 1, "subset": IndexSlice[1, :]}),
+ ("highlight_max", {"axis": 0, "subset": [0]}),
+ ("highlight_quantile", {"axis": None, "q_left": 0.6, "q_right": 0.8}),
+ ("highlight_between", {"subset": [0]}),
+ ],
+)
+@pytest.mark.parametrize(
+ "dtype",
+ [
+ int,
+ float,
+ "datetime64[ns]",
+ str,
+ "timedelta64[ns]",
+ ],
+)
+def test_all_highlight_dtypes(f, kwargs, dtype):
+ df = DataFrame([[0, 10], [20, 30]], dtype=dtype)
+ if f == "highlight_quantile" and isinstance(df.iloc[0, 0], (str)):
+ return None # quantile incompatible with str
+ if f == "highlight_between":
+ kwargs["left"] = df.iloc[1, 0] # set the range low for testing
+
+ expected = {(1, 0): [("background-color", "yellow")]}
+ result = getattr(df.style, f)(**kwargs)._compute().ctx
+ assert result == expected
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_html.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_html.py
new file mode 100644
index 0000000000000000000000000000000000000000..0b225126b0a70d9eeca414dc0c11a7e9ca80befc
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_html.py
@@ -0,0 +1,1038 @@
+import pathlib
+from textwrap import (
+ dedent,
+ indent,
+)
+
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ MultiIndex,
+ option_context,
+)
+
+jinja2 = pytest.importorskip("jinja2")
+from pandas.io.formats.style import Styler
+
+
+@pytest.fixture
+def env():
+ project_dir = pathlib.Path(__file__).parent.parent.parent.parent.parent.resolve()
+ template_dir = project_dir / "io" / "formats" / "templates"
+ loader = jinja2.FileSystemLoader(template_dir)
+ env = jinja2.Environment(loader=loader, trim_blocks=True)
+ return env
+
+
+@pytest.fixture
+def styler():
+ return Styler(DataFrame([[2.61], [2.69]], index=["a", "b"], columns=["A"]))
+
+
+@pytest.fixture
+def styler_mi():
+ midx = MultiIndex.from_product([["a", "b"], ["c", "d"]])
+ return Styler(DataFrame(np.arange(16).reshape(4, 4), index=midx, columns=midx))
+
+
+@pytest.fixture
+def styler_multi():
+ df = DataFrame(
+ data=np.arange(16).reshape(4, 4),
+ columns=MultiIndex.from_product([["A", "B"], ["a", "b"]], names=["A&", "b&"]),
+ index=MultiIndex.from_product([["X", "Y"], ["x", "y"]], names=["X>", "y_"]),
+ )
+ return Styler(df)
+
+
+@pytest.fixture
+def tpl_style(env):
+ return env.get_template("html_style.tpl")
+
+
+@pytest.fixture
+def tpl_table(env):
+ return env.get_template("html_table.tpl")
+
+
+def test_html_template_extends_options(datapath):
+ # make sure if templates are edited tests are updated as are setup fixtures
+ # to understand the dependency
+ path = datapath("..", "io", "formats", "templates", "html.tpl")
+ with open(path, encoding="utf-8") as file:
+ result = file.read()
+ assert "{% include html_style_tpl %}" in result
+ assert "{% include html_table_tpl %}" in result
+
+
+def test_exclude_styles(styler):
+ result = styler.to_html(exclude_styles=True, doctype_html=True)
+ expected = dedent(
+ """\
+
+
+
+
+
+
+
+
+
+
+ A
+
+
+
+
+ a
+ 2.610000
+
+
+ b
+ 2.690000
+
+
+
+
+
+ """
+ )
+ assert result == expected
+
+
+def test_w3_html_format(styler):
+ styler.set_uuid("").set_table_styles([{"selector": "th", "props": "att2:v2;"}]).map(
+ lambda x: "att1:v1;"
+ ).set_table_attributes('class="my-cls1" style="attr3:v3;"').set_td_classes(
+ DataFrame(["my-cls2"], index=["a"], columns=["A"])
+ ).format("{:.1f}").set_caption("A comprehensive test")
+ expected = dedent(
+ """\
+
+
+ A comprehensive test
+
+
+
+ A
+
+
+
+
+ a
+ 2.6
+
+
+ b
+ 2.7
+
+
+
+ """
+ )
+ assert expected == styler.to_html()
+
+
+def test_colspan_w3():
+ # GH 36223
+ df = DataFrame(data=[[1, 2]], columns=[["l0", "l0"], ["l1a", "l1b"]])
+ styler = Styler(df, uuid="_", cell_ids=False)
+ assert 'l0 ' in styler.to_html()
+
+
+def test_rowspan_w3():
+ # GH 38533
+ df = DataFrame(data=[[1, 2]], index=[["l0", "l0"], ["l1a", "l1b"]])
+ styler = Styler(df, uuid="_", cell_ids=False)
+ assert 'l0 ' in styler.to_html()
+
+
+def test_styles(styler):
+ styler.set_uuid("abc")
+ styler.set_table_styles([{"selector": "td", "props": "color: red;"}])
+ result = styler.to_html(doctype_html=True)
+ expected = dedent(
+ """\
+
+
+
+
+
+
+
+
+
+
+
+ A
+
+
+
+
+ a
+ 2.610000
+
+
+ b
+ 2.690000
+
+
+
+
+
+ """
+ )
+ assert result == expected
+
+
+def test_doctype(styler):
+ result = styler.to_html(doctype_html=False)
+ assert "" not in result
+ assert "" not in result
+ assert "" not in result
+ assert "" not in result
+
+
+def test_doctype_encoding(styler):
+ with option_context("styler.render.encoding", "ASCII"):
+ result = styler.to_html(doctype_html=True)
+ assert ' ' in result
+ result = styler.to_html(doctype_html=True, encoding="ANSI")
+ assert ' ' in result
+
+
+def test_bold_headers_arg(styler):
+ result = styler.to_html(bold_headers=True)
+ assert "th {\n font-weight: bold;\n}" in result
+ result = styler.to_html()
+ assert "th {\n font-weight: bold;\n}" not in result
+
+
+def test_caption_arg(styler):
+ result = styler.to_html(caption="foo bar")
+ assert "foo bar " in result
+ result = styler.to_html()
+ assert "foo bar " not in result
+
+
+def test_block_names(tpl_style, tpl_table):
+ # catch accidental removal of a block
+ expected_style = {
+ "before_style",
+ "style",
+ "table_styles",
+ "before_cellstyle",
+ "cellstyle",
+ }
+ expected_table = {
+ "before_table",
+ "table",
+ "caption",
+ "thead",
+ "tbody",
+ "after_table",
+ "before_head_rows",
+ "head_tr",
+ "after_head_rows",
+ "before_rows",
+ "tr",
+ "after_rows",
+ }
+ result1 = set(tpl_style.blocks)
+ assert result1 == expected_style
+
+ result2 = set(tpl_table.blocks)
+ assert result2 == expected_table
+
+
+def test_from_custom_template_table(tmpdir):
+ p = tmpdir.mkdir("tpl").join("myhtml_table.tpl")
+ p.write(
+ dedent(
+ """\
+ {% extends "html_table.tpl" %}
+ {% block table %}
+ {{custom_title}}
+ {{ super() }}
+ {% endblock table %}"""
+ )
+ )
+ result = Styler.from_custom_template(str(tmpdir.join("tpl")), "myhtml_table.tpl")
+ assert issubclass(result, Styler)
+ assert result.env is not Styler.env
+ assert result.template_html_table is not Styler.template_html_table
+ styler = result(DataFrame({"A": [1, 2]}))
+ assert "My Title \n\n\n
+ {{ super() }}
+ {% endblock style %}"""
+ )
+ )
+ result = Styler.from_custom_template(
+ str(tmpdir.join("tpl")), html_style="myhtml_style.tpl"
+ )
+ assert issubclass(result, Styler)
+ assert result.env is not Styler.env
+ assert result.template_html_style is not Styler.template_html_style
+ styler = result(DataFrame({"A": [1, 2]}))
+ assert ' \n\n
+
+
+
+
+ n1
+ a
+
+
+
+ n2
+ c
+
+
+ n1
+ n2
+
+
+
+
+
+ a
+ c
+ 0
+
+
+
+ """
+ )
+ result = styler_mi.to_html()
+ assert result == expected
+
+
+def test_include_css_style_rules_only_for_visible_cells(styler_mi):
+ # GH 43619
+ result = (
+ styler_mi.set_uuid("")
+ .map(lambda v: "color: blue;")
+ .hide(styler_mi.data.columns[1:], axis="columns")
+ .hide(styler_mi.data.index[1:], axis="index")
+ .to_html()
+ )
+ expected_styles = dedent(
+ """\
+
+ """
+ )
+ assert expected_styles in result
+
+
+def test_include_css_style_rules_only_for_visible_index_labels(styler_mi):
+ # GH 43619
+ result = (
+ styler_mi.set_uuid("")
+ .map_index(lambda v: "color: blue;", axis="index")
+ .hide(styler_mi.data.columns, axis="columns")
+ .hide(styler_mi.data.index[1:], axis="index")
+ .to_html()
+ )
+ expected_styles = dedent(
+ """\
+
+ """
+ )
+ assert expected_styles in result
+
+
+def test_include_css_style_rules_only_for_visible_column_labels(styler_mi):
+ # GH 43619
+ result = (
+ styler_mi.set_uuid("")
+ .map_index(lambda v: "color: blue;", axis="columns")
+ .hide(styler_mi.data.columns[1:], axis="columns")
+ .hide(styler_mi.data.index, axis="index")
+ .to_html()
+ )
+ expected_styles = dedent(
+ """\
+
+ """
+ )
+ assert expected_styles in result
+
+
+def test_hiding_index_columns_multiindex_alignment():
+ # gh 43644
+ midx = MultiIndex.from_product(
+ [["i0", "j0"], ["i1"], ["i2", "j2"]], names=["i-0", "i-1", "i-2"]
+ )
+ cidx = MultiIndex.from_product(
+ [["c0"], ["c1", "d1"], ["c2", "d2"]], names=["c-0", "c-1", "c-2"]
+ )
+ df = DataFrame(np.arange(16).reshape(4, 4), index=midx, columns=cidx)
+ styler = Styler(df, uuid_len=0)
+ styler.hide(level=1, axis=0).hide(level=0, axis=1)
+ styler.hide([("j0", "i1", "j2")], axis=0)
+ styler.hide([("c0", "d1", "d2")], axis=1)
+ result = styler.to_html()
+ expected = dedent(
+ """\
+
+
+
+
+
+ c-1
+ c1
+ d1
+
+
+
+ c-2
+ c2
+ d2
+ c2
+
+
+ i-0
+ i-2
+
+
+
+
+
+
+
+ i0
+ i2
+ 0
+ 1
+ 2
+
+
+ j2
+ 4
+ 5
+ 6
+
+
+ j0
+ i2
+ 8
+ 9
+ 10
+
+
+
+ """
+ )
+ assert result == expected
+
+
+def test_hiding_index_columns_multiindex_trimming():
+ # gh 44272
+ df = DataFrame(np.arange(64).reshape(8, 8))
+ df.columns = MultiIndex.from_product([[0, 1, 2, 3], [0, 1]])
+ df.index = MultiIndex.from_product([[0, 1, 2, 3], [0, 1]])
+ df.index.names, df.columns.names = ["a", "b"], ["c", "d"]
+ styler = Styler(df, cell_ids=False, uuid_len=0)
+ styler.hide([(0, 0), (0, 1), (1, 0)], axis=1).hide([(0, 0), (0, 1), (1, 0)], axis=0)
+ with option_context("styler.render.max_rows", 4, "styler.render.max_columns", 4):
+ result = styler.to_html()
+
+ expected = dedent(
+ """\
+
+
+
+
+
+ c
+ 1
+ 2
+ 3
+
+
+
+ d
+ 1
+ 0
+ 1
+ 0
+ ...
+
+
+ a
+ b
+
+
+
+
+
+
+
+
+
+ 1
+ 1
+ 27
+ 28
+ 29
+ 30
+ ...
+
+
+ 2
+ 0
+ 35
+ 36
+ 37
+ 38
+ ...
+
+
+ 1
+ 43
+ 44
+ 45
+ 46
+ ...
+
+
+ 3
+ 0
+ 51
+ 52
+ 53
+ 54
+ ...
+
+
+ ...
+ ...
+ ...
+ ...
+ ...
+ ...
+ ...
+
+
+
+ """
+ )
+
+ assert result == expected
+
+
+@pytest.mark.parametrize("type", ["data", "index"])
+@pytest.mark.parametrize(
+ "text, exp, found",
+ [
+ ("no link, just text", False, ""),
+ ("subdomain not www: sub.web.com", False, ""),
+ ("www subdomain: www.web.com other", True, "www.web.com"),
+ ("scheme full structure: http://www.web.com", True, "http://www.web.com"),
+ ("scheme no top-level: http://www.web", True, "http://www.web"),
+ ("no scheme, no top-level: www.web", False, "www.web"),
+ ("https scheme: https://www.web.com", True, "https://www.web.com"),
+ ("ftp scheme: ftp://www.web", True, "ftp://www.web"),
+ ("ftps scheme: ftps://www.web", True, "ftps://www.web"),
+ ("subdirectories: www.web.com/directory", True, "www.web.com/directory"),
+ ("Multiple domains: www.1.2.3.4", True, "www.1.2.3.4"),
+ ("with port: http://web.com:80", True, "http://web.com:80"),
+ (
+ "full net_loc scheme: http://user:pass@web.com",
+ True,
+ "http://user:pass@web.com",
+ ),
+ (
+ "with valid special chars: http://web.com/,.':;~!@#$*()[]",
+ True,
+ "http://web.com/,.':;~!@#$*()[]",
+ ),
+ ],
+)
+def test_rendered_links(type, text, exp, found):
+ if type == "data":
+ df = DataFrame([text])
+ styler = df.style.format(hyperlinks="html")
+ else:
+ df = DataFrame([0], index=[text])
+ styler = df.style.format_index(hyperlinks="html")
+
+ rendered = f'{found} '
+ result = styler.to_html()
+ assert (rendered in result) is exp
+ assert (text in result) is not exp # test conversion done when expected and not
+
+
+def test_multiple_rendered_links():
+ links = ("www.a.b", "http://a.c", "https://a.d", "ftp://a.e")
+ df = DataFrame(["text {} {} text {} {}".format(*links)])
+ result = df.style.format(hyperlinks="html").to_html()
+ href = '{0} '
+ for link in links:
+ assert href.format(link) in result
+ assert href.format("text") not in result
+
+
+def test_concat(styler):
+ other = styler.data.agg(["mean"]).style
+ styler.concat(other).set_uuid("X")
+ result = styler.to_html()
+ fp = "foot0_"
+ expected = dedent(
+ f"""\
+
+ b
+ 2.690000
+
+
+ mean
+ 2.650000
+
+
+
+ """
+ )
+ assert expected in result
+
+
+def test_concat_recursion(styler):
+ df = styler.data
+ styler1 = styler
+ styler2 = Styler(df.agg(["mean"]), precision=3)
+ styler3 = Styler(df.agg(["mean"]), precision=4)
+ styler1.concat(styler2.concat(styler3)).set_uuid("X")
+ result = styler.to_html()
+ # notice that the second concat (last of the output html),
+ # there are two `foot_` in the id and class
+ fp1 = "foot0_"
+ fp2 = "foot0_foot0_"
+ expected = dedent(
+ f"""\
+
+ b
+ 2.690000
+
+
+ mean
+ 2.650
+
+
+ mean
+ 2.6500
+
+
+
+ """
+ )
+ assert expected in result
+
+
+def test_concat_chain(styler):
+ df = styler.data
+ styler1 = styler
+ styler2 = Styler(df.agg(["mean"]), precision=3)
+ styler3 = Styler(df.agg(["mean"]), precision=4)
+ styler1.concat(styler2).concat(styler3).set_uuid("X")
+ result = styler.to_html()
+ fp1 = "foot0_"
+ fp2 = "foot1_"
+ expected = dedent(
+ f"""\
+
+ b
+ 2.690000
+
+
+ mean
+ 2.650
+
+
+ mean
+ 2.6500
+
+
+
+ """
+ )
+ assert expected in result
+
+
+def test_concat_combined():
+ def html_lines(foot_prefix: str):
+ assert foot_prefix.endswith("_") or foot_prefix == ""
+ fp = foot_prefix
+ return indent(
+ dedent(
+ f"""\
+
+ a
+ 2.610000
+
+
+ b
+ 2.690000
+
+ """
+ ),
+ prefix=" " * 4,
+ )
+
+ df = DataFrame([[2.61], [2.69]], index=["a", "b"], columns=["A"])
+ s1 = df.style.highlight_max(color="red")
+ s2 = df.style.highlight_max(color="green")
+ s3 = df.style.highlight_max(color="blue")
+ s4 = df.style.highlight_max(color="yellow")
+
+ result = s1.concat(s2).concat(s3.concat(s4)).set_uuid("X").to_html()
+ expected_css = dedent(
+ """\
+
+ """
+ )
+ expected_table = (
+ dedent(
+ """\
+
+
+
+
+ A
+
+
+
+ """
+ )
+ + html_lines("")
+ + html_lines("foot0_")
+ + html_lines("foot1_")
+ + html_lines("foot1_foot0_")
+ + dedent(
+ """\
+
+
+ """
+ )
+ )
+ assert expected_css + expected_table == result
+
+
+def test_to_html_na_rep_non_scalar_data(datapath):
+ # GH47103
+ df = DataFrame([{"a": 1, "b": [1, 2, 3], "c": np.nan}])
+ result = df.style.format(na_rep="-").to_html(table_uuid="test")
+ expected = """\
+
+
+
+
+
+ a
+ b
+ c
+
+
+
+
+ 0
+ 1
+ [1, 2, 3]
+ -
+
+
+
+"""
+ assert result == expected
+
+
+@pytest.mark.parametrize("escape_axis_0", [True, False])
+@pytest.mark.parametrize("escape_axis_1", [True, False])
+def test_format_index_names(styler_multi, escape_axis_0, escape_axis_1):
+ if escape_axis_0:
+ styler_multi.format_index_names(axis=0, escape="html")
+ expected_index = ["X>", "y_"]
+ else:
+ expected_index = ["X>", "y_"]
+
+ if escape_axis_1:
+ styler_multi.format_index_names(axis=1, escape="html")
+ expected_columns = ["A&", "b&"]
+ else:
+ expected_columns = ["A&", "b&"]
+
+ result = styler_multi.to_html(table_uuid="test")
+ for expected_str in expected_index + expected_columns:
+ assert f"{expected_str}" in result
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_matplotlib.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_matplotlib.py
new file mode 100644
index 0000000000000000000000000000000000000000..490bd45bfb2eea246f3c997a8a9e8298b2a02bf4
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_matplotlib.py
@@ -0,0 +1,305 @@
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ IndexSlice,
+ Series,
+)
+
+mpl = pytest.importorskip("matplotlib")
+pytest.importorskip("jinja2")
+
+from pandas.io.formats.style import Styler
+
+pytestmark = pytest.mark.usefixtures("mpl_cleanup")
+
+
+@pytest.fixture
+def df():
+ return DataFrame([[1, 2], [2, 4]], columns=["A", "B"])
+
+
+@pytest.fixture
+def styler(df):
+ return Styler(df, uuid_len=0)
+
+
+@pytest.fixture
+def df_blank():
+ return DataFrame([[0, 0], [0, 0]], columns=["A", "B"], index=["X", "Y"])
+
+
+@pytest.fixture
+def styler_blank(df_blank):
+ return Styler(df_blank, uuid_len=0)
+
+
+@pytest.mark.parametrize("f", ["background_gradient", "text_gradient"])
+def test_function_gradient(styler, f):
+ for c_map in [None, "YlOrRd"]:
+ result = getattr(styler, f)(cmap=c_map)._compute().ctx
+ assert all("#" in x[0][1] for x in result.values())
+ assert result[(0, 0)] == result[(0, 1)]
+ assert result[(1, 0)] == result[(1, 1)]
+
+
+@pytest.mark.parametrize("f", ["background_gradient", "text_gradient"])
+def test_background_gradient_color(styler, f):
+ result = getattr(styler, f)(subset=IndexSlice[1, "A"])._compute().ctx
+ if f == "background_gradient":
+ assert result[(1, 0)] == [("background-color", "#fff7fb"), ("color", "#000000")]
+ elif f == "text_gradient":
+ assert result[(1, 0)] == [("color", "#fff7fb")]
+
+
+@pytest.mark.parametrize(
+ "axis, expected",
+ [
+ (0, ["low", "low", "high", "high"]),
+ (1, ["low", "high", "low", "high"]),
+ (None, ["low", "mid", "mid", "high"]),
+ ],
+)
+@pytest.mark.parametrize("f", ["background_gradient", "text_gradient"])
+def test_background_gradient_axis(styler, axis, expected, f):
+ if f == "background_gradient":
+ colors = {
+ "low": [("background-color", "#f7fbff"), ("color", "#000000")],
+ "mid": [("background-color", "#abd0e6"), ("color", "#000000")],
+ "high": [("background-color", "#08306b"), ("color", "#f1f1f1")],
+ }
+ elif f == "text_gradient":
+ colors = {
+ "low": [("color", "#f7fbff")],
+ "mid": [("color", "#abd0e6")],
+ "high": [("color", "#08306b")],
+ }
+ result = getattr(styler, f)(cmap="Blues", axis=axis)._compute().ctx
+ for i, cell in enumerate([(0, 0), (0, 1), (1, 0), (1, 1)]):
+ assert result[cell] == colors[expected[i]]
+
+
+@pytest.mark.parametrize(
+ "cmap, expected",
+ [
+ (
+ "PuBu",
+ {
+ (4, 5): [("background-color", "#86b0d3"), ("color", "#000000")],
+ (4, 6): [("background-color", "#83afd3"), ("color", "#f1f1f1")],
+ },
+ ),
+ (
+ "YlOrRd",
+ {
+ (4, 8): [("background-color", "#fd913e"), ("color", "#000000")],
+ (4, 9): [("background-color", "#fd8f3d"), ("color", "#f1f1f1")],
+ },
+ ),
+ (
+ None,
+ {
+ (7, 0): [("background-color", "#48c16e"), ("color", "#f1f1f1")],
+ (7, 1): [("background-color", "#4cc26c"), ("color", "#000000")],
+ },
+ ),
+ ],
+)
+def test_text_color_threshold(cmap, expected):
+ # GH 39888
+ df = DataFrame(np.arange(100).reshape(10, 10))
+ result = df.style.background_gradient(cmap=cmap, axis=None)._compute().ctx
+ for k in expected.keys():
+ assert result[k] == expected[k]
+
+
+def test_background_gradient_vmin_vmax():
+ # GH 12145
+ df = DataFrame(range(5))
+ ctx = df.style.background_gradient(vmin=1, vmax=3)._compute().ctx
+ assert ctx[(0, 0)] == ctx[(1, 0)]
+ assert ctx[(4, 0)] == ctx[(3, 0)]
+
+
+def test_background_gradient_int64():
+ # GH 28869
+ df1 = Series(range(3)).to_frame()
+ df2 = Series(range(3), dtype="Int64").to_frame()
+ ctx1 = df1.style.background_gradient()._compute().ctx
+ ctx2 = df2.style.background_gradient()._compute().ctx
+ assert ctx2[(0, 0)] == ctx1[(0, 0)]
+ assert ctx2[(1, 0)] == ctx1[(1, 0)]
+ assert ctx2[(2, 0)] == ctx1[(2, 0)]
+
+
+@pytest.mark.parametrize(
+ "axis, gmap, expected",
+ [
+ (
+ 0,
+ [1, 2],
+ {
+ (0, 0): [("background-color", "#fff7fb"), ("color", "#000000")],
+ (1, 0): [("background-color", "#023858"), ("color", "#f1f1f1")],
+ (0, 1): [("background-color", "#fff7fb"), ("color", "#000000")],
+ (1, 1): [("background-color", "#023858"), ("color", "#f1f1f1")],
+ },
+ ),
+ (
+ 1,
+ [1, 2],
+ {
+ (0, 0): [("background-color", "#fff7fb"), ("color", "#000000")],
+ (1, 0): [("background-color", "#fff7fb"), ("color", "#000000")],
+ (0, 1): [("background-color", "#023858"), ("color", "#f1f1f1")],
+ (1, 1): [("background-color", "#023858"), ("color", "#f1f1f1")],
+ },
+ ),
+ (
+ None,
+ np.array([[2, 1], [1, 2]]),
+ {
+ (0, 0): [("background-color", "#023858"), ("color", "#f1f1f1")],
+ (1, 0): [("background-color", "#fff7fb"), ("color", "#000000")],
+ (0, 1): [("background-color", "#fff7fb"), ("color", "#000000")],
+ (1, 1): [("background-color", "#023858"), ("color", "#f1f1f1")],
+ },
+ ),
+ ],
+)
+def test_background_gradient_gmap_array(styler_blank, axis, gmap, expected):
+ # tests when gmap is given as a sequence and converted to ndarray
+ result = styler_blank.background_gradient(axis=axis, gmap=gmap)._compute().ctx
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "gmap, axis", [([1, 2, 3], 0), ([1, 2], 1), (np.array([[1, 2], [1, 2]]), None)]
+)
+def test_background_gradient_gmap_array_raises(gmap, axis):
+ # test when gmap as converted ndarray is bad shape
+ df = DataFrame([[0, 0, 0], [0, 0, 0]])
+ msg = "supplied 'gmap' is not correct shape"
+ with pytest.raises(ValueError, match=msg):
+ df.style.background_gradient(gmap=gmap, axis=axis)._compute()
+
+
+@pytest.mark.parametrize(
+ "gmap",
+ [
+ DataFrame( # reverse the columns
+ [[2, 1], [1, 2]], columns=["B", "A"], index=["X", "Y"]
+ ),
+ DataFrame( # reverse the index
+ [[2, 1], [1, 2]], columns=["A", "B"], index=["Y", "X"]
+ ),
+ DataFrame( # reverse the index and columns
+ [[1, 2], [2, 1]], columns=["B", "A"], index=["Y", "X"]
+ ),
+ DataFrame( # add unnecessary columns
+ [[1, 2, 3], [2, 1, 3]], columns=["A", "B", "C"], index=["X", "Y"]
+ ),
+ DataFrame( # add unnecessary index
+ [[1, 2], [2, 1], [3, 3]], columns=["A", "B"], index=["X", "Y", "Z"]
+ ),
+ ],
+)
+@pytest.mark.parametrize(
+ "subset, exp_gmap", # exp_gmap is underlying map DataFrame should conform to
+ [
+ (None, [[1, 2], [2, 1]]),
+ (["A"], [[1], [2]]), # slice only column "A" in data and gmap
+ (["B", "A"], [[2, 1], [1, 2]]), # reverse the columns in data
+ (IndexSlice["X", :], [[1, 2]]), # slice only index "X" in data and gmap
+ (IndexSlice[["Y", "X"], :], [[2, 1], [1, 2]]), # reverse the index in data
+ ],
+)
+def test_background_gradient_gmap_dataframe_align(styler_blank, gmap, subset, exp_gmap):
+ # test gmap given as DataFrame that it aligns to the data including subset
+ expected = styler_blank.background_gradient(axis=None, gmap=exp_gmap, subset=subset)
+ result = styler_blank.background_gradient(axis=None, gmap=gmap, subset=subset)
+ assert expected._compute().ctx == result._compute().ctx
+
+
+@pytest.mark.parametrize(
+ "gmap, axis, exp_gmap",
+ [
+ (Series([2, 1], index=["Y", "X"]), 0, [[1, 1], [2, 2]]), # reverse the index
+ (Series([2, 1], index=["B", "A"]), 1, [[1, 2], [1, 2]]), # reverse the cols
+ (Series([1, 2, 3], index=["X", "Y", "Z"]), 0, [[1, 1], [2, 2]]), # add idx
+ (Series([1, 2, 3], index=["A", "B", "C"]), 1, [[1, 2], [1, 2]]), # add col
+ ],
+)
+def test_background_gradient_gmap_series_align(styler_blank, gmap, axis, exp_gmap):
+ # test gmap given as Series that it aligns to the data including subset
+ expected = styler_blank.background_gradient(axis=None, gmap=exp_gmap)._compute()
+ result = styler_blank.background_gradient(axis=axis, gmap=gmap)._compute()
+ assert expected.ctx == result.ctx
+
+
+@pytest.mark.parametrize("axis", [1, 0])
+def test_background_gradient_gmap_wrong_dataframe(styler_blank, axis):
+ # test giving a gmap in DataFrame but with wrong axis
+ gmap = DataFrame([[1, 2], [2, 1]], columns=["A", "B"], index=["X", "Y"])
+ msg = "'gmap' is a DataFrame but underlying data for operations is a Series"
+ with pytest.raises(ValueError, match=msg):
+ styler_blank.background_gradient(gmap=gmap, axis=axis)._compute()
+
+
+def test_background_gradient_gmap_wrong_series(styler_blank):
+ # test giving a gmap in Series form but with wrong axis
+ msg = "'gmap' is a Series but underlying data for operations is a DataFrame"
+ gmap = Series([1, 2], index=["X", "Y"])
+ with pytest.raises(ValueError, match=msg):
+ styler_blank.background_gradient(gmap=gmap, axis=None)._compute()
+
+
+def test_background_gradient_nullable_dtypes():
+ # GH 50712
+ df1 = DataFrame([[1], [0], [np.nan]], dtype=float)
+ df2 = DataFrame([[1], [0], [None]], dtype="Int64")
+
+ ctx1 = df1.style.background_gradient()._compute().ctx
+ ctx2 = df2.style.background_gradient()._compute().ctx
+ assert ctx1 == ctx2
+
+
+@pytest.mark.parametrize(
+ "cmap",
+ ["PuBu", mpl.colormaps["PuBu"]],
+)
+def test_bar_colormap(cmap):
+ data = DataFrame([[1, 2], [3, 4]])
+ ctx = data.style.bar(cmap=cmap, axis=None)._compute().ctx
+ pubu_colors = {
+ (0, 0): "#d0d1e6",
+ (1, 0): "#056faf",
+ (0, 1): "#73a9cf",
+ (1, 1): "#023858",
+ }
+ for k, v in pubu_colors.items():
+ assert v in ctx[k][1][1]
+
+
+def test_bar_color_raises(df):
+ msg = "`color` must be string or list or tuple of 2 strings"
+ with pytest.raises(ValueError, match=msg):
+ df.style.bar(color={"a", "b"}).to_html()
+ with pytest.raises(ValueError, match=msg):
+ df.style.bar(color=["a", "b", "c"]).to_html()
+
+ msg = "`color` and `cmap` cannot both be given"
+ with pytest.raises(ValueError, match=msg):
+ df.style.bar(color="something", cmap="something else").to_html()
+
+
+@pytest.mark.parametrize("plot_method", ["scatter", "hexbin"])
+def test_pass_colormap_instance(df, plot_method):
+ # https://github.com/pandas-dev/pandas/issues/49374
+ cmap = mpl.colors.ListedColormap([[1, 1, 1], [0, 0, 0]])
+ df["c"] = df.A + df.B
+ kwargs = {"x": "A", "y": "B", "c": "c", "colormap": cmap}
+ if plot_method == "hexbin":
+ kwargs["C"] = kwargs.pop("c")
+ getattr(df.plot, plot_method)(**kwargs)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_non_unique.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_non_unique.py
new file mode 100644
index 0000000000000000000000000000000000000000..e4d31fe21f2c9cf3454a67f8c7443382f7f1c0ef
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_non_unique.py
@@ -0,0 +1,140 @@
+from textwrap import dedent
+
+import pytest
+
+from pandas import (
+ DataFrame,
+ IndexSlice,
+)
+
+pytest.importorskip("jinja2")
+
+from pandas.io.formats.style import Styler
+
+
+@pytest.fixture
+def df():
+ return DataFrame(
+ [[1, 2, 3], [4, 5, 6], [7, 8, 9]],
+ index=["i", "j", "j"],
+ columns=["c", "d", "d"],
+ dtype=float,
+ )
+
+
+@pytest.fixture
+def styler(df):
+ return Styler(df, uuid_len=0)
+
+
+def test_format_non_unique(df):
+ # GH 41269
+
+ # test dict
+ html = df.style.format({"d": "{:.1f}"}).to_html()
+ for val in ["1.000000<", "4.000000<", "7.000000<"]:
+ assert val in html
+ for val in ["2.0<", "3.0<", "5.0<", "6.0<", "8.0<", "9.0<"]:
+ assert val in html
+
+ # test subset
+ html = df.style.format(precision=1, subset=IndexSlice["j", "d"]).to_html()
+ for val in ["1.000000<", "4.000000<", "7.000000<", "2.000000<", "3.000000<"]:
+ assert val in html
+ for val in ["5.0<", "6.0<", "8.0<", "9.0<"]:
+ assert val in html
+
+
+@pytest.mark.parametrize("func", ["apply", "map"])
+def test_apply_map_non_unique_raises(df, func):
+ # GH 41269
+ if func == "apply":
+ op = lambda s: ["color: red;"] * len(s)
+ else:
+ op = lambda v: "color: red;"
+
+ with pytest.raises(KeyError, match="`Styler.apply` and `.map` are not"):
+ getattr(df.style, func)(op)._compute()
+
+
+def test_table_styles_dict_non_unique_index(styler):
+ styles = styler.set_table_styles(
+ {"j": [{"selector": "td", "props": "a: v;"}]}, axis=1
+ ).table_styles
+ assert styles == [
+ {"selector": "td.row1", "props": [("a", "v")]},
+ {"selector": "td.row2", "props": [("a", "v")]},
+ ]
+
+
+def test_table_styles_dict_non_unique_columns(styler):
+ styles = styler.set_table_styles(
+ {"d": [{"selector": "td", "props": "a: v;"}]}, axis=0
+ ).table_styles
+ assert styles == [
+ {"selector": "td.col1", "props": [("a", "v")]},
+ {"selector": "td.col2", "props": [("a", "v")]},
+ ]
+
+
+def test_tooltips_non_unique_raises(styler):
+ # ttips has unique keys
+ ttips = DataFrame([["1", "2"], ["3", "4"]], columns=["c", "d"], index=["a", "b"])
+ styler.set_tooltips(ttips=ttips) # OK
+
+ # ttips has non-unique columns
+ ttips = DataFrame([["1", "2"], ["3", "4"]], columns=["c", "c"], index=["a", "b"])
+ with pytest.raises(KeyError, match="Tooltips render only if `ttips` has unique"):
+ styler.set_tooltips(ttips=ttips)
+
+ # ttips has non-unique index
+ ttips = DataFrame([["1", "2"], ["3", "4"]], columns=["c", "d"], index=["a", "a"])
+ with pytest.raises(KeyError, match="Tooltips render only if `ttips` has unique"):
+ styler.set_tooltips(ttips=ttips)
+
+
+def test_set_td_classes_non_unique_raises(styler):
+ # classes has unique keys
+ classes = DataFrame([["1", "2"], ["3", "4"]], columns=["c", "d"], index=["a", "b"])
+ styler.set_td_classes(classes=classes) # OK
+
+ # classes has non-unique columns
+ classes = DataFrame([["1", "2"], ["3", "4"]], columns=["c", "c"], index=["a", "b"])
+ with pytest.raises(KeyError, match="Classes render only if `classes` has unique"):
+ styler.set_td_classes(classes=classes)
+
+ # classes has non-unique index
+ classes = DataFrame([["1", "2"], ["3", "4"]], columns=["c", "d"], index=["a", "a"])
+ with pytest.raises(KeyError, match="Classes render only if `classes` has unique"):
+ styler.set_td_classes(classes=classes)
+
+
+def test_hide_columns_non_unique(styler):
+ ctx = styler.hide(["d"], axis="columns")._translate(True, True)
+
+ assert ctx["head"][0][1]["display_value"] == "c"
+ assert ctx["head"][0][1]["is_visible"] is True
+
+ assert ctx["head"][0][2]["display_value"] == "d"
+ assert ctx["head"][0][2]["is_visible"] is False
+
+ assert ctx["head"][0][3]["display_value"] == "d"
+ assert ctx["head"][0][3]["is_visible"] is False
+
+ assert ctx["body"][0][1]["is_visible"] is True
+ assert ctx["body"][0][2]["is_visible"] is False
+ assert ctx["body"][0][3]["is_visible"] is False
+
+
+def test_latex_non_unique(styler):
+ result = styler.to_latex()
+ assert result == dedent(
+ """\
+ \\begin{tabular}{lrrr}
+ & c & d & d \\\\
+ i & 1.000000 & 2.000000 & 3.000000 \\\\
+ j & 4.000000 & 5.000000 & 6.000000 \\\\
+ j & 7.000000 & 8.000000 & 9.000000 \\\\
+ \\end{tabular}
+ """
+ )
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_style.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_style.py
new file mode 100644
index 0000000000000000000000000000000000000000..d78403771e227dd97eba2a130b4e60a0303c4400
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_style.py
@@ -0,0 +1,1602 @@
+import contextlib
+import copy
+import re
+from textwrap import dedent
+
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ IndexSlice,
+ MultiIndex,
+ Series,
+ option_context,
+)
+import pandas._testing as tm
+
+jinja2 = pytest.importorskip("jinja2")
+from pandas.io.formats.style import ( # isort:skip
+ Styler,
+)
+from pandas.io.formats.style_render import (
+ _get_level_lengths,
+ _get_trimming_maximums,
+ maybe_convert_css_to_tuples,
+ non_reducing_slice,
+)
+
+
+@pytest.fixture
+def mi_df():
+ return DataFrame(
+ [[1, 2], [3, 4]],
+ index=MultiIndex.from_product([["i0"], ["i1_a", "i1_b"]]),
+ columns=MultiIndex.from_product([["c0"], ["c1_a", "c1_b"]]),
+ dtype=int,
+ )
+
+
+@pytest.fixture
+def mi_styler(mi_df):
+ return Styler(mi_df, uuid_len=0)
+
+
+@pytest.fixture
+def mi_styler_comp(mi_styler):
+ # comprehensively add features to mi_styler
+ mi_styler = mi_styler._copy(deepcopy=True)
+ mi_styler.css = {**mi_styler.css, "row": "ROW", "col": "COL"}
+ mi_styler.uuid_len = 5
+ mi_styler.uuid = "abcde"
+ mi_styler.set_caption("capt")
+ mi_styler.set_table_styles([{"selector": "a", "props": "a:v;"}])
+ mi_styler.hide(axis="columns")
+ mi_styler.hide([("c0", "c1_a")], axis="columns", names=True)
+ mi_styler.hide(axis="index")
+ mi_styler.hide([("i0", "i1_a")], axis="index", names=True)
+ mi_styler.set_table_attributes('class="box"')
+ other = mi_styler.data.agg(["mean"])
+ other.index = MultiIndex.from_product([[""], other.index])
+ mi_styler.concat(other.style)
+ mi_styler.format(na_rep="MISSING", precision=3)
+ mi_styler.format_index(precision=2, axis=0)
+ mi_styler.format_index(precision=4, axis=1)
+ mi_styler.highlight_max(axis=None)
+ mi_styler.map_index(lambda x: "color: white;", axis=0)
+ mi_styler.map_index(lambda x: "color: black;", axis=1)
+ mi_styler.set_td_classes(
+ DataFrame(
+ [["a", "b"], ["a", "c"]], index=mi_styler.index, columns=mi_styler.columns
+ )
+ )
+ mi_styler.set_tooltips(
+ DataFrame(
+ [["a2", "b2"], ["a2", "c2"]],
+ index=mi_styler.index,
+ columns=mi_styler.columns,
+ )
+ )
+ mi_styler.format_index_names(escape="html", axis=0)
+ mi_styler.format_index_names(escape="html", axis=1)
+ return mi_styler
+
+
+@pytest.fixture
+def blank_value():
+ return " "
+
+
+@pytest.fixture
+def df():
+ df = DataFrame({"A": [0, 1], "B": np.random.default_rng(2).standard_normal(2)})
+ return df
+
+
+@pytest.fixture
+def styler(df):
+ df = DataFrame({"A": [0, 1], "B": np.random.default_rng(2).standard_normal(2)})
+ return Styler(df)
+
+
+@pytest.mark.parametrize(
+ "sparse_columns, exp_cols",
+ [
+ (
+ True,
+ [
+ {"is_visible": True, "attributes": 'colspan="2"', "value": "c0"},
+ {"is_visible": False, "attributes": "", "value": "c0"},
+ ],
+ ),
+ (
+ False,
+ [
+ {"is_visible": True, "attributes": "", "value": "c0"},
+ {"is_visible": True, "attributes": "", "value": "c0"},
+ ],
+ ),
+ ],
+)
+def test_mi_styler_sparsify_columns(mi_styler, sparse_columns, exp_cols):
+ exp_l1_c0 = {"is_visible": True, "attributes": "", "display_value": "c1_a"}
+ exp_l1_c1 = {"is_visible": True, "attributes": "", "display_value": "c1_b"}
+
+ ctx = mi_styler._translate(True, sparse_columns)
+
+ assert exp_cols[0].items() <= ctx["head"][0][2].items()
+ assert exp_cols[1].items() <= ctx["head"][0][3].items()
+ assert exp_l1_c0.items() <= ctx["head"][1][2].items()
+ assert exp_l1_c1.items() <= ctx["head"][1][3].items()
+
+
+@pytest.mark.parametrize(
+ "sparse_index, exp_rows",
+ [
+ (
+ True,
+ [
+ {"is_visible": True, "attributes": 'rowspan="2"', "value": "i0"},
+ {"is_visible": False, "attributes": "", "value": "i0"},
+ ],
+ ),
+ (
+ False,
+ [
+ {"is_visible": True, "attributes": "", "value": "i0"},
+ {"is_visible": True, "attributes": "", "value": "i0"},
+ ],
+ ),
+ ],
+)
+def test_mi_styler_sparsify_index(mi_styler, sparse_index, exp_rows):
+ exp_l1_r0 = {"is_visible": True, "attributes": "", "display_value": "i1_a"}
+ exp_l1_r1 = {"is_visible": True, "attributes": "", "display_value": "i1_b"}
+
+ ctx = mi_styler._translate(sparse_index, True)
+
+ assert exp_rows[0].items() <= ctx["body"][0][0].items()
+ assert exp_rows[1].items() <= ctx["body"][1][0].items()
+ assert exp_l1_r0.items() <= ctx["body"][0][1].items()
+ assert exp_l1_r1.items() <= ctx["body"][1][1].items()
+
+
+def test_mi_styler_sparsify_options(mi_styler):
+ with option_context("styler.sparse.index", False):
+ html1 = mi_styler.to_html()
+ with option_context("styler.sparse.index", True):
+ html2 = mi_styler.to_html()
+
+ assert html1 != html2
+
+ with option_context("styler.sparse.columns", False):
+ html1 = mi_styler.to_html()
+ with option_context("styler.sparse.columns", True):
+ html2 = mi_styler.to_html()
+
+ assert html1 != html2
+
+
+@pytest.mark.parametrize(
+ "rn, cn, max_els, max_rows, max_cols, exp_rn, exp_cn",
+ [
+ (100, 100, 100, None, None, 12, 6), # reduce to (12, 6) < 100 elements
+ (1000, 3, 750, None, None, 250, 3), # dynamically reduce rows to 250, keep cols
+ (4, 1000, 500, None, None, 4, 125), # dynamically reduce cols to 125, keep rows
+ (1000, 3, 750, 10, None, 10, 3), # overwrite above dynamics with max_row
+ (4, 1000, 500, None, 5, 4, 5), # overwrite above dynamics with max_col
+ (100, 100, 700, 50, 50, 25, 25), # rows cols below given maxes so < 700 elmts
+ ],
+)
+def test_trimming_maximum(rn, cn, max_els, max_rows, max_cols, exp_rn, exp_cn):
+ rn, cn = _get_trimming_maximums(
+ rn, cn, max_els, max_rows, max_cols, scaling_factor=0.5
+ )
+ assert (rn, cn) == (exp_rn, exp_cn)
+
+
+@pytest.mark.parametrize(
+ "option, val",
+ [
+ ("styler.render.max_elements", 6),
+ ("styler.render.max_rows", 3),
+ ],
+)
+def test_render_trimming_rows(option, val):
+ # test auto and specific trimming of rows
+ df = DataFrame(np.arange(120).reshape(60, 2))
+ with option_context(option, val):
+ ctx = df.style._translate(True, True)
+ assert len(ctx["head"][0]) == 3 # index + 2 data cols
+ assert len(ctx["body"]) == 4 # 3 data rows + trimming row
+ assert len(ctx["body"][0]) == 3 # index + 2 data cols
+
+
+@pytest.mark.parametrize(
+ "option, val",
+ [
+ ("styler.render.max_elements", 6),
+ ("styler.render.max_columns", 2),
+ ],
+)
+def test_render_trimming_cols(option, val):
+ # test auto and specific trimming of cols
+ df = DataFrame(np.arange(30).reshape(3, 10))
+ with option_context(option, val):
+ ctx = df.style._translate(True, True)
+ assert len(ctx["head"][0]) == 4 # index + 2 data cols + trimming col
+ assert len(ctx["body"]) == 3 # 3 data rows
+ assert len(ctx["body"][0]) == 4 # index + 2 data cols + trimming col
+
+
+def test_render_trimming_mi():
+ midx = MultiIndex.from_product([[1, 2], [1, 2, 3]])
+ df = DataFrame(np.arange(36).reshape(6, 6), columns=midx, index=midx)
+ with option_context("styler.render.max_elements", 4):
+ ctx = df.style._translate(True, True)
+
+ assert len(ctx["body"][0]) == 5 # 2 indexes + 2 data cols + trimming row
+ assert {"attributes": 'rowspan="2"'}.items() <= ctx["body"][0][0].items()
+ assert {"class": "data row0 col_trim"}.items() <= ctx["body"][0][4].items()
+ assert {"class": "data row_trim col_trim"}.items() <= ctx["body"][2][4].items()
+ assert len(ctx["body"]) == 3 # 2 data rows + trimming row
+
+
+def test_render_empty_mi():
+ # GH 43305
+ df = DataFrame(index=MultiIndex.from_product([["A"], [0, 1]], names=[None, "one"]))
+ expected = dedent(
+ """\
+ >
+
+
+
+ one
+
+
+ """
+ )
+ assert expected in df.style.to_html()
+
+
+@pytest.mark.parametrize("comprehensive", [True, False])
+@pytest.mark.parametrize("render", [True, False])
+@pytest.mark.parametrize("deepcopy", [True, False])
+def test_copy(comprehensive, render, deepcopy, mi_styler, mi_styler_comp):
+ styler = mi_styler_comp if comprehensive else mi_styler
+ styler.uuid_len = 5
+
+ s2 = copy.deepcopy(styler) if deepcopy else copy.copy(styler) # make copy and check
+ assert s2 is not styler
+
+ if render:
+ styler.to_html()
+
+ excl = [
+ "cellstyle_map", # render time vars..
+ "cellstyle_map_columns",
+ "cellstyle_map_index",
+ "template_latex", # render templates are class level
+ "template_html",
+ "template_html_style",
+ "template_html_table",
+ ]
+ if not deepcopy: # check memory locations are equal for all included attributes
+ for attr in [a for a in styler.__dict__ if (not callable(a) and a not in excl)]:
+ assert id(getattr(s2, attr)) == id(getattr(styler, attr))
+ else: # check memory locations are different for nested or mutable vars
+ shallow = [
+ "data",
+ "columns",
+ "index",
+ "uuid_len",
+ "uuid",
+ "caption",
+ "cell_ids",
+ "hide_index_",
+ "hide_columns_",
+ "hide_index_names",
+ "hide_column_names",
+ "table_attributes",
+ ]
+ for attr in shallow:
+ assert id(getattr(s2, attr)) == id(getattr(styler, attr))
+
+ for attr in [
+ a
+ for a in styler.__dict__
+ if (not callable(a) and a not in excl and a not in shallow)
+ ]:
+ if getattr(s2, attr) is None:
+ assert id(getattr(s2, attr)) == id(getattr(styler, attr))
+ else:
+ assert id(getattr(s2, attr)) != id(getattr(styler, attr))
+
+
+@pytest.mark.parametrize("deepcopy", [True, False])
+def test_inherited_copy(mi_styler, deepcopy):
+ # Ensure that the inherited class is preserved when a Styler object is copied.
+ # GH 52728
+ class CustomStyler(Styler):
+ pass
+
+ custom_styler = CustomStyler(mi_styler.data)
+ custom_styler_copy = (
+ copy.deepcopy(custom_styler) if deepcopy else copy.copy(custom_styler)
+ )
+ assert isinstance(custom_styler_copy, CustomStyler)
+
+
+def test_clear(mi_styler_comp):
+ # NOTE: if this test fails for new features then 'mi_styler_comp' should be updated
+ # to ensure proper testing of the 'copy', 'clear', 'export' methods with new feature
+ # GH 40675
+ styler = mi_styler_comp
+ styler._compute() # execute applied methods
+
+ clean_copy = Styler(styler.data, uuid=styler.uuid)
+
+ excl = [
+ "data",
+ "index",
+ "columns",
+ "uuid",
+ "uuid_len", # uuid is set to be the same on styler and clean_copy
+ "cell_ids",
+ "cellstyle_map", # execution time only
+ "cellstyle_map_columns", # execution time only
+ "cellstyle_map_index", # execution time only
+ "template_latex", # render templates are class level
+ "template_html",
+ "template_html_style",
+ "template_html_table",
+ ]
+ # tests vars are not same vals on obj and clean copy before clear (except for excl)
+ for attr in [a for a in styler.__dict__ if not (callable(a) or a in excl)]:
+ res = getattr(styler, attr) == getattr(clean_copy, attr)
+ if hasattr(res, "__iter__") and len(res) > 0:
+ assert not all(res) # some element in iterable differs
+ elif hasattr(res, "__iter__") and len(res) == 0:
+ pass # empty array
+ else:
+ assert not res # explicit var differs
+
+ # test vars have same vales on obj and clean copy after clearing
+ styler.clear()
+ for attr in [a for a in styler.__dict__ if not callable(a)]:
+ res = getattr(styler, attr) == getattr(clean_copy, attr)
+ assert all(res) if hasattr(res, "__iter__") else res
+
+
+def test_export(mi_styler_comp, mi_styler):
+ exp_attrs = [
+ "_todo",
+ "hide_index_",
+ "hide_index_names",
+ "hide_columns_",
+ "hide_column_names",
+ "table_attributes",
+ "table_styles",
+ "css",
+ ]
+ for attr in exp_attrs:
+ check = getattr(mi_styler, attr) == getattr(mi_styler_comp, attr)
+ assert not (
+ all(check) if (hasattr(check, "__iter__") and len(check) > 0) else check
+ )
+
+ export = mi_styler_comp.export()
+ used = mi_styler.use(export)
+ for attr in exp_attrs:
+ check = getattr(used, attr) == getattr(mi_styler_comp, attr)
+ assert all(check) if (hasattr(check, "__iter__") and len(check) > 0) else check
+
+ used.to_html()
+
+
+def test_hide_raises(mi_styler):
+ msg = "`subset` and `level` cannot be passed simultaneously"
+ with pytest.raises(ValueError, match=msg):
+ mi_styler.hide(axis="index", subset="something", level="something else")
+
+ msg = "`level` must be of type `int`, `str` or list of such"
+ with pytest.raises(ValueError, match=msg):
+ mi_styler.hide(axis="index", level={"bad": 1, "type": 2})
+
+
+@pytest.mark.parametrize("level", [1, "one", [1], ["one"]])
+def test_hide_index_level(mi_styler, level):
+ mi_styler.index.names, mi_styler.columns.names = ["zero", "one"], ["zero", "one"]
+ ctx = mi_styler.hide(axis="index", level=level)._translate(False, True)
+ assert len(ctx["head"][0]) == 3
+ assert len(ctx["head"][1]) == 3
+ assert len(ctx["head"][2]) == 4
+ assert ctx["head"][2][0]["is_visible"]
+ assert not ctx["head"][2][1]["is_visible"]
+
+ assert ctx["body"][0][0]["is_visible"]
+ assert not ctx["body"][0][1]["is_visible"]
+ assert ctx["body"][1][0]["is_visible"]
+ assert not ctx["body"][1][1]["is_visible"]
+
+
+@pytest.mark.parametrize("level", [1, "one", [1], ["one"]])
+@pytest.mark.parametrize("names", [True, False])
+def test_hide_columns_level(mi_styler, level, names):
+ mi_styler.columns.names = ["zero", "one"]
+ if names:
+ mi_styler.index.names = ["zero", "one"]
+ ctx = mi_styler.hide(axis="columns", level=level)._translate(True, False)
+ assert len(ctx["head"]) == (2 if names else 1)
+
+
+@pytest.mark.parametrize("method", ["map", "apply"])
+@pytest.mark.parametrize("axis", ["index", "columns"])
+def test_apply_map_header(method, axis):
+ # GH 41893
+ df = DataFrame({"A": [0, 0], "B": [1, 1]}, index=["C", "D"])
+ func = {
+ "apply": lambda s: ["attr: val" if ("A" in v or "C" in v) else "" for v in s],
+ "map": lambda v: "attr: val" if ("A" in v or "C" in v) else "",
+ }
+
+ # test execution added to todo
+ result = getattr(df.style, f"{method}_index")(func[method], axis=axis)
+ assert len(result._todo) == 1
+ assert len(getattr(result, f"ctx_{axis}")) == 0
+
+ # test ctx object on compute
+ result._compute()
+ expected = {
+ (0, 0): [("attr", "val")],
+ }
+ assert getattr(result, f"ctx_{axis}") == expected
+
+
+@pytest.mark.parametrize("method", ["apply", "map"])
+@pytest.mark.parametrize("axis", ["index", "columns"])
+def test_apply_map_header_mi(mi_styler, method, axis):
+ # GH 41893
+ func = {
+ "apply": lambda s: ["attr: val;" if "b" in v else "" for v in s],
+ "map": lambda v: "attr: val" if "b" in v else "",
+ }
+ result = getattr(mi_styler, f"{method}_index")(func[method], axis=axis)._compute()
+ expected = {(1, 1): [("attr", "val")]}
+ assert getattr(result, f"ctx_{axis}") == expected
+
+
+def test_apply_map_header_raises(mi_styler):
+ # GH 41893
+ with pytest.raises(ValueError, match="No axis named bad for object type DataFrame"):
+ mi_styler.map_index(lambda v: "attr: val;", axis="bad")._compute()
+
+
+class TestStyler:
+ def test_init_non_pandas(self):
+ msg = "``data`` must be a Series or DataFrame"
+ with pytest.raises(TypeError, match=msg):
+ Styler([1, 2, 3])
+
+ def test_init_series(self):
+ result = Styler(Series([1, 2]))
+ assert result.data.ndim == 2
+
+ def test_repr_html_ok(self, styler):
+ styler._repr_html_()
+
+ def test_repr_html_mathjax(self, styler):
+ # gh-19824 / 41395
+ assert "tex2jax_ignore" not in styler._repr_html_()
+ assert "mathjax_ignore" not in styler._repr_html_()
+
+ with option_context("styler.html.mathjax", False):
+ assert "tex2jax_ignore" in styler._repr_html_()
+ assert "mathjax_ignore" in styler._repr_html_()
+
+ def test_update_ctx(self, styler):
+ styler._update_ctx(DataFrame({"A": ["color: red", "color: blue"]}))
+ expected = {(0, 0): [("color", "red")], (1, 0): [("color", "blue")]}
+ assert styler.ctx == expected
+
+ def test_update_ctx_flatten_multi_and_trailing_semi(self, styler):
+ attrs = DataFrame({"A": ["color: red; foo: bar", "color:blue ; foo: baz;"]})
+ styler._update_ctx(attrs)
+ expected = {
+ (0, 0): [("color", "red"), ("foo", "bar")],
+ (1, 0): [("color", "blue"), ("foo", "baz")],
+ }
+ assert styler.ctx == expected
+
+ def test_render(self):
+ df = DataFrame({"A": [0, 1]})
+ style = lambda x: Series(["color: red", "color: blue"], name=x.name)
+ s = Styler(df, uuid="AB").apply(style)
+ s.to_html()
+ # it worked?
+
+ def test_multiple_render(self, df):
+ # GH 39396
+ s = Styler(df, uuid_len=0).map(lambda x: "color: red;", subset=["A"])
+ s.to_html() # do 2 renders to ensure css styles not duplicated
+ assert (
+ '" in s.to_html()
+ )
+
+ def test_render_empty_dfs(self):
+ empty_df = DataFrame()
+ es = Styler(empty_df)
+ es.to_html()
+ # An index but no columns
+ DataFrame(columns=["a"]).style.to_html()
+ # A column but no index
+ DataFrame(index=["a"]).style.to_html()
+ # No IndexError raised?
+
+ def test_render_double(self):
+ df = DataFrame({"A": [0, 1]})
+ style = lambda x: Series(
+ ["color: red; border: 1px", "color: blue; border: 2px"], name=x.name
+ )
+ s = Styler(df, uuid="AB").apply(style)
+ s.to_html()
+ # it worked?
+
+ def test_set_properties(self):
+ df = DataFrame({"A": [0, 1]})
+ result = df.style.set_properties(color="white", size="10px")._compute().ctx
+ # order is deterministic
+ v = [("color", "white"), ("size", "10px")]
+ expected = {(0, 0): v, (1, 0): v}
+ assert result.keys() == expected.keys()
+ for v1, v2 in zip(result.values(), expected.values()):
+ assert sorted(v1) == sorted(v2)
+
+ def test_set_properties_subset(self):
+ df = DataFrame({"A": [0, 1]})
+ result = (
+ df.style.set_properties(subset=IndexSlice[0, "A"], color="white")
+ ._compute()
+ .ctx
+ )
+ expected = {(0, 0): [("color", "white")]}
+ assert result == expected
+
+ def test_empty_index_name_doesnt_display(self, blank_value):
+ # https://github.com/pandas-dev/pandas/pull/12090#issuecomment-180695902
+ df = DataFrame({"A": [1, 2], "B": [3, 4], "C": [5, 6]})
+ result = df.style._translate(True, True)
+ assert len(result["head"]) == 1
+ expected = {
+ "class": "blank level0",
+ "type": "th",
+ "value": blank_value,
+ "is_visible": True,
+ "display_value": blank_value,
+ }
+ assert expected.items() <= result["head"][0][0].items()
+
+ def test_index_name(self):
+ # https://github.com/pandas-dev/pandas/issues/11655
+ df = DataFrame({"A": [1, 2], "B": [3, 4], "C": [5, 6]})
+ result = df.set_index("A").style._translate(True, True)
+ expected = {
+ "class": "index_name level0",
+ "type": "th",
+ "value": "A",
+ "is_visible": True,
+ "display_value": "A",
+ }
+ assert expected.items() <= result["head"][1][0].items()
+
+ def test_numeric_columns(self):
+ # https://github.com/pandas-dev/pandas/issues/12125
+ # smoke test for _translate
+ df = DataFrame({0: [1, 2, 3]})
+ df.style._translate(True, True)
+
+ def test_apply_axis(self):
+ df = DataFrame({"A": [0, 0], "B": [1, 1]})
+ f = lambda x: [f"val: {x.max()}" for v in x]
+ result = df.style.apply(f, axis=1)
+ assert len(result._todo) == 1
+ assert len(result.ctx) == 0
+ result._compute()
+ expected = {
+ (0, 0): [("val", "1")],
+ (0, 1): [("val", "1")],
+ (1, 0): [("val", "1")],
+ (1, 1): [("val", "1")],
+ }
+ assert result.ctx == expected
+
+ result = df.style.apply(f, axis=0)
+ expected = {
+ (0, 0): [("val", "0")],
+ (0, 1): [("val", "1")],
+ (1, 0): [("val", "0")],
+ (1, 1): [("val", "1")],
+ }
+ result._compute()
+ assert result.ctx == expected
+ result = df.style.apply(f) # default
+ result._compute()
+ assert result.ctx == expected
+
+ @pytest.mark.parametrize("axis", [0, 1])
+ def test_apply_series_return(self, axis):
+ # GH 42014
+ df = DataFrame([[1, 2], [3, 4]], index=["X", "Y"], columns=["X", "Y"])
+
+ # test Series return where len(Series) < df.index or df.columns but labels OK
+ func = lambda s: Series(["color: red;"], index=["Y"])
+ result = df.style.apply(func, axis=axis)._compute().ctx
+ assert result[(1, 1)] == [("color", "red")]
+ assert result[(1 - axis, axis)] == [("color", "red")]
+
+ # test Series return where labels align but different order
+ func = lambda s: Series(["color: red;", "color: blue;"], index=["Y", "X"])
+ result = df.style.apply(func, axis=axis)._compute().ctx
+ assert result[(0, 0)] == [("color", "blue")]
+ assert result[(1, 1)] == [("color", "red")]
+ assert result[(1 - axis, axis)] == [("color", "red")]
+ assert result[(axis, 1 - axis)] == [("color", "blue")]
+
+ @pytest.mark.parametrize("index", [False, True])
+ @pytest.mark.parametrize("columns", [False, True])
+ def test_apply_dataframe_return(self, index, columns):
+ # GH 42014
+ df = DataFrame([[1, 2], [3, 4]], index=["X", "Y"], columns=["X", "Y"])
+ idxs = ["X", "Y"] if index else ["Y"]
+ cols = ["X", "Y"] if columns else ["Y"]
+ df_styles = DataFrame("color: red;", index=idxs, columns=cols)
+ result = df.style.apply(lambda x: df_styles, axis=None)._compute().ctx
+
+ assert result[(1, 1)] == [("color", "red")] # (Y,Y) styles always present
+ assert (result[(0, 1)] == [("color", "red")]) is index # (X,Y) only if index
+ assert (result[(1, 0)] == [("color", "red")]) is columns # (Y,X) only if cols
+ assert (result[(0, 0)] == [("color", "red")]) is (index and columns) # (X,X)
+
+ @pytest.mark.parametrize(
+ "slice_",
+ [
+ IndexSlice[:],
+ IndexSlice[:, ["A"]],
+ IndexSlice[[1], :],
+ IndexSlice[[1], ["A"]],
+ IndexSlice[:2, ["A", "B"]],
+ ],
+ )
+ @pytest.mark.parametrize("axis", [0, 1])
+ def test_apply_subset(self, slice_, axis, df):
+ def h(x, color="bar"):
+ return Series(f"color: {color}", index=x.index, name=x.name)
+
+ result = df.style.apply(h, axis=axis, subset=slice_, color="baz")._compute().ctx
+ expected = {
+ (r, c): [("color", "baz")]
+ for r, row in enumerate(df.index)
+ for c, col in enumerate(df.columns)
+ if row in df.loc[slice_].index and col in df.loc[slice_].columns
+ }
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "slice_",
+ [
+ IndexSlice[:],
+ IndexSlice[:, ["A"]],
+ IndexSlice[[1], :],
+ IndexSlice[[1], ["A"]],
+ IndexSlice[:2, ["A", "B"]],
+ ],
+ )
+ def test_map_subset(self, slice_, df):
+ result = df.style.map(lambda x: "color:baz;", subset=slice_)._compute().ctx
+ expected = {
+ (r, c): [("color", "baz")]
+ for r, row in enumerate(df.index)
+ for c, col in enumerate(df.columns)
+ if row in df.loc[slice_].index and col in df.loc[slice_].columns
+ }
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "slice_",
+ [
+ IndexSlice[:, IndexSlice["x", "A"]],
+ IndexSlice[:, IndexSlice[:, "A"]],
+ IndexSlice[:, IndexSlice[:, ["A", "C"]]], # missing col element
+ IndexSlice[IndexSlice["a", 1], :],
+ IndexSlice[IndexSlice[:, 1], :],
+ IndexSlice[IndexSlice[:, [1, 3]], :], # missing row element
+ IndexSlice[:, ("x", "A")],
+ IndexSlice[("a", 1), :],
+ ],
+ )
+ def test_map_subset_multiindex(self, slice_):
+ # GH 19861
+ # edited for GH 33562
+ if (
+ isinstance(slice_[-1], tuple)
+ and isinstance(slice_[-1][-1], list)
+ and "C" in slice_[-1][-1]
+ ):
+ ctx = pytest.raises(KeyError, match="C")
+ elif (
+ isinstance(slice_[0], tuple)
+ and isinstance(slice_[0][1], list)
+ and 3 in slice_[0][1]
+ ):
+ ctx = pytest.raises(KeyError, match="3")
+ else:
+ ctx = contextlib.nullcontext()
+
+ idx = MultiIndex.from_product([["a", "b"], [1, 2]])
+ col = MultiIndex.from_product([["x", "y"], ["A", "B"]])
+ df = DataFrame(np.random.default_rng(2).random((4, 4)), columns=col, index=idx)
+
+ with ctx:
+ df.style.map(lambda x: "color: red;", subset=slice_).to_html()
+
+ def test_map_subset_multiindex_code(self):
+ # https://github.com/pandas-dev/pandas/issues/25858
+ # Checks styler.map works with multindex when codes are provided
+ codes = np.array([[0, 0, 1, 1], [0, 1, 0, 1]])
+ columns = MultiIndex(
+ levels=[["a", "b"], ["%", "#"]], codes=codes, names=["", ""]
+ )
+ df = DataFrame(
+ [[1, -1, 1, 1], [-1, 1, 1, 1]], index=["hello", "world"], columns=columns
+ )
+ pct_subset = IndexSlice[:, IndexSlice[:, "%":"%"]]
+
+ def color_negative_red(val):
+ color = "red" if val < 0 else "black"
+ return f"color: {color}"
+
+ df.loc[pct_subset]
+ df.style.map(color_negative_red, subset=pct_subset)
+
+ @pytest.mark.parametrize(
+ "stylefunc", ["background_gradient", "bar", "text_gradient"]
+ )
+ def test_subset_for_boolean_cols(self, stylefunc):
+ # GH47838
+ df = DataFrame(
+ [
+ [1, 2],
+ [3, 4],
+ ],
+ columns=[False, True],
+ )
+ styled = getattr(df.style, stylefunc)()
+ styled._compute()
+ assert set(styled.ctx) == {(0, 0), (0, 1), (1, 0), (1, 1)}
+
+ def test_empty(self):
+ df = DataFrame({"A": [1, 0]})
+ s = df.style
+ s.ctx = {(0, 0): [("color", "red")], (1, 0): [("", "")]}
+
+ result = s._translate(True, True)["cellstyle"]
+ expected = [
+ {"props": [("color", "red")], "selectors": ["row0_col0"]},
+ {"props": [("", "")], "selectors": ["row1_col0"]},
+ ]
+ assert result == expected
+
+ def test_duplicate(self):
+ df = DataFrame({"A": [1, 0]})
+ s = df.style
+ s.ctx = {(0, 0): [("color", "red")], (1, 0): [("color", "red")]}
+
+ result = s._translate(True, True)["cellstyle"]
+ expected = [
+ {"props": [("color", "red")], "selectors": ["row0_col0", "row1_col0"]}
+ ]
+ assert result == expected
+
+ def test_init_with_na_rep(self):
+ # GH 21527 28358
+ df = DataFrame([[None, None], [1.1, 1.2]], columns=["A", "B"])
+
+ ctx = Styler(df, na_rep="NA")._translate(True, True)
+ assert ctx["body"][0][1]["display_value"] == "NA"
+ assert ctx["body"][0][2]["display_value"] == "NA"
+
+ def test_caption(self, df):
+ styler = Styler(df, caption="foo")
+ result = styler.to_html()
+ assert all(["caption" in result, "foo" in result])
+
+ styler = df.style
+ result = styler.set_caption("baz")
+ assert styler is result
+ assert styler.caption == "baz"
+
+ def test_uuid(self, df):
+ styler = Styler(df, uuid="abc123")
+ result = styler.to_html()
+ assert "abc123" in result
+
+ styler = df.style
+ result = styler.set_uuid("aaa")
+ assert result is styler
+ assert result.uuid == "aaa"
+
+ def test_unique_id(self):
+ # See https://github.com/pandas-dev/pandas/issues/16780
+ df = DataFrame({"a": [1, 3, 5, 6], "b": [2, 4, 12, 21]})
+ result = df.style.to_html(uuid="test")
+ assert "test" in result
+ ids = re.findall('id="(.*?)"', result)
+ assert np.unique(ids).size == len(ids)
+
+ def test_table_styles(self, df):
+ style = [{"selector": "th", "props": [("foo", "bar")]}] # default format
+ styler = Styler(df, table_styles=style)
+ result = " ".join(styler.to_html().split())
+ assert "th { foo: bar; }" in result
+
+ styler = df.style
+ result = styler.set_table_styles(style)
+ assert styler is result
+ assert styler.table_styles == style
+
+ # GH 39563
+ style = [{"selector": "th", "props": "foo:bar;"}] # css string format
+ styler = df.style.set_table_styles(style)
+ result = " ".join(styler.to_html().split())
+ assert "th { foo: bar; }" in result
+
+ def test_table_styles_multiple(self, df):
+ ctx = df.style.set_table_styles(
+ [
+ {"selector": "th,td", "props": "color:red;"},
+ {"selector": "tr", "props": "color:green;"},
+ ]
+ )._translate(True, True)["table_styles"]
+ assert ctx == [
+ {"selector": "th", "props": [("color", "red")]},
+ {"selector": "td", "props": [("color", "red")]},
+ {"selector": "tr", "props": [("color", "green")]},
+ ]
+
+ def test_table_styles_dict_multiple_selectors(self, df):
+ # GH 44011
+ result = df.style.set_table_styles(
+ {
+ "B": [
+ {"selector": "th,td", "props": [("border-left", "2px solid black")]}
+ ]
+ }
+ )._translate(True, True)["table_styles"]
+
+ expected = [
+ {"selector": "th.col1", "props": [("border-left", "2px solid black")]},
+ {"selector": "td.col1", "props": [("border-left", "2px solid black")]},
+ ]
+
+ assert result == expected
+
+ def test_maybe_convert_css_to_tuples(self):
+ expected = [("a", "b"), ("c", "d e")]
+ assert maybe_convert_css_to_tuples("a:b;c:d e;") == expected
+ assert maybe_convert_css_to_tuples("a: b ;c: d e ") == expected
+ expected = []
+ assert maybe_convert_css_to_tuples("") == expected
+
+ # issue #59623
+ expected = [("a", "b"), ("c", "url('data:123')")]
+ assert maybe_convert_css_to_tuples("a:b;c: url('data:123');") == expected
+
+ # if no value, return attr and empty string
+ expected = [("a", ""), ("c", "")]
+ assert maybe_convert_css_to_tuples("a:;c: ") == expected
+
+ def test_maybe_convert_css_to_tuples_err(self):
+ msg = (
+ "Styles supplied as string must follow CSS rule formats, "
+ "for example 'attr: val;'. 'err' was given."
+ )
+ with pytest.raises(ValueError, match=msg):
+ maybe_convert_css_to_tuples("err")
+
+ def test_table_attributes(self, df):
+ attributes = 'class="foo" data-bar'
+ styler = Styler(df, table_attributes=attributes)
+ result = styler.to_html()
+ assert 'class="foo" data-bar' in result
+
+ result = df.style.set_table_attributes(attributes).to_html()
+ assert 'class="foo" data-bar' in result
+
+ def test_apply_none(self):
+ def f(x):
+ return DataFrame(
+ np.where(x == x.max(), "color: red", ""),
+ index=x.index,
+ columns=x.columns,
+ )
+
+ result = DataFrame([[1, 2], [3, 4]]).style.apply(f, axis=None)._compute().ctx
+ assert result[(1, 1)] == [("color", "red")]
+
+ def test_trim(self, df):
+ result = df.style.to_html() # trim=True
+ assert result.count("#") == 0
+
+ result = df.style.highlight_max().to_html()
+ assert result.count("#") == len(df.columns)
+
+ def test_export(self, df, styler):
+ f = lambda x: "color: red" if x > 0 else "color: blue"
+ g = lambda x, z: f"color: {z}"
+ style1 = styler
+ style1.map(f).map(g, z="b").highlight_max()._compute() # = render
+ result = style1.export()
+ style2 = df.style
+ style2.use(result)
+ assert style1._todo == style2._todo
+ style2.to_html()
+
+ def test_bad_apply_shape(self):
+ df = DataFrame([[1, 2], [3, 4]], index=["A", "B"], columns=["X", "Y"])
+
+ msg = "resulted in the apply method collapsing to a Series."
+ with pytest.raises(ValueError, match=msg):
+ df.style._apply(lambda x: "x")
+
+ msg = "created invalid {} labels"
+ with pytest.raises(ValueError, match=msg.format("index")):
+ df.style._apply(lambda x: [""])
+
+ with pytest.raises(ValueError, match=msg.format("index")):
+ df.style._apply(lambda x: ["", "", "", ""])
+
+ with pytest.raises(ValueError, match=msg.format("index")):
+ df.style._apply(lambda x: Series(["a:v;", ""], index=["A", "C"]), axis=0)
+
+ with pytest.raises(ValueError, match=msg.format("columns")):
+ df.style._apply(lambda x: ["", "", ""], axis=1)
+
+ with pytest.raises(ValueError, match=msg.format("columns")):
+ df.style._apply(lambda x: Series(["a:v;", ""], index=["X", "Z"]), axis=1)
+
+ msg = "returned ndarray with wrong shape"
+ with pytest.raises(ValueError, match=msg):
+ df.style._apply(lambda x: np.array([[""], [""]]), axis=None)
+
+ def test_apply_bad_return(self):
+ def f(x):
+ return ""
+
+ df = DataFrame([[1, 2], [3, 4]])
+ msg = (
+ "must return a DataFrame or ndarray when passed to `Styler.apply` "
+ "with axis=None"
+ )
+ with pytest.raises(TypeError, match=msg):
+ df.style._apply(f, axis=None)
+
+ @pytest.mark.parametrize("axis", ["index", "columns"])
+ def test_apply_bad_labels(self, axis):
+ def f(x):
+ return DataFrame(**{axis: ["bad", "labels"]})
+
+ df = DataFrame([[1, 2], [3, 4]])
+ msg = f"created invalid {axis} labels."
+ with pytest.raises(ValueError, match=msg):
+ df.style._apply(f, axis=None)
+
+ def test_get_level_lengths(self):
+ index = MultiIndex.from_product([["a", "b"], [0, 1, 2]])
+ expected = {
+ (0, 0): 3,
+ (0, 3): 3,
+ (1, 0): 1,
+ (1, 1): 1,
+ (1, 2): 1,
+ (1, 3): 1,
+ (1, 4): 1,
+ (1, 5): 1,
+ }
+ result = _get_level_lengths(index, sparsify=True, max_index=100)
+ tm.assert_dict_equal(result, expected)
+
+ expected = {
+ (0, 0): 1,
+ (0, 1): 1,
+ (0, 2): 1,
+ (0, 3): 1,
+ (0, 4): 1,
+ (0, 5): 1,
+ (1, 0): 1,
+ (1, 1): 1,
+ (1, 2): 1,
+ (1, 3): 1,
+ (1, 4): 1,
+ (1, 5): 1,
+ }
+ result = _get_level_lengths(index, sparsify=False, max_index=100)
+ tm.assert_dict_equal(result, expected)
+
+ def test_get_level_lengths_un_sorted(self):
+ index = MultiIndex.from_arrays([[1, 1, 2, 1], ["a", "b", "b", "d"]])
+ expected = {
+ (0, 0): 2,
+ (0, 2): 1,
+ (0, 3): 1,
+ (1, 0): 1,
+ (1, 1): 1,
+ (1, 2): 1,
+ (1, 3): 1,
+ }
+ result = _get_level_lengths(index, sparsify=True, max_index=100)
+ tm.assert_dict_equal(result, expected)
+
+ expected = {
+ (0, 0): 1,
+ (0, 1): 1,
+ (0, 2): 1,
+ (0, 3): 1,
+ (1, 0): 1,
+ (1, 1): 1,
+ (1, 2): 1,
+ (1, 3): 1,
+ }
+ result = _get_level_lengths(index, sparsify=False, max_index=100)
+ tm.assert_dict_equal(result, expected)
+
+ def test_mi_sparse_index_names(self, blank_value):
+ # Test the class names and displayed value are correct on rendering MI names
+ df = DataFrame(
+ {"A": [1, 2]},
+ index=MultiIndex.from_arrays(
+ [["a", "a"], [0, 1]], names=["idx_level_0", "idx_level_1"]
+ ),
+ )
+ result = df.style._translate(True, True)
+ head = result["head"][1]
+ expected = [
+ {
+ "class": "index_name level0",
+ "display_value": "idx_level_0",
+ "is_visible": True,
+ },
+ {
+ "class": "index_name level1",
+ "display_value": "idx_level_1",
+ "is_visible": True,
+ },
+ {
+ "class": "blank col0",
+ "display_value": blank_value,
+ "is_visible": True,
+ },
+ ]
+ for i, expected_dict in enumerate(expected):
+ assert expected_dict.items() <= head[i].items()
+
+ def test_mi_sparse_column_names(self, blank_value):
+ df = DataFrame(
+ np.arange(16).reshape(4, 4),
+ index=MultiIndex.from_arrays(
+ [["a", "a", "b", "a"], [0, 1, 1, 2]],
+ names=["idx_level_0", "idx_level_1"],
+ ),
+ columns=MultiIndex.from_arrays(
+ [["C1", "C1", "C2", "C2"], [1, 0, 1, 0]], names=["colnam_0", "colnam_1"]
+ ),
+ )
+ result = Styler(df, cell_ids=False)._translate(True, True)
+
+ for level in [0, 1]:
+ head = result["head"][level]
+ expected = [
+ {
+ "class": "blank",
+ "display_value": blank_value,
+ "is_visible": True,
+ },
+ {
+ "class": f"index_name level{level}",
+ "display_value": f"colnam_{level}",
+ "is_visible": True,
+ },
+ ]
+ for i, expected_dict in enumerate(expected):
+ assert expected_dict.items() <= head[i].items()
+
+ def test_hide_column_headers(self, df, styler):
+ ctx = styler.hide(axis="columns")._translate(True, True)
+ assert len(ctx["head"]) == 0 # no header entries with an unnamed index
+
+ df.index.name = "some_name"
+ ctx = df.style.hide(axis="columns")._translate(True, True)
+ assert len(ctx["head"]) == 1
+ # index names still visible, changed in #42101, reverted in 43404
+
+ def test_hide_single_index(self, df):
+ # GH 14194
+ # single unnamed index
+ ctx = df.style._translate(True, True)
+ assert ctx["body"][0][0]["is_visible"]
+ assert ctx["head"][0][0]["is_visible"]
+ ctx2 = df.style.hide(axis="index")._translate(True, True)
+ assert not ctx2["body"][0][0]["is_visible"]
+ assert not ctx2["head"][0][0]["is_visible"]
+
+ # single named index
+ ctx3 = df.set_index("A").style._translate(True, True)
+ assert ctx3["body"][0][0]["is_visible"]
+ assert len(ctx3["head"]) == 2 # 2 header levels
+ assert ctx3["head"][0][0]["is_visible"]
+
+ ctx4 = df.set_index("A").style.hide(axis="index")._translate(True, True)
+ assert not ctx4["body"][0][0]["is_visible"]
+ assert len(ctx4["head"]) == 1 # only 1 header levels
+ assert not ctx4["head"][0][0]["is_visible"]
+
+ def test_hide_multiindex(self):
+ # GH 14194
+ df = DataFrame(
+ {"A": [1, 2], "B": [1, 2]},
+ index=MultiIndex.from_arrays(
+ [["a", "a"], [0, 1]], names=["idx_level_0", "idx_level_1"]
+ ),
+ )
+ ctx1 = df.style._translate(True, True)
+ # tests for 'a' and '0'
+ assert ctx1["body"][0][0]["is_visible"]
+ assert ctx1["body"][0][1]["is_visible"]
+ # check for blank header rows
+ assert len(ctx1["head"][0]) == 4 # two visible indexes and two data columns
+
+ ctx2 = df.style.hide(axis="index")._translate(True, True)
+ # tests for 'a' and '0'
+ assert not ctx2["body"][0][0]["is_visible"]
+ assert not ctx2["body"][0][1]["is_visible"]
+ # check for blank header rows
+ assert len(ctx2["head"][0]) == 3 # one hidden (col name) and two data columns
+ assert not ctx2["head"][0][0]["is_visible"]
+
+ def test_hide_columns_single_level(self, df):
+ # GH 14194
+ # test hiding single column
+ ctx = df.style._translate(True, True)
+ assert ctx["head"][0][1]["is_visible"]
+ assert ctx["head"][0][1]["display_value"] == "A"
+ assert ctx["head"][0][2]["is_visible"]
+ assert ctx["head"][0][2]["display_value"] == "B"
+ assert ctx["body"][0][1]["is_visible"] # col A, row 1
+ assert ctx["body"][1][2]["is_visible"] # col B, row 1
+
+ ctx = df.style.hide("A", axis="columns")._translate(True, True)
+ assert not ctx["head"][0][1]["is_visible"]
+ assert not ctx["body"][0][1]["is_visible"] # col A, row 1
+ assert ctx["body"][1][2]["is_visible"] # col B, row 1
+
+ # test hiding multiple columns
+ ctx = df.style.hide(["A", "B"], axis="columns")._translate(True, True)
+ assert not ctx["head"][0][1]["is_visible"]
+ assert not ctx["head"][0][2]["is_visible"]
+ assert not ctx["body"][0][1]["is_visible"] # col A, row 1
+ assert not ctx["body"][1][2]["is_visible"] # col B, row 1
+
+ def test_hide_columns_index_mult_levels(self):
+ # GH 14194
+ # setup dataframe with multiple column levels and indices
+ i1 = MultiIndex.from_arrays(
+ [["a", "a"], [0, 1]], names=["idx_level_0", "idx_level_1"]
+ )
+ i2 = MultiIndex.from_arrays(
+ [["b", "b"], [0, 1]], names=["col_level_0", "col_level_1"]
+ )
+ df = DataFrame([[1, 2], [3, 4]], index=i1, columns=i2)
+ ctx = df.style._translate(True, True)
+ # column headers
+ assert ctx["head"][0][2]["is_visible"]
+ assert ctx["head"][1][2]["is_visible"]
+ assert ctx["head"][1][3]["display_value"] == "1"
+ # indices
+ assert ctx["body"][0][0]["is_visible"]
+ # data
+ assert ctx["body"][1][2]["is_visible"]
+ assert ctx["body"][1][2]["display_value"] == "3"
+ assert ctx["body"][1][3]["is_visible"]
+ assert ctx["body"][1][3]["display_value"] == "4"
+
+ # hide top column level, which hides both columns
+ ctx = df.style.hide("b", axis="columns")._translate(True, True)
+ assert not ctx["head"][0][2]["is_visible"] # b
+ assert not ctx["head"][1][2]["is_visible"] # 0
+ assert not ctx["body"][1][2]["is_visible"] # 3
+ assert ctx["body"][0][0]["is_visible"] # index
+
+ # hide first column only
+ ctx = df.style.hide([("b", 0)], axis="columns")._translate(True, True)
+ assert not ctx["head"][0][2]["is_visible"] # b
+ assert ctx["head"][0][3]["is_visible"] # b
+ assert not ctx["head"][1][2]["is_visible"] # 0
+ assert not ctx["body"][1][2]["is_visible"] # 3
+ assert ctx["body"][1][3]["is_visible"]
+ assert ctx["body"][1][3]["display_value"] == "4"
+
+ # hide second column and index
+ ctx = df.style.hide([("b", 1)], axis=1).hide(axis=0)._translate(True, True)
+ assert not ctx["body"][0][0]["is_visible"] # index
+ assert len(ctx["head"][0]) == 3
+ assert ctx["head"][0][1]["is_visible"] # b
+ assert ctx["head"][1][1]["is_visible"] # 0
+ assert not ctx["head"][1][2]["is_visible"] # 1
+ assert not ctx["body"][1][3]["is_visible"] # 4
+ assert ctx["body"][1][2]["is_visible"]
+ assert ctx["body"][1][2]["display_value"] == "3"
+
+ # hide top row level, which hides both rows so body empty
+ ctx = df.style.hide("a", axis="index")._translate(True, True)
+ assert ctx["body"] == []
+
+ # hide first row only
+ ctx = df.style.hide(("a", 0), axis="index")._translate(True, True)
+ for i in [0, 1, 2, 3]:
+ assert "row1" in ctx["body"][0][i]["class"] # row0 not included in body
+ assert ctx["body"][0][i]["is_visible"]
+
+ def test_pipe(self, df):
+ def set_caption_from_template(styler, a, b):
+ return styler.set_caption(f"Dataframe with a = {a} and b = {b}")
+
+ styler = df.style.pipe(set_caption_from_template, "A", b="B")
+ assert "Dataframe with a = A and b = B" in styler.to_html()
+
+ # Test with an argument that is a (callable, keyword_name) pair.
+ def f(a, b, styler):
+ return (a, b, styler)
+
+ styler = df.style
+ result = styler.pipe((f, "styler"), a=1, b=2)
+ assert result == (1, 2, styler)
+
+ def test_no_cell_ids(self):
+ # GH 35588
+ # GH 35663
+ df = DataFrame(data=[[0]])
+ styler = Styler(df, uuid="_", cell_ids=False)
+ styler.to_html()
+ s = styler.to_html() # render twice to ensure ctx is not updated
+ assert s.find('') != -1
+
+ @pytest.mark.parametrize(
+ "classes",
+ [
+ DataFrame(
+ data=[["", "test-class"], [np.nan, None]],
+ columns=["A", "B"],
+ index=["a", "b"],
+ ),
+ DataFrame(data=[["test-class"]], columns=["B"], index=["a"]),
+ DataFrame(data=[["test-class", "unused"]], columns=["B", "C"], index=["a"]),
+ ],
+ )
+ def test_set_data_classes(self, classes):
+ # GH 36159
+ df = DataFrame(data=[[0, 1], [2, 3]], columns=["A", "B"], index=["a", "b"])
+ s = Styler(df, uuid_len=0, cell_ids=False).set_td_classes(classes).to_html()
+ assert ' 0 ' in s
+ assert '1 ' in s
+ assert '2 ' in s
+ assert '3 ' in s
+ # GH 39317
+ s = Styler(df, uuid_len=0, cell_ids=True).set_td_classes(classes).to_html()
+ assert '0 ' in s
+ assert '1 ' in s
+ assert '2 ' in s
+ assert '3 ' in s
+
+ def test_set_data_classes_reindex(self):
+ # GH 39317
+ df = DataFrame(
+ data=[[0, 1, 2], [3, 4, 5], [6, 7, 8]], columns=[0, 1, 2], index=[0, 1, 2]
+ )
+ classes = DataFrame(
+ data=[["mi", "ma"], ["mu", "mo"]],
+ columns=[0, 2],
+ index=[0, 2],
+ )
+ s = Styler(df, uuid_len=0).set_td_classes(classes).to_html()
+ assert '0 ' in s
+ assert '2 ' in s
+ assert '4 ' in s
+ assert '6 ' in s
+ assert '8 ' in s
+
+ def test_chaining_table_styles(self):
+ # GH 35607
+ df = DataFrame(data=[[0, 1], [1, 2]], columns=["A", "B"])
+ styler = df.style.set_table_styles(
+ [{"selector": "", "props": [("background-color", "yellow")]}]
+ ).set_table_styles(
+ [{"selector": ".col0", "props": [("background-color", "blue")]}],
+ overwrite=False,
+ )
+ assert len(styler.table_styles) == 2
+
+ def test_column_and_row_styling(self):
+ # GH 35607
+ df = DataFrame(data=[[0, 1], [1, 2]], columns=["A", "B"])
+ s = Styler(df, uuid_len=0)
+ s = s.set_table_styles({"A": [{"selector": "", "props": [("color", "blue")]}]})
+ assert "#T_ .col0 {\n color: blue;\n}" in s.to_html()
+ s = s.set_table_styles(
+ {0: [{"selector": "", "props": [("color", "blue")]}]}, axis=1
+ )
+ assert "#T_ .row0 {\n color: blue;\n}" in s.to_html()
+
+ @pytest.mark.parametrize("len_", [1, 5, 32, 33, 100])
+ def test_uuid_len(self, len_):
+ # GH 36345
+ df = DataFrame(data=[["A"]])
+ s = Styler(df, uuid_len=len_, cell_ids=False).to_html()
+ strt = s.find('id="T_')
+ end = s[strt + 6 :].find('"')
+ if len_ > 32:
+ assert end == 32
+ else:
+ assert end == len_
+
+ @pytest.mark.parametrize("len_", [-2, "bad", None])
+ def test_uuid_len_raises(self, len_):
+ # GH 36345
+ df = DataFrame(data=[["A"]])
+ msg = "``uuid_len`` must be an integer in range \\[0, 32\\]."
+ with pytest.raises(TypeError, match=msg):
+ Styler(df, uuid_len=len_, cell_ids=False).to_html()
+
+ @pytest.mark.parametrize(
+ "slc",
+ [
+ IndexSlice[:, :],
+ IndexSlice[:, 1],
+ IndexSlice[1, :],
+ IndexSlice[[1], [1]],
+ IndexSlice[1, [1]],
+ IndexSlice[[1], 1],
+ IndexSlice[1],
+ IndexSlice[1, 1],
+ slice(None, None, None),
+ [0, 1],
+ np.array([0, 1]),
+ Series([0, 1]),
+ ],
+ )
+ def test_non_reducing_slice(self, slc):
+ df = DataFrame([[0, 1], [2, 3]])
+
+ tslice_ = non_reducing_slice(slc)
+ assert isinstance(df.loc[tslice_], DataFrame)
+
+ @pytest.mark.parametrize("box", [list, Series, np.array])
+ def test_list_slice(self, box):
+ # like dataframe getitem
+ subset = box(["A"])
+
+ df = DataFrame({"A": [1, 2], "B": [3, 4]}, index=["A", "B"])
+ expected = IndexSlice[:, ["A"]]
+
+ result = non_reducing_slice(subset)
+ tm.assert_frame_equal(df.loc[result], df.loc[expected])
+
+ def test_non_reducing_slice_on_multiindex(self):
+ # GH 19861
+ dic = {
+ ("a", "d"): [1, 4],
+ ("a", "c"): [2, 3],
+ ("b", "c"): [3, 2],
+ ("b", "d"): [4, 1],
+ }
+ df = DataFrame(dic, index=[0, 1])
+ idx = IndexSlice
+ slice_ = idx[:, idx["b", "d"]]
+ tslice_ = non_reducing_slice(slice_)
+
+ result = df.loc[tslice_]
+ expected = DataFrame({("b", "d"): [4, 1]})
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "slice_",
+ [
+ IndexSlice[:, :],
+ # check cols
+ IndexSlice[:, IndexSlice[["a"]]], # inferred deeper need list
+ IndexSlice[:, IndexSlice[["a"], ["c"]]], # inferred deeper need list
+ IndexSlice[:, IndexSlice["a", "c", :]],
+ IndexSlice[:, IndexSlice["a", :, "e"]],
+ IndexSlice[:, IndexSlice[:, "c", "e"]],
+ IndexSlice[:, IndexSlice["a", ["c", "d"], :]], # check list
+ IndexSlice[:, IndexSlice["a", ["c", "d", "-"], :]], # don't allow missing
+ IndexSlice[:, IndexSlice["a", ["c", "d", "-"], "e"]], # no slice
+ # check rows
+ IndexSlice[IndexSlice[["U"]], :], # inferred deeper need list
+ IndexSlice[IndexSlice[["U"], ["W"]], :], # inferred deeper need list
+ IndexSlice[IndexSlice["U", "W", :], :],
+ IndexSlice[IndexSlice["U", :, "Y"], :],
+ IndexSlice[IndexSlice[:, "W", "Y"], :],
+ IndexSlice[IndexSlice[:, "W", ["Y", "Z"]], :], # check list
+ IndexSlice[IndexSlice[:, "W", ["Y", "Z", "-"]], :], # don't allow missing
+ IndexSlice[IndexSlice["U", "W", ["Y", "Z", "-"]], :], # no slice
+ # check simultaneous
+ IndexSlice[IndexSlice[:, "W", "Y"], IndexSlice["a", "c", :]],
+ ],
+ )
+ def test_non_reducing_multi_slice_on_multiindex(self, slice_):
+ # GH 33562
+ cols = MultiIndex.from_product([["a", "b"], ["c", "d"], ["e", "f"]])
+ idxs = MultiIndex.from_product([["U", "V"], ["W", "X"], ["Y", "Z"]])
+ df = DataFrame(np.arange(64).reshape(8, 8), columns=cols, index=idxs)
+
+ for lvl in [0, 1]:
+ key = slice_[lvl]
+ if isinstance(key, tuple):
+ for subkey in key:
+ if isinstance(subkey, list) and "-" in subkey:
+ # not present in the index level, raises KeyError since 2.0
+ with pytest.raises(KeyError, match="-"):
+ df.loc[slice_]
+ return
+
+ expected = df.loc[slice_]
+ result = df.loc[non_reducing_slice(slice_)]
+ tm.assert_frame_equal(result, expected)
+
+
+def test_hidden_index_names(mi_df):
+ mi_df.index.names = ["Lev0", "Lev1"]
+ mi_styler = mi_df.style
+ ctx = mi_styler._translate(True, True)
+ assert len(ctx["head"]) == 3 # 2 column index levels + 1 index names row
+
+ mi_styler.hide(axis="index", names=True)
+ ctx = mi_styler._translate(True, True)
+ assert len(ctx["head"]) == 2 # index names row is unparsed
+ for i in range(4):
+ assert ctx["body"][0][i]["is_visible"] # 2 index levels + 2 data values visible
+
+ mi_styler.hide(axis="index", level=1)
+ ctx = mi_styler._translate(True, True)
+ assert len(ctx["head"]) == 2 # index names row is still hidden
+ assert ctx["body"][0][0]["is_visible"] is True
+ assert ctx["body"][0][1]["is_visible"] is False
+
+
+def test_hidden_column_names(mi_df):
+ mi_df.columns.names = ["Lev0", "Lev1"]
+ mi_styler = mi_df.style
+ ctx = mi_styler._translate(True, True)
+ assert ctx["head"][0][1]["display_value"] == "Lev0"
+ assert ctx["head"][1][1]["display_value"] == "Lev1"
+
+ mi_styler.hide(names=True, axis="columns")
+ ctx = mi_styler._translate(True, True)
+ assert ctx["head"][0][1]["display_value"] == " "
+ assert ctx["head"][1][1]["display_value"] == " "
+
+ mi_styler.hide(level=0, axis="columns")
+ ctx = mi_styler._translate(True, True)
+ assert len(ctx["head"]) == 1 # no index names and only one visible column headers
+ assert ctx["head"][0][1]["display_value"] == " "
+
+
+@pytest.mark.parametrize("caption", [1, ("a", "b", "c"), (1, "s")])
+def test_caption_raises(mi_styler, caption):
+ msg = "`caption` must be either a string or 2-tuple of strings."
+ with pytest.raises(ValueError, match=msg):
+ mi_styler.set_caption(caption)
+
+
+def test_hiding_headers_over_index_no_sparsify():
+ # GH 43464
+ midx = MultiIndex.from_product([[1, 2], ["a", "a", "b"]])
+ df = DataFrame(9, index=midx, columns=[0])
+ ctx = df.style._translate(False, False)
+ assert len(ctx["body"]) == 6
+ ctx = df.style.hide((1, "a"), axis=0)._translate(False, False)
+ assert len(ctx["body"]) == 4
+ assert "row2" in ctx["body"][0][0]["class"]
+
+
+def test_hiding_headers_over_columns_no_sparsify():
+ # GH 43464
+ midx = MultiIndex.from_product([[1, 2], ["a", "a", "b"]])
+ df = DataFrame(9, columns=midx, index=[0])
+ ctx = df.style._translate(False, False)
+ for ix in [(0, 1), (0, 2), (1, 1), (1, 2)]:
+ assert ctx["head"][ix[0]][ix[1]]["is_visible"] is True
+ ctx = df.style.hide((1, "a"), axis="columns")._translate(False, False)
+ for ix in [(0, 1), (0, 2), (1, 1), (1, 2)]:
+ assert ctx["head"][ix[0]][ix[1]]["is_visible"] is False
+
+
+def test_get_level_lengths_mi_hidden():
+ # GH 43464
+ index = MultiIndex.from_arrays([[1, 1, 1, 2, 2, 2], ["a", "a", "b", "a", "a", "b"]])
+ expected = {
+ (0, 2): 1,
+ (0, 3): 1,
+ (0, 4): 1,
+ (0, 5): 1,
+ (1, 2): 1,
+ (1, 3): 1,
+ (1, 4): 1,
+ (1, 5): 1,
+ }
+ result = _get_level_lengths(
+ index,
+ sparsify=False,
+ max_index=100,
+ hidden_elements=[0, 1, 0, 1], # hidden element can repeat if duplicated index
+ )
+ tm.assert_dict_equal(result, expected)
+
+
+def test_row_trimming_hide_index():
+ # gh 43703
+ df = DataFrame([[1], [2], [3], [4], [5]])
+ with option_context("styler.render.max_rows", 2):
+ ctx = df.style.hide([0, 1], axis="index")._translate(True, True)
+ assert len(ctx["body"]) == 3
+ for r, val in enumerate(["3", "4", "..."]):
+ assert ctx["body"][r][1]["display_value"] == val
+
+
+def test_row_trimming_hide_index_mi():
+ # gh 44247
+ df = DataFrame([[1], [2], [3], [4], [5]])
+ df.index = MultiIndex.from_product([[0], [0, 1, 2, 3, 4]])
+ with option_context("styler.render.max_rows", 2):
+ ctx = df.style.hide([(0, 0), (0, 1)], axis="index")._translate(True, True)
+ assert len(ctx["body"]) == 3
+
+ # level 0 index headers (sparsified)
+ assert {"value": 0, "attributes": 'rowspan="2"', "is_visible": True}.items() <= ctx[
+ "body"
+ ][0][0].items()
+ assert {"value": 0, "attributes": "", "is_visible": False}.items() <= ctx["body"][
+ 1
+ ][0].items()
+ assert {"value": "...", "is_visible": True}.items() <= ctx["body"][2][0].items()
+
+ for r, val in enumerate(["2", "3", "..."]):
+ assert ctx["body"][r][1]["display_value"] == val # level 1 index headers
+ for r, val in enumerate(["3", "4", "..."]):
+ assert ctx["body"][r][2]["display_value"] == val # data values
+
+
+def test_col_trimming_hide_columns():
+ # gh 44272
+ df = DataFrame([[1, 2, 3, 4, 5]])
+ with option_context("styler.render.max_columns", 2):
+ ctx = df.style.hide([0, 1], axis="columns")._translate(True, True)
+
+ assert len(ctx["head"][0]) == 6 # blank, [0, 1 (hidden)], [2 ,3 (visible)], + trim
+ for c, vals in enumerate([(1, False), (2, True), (3, True), ("...", True)]):
+ assert ctx["head"][0][c + 2]["value"] == vals[0]
+ assert ctx["head"][0][c + 2]["is_visible"] == vals[1]
+
+ assert len(ctx["body"][0]) == 6 # index + 2 hidden + 2 visible + trimming col
+
+
+def test_no_empty_apply(mi_styler):
+ # 45313
+ mi_styler.apply(lambda s: ["a:v;"] * 2, subset=[False, False])
+ mi_styler._compute()
+
+
+@pytest.mark.parametrize("format", ["html", "latex", "string"])
+def test_output_buffer(mi_styler, format, temp_file):
+ # gh 47053
+ getattr(mi_styler, f"to_{format}")(temp_file)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_to_latex.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_to_latex.py
new file mode 100644
index 0000000000000000000000000000000000000000..eb221686dd165be0dd55dac77c25f11c047f1f23
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/style/test_to_latex.py
@@ -0,0 +1,1091 @@
+from textwrap import dedent
+
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ MultiIndex,
+ Series,
+ option_context,
+)
+
+pytest.importorskip("jinja2")
+from pandas.io.formats.style import Styler
+from pandas.io.formats.style_render import (
+ _parse_latex_cell_styles,
+ _parse_latex_css_conversion,
+ _parse_latex_header_span,
+ _parse_latex_table_styles,
+ _parse_latex_table_wrapping,
+)
+
+
+@pytest.fixture
+def df():
+ return DataFrame(
+ {"A": [0, 1], "B": [-0.61, -1.22], "C": Series(["ab", "cd"], dtype=object)}
+ )
+
+
+@pytest.fixture
+def df_ext():
+ return DataFrame(
+ {"A": [0, 1, 2], "B": [-0.61, -1.22, -2.22], "C": ["ab", "cd", "de"]}
+ )
+
+
+@pytest.fixture
+def styler(df):
+ return Styler(df, uuid_len=0, precision=2)
+
+
+def test_minimal_latex_tabular(styler):
+ expected = dedent(
+ """\
+ \\begin{tabular}{lrrl}
+ & A & B & C \\\\
+ 0 & 0 & -0.61 & ab \\\\
+ 1 & 1 & -1.22 & cd \\\\
+ \\end{tabular}
+ """
+ )
+ assert styler.to_latex() == expected
+
+
+def test_tabular_hrules(styler):
+ expected = dedent(
+ """\
+ \\begin{tabular}{lrrl}
+ \\toprule
+ & A & B & C \\\\
+ \\midrule
+ 0 & 0 & -0.61 & ab \\\\
+ 1 & 1 & -1.22 & cd \\\\
+ \\bottomrule
+ \\end{tabular}
+ """
+ )
+ assert styler.to_latex(hrules=True) == expected
+
+
+def test_tabular_custom_hrules(styler):
+ styler.set_table_styles(
+ [
+ {"selector": "toprule", "props": ":hline"},
+ {"selector": "bottomrule", "props": ":otherline"},
+ ]
+ ) # no midrule
+ expected = dedent(
+ """\
+ \\begin{tabular}{lrrl}
+ \\hline
+ & A & B & C \\\\
+ 0 & 0 & -0.61 & ab \\\\
+ 1 & 1 & -1.22 & cd \\\\
+ \\otherline
+ \\end{tabular}
+ """
+ )
+ assert styler.to_latex() == expected
+
+
+def test_column_format(styler):
+ # default setting is already tested in `test_latex_minimal_tabular`
+ styler.set_table_styles([{"selector": "column_format", "props": ":cccc"}])
+
+ assert "\\begin{tabular}{rrrr}" in styler.to_latex(column_format="rrrr")
+ styler.set_table_styles([{"selector": "column_format", "props": ":r|r|cc"}])
+ assert "\\begin{tabular}{r|r|cc}" in styler.to_latex()
+
+
+def test_siunitx_cols(styler):
+ expected = dedent(
+ """\
+ \\begin{tabular}{lSSl}
+ {} & {A} & {B} & {C} \\\\
+ 0 & 0 & -0.61 & ab \\\\
+ 1 & 1 & -1.22 & cd \\\\
+ \\end{tabular}
+ """
+ )
+ assert styler.to_latex(siunitx=True) == expected
+
+
+def test_position(styler):
+ assert "\\begin{table}[h!]" in styler.to_latex(position="h!")
+ assert "\\end{table}" in styler.to_latex(position="h!")
+ styler.set_table_styles([{"selector": "position", "props": ":b!"}])
+ assert "\\begin{table}[b!]" in styler.to_latex()
+ assert "\\end{table}" in styler.to_latex()
+
+
+@pytest.mark.parametrize("env", [None, "longtable"])
+def test_label(styler, env):
+ assert "\n\\label{text}" in styler.to_latex(label="text", environment=env)
+ styler.set_table_styles([{"selector": "label", "props": ":{more §text}"}])
+ assert "\n\\label{more :text}" in styler.to_latex(environment=env)
+
+
+def test_position_float_raises(styler):
+ msg = "`position_float` should be one of 'raggedright', 'raggedleft', 'centering',"
+ with pytest.raises(ValueError, match=msg):
+ styler.to_latex(position_float="bad_string")
+
+ msg = "`position_float` cannot be used in 'longtable' `environment`"
+ with pytest.raises(ValueError, match=msg):
+ styler.to_latex(position_float="centering", environment="longtable")
+
+
+@pytest.mark.parametrize("label", [(None, ""), ("text", "\\label{text}")])
+@pytest.mark.parametrize("position", [(None, ""), ("h!", "{table}[h!]")])
+@pytest.mark.parametrize("caption", [(None, ""), ("text", "\\caption{text}")])
+@pytest.mark.parametrize("column_format", [(None, ""), ("rcrl", "{tabular}{rcrl}")])
+@pytest.mark.parametrize("position_float", [(None, ""), ("centering", "\\centering")])
+def test_kwargs_combinations(
+ styler, label, position, caption, column_format, position_float
+):
+ result = styler.to_latex(
+ label=label[0],
+ position=position[0],
+ caption=caption[0],
+ column_format=column_format[0],
+ position_float=position_float[0],
+ )
+ assert label[1] in result
+ assert position[1] in result
+ assert caption[1] in result
+ assert column_format[1] in result
+ assert position_float[1] in result
+
+
+def test_custom_table_styles(styler):
+ styler.set_table_styles(
+ [
+ {"selector": "mycommand", "props": ":{myoptions}"},
+ {"selector": "mycommand2", "props": ":{myoptions2}"},
+ ]
+ )
+ expected = dedent(
+ """\
+ \\begin{table}
+ \\mycommand{myoptions}
+ \\mycommand2{myoptions2}
+ """
+ )
+ assert expected in styler.to_latex()
+
+
+def test_cell_styling(styler):
+ styler.highlight_max(props="itshape:;Huge:--wrap;")
+ expected = dedent(
+ """\
+ \\begin{tabular}{lrrl}
+ & A & B & C \\\\
+ 0 & 0 & \\itshape {\\Huge -0.61} & ab \\\\
+ 1 & \\itshape {\\Huge 1} & -1.22 & \\itshape {\\Huge cd} \\\\
+ \\end{tabular}
+ """
+ )
+ assert expected == styler.to_latex()
+
+
+def test_multiindex_columns(df):
+ cidx = MultiIndex.from_tuples([("A", "a"), ("A", "b"), ("B", "c")])
+ df.columns = cidx
+ expected = dedent(
+ """\
+ \\begin{tabular}{lrrl}
+ & \\multicolumn{2}{r}{A} & B \\\\
+ & a & b & c \\\\
+ 0 & 0 & -0.61 & ab \\\\
+ 1 & 1 & -1.22 & cd \\\\
+ \\end{tabular}
+ """
+ )
+ s = df.style.format(precision=2)
+ assert expected == s.to_latex()
+
+ # non-sparse
+ expected = dedent(
+ """\
+ \\begin{tabular}{lrrl}
+ & A & A & B \\\\
+ & a & b & c \\\\
+ 0 & 0 & -0.61 & ab \\\\
+ 1 & 1 & -1.22 & cd \\\\
+ \\end{tabular}
+ """
+ )
+ s = df.style.format(precision=2)
+ assert expected == s.to_latex(sparse_columns=False)
+
+
+def test_multiindex_row(df_ext):
+ ridx = MultiIndex.from_tuples([("A", "a"), ("A", "b"), ("B", "c")])
+ df_ext.index = ridx
+ expected = dedent(
+ """\
+ \\begin{tabular}{llrrl}
+ & & A & B & C \\\\
+ \\multirow[c]{2}{*}{A} & a & 0 & -0.61 & ab \\\\
+ & b & 1 & -1.22 & cd \\\\
+ B & c & 2 & -2.22 & de \\\\
+ \\end{tabular}
+ """
+ )
+ styler = df_ext.style.format(precision=2)
+ result = styler.to_latex()
+ assert expected == result
+
+ # non-sparse
+ expected = dedent(
+ """\
+ \\begin{tabular}{llrrl}
+ & & A & B & C \\\\
+ A & a & 0 & -0.61 & ab \\\\
+ A & b & 1 & -1.22 & cd \\\\
+ B & c & 2 & -2.22 & de \\\\
+ \\end{tabular}
+ """
+ )
+ result = styler.to_latex(sparse_index=False)
+ assert expected == result
+
+
+def test_multirow_naive(df_ext):
+ ridx = MultiIndex.from_tuples([("X", "x"), ("X", "y"), ("Y", "z")])
+ df_ext.index = ridx
+ expected = dedent(
+ """\
+ \\begin{tabular}{llrrl}
+ & & A & B & C \\\\
+ X & x & 0 & -0.61 & ab \\\\
+ & y & 1 & -1.22 & cd \\\\
+ Y & z & 2 & -2.22 & de \\\\
+ \\end{tabular}
+ """
+ )
+ styler = df_ext.style.format(precision=2)
+ result = styler.to_latex(multirow_align="naive")
+ assert expected == result
+
+
+def test_multiindex_row_and_col(df_ext):
+ cidx = MultiIndex.from_tuples([("Z", "a"), ("Z", "b"), ("Y", "c")])
+ ridx = MultiIndex.from_tuples([("A", "a"), ("A", "b"), ("B", "c")])
+ df_ext.index, df_ext.columns = ridx, cidx
+ expected = dedent(
+ """\
+ \\begin{tabular}{llrrl}
+ & & \\multicolumn{2}{l}{Z} & Y \\\\
+ & & a & b & c \\\\
+ \\multirow[b]{2}{*}{A} & a & 0 & -0.61 & ab \\\\
+ & b & 1 & -1.22 & cd \\\\
+ B & c & 2 & -2.22 & de \\\\
+ \\end{tabular}
+ """
+ )
+ styler = df_ext.style.format(precision=2)
+ result = styler.to_latex(multirow_align="b", multicol_align="l")
+ assert result == expected
+
+ # non-sparse
+ expected = dedent(
+ """\
+ \\begin{tabular}{llrrl}
+ & & Z & Z & Y \\\\
+ & & a & b & c \\\\
+ A & a & 0 & -0.61 & ab \\\\
+ A & b & 1 & -1.22 & cd \\\\
+ B & c & 2 & -2.22 & de \\\\
+ \\end{tabular}
+ """
+ )
+ result = styler.to_latex(sparse_index=False, sparse_columns=False)
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "multicol_align, siunitx, header",
+ [
+ ("naive-l", False, " & A & &"),
+ ("naive-r", False, " & & & A"),
+ ("naive-l", True, "{} & {A} & {} & {}"),
+ ("naive-r", True, "{} & {} & {} & {A}"),
+ ],
+)
+def test_multicol_naive(df, multicol_align, siunitx, header):
+ ridx = MultiIndex.from_tuples([("A", "a"), ("A", "b"), ("A", "c")])
+ df.columns = ridx
+ level1 = " & a & b & c" if not siunitx else "{} & {a} & {b} & {c}"
+ col_format = "lrrl" if not siunitx else "lSSl"
+ expected = dedent(
+ f"""\
+ \\begin{{tabular}}{{{col_format}}}
+ {header} \\\\
+ {level1} \\\\
+ 0 & 0 & -0.61 & ab \\\\
+ 1 & 1 & -1.22 & cd \\\\
+ \\end{{tabular}}
+ """
+ )
+ styler = df.style.format(precision=2)
+ result = styler.to_latex(multicol_align=multicol_align, siunitx=siunitx)
+ assert expected == result
+
+
+def test_multi_options(df_ext):
+ cidx = MultiIndex.from_tuples([("Z", "a"), ("Z", "b"), ("Y", "c")])
+ ridx = MultiIndex.from_tuples([("A", "a"), ("A", "b"), ("B", "c")])
+ df_ext.index, df_ext.columns = ridx, cidx
+ styler = df_ext.style.format(precision=2)
+
+ expected = dedent(
+ """\
+ & & \\multicolumn{2}{r}{Z} & Y \\\\
+ & & a & b & c \\\\
+ \\multirow[c]{2}{*}{A} & a & 0 & -0.61 & ab \\\\
+ """
+ )
+ result = styler.to_latex()
+ assert expected in result
+
+ with option_context("styler.latex.multicol_align", "l"):
+ assert " & & \\multicolumn{2}{l}{Z} & Y \\\\" in styler.to_latex()
+
+ with option_context("styler.latex.multirow_align", "b"):
+ assert "\\multirow[b]{2}{*}{A} & a & 0 & -0.61 & ab \\\\" in styler.to_latex()
+
+
+def test_multiindex_columns_hidden():
+ df = DataFrame([[1, 2, 3, 4]])
+ df.columns = MultiIndex.from_tuples([("A", 1), ("A", 2), ("A", 3), ("B", 1)])
+ s = df.style
+ assert "{tabular}{lrrrr}" in s.to_latex()
+ s.set_table_styles([]) # reset the position command
+ s.hide([("A", 2)], axis="columns")
+ assert "{tabular}{lrrr}" in s.to_latex()
+
+
+@pytest.mark.parametrize(
+ "option, value",
+ [
+ ("styler.sparse.index", True),
+ ("styler.sparse.index", False),
+ ("styler.sparse.columns", True),
+ ("styler.sparse.columns", False),
+ ],
+)
+def test_sparse_options(df_ext, option, value):
+ cidx = MultiIndex.from_tuples([("Z", "a"), ("Z", "b"), ("Y", "c")])
+ ridx = MultiIndex.from_tuples([("A", "a"), ("A", "b"), ("B", "c")])
+ df_ext.index, df_ext.columns = ridx, cidx
+ styler = df_ext.style
+
+ latex1 = styler.to_latex()
+ with option_context(option, value):
+ latex2 = styler.to_latex()
+ assert (latex1 == latex2) is value
+
+
+def test_hidden_index(styler):
+ styler.hide(axis="index")
+ expected = dedent(
+ """\
+ \\begin{tabular}{rrl}
+ A & B & C \\\\
+ 0 & -0.61 & ab \\\\
+ 1 & -1.22 & cd \\\\
+ \\end{tabular}
+ """
+ )
+ assert styler.to_latex() == expected
+
+
+@pytest.mark.parametrize("environment", ["table", "figure*", None])
+def test_comprehensive(df_ext, environment):
+ # test as many low level features simultaneously as possible
+ cidx = MultiIndex.from_tuples([("Z", "a"), ("Z", "b"), ("Y", "c")])
+ ridx = MultiIndex.from_tuples([("A", "a"), ("A", "b"), ("B", "c")])
+ df_ext.index, df_ext.columns = ridx, cidx
+ stlr = df_ext.style
+ stlr.set_caption("mycap")
+ stlr.set_table_styles(
+ [
+ {"selector": "label", "props": ":{fig§item}"},
+ {"selector": "position", "props": ":h!"},
+ {"selector": "position_float", "props": ":centering"},
+ {"selector": "column_format", "props": ":rlrlr"},
+ {"selector": "toprule", "props": ":toprule"},
+ {"selector": "midrule", "props": ":midrule"},
+ {"selector": "bottomrule", "props": ":bottomrule"},
+ {"selector": "rowcolors", "props": ":{3}{pink}{}"}, # custom command
+ ]
+ )
+ stlr.highlight_max(axis=0, props="textbf:--rwrap;cellcolor:[rgb]{1,1,0.6}--rwrap")
+ stlr.highlight_max(axis=None, props="Huge:--wrap;", subset=[("Z", "a"), ("Z", "b")])
+
+ expected = (
+ """\
+\\begin{table}[h!]
+\\centering
+\\caption{mycap}
+\\label{fig:item}
+\\rowcolors{3}{pink}{}
+\\begin{tabular}{rlrlr}
+\\toprule
+ & & \\multicolumn{2}{r}{Z} & Y \\\\
+ & & a & b & c \\\\
+\\midrule
+\\multirow[c]{2}{*}{A} & a & 0 & \\textbf{\\cellcolor[rgb]{1,1,0.6}{-0.61}} & ab \\\\
+ & b & 1 & -1.22 & cd \\\\
+B & c & \\textbf{\\cellcolor[rgb]{1,1,0.6}{{\\Huge 2}}} & -2.22 & """
+ """\
+\\textbf{\\cellcolor[rgb]{1,1,0.6}{de}} \\\\
+\\bottomrule
+\\end{tabular}
+\\end{table}
+"""
+ ).replace("table", environment if environment else "table")
+ result = stlr.format(precision=2).to_latex(environment=environment)
+ assert result == expected
+
+
+def test_environment_option(styler):
+ with option_context("styler.latex.environment", "bar-env"):
+ assert "\\begin{bar-env}" in styler.to_latex()
+ assert "\\begin{foo-env}" in styler.to_latex(environment="foo-env")
+
+
+def test_parse_latex_table_styles(styler):
+ styler.set_table_styles(
+ [
+ {"selector": "foo", "props": [("attr", "value")]},
+ {"selector": "bar", "props": [("attr", "overwritten")]},
+ {"selector": "bar", "props": [("attr", "baz"), ("attr2", "ignored")]},
+ {"selector": "label", "props": [("", "{fig§item}")]},
+ ]
+ )
+ assert _parse_latex_table_styles(styler.table_styles, "bar") == "baz"
+
+ # test '§' replaced by ':' [for CSS compatibility]
+ assert _parse_latex_table_styles(styler.table_styles, "label") == "{fig:item}"
+
+
+def test_parse_latex_cell_styles_basic(): # test nesting
+ cell_style = [("itshape", "--rwrap"), ("cellcolor", "[rgb]{0,1,1}--rwrap")]
+ expected = "\\itshape{\\cellcolor[rgb]{0,1,1}{text}}"
+ assert _parse_latex_cell_styles(cell_style, "text") == expected
+
+
+@pytest.mark.parametrize(
+ "wrap_arg, expected",
+ [ # test wrapping
+ ("", "\\ "),
+ ("--wrap", "{\\ }"),
+ ("--nowrap", "\\ "),
+ ("--lwrap", "{\\} "),
+ ("--dwrap", "{\\}{}"),
+ ("--rwrap", "\\{}"),
+ ],
+)
+def test_parse_latex_cell_styles_braces(wrap_arg, expected):
+ cell_style = [("", f"{wrap_arg}")]
+ assert _parse_latex_cell_styles(cell_style, "") == expected
+
+
+def test_parse_latex_header_span():
+ cell = {"attributes": 'colspan="3"', "display_value": "text", "cellstyle": []}
+ expected = "\\multicolumn{3}{Y}{text}"
+ assert _parse_latex_header_span(cell, "X", "Y") == expected
+
+ cell = {"attributes": 'rowspan="5"', "display_value": "text", "cellstyle": []}
+ expected = "\\multirow[X]{5}{*}{text}"
+ assert _parse_latex_header_span(cell, "X", "Y") == expected
+
+ cell = {"display_value": "text", "cellstyle": []}
+ assert _parse_latex_header_span(cell, "X", "Y") == "text"
+
+ cell = {"display_value": "text", "cellstyle": [("bfseries", "--rwrap")]}
+ assert _parse_latex_header_span(cell, "X", "Y") == "\\bfseries{text}"
+
+
+def test_parse_latex_table_wrapping(styler):
+ styler.set_table_styles(
+ [
+ {"selector": "toprule", "props": ":value"},
+ {"selector": "bottomrule", "props": ":value"},
+ {"selector": "midrule", "props": ":value"},
+ {"selector": "column_format", "props": ":value"},
+ ]
+ )
+ assert _parse_latex_table_wrapping(styler.table_styles, styler.caption) is False
+ assert _parse_latex_table_wrapping(styler.table_styles, "some caption") is True
+ styler.set_table_styles(
+ [
+ {"selector": "not-ignored", "props": ":value"},
+ ],
+ overwrite=False,
+ )
+ assert _parse_latex_table_wrapping(styler.table_styles, None) is True
+
+
+def test_short_caption(styler):
+ result = styler.to_latex(caption=("full cap", "short cap"))
+ assert "\\caption[short cap]{full cap}" in result
+
+
+@pytest.mark.parametrize(
+ "css, expected",
+ [
+ ([("color", "red")], [("color", "{red}")]), # test color and input format types
+ (
+ [("color", "rgb(128, 128, 128 )")],
+ [("color", "[rgb]{0.502, 0.502, 0.502}")],
+ ),
+ (
+ [("color", "rgb(128, 50%, 25% )")],
+ [("color", "[rgb]{0.502, 0.500, 0.250}")],
+ ),
+ (
+ [("color", "rgba(128,128,128,1)")],
+ [("color", "[rgb]{0.502, 0.502, 0.502}")],
+ ),
+ ([("color", "#FF00FF")], [("color", "[HTML]{FF00FF}")]),
+ ([("color", "#F0F")], [("color", "[HTML]{FF00FF}")]),
+ ([("font-weight", "bold")], [("bfseries", "")]), # test font-weight and types
+ ([("font-weight", "bolder")], [("bfseries", "")]),
+ ([("font-weight", "normal")], []),
+ ([("background-color", "red")], [("cellcolor", "{red}--lwrap")]),
+ (
+ [("background-color", "#FF00FF")], # test background-color command and wrap
+ [("cellcolor", "[HTML]{FF00FF}--lwrap")],
+ ),
+ ([("font-style", "italic")], [("itshape", "")]), # test font-style and types
+ ([("font-style", "oblique")], [("slshape", "")]),
+ ([("font-style", "normal")], []),
+ ([("color", "red /*--dwrap*/")], [("color", "{red}--dwrap")]), # css comments
+ ([("background-color", "red /* --dwrap */")], [("cellcolor", "{red}--dwrap")]),
+ ],
+)
+def test_parse_latex_css_conversion(css, expected):
+ result = _parse_latex_css_conversion(css)
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "env, inner_env",
+ [
+ (None, "tabular"),
+ ("table", "tabular"),
+ ("longtable", "longtable"),
+ ],
+)
+@pytest.mark.parametrize(
+ "convert, exp", [(True, "bfseries"), (False, "font-weightbold")]
+)
+def test_parse_latex_css_convert_minimal(styler, env, inner_env, convert, exp):
+ # parameters ensure longtable template is also tested
+ styler.highlight_max(props="font-weight:bold;")
+ result = styler.to_latex(convert_css=convert, environment=env)
+ expected = dedent(
+ f"""\
+ 0 & 0 & \\{exp} -0.61 & ab \\\\
+ 1 & \\{exp} 1 & -1.22 & \\{exp} cd \\\\
+ \\end{{{inner_env}}}
+ """
+ )
+ assert expected in result
+
+
+def test_parse_latex_css_conversion_option():
+ css = [("command", "option--latex--wrap")]
+ expected = [("command", "option--wrap")]
+ result = _parse_latex_css_conversion(css)
+ assert result == expected
+
+
+def test_styler_object_after_render(styler):
+ # GH 42320
+ pre_render = styler._copy(deepcopy=True)
+ styler.to_latex(
+ column_format="rllr",
+ position="h",
+ position_float="centering",
+ hrules=True,
+ label="my lab",
+ caption="my cap",
+ )
+
+ assert pre_render.table_styles == styler.table_styles
+ assert pre_render.caption == styler.caption
+
+
+def test_longtable_comprehensive(styler):
+ result = styler.to_latex(
+ environment="longtable", hrules=True, label="fig:A", caption=("full", "short")
+ )
+ expected = dedent(
+ """\
+ \\begin{longtable}{lrrl}
+ \\caption[short]{full} \\label{fig:A} \\\\
+ \\toprule
+ & A & B & C \\\\
+ \\midrule
+ \\endfirsthead
+ \\caption[]{full} \\\\
+ \\toprule
+ & A & B & C \\\\
+ \\midrule
+ \\endhead
+ \\midrule
+ \\multicolumn{4}{r}{Continued on next page} \\\\
+ \\midrule
+ \\endfoot
+ \\bottomrule
+ \\endlastfoot
+ 0 & 0 & -0.61 & ab \\\\
+ 1 & 1 & -1.22 & cd \\\\
+ \\end{longtable}
+ """
+ )
+ assert result == expected
+
+
+def test_longtable_minimal(styler):
+ result = styler.to_latex(environment="longtable")
+ expected = dedent(
+ """\
+ \\begin{longtable}{lrrl}
+ & A & B & C \\\\
+ \\endfirsthead
+ & A & B & C \\\\
+ \\endhead
+ \\multicolumn{4}{r}{Continued on next page} \\\\
+ \\endfoot
+ \\endlastfoot
+ 0 & 0 & -0.61 & ab \\\\
+ 1 & 1 & -1.22 & cd \\\\
+ \\end{longtable}
+ """
+ )
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "sparse, exp, siunitx",
+ [
+ (True, "{} & \\multicolumn{2}{r}{A} & {B}", True),
+ (False, "{} & {A} & {A} & {B}", True),
+ (True, " & \\multicolumn{2}{r}{A} & B", False),
+ (False, " & A & A & B", False),
+ ],
+)
+def test_longtable_multiindex_columns(df, sparse, exp, siunitx):
+ cidx = MultiIndex.from_tuples([("A", "a"), ("A", "b"), ("B", "c")])
+ df.columns = cidx
+ with_si = "{} & {a} & {b} & {c} \\\\"
+ without_si = " & a & b & c \\\\"
+ expected = dedent(
+ f"""\
+ \\begin{{longtable}}{{l{"SS" if siunitx else "rr"}l}}
+ {exp} \\\\
+ {with_si if siunitx else without_si}
+ \\endfirsthead
+ {exp} \\\\
+ {with_si if siunitx else without_si}
+ \\endhead
+ """
+ )
+ result = df.style.to_latex(
+ environment="longtable", sparse_columns=sparse, siunitx=siunitx
+ )
+ assert expected in result
+
+
+@pytest.mark.parametrize(
+ "caption, cap_exp",
+ [
+ ("full", ("{full}", "")),
+ (("full", "short"), ("{full}", "[short]")),
+ ],
+)
+@pytest.mark.parametrize("label, lab_exp", [(None, ""), ("tab:A", " \\label{tab:A}")])
+def test_longtable_caption_label(styler, caption, cap_exp, label, lab_exp):
+ cap_exp1 = f"\\caption{cap_exp[1]}{cap_exp[0]}"
+ cap_exp2 = f"\\caption[]{cap_exp[0]}"
+
+ expected = dedent(
+ f"""\
+ {cap_exp1}{lab_exp} \\\\
+ & A & B & C \\\\
+ \\endfirsthead
+ {cap_exp2} \\\\
+ """
+ )
+ assert expected in styler.to_latex(
+ environment="longtable", caption=caption, label=label
+ )
+
+
+@pytest.mark.parametrize("index", [True, False])
+@pytest.mark.parametrize(
+ "columns, siunitx",
+ [
+ (True, True),
+ (True, False),
+ (False, False),
+ ],
+)
+def test_apply_map_header_render_mi(df_ext, index, columns, siunitx):
+ cidx = MultiIndex.from_tuples([("Z", "a"), ("Z", "b"), ("Y", "c")])
+ ridx = MultiIndex.from_tuples([("A", "a"), ("A", "b"), ("B", "c")])
+ df_ext.index, df_ext.columns = ridx, cidx
+ styler = df_ext.style
+
+ func = lambda v: "bfseries: --rwrap" if "A" in v or "Z" in v or "c" in v else None
+
+ if index:
+ styler.map_index(func, axis="index")
+ if columns:
+ styler.map_index(func, axis="columns")
+
+ result = styler.to_latex(siunitx=siunitx)
+
+ expected_index = dedent(
+ """\
+ \\multirow[c]{2}{*}{\\bfseries{A}} & a & 0 & -0.610000 & ab \\\\
+ \\bfseries{} & b & 1 & -1.220000 & cd \\\\
+ B & \\bfseries{c} & 2 & -2.220000 & de \\\\
+ """
+ )
+ assert (expected_index in result) is index
+
+ exp_cols_si = dedent(
+ """\
+ {} & {} & \\multicolumn{2}{r}{\\bfseries{Z}} & {Y} \\\\
+ {} & {} & {a} & {b} & {\\bfseries{c}} \\\\
+ """
+ )
+ exp_cols_no_si = """\
+ & & \\multicolumn{2}{r}{\\bfseries{Z}} & Y \\\\
+ & & a & b & \\bfseries{c} \\\\
+"""
+ assert ((exp_cols_si if siunitx else exp_cols_no_si) in result) is columns
+
+
+def test_repr_option(styler):
+ assert "' in result
+ assert ' ' not in result
+ assert 'title="' not in result
+
+
+def test_tooltip_css_class(styler):
+ # GH 21266
+ result = styler.set_tooltips(
+ DataFrame([["tooltip"]], index=["x"], columns=["A"]),
+ css_class="other-class",
+ props=[("color", "green")],
+ ).to_html()
+ assert "#T_ .other-class {\n color: green;\n" in result
+ assert '#T_ #T__row0_col0 .other-class::after {\n content: "tooltip";\n' in result
+
+ # GH 39563
+ result = styler.set_tooltips( # set_tooltips overwrites previous
+ DataFrame([["tooltip"]], index=["x"], columns=["A"]),
+ css_class="another-class",
+ props="color:green;color:red;",
+ ).to_html()
+ assert "#T_ .another-class {\n color: green;\n color: red;\n}" in result
+
+
+@pytest.mark.parametrize(
+ "data, columns, index",
+ [
+ # Test basic reindex and ignoring blank
+ ([["Min", "Max"], [np.nan, ""]], ["A", "C"], ["x", "y"]),
+ # Test non-referenced columns, reversed col names, short index
+ ([["Max", "Min", "Bad-Col"]], ["C", "A", "D"], ["x"]),
+ ],
+)
+def test_tooltip_render_as_title(data, columns, index, styler):
+ ttips = DataFrame(data=data, columns=columns, index=index)
+ # GH 56605
+ result = styler.set_tooltips(ttips, as_title_attribute=True).to_html()
+
+ # test css not added
+ assert "#T_ .pd-t {\n visibility: hidden;\n" not in result
+
+ # test 'Min' tooltip added as title attribute and css does not exist
+ assert "#T_ #T__row0_col0:hover .pd-t {\n visibility: visible;\n}" not in result
+ assert '#T_ #T__row0_col0 .pd-t::after {\n content: "Min";\n}' not in result
+ assert 'class="data row0 col0" title="Min">0' in result
+
+ # test 'Max' tooltip added as title attribute and css does not exist
+ assert "#T_ #T__row0_col2:hover .pd-t {\n visibility: visible;\n}" not in result
+ assert '#T_ #T__row0_col2 .pd-t::after {\n content: "Max";\n}' not in result
+ assert 'class="data row0 col2" title="Max">2' in result
+
+ # test Nan, empty string and bad column ignored
+ assert "#T_ #T__row1_col0:hover .pd-t {\n visibility: visible;\n}" not in result
+ assert "#T_ #T__row1_col1:hover .pd-t {\n visibility: visible;\n}" not in result
+ assert "#T_ #T__row0_col1:hover .pd-t {\n visibility: visible;\n}" not in result
+ assert "#T_ #T__row1_col2:hover .pd-t {\n visibility: visible;\n}" not in result
+ assert "Bad-Col" not in result
+ assert 'class="data row0 col1" >1' in result
+ assert 'class="data row1 col0" >3' in result
+ assert 'class="data row1 col1" >4' in result
+ assert 'class="data row1 col2" >5' in result
+ assert 'class="data row2 col0" >6' in result
+ assert 'class="data row2 col1" >7' in result
+ assert 'class="data row2 col2" >8' in result
+
+
+def test_tooltip_render_as_title_with_hidden_index_level():
+ df = DataFrame(
+ data=[[0, 1, 2], [3, 4, 5], [6, 7, 8]],
+ columns=["A", "B", "C"],
+ index=MultiIndex.from_arrays(
+ [["x", "y", "z"], [1, 2, 3], ["aa", "bb", "cc"]],
+ names=["alpha", "num", "char"],
+ ),
+ )
+ ttips = DataFrame(
+ # Test basic reindex and ignoring blank, and hide level 2 (num) from index
+ data=[["Min", "Max"], [np.nan, ""]],
+ columns=["A", "C"],
+ index=MultiIndex.from_arrays(
+ [["x", "y"], [1, 2], ["aa", "bb"]], names=["alpha", "num", "char"]
+ ),
+ )
+ styler = Styler(df, uuid_len=0)
+ styler = styler.hide(axis=0, level=-1, names=True)
+ # GH 56605
+ result = styler.set_tooltips(ttips, as_title_attribute=True).to_html()
+
+ # test css not added
+ assert "#T_ .pd-t {\n visibility: hidden;\n" not in result
+
+ # test 'Min' tooltip added as title attribute and css does not exist
+ assert "#T_ #T__row0_col0:hover .pd-t {\n visibility: visible;\n}" not in result
+ assert '#T_ #T__row0_col0 .pd-t::after {\n content: "Min";\n}' not in result
+ assert 'class="data row0 col0" title="Min">0' in result
+
+ # test 'Max' tooltip added as title attribute and css does not exist
+ assert "#T_ #T__row0_col2:hover .pd-t {\n visibility: visible;\n}" not in result
+ assert '#T_ #T__row0_col2 .pd-t::after {\n content: "Max";\n}' not in result
+ assert 'class="data row0 col2" title="Max">2' in result
+
+ # test Nan, empty string and bad column ignored
+ assert "#T_ #T__row1_col0:hover .pd-t {\n visibility: visible;\n}" not in result
+ assert "#T_ #T__row1_col1:hover .pd-t {\n visibility: visible;\n}" not in result
+ assert "#T_ #T__row0_col1:hover .pd-t {\n visibility: visible;\n}" not in result
+ assert "#T_ #T__row1_col2:hover .pd-t {\n visibility: visible;\n}" not in result
+ assert "Bad-Col" not in result
+ assert 'class="data row0 col1" >1' in result
+ assert 'class="data row1 col0" >3' in result
+ assert 'class="data row1 col1" >4' in result
+ assert 'class="data row1 col2" >5' in result
+ assert 'class="data row2 col0" >6' in result
+ assert 'class="data row2 col1" >7' in result
+ assert 'class="data row2 col2" >8' in result
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_console.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_console.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd7b57df9baed18b172dc8398a61a49e9435f82a
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_console.py
@@ -0,0 +1,72 @@
+import locale
+
+import pytest
+
+from pandas._config import detect_console_encoding
+
+
+class MockEncoding:
+ """
+ Used to add a side effect when accessing the 'encoding' property. If the
+ side effect is a str in nature, the value will be returned. Otherwise, the
+ side effect should be an exception that will be raised.
+ """
+
+ def __init__(self, encoding) -> None:
+ super().__init__()
+ self.val = encoding
+
+ @property
+ def encoding(self):
+ return self.raise_or_return(self.val)
+
+ @staticmethod
+ def raise_or_return(val):
+ if isinstance(val, str):
+ return val
+ else:
+ raise val
+
+
+@pytest.mark.parametrize("empty,filled", [["stdin", "stdout"], ["stdout", "stdin"]])
+def test_detect_console_encoding_from_stdout_stdin(monkeypatch, empty, filled):
+ # Ensures that when sys.stdout.encoding or sys.stdin.encoding is used when
+ # they have values filled.
+ # GH 21552
+ with monkeypatch.context() as context:
+ context.setattr(f"sys.{empty}", MockEncoding(""))
+ context.setattr(f"sys.{filled}", MockEncoding(filled))
+ assert detect_console_encoding() == filled
+
+
+@pytest.mark.parametrize("encoding", [AttributeError, OSError, "ascii"])
+def test_detect_console_encoding_fallback_to_locale(monkeypatch, encoding):
+ # GH 21552
+ with monkeypatch.context() as context:
+ context.setattr("locale.getpreferredencoding", lambda: "foo")
+ context.setattr("sys.stdout", MockEncoding(encoding))
+ assert detect_console_encoding() == "foo"
+
+
+@pytest.mark.parametrize(
+ "std,locale",
+ [
+ ["ascii", "ascii"],
+ ["ascii", locale.Error],
+ [AttributeError, "ascii"],
+ [AttributeError, locale.Error],
+ [OSError, "ascii"],
+ [OSError, locale.Error],
+ ],
+)
+def test_detect_console_encoding_fallback_to_default(monkeypatch, std, locale):
+ # When both the stdout/stdin encoding and locale preferred encoding checks
+ # fail (or return 'ascii', we should default to the sys default encoding.
+ # GH 21552
+ with monkeypatch.context() as context:
+ context.setattr(
+ "locale.getpreferredencoding", lambda: MockEncoding.raise_or_return(locale)
+ )
+ context.setattr("sys.stdout", MockEncoding(std))
+ context.setattr("sys.getdefaultencoding", lambda: "sysDefaultEncoding")
+ assert detect_console_encoding() == "sysDefaultEncoding"
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_css.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_css.py
new file mode 100644
index 0000000000000000000000000000000000000000..642a56270434424ac2347cff55bb37eaf4608e50
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_css.py
@@ -0,0 +1,288 @@
+import pytest
+
+from pandas.errors import CSSWarning
+
+import pandas._testing as tm
+
+from pandas.io.formats.css import CSSResolver
+
+
+def assert_resolves(css, props, inherited=None):
+ resolve = CSSResolver()
+ actual = resolve(css, inherited=inherited)
+ assert props == actual
+
+
+def assert_same_resolution(css1, css2, inherited=None):
+ resolve = CSSResolver()
+ resolved1 = resolve(css1, inherited=inherited)
+ resolved2 = resolve(css2, inherited=inherited)
+ assert resolved1 == resolved2
+
+
+@pytest.mark.parametrize(
+ "name,norm,abnorm",
+ [
+ (
+ "whitespace",
+ "hello: world; foo: bar",
+ " \t hello \t :\n world \n ; \n foo: \tbar\n\n",
+ ),
+ ("case", "hello: world; foo: bar", "Hello: WORLD; foO: bar"),
+ ("empty-decl", "hello: world; foo: bar", "; hello: world;; foo: bar;\n; ;"),
+ ("empty-list", "", ";"),
+ ],
+)
+def test_css_parse_normalisation(name, norm, abnorm):
+ assert_same_resolution(norm, abnorm)
+
+
+@pytest.mark.parametrize(
+ "invalid_css,remainder,msg",
+ [
+ # No colon
+ ("hello-world", "", "expected a colon"),
+ ("border-style: solid; hello-world", "border-style: solid", "expected a colon"),
+ (
+ "border-style: solid; hello-world; font-weight: bold",
+ "border-style: solid; font-weight: bold",
+ "expected a colon",
+ ),
+ # Unclosed string fail
+ # Invalid size
+ ("font-size: blah", "font-size: 1em", "Unhandled size"),
+ ("font-size: 1a2b", "font-size: 1em", "Unhandled size"),
+ ("font-size: 1e5pt", "font-size: 1em", "Unhandled size"),
+ ("font-size: 1+6pt", "font-size: 1em", "Unhandled size"),
+ ("font-size: 1unknownunit", "font-size: 1em", "Unhandled size"),
+ ("font-size: 10", "font-size: 1em", "Unhandled size"),
+ ("font-size: 10 pt", "font-size: 1em", "Unhandled size"),
+ # Too many args
+ ("border-top: 1pt solid red green", "border-top: 1pt solid green", "Too many"),
+ ],
+)
+def test_css_parse_invalid(invalid_css, remainder, msg):
+ with tm.assert_produces_warning(CSSWarning, match=msg):
+ assert_same_resolution(invalid_css, remainder)
+
+
+@pytest.mark.parametrize(
+ "shorthand,expansions",
+ [
+ ("margin", ["margin-top", "margin-right", "margin-bottom", "margin-left"]),
+ ("padding", ["padding-top", "padding-right", "padding-bottom", "padding-left"]),
+ (
+ "border-width",
+ [
+ "border-top-width",
+ "border-right-width",
+ "border-bottom-width",
+ "border-left-width",
+ ],
+ ),
+ (
+ "border-color",
+ [
+ "border-top-color",
+ "border-right-color",
+ "border-bottom-color",
+ "border-left-color",
+ ],
+ ),
+ (
+ "border-style",
+ [
+ "border-top-style",
+ "border-right-style",
+ "border-bottom-style",
+ "border-left-style",
+ ],
+ ),
+ ],
+)
+def test_css_side_shorthands(shorthand, expansions):
+ top, right, bottom, left = expansions
+
+ assert_resolves(
+ f"{shorthand}: 1pt", {top: "1pt", right: "1pt", bottom: "1pt", left: "1pt"}
+ )
+
+ assert_resolves(
+ f"{shorthand}: 1pt 4pt", {top: "1pt", right: "4pt", bottom: "1pt", left: "4pt"}
+ )
+
+ assert_resolves(
+ f"{shorthand}: 1pt 4pt 2pt",
+ {top: "1pt", right: "4pt", bottom: "2pt", left: "4pt"},
+ )
+
+ assert_resolves(
+ f"{shorthand}: 1pt 4pt 2pt 0pt",
+ {top: "1pt", right: "4pt", bottom: "2pt", left: "0pt"},
+ )
+
+ with tm.assert_produces_warning(CSSWarning, match="Could not expand"):
+ assert_resolves(f"{shorthand}: 1pt 1pt 1pt 1pt 1pt", {})
+
+
+@pytest.mark.parametrize(
+ "shorthand,sides",
+ [
+ ("border-top", ["top"]),
+ ("border-right", ["right"]),
+ ("border-bottom", ["bottom"]),
+ ("border-left", ["left"]),
+ ("border", ["top", "right", "bottom", "left"]),
+ ],
+)
+def test_css_border_shorthand_sides(shorthand, sides):
+ def create_border_dict(sides, color=None, style=None, width=None):
+ resolved = {}
+ for side in sides:
+ if color:
+ resolved[f"border-{side}-color"] = color
+ if style:
+ resolved[f"border-{side}-style"] = style
+ if width:
+ resolved[f"border-{side}-width"] = width
+ return resolved
+
+ assert_resolves(
+ f"{shorthand}: 1pt red solid", create_border_dict(sides, "red", "solid", "1pt")
+ )
+
+
+@pytest.mark.parametrize(
+ "prop, expected",
+ [
+ ("1pt red solid", ("red", "solid", "1pt")),
+ ("red 1pt solid", ("red", "solid", "1pt")),
+ ("red solid 1pt", ("red", "solid", "1pt")),
+ ("solid 1pt red", ("red", "solid", "1pt")),
+ ("red solid", ("red", "solid", "1.500000pt")),
+ # Note: color=black is not CSS conforming
+ # (See https://drafts.csswg.org/css-backgrounds/#border-shorthands)
+ ("1pt solid", ("black", "solid", "1pt")),
+ ("1pt red", ("red", "none", "1pt")),
+ ("red", ("red", "none", "1.500000pt")),
+ ("1pt", ("black", "none", "1pt")),
+ ("solid", ("black", "solid", "1.500000pt")),
+ # Sizes
+ ("1em", ("black", "none", "12pt")),
+ ],
+)
+def test_css_border_shorthands(prop, expected):
+ color, style, width = expected
+
+ assert_resolves(
+ f"border-left: {prop}",
+ {
+ "border-left-color": color,
+ "border-left-style": style,
+ "border-left-width": width,
+ },
+ )
+
+
+@pytest.mark.parametrize(
+ "style,inherited,equiv",
+ [
+ ("margin: 1px; margin: 2px", "", "margin: 2px"),
+ ("margin: 1px", "margin: 2px", "margin: 1px"),
+ ("margin: 1px; margin: inherit", "margin: 2px", "margin: 2px"),
+ (
+ "margin: 1px; margin-top: 2px",
+ "",
+ "margin-left: 1px; margin-right: 1px; margin-bottom: 1px; margin-top: 2px",
+ ),
+ ("margin-top: 2px", "margin: 1px", "margin: 1px; margin-top: 2px"),
+ ("margin: 1px", "margin-top: 2px", "margin: 1px"),
+ (
+ "margin: 1px; margin-top: inherit",
+ "margin: 2px",
+ "margin: 1px; margin-top: 2px",
+ ),
+ ],
+)
+def test_css_precedence(style, inherited, equiv):
+ resolve = CSSResolver()
+ inherited_props = resolve(inherited)
+ style_props = resolve(style, inherited=inherited_props)
+ equiv_props = resolve(equiv)
+ assert style_props == equiv_props
+
+
+@pytest.mark.parametrize(
+ "style,equiv",
+ [
+ (
+ "margin: 1px; margin-top: inherit",
+ "margin-bottom: 1px; margin-right: 1px; margin-left: 1px",
+ ),
+ ("margin-top: inherit", ""),
+ ("margin-top: initial", ""),
+ ],
+)
+def test_css_none_absent(style, equiv):
+ assert_same_resolution(style, equiv)
+
+
+@pytest.mark.parametrize(
+ "size,resolved",
+ [
+ ("xx-small", "6pt"),
+ ("x-small", f"{7.5:f}pt"),
+ ("small", f"{9.6:f}pt"),
+ ("medium", "12pt"),
+ ("large", f"{13.5:f}pt"),
+ ("x-large", "18pt"),
+ ("xx-large", "24pt"),
+ ("8px", "6pt"),
+ ("1.25pc", "15pt"),
+ (".25in", "18pt"),
+ ("02.54cm", "72pt"),
+ ("25.4mm", "72pt"),
+ ("101.6q", "72pt"),
+ ],
+)
+@pytest.mark.parametrize("relative_to", [None, "16pt"]) # invariant to inherited size
+def test_css_absolute_font_size(size, relative_to, resolved):
+ if relative_to is None:
+ inherited = None
+ else:
+ inherited = {"font-size": relative_to}
+ assert_resolves(f"font-size: {size}", {"font-size": resolved}, inherited=inherited)
+
+
+@pytest.mark.parametrize(
+ "size,relative_to,resolved",
+ [
+ ("1em", None, "12pt"),
+ ("1.0em", None, "12pt"),
+ ("1.25em", None, "15pt"),
+ ("1em", "16pt", "16pt"),
+ ("1.0em", "16pt", "16pt"),
+ ("1.25em", "16pt", "20pt"),
+ ("1rem", "16pt", "12pt"),
+ ("1.0rem", "16pt", "12pt"),
+ ("1.25rem", "16pt", "15pt"),
+ ("100%", None, "12pt"),
+ ("125%", None, "15pt"),
+ ("100%", "16pt", "16pt"),
+ ("125%", "16pt", "20pt"),
+ ("2ex", None, "12pt"),
+ ("2.0ex", None, "12pt"),
+ ("2.50ex", None, "15pt"),
+ ("inherit", "16pt", "16pt"),
+ ("smaller", None, "10pt"),
+ ("smaller", "18pt", "15pt"),
+ ("larger", None, f"{14.4:f}pt"),
+ ("larger", "15pt", "18pt"),
+ ],
+)
+def test_css_relative_font_size(size, relative_to, resolved):
+ if relative_to is None:
+ inherited = None
+ else:
+ inherited = {"font-size": relative_to}
+ assert_resolves(f"font-size: {size}", {"font-size": resolved}, inherited=inherited)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_eng_formatting.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_eng_formatting.py
new file mode 100644
index 0000000000000000000000000000000000000000..6d581b5b92e0c8cbcfe21dbbdfb0f99ca05c1a4e
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_eng_formatting.py
@@ -0,0 +1,254 @@
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ reset_option,
+ set_eng_float_format,
+)
+
+from pandas.io.formats.format import EngFormatter
+
+
+@pytest.fixture(autouse=True)
+def reset_float_format():
+ yield
+ reset_option("display.float_format")
+
+
+class TestEngFormatter:
+ def test_eng_float_formatter2(self, float_frame):
+ df = float_frame
+ df.loc[5] = 0
+
+ set_eng_float_format()
+ repr(df)
+
+ set_eng_float_format(use_eng_prefix=True)
+ repr(df)
+
+ set_eng_float_format(accuracy=0)
+ repr(df)
+
+ def test_eng_float_formatter(self):
+ df = DataFrame({"A": [1.41, 141.0, 14100, 1410000.0]})
+
+ set_eng_float_format()
+ result = df.to_string()
+ expected = (
+ " A\n"
+ "0 1.410E+00\n"
+ "1 141.000E+00\n"
+ "2 14.100E+03\n"
+ "3 1.410E+06"
+ )
+ assert result == expected
+
+ set_eng_float_format(use_eng_prefix=True)
+ result = df.to_string()
+ expected = " A\n0 1.410\n1 141.000\n2 14.100k\n3 1.410M"
+ assert result == expected
+
+ set_eng_float_format(accuracy=0)
+ result = df.to_string()
+ expected = " A\n0 1E+00\n1 141E+00\n2 14E+03\n3 1E+06"
+ assert result == expected
+
+ def compare(self, formatter, input, output):
+ formatted_input = formatter(input)
+ assert formatted_input == output
+
+ def compare_all(self, formatter, in_out):
+ """
+ Parameters:
+ -----------
+ formatter: EngFormatter under test
+ in_out: list of tuples. Each tuple = (number, expected_formatting)
+
+ It is tested if 'formatter(number) == expected_formatting'.
+ *number* should be >= 0 because formatter(-number) == fmt is also
+ tested. *fmt* is derived from *expected_formatting*
+ """
+ for input, output in in_out:
+ self.compare(formatter, input, output)
+ self.compare(formatter, -input, "-" + output[1:])
+
+ def test_exponents_with_eng_prefix(self):
+ formatter = EngFormatter(accuracy=3, use_eng_prefix=True)
+ f = np.sqrt(2)
+ in_out = [
+ (f * 10**-24, " 1.414y"),
+ (f * 10**-23, " 14.142y"),
+ (f * 10**-22, " 141.421y"),
+ (f * 10**-21, " 1.414z"),
+ (f * 10**-20, " 14.142z"),
+ (f * 10**-19, " 141.421z"),
+ (f * 10**-18, " 1.414a"),
+ (f * 10**-17, " 14.142a"),
+ (f * 10**-16, " 141.421a"),
+ (f * 10**-15, " 1.414f"),
+ (f * 10**-14, " 14.142f"),
+ (f * 10**-13, " 141.421f"),
+ (f * 10**-12, " 1.414p"),
+ (f * 10**-11, " 14.142p"),
+ (f * 10**-10, " 141.421p"),
+ (f * 10**-9, " 1.414n"),
+ (f * 10**-8, " 14.142n"),
+ (f * 10**-7, " 141.421n"),
+ (f * 10**-6, " 1.414u"),
+ (f * 10**-5, " 14.142u"),
+ (f * 10**-4, " 141.421u"),
+ (f * 10**-3, " 1.414m"),
+ (f * 10**-2, " 14.142m"),
+ (f * 10**-1, " 141.421m"),
+ (f * 10**0, " 1.414"),
+ (f * 10**1, " 14.142"),
+ (f * 10**2, " 141.421"),
+ (f * 10**3, " 1.414k"),
+ (f * 10**4, " 14.142k"),
+ (f * 10**5, " 141.421k"),
+ (f * 10**6, " 1.414M"),
+ (f * 10**7, " 14.142M"),
+ (f * 10**8, " 141.421M"),
+ (f * 10**9, " 1.414G"),
+ (f * 10**10, " 14.142G"),
+ (f * 10**11, " 141.421G"),
+ (f * 10**12, " 1.414T"),
+ (f * 10**13, " 14.142T"),
+ (f * 10**14, " 141.421T"),
+ (f * 10**15, " 1.414P"),
+ (f * 10**16, " 14.142P"),
+ (f * 10**17, " 141.421P"),
+ (f * 10**18, " 1.414E"),
+ (f * 10**19, " 14.142E"),
+ (f * 10**20, " 141.421E"),
+ (f * 10**21, " 1.414Z"),
+ (f * 10**22, " 14.142Z"),
+ (f * 10**23, " 141.421Z"),
+ (f * 10**24, " 1.414Y"),
+ (f * 10**25, " 14.142Y"),
+ (f * 10**26, " 141.421Y"),
+ ]
+ self.compare_all(formatter, in_out)
+
+ def test_exponents_without_eng_prefix(self):
+ formatter = EngFormatter(accuracy=4, use_eng_prefix=False)
+ f = np.pi
+ in_out = [
+ (f * 10**-24, " 3.1416E-24"),
+ (f * 10**-23, " 31.4159E-24"),
+ (f * 10**-22, " 314.1593E-24"),
+ (f * 10**-21, " 3.1416E-21"),
+ (f * 10**-20, " 31.4159E-21"),
+ (f * 10**-19, " 314.1593E-21"),
+ (f * 10**-18, " 3.1416E-18"),
+ (f * 10**-17, " 31.4159E-18"),
+ (f * 10**-16, " 314.1593E-18"),
+ (f * 10**-15, " 3.1416E-15"),
+ (f * 10**-14, " 31.4159E-15"),
+ (f * 10**-13, " 314.1593E-15"),
+ (f * 10**-12, " 3.1416E-12"),
+ (f * 10**-11, " 31.4159E-12"),
+ (f * 10**-10, " 314.1593E-12"),
+ (f * 10**-9, " 3.1416E-09"),
+ (f * 10**-8, " 31.4159E-09"),
+ (f * 10**-7, " 314.1593E-09"),
+ (f * 10**-6, " 3.1416E-06"),
+ (f * 10**-5, " 31.4159E-06"),
+ (f * 10**-4, " 314.1593E-06"),
+ (f * 10**-3, " 3.1416E-03"),
+ (f * 10**-2, " 31.4159E-03"),
+ (f * 10**-1, " 314.1593E-03"),
+ (f * 10**0, " 3.1416E+00"),
+ (f * 10**1, " 31.4159E+00"),
+ (f * 10**2, " 314.1593E+00"),
+ (f * 10**3, " 3.1416E+03"),
+ (f * 10**4, " 31.4159E+03"),
+ (f * 10**5, " 314.1593E+03"),
+ (f * 10**6, " 3.1416E+06"),
+ (f * 10**7, " 31.4159E+06"),
+ (f * 10**8, " 314.1593E+06"),
+ (f * 10**9, " 3.1416E+09"),
+ (f * 10**10, " 31.4159E+09"),
+ (f * 10**11, " 314.1593E+09"),
+ (f * 10**12, " 3.1416E+12"),
+ (f * 10**13, " 31.4159E+12"),
+ (f * 10**14, " 314.1593E+12"),
+ (f * 10**15, " 3.1416E+15"),
+ (f * 10**16, " 31.4159E+15"),
+ (f * 10**17, " 314.1593E+15"),
+ (f * 10**18, " 3.1416E+18"),
+ (f * 10**19, " 31.4159E+18"),
+ (f * 10**20, " 314.1593E+18"),
+ (f * 10**21, " 3.1416E+21"),
+ (f * 10**22, " 31.4159E+21"),
+ (f * 10**23, " 314.1593E+21"),
+ (f * 10**24, " 3.1416E+24"),
+ (f * 10**25, " 31.4159E+24"),
+ (f * 10**26, " 314.1593E+24"),
+ ]
+ self.compare_all(formatter, in_out)
+
+ def test_rounding(self):
+ formatter = EngFormatter(accuracy=3, use_eng_prefix=True)
+ in_out = [
+ (5.55555, " 5.556"),
+ (55.5555, " 55.556"),
+ (555.555, " 555.555"),
+ (5555.55, " 5.556k"),
+ (55555.5, " 55.556k"),
+ (555555, " 555.555k"),
+ ]
+ self.compare_all(formatter, in_out)
+
+ formatter = EngFormatter(accuracy=1, use_eng_prefix=True)
+ in_out = [
+ (5.55555, " 5.6"),
+ (55.5555, " 55.6"),
+ (555.555, " 555.6"),
+ (5555.55, " 5.6k"),
+ (55555.5, " 55.6k"),
+ (555555, " 555.6k"),
+ ]
+ self.compare_all(formatter, in_out)
+
+ formatter = EngFormatter(accuracy=0, use_eng_prefix=True)
+ in_out = [
+ (5.55555, " 6"),
+ (55.5555, " 56"),
+ (555.555, " 556"),
+ (5555.55, " 6k"),
+ (55555.5, " 56k"),
+ (555555, " 556k"),
+ ]
+ self.compare_all(formatter, in_out)
+
+ formatter = EngFormatter(accuracy=3, use_eng_prefix=True)
+ result = formatter(0)
+ assert result == " 0.000"
+
+ def test_nan(self):
+ # Issue #11981
+
+ formatter = EngFormatter(accuracy=1, use_eng_prefix=True)
+ result = formatter(np.nan)
+ assert result == "NaN"
+
+ df = DataFrame(
+ {
+ "a": [1.5, 10.3, 20.5],
+ "b": [50.3, 60.67, 70.12],
+ "c": [100.2, 101.33, 120.33],
+ }
+ )
+ pt = df.pivot_table(values="a", index="b", columns="c")
+ set_eng_float_format(accuracy=1)
+ result = pt.to_string()
+ assert "NaN" in result
+
+ def test_inf(self):
+ # Issue #11981
+
+ formatter = EngFormatter(accuracy=1, use_eng_prefix=True)
+ result = formatter(np.inf)
+ assert result == "inf"
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_format.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_format.py
new file mode 100644
index 0000000000000000000000000000000000000000..9602aa723e4ed75fe3ac7f2a4f00b8dbfa46ae53
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_format.py
@@ -0,0 +1,2302 @@
+"""
+Tests for the file pandas.io.formats.format, *not* tests for general formatting
+of pandas objects.
+"""
+
+from datetime import datetime
+from io import StringIO
+import re
+from shutil import get_terminal_size
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ MultiIndex,
+ NaT,
+ Series,
+ Timestamp,
+ date_range,
+ get_option,
+ option_context,
+ read_csv,
+ reset_option,
+)
+
+from pandas.io.formats import printing
+import pandas.io.formats.format as fmt
+
+
+def has_info_repr(df):
+ r = repr(df)
+ c1 = r.split("\n")[0].startswith("
+ # 2. Index
+ # 3. Columns
+ # 4. dtype
+ # 5. memory usage
+ # 6. trailing newline
+ nv = len(r.split("\n")) == 6
+ return has_info and nv
+
+
+def has_horizontally_truncated_repr(df):
+ try: # Check header row
+ fst_line = np.array(repr(df).splitlines()[0].split())
+ cand_col = np.where(fst_line == "...")[0][0]
+ except IndexError:
+ return False
+ # Make sure each row has this ... in the same place
+ r = repr(df)
+ for ix, _ in enumerate(r.splitlines()):
+ if not r.split()[cand_col] == "...":
+ return False
+ return True
+
+
+def has_vertically_truncated_repr(df):
+ r = repr(df)
+ only_dot_row = False
+ for row in r.splitlines():
+ if re.match(r"^[\.\ ]+$", row):
+ only_dot_row = True
+ return only_dot_row
+
+
+def has_truncated_repr(df):
+ return has_horizontally_truncated_repr(df) or has_vertically_truncated_repr(df)
+
+
+def has_doubly_truncated_repr(df):
+ return has_horizontally_truncated_repr(df) and has_vertically_truncated_repr(df)
+
+
+def has_expanded_repr(df):
+ r = repr(df)
+ for line in r.split("\n"):
+ if line.endswith("\\"):
+ return True
+ return False
+
+
+class TestDataFrameFormatting:
+ def test_repr_truncation(self):
+ max_len = 20
+ with option_context("display.max_colwidth", max_len):
+ df = DataFrame(
+ {
+ "A": np.random.default_rng(2).standard_normal(10),
+ "B": [
+ "a"
+ * np.random.default_rng(2).integers(max_len - 1, max_len + 1)
+ for _ in range(10)
+ ],
+ }
+ )
+ r = repr(df)
+ r = r[r.find("\n") + 1 :]
+
+ adj = printing.get_adjustment()
+
+ for line, value in zip(r.split("\n"), df["B"]):
+ if adj.len(value) + 1 > max_len:
+ assert "..." in line
+ else:
+ assert "..." not in line
+
+ with option_context("display.max_colwidth", 999999):
+ assert "..." not in repr(df)
+
+ with option_context("display.max_colwidth", max_len + 2):
+ assert "..." not in repr(df)
+
+ def test_repr_truncation_preserves_na(self):
+ # https://github.com/pandas-dev/pandas/issues/55630
+ df = DataFrame({"a": [pd.NA for _ in range(10)]})
+ with option_context("display.max_rows", 2, "display.show_dimensions", False):
+ assert repr(df) == " a\n0 \n.. ...\n9 "
+
+ def test_repr_truncation_dataframe_attrs(self):
+ # GH#60455
+ df = DataFrame([[0] * 10])
+ df.attrs["b"] = DataFrame([])
+ with option_context("display.max_columns", 2, "display.show_dimensions", False):
+ assert repr(df) == " 0 ... 9\n0 0 ... 0"
+
+ def test_repr_truncation_series_with_dataframe_attrs(self):
+ # GH#60568
+ ser = Series([0] * 10)
+ ser.attrs["b"] = DataFrame([])
+ with option_context("display.max_rows", 2, "display.show_dimensions", False):
+ assert repr(ser) == "0 0\n ..\n9 0\ndtype: int64"
+
+ def test_max_colwidth_negative_int_raises(self):
+ # Deprecation enforced from:
+ # https://github.com/pandas-dev/pandas/issues/31532
+ with pytest.raises(
+ ValueError, match="Value must be a nonnegative integer or None"
+ ):
+ with option_context("display.max_colwidth", -1):
+ pass
+
+ def test_repr_chop_threshold(self):
+ df = DataFrame([[0.1, 0.5], [0.5, -0.1]])
+ reset_option("display.chop_threshold") # default None
+ assert repr(df) == " 0 1\n0 0.1 0.5\n1 0.5 -0.1"
+
+ with option_context("display.chop_threshold", 0.2):
+ assert repr(df) == " 0 1\n0 0.0 0.5\n1 0.5 0.0"
+
+ with option_context("display.chop_threshold", 0.6):
+ assert repr(df) == " 0 1\n0 0.0 0.0\n1 0.0 0.0"
+
+ with option_context("display.chop_threshold", None):
+ assert repr(df) == " 0 1\n0 0.1 0.5\n1 0.5 -0.1"
+
+ def test_repr_chop_threshold_column_below(self):
+ # GH 6839: validation case
+
+ df = DataFrame([[10, 20, 30, 40], [8e-10, -1e-11, 2e-9, -2e-11]]).T
+
+ with option_context("display.chop_threshold", 0):
+ assert repr(df) == (
+ " 0 1\n"
+ "0 10.0 8.000000e-10\n"
+ "1 20.0 -1.000000e-11\n"
+ "2 30.0 2.000000e-09\n"
+ "3 40.0 -2.000000e-11"
+ )
+
+ with option_context("display.chop_threshold", 1e-8):
+ assert repr(df) == (
+ " 0 1\n"
+ "0 10.0 0.000000e+00\n"
+ "1 20.0 0.000000e+00\n"
+ "2 30.0 0.000000e+00\n"
+ "3 40.0 0.000000e+00"
+ )
+
+ with option_context("display.chop_threshold", 5e-11):
+ assert repr(df) == (
+ " 0 1\n"
+ "0 10.0 8.000000e-10\n"
+ "1 20.0 0.000000e+00\n"
+ "2 30.0 2.000000e-09\n"
+ "3 40.0 0.000000e+00"
+ )
+
+ def test_repr_no_backslash(self):
+ with option_context("mode.sim_interactive", True):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 4)))
+ assert "\\" not in repr(df)
+
+ def test_expand_frame_repr(self):
+ df_small = DataFrame("hello", index=[0], columns=[0])
+ df_wide = DataFrame("hello", index=[0], columns=range(10))
+ df_tall = DataFrame("hello", index=range(30), columns=range(5))
+
+ with option_context("mode.sim_interactive", True):
+ with option_context(
+ "display.max_columns",
+ 10,
+ "display.width",
+ 20,
+ "display.max_rows",
+ 20,
+ "display.show_dimensions",
+ True,
+ ):
+ with option_context("display.expand_frame_repr", True):
+ assert not has_truncated_repr(df_small)
+ assert not has_expanded_repr(df_small)
+ assert not has_truncated_repr(df_wide)
+ assert has_expanded_repr(df_wide)
+ assert has_vertically_truncated_repr(df_tall)
+ assert has_expanded_repr(df_tall)
+
+ with option_context("display.expand_frame_repr", False):
+ assert not has_truncated_repr(df_small)
+ assert not has_expanded_repr(df_small)
+ assert not has_horizontally_truncated_repr(df_wide)
+ assert not has_expanded_repr(df_wide)
+ assert has_vertically_truncated_repr(df_tall)
+ assert not has_expanded_repr(df_tall)
+
+ def test_repr_non_interactive(self):
+ # in non interactive mode, there can be no dependency on the
+ # result of terminal auto size detection
+ df = DataFrame("hello", index=range(1000), columns=range(5))
+
+ with option_context(
+ "mode.sim_interactive", False, "display.width", 0, "display.max_rows", 5000
+ ):
+ assert not has_truncated_repr(df)
+ assert not has_expanded_repr(df)
+
+ def test_repr_truncates_terminal_size(self, monkeypatch):
+ # see gh-21180
+
+ terminal_size = (118, 96)
+ monkeypatch.setattr(
+ "pandas.io.formats.format.get_terminal_size", lambda: terminal_size
+ )
+
+ index = range(5)
+ columns = MultiIndex.from_tuples(
+ [
+ ("This is a long title with > 37 chars.", "cat"),
+ ("This is a loooooonger title with > 43 chars.", "dog"),
+ ]
+ )
+ df = DataFrame(1, index=index, columns=columns)
+
+ result = repr(df)
+
+ h1, h2 = result.split("\n")[:2]
+ assert "long" in h1
+ assert "loooooonger" in h1
+ assert "cat" in h2
+ assert "dog" in h2
+
+ # regular columns
+ df2 = DataFrame({"A" * 41: [1, 2], "B" * 41: [1, 2]})
+ result = repr(df2)
+
+ assert df2.columns[0] in result.split("\n")[0]
+
+ def test_repr_truncates_terminal_size_full(self, monkeypatch):
+ # GH 22984 ensure entire window is filled
+ terminal_size = (80, 24)
+ df = DataFrame(np.random.default_rng(2).random((1, 7)))
+
+ monkeypatch.setattr(
+ "pandas.io.formats.format.get_terminal_size", lambda: terminal_size
+ )
+ assert "..." not in str(df)
+
+ def test_repr_truncation_column_size(self):
+ # dataframe with last column very wide -> check it is not used to
+ # determine size of truncation (...) column
+ df = DataFrame(
+ {
+ "a": [108480, 30830],
+ "b": [12345, 12345],
+ "c": [12345, 12345],
+ "d": [12345, 12345],
+ "e": ["a" * 50] * 2,
+ }
+ )
+ assert "..." in str(df)
+ assert " ... " not in str(df)
+
+ def test_repr_max_columns_max_rows(self):
+ term_width, term_height = get_terminal_size()
+ if term_width < 10 or term_height < 10:
+ pytest.skip(f"terminal size too small, {term_width} x {term_height}")
+
+ def mkframe(n):
+ index = [f"{i:05d}" for i in range(n)]
+ return DataFrame(0, index, index)
+
+ df6 = mkframe(6)
+ df10 = mkframe(10)
+ with option_context("mode.sim_interactive", True):
+ with option_context("display.width", term_width * 2):
+ with option_context("display.max_rows", 5, "display.max_columns", 5):
+ assert not has_expanded_repr(mkframe(4))
+ assert not has_expanded_repr(mkframe(5))
+ assert not has_expanded_repr(df6)
+ assert has_doubly_truncated_repr(df6)
+
+ with option_context("display.max_rows", 20, "display.max_columns", 10):
+ # Out off max_columns boundary, but no extending
+ # since not exceeding width
+ assert not has_expanded_repr(df6)
+ assert not has_truncated_repr(df6)
+
+ with option_context("display.max_rows", 9, "display.max_columns", 10):
+ # out vertical bounds can not result in expanded repr
+ assert not has_expanded_repr(df10)
+ assert has_vertically_truncated_repr(df10)
+
+ # width=None in terminal, auto detection
+ with option_context(
+ "display.max_columns",
+ 100,
+ "display.max_rows",
+ term_width * 20,
+ "display.width",
+ None,
+ ):
+ df = mkframe((term_width // 7) - 2)
+ assert not has_expanded_repr(df)
+ df = mkframe((term_width // 7) + 2)
+ printing.pprint_thing(df._repr_fits_horizontal_())
+ assert has_expanded_repr(df)
+
+ def test_repr_min_rows(self):
+ df = DataFrame({"a": range(20)})
+
+ # default setting no truncation even if above min_rows
+ assert ".." not in repr(df)
+ assert ".." not in df._repr_html_()
+
+ df = DataFrame({"a": range(61)})
+
+ # default of max_rows 60 triggers truncation if above
+ assert ".." in repr(df)
+ assert ".." in df._repr_html_()
+
+ with option_context("display.max_rows", 10, "display.min_rows", 4):
+ # truncated after first two rows
+ assert ".." in repr(df)
+ assert "2 " not in repr(df)
+ assert "..." in df._repr_html_()
+ assert "2 " not in df._repr_html_()
+
+ with option_context("display.max_rows", 12, "display.min_rows", None):
+ # when set to None, follow value of max_rows
+ assert "5 5" in repr(df)
+ assert "5 " in df._repr_html_()
+
+ with option_context("display.max_rows", 10, "display.min_rows", 12):
+ # when set value higher as max_rows, use the minimum
+ assert "5 5" not in repr(df)
+ assert "5 " not in df._repr_html_()
+
+ with option_context("display.max_rows", None, "display.min_rows", 12):
+ # max_rows of None -> never truncate
+ assert ".." not in repr(df)
+ assert ".." not in df._repr_html_()
+
+ @pytest.mark.parametrize(
+ "data, format_option, expected_values",
+ [
+ (12345.6789, "{:12.3f}", "12345.679"),
+ (None, "{:.3f}", "None"),
+ ("", "{:.2f}", ""),
+ (112345.6789, "{:6.3f}", "112345.679"),
+ ("foo foo", None, "foo foo"),
+ (" foo", None, "foo"),
+ (
+ "foo foo foo",
+ None,
+ "foo foo foo",
+ ), # odd no.of spaces
+ (
+ "foo foo foo",
+ None,
+ "foo foo foo",
+ ), # even no.of spaces
+ ],
+ )
+ def test_repr_float_formatting_html_output(
+ self, data, format_option, expected_values
+ ):
+ if format_option is not None:
+ with option_context("display.float_format", format_option.format):
+ df = DataFrame({"A": [data]})
+ html_output = df._repr_html_()
+ assert expected_values in html_output
+ else:
+ df = DataFrame({"A": [data]})
+ html_output = df._repr_html_()
+ assert expected_values in html_output
+
+ def test_str_max_colwidth(self):
+ # GH 7856
+ df = DataFrame(
+ [
+ {
+ "a": "foo",
+ "b": "bar",
+ "c": "uncomfortably long line with lots of stuff",
+ "d": 1,
+ },
+ {"a": "foo", "b": "bar", "c": "stuff", "d": 1},
+ ]
+ )
+ df.set_index(["a", "b", "c"])
+ assert str(df) == (
+ " a b c d\n"
+ "0 foo bar uncomfortably long line with lots of stuff 1\n"
+ "1 foo bar stuff 1"
+ )
+ with option_context("max_colwidth", 20):
+ assert str(df) == (
+ " a b c d\n"
+ "0 foo bar uncomfortably lo... 1\n"
+ "1 foo bar stuff 1"
+ )
+
+ def test_auto_detect(self):
+ term_width, term_height = get_terminal_size()
+ fac = 1.05 # Arbitrary large factor to exceed term width
+ cols = range(int(term_width * fac))
+ index = range(10)
+ df = DataFrame(index=index, columns=cols)
+ with option_context("mode.sim_interactive", True):
+ with option_context("display.max_rows", None):
+ with option_context("display.max_columns", None):
+ # Wrap around with None
+ assert has_expanded_repr(df)
+ with option_context("display.max_rows", 0):
+ with option_context("display.max_columns", 0):
+ # Truncate with auto detection.
+ assert has_horizontally_truncated_repr(df)
+
+ index = range(int(term_height * fac))
+ df = DataFrame(index=index, columns=cols)
+ with option_context("display.max_rows", 0):
+ with option_context("display.max_columns", None):
+ # Wrap around with None
+ assert has_expanded_repr(df)
+ # Truncate vertically
+ assert has_vertically_truncated_repr(df)
+
+ with option_context("display.max_rows", None):
+ with option_context("display.max_columns", 0):
+ assert has_horizontally_truncated_repr(df)
+
+ def test_to_string_repr_unicode2(self):
+ idx = Index(["abc", "\u03c3a", "aegdvg"])
+ ser = Series(np.random.default_rng(2).standard_normal(len(idx)), idx)
+ rs = repr(ser).split("\n")
+ line_len = len(rs[0])
+ for line in rs[1:]:
+ try:
+ line = line.decode(get_option("display.encoding"))
+ except AttributeError:
+ pass
+ if not line.startswith("dtype:"):
+ assert len(line) == line_len
+
+ def test_east_asian_unicode_false(self):
+ # not aligned properly because of east asian width
+
+ # mid col
+ df = DataFrame(
+ {"a": ["あ", "いいい", "う", "ええええええ"], "b": [1, 222, 33333, 4]},
+ index=["a", "bb", "c", "ddd"],
+ )
+ expected = (
+ " a b\na あ 1\n"
+ "bb いいい 222\nc う 33333\n"
+ "ddd ええええええ 4"
+ )
+ assert repr(df) == expected
+
+ # last col
+ df = DataFrame(
+ {"a": [1, 222, 33333, 4], "b": ["あ", "いいい", "う", "ええええええ"]},
+ index=["a", "bb", "c", "ddd"],
+ )
+ expected = (
+ " a b\na 1 あ\n"
+ "bb 222 いいい\nc 33333 う\n"
+ "ddd 4 ええええええ"
+ )
+ assert repr(df) == expected
+
+ # all col
+ df = DataFrame(
+ {
+ "a": ["あああああ", "い", "う", "えええ"],
+ "b": ["あ", "いいい", "う", "ええええええ"],
+ },
+ index=["a", "bb", "c", "ddd"],
+ )
+ expected = (
+ " a b\na あああああ あ\n"
+ "bb い いいい\nc う う\n"
+ "ddd えええ ええええええ"
+ )
+ assert repr(df) == expected
+
+ # column name
+ df = DataFrame(
+ {
+ "b": ["あ", "いいい", "う", "ええええええ"],
+ "あああああ": [1, 222, 33333, 4],
+ },
+ index=["a", "bb", "c", "ddd"],
+ )
+ expected = (
+ " b あああああ\na あ 1\n"
+ "bb いいい 222\nc う 33333\n"
+ "ddd ええええええ 4"
+ )
+ assert repr(df) == expected
+
+ # index
+ df = DataFrame(
+ {
+ "a": ["あああああ", "い", "う", "えええ"],
+ "b": ["あ", "いいい", "う", "ええええええ"],
+ },
+ index=["あああ", "いいいいいい", "うう", "え"],
+ )
+ expected = (
+ " a b\nあああ あああああ あ\n"
+ "いいいいいい い いいい\nうう う う\n"
+ "え えええ ええええええ"
+ )
+ assert repr(df) == expected
+
+ # index name
+ df = DataFrame(
+ {
+ "a": ["あああああ", "い", "う", "えええ"],
+ "b": ["あ", "いいい", "う", "ええええええ"],
+ },
+ index=Index(["あ", "い", "うう", "え"], name="おおおお"),
+ )
+ expected = (
+ " a b\n"
+ "おおおお \n"
+ "あ あああああ あ\n"
+ "い い いいい\n"
+ "うう う う\n"
+ "え えええ ええええええ"
+ )
+ assert repr(df) == expected
+
+ # all
+ df = DataFrame(
+ {
+ "あああ": ["あああ", "い", "う", "えええええ"],
+ "いいいいい": ["あ", "いいい", "う", "ええ"],
+ },
+ index=Index(["あ", "いいい", "うう", "え"], name="お"),
+ )
+ expected = (
+ " あああ いいいいい\n"
+ "お \n"
+ "あ あああ あ\n"
+ "いいい い いいい\n"
+ "うう う う\n"
+ "え えええええ ええ"
+ )
+ assert repr(df) == expected
+
+ # MultiIndex
+ idx = MultiIndex.from_tuples(
+ [("あ", "いい"), ("う", "え"), ("おおお", "かかかか"), ("き", "くく")]
+ )
+ df = DataFrame(
+ {
+ "a": ["あああああ", "い", "う", "えええ"],
+ "b": ["あ", "いいい", "う", "ええええええ"],
+ },
+ index=idx,
+ )
+ expected = (
+ " a b\n"
+ "あ いい あああああ あ\n"
+ "う え い いいい\n"
+ "おおお かかかか う う\n"
+ "き くく えええ ええええええ"
+ )
+ assert repr(df) == expected
+
+ # truncate
+ with option_context("display.max_rows", 3, "display.max_columns", 3):
+ df = DataFrame(
+ {
+ "a": ["あああああ", "い", "う", "えええ"],
+ "b": ["あ", "いいい", "う", "ええええええ"],
+ "c": ["お", "か", "ききき", "くくくくくく"],
+ "ああああ": ["さ", "し", "す", "せ"],
+ },
+ columns=["a", "b", "c", "ああああ"],
+ )
+
+ expected = (
+ " a ... ああああ\n0 あああああ ... さ\n"
+ ".. ... ... ...\n3 えええ ... せ\n"
+ "\n[4 rows x 4 columns]"
+ )
+ assert repr(df) == expected
+
+ df.index = ["あああ", "いいいい", "う", "aaa"]
+ expected = (
+ " a ... ああああ\nあああ あああああ ... さ\n"
+ ".. ... ... ...\naaa えええ ... せ\n"
+ "\n[4 rows x 4 columns]"
+ )
+ assert repr(df) == expected
+
+ def test_east_asian_unicode_true(self):
+ # Enable Unicode option -----------------------------------------
+ with option_context("display.unicode.east_asian_width", True):
+ # mid col
+ df = DataFrame(
+ {"a": ["あ", "いいい", "う", "ええええええ"], "b": [1, 222, 33333, 4]},
+ index=["a", "bb", "c", "ddd"],
+ )
+ expected = (
+ " a b\na あ 1\n"
+ "bb いいい 222\nc う 33333\n"
+ "ddd ええええええ 4"
+ )
+ assert repr(df) == expected
+
+ # last col
+ df = DataFrame(
+ {"a": [1, 222, 33333, 4], "b": ["あ", "いいい", "う", "ええええええ"]},
+ index=["a", "bb", "c", "ddd"],
+ )
+ expected = (
+ " a b\na 1 あ\n"
+ "bb 222 いいい\nc 33333 う\n"
+ "ddd 4 ええええええ"
+ )
+ assert repr(df) == expected
+
+ # all col
+ df = DataFrame(
+ {
+ "a": ["あああああ", "い", "う", "えええ"],
+ "b": ["あ", "いいい", "う", "ええええええ"],
+ },
+ index=["a", "bb", "c", "ddd"],
+ )
+ expected = (
+ " a b\n"
+ "a あああああ あ\n"
+ "bb い いいい\n"
+ "c う う\n"
+ "ddd えええ ええええええ"
+ )
+ assert repr(df) == expected
+
+ # column name
+ df = DataFrame(
+ {
+ "b": ["あ", "いいい", "う", "ええええええ"],
+ "あああああ": [1, 222, 33333, 4],
+ },
+ index=["a", "bb", "c", "ddd"],
+ )
+ expected = (
+ " b あああああ\n"
+ "a あ 1\n"
+ "bb いいい 222\n"
+ "c う 33333\n"
+ "ddd ええええええ 4"
+ )
+ assert repr(df) == expected
+
+ # index
+ df = DataFrame(
+ {
+ "a": ["あああああ", "い", "う", "えええ"],
+ "b": ["あ", "いいい", "う", "ええええええ"],
+ },
+ index=["あああ", "いいいいいい", "うう", "え"],
+ )
+ expected = (
+ " a b\n"
+ "あああ あああああ あ\n"
+ "いいいいいい い いいい\n"
+ "うう う う\n"
+ "え えええ ええええええ"
+ )
+ assert repr(df) == expected
+
+ # index name
+ df = DataFrame(
+ {
+ "a": ["あああああ", "い", "う", "えええ"],
+ "b": ["あ", "いいい", "う", "ええええええ"],
+ },
+ index=Index(["あ", "い", "うう", "え"], name="おおおお"),
+ )
+ expected = (
+ " a b\n"
+ "おおおお \n"
+ "あ あああああ あ\n"
+ "い い いいい\n"
+ "うう う う\n"
+ "え えええ ええええええ"
+ )
+ assert repr(df) == expected
+
+ # all
+ df = DataFrame(
+ {
+ "あああ": ["あああ", "い", "う", "えええええ"],
+ "いいいいい": ["あ", "いいい", "う", "ええ"],
+ },
+ index=Index(["あ", "いいい", "うう", "え"], name="お"),
+ )
+ expected = (
+ " あああ いいいいい\n"
+ "お \n"
+ "あ あああ あ\n"
+ "いいい い いいい\n"
+ "うう う う\n"
+ "え えええええ ええ"
+ )
+ assert repr(df) == expected
+
+ # MultiIndex
+ idx = MultiIndex.from_tuples(
+ [("あ", "いい"), ("う", "え"), ("おおお", "かかかか"), ("き", "くく")]
+ )
+ df = DataFrame(
+ {
+ "a": ["あああああ", "い", "う", "えええ"],
+ "b": ["あ", "いいい", "う", "ええええええ"],
+ },
+ index=idx,
+ )
+ expected = (
+ " a b\n"
+ "あ いい あああああ あ\n"
+ "う え い いいい\n"
+ "おおお かかかか う う\n"
+ "き くく えええ ええええええ"
+ )
+ assert repr(df) == expected
+
+ # truncate
+ with option_context("display.max_rows", 3, "display.max_columns", 3):
+ df = DataFrame(
+ {
+ "a": ["あああああ", "い", "う", "えええ"],
+ "b": ["あ", "いいい", "う", "ええええええ"],
+ "c": ["お", "か", "ききき", "くくくくくく"],
+ "ああああ": ["さ", "し", "す", "せ"],
+ },
+ columns=["a", "b", "c", "ああああ"],
+ )
+
+ expected = (
+ " a ... ああああ\n"
+ "0 あああああ ... さ\n"
+ ".. ... ... ...\n"
+ "3 えええ ... せ\n"
+ "\n[4 rows x 4 columns]"
+ )
+ assert repr(df) == expected
+
+ df.index = ["あああ", "いいいい", "う", "aaa"]
+ expected = (
+ " a ... ああああ\n"
+ "あああ あああああ ... さ\n"
+ "... ... ... ...\n"
+ "aaa えええ ... せ\n"
+ "\n[4 rows x 4 columns]"
+ )
+ assert repr(df) == expected
+
+ # ambiguous unicode
+ df = DataFrame(
+ {
+ "b": ["あ", "いいい", "¡¡", "ええええええ"],
+ "あああああ": [1, 222, 33333, 4],
+ },
+ index=["a", "bb", "c", "¡¡¡"],
+ )
+ expected = (
+ " b あああああ\n"
+ "a あ 1\n"
+ "bb いいい 222\n"
+ "c ¡¡ 33333\n"
+ "¡¡¡ ええええええ 4"
+ )
+ assert repr(df) == expected
+
+ def test_to_string_buffer_all_unicode(self):
+ buf = StringIO()
+
+ empty = DataFrame({"c/\u03c3": Series(dtype=object)})
+ nonempty = DataFrame({"c/\u03c3": Series([1, 2, 3])})
+
+ print(empty, file=buf)
+ print(nonempty, file=buf)
+
+ # this should work
+ buf.getvalue()
+
+ @pytest.mark.parametrize(
+ "index_scalar",
+ [
+ "a" * 10,
+ 1,
+ Timestamp(2020, 1, 1),
+ pd.Period("2020-01-01"),
+ ],
+ )
+ @pytest.mark.parametrize("h", [10, 20])
+ @pytest.mark.parametrize("w", [10, 20])
+ def test_to_string_truncate_indices(self, index_scalar, h, w):
+ with option_context("display.expand_frame_repr", False):
+ df = DataFrame(
+ index=[index_scalar] * h, columns=[str(i) * 10 for i in range(w)]
+ )
+ with option_context("display.max_rows", 15):
+ if h == 20:
+ assert has_vertically_truncated_repr(df)
+ else:
+ assert not has_vertically_truncated_repr(df)
+ with option_context("display.max_columns", 15):
+ if w == 20:
+ assert has_horizontally_truncated_repr(df)
+ else:
+ assert not has_horizontally_truncated_repr(df)
+ with option_context("display.max_rows", 15, "display.max_columns", 15):
+ if h == 20 and w == 20:
+ assert has_doubly_truncated_repr(df)
+ else:
+ assert not has_doubly_truncated_repr(df)
+
+ def test_to_string_truncate_multilevel(self):
+ arrays = [
+ ["bar", "bar", "baz", "baz", "foo", "foo", "qux", "qux"],
+ ["one", "two", "one", "two", "one", "two", "one", "two"],
+ ]
+ df = DataFrame(index=arrays, columns=arrays)
+ with option_context("display.max_rows", 7, "display.max_columns", 7):
+ assert has_doubly_truncated_repr(df)
+
+ @pytest.mark.parametrize("dtype", ["object", "datetime64[us]"])
+ def test_truncate_with_different_dtypes(self, dtype):
+ # 11594, 12045
+ # when truncated the dtypes of the splits can differ
+
+ # 11594
+ ser = Series(
+ [datetime(2012, 1, 1)] * 10
+ + [datetime(1012, 1, 2)]
+ + [datetime(2012, 1, 3)] * 10,
+ dtype=dtype,
+ )
+
+ with option_context("display.max_rows", 8):
+ result = str(ser)
+ assert dtype in result
+
+ def test_truncate_with_different_dtypes2(self):
+ # 12045
+ df = DataFrame({"text": ["some words"] + [None] * 9}, dtype=object)
+
+ with option_context("display.max_rows", 8, "display.max_columns", 3):
+ result = str(df)
+ assert "None" in result
+ assert "NaN" not in result
+
+ def test_truncate_with_different_dtypes_multiindex(self):
+ # GH#13000
+ df = DataFrame({"Vals": range(100)})
+ frame = pd.concat([df], keys=["Sweep"], names=["Sweep", "Index"])
+ result = repr(frame)
+
+ result2 = repr(frame.iloc[:5])
+ assert result.startswith(result2)
+
+ def test_datetimelike_frame(self):
+ # GH 12211
+ df = DataFrame({"date": [Timestamp("20130101").tz_localize("UTC")] + [NaT] * 5})
+
+ with option_context("display.max_rows", 5):
+ result = str(df)
+ assert "2013-01-01 00:00:00+00:00" in result
+ assert "NaT" in result
+ assert "..." in result
+ assert "[6 rows x 1 columns]" in result
+
+ dts = [Timestamp("2011-01-01", tz="US/Eastern")] * 5 + [NaT] * 5
+ df = DataFrame({"dt": dts, "x": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]})
+ with option_context("display.max_rows", 5):
+ expected = (
+ " dt x\n"
+ "0 2011-01-01 00:00:00-05:00 1\n"
+ "1 2011-01-01 00:00:00-05:00 2\n"
+ ".. ... ..\n"
+ "8 NaT 9\n"
+ "9 NaT 10\n\n"
+ "[10 rows x 2 columns]"
+ )
+ assert repr(df) == expected
+
+ dts = [NaT] * 5 + [Timestamp("2011-01-01", tz="US/Eastern")] * 5
+ df = DataFrame({"dt": dts, "x": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]})
+ with option_context("display.max_rows", 5):
+ expected = (
+ " dt x\n"
+ "0 NaT 1\n"
+ "1 NaT 2\n"
+ ".. ... ..\n"
+ "8 2011-01-01 00:00:00-05:00 9\n"
+ "9 2011-01-01 00:00:00-05:00 10\n\n"
+ "[10 rows x 2 columns]"
+ )
+ assert repr(df) == expected
+
+ dts = [Timestamp("2011-01-01", tz="Asia/Tokyo")] * 5 + [
+ Timestamp("2011-01-01", tz="US/Eastern")
+ ] * 5
+ df = DataFrame({"dt": dts, "x": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]})
+ with option_context("display.max_rows", 5):
+ expected = (
+ " dt x\n"
+ "0 2011-01-01 00:00:00+09:00 1\n"
+ "1 2011-01-01 00:00:00+09:00 2\n"
+ ".. ... ..\n"
+ "8 2011-01-01 00:00:00-05:00 9\n"
+ "9 2011-01-01 00:00:00-05:00 10\n\n"
+ "[10 rows x 2 columns]"
+ )
+ assert repr(df) == expected
+
+ @pytest.mark.parametrize(
+ "start_date",
+ [
+ "2017-01-01 23:59:59.999999999",
+ "2017-01-01 23:59:59.99999999",
+ "2017-01-01 23:59:59.9999999",
+ "2017-01-01 23:59:59.999999",
+ "2017-01-01 23:59:59.99999",
+ "2017-01-01 23:59:59.9999",
+ ],
+ )
+ def test_datetimeindex_highprecision(self, start_date):
+ # GH19030
+ # Check that high-precision time values for the end of day are
+ # included in repr for DatetimeIndex
+ df = DataFrame({"A": date_range(start=start_date, freq="D", periods=5)})
+ result = str(df)
+ assert start_date in result
+
+ dti = date_range(start=start_date, freq="D", periods=5)
+ df = DataFrame({"A": range(5)}, index=dti)
+ result = str(df.index)
+ assert start_date in result
+
+ def test_string_repr_encoding(self, datapath):
+ filepath = datapath("io", "parser", "data", "unicode_series.csv")
+ df = read_csv(filepath, header=None, encoding="latin1")
+ repr(df)
+ repr(df[1])
+
+ def test_repr_corner(self):
+ # representing infs poses no problems
+ df = DataFrame({"foo": [-np.inf, np.inf]})
+ repr(df)
+
+ def test_frame_info_encoding(self):
+ index = ["'Til There Was You (1997)", "ldum klaka (Cold Fever) (1994)"]
+ with option_context("display.max_rows", 1):
+ df = DataFrame(columns=["a", "b", "c"], index=index)
+ repr(df)
+ repr(df.T)
+
+ def test_wide_repr(self):
+ with option_context(
+ "mode.sim_interactive",
+ True,
+ "display.show_dimensions",
+ True,
+ "display.max_columns",
+ 20,
+ ):
+ max_cols = get_option("display.max_columns")
+ df = DataFrame([["a" * 25] * (max_cols - 1)] * 10)
+ with option_context("display.expand_frame_repr", False):
+ rep_str = repr(df)
+
+ assert f"10 rows x {max_cols - 1} columns" in rep_str
+ with option_context("display.expand_frame_repr", True):
+ wide_repr = repr(df)
+ assert rep_str != wide_repr
+
+ with option_context("display.width", 120):
+ wider_repr = repr(df)
+ assert len(wider_repr) < len(wide_repr)
+
+ def test_wide_repr_wide_columns(self):
+ with option_context("mode.sim_interactive", True, "display.max_columns", 20):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((5, 3)),
+ columns=["a" * 90, "b" * 90, "c" * 90],
+ )
+ rep_str = repr(df)
+
+ assert len(rep_str.splitlines()) == 20
+
+ def test_wide_repr_named(self):
+ with option_context("mode.sim_interactive", True, "display.max_columns", 20):
+ max_cols = get_option("display.max_columns")
+ df = DataFrame([["a" * 25] * (max_cols - 1)] * 10)
+ df.index.name = "DataFrame Index"
+ with option_context("display.expand_frame_repr", False):
+ rep_str = repr(df)
+ with option_context("display.expand_frame_repr", True):
+ wide_repr = repr(df)
+ assert rep_str != wide_repr
+
+ with option_context("display.width", 150):
+ wider_repr = repr(df)
+ assert len(wider_repr) < len(wide_repr)
+
+ for line in wide_repr.splitlines()[1::13]:
+ assert "DataFrame Index" in line
+
+ def test_wide_repr_multiindex(self):
+ with option_context("mode.sim_interactive", True, "display.max_columns", 20):
+ midx = MultiIndex.from_arrays([["a" * 5] * 10] * 2)
+ max_cols = get_option("display.max_columns")
+ df = DataFrame([["a" * 25] * (max_cols - 1)] * 10, index=midx)
+ df.index.names = ["Level 0", "Level 1"]
+ with option_context("display.expand_frame_repr", False):
+ rep_str = repr(df)
+ with option_context("display.expand_frame_repr", True):
+ wide_repr = repr(df)
+ assert rep_str != wide_repr
+
+ with option_context("display.width", 150):
+ wider_repr = repr(df)
+ assert len(wider_repr) < len(wide_repr)
+
+ for line in wide_repr.splitlines()[1::13]:
+ assert "Level 0 Level 1" in line
+
+ def test_wide_repr_multiindex_cols(self):
+ with option_context("mode.sim_interactive", True, "display.max_columns", 20):
+ max_cols = get_option("display.max_columns")
+ midx = MultiIndex.from_arrays([["a" * 5] * 10] * 2)
+ mcols = MultiIndex.from_arrays([["b" * 3] * (max_cols - 1)] * 2)
+ df = DataFrame(
+ [["c" * 25] * (max_cols - 1)] * 10, index=midx, columns=mcols
+ )
+ df.index.names = ["Level 0", "Level 1"]
+ with option_context("display.expand_frame_repr", False):
+ rep_str = repr(df)
+ with option_context("display.expand_frame_repr", True):
+ wide_repr = repr(df)
+ assert rep_str != wide_repr
+
+ with option_context("display.width", 150, "display.max_columns", 20):
+ wider_repr = repr(df)
+ assert len(wider_repr) < len(wide_repr)
+
+ def test_wide_repr_unicode(self):
+ with option_context("mode.sim_interactive", True, "display.max_columns", 20):
+ max_cols = 20
+ df = DataFrame([["a" * 25] * 10] * (max_cols - 1))
+ with option_context("display.expand_frame_repr", False):
+ rep_str = repr(df)
+ with option_context("display.expand_frame_repr", True):
+ wide_repr = repr(df)
+ assert rep_str != wide_repr
+
+ with option_context("display.width", 150):
+ wider_repr = repr(df)
+ assert len(wider_repr) < len(wide_repr)
+
+ def test_wide_repr_wide_long_columns(self):
+ with option_context("mode.sim_interactive", True):
+ df = DataFrame({"a": ["a" * 30, "b" * 30], "b": ["c" * 70, "d" * 80]})
+
+ result = repr(df)
+ assert "ccccc" in result
+ assert "ddddd" in result
+
+ def test_long_series(self):
+ n = 1000
+ s = Series(
+ np.random.default_rng(2).integers(-50, 50, n),
+ index=[f"s{x:04d}" for x in range(n)],
+ dtype="int64",
+ )
+
+ str_rep = str(s)
+ nmatches = len(re.findall("dtype", str_rep))
+ assert nmatches == 1
+
+ def test_to_string_ascii_error(self):
+ data = [
+ (
+ "0 ",
+ " .gitignore ",
+ " 5 ",
+ " \xe2\x80\xa2\xe2\x80\xa2\xe2\x80\xa2\xe2\x80\xa2\xe2\x80\xa2",
+ )
+ ]
+ df = DataFrame(data)
+
+ # it works!
+ repr(df)
+
+ def test_show_dimensions(self):
+ df = DataFrame(123, index=range(10, 15), columns=range(30))
+
+ with option_context(
+ "display.max_rows",
+ 10,
+ "display.max_columns",
+ 40,
+ "display.width",
+ 500,
+ "display.expand_frame_repr",
+ "info",
+ "display.show_dimensions",
+ True,
+ ):
+ assert "5 rows" in str(df)
+ assert "5 rows" in df._repr_html_()
+ with option_context(
+ "display.max_rows",
+ 10,
+ "display.max_columns",
+ 40,
+ "display.width",
+ 500,
+ "display.expand_frame_repr",
+ "info",
+ "display.show_dimensions",
+ False,
+ ):
+ assert "5 rows" not in str(df)
+ assert "5 rows" not in df._repr_html_()
+ with option_context(
+ "display.max_rows",
+ 2,
+ "display.max_columns",
+ 2,
+ "display.width",
+ 500,
+ "display.expand_frame_repr",
+ "info",
+ "display.show_dimensions",
+ "truncate",
+ ):
+ assert "5 rows" in str(df)
+ assert "5 rows" in df._repr_html_()
+ with option_context(
+ "display.max_rows",
+ 10,
+ "display.max_columns",
+ 40,
+ "display.width",
+ 500,
+ "display.expand_frame_repr",
+ "info",
+ "display.show_dimensions",
+ "truncate",
+ ):
+ assert "5 rows" not in str(df)
+ assert "5 rows" not in df._repr_html_()
+
+ def test_info_repr(self):
+ # GH#21746 For tests inside a terminal (i.e. not CI) we need to detect
+ # the terminal size to ensure that we try to print something "too big"
+ term_width, term_height = get_terminal_size()
+
+ max_rows = 60
+ max_cols = 20 + (max(term_width, 80) - 80) // 4
+ # Long
+ h, w = max_rows + 1, max_cols - 1
+ df = DataFrame({k: np.arange(1, 1 + h) for k in np.arange(w)})
+ assert has_vertically_truncated_repr(df)
+ with option_context("display.large_repr", "info"):
+ assert has_info_repr(df)
+
+ # Wide
+ h, w = max_rows - 1, max_cols + 1
+ df = DataFrame({k: np.arange(1, 1 + h) for k in np.arange(w)})
+ assert has_horizontally_truncated_repr(df)
+ with option_context(
+ "display.large_repr", "info", "display.max_columns", max_cols
+ ):
+ assert has_info_repr(df)
+
+ def test_info_repr_max_cols(self):
+ # GH #6939
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 5)))
+ with option_context(
+ "display.large_repr",
+ "info",
+ "display.max_columns",
+ 1,
+ "display.max_info_columns",
+ 4,
+ ):
+ assert has_non_verbose_info_repr(df)
+
+ with option_context(
+ "display.large_repr",
+ "info",
+ "display.max_columns",
+ 1,
+ "display.max_info_columns",
+ 5,
+ ):
+ assert not has_non_verbose_info_repr(df)
+
+ # FIXME: don't leave commented-out
+ # test verbose overrides
+ # set_option('display.max_info_columns', 4) # exceeded
+
+ def test_pprint_pathological_object(self):
+ """
+ If the test fails, it at least won't hang.
+ """
+
+ class A:
+ def __getitem__(self, key):
+ return 3 # obviously simplified
+
+ df = DataFrame([A()])
+ repr(df) # just don't die
+
+ def test_float_trim_zeros(self):
+ vals = [
+ 2.08430917305e10,
+ 3.52205017305e10,
+ 2.30674817305e10,
+ 2.03954217305e10,
+ 5.59897817305e10,
+ ]
+ skip = True
+ for line in repr(DataFrame({"A": vals})).split("\n")[:-2]:
+ if line.startswith("dtype:"):
+ continue
+ if _three_digit_exp():
+ assert ("+010" in line) or skip
+ else:
+ assert ("+10" in line) or skip
+ skip = False
+
+ @pytest.mark.parametrize(
+ "data, expected",
+ [
+ (["3.50"], "0 3.50\ndtype: object"),
+ ([1.20, "1.00"], "0 1.2\n1 1.00\ndtype: object"),
+ ([np.nan], "0 NaN\ndtype: float64"),
+ ([None], "0 None\ndtype: object"),
+ (["3.50", np.nan], "0 3.50\n1 NaN\ndtype: object"),
+ ([3.50, np.nan], "0 3.5\n1 NaN\ndtype: float64"),
+ ([3.50, np.nan, "3.50"], "0 3.5\n1 NaN\n2 3.50\ndtype: object"),
+ ([3.50, None, "3.50"], "0 3.5\n1 None\n2 3.50\ndtype: object"),
+ ],
+ )
+ def test_repr_str_float_truncation(self, data, expected, using_infer_string):
+ # GH#38708
+ series = Series(data, dtype=object if "3.50" in data else None)
+ result = repr(series)
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "float_format,expected",
+ [
+ ("{:,.0f}".format, "0 1,000\n1 test\ndtype: object"),
+ ("{:.4f}".format, "0 1000.0000\n1 test\ndtype: object"),
+ ],
+ )
+ def test_repr_float_format_in_object_col(self, float_format, expected):
+ # GH#40024
+ df = Series([1000.0, "test"])
+ with option_context("display.float_format", float_format):
+ result = repr(df)
+
+ assert result == expected
+
+ def test_period(self):
+ # GH 12615
+ df = DataFrame(
+ {
+ "A": pd.period_range("2013-01", periods=4, freq="M"),
+ "B": [
+ pd.Period("2011-01", freq="M"),
+ pd.Period("2011-02-01", freq="D"),
+ pd.Period("2011-03-01 09:00", freq="h"),
+ pd.Period("2011-04", freq="M"),
+ ],
+ "C": list("abcd"),
+ }
+ )
+ exp = (
+ " A B C\n"
+ "0 2013-01 2011-01 a\n"
+ "1 2013-02 2011-02-01 b\n"
+ "2 2013-03 2011-03-01 09:00 c\n"
+ "3 2013-04 2011-04 d"
+ )
+ assert str(df) == exp
+
+ @pytest.mark.parametrize(
+ "length, max_rows, min_rows, expected",
+ [
+ (10, 10, 10, 10),
+ (10, 10, None, 10),
+ (10, 8, None, 8),
+ (20, 30, 10, 30), # max_rows > len(frame), hence max_rows
+ (50, 30, 10, 10), # max_rows < len(frame), hence min_rows
+ (100, 60, 10, 10), # same
+ (60, 60, 10, 60), # edge case
+ (61, 60, 10, 10), # edge case
+ ],
+ )
+ def test_max_rows_fitted(self, length, min_rows, max_rows, expected):
+ """Check that display logic is correct.
+
+ GH #37359
+
+ See description here:
+ https://pandas.pydata.org/docs/dev/user_guide/options.html#frequently-used-options
+ """
+ formatter = fmt.DataFrameFormatter(
+ DataFrame(np.random.default_rng(2).random((length, 3))),
+ max_rows=max_rows,
+ min_rows=min_rows,
+ )
+ result = formatter.max_rows_fitted
+ assert result == expected
+
+
+def gen_series_formatting():
+ s1 = Series(["a"] * 100)
+ s2 = Series(["ab"] * 100)
+ s3 = Series(["a", "ab", "abc", "abcd", "abcde", "abcdef"])
+ s4 = s3[::-1]
+ test_sers = {"onel": s1, "twol": s2, "asc": s3, "desc": s4}
+ return test_sers
+
+
+class TestSeriesFormatting:
+ def test_freq_name_separation(self):
+ s = Series(
+ np.random.default_rng(2).standard_normal(10),
+ index=date_range("1/1/2000", periods=10),
+ name=0,
+ )
+
+ result = repr(s)
+ assert "Freq: D, Name: 0" in result
+
+ def test_unicode_name_in_footer(self):
+ s = Series([1, 2], name="\u05e2\u05d1\u05e8\u05d9\u05ea")
+ sf = fmt.SeriesFormatter(s, name="\u05e2\u05d1\u05e8\u05d9\u05ea")
+ sf._get_footer() # should not raise exception
+
+ def test_east_asian_unicode_series(self, using_infer_string):
+ # not aligned properly because of east asian width
+
+ # unicode index
+ s = Series(["a", "bb", "CCC", "D"], index=["あ", "いい", "ううう", "ええええ"])
+ expected = "".join(
+ [
+ "あ a\n",
+ "いい bb\n",
+ "ううう CCC\n",
+ "ええええ D\ndtype: object",
+ ]
+ )
+ if using_infer_string:
+ expected = expected.replace("dtype: object", "dtype: str")
+ assert repr(s) == expected
+
+ # unicode values
+ s = Series(["あ", "いい", "ううう", "ええええ"], index=["a", "bb", "c", "ddd"])
+ expected = "".join(
+ [
+ "a あ\n",
+ "bb いい\n",
+ "c ううう\n",
+ "ddd ええええ\n",
+ "dtype: object",
+ ]
+ )
+ if using_infer_string:
+ expected = expected.replace("dtype: object", "dtype: str")
+ assert repr(s) == expected
+
+ # both
+ s = Series(
+ ["あ", "いい", "ううう", "ええええ"],
+ index=["ああ", "いいいい", "う", "えええ"],
+ )
+ expected = "".join(
+ [
+ "ああ あ\n",
+ "いいいい いい\n",
+ "う ううう\n",
+ "えええ ええええ\n",
+ "dtype: object",
+ ]
+ )
+ if using_infer_string:
+ expected = expected.replace("dtype: object", "dtype: str")
+ assert repr(s) == expected
+
+ # unicode footer
+ s = Series(
+ ["あ", "いい", "ううう", "ええええ"],
+ index=["ああ", "いいいい", "う", "えええ"],
+ name="おおおおおおお",
+ )
+ expected = (
+ "ああ あ\nいいいい いい\nう ううう\n"
+ "えええ ええええ\nName: おおおおおおお, dtype: object"
+ )
+ if using_infer_string:
+ expected = expected.replace("dtype: object", "dtype: str")
+ assert repr(s) == expected
+
+ # MultiIndex
+ idx = MultiIndex.from_tuples(
+ [("あ", "いい"), ("う", "え"), ("おおお", "かかかか"), ("き", "くく")]
+ )
+ s = Series([1, 22, 3333, 44444], index=idx)
+ expected = (
+ "あ いい 1\n"
+ "う え 22\n"
+ "おおお かかかか 3333\n"
+ "き くく 44444\ndtype: int64"
+ )
+ assert repr(s) == expected
+
+ # object dtype, shorter than unicode repr
+ s = Series([1, 22, 3333, 44444], index=[1, "AB", np.nan, "あああ"])
+ expected = (
+ "1 1\nAB 22\nNaN 3333\nあああ 44444\ndtype: int64"
+ )
+ assert repr(s) == expected
+
+ # object dtype, longer than unicode repr
+ s = Series(
+ [1, 22, 3333, 44444], index=[1, "AB", Timestamp("2011-01-01"), "あああ"]
+ )
+ expected = (
+ "1 1\n"
+ "AB 22\n"
+ "2011-01-01 00:00:00 3333\n"
+ "あああ 44444\ndtype: int64"
+ )
+ assert repr(s) == expected
+
+ # truncate
+ with option_context("display.max_rows", 3):
+ s = Series(["あ", "いい", "ううう", "ええええ"], name="おおおおおおお")
+
+ expected = (
+ "0 あ\n ... \n"
+ "3 ええええ\n"
+ "Name: おおおおおおお, Length: 4, dtype: object"
+ )
+ if using_infer_string:
+ expected = expected.replace("dtype: object", "dtype: str")
+ assert repr(s) == expected
+
+ s.index = ["ああ", "いいいい", "う", "えええ"]
+ expected = (
+ "ああ あ\n ... \n"
+ "えええ ええええ\n"
+ "Name: おおおおおおお, Length: 4, dtype: object"
+ )
+ if using_infer_string:
+ expected = expected.replace("dtype: object", "dtype: str")
+ assert repr(s) == expected
+
+ # Enable Unicode option -----------------------------------------
+ with option_context("display.unicode.east_asian_width", True):
+ # unicode index
+ s = Series(
+ ["a", "bb", "CCC", "D"],
+ index=["あ", "いい", "ううう", "ええええ"],
+ )
+ expected = (
+ "あ a\nいい bb\nううう CCC\n"
+ "ええええ D\ndtype: object"
+ )
+ if using_infer_string:
+ expected = expected.replace("dtype: object", "dtype: str")
+ assert repr(s) == expected
+
+ # unicode values
+ s = Series(
+ ["あ", "いい", "ううう", "ええええ"],
+ index=["a", "bb", "c", "ddd"],
+ )
+ expected = (
+ "a あ\nbb いい\nc ううう\n"
+ "ddd ええええ\ndtype: object"
+ )
+ if using_infer_string:
+ expected = expected.replace("dtype: object", "dtype: str")
+ assert repr(s) == expected
+ # both
+ s = Series(
+ ["あ", "いい", "ううう", "ええええ"],
+ index=["ああ", "いいいい", "う", "えええ"],
+ )
+ expected = (
+ "ああ あ\n"
+ "いいいい いい\n"
+ "う ううう\n"
+ "えええ ええええ\ndtype: object"
+ )
+ if using_infer_string:
+ expected = expected.replace("dtype: object", "dtype: str")
+ assert repr(s) == expected
+
+ # unicode footer
+ s = Series(
+ ["あ", "いい", "ううう", "ええええ"],
+ index=["ああ", "いいいい", "う", "えええ"],
+ name="おおおおおおお",
+ )
+ expected = (
+ "ああ あ\n"
+ "いいいい いい\n"
+ "う ううう\n"
+ "えええ ええええ\n"
+ "Name: おおおおおおお, dtype: object"
+ )
+ if using_infer_string:
+ expected = expected.replace("dtype: object", "dtype: str")
+ assert repr(s) == expected
+
+ # MultiIndex
+ idx = MultiIndex.from_tuples(
+ [("あ", "いい"), ("う", "え"), ("おおお", "かかかか"), ("き", "くく")]
+ )
+ s = Series([1, 22, 3333, 44444], index=idx)
+ expected = (
+ "あ いい 1\n"
+ "う え 22\n"
+ "おおお かかかか 3333\n"
+ "き くく 44444\n"
+ "dtype: int64"
+ )
+ assert repr(s) == expected
+
+ # object dtype, shorter than unicode repr
+ s = Series([1, 22, 3333, 44444], index=[1, "AB", np.nan, "あああ"])
+ expected = (
+ "1 1\nAB 22\nNaN 3333\n"
+ "あああ 44444\ndtype: int64"
+ )
+ assert repr(s) == expected
+
+ # object dtype, longer than unicode repr
+ s = Series(
+ [1, 22, 3333, 44444],
+ index=[1, "AB", Timestamp("2011-01-01"), "あああ"],
+ )
+ expected = (
+ "1 1\n"
+ "AB 22\n"
+ "2011-01-01 00:00:00 3333\n"
+ "あああ 44444\ndtype: int64"
+ )
+ assert repr(s) == expected
+
+ # truncate
+ with option_context("display.max_rows", 3):
+ s = Series(["あ", "いい", "ううう", "ええええ"], name="おおおおおおお")
+ expected = (
+ "0 あ\n ... \n"
+ "3 ええええ\n"
+ "Name: おおおおおおお, Length: 4, dtype: object"
+ )
+ if using_infer_string:
+ expected = expected.replace("dtype: object", "dtype: str")
+ assert repr(s) == expected
+
+ s.index = ["ああ", "いいいい", "う", "えええ"]
+ expected = (
+ "ああ あ\n"
+ " ... \n"
+ "えええ ええええ\n"
+ "Name: おおおおおおお, Length: 4, dtype: object"
+ )
+ if using_infer_string:
+ expected = expected.replace("dtype: object", "dtype: str")
+ assert repr(s) == expected
+
+ # ambiguous unicode
+ s = Series(
+ ["¡¡", "い¡¡", "ううう", "ええええ"],
+ index=["ああ", "¡¡¡¡いい", "¡¡", "えええ"],
+ )
+ expected = (
+ "ああ ¡¡\n"
+ "¡¡¡¡いい い¡¡\n"
+ "¡¡ ううう\n"
+ "えええ ええええ\ndtype: object"
+ )
+ if using_infer_string:
+ expected = expected.replace("dtype: object", "dtype: str")
+ assert repr(s) == expected
+
+ def test_float_trim_zeros(self):
+ vals = [
+ 2.08430917305e10,
+ 3.52205017305e10,
+ 2.30674817305e10,
+ 2.03954217305e10,
+ 5.59897817305e10,
+ ]
+ for line in repr(Series(vals)).split("\n"):
+ if line.startswith("dtype:"):
+ continue
+ if _three_digit_exp():
+ assert "+010" in line
+ else:
+ assert "+10" in line
+
+ @pytest.mark.parametrize(
+ "start_date",
+ [
+ "2017-01-01 23:59:59.999999999",
+ "2017-01-01 23:59:59.99999999",
+ "2017-01-01 23:59:59.9999999",
+ "2017-01-01 23:59:59.999999",
+ "2017-01-01 23:59:59.99999",
+ "2017-01-01 23:59:59.9999",
+ ],
+ )
+ def test_datetimeindex_highprecision(self, start_date):
+ # GH19030
+ # Check that high-precision time values for the end of day are
+ # included in repr for DatetimeIndex
+ s1 = Series(date_range(start=start_date, freq="D", periods=5))
+ result = str(s1)
+ assert start_date in result
+
+ dti = date_range(start=start_date, freq="D", periods=5)
+ s2 = Series(3, index=dti)
+ result = str(s2.index)
+ assert start_date in result
+
+ def test_mixed_datetime64(self):
+ df = DataFrame({"A": [1, 2], "B": ["2012-01-01", "2012-01-02"]})
+ df["B"] = pd.to_datetime(df.B)
+
+ result = repr(df.loc[0])
+ assert "2012-01-01" in result
+
+ def test_period(self):
+ # GH 12615
+ index = pd.period_range("2013-01", periods=6, freq="M")
+ s = Series(np.arange(6, dtype="int64"), index=index)
+ exp = (
+ "2013-01 0\n"
+ "2013-02 1\n"
+ "2013-03 2\n"
+ "2013-04 3\n"
+ "2013-05 4\n"
+ "2013-06 5\n"
+ "Freq: M, dtype: int64"
+ )
+ assert str(s) == exp
+
+ s = Series(index)
+ exp = (
+ "0 2013-01\n"
+ "1 2013-02\n"
+ "2 2013-03\n"
+ "3 2013-04\n"
+ "4 2013-05\n"
+ "5 2013-06\n"
+ "dtype: period[M]"
+ )
+ assert str(s) == exp
+
+ # periods with mixed freq
+ s = Series(
+ [
+ pd.Period("2011-01", freq="M"),
+ pd.Period("2011-02-01", freq="D"),
+ pd.Period("2011-03-01 09:00", freq="h"),
+ ]
+ )
+ exp = (
+ "0 2011-01\n1 2011-02-01\n"
+ "2 2011-03-01 09:00\ndtype: object"
+ )
+ assert str(s) == exp
+
+ def test_max_multi_index_display(self):
+ # GH 7101
+
+ # doc example (indexing.rst)
+
+ # multi-index
+ arrays = [
+ ["bar", "bar", "baz", "baz", "foo", "foo", "qux", "qux"],
+ ["one", "two", "one", "two", "one", "two", "one", "two"],
+ ]
+ tuples = list(zip(*arrays))
+ index = MultiIndex.from_tuples(tuples, names=["first", "second"])
+ s = Series(np.random.default_rng(2).standard_normal(8), index=index)
+
+ with option_context("display.max_rows", 10):
+ assert len(str(s).split("\n")) == 10
+ with option_context("display.max_rows", 3):
+ assert len(str(s).split("\n")) == 5
+ with option_context("display.max_rows", 2):
+ assert len(str(s).split("\n")) == 5
+ with option_context("display.max_rows", 1):
+ assert len(str(s).split("\n")) == 4
+ with option_context("display.max_rows", 0):
+ assert len(str(s).split("\n")) == 10
+
+ # index
+ s = Series(np.random.default_rng(2).standard_normal(8), None)
+
+ with option_context("display.max_rows", 10):
+ assert len(str(s).split("\n")) == 9
+ with option_context("display.max_rows", 3):
+ assert len(str(s).split("\n")) == 4
+ with option_context("display.max_rows", 2):
+ assert len(str(s).split("\n")) == 4
+ with option_context("display.max_rows", 1):
+ assert len(str(s).split("\n")) == 3
+ with option_context("display.max_rows", 0):
+ assert len(str(s).split("\n")) == 9
+
+ # Make sure #8532 is fixed
+ def test_consistent_format(self):
+ s = Series([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0.9999, 1, 1] * 10)
+ with option_context("display.max_rows", 10, "display.show_dimensions", False):
+ res = repr(s)
+ exp = (
+ "0 1.0000\n1 1.0000\n2 1.0000\n3 "
+ "1.0000\n4 1.0000\n ... \n125 "
+ "1.0000\n126 1.0000\n127 0.9999\n128 "
+ "1.0000\n129 1.0000\ndtype: float64"
+ )
+ assert res == exp
+
+ def chck_ncols(self, s):
+ lines = [
+ line for line in repr(s).split("\n") if not re.match(r"[^\.]*\.+", line)
+ ][:-1]
+ ncolsizes = len({len(line.strip()) for line in lines})
+ assert ncolsizes == 1
+
+ def test_format_explicit(self, using_infer_string):
+ test_sers = gen_series_formatting()
+ with option_context("display.max_rows", 4, "display.show_dimensions", False):
+ res = repr(test_sers["onel"])
+ exp = "0 a\n1 a\n ..\n98 a\n99 a\ndtype: object"
+ if using_infer_string:
+ exp = exp.replace("dtype: object", "dtype: str")
+ assert exp == res
+ res = repr(test_sers["twol"])
+ exp = "0 ab\n1 ab\n ..\n98 ab\n99 ab\ndtype: object"
+ if using_infer_string:
+ exp = exp.replace("dtype: object", "dtype: str")
+ assert exp == res
+ res = repr(test_sers["asc"])
+ exp = (
+ "0 a\n1 ab\n ... \n4 abcde\n5 "
+ "abcdef\ndtype: object"
+ )
+ if using_infer_string:
+ exp = exp.replace("dtype: object", "dtype: str")
+ assert exp == res
+ res = repr(test_sers["desc"])
+ exp = (
+ "5 abcdef\n4 abcde\n ... \n1 ab\n0 "
+ "a\ndtype: object"
+ )
+ if using_infer_string:
+ exp = exp.replace("dtype: object", "dtype: str")
+ assert exp == res
+
+ def test_ncols(self):
+ test_sers = gen_series_formatting()
+ for s in test_sers.values():
+ self.chck_ncols(s)
+
+ def test_max_rows_eq_one(self):
+ s = Series(range(10), dtype="int64")
+ with option_context("display.max_rows", 1):
+ strrepr = repr(s).split("\n")
+ exp1 = ["0", "0"]
+ res1 = strrepr[0].split()
+ assert exp1 == res1
+ exp2 = [".."]
+ res2 = strrepr[1].split()
+ assert exp2 == res2
+
+ def test_truncate_ndots(self):
+ def getndots(s):
+ return len(re.match(r"[^\.]*(\.*)", s).groups()[0])
+
+ s = Series([0, 2, 3, 6])
+ with option_context("display.max_rows", 2):
+ strrepr = repr(s).replace("\n", "")
+ assert getndots(strrepr) == 2
+
+ s = Series([0, 100, 200, 400])
+ with option_context("display.max_rows", 2):
+ strrepr = repr(s).replace("\n", "")
+ assert getndots(strrepr) == 3
+
+ def test_show_dimensions(self):
+ # gh-7117
+ s = Series(range(5))
+
+ assert "Length" not in repr(s)
+
+ with option_context("display.max_rows", 4):
+ assert "Length" in repr(s)
+
+ with option_context("display.show_dimensions", True):
+ assert "Length" in repr(s)
+
+ with option_context("display.max_rows", 4, "display.show_dimensions", False):
+ assert "Length" not in repr(s)
+
+ def test_repr_min_rows(self):
+ s = Series(range(20))
+
+ # default setting no truncation even if above min_rows
+ assert ".." not in repr(s)
+
+ s = Series(range(61))
+
+ # default of max_rows 60 triggers truncation if above
+ assert ".." in repr(s)
+
+ with option_context("display.max_rows", 10, "display.min_rows", 4):
+ # truncated after first two rows
+ assert ".." in repr(s)
+ assert "2 " not in repr(s)
+
+ with option_context("display.max_rows", 12, "display.min_rows", None):
+ # when set to None, follow value of max_rows
+ assert "5 5" in repr(s)
+
+ with option_context("display.max_rows", 10, "display.min_rows", 12):
+ # when set value higher as max_rows, use the minimum
+ assert "5 5" not in repr(s)
+
+ with option_context("display.max_rows", None, "display.min_rows", 12):
+ # max_rows of None -> never truncate
+ assert ".." not in repr(s)
+
+
+class TestGenericArrayFormatter:
+ def test_1d_array(self):
+ # _GenericArrayFormatter is used on types for which there isn't a dedicated
+ # formatter. np.bool_ is one of those types.
+ obj = fmt._GenericArrayFormatter(np.array([True, False]))
+ res = obj.get_result()
+ assert len(res) == 2
+ # Results should be right-justified.
+ assert res[0] == " True"
+ assert res[1] == " False"
+
+ def test_2d_array(self):
+ obj = fmt._GenericArrayFormatter(np.array([[True, False], [False, True]]))
+ res = obj.get_result()
+ assert len(res) == 2
+ assert res[0] == " [True, False]"
+ assert res[1] == " [False, True]"
+
+ def test_3d_array(self):
+ obj = fmt._GenericArrayFormatter(
+ np.array([[[True, True], [False, False]], [[False, True], [True, False]]])
+ )
+ res = obj.get_result()
+ assert len(res) == 2
+ assert res[0] == " [[True, True], [False, False]]"
+ assert res[1] == " [[False, True], [True, False]]"
+
+
+def _three_digit_exp():
+ return f"{1.7e8:.4g}" == "1.7e+008"
+
+
+class TestFloatArrayFormatter:
+ def test_misc(self):
+ obj = fmt.FloatArrayFormatter(np.array([], dtype=np.float64))
+ result = obj.get_result()
+ assert len(result) == 0
+
+ def test_format(self):
+ obj = fmt.FloatArrayFormatter(np.array([12, 0], dtype=np.float64))
+ result = obj.get_result()
+ assert result[0] == " 12.0"
+ assert result[1] == " 0.0"
+
+ def test_output_display_precision_trailing_zeroes(self):
+ # Issue #20359: trimming zeros while there is no decimal point
+
+ # Happens when display precision is set to zero
+ with option_context("display.precision", 0):
+ s = Series([840.0, 4200.0])
+ expected_output = "0 840\n1 4200\ndtype: float64"
+ assert str(s) == expected_output
+
+ @pytest.mark.parametrize(
+ "value,expected",
+ [
+ ([9.4444], " 0\n0 9"),
+ ([0.49], " 0\n0 5e-01"),
+ ([10.9999], " 0\n0 11"),
+ ([9.5444, 9.6], " 0\n0 10\n1 10"),
+ ([0.46, 0.78, -9.9999], " 0\n0 5e-01\n1 8e-01\n2 -1e+01"),
+ ],
+ )
+ def test_set_option_precision(self, value, expected):
+ # Issue #30122
+ # Precision was incorrectly shown
+
+ with option_context("display.precision", 0):
+ df_value = DataFrame(value)
+ assert str(df_value) == expected
+
+ def test_output_significant_digits(self):
+ # Issue #9764
+
+ # In case default display precision changes:
+ with option_context("display.precision", 6):
+ # DataFrame example from issue #9764
+ d = DataFrame(
+ {
+ "col1": [
+ 9.999e-8,
+ 1e-7,
+ 1.0001e-7,
+ 2e-7,
+ 4.999e-7,
+ 5e-7,
+ 5.0001e-7,
+ 6e-7,
+ 9.999e-7,
+ 1e-6,
+ 1.0001e-6,
+ 2e-6,
+ 4.999e-6,
+ 5e-6,
+ 5.0001e-6,
+ 6e-6,
+ ]
+ }
+ )
+
+ expected_output = {
+ (0, 6): " col1\n"
+ "0 9.999000e-08\n"
+ "1 1.000000e-07\n"
+ "2 1.000100e-07\n"
+ "3 2.000000e-07\n"
+ "4 4.999000e-07\n"
+ "5 5.000000e-07",
+ (1, 6): " col1\n"
+ "1 1.000000e-07\n"
+ "2 1.000100e-07\n"
+ "3 2.000000e-07\n"
+ "4 4.999000e-07\n"
+ "5 5.000000e-07",
+ (1, 8): " col1\n"
+ "1 1.000000e-07\n"
+ "2 1.000100e-07\n"
+ "3 2.000000e-07\n"
+ "4 4.999000e-07\n"
+ "5 5.000000e-07\n"
+ "6 5.000100e-07\n"
+ "7 6.000000e-07",
+ (8, 16): " col1\n"
+ "8 9.999000e-07\n"
+ "9 1.000000e-06\n"
+ "10 1.000100e-06\n"
+ "11 2.000000e-06\n"
+ "12 4.999000e-06\n"
+ "13 5.000000e-06\n"
+ "14 5.000100e-06\n"
+ "15 6.000000e-06",
+ (9, 16): " col1\n"
+ "9 0.000001\n"
+ "10 0.000001\n"
+ "11 0.000002\n"
+ "12 0.000005\n"
+ "13 0.000005\n"
+ "14 0.000005\n"
+ "15 0.000006",
+ }
+
+ for (start, stop), v in expected_output.items():
+ assert str(d[start:stop]) == v
+
+ def test_too_long(self):
+ # GH 10451
+ with option_context("display.precision", 4):
+ # need both a number > 1e6 and something that normally formats to
+ # having length > display.precision + 6
+ df = DataFrame({"x": [12345.6789]})
+ assert str(df) == " x\n0 12345.6789"
+ df = DataFrame({"x": [2e6]})
+ assert str(df) == " x\n0 2000000.0"
+ df = DataFrame({"x": [12345.6789, 2e6]})
+ assert str(df) == " x\n0 1.2346e+04\n1 2.0000e+06"
+
+
+class TestTimedelta64Formatter:
+ def test_days(self):
+ x = pd.to_timedelta([*list(range(5)), NaT], unit="D")._values
+ result = fmt._Timedelta64Formatter(x).get_result()
+ assert result[0].strip() == "0 days"
+ assert result[1].strip() == "1 days"
+
+ result = fmt._Timedelta64Formatter(x[1:2]).get_result()
+ assert result[0].strip() == "1 days"
+
+ result = fmt._Timedelta64Formatter(x).get_result()
+ assert result[0].strip() == "0 days"
+ assert result[1].strip() == "1 days"
+
+ result = fmt._Timedelta64Formatter(x[1:2]).get_result()
+ assert result[0].strip() == "1 days"
+
+ def test_days_neg(self):
+ x = pd.to_timedelta([*list(range(5)), NaT], unit="D")._values
+ result = fmt._Timedelta64Formatter(-x).get_result()
+ assert result[0].strip() == "0 days"
+ assert result[1].strip() == "-1 days"
+
+ def test_subdays(self):
+ y = pd.to_timedelta([*list(range(5)), NaT], unit="s")._values
+ result = fmt._Timedelta64Formatter(y).get_result()
+ assert result[0].strip() == "0 days 00:00:00"
+ assert result[1].strip() == "0 days 00:00:01"
+
+ def test_subdays_neg(self):
+ y = pd.to_timedelta([*list(range(5)), NaT], unit="s")._values
+ result = fmt._Timedelta64Formatter(-y).get_result()
+ assert result[0].strip() == "0 days 00:00:00"
+ assert result[1].strip() == "-1 days +23:59:59"
+
+ def test_zero(self):
+ x = pd.to_timedelta([*list(range(1)), NaT], unit="D")._values
+ result = fmt._Timedelta64Formatter(x).get_result()
+ assert result[0].strip() == "0 days"
+
+ x = pd.to_timedelta(list(range(1)), unit="D")._values
+ result = fmt._Timedelta64Formatter(x).get_result()
+ assert result[0].strip() == "0 days"
+
+
+class TestDatetime64Formatter:
+ def test_mixed(self):
+ x = Series([datetime(2013, 1, 1), datetime(2013, 1, 1, 12), NaT])._values
+ result = fmt._Datetime64Formatter(x).get_result()
+ assert result[0].strip() == "2013-01-01 00:00:00"
+ assert result[1].strip() == "2013-01-01 12:00:00"
+
+ def test_dates(self):
+ x = Series([datetime(2013, 1, 1), datetime(2013, 1, 2), NaT])._values
+ result = fmt._Datetime64Formatter(x).get_result()
+ assert result[0].strip() == "2013-01-01"
+ assert result[1].strip() == "2013-01-02"
+
+ def test_date_nanos(self):
+ x = Series([Timestamp(200)])._values
+ result = fmt._Datetime64Formatter(x).get_result()
+ assert result[0].strip() == "1970-01-01 00:00:00.000000200"
+
+ def test_dates_display(self):
+ # 10170
+ # make sure that we are consistently display date formatting
+ x = Series(date_range("20130101 09:00:00", periods=5, freq="D"))
+ x.iloc[1] = np.nan
+ result = fmt._Datetime64Formatter(x._values).get_result()
+ assert result[0].strip() == "2013-01-01 09:00:00"
+ assert result[1].strip() == "NaT"
+ assert result[4].strip() == "2013-01-05 09:00:00"
+
+ x = Series(date_range("20130101 09:00:00", periods=5, freq="s"))
+ x.iloc[1] = np.nan
+ result = fmt._Datetime64Formatter(x._values).get_result()
+ assert result[0].strip() == "2013-01-01 09:00:00"
+ assert result[1].strip() == "NaT"
+ assert result[4].strip() == "2013-01-01 09:00:04"
+
+ x = Series(date_range("20130101 09:00:00", periods=5, freq="ms"))
+ x.iloc[1] = np.nan
+ result = fmt._Datetime64Formatter(x._values).get_result()
+ assert result[0].strip() == "2013-01-01 09:00:00.000"
+ assert result[1].strip() == "NaT"
+ assert result[4].strip() == "2013-01-01 09:00:00.004"
+
+ x = Series(date_range("20130101 09:00:00", periods=5, freq="us"))
+ x.iloc[1] = np.nan
+ result = fmt._Datetime64Formatter(x._values).get_result()
+ assert result[0].strip() == "2013-01-01 09:00:00.000000"
+ assert result[1].strip() == "NaT"
+ assert result[4].strip() == "2013-01-01 09:00:00.000004"
+
+ x = Series(date_range("20130101 09:00:00", periods=5, freq="ns"))
+ x.iloc[1] = np.nan
+ result = fmt._Datetime64Formatter(x._values).get_result()
+ assert result[0].strip() == "2013-01-01 09:00:00.000000000"
+ assert result[1].strip() == "NaT"
+ assert result[4].strip() == "2013-01-01 09:00:00.000000004"
+
+ def test_datetime64formatter_yearmonth(self):
+ x = Series([datetime(2016, 1, 1), datetime(2016, 2, 2)])._values
+
+ def format_func(x):
+ return x.strftime("%Y-%m")
+
+ formatter = fmt._Datetime64Formatter(x, formatter=format_func)
+ result = formatter.get_result()
+ assert result == ["2016-01", "2016-02"]
+
+ def test_datetime64formatter_hoursecond(self):
+ x = Series(
+ pd.to_datetime(["10:10:10.100", "12:12:12.120"], format="%H:%M:%S.%f")
+ )._values
+
+ def format_func(x):
+ return x.strftime("%H:%M")
+
+ formatter = fmt._Datetime64Formatter(x, formatter=format_func)
+ result = formatter.get_result()
+ assert result == ["10:10", "12:12"]
+
+ def test_datetime64formatter_tz_ms(self):
+ x = (
+ Series(
+ np.array(["2999-01-01", "2999-01-02", "NaT"], dtype="datetime64[ms]")
+ )
+ .dt.tz_localize("US/Pacific")
+ ._values
+ )
+ result = fmt._Datetime64TZFormatter(x).get_result()
+ assert result[0].strip() == "2999-01-01 00:00:00-08:00"
+ assert result[1].strip() == "2999-01-02 00:00:00-08:00"
+
+
+class TestFormatPercentiles:
+ @pytest.mark.parametrize(
+ "percentiles, expected",
+ [
+ (
+ [0.01999, 0.02001, 0.5, 0.666666, 0.9999],
+ ["1.999%", "2.001%", "50%", "66.667%", "99.99%"],
+ ),
+ (
+ [0, 0.5, 0.02001, 0.5, 0.666666, 0.9999],
+ ["0%", "50%", "2.0%", "50%", "66.67%", "99.99%"],
+ ),
+ ([0.281, 0.29, 0.57, 0.58], ["28.1%", "29%", "57%", "58%"]),
+ ([0.28, 0.29, 0.57, 0.58], ["28%", "29%", "57%", "58%"]),
+ (
+ [0.9, 0.99, 0.999, 0.9999, 0.99999],
+ ["90%", "99%", "99.9%", "99.99%", "99.999%"],
+ ),
+ ],
+ )
+ def test_format_percentiles(self, percentiles, expected):
+ result = fmt.format_percentiles(percentiles)
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "percentiles",
+ [
+ ([0.1, np.nan, 0.5]),
+ ([-0.001, 0.1, 0.5]),
+ ([2, 0.1, 0.5]),
+ ([0.1, 0.5, "a"]),
+ ],
+ )
+ def test_error_format_percentiles(self, percentiles):
+ msg = r"percentiles should all be in the interval \[0,1\]"
+ with pytest.raises(ValueError, match=msg):
+ fmt.format_percentiles(percentiles)
+
+ def test_format_percentiles_integer_idx(self):
+ # Issue #26660
+ result = fmt.format_percentiles(np.linspace(0, 1, 10 + 1))
+ expected = [
+ "0%",
+ "10%",
+ "20%",
+ "30%",
+ "40%",
+ "50%",
+ "60%",
+ "70%",
+ "80%",
+ "90%",
+ "100%",
+ ]
+ assert result == expected
+
+
+@pytest.mark.parametrize("method", ["to_string", "to_html", "to_latex"])
+@pytest.mark.parametrize(
+ "encoding, data",
+ [(None, "abc"), ("utf-8", "abc"), ("gbk", "造成输出中文显示乱码"), ("foo", "abc")],
+)
+@pytest.mark.parametrize("filepath_or_buffer_id", ["string", "pathlike", "buffer"])
+def test_filepath_or_buffer_arg(
+ method,
+ tmp_path,
+ encoding,
+ data,
+ filepath_or_buffer_id,
+):
+ if filepath_or_buffer_id == "buffer":
+ filepath_or_buffer = StringIO()
+ elif filepath_or_buffer_id == "pathlike":
+ filepath_or_buffer = tmp_path / "foo"
+ else:
+ filepath_or_buffer = str(tmp_path / "foo")
+
+ df = DataFrame([data])
+ if method in ["to_latex"]: # uses styler implementation
+ pytest.importorskip("jinja2")
+
+ if filepath_or_buffer_id not in ["string", "pathlike"] and encoding is not None:
+ with pytest.raises(
+ ValueError, match="buf is not a file name and encoding is specified."
+ ):
+ getattr(df, method)(buf=filepath_or_buffer, encoding=encoding)
+ elif encoding == "foo":
+ with pytest.raises(LookupError, match="unknown encoding"):
+ getattr(df, method)(buf=filepath_or_buffer, encoding=encoding)
+ else:
+ expected = getattr(df, method)()
+ getattr(df, method)(buf=filepath_or_buffer, encoding=encoding)
+ encoding = encoding or "utf-8"
+ if filepath_or_buffer_id == "string":
+ with open(filepath_or_buffer, encoding=encoding) as f:
+ result = f.read()
+ elif filepath_or_buffer_id == "pathlike":
+ result = filepath_or_buffer.read_text(encoding=encoding)
+ elif filepath_or_buffer_id == "buffer":
+ result = filepath_or_buffer.getvalue()
+ assert result == expected
+ if filepath_or_buffer_id == "buffer":
+ assert not filepath_or_buffer.closed
+
+
+@pytest.mark.parametrize("method", ["to_string", "to_html", "to_latex"])
+def test_filepath_or_buffer_bad_arg_raises(float_frame, method):
+ if method in ["to_latex"]: # uses styler implementation
+ pytest.importorskip("jinja2")
+ msg = "buf is not a file name and it has no write method"
+ with pytest.raises(TypeError, match=msg):
+ getattr(float_frame, method)(buf=object())
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_ipython_compat.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_ipython_compat.py
new file mode 100644
index 0000000000000000000000000000000000000000..df202dbd7d9fb9d754af234643c1d27ed0c78414
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_ipython_compat.py
@@ -0,0 +1,91 @@
+import numpy as np
+
+import pandas._config.config as cf
+
+from pandas import (
+ DataFrame,
+ MultiIndex,
+)
+
+
+class TestTableSchemaRepr:
+ def test_publishes(self, ip):
+ ipython = ip.instance(config=ip.config)
+ df = DataFrame({"A": [1, 2]})
+ objects = [df["A"], df] # dataframe / series
+ expected_keys = [
+ {"text/plain", "application/vnd.dataresource+json"},
+ {"text/plain", "text/html", "application/vnd.dataresource+json"},
+ ]
+
+ opt = cf.option_context("display.html.table_schema", True)
+ last_obj = None
+ for obj, expected in zip(objects, expected_keys):
+ last_obj = obj
+ with cf.option_context("display.html.table_schema", True):
+ # Can't reuse opt on all systems GH#58055
+ formatted = ipython.display_formatter.format(obj)
+ assert set(formatted[0].keys()) == expected
+
+ with_latex = cf.option_context("styler.render.repr", "latex")
+
+ with opt, with_latex:
+ formatted = ipython.display_formatter.format(last_obj)
+
+ expected = {
+ "text/plain",
+ "text/html",
+ "text/latex",
+ "application/vnd.dataresource+json",
+ }
+ assert set(formatted[0].keys()) == expected
+
+ def test_publishes_not_implemented(self, ip):
+ # column MultiIndex
+ # GH#15996
+ midx = MultiIndex.from_product([["A", "B"], ["a", "b", "c"]])
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((5, len(midx))), columns=midx
+ )
+
+ opt = cf.option_context("display.html.table_schema", True)
+
+ with opt:
+ formatted = ip.instance(config=ip.config).display_formatter.format(df)
+
+ expected = {"text/plain", "text/html"}
+ assert set(formatted[0].keys()) == expected
+
+ def test_config_on(self):
+ df = DataFrame({"A": [1, 2]})
+ with cf.option_context("display.html.table_schema", True):
+ result = df._repr_data_resource_()
+
+ assert result is not None
+
+ def test_config_default_off(self):
+ df = DataFrame({"A": [1, 2]})
+ with cf.option_context("display.html.table_schema", False):
+ result = df._repr_data_resource_()
+
+ assert result is None
+
+ def test_enable_data_resource_formatter(self, ip):
+ # GH#10491
+ formatters = ip.instance(config=ip.config).display_formatter.formatters
+ mimetype = "application/vnd.dataresource+json"
+
+ with cf.option_context("display.html.table_schema", True):
+ assert "application/vnd.dataresource+json" in formatters
+ assert formatters[mimetype].enabled
+
+ # still there, just disabled
+ assert "application/vnd.dataresource+json" in formatters
+ assert not formatters[mimetype].enabled
+
+ # able to re-set
+ with cf.option_context("display.html.table_schema", True):
+ assert "application/vnd.dataresource+json" in formatters
+ assert formatters[mimetype].enabled
+ # smoke test that it works
+ ip.instance(config=ip.config).display_formatter.format(cf)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_printing.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_printing.py
new file mode 100644
index 0000000000000000000000000000000000000000..7d154235d2c4a6d3cb5ee481127221017b5e1bdc
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_printing.py
@@ -0,0 +1,172 @@
+# Note! This file is aimed specifically at pandas.io.formats.printing utility
+# functions, not the general printing of pandas objects.
+from collections.abc import Mapping
+import string
+
+import pytest
+
+import pandas._config.config as cf
+
+import pandas as pd
+
+from pandas.io.formats import printing
+
+
+@pytest.mark.parametrize(
+ "input_names, expected_names",
+ [
+ (["'a b"], "['\\'a b']"), # Escape leading quote
+ (["test's b"], "['test\\'s b']"), # Escape apostrophe
+ (["'test' b"], "['\\'test\\' b']"), # Escape surrounding quotes
+ (["test b'"], "['test b\\'']"), # Escape single quote
+ (["test\n' b"], "['test\\n\\' b']"), # Escape quotes, preserve newline
+ ],
+)
+def test_formatted_index_names(input_names, expected_names):
+ # GH#60190
+ df = pd.DataFrame({name: [1, 2, 3] for name in input_names}).set_index(input_names)
+ formatted_names = str(df.index.names)
+
+ assert formatted_names == expected_names
+
+
+def test_adjoin():
+ data = [["a", "b", "c"], ["dd", "ee", "ff"], ["ggg", "hhh", "iii"]]
+ expected = "a dd ggg\nb ee hhh\nc ff iii"
+
+ adjoined = printing.adjoin(2, *data)
+
+ assert adjoined == expected
+
+
+class MyMapping(Mapping):
+ def __getitem__(self, key):
+ return 4
+
+ def __iter__(self):
+ return iter(["a", "b"])
+
+ def __len__(self):
+ return 2
+
+
+class TestPPrintThing:
+ def test_repr_binary_type(self):
+ letters = string.ascii_letters
+ try:
+ raw = bytes(letters, encoding=cf.get_option("display.encoding"))
+ except TypeError:
+ raw = bytes(letters)
+ b = str(raw.decode("utf-8"))
+ res = printing.pprint_thing(b, quote_strings=True)
+ assert res == repr(b)
+ res = printing.pprint_thing(b, quote_strings=False)
+ assert res == b
+
+ def test_repr_obeys_max_seq_limit(self):
+ with cf.option_context("display.max_seq_items", 2000):
+ assert len(printing.pprint_thing(list(range(1000)))) > 1000
+
+ with cf.option_context("display.max_seq_items", 5):
+ assert len(printing.pprint_thing(list(range(1000)))) < 100
+
+ with cf.option_context("display.max_seq_items", 1):
+ assert len(printing.pprint_thing(list(range(1000)))) < 9
+
+ def test_repr_set(self):
+ assert printing.pprint_thing({1}) == "{1}"
+
+ def test_repr_dict(self):
+ assert printing.pprint_thing({"a": 4, "b": 4}) == "{'a': 4, 'b': 4}"
+
+ def test_repr_mapping(self):
+ assert printing.pprint_thing(MyMapping()) == "{'a': 4, 'b': 4}"
+
+ def test_repr_frozenset(self):
+ assert printing.pprint_thing(frozenset([1, 2])) == "frozenset({1, 2})"
+
+
+class TestFormatBase:
+ def test_adjoin(self):
+ data = [["a", "b", "c"], ["dd", "ee", "ff"], ["ggg", "hhh", "iii"]]
+ expected = "a dd ggg\nb ee hhh\nc ff iii"
+
+ adjoined = printing.adjoin(2, *data)
+
+ assert adjoined == expected
+
+ def test_adjoin_unicode(self):
+ data = [["あ", "b", "c"], ["dd", "ええ", "ff"], ["ggg", "hhh", "いいい"]]
+ expected = "あ dd ggg\nb ええ hhh\nc ff いいい"
+ adjoined = printing.adjoin(2, *data)
+ assert adjoined == expected
+
+ adj = printing._EastAsianTextAdjustment()
+
+ expected = """あ dd ggg
+b ええ hhh
+c ff いいい"""
+
+ adjoined = adj.adjoin(2, *data)
+ assert adjoined == expected
+ cols = adjoined.split("\n")
+ assert adj.len(cols[0]) == 13
+ assert adj.len(cols[1]) == 13
+ assert adj.len(cols[2]) == 16
+
+ expected = """あ dd ggg
+b ええ hhh
+c ff いいい"""
+
+ adjoined = adj.adjoin(7, *data)
+ assert adjoined == expected
+ cols = adjoined.split("\n")
+ assert adj.len(cols[0]) == 23
+ assert adj.len(cols[1]) == 23
+ assert adj.len(cols[2]) == 26
+
+ def test_justify(self):
+ adj = printing._EastAsianTextAdjustment()
+
+ def just(x, *args, **kwargs):
+ # wrapper to test single str
+ return adj.justify([x], *args, **kwargs)[0]
+
+ assert just("abc", 5, mode="left") == "abc "
+ assert just("abc", 5, mode="center") == " abc "
+ assert just("abc", 5, mode="right") == " abc"
+ assert just("abc", 5, mode="left") == "abc "
+ assert just("abc", 5, mode="center") == " abc "
+ assert just("abc", 5, mode="right") == " abc"
+
+ assert just("パンダ", 5, mode="left") == "パンダ"
+ assert just("パンダ", 5, mode="center") == "パンダ"
+ assert just("パンダ", 5, mode="right") == "パンダ"
+
+ assert just("パンダ", 10, mode="left") == "パンダ "
+ assert just("パンダ", 10, mode="center") == " パンダ "
+ assert just("パンダ", 10, mode="right") == " パンダ"
+
+ def test_east_asian_len(self):
+ adj = printing._EastAsianTextAdjustment()
+
+ assert adj.len("abc") == 3
+ assert adj.len("abc") == 3
+
+ assert adj.len("パンダ") == 6
+ assert adj.len("パンダ") == 5
+ assert adj.len("パンダpanda") == 11
+ assert adj.len("パンダpanda") == 10
+
+ def test_ambiguous_width(self):
+ adj = printing._EastAsianTextAdjustment()
+ assert adj.len("¡¡ab") == 4
+
+ with cf.option_context("display.unicode.ambiguous_as_wide", True):
+ adj = printing._EastAsianTextAdjustment()
+ assert adj.len("¡¡ab") == 6
+
+ data = [["あ", "b", "c"], ["dd", "ええ", "ff"], ["ggg", "¡¡ab", "いいい"]]
+ expected = "あ dd ggg \nb ええ ¡¡ab\nc ff いいい"
+ adjoined = adj.adjoin(2, *data)
+ assert adjoined == expected
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_csv.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_csv.py
new file mode 100644
index 0000000000000000000000000000000000000000..f989bd153ed332b40d9222c312fe5e9a024b8488
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_csv.py
@@ -0,0 +1,886 @@
+import io
+import os
+import sys
+from zipfile import ZipFile
+
+from _csv import Error
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ compat,
+)
+import pandas._testing as tm
+
+
+class TestToCSV:
+ def test_to_csv_with_single_column(self, temp_file):
+ # see gh-18676, https://bugs.python.org/issue32255
+ #
+ # Python's CSV library adds an extraneous '""'
+ # before the newline when the NaN-value is in
+ # the first row. Otherwise, only the newline
+ # character is added. This behavior is inconsistent
+ # and was patched in https://bugs.python.org/pull_request4672.
+ df1 = DataFrame([None, 1])
+ expected1 = """\
+""
+1.0
+"""
+ df1.to_csv(temp_file, header=None, index=None)
+ with open(temp_file, encoding="utf-8") as f:
+ assert f.read() == expected1
+
+ df2 = DataFrame([1, None])
+ expected2 = """\
+1.0
+""
+"""
+ df2.to_csv(temp_file, header=None, index=None)
+ with open(temp_file, encoding="utf-8") as f:
+ assert f.read() == expected2
+
+ def test_to_csv_default_encoding(self, temp_file):
+ # GH17097
+ df = DataFrame({"col": ["AAAAA", "ÄÄÄÄÄ", "ßßßßß", "聞聞聞聞聞"]})
+
+ # the default to_csv encoding is uft-8.
+ df.to_csv(temp_file)
+ tm.assert_frame_equal(pd.read_csv(temp_file, index_col=0), df)
+
+ def test_to_csv_quotechar(self, temp_file):
+ df = DataFrame({"col": [1, 2]})
+ expected = """\
+"","col"
+"0","1"
+"1","2"
+"""
+
+ df.to_csv(temp_file, quoting=1) # 1=QUOTE_ALL
+ with open(temp_file, encoding="utf-8") as f:
+ assert f.read() == expected
+
+ expected = """\
+$$,$col$
+$0$,$1$
+$1$,$2$
+"""
+
+ df.to_csv(temp_file, quoting=1, quotechar="$")
+ with open(temp_file, encoding="utf-8") as f:
+ assert f.read() == expected
+
+ with pytest.raises(TypeError, match="quotechar"):
+ df.to_csv(temp_file, quoting=1, quotechar=None)
+
+ def test_to_csv_doublequote(self, temp_file):
+ df = DataFrame({"col": ['a"a', '"bb"']})
+ expected = '''\
+"","col"
+"0","a""a"
+"1","""bb"""
+'''
+
+ df.to_csv(temp_file, quoting=1, doublequote=True) # QUOTE_ALL
+ with open(temp_file, encoding="utf-8") as f:
+ assert f.read() == expected
+
+ with pytest.raises(Error, match="escapechar"):
+ df.to_csv(temp_file, doublequote=False) # no escapechar set
+
+ def test_to_csv_escapechar(self, temp_file):
+ df = DataFrame({"col": ['a"a', '"bb"']})
+ expected = """\
+"","col"
+"0","a\\"a"
+"1","\\"bb\\""
+"""
+
+ df.to_csv(temp_file, quoting=1, doublequote=False, escapechar="\\")
+ with open(temp_file, encoding="utf-8") as f:
+ assert f.read() == expected
+
+ df = DataFrame({"col": ["a,a", ",bb,"]})
+ expected = """\
+,col
+0,a\\,a
+1,\\,bb\\,
+"""
+
+ df.to_csv(temp_file, quoting=3, escapechar="\\") # QUOTE_NONE
+ with open(temp_file, encoding="utf-8") as f:
+ assert f.read() == expected
+
+ def test_csv_to_string(self):
+ df = DataFrame({"col": [1, 2]})
+ expected_rows = [",col", "0,1", "1,2"]
+ expected = tm.convert_rows_list_to_csv_str(expected_rows)
+ assert df.to_csv() == expected
+
+ def test_to_csv_decimal(self):
+ # see gh-781
+ df = DataFrame({"col1": [1], "col2": ["a"], "col3": [10.1]})
+
+ expected_rows = [",col1,col2,col3", "0,1,a,10.1"]
+ expected_default = tm.convert_rows_list_to_csv_str(expected_rows)
+ assert df.to_csv() == expected_default
+
+ expected_rows = [";col1;col2;col3", "0;1;a;10,1"]
+ expected_european_excel = tm.convert_rows_list_to_csv_str(expected_rows)
+ assert df.to_csv(decimal=",", sep=";") == expected_european_excel
+
+ expected_rows = [",col1,col2,col3", "0,1,a,10.10"]
+ expected_float_format_default = tm.convert_rows_list_to_csv_str(expected_rows)
+ assert df.to_csv(float_format="%.2f") == expected_float_format_default
+
+ expected_rows = [";col1;col2;col3", "0;1;a;10,10"]
+ expected_float_format = tm.convert_rows_list_to_csv_str(expected_rows)
+ assert (
+ df.to_csv(decimal=",", sep=";", float_format="%.2f")
+ == expected_float_format
+ )
+
+ # see gh-11553: testing if decimal is taken into account for '0.0'
+ df = DataFrame({"a": [0, 1.1], "b": [2.2, 3.3], "c": 1})
+
+ expected_rows = ["a,b,c", "0^0,2^2,1", "1^1,3^3,1"]
+ expected = tm.convert_rows_list_to_csv_str(expected_rows)
+ assert df.to_csv(index=False, decimal="^") == expected
+
+ # same but for an index
+ assert df.set_index("a").to_csv(decimal="^") == expected
+
+ # same for a multi-index
+ assert df.set_index(["a", "b"]).to_csv(decimal="^") == expected
+
+ def test_to_csv_float_format(self):
+ # testing if float_format is taken into account for the index
+ # GH 11553
+ df = DataFrame({"a": [0, 1], "b": [2.2, 3.3], "c": 1})
+
+ expected_rows = ["a,b,c", "0,2.20,1", "1,3.30,1"]
+ expected = tm.convert_rows_list_to_csv_str(expected_rows)
+ assert df.set_index("a").to_csv(float_format="%.2f") == expected
+
+ # same for a multi-index
+ assert df.set_index(["a", "b"]).to_csv(float_format="%.2f") == expected
+
+ def test_to_csv_na_rep(self):
+ # see gh-11553
+ #
+ # Testing if NaN values are correctly represented in the index.
+ df = DataFrame({"a": [0, np.nan], "b": [0, 1], "c": [2, 3]})
+ expected_rows = ["a,b,c", "0.0,0,2", "_,1,3"]
+ expected = tm.convert_rows_list_to_csv_str(expected_rows)
+
+ assert df.set_index("a").to_csv(na_rep="_") == expected
+ assert df.set_index(["a", "b"]).to_csv(na_rep="_") == expected
+
+ # now with an index containing only NaNs
+ df = DataFrame({"a": np.nan, "b": [0, 1], "c": [2, 3]})
+ expected_rows = ["a,b,c", "_,0,2", "_,1,3"]
+ expected = tm.convert_rows_list_to_csv_str(expected_rows)
+
+ assert df.set_index("a").to_csv(na_rep="_") == expected
+ assert df.set_index(["a", "b"]).to_csv(na_rep="_") == expected
+
+ # check if na_rep parameter does not break anything when no NaN
+ df = DataFrame({"a": 0, "b": [0, 1], "c": [2, 3]})
+ expected_rows = ["a,b,c", "0,0,2", "0,1,3"]
+ expected = tm.convert_rows_list_to_csv_str(expected_rows)
+
+ assert df.set_index("a").to_csv(na_rep="_") == expected
+ assert df.set_index(["a", "b"]).to_csv(na_rep="_") == expected
+
+ csv = pd.Series(["a", pd.NA, "c"]).to_csv(na_rep="ZZZZZ")
+ expected = tm.convert_rows_list_to_csv_str([",0", "0,a", "1,ZZZZZ", "2,c"])
+ assert expected == csv
+
+ def test_to_csv_na_rep_nullable_string(self, nullable_string_dtype):
+ # GH 29975
+ # Make sure full na_rep shows up when a dtype is provided
+ expected = tm.convert_rows_list_to_csv_str([",0", "0,a", "1,ZZZZZ", "2,c"])
+ csv = pd.Series(["a", pd.NA, "c"], dtype=nullable_string_dtype).to_csv(
+ na_rep="ZZZZZ"
+ )
+ assert expected == csv
+
+ def test_to_csv_date_format(self):
+ # GH 10209
+ df_sec = DataFrame({"A": pd.date_range("20130101", periods=5, freq="s")})
+ df_day = DataFrame({"A": pd.date_range("20130101", periods=5, freq="D")})
+
+ expected_rows = [
+ ",A",
+ "0,2013-01-01 00:00:00",
+ "1,2013-01-01 00:00:01",
+ "2,2013-01-01 00:00:02",
+ "3,2013-01-01 00:00:03",
+ "4,2013-01-01 00:00:04",
+ ]
+ expected_default_sec = tm.convert_rows_list_to_csv_str(expected_rows)
+ assert df_sec.to_csv() == expected_default_sec
+
+ expected_rows = [
+ ",A",
+ "0,2013-01-01 00:00:00",
+ "1,2013-01-02 00:00:00",
+ "2,2013-01-03 00:00:00",
+ "3,2013-01-04 00:00:00",
+ "4,2013-01-05 00:00:00",
+ ]
+ expected_ymdhms_day = tm.convert_rows_list_to_csv_str(expected_rows)
+ assert df_day.to_csv(date_format="%Y-%m-%d %H:%M:%S") == expected_ymdhms_day
+
+ expected_rows = [
+ ",A",
+ "0,2013-01-01",
+ "1,2013-01-01",
+ "2,2013-01-01",
+ "3,2013-01-01",
+ "4,2013-01-01",
+ ]
+ expected_ymd_sec = tm.convert_rows_list_to_csv_str(expected_rows)
+ assert df_sec.to_csv(date_format="%Y-%m-%d") == expected_ymd_sec
+
+ expected_rows = [
+ ",A",
+ "0,2013-01-01",
+ "1,2013-01-02",
+ "2,2013-01-03",
+ "3,2013-01-04",
+ "4,2013-01-05",
+ ]
+ expected_default_day = tm.convert_rows_list_to_csv_str(expected_rows)
+ assert df_day.to_csv() == expected_default_day
+ assert df_day.to_csv(date_format="%Y-%m-%d") == expected_default_day
+
+ # see gh-7791
+ #
+ # Testing if date_format parameter is taken into account
+ # for multi-indexed DataFrames.
+ df_sec["B"] = 0
+ df_sec["C"] = 1
+
+ expected_rows = ["A,B,C", "2013-01-01,0,1.0"]
+ expected_ymd_sec = tm.convert_rows_list_to_csv_str(expected_rows)
+
+ df_sec_grouped = df_sec.groupby([pd.Grouper(key="A", freq="1h"), "B"])
+ assert df_sec_grouped.mean().to_csv(date_format="%Y-%m-%d") == expected_ymd_sec
+
+ def test_to_csv_different_datetime_formats(self):
+ # GH#21734
+ df = DataFrame(
+ {
+ "date": pd.to_datetime("1970-01-01"),
+ "datetime": pd.date_range("1970-01-01", periods=2, freq="h"),
+ }
+ )
+ expected_rows = [
+ "date,datetime",
+ "1970-01-01,1970-01-01 00:00:00",
+ "1970-01-01,1970-01-01 01:00:00",
+ ]
+ expected = tm.convert_rows_list_to_csv_str(expected_rows)
+ assert df.to_csv(index=False) == expected
+
+ def test_to_csv_date_format_in_categorical(self):
+ # GH#40754
+ ser = pd.Series(pd.to_datetime(["2021-03-27", pd.NaT], format="%Y-%m-%d"))
+ ser = ser.astype("category")
+ expected = tm.convert_rows_list_to_csv_str(["0", "2021-03-27", '""'])
+ assert ser.to_csv(index=False) == expected
+
+ ser = pd.Series(
+ pd.date_range(
+ start="2021-03-27", freq="D", periods=1, tz="Europe/Berlin"
+ ).append(pd.DatetimeIndex([pd.NaT]))
+ )
+ ser = ser.astype("category")
+ assert ser.to_csv(index=False, date_format="%Y-%m-%d") == expected
+
+ def test_to_csv_float_ea_float_format(self):
+ # GH#45991
+ df = DataFrame({"a": [1.1, 2.02, pd.NA, 6.000006], "b": "c"})
+ df["a"] = df["a"].astype("Float64")
+ result = df.to_csv(index=False, float_format="%.5f")
+ expected = tm.convert_rows_list_to_csv_str(
+ ["a,b", "1.10000,c", "2.02000,c", ",c", "6.00001,c"]
+ )
+ assert result == expected
+
+ def test_to_csv_float_ea_no_float_format(self):
+ # GH#45991
+ df = DataFrame({"a": [1.1, 2.02, pd.NA, 6.000006], "b": "c"})
+ df["a"] = df["a"].astype("Float64")
+ result = df.to_csv(index=False)
+ expected = tm.convert_rows_list_to_csv_str(
+ ["a,b", "1.1,c", "2.02,c", ",c", "6.000006,c"]
+ )
+ assert result == expected
+
+ def test_to_csv_float_ea_nan_distinguish(self, using_nan_is_na):
+ # GH#61617, GH#65227 - to_csv should not crash when FloatingArray
+ # contains unmasked NaN (with distinguish_nan_and_na=True)
+ df = DataFrame({"a": pd.array([np.nan, pd.NA, 3.0], dtype="Float64"), "b": "c"})
+ result = df.to_csv(index=False)
+ if using_nan_is_na:
+ expected = tm.convert_rows_list_to_csv_str(["a,b", ",c", ",c", "3.0,c"])
+ else:
+ expected = tm.convert_rows_list_to_csv_str(["a,b", "nan,c", ",c", "3.0,c"])
+ assert result == expected
+
+ def test_to_csv_float_ea_nan_distinguish_series(self, using_nan_is_na):
+ # GH#65227 - Series.to_csv with FloatingArray containing both NaN and NA
+ ser = pd.Series((1, pd.NA, 0), index=["a", "b", "c"], dtype="Float64", name="x")
+ ser = ser / ser
+ result = ser.to_csv()
+ if using_nan_is_na:
+ expected = tm.convert_rows_list_to_csv_str([",x", "a,1.0", "b,", "c,"])
+ else:
+ expected = tm.convert_rows_list_to_csv_str([",x", "a,1.0", "b,", "c,nan"])
+ assert result == expected
+
+ def test_to_csv_multi_index(self):
+ # see gh-6618
+ df = DataFrame([1], columns=pd.MultiIndex.from_arrays([[1], [2]]))
+
+ exp_rows = [",1", ",2", "0,1"]
+ exp = tm.convert_rows_list_to_csv_str(exp_rows)
+ assert df.to_csv() == exp
+
+ exp_rows = ["1", "2", "1"]
+ exp = tm.convert_rows_list_to_csv_str(exp_rows)
+ assert df.to_csv(index=False) == exp
+
+ df = DataFrame(
+ [1],
+ columns=pd.MultiIndex.from_arrays([[1], [2]]),
+ index=pd.MultiIndex.from_arrays([[1], [2]]),
+ )
+
+ exp_rows = [",,1", ",,2", "1,2,1"]
+ exp = tm.convert_rows_list_to_csv_str(exp_rows)
+ assert df.to_csv() == exp
+
+ exp_rows = ["1", "2", "1"]
+ exp = tm.convert_rows_list_to_csv_str(exp_rows)
+ assert df.to_csv(index=False) == exp
+
+ df = DataFrame([1], columns=pd.MultiIndex.from_arrays([["foo"], ["bar"]]))
+
+ exp_rows = [",foo", ",bar", "0,1"]
+ exp = tm.convert_rows_list_to_csv_str(exp_rows)
+ assert df.to_csv() == exp
+
+ exp_rows = ["foo", "bar", "1"]
+ exp = tm.convert_rows_list_to_csv_str(exp_rows)
+ assert df.to_csv(index=False) == exp
+
+ @pytest.mark.parametrize(
+ "ind,expected",
+ [
+ (
+ pd.MultiIndex(levels=[[1.0]], codes=[[0]], names=["x"]),
+ "x,data\n1.0,1\n",
+ ),
+ (
+ pd.MultiIndex(
+ levels=[[1.0], [2.0]], codes=[[0], [0]], names=["x", "y"]
+ ),
+ "x,y,data\n1.0,2.0,1\n",
+ ),
+ ],
+ )
+ def test_to_csv_single_level_multi_index(self, ind, expected, frame_or_series):
+ # see gh-19589
+ obj = frame_or_series(pd.Series([1], ind, name="data"))
+
+ result = obj.to_csv(lineterminator="\n", header=True)
+ assert result == expected
+
+ def test_to_csv_string_array_ascii(self, temp_file):
+ # GH 10813
+ str_array = [{"names": ["foo", "bar"]}, {"names": ["baz", "qux"]}]
+ df = DataFrame(str_array)
+ expected_ascii = """\
+,names
+0,"['foo', 'bar']"
+1,"['baz', 'qux']"
+"""
+ df.to_csv(temp_file, encoding="ascii")
+ with open(temp_file, encoding="utf-8") as f:
+ assert f.read() == expected_ascii
+
+ def test_to_csv_string_array_utf8(self, temp_file):
+ # GH 10813
+ str_array = [{"names": ["foo", "bar"]}, {"names": ["baz", "qux"]}]
+ df = DataFrame(str_array)
+ expected_utf8 = """\
+,names
+0,"['foo', 'bar']"
+1,"['baz', 'qux']"
+"""
+ df.to_csv(temp_file, encoding="utf-8")
+ with open(temp_file, encoding="utf-8") as f:
+ assert f.read() == expected_utf8
+
+ def test_to_csv_string_with_lf(self, temp_file):
+ # GH 20353
+ data = {"int": [1, 2, 3], "str_lf": ["abc", "d\nef", "g\nh\n\ni"]}
+ df = DataFrame(data)
+
+ # case 1: The default line terminator(=os.linesep)(PR 21406)
+ os_linesep = os.linesep.encode("utf-8")
+ expected_noarg = (
+ b"int,str_lf"
+ + os_linesep
+ + b"1,abc"
+ + os_linesep
+ + b'2,"d\nef"'
+ + os_linesep
+ + b'3,"g\nh\n\ni"'
+ + os_linesep
+ )
+ df.to_csv(temp_file, index=False)
+ with open(temp_file, "rb") as f:
+ assert f.read() == expected_noarg
+
+ # case 2: LF as line terminator
+ expected_lf = b'int,str_lf\n1,abc\n2,"d\nef"\n3,"g\nh\n\ni"\n'
+ df.to_csv(temp_file, lineterminator="\n", index=False)
+ with open(temp_file, "rb") as f:
+ assert f.read() == expected_lf
+
+ # case 3: CRLF as line terminator
+ # 'lineterminator' should not change inner element
+ expected_crlf = b'int,str_lf\r\n1,abc\r\n2,"d\nef"\r\n3,"g\nh\n\ni"\r\n'
+ df.to_csv(temp_file, lineterminator="\r\n", index=False)
+ with open(temp_file, "rb") as f:
+ assert f.read() == expected_crlf
+
+ def test_to_csv_string_with_crlf(self, temp_file):
+ # GH 20353
+ data = {"int": [1, 2, 3], "str_crlf": ["abc", "d\r\nef", "g\r\nh\r\n\r\ni"]}
+ df = DataFrame(data)
+ # case 1: The default line terminator(=os.linesep)(PR 21406)
+ os_linesep = os.linesep.encode("utf-8")
+ expected_noarg = (
+ b"int,str_crlf"
+ + os_linesep
+ + b"1,abc"
+ + os_linesep
+ + b'2,"d\r\nef"'
+ + os_linesep
+ + b'3,"g\r\nh\r\n\r\ni"'
+ + os_linesep
+ )
+ df.to_csv(temp_file, index=False)
+ with open(temp_file, "rb") as f:
+ assert f.read() == expected_noarg
+
+ # case 2: LF as line terminator
+ expected_lf = b'int,str_crlf\n1,abc\n2,"d\r\nef"\n3,"g\r\nh\r\n\r\ni"\n'
+ df.to_csv(temp_file, lineterminator="\n", index=False)
+ with open(temp_file, "rb") as f:
+ assert f.read() == expected_lf
+
+ # case 3: CRLF as line terminator
+ # 'lineterminator' should not change inner element
+ expected_crlf = (
+ b'int,str_crlf\r\n1,abc\r\n2,"d\r\nef"\r\n3,"g\r\nh\r\n\r\ni"\r\n'
+ )
+ df.to_csv(temp_file, lineterminator="\r\n", index=False)
+ with open(temp_file, "rb") as f:
+ assert f.read() == expected_crlf
+
+ def test_to_csv_stdout_file(self, capsys):
+ # GH 21561
+ df = DataFrame([["foo", "bar"], ["baz", "qux"]], columns=["name_1", "name_2"])
+ expected_rows = [",name_1,name_2", "0,foo,bar", "1,baz,qux"]
+ expected_ascii = tm.convert_rows_list_to_csv_str(expected_rows)
+
+ df.to_csv(sys.stdout, encoding="ascii")
+ captured = capsys.readouterr()
+
+ assert captured.out == expected_ascii
+ assert not sys.stdout.closed
+
+ @pytest.mark.xfail(
+ compat.is_platform_windows(),
+ reason=(
+ "Especially in Windows, file stream should not be passed"
+ "to csv writer without newline='' option."
+ "(https://docs.python.org/3/library/csv.html#csv.writer)"
+ ),
+ )
+ def test_to_csv_write_to_open_file(self, temp_file):
+ # GH 21696
+ df = DataFrame({"a": ["x", "y", "z"]})
+ expected = """\
+manual header
+x
+y
+z
+"""
+ with open(temp_file, "w", encoding="utf-8") as f:
+ f.write("manual header\n")
+ df.to_csv(f, header=None, index=None)
+ with open(temp_file, encoding="utf-8") as f:
+ assert f.read() == expected
+
+ def test_to_csv_write_to_open_file_with_newline_py3(self, temp_file):
+ # see gh-21696
+ # see gh-20353
+ df = DataFrame({"a": ["x", "y", "z"]})
+ expected_rows = ["x", "y", "z"]
+ expected = "manual header\n" + tm.convert_rows_list_to_csv_str(expected_rows)
+
+ with open(temp_file, "w", newline="", encoding="utf-8") as f:
+ f.write("manual header\n")
+ df.to_csv(f, header=None, index=None)
+
+ with open(temp_file, "rb") as f:
+ assert f.read() == bytes(expected, "utf-8")
+
+ @pytest.mark.parametrize("to_infer", [True, False])
+ @pytest.mark.parametrize("read_infer", [True, False])
+ def test_to_csv_compression(
+ self,
+ compression_only,
+ read_infer,
+ to_infer,
+ compression_to_extension,
+ temp_file,
+ ):
+ # see gh-15008
+ compression = compression_only
+
+ df = DataFrame({"A": [1]})
+
+ to_compression = "infer" if to_infer else compression
+ read_compression = "infer" if read_infer else compression
+
+ path_ext = str(temp_file) + "." + compression_to_extension[compression]
+ df.to_csv(path_ext, compression=to_compression)
+ result = pd.read_csv(path_ext, index_col=0, compression=read_compression)
+ tm.assert_frame_equal(result, df)
+
+ def test_to_csv_compression_dict(self, compression_only, temp_file):
+ # GH 26023
+ method = compression_only
+ df = DataFrame({"ABC": [1]})
+ extension = {
+ "gzip": "gz",
+ "zstd": "zst",
+ }.get(method, method)
+
+ path = str(temp_file) + "." + extension
+ df.to_csv(path, compression={"method": method})
+ read_df = pd.read_csv(path, index_col=0)
+ tm.assert_frame_equal(read_df, df)
+
+ def test_to_csv_compression_dict_no_method_raises(self, temp_file):
+ # GH 26023
+ df = DataFrame({"ABC": [1]})
+ compression = {"some_option": True}
+ msg = "must have key 'method'"
+
+ with pytest.raises(ValueError, match=msg):
+ df.to_csv(temp_file, compression=compression)
+
+ @pytest.mark.parametrize("compression", ["zip", "infer"])
+ @pytest.mark.parametrize("archive_name", ["test_to_csv.csv", "test_to_csv.zip"])
+ def test_to_csv_zip_arguments(self, compression, archive_name, temp_file):
+ # GH 26023
+ df = DataFrame({"ABC": [1]})
+
+ path = str(temp_file) + ".zip"
+ df.to_csv(
+ path, compression={"method": compression, "archive_name": archive_name}
+ )
+ with ZipFile(path) as zp:
+ assert len(zp.filelist) == 1
+ archived_file = zp.filelist[0].filename
+ assert archived_file == archive_name
+
+ @pytest.mark.parametrize(
+ "filename,expected_arcname",
+ [
+ ("archive.csv", "archive.csv"),
+ ("archive.tsv", "archive.tsv"),
+ ("archive.csv.zip", "archive.csv"),
+ ("archive.tsv.zip", "archive.tsv"),
+ ("archive.zip", "archive"),
+ ],
+ )
+ def test_to_csv_zip_infer_name(self, tmp_path, filename, expected_arcname):
+ # GH 39465
+ df = DataFrame({"ABC": [1]})
+ path = tmp_path / filename
+ df.to_csv(path, compression="zip")
+ with ZipFile(path) as zp:
+ assert len(zp.filelist) == 1
+ archived_file = zp.filelist[0].filename
+ assert archived_file == expected_arcname
+
+ @pytest.mark.parametrize("df_new_type", ["Int64"])
+ def test_to_csv_na_rep_long_string(self, df_new_type):
+ # see gh-25099
+ df = DataFrame({"c": [pd.NA] * 3})
+ df = df.astype(df_new_type)
+ expected_rows = ["c", "mynull", "mynull", "mynull"]
+ expected = tm.convert_rows_list_to_csv_str(expected_rows)
+
+ result = df.to_csv(index=False, na_rep="mynull", encoding="ascii")
+
+ assert expected == result
+
+ def test_to_csv_timedelta_precision(self):
+ # GH 6783
+ s = pd.Series([1, 1]).astype("timedelta64[ns]")
+ buf = io.StringIO()
+ s.to_csv(buf)
+ result = buf.getvalue()
+ expected_rows = [
+ ",0",
+ "0,0 days 00:00:00.000000001",
+ "1,0 days 00:00:00.000000001",
+ ]
+ expected = tm.convert_rows_list_to_csv_str(expected_rows)
+ assert result == expected
+
+ def test_na_rep_truncated(self):
+ # https://github.com/pandas-dev/pandas/issues/31447
+ result = pd.Series(range(8, 12)).to_csv(na_rep="-")
+ expected = tm.convert_rows_list_to_csv_str([",0", "0,8", "1,9", "2,10", "3,11"])
+ assert result == expected
+
+ result = pd.Series([True, False]).to_csv(na_rep="nan")
+ expected = tm.convert_rows_list_to_csv_str([",0", "0,True", "1,False"])
+ assert result == expected
+
+ result = pd.Series([1.1, 2.2]).to_csv(na_rep=".")
+ expected = tm.convert_rows_list_to_csv_str([",0", "0,1.1", "1,2.2"])
+ assert result == expected
+
+ @pytest.mark.parametrize("errors", ["surrogatepass", "ignore", "replace"])
+ def test_to_csv_errors(self, errors, temp_file):
+ # GH 22610
+ data = ["\ud800foo"]
+ ser = pd.Series(data, index=Index(data, dtype=object), dtype=object)
+
+ ser.to_csv(temp_file, errors=errors)
+ # No use in reading back the data as it is not the same anymore
+ # due to the error handling
+
+ @pytest.mark.parametrize("mode", ["wb", "w"])
+ def test_to_csv_binary_handle(self, mode, temp_file):
+ """
+ Binary file objects should work (if 'mode' contains a 'b') or even without
+ it in most cases.
+
+ GH 35058 and GH 19827
+ """
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+
+ with open(temp_file, mode="w+b") as handle:
+ df.to_csv(handle, mode=mode)
+ tm.assert_frame_equal(df, pd.read_csv(temp_file, index_col=0))
+
+ @pytest.mark.parametrize("mode", ["wb", "w"])
+ def test_to_csv_encoding_binary_handle(self, mode, temp_file):
+ """
+ Binary file objects should honor a specified encoding.
+
+ GH 23854 and GH 13068 with binary handles
+ """
+ # example from GH 23854
+ content = "a, b, 🐟".encode("utf-8-sig")
+ buffer = io.BytesIO(content)
+ df = pd.read_csv(buffer, encoding="utf-8-sig")
+
+ buffer = io.BytesIO()
+ df.to_csv(buffer, mode=mode, encoding="utf-8-sig", index=False)
+ buffer.seek(0) # tests whether file handle wasn't closed
+ assert buffer.getvalue().startswith(content)
+
+ # example from GH 13068
+ with open(temp_file, "w+b") as handle:
+ DataFrame().to_csv(handle, mode=mode, encoding="utf-8-sig")
+
+ handle.seek(0)
+ assert handle.read().startswith(b'\xef\xbb\xbf""')
+
+
+def test_to_csv_iterative_compression_name(compression, temp_file):
+ # GH 38714
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ df.to_csv(temp_file, compression=compression, chunksize=1)
+ tm.assert_frame_equal(
+ pd.read_csv(temp_file, compression=compression, index_col=0), df
+ )
+
+
+def test_to_csv_iterative_compression_buffer(compression):
+ # GH 38714
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ with io.BytesIO() as buffer:
+ df.to_csv(buffer, compression=compression, chunksize=1)
+ buffer.seek(0)
+ tm.assert_frame_equal(
+ pd.read_csv(buffer, compression=compression, index_col=0), df
+ )
+ assert not buffer.closed
+
+
+def test_new_style_float_format_basic():
+ df = DataFrame({"A": [1234.56789, 9876.54321]})
+ result = df.to_csv(float_format="{:.2f}", lineterminator="\n")
+ expected = ",A\n0,1234.57\n1,9876.54\n"
+ assert result == expected
+
+
+def test_new_style_float_format_thousands():
+ df = DataFrame({"A": [1234.56789, 9876.54321]})
+ result = df.to_csv(float_format="{:,.2f}", lineterminator="\n")
+ expected = ',A\n0,"1,234.57"\n1,"9,876.54"\n'
+ assert result == expected
+
+
+def test_new_style_scientific_format():
+ df = DataFrame({"A": [0.000123, 0.000456]})
+ result = df.to_csv(float_format="{:.2e}", lineterminator="\n")
+ expected = ",A\n0,1.23e-04\n1,4.56e-04\n"
+ assert result == expected
+
+
+def test_new_style_with_nan():
+ df = DataFrame({"A": [1.23, np.nan, 4.56]})
+ result = df.to_csv(float_format="{:.2f}", na_rep="NA", lineterminator="\n")
+ expected = ",A\n0,1.23\n1,NA\n2,4.56\n"
+ assert result == expected
+
+
+def test_new_style_with_mixed_types():
+ df = DataFrame({"A": [1.23, 4.56], "B": ["x", "y"]})
+ result = df.to_csv(float_format="{:.2f}", lineterminator="\n")
+ expected = ",A,B\n0,1.23,x\n1,4.56,y\n"
+ assert result == expected
+
+
+def test_new_style_with_mixed_types_in_column():
+ df = DataFrame({"A": [1.23, "text", 4.56]})
+ result = df.to_csv(float_format="{:.2f}", lineterminator="\n")
+ expected = ",A\n0,1.23\n1,text\n2,4.56\n"
+ assert result == expected
+
+
+def test_invalid_new_style_format_missing_brace():
+ df = DataFrame({"A": [1.23]})
+ with pytest.raises(ValueError, match="Invalid new-style format string '{:.2f"):
+ df.to_csv(float_format="{:.2f")
+
+
+def test_invalid_new_style_format_specifier():
+ df = DataFrame({"A": [1.23]})
+ with pytest.raises(ValueError, match="Invalid new-style format string '{:.2z}'"):
+ df.to_csv(float_format="{:.2z}")
+
+
+def test_old_style_format_compatibility():
+ df = DataFrame({"A": [1234.56789, 9876.54321]})
+ result = df.to_csv(float_format="%.2f", lineterminator="\n")
+ expected = ",A\n0,1234.57\n1,9876.54\n"
+ assert result == expected
+
+
+def test_callable_float_format_compatibility():
+ df = DataFrame({"A": [1234.56789, 9876.54321]})
+ result = df.to_csv(float_format=lambda x: f"{x:,.2f}", lineterminator="\n")
+ expected = ',A\n0,"1,234.57"\n1,"9,876.54"\n'
+ assert result == expected
+
+
+def test_no_float_format():
+ df = DataFrame({"A": [1.23, 4.56]})
+ result = df.to_csv(float_format=None, lineterminator="\n")
+ expected = ",A\n0,1.23\n1,4.56\n"
+ assert result == expected
+
+
+def test_large_numbers():
+ df = DataFrame({"A": [1e308, 2e308]})
+ result = df.to_csv(float_format="{:.2e}", lineterminator="\n")
+ expected = ",A\n0,1.00e+308\n1,inf\n"
+ assert result == expected
+
+
+def test_zero_and_negative():
+ df = DataFrame({"A": [0.0, -1.23456]})
+ result = df.to_csv(float_format="{:+.2f}", lineterminator="\n")
+ expected = ",A\n0,+0.00\n1,-1.23\n"
+ assert result == expected
+
+
+def test_unicode_format():
+ df = DataFrame({"A": [1.23, 4.56]})
+ result = df.to_csv(float_format="{:.2f}€", encoding="utf-8", lineterminator="\n")
+ expected = ",A\n0,1.23€\n1,4.56€\n"
+ assert result == expected
+
+
+def test_empty_dataframe():
+ df = DataFrame({"A": []})
+ result = df.to_csv(float_format="{:.2f}", lineterminator="\n")
+ expected = ",A\n"
+ assert result == expected
+
+
+def test_multi_column_float():
+ df = DataFrame({"A": [1.23, 4.56], "B": [7.89, 0.12]})
+ result = df.to_csv(float_format="{:.2f}", lineterminator="\n")
+ expected = ",A,B\n0,1.23,7.89\n1,4.56,0.12\n"
+ assert result == expected
+
+
+def test_invalid_float_format_type():
+ df = DataFrame({"A": [1.23]})
+ with pytest.raises(ValueError, match="float_format must be a string or callable"):
+ df.to_csv(float_format=123)
+
+
+def test_new_style_with_inf():
+ df = DataFrame({"A": [1.23, np.inf, -np.inf]})
+ result = df.to_csv(float_format="{:.2f}", na_rep="NA", lineterminator="\n")
+ expected = ",A\n0,1.23\n1,inf\n2,-inf\n"
+ assert result == expected
+
+
+def test_new_style_with_precision_edge():
+ df = DataFrame({"A": [1.23456789]})
+ result = df.to_csv(float_format="{:.10f}", lineterminator="\n")
+ expected = ",A\n0,1.2345678900\n"
+ assert result == expected
+
+
+def test_new_style_with_template():
+ df = DataFrame({"A": [1234.56789]})
+ result = df.to_csv(float_format="Value: {:,.2f}", lineterminator="\n")
+ expected = ',A\n0,"Value: 1,234.57"\n'
+ assert result == expected
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_excel.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_excel.py
new file mode 100644
index 0000000000000000000000000000000000000000..babbfcb46a6fc3969ee48372da5d1ab528d89d91
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_excel.py
@@ -0,0 +1,473 @@
+"""Tests formatting as writer-agnostic ExcelCells
+
+ExcelFormatter is tested implicitly in pandas/tests/io/excel
+"""
+
+import string
+
+import pytest
+
+from pandas.errors import CSSWarning
+
+import pandas._testing as tm
+
+from pandas.io.formats.excel import (
+ CssExcelCell,
+ CSSToExcelConverter,
+)
+
+
+@pytest.mark.parametrize(
+ "css,expected",
+ [
+ # FONT
+ # - name
+ ("font-family: foo,bar", {"font": {"name": "foo"}}),
+ ('font-family: "foo bar",baz', {"font": {"name": "foo bar"}}),
+ ("font-family: foo,\nbar", {"font": {"name": "foo"}}),
+ ("font-family: foo, bar, baz", {"font": {"name": "foo"}}),
+ ("font-family: bar, foo", {"font": {"name": "bar"}}),
+ ("font-family: 'foo bar', baz", {"font": {"name": "foo bar"}}),
+ ("font-family: 'foo \\'bar', baz", {"font": {"name": "foo 'bar"}}),
+ ('font-family: "foo \\"bar", baz', {"font": {"name": 'foo "bar'}}),
+ ('font-family: "foo ,bar", baz', {"font": {"name": "foo ,bar"}}),
+ # - family
+ ("font-family: serif", {"font": {"name": "serif", "family": 1}}),
+ ("font-family: Serif", {"font": {"name": "serif", "family": 1}}),
+ ("font-family: roman, serif", {"font": {"name": "roman", "family": 1}}),
+ ("font-family: roman, sans-serif", {"font": {"name": "roman", "family": 2}}),
+ ("font-family: roman, sans serif", {"font": {"name": "roman"}}),
+ ("font-family: roman, sansserif", {"font": {"name": "roman"}}),
+ ("font-family: roman, cursive", {"font": {"name": "roman", "family": 4}}),
+ ("font-family: roman, fantasy", {"font": {"name": "roman", "family": 5}}),
+ # - size
+ ("font-size: 1em", {"font": {"size": 12}}),
+ ("font-size: xx-small", {"font": {"size": 6}}),
+ ("font-size: x-small", {"font": {"size": 7.5}}),
+ ("font-size: small", {"font": {"size": 9.6}}),
+ ("font-size: medium", {"font": {"size": 12}}),
+ ("font-size: large", {"font": {"size": 13.5}}),
+ ("font-size: x-large", {"font": {"size": 18}}),
+ ("font-size: xx-large", {"font": {"size": 24}}),
+ ("font-size: 50%", {"font": {"size": 6}}),
+ # - bold
+ ("font-weight: 100", {"font": {"bold": False}}),
+ ("font-weight: 200", {"font": {"bold": False}}),
+ ("font-weight: 300", {"font": {"bold": False}}),
+ ("font-weight: 400", {"font": {"bold": False}}),
+ ("font-weight: normal", {"font": {"bold": False}}),
+ ("font-weight: lighter", {"font": {"bold": False}}),
+ ("font-weight: bold", {"font": {"bold": True}}),
+ ("font-weight: bolder", {"font": {"bold": True}}),
+ ("font-weight: 700", {"font": {"bold": True}}),
+ ("font-weight: 800", {"font": {"bold": True}}),
+ ("font-weight: 900", {"font": {"bold": True}}),
+ # - italic
+ ("font-style: italic", {"font": {"italic": True}}),
+ ("font-style: oblique", {"font": {"italic": True}}),
+ # - underline
+ ("text-decoration: underline", {"font": {"underline": "single"}}),
+ ("text-decoration: overline", {}),
+ ("text-decoration: none", {}),
+ # - strike
+ ("text-decoration: line-through", {"font": {"strike": True}}),
+ (
+ "text-decoration: underline line-through",
+ {"font": {"strike": True, "underline": "single"}},
+ ),
+ (
+ "text-decoration: underline; text-decoration: line-through",
+ {"font": {"strike": True}},
+ ),
+ # - color
+ ("color: red", {"font": {"color": "FF0000"}}),
+ ("color: #ff0000", {"font": {"color": "FF0000"}}),
+ ("color: #f0a", {"font": {"color": "FF00AA"}}),
+ # - shadow
+ ("text-shadow: none", {"font": {"shadow": False}}),
+ ("text-shadow: 0px -0em 0px #CCC", {"font": {"shadow": False}}),
+ ("text-shadow: 0px -0em 0px #999", {"font": {"shadow": False}}),
+ ("text-shadow: 0px -0em 0px", {"font": {"shadow": False}}),
+ ("text-shadow: 2px -0em 0px #CCC", {"font": {"shadow": True}}),
+ ("text-shadow: 0px -2em 0px #CCC", {"font": {"shadow": True}}),
+ ("text-shadow: 0px -0em 2px #CCC", {"font": {"shadow": True}}),
+ ("text-shadow: 0px -0em 2px", {"font": {"shadow": True}}),
+ ("text-shadow: 0px -2em", {"font": {"shadow": True}}),
+ # FILL
+ # - color, fillType
+ (
+ "background-color: red",
+ {"fill": {"fgColor": "FF0000", "patternType": "solid"}},
+ ),
+ (
+ "background-color: #ff0000",
+ {"fill": {"fgColor": "FF0000", "patternType": "solid"}},
+ ),
+ (
+ "background-color: #f0a",
+ {"fill": {"fgColor": "FF00AA", "patternType": "solid"}},
+ ),
+ # BORDER
+ # - style
+ (
+ "border-style: solid",
+ {
+ "border": {
+ "top": {"style": "medium"},
+ "bottom": {"style": "medium"},
+ "left": {"style": "medium"},
+ "right": {"style": "medium"},
+ }
+ },
+ ),
+ (
+ "border-style: solid; border-width: thin",
+ {
+ "border": {
+ "top": {"style": "thin"},
+ "bottom": {"style": "thin"},
+ "left": {"style": "thin"},
+ "right": {"style": "thin"},
+ }
+ },
+ ),
+ (
+ "border-top-style: solid; border-top-width: thin",
+ {"border": {"top": {"style": "thin"}}},
+ ),
+ (
+ "border-top-style: solid; border-top-width: 1pt",
+ {"border": {"top": {"style": "thin"}}},
+ ),
+ ("border-top-style: solid", {"border": {"top": {"style": "medium"}}}),
+ (
+ "border-top-style: solid; border-top-width: medium",
+ {"border": {"top": {"style": "medium"}}},
+ ),
+ (
+ "border-top-style: solid; border-top-width: 2pt",
+ {"border": {"top": {"style": "medium"}}},
+ ),
+ (
+ "border-top-style: solid; border-top-width: thick",
+ {"border": {"top": {"style": "thick"}}},
+ ),
+ (
+ "border-top-style: solid; border-top-width: 4pt",
+ {"border": {"top": {"style": "thick"}}},
+ ),
+ (
+ "border-top-style: solid; border-top-width: none",
+ {"border": {"top": {"style": "none"}}},
+ ),
+ (
+ "border-top-style: solid; border-top-width: 0.000001pt",
+ {"border": {"top": {"style": "none"}}},
+ ),
+ (
+ "border-top-style: dotted",
+ {"border": {"top": {"style": "mediumDashDotDot"}}},
+ ),
+ (
+ "border-top-style: dotted; border-top-width: thin",
+ {"border": {"top": {"style": "dotted"}}},
+ ),
+ ("border-top-style: dashed", {"border": {"top": {"style": "mediumDashed"}}}),
+ (
+ "border-top-style: dashed; border-top-width: thin",
+ {"border": {"top": {"style": "dashed"}}},
+ ),
+ ("border-top-style: double", {"border": {"top": {"style": "double"}}}),
+ # - color
+ (
+ "border-style: solid; border-color: #0000ff",
+ {
+ "border": {
+ "top": {"style": "medium", "color": "0000FF"},
+ "right": {"style": "medium", "color": "0000FF"},
+ "bottom": {"style": "medium", "color": "0000FF"},
+ "left": {"style": "medium", "color": "0000FF"},
+ }
+ },
+ ),
+ (
+ "border-top-style: double; border-top-color: blue",
+ {"border": {"top": {"style": "double", "color": "0000FF"}}},
+ ),
+ (
+ "border-top-style: solid; border-top-color: #06c",
+ {"border": {"top": {"style": "medium", "color": "0066CC"}}},
+ ),
+ (
+ "border-top-color: blue",
+ {"border": {"top": {"color": "0000FF", "style": "none"}}},
+ ),
+ (
+ "border-top-style: slantDashDot; border-top-color: blue",
+ {"border": {"top": {"style": "slantDashDot", "color": "0000FF"}}},
+ ),
+ # ALIGNMENT
+ # - horizontal
+ ("text-align: center", {"alignment": {"horizontal": "center"}}),
+ ("text-align: left", {"alignment": {"horizontal": "left"}}),
+ ("text-align: right", {"alignment": {"horizontal": "right"}}),
+ ("text-align: justify", {"alignment": {"horizontal": "justify"}}),
+ # - vertical
+ ("vertical-align: top", {"alignment": {"vertical": "top"}}),
+ ("vertical-align: text-top", {"alignment": {"vertical": "top"}}),
+ ("vertical-align: middle", {"alignment": {"vertical": "center"}}),
+ ("vertical-align: bottom", {"alignment": {"vertical": "bottom"}}),
+ ("vertical-align: text-bottom", {"alignment": {"vertical": "bottom"}}),
+ # - wrap_text
+ ("white-space: nowrap", {"alignment": {"wrap_text": False}}),
+ ("white-space: pre", {"alignment": {"wrap_text": False}}),
+ ("white-space: pre-line", {"alignment": {"wrap_text": False}}),
+ ("white-space: normal", {"alignment": {"wrap_text": True}}),
+ # NUMBER FORMAT
+ ("number-format: 0%", {"number_format": {"format_code": "0%"}}),
+ (
+ "number-format: 0§[Red](0)§-§@;",
+ {"number_format": {"format_code": "0;[red](0);-;@"}}, # GH 46152
+ ),
+ ],
+)
+def test_css_to_excel(css, expected):
+ convert = CSSToExcelConverter()
+ assert expected == convert(css)
+
+
+def test_css_to_excel_multiple():
+ convert = CSSToExcelConverter()
+ actual = convert(
+ """
+ font-weight: bold;
+ text-decoration: underline;
+ color: red;
+ border-width: thin;
+ text-align: center;
+ vertical-align: top;
+ unused: something;
+ """
+ )
+ assert {
+ "font": {"bold": True, "underline": "single", "color": "FF0000"},
+ "border": {
+ "top": {"style": "thin"},
+ "right": {"style": "thin"},
+ "bottom": {"style": "thin"},
+ "left": {"style": "thin"},
+ },
+ "alignment": {"horizontal": "center", "vertical": "top"},
+ } == actual
+
+
+@pytest.mark.parametrize(
+ "css",
+ [
+ "border-top-style: unhandled-border-style",
+ "border-style: another-unhandled-style",
+ ],
+)
+def test_css_to_excel_unhandled_border_style_warns(css):
+ """Test that unhandled border styles raise a CSSWarning."""
+ convert = CSSToExcelConverter()
+ with tm.assert_produces_warning(CSSWarning, match="Unhandled border style format"):
+ convert(css)
+
+
+@pytest.mark.parametrize(
+ "css,inherited,expected",
+ [
+ ("font-weight: bold", "", {"font": {"bold": True}}),
+ ("", "font-weight: bold", {"font": {"bold": True}}),
+ (
+ "font-weight: bold",
+ "font-style: italic",
+ {"font": {"bold": True, "italic": True}},
+ ),
+ ("font-style: normal", "font-style: italic", {"font": {"italic": False}}),
+ ("font-style: inherit", "", {}),
+ (
+ "font-style: normal; font-style: inherit",
+ "font-style: italic",
+ {"font": {"italic": True}},
+ ),
+ ],
+)
+def test_css_to_excel_inherited(css, inherited, expected):
+ convert = CSSToExcelConverter(inherited)
+ assert expected == convert(css)
+
+
+@pytest.mark.parametrize(
+ "input_color,output_color",
+ (
+ list(CSSToExcelConverter.NAMED_COLORS.items())
+ + [("#" + rgb, rgb) for rgb in CSSToExcelConverter.NAMED_COLORS.values()]
+ + [("#F0F", "FF00FF"), ("#ABC", "AABBCC")]
+ ),
+)
+def test_css_to_excel_good_colors(input_color, output_color):
+ # see gh-18392
+ css = (
+ f"border-top-color: {input_color}; "
+ f"border-right-color: {input_color}; "
+ f"border-bottom-color: {input_color}; "
+ f"border-left-color: {input_color}; "
+ f"background-color: {input_color}; "
+ f"color: {input_color}"
+ )
+
+ expected = {}
+
+ expected["fill"] = {"patternType": "solid", "fgColor": output_color}
+
+ expected["font"] = {"color": output_color}
+
+ expected["border"] = {
+ k: {"color": output_color, "style": "none"}
+ for k in ("top", "right", "bottom", "left")
+ }
+
+ with tm.assert_produces_warning(None):
+ convert = CSSToExcelConverter()
+ assert expected == convert(css)
+
+
+@pytest.mark.parametrize("input_color", [None, "not-a-color"])
+def test_css_to_excel_bad_colors(input_color):
+ # see gh-18392
+ css = (
+ f"border-top-color: {input_color}; "
+ f"border-right-color: {input_color}; "
+ f"border-bottom-color: {input_color}; "
+ f"border-left-color: {input_color}; "
+ f"background-color: {input_color}; "
+ f"color: {input_color}"
+ )
+
+ expected = {}
+
+ if input_color is not None:
+ expected["fill"] = {"patternType": "solid"}
+
+ with tm.assert_produces_warning(CSSWarning, match="Unhandled color format"):
+ convert = CSSToExcelConverter()
+ assert expected == convert(css)
+
+
+@pytest.mark.parametrize("input_color", ["#", "#1234567"])
+def test_css_to_excel_invalid_color_raises(input_color):
+ """Test that invalid colors raise a ValueError."""
+ css = (
+ f"border-top-color: {input_color}; "
+ f"border-right-color: {input_color}; "
+ f"border-bottom-color: {input_color}; "
+ f"border-left-color: {input_color}; "
+ f"background-color: {input_color}; "
+ f"color: {input_color}"
+ )
+
+ convert = CSSToExcelConverter()
+ with pytest.raises(ValueError, match=f"Unexpected color {input_color}"):
+ convert(css)
+
+
+def tests_css_named_colors_valid():
+ upper_hexs = set(map(str.upper, string.hexdigits))
+ for color in CSSToExcelConverter.NAMED_COLORS.values():
+ assert len(color) == 6 and all(c in upper_hexs for c in color)
+
+
+def test_css_named_colors_from_mpl_present():
+ mpl_colors = pytest.importorskip("matplotlib.colors")
+
+ pd_colors = CSSToExcelConverter.NAMED_COLORS
+ for name, color in mpl_colors.CSS4_COLORS.items():
+ assert name in pd_colors and pd_colors[name] == color[1:]
+
+
+@pytest.mark.parametrize(
+ "styles,expected",
+ [
+ ([("color", "green"), ("color", "red")], "color: red;"),
+ ([("font-weight", "bold"), ("font-weight", "normal")], "font-weight: normal;"),
+ ([("text-align", "center"), ("TEXT-ALIGN", "right")], "text-align: right;"),
+ ],
+)
+def test_css_excel_cell_precedence(styles, expected):
+ """It applies favors latter declarations over former declarations"""
+ # See GH 47371
+ converter = CSSToExcelConverter()
+ converter._call_cached.cache_clear()
+ css_styles = {(0, 0): styles}
+ cell = CssExcelCell(
+ row=0,
+ col=0,
+ val="",
+ style=None,
+ css_styles=css_styles,
+ css_row=0,
+ css_col=0,
+ css_converter=converter,
+ )
+ converter._call_cached.cache_clear()
+
+ assert cell.style == converter(expected)
+
+
+@pytest.mark.parametrize(
+ "styles,cache_hits,cache_misses",
+ [
+ ([[("color", "green"), ("color", "red"), ("color", "green")]], 0, 1),
+ (
+ [
+ [("font-weight", "bold")],
+ [("font-weight", "normal"), ("font-weight", "bold")],
+ ],
+ 1,
+ 1,
+ ),
+ ([[("text-align", "center")], [("TEXT-ALIGN", "center")]], 1, 1),
+ (
+ [
+ [("font-weight", "bold"), ("text-align", "center")],
+ [("font-weight", "bold"), ("text-align", "left")],
+ ],
+ 0,
+ 2,
+ ),
+ (
+ [
+ [("font-weight", "bold"), ("text-align", "center")],
+ [("font-weight", "bold"), ("text-align", "left")],
+ [("font-weight", "bold"), ("text-align", "center")],
+ ],
+ 1,
+ 2,
+ ),
+ ],
+)
+def test_css_excel_cell_cache(styles, cache_hits, cache_misses):
+ """It caches unique cell styles"""
+ # See GH 47371
+ converter = CSSToExcelConverter()
+ converter._call_cached.cache_clear()
+
+ css_styles = {(0, i): _style for i, _style in enumerate(styles)}
+ for css_row, css_col in css_styles:
+ CssExcelCell(
+ row=0,
+ col=0,
+ val="",
+ style=None,
+ css_styles=css_styles,
+ css_row=css_row,
+ css_col=css_col,
+ css_converter=converter,
+ )
+ cache_info = converter._call_cached.cache_info()
+ converter._call_cached.cache_clear()
+
+ assert cache_info.hits == cache_hits
+ assert cache_info.misses == cache_misses
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_html.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_html.py
new file mode 100644
index 0000000000000000000000000000000000000000..11f6d1282bf7b1e97404d8e73aa282c2f2a04a6f
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_html.py
@@ -0,0 +1,1166 @@
+from datetime import datetime
+from io import StringIO
+import itertools
+import re
+import textwrap
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ MultiIndex,
+ get_option,
+ option_context,
+)
+
+import pandas.io.formats.format as fmt
+
+lorem_ipsum = (
+ "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod "
+ "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim "
+ "veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex "
+ "ea commodo consequat. Duis aute irure dolor in reprehenderit in "
+ "voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur "
+ "sint occaecat cupidatat non proident, sunt in culpa qui officia "
+ "deserunt mollit anim id est laborum."
+)
+
+
+def expected_html(datapath, name):
+ """
+ Read HTML file from formats data directory.
+
+ Parameters
+ ----------
+ datapath : pytest fixture
+ The datapath fixture injected into a test by pytest.
+ name : str
+ The name of the HTML file without the suffix.
+
+ Returns
+ -------
+ str : contents of HTML file.
+ """
+ filename = ".".join([name, "html"])
+ filepath = datapath("io", "formats", "data", "html", filename)
+ with open(filepath, encoding="utf-8") as f:
+ html = f.read()
+ return html.rstrip()
+
+
+@pytest.fixture(params=["mixed", "empty"])
+def biggie_df_fixture(request):
+ """Fixture for a big mixed Dataframe and an empty Dataframe"""
+ if request.param == "mixed":
+ df = DataFrame(
+ {
+ "A": np.random.default_rng(2).standard_normal(200),
+ "B": Index([f"{i}?!" for i in range(200)]),
+ },
+ index=np.arange(200),
+ )
+ df.loc[:20, "A"] = np.nan
+ df.loc[:20, "B"] = np.nan
+ return df
+ elif request.param == "empty":
+ df = DataFrame(index=np.arange(200))
+ return df
+
+
+@pytest.fixture(params=fmt.VALID_JUSTIFY_PARAMETERS)
+def justify(request):
+ return request.param
+
+
+@pytest.mark.parametrize("col_space", [30, 50])
+def test_to_html_with_col_space(col_space):
+ df = DataFrame(np.random.default_rng(2).random(size=(1, 3)))
+ # check that col_space affects HTML generation
+ # and be very brittle about it.
+ result = df.to_html(col_space=col_space)
+ hdrs = [x for x in result.split(r"\n") if re.search(r"\s]", x)]
+ assert len(hdrs) > 0
+ for h in hdrs:
+ assert "min-width" in h
+ assert str(col_space) in h
+
+
+def test_to_html_with_column_specific_col_space_raises():
+ df = DataFrame(
+ np.random.default_rng(2).random(size=(3, 3)), columns=["a", "b", "c"]
+ )
+
+ msg = (
+ "Col_space length\\(\\d+\\) should match DataFrame number of columns\\(\\d+\\)"
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.to_html(col_space=[30, 40])
+
+ with pytest.raises(ValueError, match=msg):
+ df.to_html(col_space=[30, 40, 50, 60])
+
+ msg = "unknown column"
+ with pytest.raises(ValueError, match=msg):
+ df.to_html(col_space={"a": "foo", "b": 23, "d": 34})
+
+
+def test_to_html_with_column_specific_col_space():
+ df = DataFrame(
+ np.random.default_rng(2).random(size=(3, 3)), columns=["a", "b", "c"]
+ )
+
+ result = df.to_html(col_space={"a": "2em", "b": 23})
+ hdrs = [x for x in result.split("\n") if re.search(r" \s]", x)]
+ assert 'min-width: 2em;">a ' in hdrs[1]
+ assert 'min-width: 23px;">b' in hdrs[2]
+ assert "c " in hdrs[3]
+
+ result = df.to_html(col_space=["1em", 2, 3])
+ hdrs = [x for x in result.split("\n") if re.search(r"\s]", x)]
+ assert 'min-width: 1em;">a ' in hdrs[1]
+ assert 'min-width: 2px;">b' in hdrs[2]
+ assert 'min-width: 3px;">c' in hdrs[3]
+
+
+def test_to_html_with_empty_string_label():
+ # GH 3547, to_html regards empty string labels as repeated labels
+ data = {"c1": ["a", "b"], "c2": ["a", ""], "data": [1, 2]}
+ df = DataFrame(data).set_index(["c1", "c2"])
+ result = df.to_html()
+ assert "rowspan" not in result
+
+
+@pytest.mark.parametrize(
+ "df_data,expected",
+ [
+ ({"\u03c3": np.arange(10.0)}, "unicode_1"),
+ ({"A": ["\u03c3"]}, "unicode_2"),
+ ],
+)
+def test_to_html_unicode(df_data, expected, datapath):
+ df = DataFrame(df_data)
+ expected = expected_html(datapath, expected)
+ result = df.to_html()
+ assert result == expected
+
+
+def test_to_html_encoding(float_frame, temp_file):
+ # GH 28663
+ float_frame.to_html(temp_file, encoding="gbk")
+ expected = temp_file.read_text(encoding="gbk")
+ assert float_frame.to_html() == expected
+
+
+def test_to_html_decimal(datapath):
+ # GH 12031
+ df = DataFrame({"A": [6.0, 3.1, 2.2]})
+ result = df.to_html(decimal=",")
+ expected = expected_html(datapath, "gh12031_expected_output")
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "kwargs,string,expected",
+ [
+ ({}, "", "escaped"),
+ ({"escape": False}, "bold ", "escape_disabled"),
+ ],
+)
+def test_to_html_escaped(kwargs, string, expected, datapath):
+ a = "strl2": {a: string, b: string}}
+ result = DataFrame(test_dict).to_html(**kwargs)
+ expected = expected_html(datapath, expected)
+ assert result == expected
+
+
+@pytest.mark.parametrize("index_is_named", [True, False])
+def test_to_html_multiindex_index_false(index_is_named, datapath):
+ # GH 8452
+ df = DataFrame(
+ {"a": range(2), "b": range(3, 5), "c": range(5, 7), "d": range(3, 5)}
+ )
+ df.columns = MultiIndex.from_product([["a", "b"], ["c", "d"]])
+ if index_is_named:
+ df.index = Index(df.index.values, name="idx")
+ result = df.to_html(index=False)
+ expected = expected_html(datapath, "gh8452_expected_output")
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "multi_sparse,expected",
+ [
+ (False, "multiindex_sparsify_false_multi_sparse_1"),
+ (False, "multiindex_sparsify_false_multi_sparse_2"),
+ (True, "multiindex_sparsify_1"),
+ (True, "multiindex_sparsify_2"),
+ ],
+)
+def test_to_html_multiindex_sparsify(multi_sparse, expected, datapath):
+ index = MultiIndex.from_arrays([[0, 0, 1, 1], [0, 1, 0, 1]], names=["foo", None])
+ df = DataFrame([[0, 1], [2, 3], [4, 5], [6, 7]], index=index)
+ if expected.endswith("2"):
+ df.columns = index[::2]
+ with option_context("display.multi_sparse", multi_sparse):
+ result = df.to_html()
+ expected = expected_html(datapath, expected)
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "max_rows,expected",
+ [
+ (60, "gh14882_expected_output_1"),
+ # Test that ... appears in a middle level
+ (56, "gh14882_expected_output_2"),
+ ],
+)
+def test_to_html_multiindex_odd_even_truncate(max_rows, expected, datapath):
+ # GH 14882 - Issue on truncation with odd length DataFrame
+ index = MultiIndex.from_product(
+ [[100, 200, 300], [10, 20, 30], [1, 2, 3, 4, 5, 6, 7]], names=["a", "b", "c"]
+ )
+ df = DataFrame({"n": range(len(index))}, index=index)
+ result = df.to_html(max_rows=max_rows)
+ expected = expected_html(datapath, expected)
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "df,formatters,expected",
+ [
+ (
+ DataFrame(
+ [[0, 1], [2, 3], [4, 5], [6, 7]],
+ columns=Index(["foo", None], dtype=object),
+ index=np.arange(4),
+ ),
+ {"__index__": lambda x: "abcd"[x]},
+ "index_formatter",
+ ),
+ (
+ DataFrame({"months": [datetime(2016, 1, 1), datetime(2016, 2, 2)]}),
+ {"months": lambda x: x.strftime("%Y-%m")},
+ "datetime64_monthformatter",
+ ),
+ (
+ DataFrame(
+ {
+ "hod": pd.to_datetime(
+ ["10:10:10.100", "12:12:12.120"], format="%H:%M:%S.%f"
+ )
+ }
+ ),
+ {"hod": lambda x: x.strftime("%H:%M")},
+ "datetime64_hourformatter",
+ ),
+ (
+ DataFrame(
+ {
+ "i": pd.Series([1, 2], dtype="int64"),
+ "f": pd.Series([1, 2], dtype="float64"),
+ "I": pd.Series([1, 2], dtype="Int64"),
+ "s": pd.Series([1, 2], dtype="string"),
+ "b": pd.Series([True, False], dtype="boolean"),
+ "c": pd.Series(["a", "b"], dtype=pd.CategoricalDtype(["a", "b"])),
+ "o": pd.Series([1, "2"], dtype=object),
+ }
+ ),
+ [lambda x: "formatted"] * 7,
+ "various_dtypes_formatted",
+ ),
+ ],
+)
+def test_to_html_formatters(df, formatters, expected, datapath):
+ expected = expected_html(datapath, expected)
+ result = df.to_html(formatters=formatters)
+ assert result == expected
+
+
+def test_to_html_regression_GH6098():
+ df = DataFrame(
+ {
+ "clé1": ["a", "a", "b", "b", "a"],
+ "clé2": ["1er", "2ème", "1er", "2ème", "1er"],
+ "données1": np.random.default_rng(2).standard_normal(5),
+ "données2": np.random.default_rng(2).standard_normal(5),
+ }
+ )
+
+ # it works
+ df.pivot_table(index=["clé1"], columns=["clé2"])._repr_html_()
+
+
+def test_to_html_truncate(datapath):
+ index = pd.date_range(start="20010101", freq="D", periods=20)
+ df = DataFrame(index=index, columns=range(20))
+ result = df.to_html(max_rows=8, max_cols=4)
+ expected = expected_html(datapath, "truncate")
+ assert result == expected
+
+
+@pytest.mark.parametrize("size", [1, 5])
+def test_html_invalid_formatters_arg_raises(size):
+ # issue-28469
+ df = DataFrame(columns=["a", "b", "c"])
+ msg = "Formatters length({}) should match DataFrame number of columns(3)"
+ with pytest.raises(ValueError, match=re.escape(msg.format(size))):
+ df.to_html(formatters=["{}".format] * size)
+
+
+def test_to_html_truncate_formatter(datapath):
+ # issue-25955
+ data = [
+ {"A": 1, "B": 2, "C": 3, "D": 4},
+ {"A": 5, "B": 6, "C": 7, "D": 8},
+ {"A": 9, "B": 10, "C": 11, "D": 12},
+ {"A": 13, "B": 14, "C": 15, "D": 16},
+ ]
+
+ df = DataFrame(data)
+ fmt = lambda x: str(x) + "_mod"
+ formatters = [fmt, fmt, None, None]
+ result = df.to_html(formatters=formatters, max_cols=3)
+ expected = expected_html(datapath, "truncate_formatter")
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "sparsify,expected",
+ [(True, "truncate_multi_index"), (False, "truncate_multi_index_sparse_off")],
+)
+def test_to_html_truncate_multi_index(sparsify, expected, datapath):
+ arrays = [
+ ["bar", "bar", "baz", "baz", "foo", "foo", "qux", "qux"],
+ ["one", "two", "one", "two", "one", "two", "one", "two"],
+ ]
+ df = DataFrame(index=arrays, columns=arrays)
+ result = df.to_html(max_rows=7, max_cols=7, sparsify=sparsify)
+ expected = expected_html(datapath, expected)
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "option,result,expected",
+ [
+ (None, lambda df: df.to_html(), "1"),
+ (None, lambda df: df.to_html(border=2), "2"),
+ (2, lambda df: df.to_html(), "2"),
+ (2, lambda df: df._repr_html_(), "2"),
+ ],
+)
+def test_to_html_border(option, result, expected):
+ df = DataFrame({"A": [1, 2]})
+ if option is None:
+ result = result(df)
+ else:
+ with option_context("display.html.border", option):
+ result = result(df)
+ expected = f'border="{expected}"'
+ assert expected in result
+
+
+@pytest.mark.parametrize("biggie_df_fixture", ["mixed"], indirect=True)
+def test_to_html(biggie_df_fixture):
+ # TODO: split this test
+ df = biggie_df_fixture
+ s = df.to_html()
+
+ buf = StringIO()
+ retval = df.to_html(buf=buf)
+ assert retval is None
+ assert buf.getvalue() == s
+
+ assert isinstance(s, str)
+
+ df.to_html(columns=["B", "A"], col_space=17)
+ df.to_html(columns=["B", "A"], formatters={"A": lambda x: f"{x:.1f}"})
+
+ df.to_html(columns=["B", "A"], float_format=str)
+ df.to_html(columns=["B", "A"], col_space=12, float_format=str)
+
+
+@pytest.mark.parametrize("biggie_df_fixture", ["empty"], indirect=True)
+def test_to_html_empty_dataframe(biggie_df_fixture):
+ df = biggie_df_fixture
+ df.to_html()
+
+
+def test_to_html_filename(biggie_df_fixture, tmpdir):
+ df = biggie_df_fixture
+ expected = df.to_html()
+ path = tmpdir.join("test.html")
+ df.to_html(path)
+ result = path.read()
+ assert result == expected
+
+
+def test_to_html_with_no_bold():
+ df = DataFrame({"x": np.random.default_rng(2).standard_normal(5)})
+ html = df.to_html(bold_rows=False)
+ result = html[html.find("")]
+ assert "B" not in result
+
+
+@pytest.mark.parametrize(
+ "columns,justify,expected",
+ [
+ (
+ MultiIndex.from_arrays(
+ [np.arange(2).repeat(2), np.mod(range(4), 2)],
+ names=["CL0", "CL1"],
+ ),
+ "left",
+ "multiindex_1",
+ ),
+ (
+ MultiIndex.from_arrays([np.arange(4), np.mod(range(4), 2)]),
+ "right",
+ "multiindex_2",
+ ),
+ ],
+)
+def test_to_html_multiindex(columns, justify, expected, datapath):
+ df = DataFrame([list("abcd"), list("efgh")], columns=columns)
+ result = df.to_html(justify=justify)
+ expected = expected_html(datapath, expected)
+ assert result == expected
+
+
+def test_to_html_justify(justify, datapath):
+ df = DataFrame(
+ {"A": [6, 30000, 2], "B": [1, 2, 70000], "C": [223442, 0, 1]},
+ columns=["A", "B", "C"],
+ )
+ result = df.to_html(justify=justify)
+ expected = expected_html(datapath, "justify").format(justify=justify)
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "justify", ["super-right", "small-left", "noinherit", "tiny", "pandas"]
+)
+def test_to_html_invalid_justify(justify):
+ # GH 17527
+ df = DataFrame()
+ msg = "Invalid value for justify parameter"
+
+ with pytest.raises(ValueError, match=msg):
+ df.to_html(justify=justify)
+
+
+class TestHTMLIndex:
+ @pytest.fixture
+ def df(self):
+ index = ["foo", "bar", "baz"]
+ df = DataFrame(
+ {"A": [1, 2, 3], "B": [1.2, 3.4, 5.6], "C": ["one", "two", np.nan]},
+ columns=["A", "B", "C"],
+ index=index,
+ )
+ return df
+
+ @pytest.fixture
+ def expected_without_index(self, datapath):
+ return expected_html(datapath, "index_2")
+
+ def test_to_html_flat_index_without_name(
+ self, datapath, df, expected_without_index
+ ):
+ expected_with_index = expected_html(datapath, "index_1")
+ assert df.to_html() == expected_with_index
+
+ result = df.to_html(index=False)
+ for i in df.index:
+ assert i not in result
+ assert result == expected_without_index
+
+ def test_to_html_flat_index_with_name(self, datapath, df, expected_without_index):
+ df.index = Index(["foo", "bar", "baz"], name="idx")
+ expected_with_index = expected_html(datapath, "index_3")
+ assert df.to_html() == expected_with_index
+ assert df.to_html(index=False) == expected_without_index
+
+ def test_to_html_multiindex_without_names(
+ self, datapath, df, expected_without_index
+ ):
+ tuples = [("foo", "car"), ("foo", "bike"), ("bar", "car")]
+ df.index = MultiIndex.from_tuples(tuples)
+
+ expected_with_index = expected_html(datapath, "index_4")
+ assert df.to_html() == expected_with_index
+
+ result = df.to_html(index=False)
+ for i in ["foo", "bar", "car", "bike"]:
+ assert i not in result
+ # must be the same result as normal index
+ assert result == expected_without_index
+
+ def test_to_html_multiindex_with_names(self, datapath, df, expected_without_index):
+ tuples = [("foo", "car"), ("foo", "bike"), ("bar", "car")]
+ df.index = MultiIndex.from_tuples(tuples, names=["idx1", "idx2"])
+ expected_with_index = expected_html(datapath, "index_5")
+ assert df.to_html() == expected_with_index
+ assert df.to_html(index=False) == expected_without_index
+
+
+@pytest.mark.parametrize("classes", ["sortable draggable", ["sortable", "draggable"]])
+def test_to_html_with_classes(classes, datapath):
+ df = DataFrame()
+ expected = expected_html(datapath, "with_classes")
+ result = df.to_html(classes=classes)
+ assert result == expected
+
+
+def test_to_html_no_index_max_rows(datapath):
+ # GH 14998
+ df = DataFrame({"A": [1, 2, 3, 4]})
+ result = df.to_html(index=False, max_rows=1)
+ expected = expected_html(datapath, "gh14998_expected_output")
+ assert result == expected
+
+
+def test_to_html_multiindex_max_cols(datapath):
+ # GH 6131
+ index = MultiIndex(
+ levels=[["ba", "bb", "bc"], ["ca", "cb", "cc"]],
+ codes=[[0, 1, 2], [0, 1, 2]],
+ names=["b", "c"],
+ )
+ columns = MultiIndex(
+ levels=[["d"], ["aa", "ab", "ac"]],
+ codes=[[0, 0, 0], [0, 1, 2]],
+ names=[None, "a"],
+ )
+ data = np.array(
+ [[1.0, np.nan, np.nan], [np.nan, 2.0, np.nan], [np.nan, np.nan, 3.0]]
+ )
+ df = DataFrame(data, index, columns)
+ result = df.to_html(max_cols=2)
+ expected = expected_html(datapath, "gh6131_expected_output")
+ assert result == expected
+
+
+def test_to_html_multi_indexes_index_false(datapath):
+ # GH 22579
+ df = DataFrame(
+ {"a": range(10), "b": range(10, 20), "c": range(10, 20), "d": range(10, 20)}
+ )
+ df.columns = MultiIndex.from_product([["a", "b"], ["c", "d"]])
+ df.index = MultiIndex.from_product([["a", "b"], ["c", "d", "e", "f", "g"]])
+ result = df.to_html(index=False)
+ expected = expected_html(datapath, "gh22579_expected_output")
+ assert result == expected
+
+
+@pytest.mark.parametrize("index_names", [True, False])
+@pytest.mark.parametrize("header", [True, False])
+@pytest.mark.parametrize("index", [True, False])
+@pytest.mark.parametrize(
+ "column_index, column_type",
+ [
+ (Index([0, 1]), "unnamed_standard"),
+ (Index([0, 1], name="columns.name"), "named_standard"),
+ (MultiIndex.from_product([["a"], ["b", "c"]]), "unnamed_multi"),
+ (
+ MultiIndex.from_product(
+ [["a"], ["b", "c"]], names=["columns.name.0", "columns.name.1"]
+ ),
+ "named_multi",
+ ),
+ ],
+)
+@pytest.mark.parametrize(
+ "row_index, row_type",
+ [
+ (Index([0, 1]), "unnamed_standard"),
+ (Index([0, 1], name="index.name"), "named_standard"),
+ (MultiIndex.from_product([["a"], ["b", "c"]]), "unnamed_multi"),
+ (
+ MultiIndex.from_product(
+ [["a"], ["b", "c"]], names=["index.name.0", "index.name.1"]
+ ),
+ "named_multi",
+ ),
+ ],
+)
+def test_to_html_basic_alignment(
+ datapath, row_index, row_type, column_index, column_type, index, header, index_names
+):
+ # GH 22747, GH 22579
+ df = DataFrame(np.zeros((2, 2), dtype=int), index=row_index, columns=column_index)
+ result = df.to_html(index=index, header=header, index_names=index_names)
+
+ if not index:
+ row_type = "none"
+ elif not index_names and row_type.startswith("named"):
+ row_type = "un" + row_type
+
+ if not header:
+ column_type = "none"
+ elif not index_names and column_type.startswith("named"):
+ column_type = "un" + column_type
+
+ filename = "index_" + row_type + "_columns_" + column_type
+ expected = expected_html(datapath, filename)
+ assert result == expected
+
+
+@pytest.mark.parametrize("index_names", [True, False])
+@pytest.mark.parametrize("header", [True, False])
+@pytest.mark.parametrize("index", [True, False])
+@pytest.mark.parametrize(
+ "column_index, column_type",
+ [
+ (Index(np.arange(8)), "unnamed_standard"),
+ (Index(np.arange(8), name="columns.name"), "named_standard"),
+ (
+ MultiIndex.from_product([["a", "b"], ["c", "d"], ["e", "f"]]),
+ "unnamed_multi",
+ ),
+ (
+ MultiIndex.from_product(
+ [["a", "b"], ["c", "d"], ["e", "f"]], names=["foo", None, "baz"]
+ ),
+ "named_multi",
+ ),
+ ],
+)
+@pytest.mark.parametrize(
+ "row_index, row_type",
+ [
+ (Index(np.arange(8)), "unnamed_standard"),
+ (Index(np.arange(8), name="index.name"), "named_standard"),
+ (
+ MultiIndex.from_product([["a", "b"], ["c", "d"], ["e", "f"]]),
+ "unnamed_multi",
+ ),
+ (
+ MultiIndex.from_product(
+ [["a", "b"], ["c", "d"], ["e", "f"]], names=["foo", None, "baz"]
+ ),
+ "named_multi",
+ ),
+ ],
+)
+def test_to_html_alignment_with_truncation(
+ datapath, row_index, row_type, column_index, column_type, index, header, index_names
+):
+ # GH 22747, GH 22579
+ df = DataFrame(np.arange(64).reshape(8, 8), index=row_index, columns=column_index)
+ result = df.to_html(
+ max_rows=4, max_cols=4, index=index, header=header, index_names=index_names
+ )
+
+ if not index:
+ row_type = "none"
+ elif not index_names and row_type.startswith("named"):
+ row_type = "un" + row_type
+
+ if not header:
+ column_type = "none"
+ elif not index_names and column_type.startswith("named"):
+ column_type = "un" + column_type
+
+ filename = "trunc_df_index_" + row_type + "_columns_" + column_type
+ expected = expected_html(datapath, filename)
+ assert result == expected
+
+
+@pytest.mark.parametrize("index", [False, 0])
+def test_to_html_truncation_index_false_max_rows(datapath, index):
+ # GH 15019
+ data = [
+ [1.764052, 0.400157],
+ [0.978738, 2.240893],
+ [1.867558, -0.977278],
+ [0.950088, -0.151357],
+ [-0.103219, 0.410599],
+ ]
+ df = DataFrame(data)
+ result = df.to_html(max_rows=4, index=index)
+ expected = expected_html(datapath, "gh15019_expected_output")
+ assert result == expected
+
+
+@pytest.mark.parametrize("index", [False, 0])
+@pytest.mark.parametrize(
+ "col_index_named, expected_output",
+ [(False, "gh22783_expected_output"), (True, "gh22783_named_columns_index")],
+)
+def test_to_html_truncation_index_false_max_cols(
+ datapath, index, col_index_named, expected_output
+):
+ # GH 22783
+ data = [
+ [1.764052, 0.400157, 0.978738, 2.240893, 1.867558],
+ [-0.977278, 0.950088, -0.151357, -0.103219, 0.410599],
+ ]
+ df = DataFrame(data)
+ if col_index_named:
+ df.columns.rename("columns.name", inplace=True)
+ result = df.to_html(max_cols=4, index=index)
+ expected = expected_html(datapath, expected_output)
+ assert result == expected
+
+
+@pytest.mark.parametrize("notebook", [True, False])
+def test_to_html_notebook_has_style(notebook):
+ df = DataFrame({"A": [1, 2, 3]})
+ result = df.to_html(notebook=notebook)
+
+ if notebook:
+ assert "tbody tr th:only-of-type" in result
+ assert "vertical-align: middle;" in result
+ assert "thead th" in result
+ else:
+ assert "tbody tr th:only-of-type" not in result
+ assert "vertical-align: middle;" not in result
+ assert "thead th" not in result
+
+
+def test_to_html_with_index_names_false():
+ # GH 16493
+ df = DataFrame({"A": [1, 2]}, index=Index(["a", "b"], name="myindexname"))
+ result = df.to_html(index_names=False)
+ assert "myindexname" not in result
+
+
+def test_to_html_with_id():
+ # GH 8496
+ df = DataFrame({"A": [1, 2]}, index=Index(["a", "b"], name="myindexname"))
+ result = df.to_html(index_names=False, table_id="TEST_ID")
+ assert ' id="TEST_ID"' in result
+
+
+@pytest.mark.parametrize(
+ "value,float_format,expected",
+ [
+ (0.19999, "%.3f", "gh21625_expected_output"),
+ (100.0, "%.0f", "gh22270_expected_output"),
+ ],
+)
+def test_to_html_float_format_no_fixed_width(value, float_format, expected, datapath):
+ # GH 21625, GH 22270
+ df = DataFrame({"x": [value]})
+ expected = expected_html(datapath, expected)
+ result = df.to_html(float_format=float_format)
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "render_links,expected",
+ [(True, "render_links_true"), (False, "render_links_false")],
+)
+def test_to_html_render_links(render_links, expected, datapath):
+ # GH 2679
+ data = [
+ [0, "https://pandas.pydata.org/?q1=a&q2=b", "pydata.org"],
+ [0, "www.pydata.org", "pydata.org"],
+ ]
+ df = DataFrame(data, columns=Index(["foo", "bar", None], dtype=object))
+
+ result = df.to_html(render_links=render_links)
+ expected = expected_html(datapath, expected)
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "method,expected",
+ [
+ ("to_html", lambda x: lorem_ipsum),
+ ("_repr_html_", lambda x: lorem_ipsum[: x - 4] + "..."), # regression case
+ ],
+)
+@pytest.mark.parametrize("max_colwidth", [10, 20, 50, 100])
+def test_ignore_display_max_colwidth(method, expected, max_colwidth):
+ # see gh-17004
+ df = DataFrame([lorem_ipsum])
+ with option_context("display.max_colwidth", max_colwidth):
+ result = getattr(df, method)()
+ expected = expected(max_colwidth)
+ assert expected in result
+
+
+@pytest.mark.parametrize("classes", [True, 0])
+def test_to_html_invalid_classes_type(classes):
+ # GH 25608
+ df = DataFrame()
+ msg = "classes must be a string, list, or tuple"
+
+ with pytest.raises(TypeError, match=msg):
+ df.to_html(classes=classes)
+
+
+def test_to_html_round_column_headers():
+ # GH 17280
+ df = DataFrame([1], columns=[0.55555])
+ with option_context("display.precision", 3):
+ html = df.to_html(notebook=False)
+ notebook = df.to_html(notebook=True)
+ assert "0.55555" in html
+ assert "0.556" in notebook
+
+
+@pytest.mark.parametrize("unit", ["100px", "10%", "5em", 150])
+def test_to_html_with_col_space_units(unit):
+ # GH 25941
+ df = DataFrame(np.random.default_rng(2).random(size=(1, 3)))
+ result = df.to_html(col_space=unit)
+ result = result.split("tbody")[0]
+ hdrs = [x for x in result.split("\n") if re.search(r"\s]", x)]
+ if isinstance(unit, int):
+ unit = str(unit) + "px"
+ for h in hdrs:
+ expected = f' '
+ assert expected in h
+
+
+class TestReprHTML:
+ def test_html_repr_min_rows_default(self, datapath):
+ # gh-27991
+
+ # default setting no truncation even if above min_rows
+ df = DataFrame({"a": range(20)})
+ result = df._repr_html_()
+ expected = expected_html(datapath, "html_repr_min_rows_default_no_truncation")
+ assert result == expected
+
+ # default of max_rows 60 triggers truncation if above
+ df = DataFrame({"a": range(61)})
+ result = df._repr_html_()
+ expected = expected_html(datapath, "html_repr_min_rows_default_truncated")
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "max_rows,min_rows,expected",
+ [
+ # truncated after first two rows
+ (10, 4, "html_repr_max_rows_10_min_rows_4"),
+ # when set to None, follow value of max_rows
+ (12, None, "html_repr_max_rows_12_min_rows_None"),
+ # when set value higher as max_rows, use the minimum
+ (10, 12, "html_repr_max_rows_10_min_rows_12"),
+ # max_rows of None -> never truncate
+ (None, 12, "html_repr_max_rows_None_min_rows_12"),
+ ],
+ )
+ def test_html_repr_min_rows(self, datapath, max_rows, min_rows, expected):
+ # gh-27991
+
+ df = DataFrame({"a": range(61)})
+ expected = expected_html(datapath, expected)
+ with option_context("display.max_rows", max_rows, "display.min_rows", min_rows):
+ result = df._repr_html_()
+ assert result == expected
+
+ def test_repr_html_ipython_config(self, ip):
+ code = textwrap.dedent(
+ """\
+ from pandas import DataFrame
+ df = DataFrame({"A": [1, 2]})
+ df._repr_html_()
+
+ cfg = get_ipython().config
+ cfg['IPKernelApp']['parent_appname']
+ df._repr_html_()
+ """
+ )
+ result = ip.run_cell(code, silent=True)
+ assert not result.error_in_exec
+
+ def test_info_repr_html(self):
+ max_rows = 60
+ max_cols = 20
+ # Long
+ h, w = max_rows + 1, max_cols - 1
+ df = DataFrame({k: np.arange(1, 1 + h) for k in np.arange(w)})
+ assert r"<class" not in df._repr_html_()
+ with option_context("display.large_repr", "info"):
+ assert r"<class" in df._repr_html_()
+
+ # Wide
+ h, w = max_rows - 1, max_cols + 1
+ df = DataFrame({k: np.arange(1, 1 + h) for k in np.arange(w)})
+ assert "{40 + h}" in reg_repr
+
+ h = max_rows + 1
+ df = DataFrame(
+ {
+ "idx": np.linspace(-10, 10, h),
+ "A": np.arange(1, 1 + h),
+ "B": np.arange(41, 41 + h),
+ }
+ ).set_index("idx")
+ long_repr = df._repr_html_()
+ assert ".." in long_repr
+ assert "31 " not in long_repr
+ assert f"{h} rows " in long_repr
+ assert "2 columns" in long_repr
+
+ def test_repr_html_long_multiindex(self):
+ max_rows = 60
+ max_L1 = max_rows // 2
+
+ tuples = list(itertools.product(np.arange(max_L1), ["foo", "bar"]))
+ idx = MultiIndex.from_tuples(tuples, names=["first", "second"])
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((max_L1 * 2, 2)),
+ index=idx,
+ columns=["A", "B"],
+ )
+ with option_context("display.max_rows", 60, "display.max_columns", 20):
+ reg_repr = df._repr_html_()
+ assert "..." not in reg_repr
+
+ tuples = list(itertools.product(np.arange(max_L1 + 1), ["foo", "bar"]))
+ idx = MultiIndex.from_tuples(tuples, names=["first", "second"])
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal(((max_L1 + 1) * 2, 2)),
+ index=idx,
+ columns=["A", "B"],
+ )
+ long_repr = df._repr_html_()
+ assert "..." in long_repr
+
+ def test_repr_html_long_and_wide(self):
+ max_cols = 20
+ max_rows = 60
+
+ h, w = max_rows - 1, max_cols - 1
+ df = DataFrame({k: np.arange(1, 1 + h) for k in np.arange(w)})
+ with option_context("display.max_rows", 60, "display.max_columns", 20):
+ assert "..." not in df._repr_html_()
+
+ h, w = max_rows + 1, max_cols + 1
+ df = DataFrame({k: np.arange(1, 1 + h) for k in np.arange(w)})
+ with option_context("display.max_rows", 60, "display.max_columns", 20):
+ assert "..." in df._repr_html_()
+
+
+def test_to_html_multilevel(multiindex_year_month_day_dataframe_random_data):
+ ymd = multiindex_year_month_day_dataframe_random_data
+
+ ymd.columns.name = "foo"
+ ymd.to_html()
+ ymd.T.to_html()
+
+
+@pytest.mark.parametrize("na_rep", ["NaN", "Ted"])
+def test_to_html_na_rep_and_float_format(na_rep, datapath):
+ # https://github.com/pandas-dev/pandas/issues/13828
+ df = DataFrame(
+ [
+ ["A", 1.2225],
+ ["A", None],
+ ],
+ columns=["Group", "Data"],
+ )
+ result = df.to_html(na_rep=na_rep, float_format="{:.2f}".format)
+ expected = expected_html(datapath, "gh13828_expected_output")
+ expected = expected.format(na_rep=na_rep)
+ assert result == expected
+
+
+def test_to_html_na_rep_non_scalar_data(datapath):
+ # GH47103
+ df = DataFrame([{"a": 1, "b": [1, 2, 3]}])
+ result = df.to_html(na_rep="-")
+ expected = expected_html(datapath, "gh47103_expected_output")
+ assert result == expected
+
+
+def test_to_html_float_format_object_col(datapath):
+ # GH#40024
+ df = DataFrame(data={"x": [1000.0, "test"]})
+ result = df.to_html(float_format=lambda x: f"{x:,.0f}")
+ expected = expected_html(datapath, "gh40024_expected_output")
+ assert result == expected
+
+
+def test_to_html_multiindex_col_with_colspace():
+ # GH#53885
+ df = DataFrame([[1, 2]])
+ df.columns = MultiIndex.from_tuples([(1, 1), (2, 1)])
+ result = df.to_html(col_space=100)
+ expected = (
+ '\n'
+ " \n"
+ " \n"
+ ' \n'
+ ' 1 \n'
+ ' 2 \n'
+ " \n"
+ " \n"
+ ' \n'
+ ' 1 \n'
+ ' 1 \n'
+ " \n"
+ " \n"
+ " \n"
+ " \n"
+ " 0 \n"
+ " 1 \n"
+ " 2 \n"
+ " \n"
+ " \n"
+ "
"
+ )
+ assert result == expected
+
+
+def test_to_html_tuple_col_with_colspace():
+ # GH#53885
+ df = DataFrame({("a", "b"): [1], "b": [2]})
+ result = df.to_html(col_space=100)
+ expected = (
+ '\n'
+ " \n"
+ ' \n'
+ ' \n'
+ ' (a, b) \n'
+ ' b \n'
+ " \n"
+ " \n"
+ " \n"
+ " \n"
+ " 0 \n"
+ " 1 \n"
+ " 2 \n"
+ " \n"
+ " \n"
+ "
"
+ )
+ assert result == expected
+
+
+def test_to_html_empty_complex_array():
+ # GH#54167
+ df = DataFrame({"x": np.array([], dtype="complex")})
+ result = df.to_html(col_space=100)
+ expected = (
+ '\n'
+ " \n"
+ ' \n'
+ ' \n'
+ ' x \n'
+ " \n"
+ " \n"
+ " \n"
+ " \n"
+ "
"
+ )
+ assert result == expected
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_latex.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_latex.py
new file mode 100644
index 0000000000000000000000000000000000000000..6e44b42dc88a88e2fe3faf4924788a485df26b60
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_latex.py
@@ -0,0 +1,1527 @@
+from datetime import datetime
+from textwrap import dedent
+
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Series,
+)
+
+pytest.importorskip("jinja2")
+
+
+def _dedent(string):
+ """Dedent without new line in the beginning.
+
+ Built-in textwrap.dedent would keep new line character in the beginning
+ of multi-line string starting from the new line.
+ This version drops the leading new line character.
+ """
+ return dedent(string).lstrip()
+
+
+@pytest.fixture
+def df_short():
+ """Short dataframe for testing table/tabular/longtable LaTeX env."""
+ return DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+
+
+class TestToLatex:
+ def test_to_latex_to_file(self, float_frame, temp_file):
+ float_frame.to_latex(temp_file)
+ with open(temp_file, encoding="utf-8") as f:
+ assert float_frame.to_latex() == f.read()
+
+ def test_to_latex_to_file_utf8_with_encoding(self, temp_file):
+ # test with utf-8 and encoding option (GH 7061)
+ df = DataFrame([["au\xdfgangen"]])
+ df.to_latex(temp_file, encoding="utf-8")
+ with open(temp_file, encoding="utf-8") as f:
+ assert df.to_latex() == f.read()
+
+ def test_to_latex_to_file_utf8_without_encoding(self, temp_file):
+ # test with utf-8 without encoding option
+ df = DataFrame([["au\xdfgangen"]])
+ df.to_latex(temp_file)
+ with open(temp_file, encoding="utf-8") as f:
+ assert df.to_latex() == f.read()
+
+ def test_to_latex_tabular_with_index(self):
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ result = df.to_latex()
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lrl}
+ \toprule
+ & a & b \\
+ \midrule
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_tabular_without_index(self):
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ result = df.to_latex(index=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{rl}
+ \toprule
+ a & b \\
+ \midrule
+ 1 & b1 \\
+ 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "bad_column_format",
+ [5, 1.2, ["l", "r"], ("r", "c"), {"r", "c", "l"}, {"a": "r", "b": "l"}],
+ )
+ def test_to_latex_bad_column_format(self, bad_column_format):
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ msg = r"`column_format` must be str or unicode"
+ with pytest.raises(ValueError, match=msg):
+ df.to_latex(column_format=bad_column_format)
+
+ def test_to_latex_column_format_just_works(self, float_frame):
+ # GH Bug #9402
+ float_frame.to_latex(column_format="lcr")
+
+ def test_to_latex_column_format(self):
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ result = df.to_latex(column_format="lcr")
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lcr}
+ \toprule
+ & a & b \\
+ \midrule
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_float_format_object_col(self):
+ # GH#40024
+ ser = Series([1000.0, "test"])
+ result = ser.to_latex(float_format="{:,.0f}".format)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{ll}
+ \toprule
+ & 0 \\
+ \midrule
+ 0 & 1,000 \\
+ 1 & test \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_empty_tabular(self):
+ df = DataFrame()
+ result = df.to_latex()
+ expected = _dedent(
+ r"""
+ \begin{tabular}{l}
+ \toprule
+ \midrule
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_series(self):
+ s = Series(["a", "b", "c"])
+ result = s.to_latex()
+ expected = _dedent(
+ r"""
+ \begin{tabular}{ll}
+ \toprule
+ & 0 \\
+ \midrule
+ 0 & a \\
+ 1 & b \\
+ 2 & c \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_midrule_location(self):
+ # GH 18326
+ df = DataFrame({"a": [1, 2]})
+ df.index.name = "foo"
+ result = df.to_latex(index_names=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lr}
+ \toprule
+ & a \\
+ \midrule
+ 0 & 1 \\
+ 1 & 2 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+
+class TestToLatexLongtable:
+ def test_to_latex_empty_longtable(self):
+ df = DataFrame()
+ result = df.to_latex(longtable=True)
+ expected = _dedent(
+ r"""
+ \begin{longtable}{l}
+ \toprule
+ \midrule
+ \endfirsthead
+ \toprule
+ \midrule
+ \endhead
+ \midrule
+ \multicolumn{0}{r}{Continued on next page} \\
+ \midrule
+ \endfoot
+ \bottomrule
+ \endlastfoot
+ \end{longtable}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_longtable_with_index(self):
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ result = df.to_latex(longtable=True)
+ expected = _dedent(
+ r"""
+ \begin{longtable}{lrl}
+ \toprule
+ & a & b \\
+ \midrule
+ \endfirsthead
+ \toprule
+ & a & b \\
+ \midrule
+ \endhead
+ \midrule
+ \multicolumn{3}{r}{Continued on next page} \\
+ \midrule
+ \endfoot
+ \bottomrule
+ \endlastfoot
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \end{longtable}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_longtable_without_index(self):
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ result = df.to_latex(index=False, longtable=True)
+ expected = _dedent(
+ r"""
+ \begin{longtable}{rl}
+ \toprule
+ a & b \\
+ \midrule
+ \endfirsthead
+ \toprule
+ a & b \\
+ \midrule
+ \endhead
+ \midrule
+ \multicolumn{2}{r}{Continued on next page} \\
+ \midrule
+ \endfoot
+ \bottomrule
+ \endlastfoot
+ 1 & b1 \\
+ 2 & b2 \\
+ \end{longtable}
+ """
+ )
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "df_data, expected_number",
+ [
+ ({"a": [1, 2]}, 1),
+ ({"a": [1, 2], "b": [3, 4]}, 2),
+ ({"a": [1, 2], "b": [3, 4], "c": [5, 6]}, 3),
+ ],
+ )
+ def test_to_latex_longtable_continued_on_next_page(self, df_data, expected_number):
+ df = DataFrame(df_data)
+ result = df.to_latex(index=False, longtable=True)
+ assert rf"\multicolumn{{{expected_number}}}" in result
+
+
+class TestToLatexHeader:
+ def test_to_latex_no_header_with_index(self):
+ # GH 7124
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ result = df.to_latex(header=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lrl}
+ \toprule
+ \midrule
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_no_header_without_index(self):
+ # GH 7124
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ result = df.to_latex(index=False, header=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{rl}
+ \toprule
+ \midrule
+ 1 & b1 \\
+ 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_specified_header_with_index(self):
+ # GH 7124
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ result = df.to_latex(header=["AA", "BB"])
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lrl}
+ \toprule
+ & AA & BB \\
+ \midrule
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_specified_header_without_index(self):
+ # GH 7124
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ result = df.to_latex(header=["AA", "BB"], index=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{rl}
+ \toprule
+ AA & BB \\
+ \midrule
+ 1 & b1 \\
+ 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "header, num_aliases",
+ [
+ (["A"], 1),
+ (("B",), 1),
+ (("Col1", "Col2", "Col3"), 3),
+ (("Col1", "Col2", "Col3", "Col4"), 4),
+ ],
+ )
+ def test_to_latex_number_of_items_in_header_missmatch_raises(
+ self,
+ header,
+ num_aliases,
+ ):
+ # GH 7124
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ msg = f"Writing 2 cols but got {num_aliases} aliases"
+ with pytest.raises(ValueError, match=msg):
+ df.to_latex(header=header)
+
+ def test_to_latex_decimal(self):
+ # GH 12031
+ df = DataFrame({"a": [1.0, 2.1], "b": ["b1", "b2"]})
+ result = df.to_latex(decimal=",")
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lrl}
+ \toprule
+ & a & b \\
+ \midrule
+ 0 & 1,000000 & b1 \\
+ 1 & 2,100000 & b2 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+
+class TestToLatexBold:
+ def test_to_latex_bold_rows(self):
+ # GH 16707
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ result = df.to_latex(bold_rows=True)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lrl}
+ \toprule
+ & a & b \\
+ \midrule
+ \textbf{0} & 1 & b1 \\
+ \textbf{1} & 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_no_bold_rows(self):
+ # GH 16707
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ result = df.to_latex(bold_rows=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lrl}
+ \toprule
+ & a & b \\
+ \midrule
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+
+class TestToLatexCaptionLabel:
+ @pytest.fixture
+ def caption_table(self):
+ """Caption for table/tabular LaTeX environment."""
+ return "a table in a \\texttt{table/tabular} environment"
+
+ @pytest.fixture
+ def short_caption(self):
+ """Short caption for testing \\caption[short_caption]{full_caption}."""
+ return "a table"
+
+ @pytest.fixture
+ def label_table(self):
+ """Label for table/tabular LaTeX environment."""
+ return "tab:table_tabular"
+
+ @pytest.fixture
+ def caption_longtable(self):
+ """Caption for longtable LaTeX environment."""
+ return "a table in a \\texttt{longtable} environment"
+
+ @pytest.fixture
+ def label_longtable(self):
+ """Label for longtable LaTeX environment."""
+ return "tab:longtable"
+
+ def test_to_latex_caption_only(self, df_short, caption_table):
+ # GH 25436
+ result = df_short.to_latex(caption=caption_table)
+ expected = _dedent(
+ r"""
+ \begin{table}
+ \caption{a table in a \texttt{table/tabular} environment}
+ \begin{tabular}{lrl}
+ \toprule
+ & a & b \\
+ \midrule
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ \end{table}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_label_only(self, df_short, label_table):
+ # GH 25436
+ result = df_short.to_latex(label=label_table)
+ expected = _dedent(
+ r"""
+ \begin{table}
+ \label{tab:table_tabular}
+ \begin{tabular}{lrl}
+ \toprule
+ & a & b \\
+ \midrule
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ \end{table}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_caption_and_label(self, df_short, caption_table, label_table):
+ # GH 25436
+ result = df_short.to_latex(caption=caption_table, label=label_table)
+ expected = _dedent(
+ r"""
+ \begin{table}
+ \caption{a table in a \texttt{table/tabular} environment}
+ \label{tab:table_tabular}
+ \begin{tabular}{lrl}
+ \toprule
+ & a & b \\
+ \midrule
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ \end{table}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_caption_and_shortcaption(
+ self,
+ df_short,
+ caption_table,
+ short_caption,
+ ):
+ result = df_short.to_latex(caption=(caption_table, short_caption))
+ expected = _dedent(
+ r"""
+ \begin{table}
+ \caption[a table]{a table in a \texttt{table/tabular} environment}
+ \begin{tabular}{lrl}
+ \toprule
+ & a & b \\
+ \midrule
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ \end{table}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_caption_and_shortcaption_list_is_ok(self, df_short):
+ caption = ("Long-long-caption", "Short")
+ result_tuple = df_short.to_latex(caption=caption)
+ result_list = df_short.to_latex(caption=list(caption))
+ assert result_tuple == result_list
+
+ def test_to_latex_caption_shortcaption_and_label(
+ self,
+ df_short,
+ caption_table,
+ short_caption,
+ label_table,
+ ):
+ # test when the short_caption is provided alongside caption and label
+ result = df_short.to_latex(
+ caption=(caption_table, short_caption),
+ label=label_table,
+ )
+ expected = _dedent(
+ r"""
+ \begin{table}
+ \caption[a table]{a table in a \texttt{table/tabular} environment}
+ \label{tab:table_tabular}
+ \begin{tabular}{lrl}
+ \toprule
+ & a & b \\
+ \midrule
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ \end{table}
+ """
+ )
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "bad_caption",
+ [
+ ("full_caption", "short_caption", "extra_string"),
+ ("full_caption", "short_caption", 1),
+ ("full_caption", "short_caption", None),
+ ("full_caption",),
+ (None,),
+ ],
+ )
+ def test_to_latex_bad_caption_raises(self, bad_caption):
+ # test that wrong number of params is raised
+ df = DataFrame({"a": [1]})
+ msg = "`caption` must be either a string or 2-tuple of strings"
+ with pytest.raises(ValueError, match=msg):
+ df.to_latex(caption=bad_caption)
+
+ def test_to_latex_two_chars_caption(self, df_short):
+ # test that two chars caption is handled correctly
+ # it must not be unpacked into long_caption, short_caption.
+ result = df_short.to_latex(caption="xy")
+ expected = _dedent(
+ r"""
+ \begin{table}
+ \caption{xy}
+ \begin{tabular}{lrl}
+ \toprule
+ & a & b \\
+ \midrule
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ \end{table}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_longtable_caption_only(self, df_short, caption_longtable):
+ # GH 25436
+ # test when no caption and no label is provided
+ # is performed by test_to_latex_longtable()
+ result = df_short.to_latex(longtable=True, caption=caption_longtable)
+ expected = _dedent(
+ r"""
+ \begin{longtable}{lrl}
+ \caption{a table in a \texttt{longtable} environment} \\
+ \toprule
+ & a & b \\
+ \midrule
+ \endfirsthead
+ \caption[]{a table in a \texttt{longtable} environment} \\
+ \toprule
+ & a & b \\
+ \midrule
+ \endhead
+ \midrule
+ \multicolumn{3}{r}{Continued on next page} \\
+ \midrule
+ \endfoot
+ \bottomrule
+ \endlastfoot
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \end{longtable}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_longtable_label_only(self, df_short, label_longtable):
+ # GH 25436
+ result = df_short.to_latex(longtable=True, label=label_longtable)
+ expected = _dedent(
+ r"""
+ \begin{longtable}{lrl}
+ \label{tab:longtable} \\
+ \toprule
+ & a & b \\
+ \midrule
+ \endfirsthead
+ \toprule
+ & a & b \\
+ \midrule
+ \endhead
+ \midrule
+ \multicolumn{3}{r}{Continued on next page} \\
+ \midrule
+ \endfoot
+ \bottomrule
+ \endlastfoot
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \end{longtable}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_longtable_caption_and_label(
+ self,
+ df_short,
+ caption_longtable,
+ label_longtable,
+ ):
+ # GH 25436
+ result = df_short.to_latex(
+ longtable=True,
+ caption=caption_longtable,
+ label=label_longtable,
+ )
+ expected = _dedent(
+ r"""
+ \begin{longtable}{lrl}
+ \caption{a table in a \texttt{longtable} environment} \label{tab:longtable} \\
+ \toprule
+ & a & b \\
+ \midrule
+ \endfirsthead
+ \caption[]{a table in a \texttt{longtable} environment} \\
+ \toprule
+ & a & b \\
+ \midrule
+ \endhead
+ \midrule
+ \multicolumn{3}{r}{Continued on next page} \\
+ \midrule
+ \endfoot
+ \bottomrule
+ \endlastfoot
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \end{longtable}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_longtable_caption_shortcaption_and_label(
+ self,
+ df_short,
+ caption_longtable,
+ short_caption,
+ label_longtable,
+ ):
+ # test when the caption, the short_caption and the label are provided
+ result = df_short.to_latex(
+ longtable=True,
+ caption=(caption_longtable, short_caption),
+ label=label_longtable,
+ )
+ expected = _dedent(
+ r"""
+\begin{longtable}{lrl}
+\caption[a table]{a table in a \texttt{longtable} environment} \label{tab:longtable} \\
+\toprule
+ & a & b \\
+\midrule
+\endfirsthead
+\caption[]{a table in a \texttt{longtable} environment} \\
+\toprule
+ & a & b \\
+\midrule
+\endhead
+\midrule
+\multicolumn{3}{r}{Continued on next page} \\
+\midrule
+\endfoot
+\bottomrule
+\endlastfoot
+0 & 1 & b1 \\
+1 & 2 & b2 \\
+\end{longtable}
+"""
+ )
+ assert result == expected
+
+
+class TestToLatexEscape:
+ @pytest.fixture
+ def df_with_symbols(self):
+ """Dataframe with special characters for testing chars escaping."""
+ a = "a"
+ b = "b"
+ return DataFrame({"co$e^x$": {a: "a", b: "b"}, "co^l1": {a: "a", b: "b"}})
+
+ def test_to_latex_escape_false(self, df_with_symbols):
+ result = df_with_symbols.to_latex(escape=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lll}
+ \toprule
+ & co$e^x$ & co^l1 \\
+ \midrule
+ a & a & a \\
+ b & b & b \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_escape_default(self, df_with_symbols):
+ # gh50871: in v2.0 escape is False by default (styler.format.escape=None)
+ default = df_with_symbols.to_latex()
+ specified_true = df_with_symbols.to_latex(escape=True)
+ assert default != specified_true
+
+ def test_to_latex_special_escape(self):
+ df = DataFrame([r"a\b\c", r"^a^b^c", r"~a~b~c"])
+ result = df.to_latex(escape=True)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{ll}
+ \toprule
+ & 0 \\
+ \midrule
+ 0 & a\textbackslash b\textbackslash c \\
+ 1 & \textasciicircum a\textasciicircum b\textasciicircum c \\
+ 2 & \textasciitilde a\textasciitilde b\textasciitilde c \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_escape_special_chars(self):
+ special_characters = ["&", "%", "$", "#", "_", "{", "}", "~", "^", "\\"]
+ df = DataFrame(data=special_characters)
+ result = df.to_latex(escape=True)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{ll}
+ \toprule
+ & 0 \\
+ \midrule
+ 0 & \& \\
+ 1 & \% \\
+ 2 & \$ \\
+ 3 & \# \\
+ 4 & \_ \\
+ 5 & \{ \\
+ 6 & \} \\
+ 7 & \textasciitilde \\
+ 8 & \textasciicircum \\
+ 9 & \textbackslash \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_escape_special_chars_in_index_names(self):
+ # https://github.com/pandas-dev/pandas/issues/61309
+ # https://github.com/pandas-dev/pandas/issues/57362
+ index = "&%$#_{}}~^\\"
+ df = DataFrame({index: [1, 2, 3]}).set_index(index)
+ result = df.to_latex(escape=True)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{l}
+ \toprule
+ \&\%\$\#\_\{\}\}\textasciitilde \textasciicircum \textbackslash \\
+ \midrule
+ 1 \\
+ 2 \\
+ 3 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_escape_special_chars_in_column_name(self):
+ df = DataFrame({"A": [1, 2, 3], "B": ["a", "b", "c"]})
+ df.columns.name = "_^~"
+ result = df.to_latex(escape=True)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lrl}
+ \toprule
+ \_\textasciicircum \textasciitilde & A & B \\
+ \midrule
+ 0 & 1 & a \\
+ 1 & 2 & b \\
+ 2 & 3 & c \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_specified_header_special_chars_without_escape(self):
+ # GH 7124
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ result = df.to_latex(header=["$A$", "$B$"], escape=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lrl}
+ \toprule
+ & $A$ & $B$ \\
+ \midrule
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+
+class TestToLatexPosition:
+ def test_to_latex_position(self):
+ the_position = "h"
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ result = df.to_latex(position=the_position)
+ expected = _dedent(
+ r"""
+ \begin{table}[h]
+ \begin{tabular}{lrl}
+ \toprule
+ & a & b \\
+ \midrule
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \bottomrule
+ \end{tabular}
+ \end{table}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_longtable_position(self):
+ the_position = "t"
+ df = DataFrame({"a": [1, 2], "b": ["b1", "b2"]})
+ result = df.to_latex(longtable=True, position=the_position)
+ expected = _dedent(
+ r"""
+ \begin{longtable}[t]{lrl}
+ \toprule
+ & a & b \\
+ \midrule
+ \endfirsthead
+ \toprule
+ & a & b \\
+ \midrule
+ \endhead
+ \midrule
+ \multicolumn{3}{r}{Continued on next page} \\
+ \midrule
+ \endfoot
+ \bottomrule
+ \endlastfoot
+ 0 & 1 & b1 \\
+ 1 & 2 & b2 \\
+ \end{longtable}
+ """
+ )
+ assert result == expected
+
+
+class TestToLatexFormatters:
+ def test_to_latex_with_formatters(self):
+ df = DataFrame(
+ {
+ "datetime64": [
+ datetime(2016, 1, 1),
+ datetime(2016, 2, 5),
+ datetime(2016, 3, 3),
+ ],
+ "float": [1.0, 2.0, 3.0],
+ "int": [1, 2, 3],
+ "object": [(1, 2), True, False],
+ }
+ )
+
+ formatters = {
+ "datetime64": lambda x: x.strftime("%Y-%m"),
+ "float": lambda x: f"[{x: 4.1f}]",
+ "int": lambda x: f"0x{x:x}",
+ "object": lambda x: f"-{x!s}-",
+ "__index__": lambda x: f"index: {x}",
+ }
+ result = df.to_latex(formatters=dict(formatters))
+
+ expected = _dedent(
+ r"""
+ \begin{tabular}{llrrl}
+ \toprule
+ & datetime64 & float & int & object \\
+ \midrule
+ index: 0 & 2016-01 & [ 1.0] & 0x1 & -(1, 2)- \\
+ index: 1 & 2016-02 & [ 2.0] & 0x2 & -True- \\
+ index: 2 & 2016-03 & [ 3.0] & 0x3 & -False- \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_float_format_no_fixed_width_3decimals(self):
+ # GH 21625
+ df = DataFrame({"x": [0.19999]})
+ result = df.to_latex(float_format="%.3f")
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lr}
+ \toprule
+ & x \\
+ \midrule
+ 0 & 0.200 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_float_format_no_fixed_width_integer(self):
+ # GH 22270
+ df = DataFrame({"x": [100.0]})
+ result = df.to_latex(float_format="%.0f")
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lr}
+ \toprule
+ & x \\
+ \midrule
+ 0 & 100 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ @pytest.mark.parametrize("na_rep", ["NaN", "Ted"])
+ def test_to_latex_na_rep_and_float_format(self, na_rep):
+ df = DataFrame(
+ [
+ ["A", 1.2225],
+ ["A", None],
+ ],
+ columns=["Group", "Data"],
+ )
+ result = df.to_latex(na_rep=na_rep, float_format="{:.2f}".format)
+ expected = _dedent(
+ rf"""
+ \begin{{tabular}}{{llr}}
+ \toprule
+ & Group & Data \\
+ \midrule
+ 0 & A & 1.22 \\
+ 1 & A & {na_rep} \\
+ \bottomrule
+ \end{{tabular}}
+ """
+ )
+ assert result == expected
+
+
+class TestToLatexMultiindex:
+ @pytest.fixture
+ def multiindex_frame(self):
+ """Multiindex dataframe for testing multirow LaTeX macros."""
+ return DataFrame.from_dict(
+ {
+ ("c1", 0): Series({x: x for x in range(4)}),
+ ("c1", 1): Series({x: x + 4 for x in range(4)}),
+ ("c2", 0): Series({x: x for x in range(4)}),
+ ("c2", 1): Series({x: x + 4 for x in range(4)}),
+ ("c3", 0): Series({x: x for x in range(4)}),
+ }
+ ).T
+
+ @pytest.fixture
+ def multicolumn_frame(self):
+ """Multicolumn dataframe for testing multicolumn LaTeX macros."""
+ return DataFrame(
+ {
+ ("c1", 0): {x: x for x in range(5)},
+ ("c1", 1): {x: x + 5 for x in range(5)},
+ ("c2", 0): {x: x for x in range(5)},
+ ("c2", 1): {x: x + 5 for x in range(5)},
+ ("c3", 0): {x: x for x in range(5)},
+ }
+ )
+
+ def test_to_latex_multindex_header(self):
+ # GH 16718
+ df = DataFrame({"a": [0], "b": [1], "c": [2], "d": [3]})
+ df = df.set_index(["a", "b"])
+ observed = df.to_latex(header=["r1", "r2"], multirow=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{llrr}
+ \toprule
+ & & r1 & r2 \\
+ a & b & & \\
+ \midrule
+ 0 & 1 & 2 & 3 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert observed == expected
+
+ def test_to_latex_multiindex_empty_name(self):
+ # GH 18669
+ mi = pd.MultiIndex.from_product([[1, 2]], names=[""])
+ df = DataFrame(-1, index=mi, columns=range(4))
+ observed = df.to_latex()
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lrrrr}
+ \toprule
+ & 0 & 1 & 2 & 3 \\
+ & & & & \\
+ \midrule
+ 1 & -1 & -1 & -1 & -1 \\
+ 2 & -1 & -1 & -1 & -1 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert observed == expected
+
+ def test_to_latex_multiindex_column_tabular(self):
+ df = DataFrame({("x", "y"): ["a"]})
+ result = df.to_latex()
+ expected = _dedent(
+ r"""
+ \begin{tabular}{ll}
+ \toprule
+ & x \\
+ & y \\
+ \midrule
+ 0 & a \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_multiindex_small_tabular(self):
+ df = DataFrame({("x", "y"): ["a"]}).T
+ result = df.to_latex(multirow=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lll}
+ \toprule
+ & & 0 \\
+ \midrule
+ x & y & a \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_multiindex_tabular(self, multiindex_frame):
+ result = multiindex_frame.to_latex(multirow=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{llrrrr}
+ \toprule
+ & & 0 & 1 & 2 & 3 \\
+ \midrule
+ c1 & 0 & 0 & 1 & 2 & 3 \\
+ & 1 & 4 & 5 & 6 & 7 \\
+ c2 & 0 & 0 & 1 & 2 & 3 \\
+ & 1 & 4 & 5 & 6 & 7 \\
+ c3 & 0 & 0 & 1 & 2 & 3 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_multicolumn_tabular(self, multiindex_frame):
+ # GH 14184
+ df = multiindex_frame.T
+ df.columns.names = ["a", "b"]
+ result = df.to_latex(multirow=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lrrrrr}
+ \toprule
+ a & \multicolumn{2}{r}{c1} & \multicolumn{2}{r}{c2} & c3 \\
+ b & 0 & 1 & 0 & 1 & 0 \\
+ \midrule
+ 0 & 0 & 4 & 0 & 4 & 0 \\
+ 1 & 1 & 5 & 1 & 5 & 1 \\
+ 2 & 2 & 6 & 2 & 6 & 2 \\
+ 3 & 3 & 7 & 3 & 7 & 3 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_index_has_name_tabular(self):
+ # GH 10660
+ df = DataFrame({"a": [0, 0, 1, 1], "b": list("abab"), "c": [1, 2, 3, 4]})
+ result = df.set_index(["a", "b"]).to_latex(multirow=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{llr}
+ \toprule
+ & & c \\
+ a & b & \\
+ \midrule
+ 0 & a & 1 \\
+ & b & 2 \\
+ 1 & a & 3 \\
+ & b & 4 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_groupby_tabular(self):
+ # GH 10660
+ df = DataFrame({"a": [0, 0, 1, 1], "b": list("abab"), "c": [1, 2, 3, 4]})
+ result = (
+ df.groupby("a")
+ .describe()
+ .to_latex(float_format="{:.1f}".format, escape=True)
+ )
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lrrrrrrrr}
+ \toprule
+ & \multicolumn{8}{r}{c} \\
+ & count & mean & std & min & 25\% & 50\% & 75\% & max \\
+ a & & & & & & & & \\
+ \midrule
+ 0 & 2.0 & 1.5 & 0.7 & 1.0 & 1.2 & 1.5 & 1.8 & 2.0 \\
+ 1 & 2.0 & 3.5 & 0.7 & 3.0 & 3.2 & 3.5 & 3.8 & 4.0 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_multiindex_dupe_level(self):
+ # see gh-14484
+ #
+ # If an index is repeated in subsequent rows, it should be
+ # replaced with a blank in the created table. This should
+ # ONLY happen if all higher order indices (to the left) are
+ # equal too. In this test, 'c' has to be printed both times
+ # because the higher order index 'A' != 'B'.
+ df = DataFrame(
+ index=pd.MultiIndex.from_tuples([("A", "c"), ("B", "c")]), columns=["col"]
+ )
+ result = df.to_latex(multirow=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lll}
+ \toprule
+ & & col \\
+ \midrule
+ A & c & NaN \\
+ B & c & NaN \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_multicolumn_default(self, multicolumn_frame):
+ result = multicolumn_frame.to_latex()
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lrrrrr}
+ \toprule
+ & \multicolumn{2}{r}{c1} & \multicolumn{2}{r}{c2} & c3 \\
+ & 0 & 1 & 0 & 1 & 0 \\
+ \midrule
+ 0 & 0 & 5 & 0 & 5 & 0 \\
+ 1 & 1 & 6 & 1 & 6 & 1 \\
+ 2 & 2 & 7 & 2 & 7 & 2 \\
+ 3 & 3 & 8 & 3 & 8 & 3 \\
+ 4 & 4 & 9 & 4 & 9 & 4 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_multicolumn_false(self, multicolumn_frame):
+ result = multicolumn_frame.to_latex(multicolumn=False, multicolumn_format="l")
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lrrrrr}
+ \toprule
+ & c1 & & c2 & & c3 \\
+ & 0 & 1 & 0 & 1 & 0 \\
+ \midrule
+ 0 & 0 & 5 & 0 & 5 & 0 \\
+ 1 & 1 & 6 & 1 & 6 & 1 \\
+ 2 & 2 & 7 & 2 & 7 & 2 \\
+ 3 & 3 & 8 & 3 & 8 & 3 \\
+ 4 & 4 & 9 & 4 & 9 & 4 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_multirow_true(self, multicolumn_frame):
+ result = multicolumn_frame.T.to_latex(multirow=True)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{llrrrrr}
+ \toprule
+ & & 0 & 1 & 2 & 3 & 4 \\
+ \midrule
+ \multirow[t]{2}{*}{c1} & 0 & 0 & 1 & 2 & 3 & 4 \\
+ & 1 & 5 & 6 & 7 & 8 & 9 \\
+ \cline{1-7}
+ \multirow[t]{2}{*}{c2} & 0 & 0 & 1 & 2 & 3 & 4 \\
+ & 1 & 5 & 6 & 7 & 8 & 9 \\
+ \cline{1-7}
+ c3 & 0 & 0 & 1 & 2 & 3 & 4 \\
+ \cline{1-7}
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_multicolumnrow_with_multicol_format(self, multicolumn_frame):
+ multicolumn_frame.index = multicolumn_frame.T.index
+ result = multicolumn_frame.T.to_latex(
+ multirow=True,
+ multicolumn=True,
+ multicolumn_format="c",
+ )
+ expected = _dedent(
+ r"""
+ \begin{tabular}{llrrrrr}
+ \toprule
+ & & \multicolumn{2}{c}{c1} & \multicolumn{2}{c}{c2} & c3 \\
+ & & 0 & 1 & 0 & 1 & 0 \\
+ \midrule
+ \multirow[t]{2}{*}{c1} & 0 & 0 & 1 & 2 & 3 & 4 \\
+ & 1 & 5 & 6 & 7 & 8 & 9 \\
+ \cline{1-7}
+ \multirow[t]{2}{*}{c2} & 0 & 0 & 1 & 2 & 3 & 4 \\
+ & 1 & 5 & 6 & 7 & 8 & 9 \\
+ \cline{1-7}
+ c3 & 0 & 0 & 1 & 2 & 3 & 4 \\
+ \cline{1-7}
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ @pytest.mark.parametrize("name0", [None, "named0"])
+ @pytest.mark.parametrize("name1", [None, "named1"])
+ @pytest.mark.parametrize("axes", [[0], [1], [0, 1]])
+ def test_to_latex_multiindex_names(self, name0, name1, axes):
+ # GH 18667
+ names = [name0, name1]
+ mi = pd.MultiIndex.from_product([[1, 2], [3, 4]])
+ df = DataFrame(-1, index=mi.copy(), columns=mi.copy())
+ for idx in axes:
+ df.axes[idx].names = names
+
+ idx_names = tuple(n or "" for n in names)
+ idx_names_row = (
+ f"{idx_names[0]} & {idx_names[1]} & & & & \\\\\n"
+ if (0 in axes and any(names))
+ else ""
+ )
+ col_names = [n if (bool(n) and 1 in axes) else "" for n in names]
+ observed = df.to_latex(multirow=False)
+ expected = r"""\begin{{tabular}}{{llrrrr}}
+\toprule
+ & {} & \multicolumn{{2}}{{r}}{{1}} & \multicolumn{{2}}{{r}}{{2}} \\
+ & {} & 3 & 4 & 3 & 4 \\
+{}\midrule
+1 & 3 & -1 & -1 & -1 & -1 \\
+ & 4 & -1 & -1 & -1 & -1 \\
+2 & 3 & -1 & -1 & -1 & -1 \\
+ & 4 & -1 & -1 & -1 & -1 \\
+\bottomrule
+\end{{tabular}}
+""".format(*col_names, idx_names_row)
+ assert observed == expected
+
+ @pytest.mark.parametrize("one_row", [True, False])
+ def test_to_latex_multiindex_nans(self, one_row):
+ # GH 14249
+ df = DataFrame({"a": [None, 1], "b": [2, 3], "c": [4, 5]})
+ if one_row:
+ df = df.iloc[[0]]
+ observed = df.set_index(["a", "b"]).to_latex(multirow=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{llr}
+ \toprule
+ & & c \\
+ a & b & \\
+ \midrule
+ NaN & 2 & 4 \\
+ """
+ )
+ if not one_row:
+ expected += r"""1.000000 & 3 & 5 \\
+"""
+ expected += r"""\bottomrule
+\end{tabular}
+"""
+ assert observed == expected
+
+ def test_to_latex_non_string_index(self):
+ # GH 19981
+ df = DataFrame([[1, 2, 3]] * 2).set_index([0, 1])
+ result = df.to_latex(multirow=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{llr}
+ \toprule
+ & & 2 \\
+ 0 & 1 & \\
+ \midrule
+ 1 & 2 & 3 \\
+ & 2 & 3 \\
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_multiindex_multirow(self):
+ # GH 16719
+ mi = pd.MultiIndex.from_product(
+ [[0.0, 1.0], [3.0, 2.0, 1.0], ["0", "1"]], names=["i", "val0", "val1"]
+ )
+ df = DataFrame(index=mi)
+ result = df.to_latex(multirow=True, escape=False)
+ expected = _dedent(
+ r"""
+ \begin{tabular}{lll}
+ \toprule
+ i & val0 & val1 \\
+ \midrule
+ \multirow[t]{6}{*}{0.000000} & \multirow[t]{2}{*}{3.000000} & 0 \\
+ & & 1 \\
+ \cline{2-3}
+ & \multirow[t]{2}{*}{2.000000} & 0 \\
+ & & 1 \\
+ \cline{2-3}
+ & \multirow[t]{2}{*}{1.000000} & 0 \\
+ & & 1 \\
+ \cline{1-3} \cline{2-3}
+ \multirow[t]{6}{*}{1.000000} & \multirow[t]{2}{*}{3.000000} & 0 \\
+ & & 1 \\
+ \cline{2-3}
+ & \multirow[t]{2}{*}{2.000000} & 0 \\
+ & & 1 \\
+ \cline{2-3}
+ & \multirow[t]{2}{*}{1.000000} & 0 \\
+ & & 1 \\
+ \cline{1-3} \cline{2-3}
+ \bottomrule
+ \end{tabular}
+ """
+ )
+ assert result == expected
+
+ def test_to_latex_multiindex_format_single_index_hidden(self):
+ # GH 52218
+ df = DataFrame(
+ {
+ "A": [1, 2],
+ "B": [4, 5],
+ }
+ )
+ result = (
+ df.style.hide(axis="index")
+ .map_index(lambda v: "textbf:--rwrap;", axis="columns")
+ .to_latex()
+ )
+ expected = _dedent(r"""
+ \begin{tabular}{rr}
+ \textbf{A} & \textbf{B} \\
+ 1 & 4 \\
+ 2 & 5 \\
+ \end{tabular}
+ """)
+ assert result == expected
+
+ def test_to_latex_multiindex_format_triple_index_two_hidden(self):
+ # GH 52218
+ arrays = [
+ ["A", "A", "B", "B"],
+ ["one", "two", "one", "two"],
+ ["x", "x", "y", "y"],
+ ]
+ index = pd.MultiIndex.from_arrays(
+ arrays, names=["Level 0", "Level 1", "Level 2"]
+ )
+ df = DataFrame(
+ [[0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0]],
+ index=index,
+ columns=["C1", "C2", "C3"],
+ )
+ result = (
+ df.style.hide(axis="index", level=[0, 1])
+ .map_index(lambda v: "textbf:--rwrap;", axis="columns")
+ .to_latex()
+ )
+ expected = _dedent(r"""
+ \begin{tabular}{lrrr}
+ & \textbf{C1} & \textbf{C2} & \textbf{C3} \\
+ Level 2 & & & \\
+ x & 0 & 0 & 0 \\
+ x & 0 & 0 & 0 \\
+ y & 0 & 0 & 0 \\
+ y & 0 & 0 & 0 \\
+ \end{tabular}
+ """)
+ assert result == expected
+
+ def test_to_latex_multiindex_format_triple_index_all_hidden(self):
+ # GH 52218
+ arrays = [
+ ["A", "A", "B", "B"],
+ ["one", "two", "one", "two"],
+ ["x", "x", "y", "y"],
+ ]
+ index = pd.MultiIndex.from_arrays(
+ arrays, names=["Level 0", "Level 1", "Level 2"]
+ )
+ df = DataFrame(
+ [[0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0]],
+ index=index,
+ columns=["C1", "C2", "C3"],
+ )
+ result = (
+ df.style.hide(axis="index", level=[0, 1, 2])
+ .map_index(lambda v: "textbf:--rwrap;", axis="columns")
+ .to_latex()
+ )
+ expected = _dedent(r"""
+ \begin{tabular}{rrr}
+ \textbf{C1} & \textbf{C2} & \textbf{C3} \\
+ 0 & 0 & 0 \\
+ 0 & 0 & 0 \\
+ 0 & 0 & 0 \\
+ 0 & 0 & 0 \\
+ \end{tabular}
+ """)
+ assert result == expected
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_markdown.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_markdown.py
new file mode 100644
index 0000000000000000000000000000000000000000..9b1e22ce1bb06108eeb13ff9076531ea96f08f47
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_markdown.py
@@ -0,0 +1,102 @@
+from io import StringIO
+
+import pytest
+
+from pandas.errors import Pandas4Warning
+
+import pandas as pd
+import pandas._testing as tm
+
+pytest.importorskip("tabulate")
+
+
+def test_keyword_deprecation():
+ # GH 57280
+ msg = (
+ "Starting with pandas version 4.0 all arguments of to_markdown "
+ "except for the argument 'buf' will be keyword-only."
+ )
+ s = pd.Series()
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ s.to_markdown(None, "wt")
+
+
+def test_simple():
+ buf = StringIO()
+ df = pd.DataFrame([1, 2, 3])
+ df.to_markdown(buf=buf)
+ result = buf.getvalue()
+ assert (
+ result == "| | 0 |\n|---:|----:|\n| 0 | 1 |\n| 1 | 2 |\n| 2 | 3 |"
+ )
+
+
+def test_empty_frame():
+ buf = StringIO()
+ df = pd.DataFrame({"id": [], "first_name": [], "last_name": []}).set_index("id")
+ df.to_markdown(buf=buf)
+ result = buf.getvalue()
+ assert result == (
+ "| id | first_name | last_name |\n|------|--------------|-------------|"
+ )
+
+
+def test_other_tablefmt():
+ buf = StringIO()
+ df = pd.DataFrame([1, 2, 3])
+ df.to_markdown(buf=buf, tablefmt="jira")
+ result = buf.getvalue()
+ assert result == "|| || 0 ||\n| 0 | 1 |\n| 1 | 2 |\n| 2 | 3 |"
+
+
+def test_other_headers():
+ buf = StringIO()
+ df = pd.DataFrame([1, 2, 3])
+ df.to_markdown(buf=buf, headers=["foo", "bar"])
+ result = buf.getvalue()
+ assert result == (
+ "| foo | bar |\n|------:|------:|\n| 0 "
+ "| 1 |\n| 1 | 2 |\n| 2 | 3 |"
+ )
+
+
+def test_series():
+ buf = StringIO()
+ s = pd.Series([1, 2, 3], name="foo")
+ s.to_markdown(buf=buf)
+ result = buf.getvalue()
+ assert result == (
+ "| | foo |\n|---:|------:|\n| 0 | 1 |\n| 1 | 2 |\n| 2 | 3 |"
+ )
+
+
+def test_no_buf():
+ df = pd.DataFrame([1, 2, 3])
+ result = df.to_markdown()
+ assert (
+ result == "| | 0 |\n|---:|----:|\n| 0 | 1 |\n| 1 | 2 |\n| 2 | 3 |"
+ )
+
+
+@pytest.mark.parametrize("index", [True, False])
+def test_index(index):
+ # GH 32667
+
+ df = pd.DataFrame([1, 2, 3])
+
+ result = df.to_markdown(index=index)
+
+ if index:
+ expected = (
+ "| | 0 |\n|---:|----:|\n| 0 | 1 |\n| 1 | 2 |\n| 2 | 3 |"
+ )
+ else:
+ expected = "| 0 |\n|----:|\n| 1 |\n| 2 |\n| 3 |"
+ assert result == expected
+
+
+def test_showindex_disallowed_in_kwargs():
+ # GH 32667; disallowing showindex in kwargs enforced in 2.0
+ df = pd.DataFrame([1, 2, 3])
+ with pytest.raises(ValueError, match="Pass 'index' instead of 'showindex"):
+ df.to_markdown(index=True, showindex=True)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_string.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_string.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d414cef152c71ea35e10f8c910a1c39a5a051ae
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/formats/test_to_string.py
@@ -0,0 +1,1226 @@
+from datetime import (
+ datetime,
+ timedelta,
+)
+from io import StringIO
+import re
+import sys
+from textwrap import dedent
+
+import numpy as np
+import pytest
+
+from pandas._config import using_string_dtype
+
+from pandas.errors import Pandas4Warning
+
+from pandas import (
+ CategoricalIndex,
+ DataFrame,
+ Index,
+ NaT,
+ Series,
+ Timestamp,
+ concat,
+ date_range,
+ get_option,
+ option_context,
+ read_csv,
+ timedelta_range,
+ to_datetime,
+)
+import pandas._testing as tm
+
+
+def _three_digit_exp():
+ return f"{1.7e8:.4g}" == "1.7e+008"
+
+
+class TestDataFrameToStringFormatters:
+ def test_keyword_deprecation(self):
+ # GH 57280
+ msg = (
+ "Starting with pandas version 4.0 all arguments of to_string "
+ "except for the argument 'buf' will be keyword-only."
+ )
+ s = Series(["a", "b"])
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ s.to_string(None, "NaN")
+
+ def test_to_string_masked_ea_with_formatter(self):
+ # GH#39336
+ df = DataFrame(
+ {
+ "a": Series([0.123456789, 1.123456789], dtype="Float64"),
+ "b": Series([1, 2], dtype="Int64"),
+ }
+ )
+ result = df.to_string(formatters=["{:.2f}".format, "{:.2f}".format])
+ expected = dedent(
+ """\
+ a b
+ 0 0.12 1.00
+ 1 1.12 2.00"""
+ )
+ assert result == expected
+
+ def test_to_string_with_formatters(self):
+ df = DataFrame(
+ {
+ "int": [1, 2, 3],
+ "float": [1.0, 2.0, 3.0],
+ "object": [(1, 2), True, False],
+ },
+ columns=["int", "float", "object"],
+ )
+
+ formatters = [
+ ("int", lambda x: f"0x{x:x}"),
+ ("float", lambda x: f"[{x: 4.1f}]"),
+ ("object", lambda x: f"-{x!s}-"),
+ ]
+ result = df.to_string(formatters=dict(formatters))
+ result2 = df.to_string(formatters=list(zip(*formatters))[1])
+ assert result == (
+ " int float object\n"
+ "0 0x1 [ 1.0] -(1, 2)-\n"
+ "1 0x2 [ 2.0] -True-\n"
+ "2 0x3 [ 3.0] -False-"
+ )
+ assert result == result2
+
+ def test_to_string_with_datetime64_monthformatter(self):
+ months = [datetime(2016, 1, 1), datetime(2016, 2, 2)]
+ x = DataFrame({"months": months})
+
+ def format_func(x):
+ return x.strftime("%Y-%m")
+
+ result = x.to_string(formatters={"months": format_func})
+ expected = dedent(
+ """\
+ months
+ 0 2016-01
+ 1 2016-02"""
+ )
+ assert result.strip() == expected
+
+ def test_to_string_with_datetime64_hourformatter(self):
+ x = DataFrame(
+ {"hod": to_datetime(["10:10:10.100", "12:12:12.120"], format="%H:%M:%S.%f")}
+ )
+
+ def format_func(x):
+ return x.strftime("%H:%M")
+
+ result = x.to_string(formatters={"hod": format_func})
+ expected = dedent(
+ """\
+ hod
+ 0 10:10
+ 1 12:12"""
+ )
+ assert result.strip() == expected
+
+ def test_to_string_with_formatters_unicode(self):
+ df = DataFrame({"c/\u03c3": [1, 2, 3]})
+ result = df.to_string(formatters={"c/\u03c3": str})
+ expected = dedent(
+ """\
+ c/\u03c3
+ 0 1
+ 1 2
+ 2 3"""
+ )
+ assert result == expected
+
+ def test_to_string_index_formatter(self):
+ df = DataFrame([range(5), range(5, 10), range(10, 15)])
+ rs = df.to_string(formatters={"__index__": lambda x: "abc"[x]})
+ xp = dedent(
+ """\
+ 0 1 2 3 4
+ a 0 1 2 3 4
+ b 5 6 7 8 9
+ c 10 11 12 13 14"""
+ )
+ assert rs == xp
+
+ def test_no_extra_space(self):
+ # GH#52690: Check that no extra space is given
+ col1 = "TEST"
+ col2 = "PANDAS"
+ col3 = "to_string"
+ expected = f"{col1:<6s} {col2:<7s} {col3:<10s}"
+ df = DataFrame([{"col1": "TEST", "col2": "PANDAS", "col3": "to_string"}])
+ d = {"col1": "{:<6s}".format, "col2": "{:<7s}".format, "col3": "{:<10s}".format}
+ result = df.to_string(index=False, header=False, formatters=d)
+ assert result == expected
+
+
+class TestDataFrameToStringColSpace:
+ def test_to_string_with_column_specific_col_space_raises(self):
+ df = DataFrame(
+ np.random.default_rng(2).random(size=(3, 3)), columns=["a", "b", "c"]
+ )
+
+ msg = (
+ "Col_space length\\(\\d+\\) should match "
+ "DataFrame number of columns\\(\\d+\\)"
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.to_string(col_space=[30, 40])
+
+ with pytest.raises(ValueError, match=msg):
+ df.to_string(col_space=[30, 40, 50, 60])
+
+ msg = "unknown column"
+ with pytest.raises(ValueError, match=msg):
+ df.to_string(col_space={"a": "foo", "b": 23, "d": 34})
+
+ def test_to_string_with_column_specific_col_space(self):
+ df = DataFrame(
+ np.random.default_rng(2).random(size=(3, 3)), columns=["a", "b", "c"]
+ )
+
+ result = df.to_string(col_space={"a": 10, "b": 11, "c": 12})
+ # 3 separating space + each col_space for (id, a, b, c)
+ assert len(result.split("\n")[1]) == (3 + 1 + 10 + 11 + 12)
+
+ result = df.to_string(col_space=[10, 11, 12])
+ assert len(result.split("\n")[1]) == (3 + 1 + 10 + 11 + 12)
+
+ def test_to_string_with_col_space(self):
+ df = DataFrame(np.random.default_rng(2).random(size=(1, 3)))
+ c10 = len(df.to_string(col_space=10).split("\n")[1])
+ c20 = len(df.to_string(col_space=20).split("\n")[1])
+ c30 = len(df.to_string(col_space=30).split("\n")[1])
+ assert c10 < c20 < c30
+
+ # GH#8230
+ # col_space wasn't being applied with header=False
+ with_header = df.to_string(col_space=20)
+ with_header_row1 = with_header.splitlines()[1]
+ no_header = df.to_string(col_space=20, header=False)
+ assert len(with_header_row1) == len(no_header)
+
+ def test_to_string_repr_tuples(self):
+ buf = StringIO()
+
+ df = DataFrame({"tups": list(zip(range(10), range(10)))})
+ repr(df)
+ df.to_string(col_space=10, buf=buf)
+
+
+class TestDataFrameToStringHeader:
+ def test_to_string_header_false(self):
+ # GH#49230
+ df = DataFrame([1, 2])
+ df.index.name = "a"
+ s = df.to_string(header=False)
+ expected = "a \n0 1\n1 2"
+ assert s == expected
+
+ df = DataFrame([[1, 2], [3, 4]])
+ df.index.name = "a"
+ s = df.to_string(header=False)
+ expected = "a \n0 1 2\n1 3 4"
+ assert s == expected
+
+ def test_to_string_multindex_header(self):
+ # GH#16718
+ df = DataFrame({"a": [0], "b": [1], "c": [2], "d": [3]}).set_index(["a", "b"])
+ res = df.to_string(header=["r1", "r2"])
+ exp = " r1 r2\na b \n0 1 2 3"
+ assert res == exp
+
+ def test_to_string_no_header(self):
+ df = DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]})
+
+ df_s = df.to_string(header=False)
+ expected = "0 1 4\n1 2 5\n2 3 6"
+
+ assert df_s == expected
+
+ def test_to_string_specified_header(self):
+ df = DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]})
+
+ df_s = df.to_string(header=["X", "Y"])
+ expected = " X Y\n0 1 4\n1 2 5\n2 3 6"
+
+ assert df_s == expected
+
+ msg = "Writing 2 cols but got 1 aliases"
+ with pytest.raises(ValueError, match=msg):
+ df.to_string(header=["X"])
+
+
+class TestDataFrameToStringLineWidth:
+ def test_to_string_line_width(self):
+ df = DataFrame(123, index=range(10, 15), columns=range(30))
+ lines = df.to_string(line_width=80)
+ assert max(len(line) for line in lines.split("\n")) == 80
+
+ def test_to_string_line_width_no_index(self):
+ # GH#13998, GH#22505
+ df = DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]})
+
+ df_s = df.to_string(line_width=1, index=False)
+ expected = " x \\\n 1 \n 2 \n 3 \n\n y \n 4 \n 5 \n 6 "
+
+ assert df_s == expected
+
+ df = DataFrame({"x": [11, 22, 33], "y": [4, 5, 6]})
+
+ df_s = df.to_string(line_width=1, index=False)
+ expected = " x \\\n11 \n22 \n33 \n\n y \n 4 \n 5 \n 6 "
+
+ assert df_s == expected
+
+ df = DataFrame({"x": [11, 22, -33], "y": [4, 5, -6]})
+
+ df_s = df.to_string(line_width=1, index=False)
+ expected = " x \\\n 11 \n 22 \n-33 \n\n y \n 4 \n 5 \n-6 "
+
+ assert df_s == expected
+
+ def test_to_string_line_width_no_header(self):
+ # GH#53054
+ df = DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]})
+
+ df_s = df.to_string(line_width=1, header=False)
+ expected = "0 1 \\\n1 2 \n2 3 \n\n0 4 \n1 5 \n2 6 "
+
+ assert df_s == expected
+
+ df = DataFrame({"x": [11, 22, 33], "y": [4, 5, 6]})
+
+ df_s = df.to_string(line_width=1, header=False)
+ expected = "0 11 \\\n1 22 \n2 33 \n\n0 4 \n1 5 \n2 6 "
+
+ assert df_s == expected
+
+ df = DataFrame({"x": [11, 22, -33], "y": [4, 5, -6]})
+
+ df_s = df.to_string(line_width=1, header=False)
+ expected = "0 11 \\\n1 22 \n2 -33 \n\n0 4 \n1 5 \n2 -6 "
+
+ assert df_s == expected
+
+ def test_to_string_line_width_with_both_index_and_header(self):
+ # GH#53054
+ df = DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]})
+
+ df_s = df.to_string(line_width=1)
+ expected = (
+ " x \\\n0 1 \n1 2 \n2 3 \n\n y \n0 4 \n1 5 \n2 6 "
+ )
+
+ assert df_s == expected
+
+ df = DataFrame({"x": [11, 22, 33], "y": [4, 5, 6]})
+
+ df_s = df.to_string(line_width=1)
+ expected = (
+ " x \\\n0 11 \n1 22 \n2 33 \n\n y \n0 4 \n1 5 \n2 6 "
+ )
+
+ assert df_s == expected
+
+ df = DataFrame({"x": [11, 22, -33], "y": [4, 5, -6]})
+
+ df_s = df.to_string(line_width=1)
+ expected = (
+ " x \\\n0 11 \n1 22 \n2 -33 \n\n y \n0 4 \n1 5 \n2 -6 "
+ )
+
+ assert df_s == expected
+
+ def test_to_string_line_width_no_index_no_header(self):
+ # GH#53054
+ df = DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]})
+
+ df_s = df.to_string(line_width=1, index=False, header=False)
+ expected = "1 \\\n2 \n3 \n\n4 \n5 \n6 "
+
+ assert df_s == expected
+
+ df = DataFrame({"x": [11, 22, 33], "y": [4, 5, 6]})
+
+ df_s = df.to_string(line_width=1, index=False, header=False)
+ expected = "11 \\\n22 \n33 \n\n4 \n5 \n6 "
+
+ assert df_s == expected
+
+ df = DataFrame({"x": [11, 22, -33], "y": [4, 5, -6]})
+
+ df_s = df.to_string(line_width=1, index=False, header=False)
+ expected = " 11 \\\n 22 \n-33 \n\n 4 \n 5 \n-6 "
+
+ assert df_s == expected
+
+
+class TestToStringNumericFormatting:
+ def test_to_string_float_format_no_fixed_width(self):
+ # GH#21625
+ df = DataFrame({"x": [0.19999]})
+ expected = " x\n0 0.200"
+ assert df.to_string(float_format="%.3f") == expected
+
+ # GH#22270
+ df = DataFrame({"x": [100.0]})
+ expected = " x\n0 100"
+ assert df.to_string(float_format="%.0f") == expected
+
+ def test_to_string_small_float_values(self):
+ df = DataFrame({"a": [1.5, 1e-17, -5.5e-7]})
+
+ result = df.to_string()
+ # sadness per above
+ if _three_digit_exp():
+ expected = (
+ " a\n0 1.500000e+000\n1 1.000000e-017\n2 -5.500000e-007"
+ )
+ else:
+ expected = (
+ " a\n0 1.500000e+00\n1 1.000000e-17\n2 -5.500000e-07"
+ )
+ assert result == expected
+
+ # but not all exactly zero
+ df = df * 0
+ result = df.to_string()
+ expected = " 0\n0 0\n1 0\n2 -0"
+ # TODO: assert that these match??
+
+ def test_to_string_complex_float_formatting(self):
+ # GH #25514, 25745
+ with option_context("display.precision", 5):
+ df = DataFrame(
+ {
+ "x": [
+ (0.4467846931321966 + 0.0715185102060818j),
+ (0.2739442392974528 + 0.23515228785438969j),
+ (0.26974928742135185 + 0.3250604054898979j),
+ (-1j),
+ ]
+ }
+ )
+ result = df.to_string()
+ expected = (
+ " x\n0 0.44678+0.07152j\n"
+ "1 0.27394+0.23515j\n"
+ "2 0.26975+0.32506j\n"
+ "3 -0.00000-1.00000j"
+ )
+ assert result == expected
+
+ def test_to_string_complex_float_formatting_with_exponents(self):
+ # GH #60393
+ with option_context("display.precision", 6):
+ df = DataFrame(
+ {
+ "x": [
+ (1.8816e-09 + 0j),
+ (1.8816e-09 + 3.39676e-09j),
+ ]
+ }
+ )
+ result = df.to_string()
+ expected = (
+ " x\n0 1.881600e-09+0.000000e+00j\n"
+ "1 1.881600e-09+3.396760e-09j"
+ )
+ assert result == expected
+
+ def test_to_string_format_inf(self):
+ # GH#24861
+ df = DataFrame(
+ {
+ "A": [-np.inf, np.inf, -1, -2.1234, 3, 4],
+ "B": [-np.inf, np.inf, "foo", "foooo", "fooooo", "bar"],
+ }
+ )
+ result = df.to_string()
+
+ expected = (
+ " A B\n"
+ "0 -inf -inf\n"
+ "1 inf inf\n"
+ "2 -1.0000 foo\n"
+ "3 -2.1234 foooo\n"
+ "4 3.0000 fooooo\n"
+ "5 4.0000 bar"
+ )
+ assert result == expected
+
+ df = DataFrame(
+ {
+ "A": [-np.inf, np.inf, -1.0, -2.0, 3.0, 4.0],
+ "B": [-np.inf, np.inf, "foo", "foooo", "fooooo", "bar"],
+ }
+ )
+ result = df.to_string()
+
+ expected = (
+ " A B\n"
+ "0 -inf -inf\n"
+ "1 inf inf\n"
+ "2 -1.0 foo\n"
+ "3 -2.0 foooo\n"
+ "4 3.0 fooooo\n"
+ "5 4.0 bar"
+ )
+ assert result == expected
+
+ def test_to_string_int_formatting(self):
+ df = DataFrame({"x": [-15, 20, 25, -35]})
+ assert issubclass(df["x"].dtype.type, np.integer)
+
+ output = df.to_string()
+ expected = " x\n0 -15\n1 20\n2 25\n3 -35"
+ assert output == expected
+
+ def test_to_string_float_formatting(self):
+ with option_context(
+ "display.precision",
+ 5,
+ "display.notebook_repr_html",
+ False,
+ ):
+ df = DataFrame(
+ {"x": [0, 0.25, 3456.000, 12e45, 1.64e6, 1.7e8, 1.253456, np.pi, -1e6]}
+ )
+
+ df_s = df.to_string()
+
+ if _three_digit_exp():
+ expected = (
+ " x\n0 0.00000e+000\n1 2.50000e-001\n"
+ "2 3.45600e+003\n3 1.20000e+046\n4 1.64000e+006\n"
+ "5 1.70000e+008\n6 1.25346e+000\n7 3.14159e+000\n"
+ "8 -1.00000e+006"
+ )
+ else:
+ expected = (
+ " x\n0 0.00000e+00\n1 2.50000e-01\n"
+ "2 3.45600e+03\n3 1.20000e+46\n4 1.64000e+06\n"
+ "5 1.70000e+08\n6 1.25346e+00\n7 3.14159e+00\n"
+ "8 -1.00000e+06"
+ )
+ assert df_s == expected
+
+ df = DataFrame({"x": [3234, 0.253]})
+ df_s = df.to_string()
+
+ expected = " x\n0 3234.000\n1 0.253"
+ assert df_s == expected
+
+ assert get_option("display.precision") == 6
+
+ df = DataFrame({"x": [1e9, 0.2512]})
+ df_s = df.to_string()
+
+ if _three_digit_exp():
+ expected = " x\n0 1.000000e+009\n1 2.512000e-001"
+ else:
+ expected = " x\n0 1.000000e+09\n1 2.512000e-01"
+ assert df_s == expected
+
+
+class TestDataFrameToString:
+ def test_to_string_decimal(self):
+ # GH#23614
+ df = DataFrame({"A": [6.0, 3.1, 2.2]})
+ expected = " A\n0 6,0\n1 3,1\n2 2,2"
+ assert df.to_string(decimal=",") == expected
+
+ def test_to_string_left_justify_cols(self):
+ df = DataFrame({"x": [3234, 0.253]})
+ df_s = df.to_string(justify="left")
+ expected = " x \n0 3234.000\n1 0.253"
+ assert df_s == expected
+
+ def test_to_string_format_na(self):
+ df = DataFrame(
+ {
+ "A": [np.nan, -1, -2.1234, 3, 4],
+ "B": [np.nan, "foo", "foooo", "fooooo", "bar"],
+ }
+ )
+ result = df.to_string()
+
+ expected = (
+ " A B\n"
+ "0 NaN NaN\n"
+ "1 -1.0000 foo\n"
+ "2 -2.1234 foooo\n"
+ "3 3.0000 fooooo\n"
+ "4 4.0000 bar"
+ )
+ assert result == expected
+
+ df = DataFrame(
+ {
+ "A": [np.nan, -1.0, -2.0, 3.0, 4.0],
+ "B": [np.nan, "foo", "foooo", "fooooo", "bar"],
+ }
+ )
+ result = df.to_string()
+
+ expected = (
+ " A B\n"
+ "0 NaN NaN\n"
+ "1 -1.0 foo\n"
+ "2 -2.0 foooo\n"
+ "3 3.0 fooooo\n"
+ "4 4.0 bar"
+ )
+ assert result == expected
+
+ def test_to_string_with_dict_entries(self):
+ df = DataFrame({"A": [{"a": 1, "b": 2}]})
+
+ val = df.to_string()
+ assert "'a': 1" in val
+ assert "'b': 2" in val
+
+ def test_to_string_with_categorical_columns(self):
+ # GH#35439
+ data = [[4, 2], [3, 2], [4, 3]]
+ cols = ["aaaaaaaaa", "b"]
+ df = DataFrame(data, columns=cols)
+ df_cat_cols = DataFrame(data, columns=CategoricalIndex(cols))
+
+ assert df.to_string() == df_cat_cols.to_string()
+
+ def test_repr_embedded_ndarray(self):
+ arr = np.empty(10, dtype=[("err", object)])
+ for i in range(len(arr)):
+ arr["err"][i] = np.random.default_rng(2).standard_normal(i)
+
+ df = DataFrame(arr)
+ repr(df["err"])
+ repr(df)
+ df.to_string()
+
+ def test_to_string_truncate(self):
+ # GH 9784 - dont truncate when calling DataFrame.to_string
+ df = DataFrame(
+ [
+ {
+ "a": "foo",
+ "b": "bar",
+ "c": "let's make this a very VERY long line that is longer "
+ "than the default 50 character limit",
+ "d": 1,
+ },
+ {"a": "foo", "b": "bar", "c": "stuff", "d": 1},
+ ]
+ )
+ df.set_index(["a", "b", "c"])
+ assert df.to_string() == (
+ " a b "
+ " c d\n"
+ "0 foo bar let's make this a very VERY long line t"
+ "hat is longer than the default 50 character limit 1\n"
+ "1 foo bar "
+ " stuff 1"
+ )
+ with option_context("max_colwidth", 20):
+ # the display option has no effect on the to_string method
+ assert df.to_string() == (
+ " a b "
+ " c d\n"
+ "0 foo bar let's make this a very VERY long line t"
+ "hat is longer than the default 50 character limit 1\n"
+ "1 foo bar "
+ " stuff 1"
+ )
+ assert df.to_string(max_colwidth=20) == (
+ " a b c d\n"
+ "0 foo bar let's make this ... 1\n"
+ "1 foo bar stuff 1"
+ )
+
+ @pytest.mark.parametrize(
+ "input_array, expected",
+ [
+ ({"A": ["a"]}, "A\na"),
+ ({"A": ["a", "b"], "B": ["c", "dd"]}, "A B\na c\nb dd"),
+ ({"A": ["a", 1], "B": ["aa", 1]}, "A B\na aa\n1 1"),
+ ],
+ )
+ def test_format_remove_leading_space_dataframe(self, input_array, expected):
+ # GH#24980
+ df = DataFrame(input_array).to_string(index=False)
+ assert df == expected
+
+ @pytest.mark.parametrize(
+ "data,expected",
+ [
+ (
+ {"col1": [1, 2], "col2": [3, 4]},
+ " col1 col2\n0 1 3\n1 2 4",
+ ),
+ (
+ {"col1": ["Abc", 0.756], "col2": [np.nan, 4.5435]},
+ " col1 col2\n0 Abc NaN\n1 0.756 4.5435",
+ ),
+ (
+ {"col1": [np.nan, "a"], "col2": [0.009, 3.543], "col3": ["Abc", 23]},
+ " col1 col2 col3\n0 NaN 0.009 Abc\n1 a 3.543 23",
+ ),
+ ],
+ )
+ def test_to_string_max_rows_zero(self, data, expected):
+ # GH#35394
+ result = DataFrame(data=data).to_string(max_rows=0)
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "max_cols, max_rows, expected",
+ [
+ (
+ 10,
+ None,
+ " 0 1 2 3 4 ... 6 7 8 9 10\n"
+ " 0 0 0 0 0 ... 0 0 0 0 0\n"
+ " 0 0 0 0 0 ... 0 0 0 0 0\n"
+ " 0 0 0 0 0 ... 0 0 0 0 0\n"
+ " 0 0 0 0 0 ... 0 0 0 0 0",
+ ),
+ (
+ None,
+ 2,
+ " 0 1 2 3 4 5 6 7 8 9 10\n"
+ " 0 0 0 0 0 0 0 0 0 0 0\n"
+ " .. .. .. .. .. .. .. .. .. .. ..\n"
+ " 0 0 0 0 0 0 0 0 0 0 0",
+ ),
+ (
+ 10,
+ 2,
+ " 0 1 2 3 4 ... 6 7 8 9 10\n"
+ " 0 0 0 0 0 ... 0 0 0 0 0\n"
+ " .. .. .. .. .. ... .. .. .. .. ..\n"
+ " 0 0 0 0 0 ... 0 0 0 0 0",
+ ),
+ (
+ 9,
+ 2,
+ " 0 1 2 3 ... 7 8 9 10\n"
+ " 0 0 0 0 ... 0 0 0 0\n"
+ " .. .. .. .. ... .. .. .. ..\n"
+ " 0 0 0 0 ... 0 0 0 0",
+ ),
+ (
+ 1,
+ 1,
+ " 0 ...\n 0 ...\n.. ...",
+ ),
+ ],
+ )
+ def test_truncation_no_index(self, max_cols, max_rows, expected):
+ df = DataFrame([[0] * 11] * 4)
+ assert (
+ df.to_string(index=False, max_cols=max_cols, max_rows=max_rows) == expected
+ )
+
+ def test_to_string_no_index(self):
+ # GH#16839, GH#13032
+ df = DataFrame({"x": [11, 22], "y": [33, -44], "z": ["AAA", " "]})
+
+ df_s = df.to_string(index=False)
+ # Leading space is expected for positive numbers.
+ expected = " x y z\n11 33 AAA\n22 -44 "
+ assert df_s == expected
+
+ df_s = df[["y", "x", "z"]].to_string(index=False)
+ expected = " y x z\n 33 11 AAA\n-44 22 "
+ assert df_s == expected
+
+ def test_to_string_unicode_columns(self, float_frame):
+ df = DataFrame({"\u03c3": np.arange(10.0)})
+
+ buf = StringIO()
+ df.to_string(buf=buf)
+ buf.getvalue()
+
+ buf = StringIO()
+ df.info(buf=buf)
+ buf.getvalue()
+
+ result = float_frame.to_string()
+ assert isinstance(result, str)
+
+ @pytest.mark.parametrize("na_rep", ["NaN", "Ted"])
+ def test_to_string_na_rep_and_float_format(self, na_rep):
+ # GH#13828
+ df = DataFrame([["A", 1.2225], ["A", None]], columns=["Group", "Data"])
+ result = df.to_string(na_rep=na_rep, float_format="{:.2f}".format)
+ expected = dedent(
+ f"""\
+ Group Data
+ 0 A 1.22
+ 1 A {na_rep}"""
+ )
+ assert result == expected
+
+ def test_to_string_string_dtype(self):
+ # GH#50099
+ pytest.importorskip("pyarrow")
+ df = DataFrame(
+ {"x": ["foo", "bar", "baz"], "y": ["a", "b", "c"], "z": [1, 2, 3]}
+ )
+ df = df.astype(
+ {"x": "string[pyarrow]", "y": "string[python]", "z": "int64[pyarrow]"}
+ )
+ result = df.dtypes.to_string()
+ expected = dedent(
+ """\
+ x string
+ y string
+ z int64[pyarrow]"""
+ )
+ assert result == expected
+
+ def test_to_string_utf8_columns(self):
+ n = "\u05d0".encode()
+ df = DataFrame([1, 2], columns=[n])
+
+ with option_context("display.max_rows", 1):
+ repr(df)
+
+ def test_to_string_unicode_two(self):
+ dm = DataFrame({"c/\u03c3": []})
+ buf = StringIO()
+ dm.to_string(buf)
+
+ def test_to_string_unicode_three(self):
+ dm = DataFrame(["\xc2"])
+ buf = StringIO()
+ dm.to_string(buf)
+
+ def test_to_string_with_float_index(self):
+ index = Index([1.5, 2, 3, 4, 5])
+ df = DataFrame(np.arange(5), index=index)
+
+ result = df.to_string()
+ expected = " 0\n1.5 0\n2.0 1\n3.0 2\n4.0 3\n5.0 4"
+ assert result == expected
+
+ def test_to_string(self):
+ # big mixed
+ biggie = DataFrame(
+ {
+ "A": np.random.default_rng(2).standard_normal(200),
+ "B": Index([f"{i}?!" for i in range(200)]),
+ },
+ )
+
+ biggie.loc[:20, "A"] = np.nan
+ biggie.loc[:20, "B"] = np.nan
+ s = biggie.to_string()
+
+ buf = StringIO()
+ retval = biggie.to_string(buf=buf)
+ assert retval is None
+ assert buf.getvalue() == s
+
+ assert isinstance(s, str)
+
+ # print in right order
+ result = biggie.to_string(
+ columns=["B", "A"], col_space=17, float_format="%.5f".__mod__
+ )
+ lines = result.split("\n")
+ header = lines[0].strip().split()
+ joined = "\n".join([re.sub(r"\s+", " ", x).strip() for x in lines[1:]])
+ recons = read_csv(StringIO(joined), names=header, header=None, sep=" ")
+ tm.assert_series_equal(recons["B"], biggie["B"])
+ assert recons["A"].count() == biggie["A"].count()
+ assert (np.abs(recons["A"].dropna() - biggie["A"].dropna()) < 0.1).all()
+
+ # FIXME: don't leave commented-out
+ # expected = ['B', 'A']
+ # assert header == expected
+
+ result = biggie.to_string(columns=["A"], col_space=17)
+ header = result.split("\n")[0].strip().split()
+ expected = ["A"]
+ assert header == expected
+
+ biggie.to_string(columns=["B", "A"], formatters={"A": lambda x: f"{x:.1f}"})
+
+ biggie.to_string(columns=["B", "A"], float_format=str)
+ biggie.to_string(columns=["B", "A"], col_space=12, float_format=str)
+
+ frame = DataFrame(index=np.arange(200))
+ frame.to_string()
+
+ # TODO: split or simplify this test?
+ @pytest.mark.xfail(using_string_dtype(), reason="fix when arrow is default")
+ def test_to_string_index_with_nan(self):
+ # GH#2850
+ df = DataFrame(
+ {
+ "id1": {0: "1a3", 1: "9h4"},
+ "id2": {0: np.nan, 1: "d67"},
+ "id3": {0: "78d", 1: "79d"},
+ "value": {0: 123, 1: 64},
+ }
+ )
+
+ # multi-index
+ y = df.set_index(["id1", "id2", "id3"])
+ result = y.to_string()
+ expected = (
+ " value\nid1 id2 id3 \n"
+ "1a3 NaN 78d 123\n9h4 d67 79d 64"
+ )
+ assert result == expected
+
+ # index
+ y = df.set_index("id2")
+ result = y.to_string()
+ expected = (
+ " id1 id3 value\nid2 \n"
+ "NaN 1a3 78d 123\nd67 9h4 79d 64"
+ )
+ assert result == expected
+
+ # with append (this failed in 0.12)
+ y = df.set_index(["id1", "id2"]).set_index("id3", append=True)
+ result = y.to_string()
+ expected = (
+ " value\nid1 id2 id3 \n"
+ "1a3 NaN 78d 123\n9h4 d67 79d 64"
+ )
+ assert result == expected
+
+ # all-nan in mi
+ df2 = df.copy()
+ df2.loc[:, "id2"] = np.nan
+ y = df2.set_index("id2")
+ result = y.to_string()
+ expected = (
+ " id1 id3 value\nid2 \n"
+ "NaN 1a3 78d 123\nNaN 9h4 79d 64"
+ )
+ assert result == expected
+
+ # partial nan in mi
+ df2 = df.copy()
+ df2.loc[:, "id2"] = np.nan
+ y = df2.set_index(["id2", "id3"])
+ result = y.to_string()
+ expected = (
+ " id1 value\nid2 id3 \n"
+ "NaN 78d 1a3 123\n 79d 9h4 64"
+ )
+ assert result == expected
+
+ df = DataFrame(
+ {
+ "id1": {0: np.nan, 1: "9h4"},
+ "id2": {0: np.nan, 1: "d67"},
+ "id3": {0: np.nan, 1: "79d"},
+ "value": {0: 123, 1: 64},
+ }
+ )
+
+ y = df.set_index(["id1", "id2", "id3"])
+ result = y.to_string()
+ expected = (
+ " value\nid1 id2 id3 \n"
+ "NaN NaN NaN 123\n9h4 d67 79d 64"
+ )
+ assert result == expected
+
+ def test_to_string_nonunicode_nonascii_alignment(self):
+ df = DataFrame([["aa\xc3\xa4\xc3\xa4", 1], ["bbbb", 2]])
+ rep_str = df.to_string()
+ lines = rep_str.split("\n")
+ assert len(lines[1]) == len(lines[2])
+
+ def test_unicode_problem_decoding_as_ascii(self):
+ df = DataFrame({"c/\u03c3": Series({"test": np.nan})})
+ str(df.to_string())
+
+ def test_to_string_repr_unicode(self):
+ buf = StringIO()
+
+ unicode_values = ["\u03c3"] * 10
+ unicode_values = np.array(unicode_values, dtype=object)
+ df = DataFrame({"unicode": unicode_values})
+ df.to_string(col_space=10, buf=buf)
+
+ # it works!
+ repr(df)
+ # it works even if sys.stdin in None
+ _stdin = sys.stdin
+ try:
+ sys.stdin = None
+ repr(df)
+ finally:
+ sys.stdin = _stdin
+
+ def test_nested_dataframe(self):
+ df1 = DataFrame({"level1": [["row1"], ["row2"]]})
+ df2 = DataFrame({"level3": [{"level2": df1}]})
+ result = df2.to_string()
+ expected = " level3\n0 {'level2': ['level1']}"
+ assert result == expected
+
+
+class TestSeriesToString:
+ def test_to_string_without_index(self):
+ # GH#11729 Test index=False option
+ ser = Series([1, 2, 3, 4])
+ result = ser.to_string(index=False)
+ expected = "\n".join(["1", "2", "3", "4"])
+ assert result == expected
+
+ def test_to_string_name(self):
+ ser = Series(range(100), dtype="int64")
+ ser.name = "myser"
+ res = ser.to_string(max_rows=2, name=True)
+ exp = "0 0\n ..\n99 99\nName: myser"
+ assert res == exp
+ res = ser.to_string(max_rows=2, name=False)
+ exp = "0 0\n ..\n99 99"
+ assert res == exp
+
+ def test_to_string_dtype(self):
+ ser = Series(range(100), dtype="int64")
+ res = ser.to_string(max_rows=2, dtype=True)
+ exp = "0 0\n ..\n99 99\ndtype: int64"
+ assert res == exp
+ res = ser.to_string(max_rows=2, dtype=False)
+ exp = "0 0\n ..\n99 99"
+ assert res == exp
+
+ def test_to_string_length(self):
+ ser = Series(range(100), dtype="int64")
+ res = ser.to_string(max_rows=2, length=True)
+ exp = "0 0\n ..\n99 99\nLength: 100"
+ assert res == exp
+
+ def test_to_string_na_rep(self):
+ ser = Series(index=range(100), dtype=np.float64)
+ res = ser.to_string(na_rep="foo", max_rows=2)
+ exp = "0 foo\n ..\n99 foo"
+ assert res == exp
+
+ def test_to_string_float_format(self):
+ ser = Series(range(10), dtype="float64")
+ res = ser.to_string(float_format=lambda x: f"{x:2.1f}", max_rows=2)
+ exp = "0 0.0\n ..\n9 9.0"
+ assert res == exp
+
+ def test_to_string_header(self):
+ ser = Series(range(10), dtype="int64")
+ ser.index.name = "foo"
+ res = ser.to_string(header=True, max_rows=2)
+ exp = "foo\n0 0\n ..\n9 9"
+ assert res == exp
+ res = ser.to_string(header=False, max_rows=2)
+ exp = "0 0\n ..\n9 9"
+ assert res == exp
+
+ def test_to_string_empty_col(self):
+ # GH#13653
+ ser = Series(["", "Hello", "World", "", "", "Mooooo", "", ""])
+ res = ser.to_string(index=False)
+ exp = " \n Hello\n World\n \n \nMooooo\n \n "
+ assert re.match(exp, res)
+
+ def test_to_string_timedelta64(self):
+ Series(np.array([1100, 20], dtype="timedelta64[ns]")).to_string()
+
+ ser = Series(date_range("2012-1-1", periods=3, freq="D"))
+
+ # GH#2146
+
+ # adding NaTs
+ y = ser - ser.shift(1)
+ result = y.to_string()
+ assert "1 days" in result
+ assert "00:00:00" not in result
+ assert "NaT" in result
+
+ # with frac seconds
+ o = Series([datetime(2012, 1, 1, microsecond=150)] * 3)
+ y = ser - o
+ result = y.to_string()
+ assert "-1 days +23:59:59.999850" in result
+
+ # rounding?
+ o = Series([datetime(2012, 1, 1, 1)] * 3)
+ y = ser - o
+ result = y.to_string()
+ assert "-1 days +23:00:00" in result
+ assert "1 days 23:00:00" in result
+
+ o = Series([datetime(2012, 1, 1, 1, 1)] * 3)
+ y = ser - o
+ result = y.to_string()
+ assert "-1 days +22:59:00" in result
+ assert "1 days 22:59:00" in result
+
+ o = Series([datetime(2012, 1, 1, 1, 1, microsecond=150)] * 3)
+ y = ser - o
+ result = y.to_string()
+ assert "-1 days +22:58:59.999850" in result
+ assert "0 days 22:58:59.999850" in result
+
+ # neg time
+ td = timedelta(minutes=5, seconds=3)
+ s2 = Series(date_range("2012-1-1", periods=3, freq="D")) + td
+ y = ser - s2
+ result = y.to_string()
+ assert "-1 days +23:54:57" in result
+
+ td = timedelta(microseconds=550)
+ s2 = Series(date_range("2012-1-1", periods=3, freq="D")) + td
+ y = ser - td
+ result = y.to_string()
+ assert "2012-01-01 23:59:59.999450" in result
+
+ # no boxing of the actual elements
+ td = Series(timedelta_range("1 days", periods=3))
+ result = td.to_string()
+ assert result == "0 1 days\n1 2 days\n2 3 days"
+
+ def test_to_string(self):
+ ts = Series(
+ np.arange(10, dtype=np.float64),
+ index=date_range("2020-01-01", periods=10, freq="B"),
+ )
+ buf = StringIO()
+
+ s = ts.to_string()
+
+ retval = ts.to_string(buf=buf)
+ assert retval is None
+ assert buf.getvalue().strip() == s
+
+ # pass float_format
+ format = "%.4f".__mod__
+ result = ts.to_string(float_format=format)
+ result = [x.split()[1] for x in result.split("\n")[:-1]]
+ expected = [format(x) for x in ts]
+ assert result == expected
+
+ # empty string
+ result = ts[:0].to_string()
+ assert result == "Series([], Freq: B)"
+
+ result = ts[:0].to_string(length=0)
+ assert result == "Series([], Freq: B)"
+
+ # name and length
+ cp = ts.copy()
+ cp.name = "foo"
+ result = cp.to_string(length=True, name=True, dtype=True)
+ last_line = result.split("\n")[-1].strip()
+ assert last_line == (f"Freq: B, Name: foo, Length: {len(cp)}, dtype: float64")
+
+ @pytest.mark.parametrize(
+ "input_array, expected",
+ [
+ ("a", "a"),
+ (["a", "b"], "a\nb"),
+ ([1, "a"], "1\na"),
+ (1, "1"),
+ ([0, -1], " 0\n-1"),
+ (1.0, "1.0"),
+ ([" a", " b"], " a\n b"),
+ ([".1", "1"], ".1\n 1"),
+ (["10", "-10"], " 10\n-10"),
+ ],
+ )
+ def test_format_remove_leading_space_series(self, input_array, expected):
+ # GH: 24980
+ ser = Series(input_array)
+ result = ser.to_string(index=False)
+ assert result == expected
+
+ def test_to_string_complex_number_trims_zeros(self):
+ ser = Series([1.000000 + 1.000000j, 1.0 + 1.0j, 1.05 + 1.0j])
+ result = ser.to_string()
+ expected = dedent(
+ """\
+ 0 1.00+1.00j
+ 1 1.00+1.00j
+ 2 1.05+1.00j"""
+ )
+ assert result == expected
+
+ def test_nullable_float_to_string(self, float_ea_dtype):
+ # https://github.com/pandas-dev/pandas/issues/36775
+ dtype = float_ea_dtype
+ ser = Series([0.0, 1.0, None], dtype=dtype)
+ result = ser.to_string()
+ expected = dedent(
+ """\
+ 0 0.0
+ 1 1.0
+ 2 """
+ )
+ assert result == expected
+
+ def test_nullable_int_to_string(self, any_int_ea_dtype):
+ # https://github.com/pandas-dev/pandas/issues/36775
+ dtype = any_int_ea_dtype
+ ser = Series([0, 1, None], dtype=dtype)
+ result = ser.to_string()
+ expected = dedent(
+ """\
+ 0 0
+ 1 1
+ 2 """
+ )
+ assert result == expected
+
+ def test_to_string_mixed(self):
+ ser = Series(["foo", np.nan, -1.23, 4.56])
+ result = ser.to_string()
+ expected = "".join(["0 foo\n", "1 NaN\n", "2 -1.23\n", "3 4.56"])
+ assert result == expected
+
+ # but don't count NAs as floats
+ ser = Series(["foo", np.nan, "bar", "baz"])
+ result = ser.to_string()
+ expected = "".join(["0 foo\n", "1 NaN\n", "2 bar\n", "3 baz"])
+ assert result == expected
+
+ ser = Series(["foo", 5, "bar", "baz"])
+ result = ser.to_string()
+ expected = "".join(["0 foo\n", "1 5\n", "2 bar\n", "3 baz"])
+ assert result == expected
+
+ def test_to_string_float_na_spacing(self):
+ ser = Series([0.0, 1.5678, 2.0, -3.0, 4.0])
+ ser[::2] = np.nan
+
+ result = ser.to_string()
+ expected = "0 NaN\n1 1.5678\n2 NaN\n3 -3.0000\n4 NaN"
+ assert result == expected
+
+ def test_to_string_with_datetimeindex(self):
+ index = date_range("20130102", periods=6)
+ ser = Series(1, index=index)
+ result = ser.to_string()
+ assert "2013-01-02" in result
+
+ # nat in index
+ s2 = Series(2, index=[Timestamp("20130111"), NaT])
+ ser = concat([s2, ser])
+ result = ser.to_string()
+ assert "NaT" in result
+
+ # nat in summary
+ result = str(s2.index)
+ assert "NaT" in result
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/generate_legacy_storage_files.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/generate_legacy_storage_files.py
new file mode 100644
index 0000000000000000000000000000000000000000..04f176a550edf0b927727788e958421b422c26d0
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/generate_legacy_storage_files.py
@@ -0,0 +1,421 @@
+"""
+self-contained to write legacy storage pickle files
+
+To use this script. Create an environment where you want
+generate pickles, say its for 0.20.3, with your pandas clone
+in ~/pandas
+
+. activate pandas_0.20.3
+cd ~/pandas/pandas
+
+$ python -m tests.io.generate_legacy_storage_files \
+ tests/io/data/legacy_pickle/0.20.3/ pickle
+
+This script generates a storage file for the current arch, system,
+and python version
+ pandas version: 0.20.3
+ output dir : pandas/pandas/tests/io/data/legacy_pickle/0.20.3/
+ storage format: pickle
+created pickle file: 0.20.3_x86_64_darwin_3.5.2.pickle
+
+The idea here is you are using the *current* version of the
+generate_legacy_storage_files with an *older* version of pandas to
+generate a pickle file. We will then check this file into a current
+branch, and test using test_pickle.py. This will load the *older*
+pickles and test versus the current data that is generated
+(with main). These are then compared.
+
+If we have cases where we changed the signature (e.g. we renamed
+offset -> freq in Timestamp). Then we have to conditionally execute
+in the generate_legacy_storage_files.py to make it
+run under the older AND the newer version.
+
+"""
+
+from datetime import timedelta
+import os
+import pickle
+import platform as pl
+import sys
+
+# Remove script directory from path, otherwise Python will try to
+# import the JSON test directory as the json module
+sys.path.pop(0)
+
+import numpy as np
+
+import pandas
+from pandas import (
+ Categorical,
+ DataFrame,
+ Index,
+ MultiIndex,
+ NaT,
+ Period,
+ RangeIndex,
+ Series,
+ Timestamp,
+ bdate_range,
+ date_range,
+ interval_range,
+ period_range,
+ timedelta_range,
+)
+from pandas.arrays import SparseArray
+
+from pandas.tseries.offsets import (
+ FY5253,
+ BusinessDay,
+ BusinessHour,
+ CustomBusinessDay,
+ DateOffset,
+ Day,
+ Easter,
+ Hour,
+ LastWeekOfMonth,
+ Minute,
+ MonthBegin,
+ MonthEnd,
+ QuarterBegin,
+ QuarterEnd,
+ SemiMonthBegin,
+ SemiMonthEnd,
+ Week,
+ WeekOfMonth,
+ YearBegin,
+ YearEnd,
+)
+
+
+def _create_sp_series():
+ nan = np.nan
+
+ # nan-based
+ arr = np.arange(15, dtype=np.float64)
+ arr[7:12] = nan
+ arr[-1:] = nan
+
+ bseries = Series(SparseArray(arr, kind="block"))
+ bseries.name = "bseries"
+ return bseries
+
+
+def _create_sp_tsseries():
+ nan = np.nan
+
+ # nan-based
+ arr = np.arange(15, dtype=np.float64)
+ arr[7:12] = nan
+ arr[-1:] = nan
+
+ date_index = bdate_range("1/1/2011", periods=len(arr))
+ bseries = Series(SparseArray(arr, kind="block"), index=date_index)
+ bseries.name = "btsseries"
+ return bseries
+
+
+def _create_sp_frame():
+ nan = np.nan
+
+ data = {
+ "A": [nan, nan, nan, 0, 1, 2, 3, 4, 5, 6],
+ "B": [0, 1, 2, nan, nan, nan, 3, 4, 5, 6],
+ "C": np.arange(10).astype(np.int64),
+ "D": [0, 1, 2, 3, 4, 5, nan, nan, nan, nan],
+ }
+
+ dates = bdate_range("1/1/2011", periods=10)
+ return DataFrame(data, index=dates).apply(SparseArray)
+
+
+def create_pickle_data(test: bool = True):
+ """create the pickle data"""
+ data = {
+ "A": [0.0, 1.0, 2.0, 3.0, np.nan],
+ "B": [0, 1, 0, 1, 0],
+ "C": ["foo1", "foo2", "foo3", "foo4", "foo5"],
+ "D": date_range("1/1/2009", periods=5),
+ "E": [0.0, 1, Timestamp("20100101"), "foo", 2.0],
+ }
+
+ scalars = {"timestamp": Timestamp("20130101"), "period": Period("2012", "M")}
+
+ index = {
+ "int": Index(np.arange(10)),
+ "date": date_range("20130101", periods=10),
+ "period": period_range("2013-01-01", freq="M", periods=10),
+ "float": Index(np.arange(10, dtype=np.float64)),
+ "uint": Index(np.arange(10, dtype=np.uint64)),
+ "timedelta": timedelta_range("00:00:00", freq="30min", periods=10),
+ "string": Index(["foo", "bar", "baz", "qux", "quux"], dtype="string"),
+ }
+
+ index["range"] = RangeIndex(10)
+
+ index["interval"] = interval_range(0, periods=10)
+
+ mi = {
+ "reg2": MultiIndex.from_tuples(
+ tuple(
+ zip(
+ *[
+ ["bar", "bar", "baz", "baz", "foo", "foo", "qux", "qux"],
+ ["one", "two", "one", "two", "one", "two", "one", "two"],
+ ]
+ )
+ ),
+ names=["first", "second"],
+ )
+ }
+
+ series = {
+ "float": Series(data["A"]),
+ "int": Series(data["B"]),
+ "mixed": Series(data["E"]),
+ "ts": Series(
+ np.arange(10).astype(np.int64), index=date_range("20130101", periods=10)
+ ),
+ "mi": Series(
+ np.arange(5).astype(np.float64),
+ index=MultiIndex.from_tuples(
+ tuple(zip(*[[1, 1, 2, 2, 2], [3, 4, 3, 4, 5]])), names=["one", "two"]
+ ),
+ ),
+ "dup": Series(np.arange(5).astype(np.float64), index=["A", "B", "C", "D", "A"]),
+ "cat": Series(Categorical(["foo", "bar", "baz"])),
+ "dt": Series(date_range("20130101", periods=5)),
+ "dt_tz": Series(date_range("20130101", periods=5, tz="US/Eastern")),
+ "period": Series([Period("2000Q1")] * 5),
+ "string": Series(["foo", "bar", "baz", "qux", "quux"], dtype="string"),
+ }
+
+ mixed_dup_df = DataFrame(data)
+ mixed_dup_df.columns = list("ABCDA")
+ frame = {
+ "float": DataFrame({"A": series["float"], "B": series["float"] + 1}),
+ "int": DataFrame({"A": series["int"], "B": series["int"] + 1}),
+ "mixed": DataFrame({k: data[k] for k in ["A", "B", "C", "D"]}),
+ "mi": DataFrame(
+ {"A": np.arange(5).astype(np.float64), "B": np.arange(5).astype(np.int64)},
+ index=MultiIndex.from_tuples(
+ tuple(
+ zip(
+ *[
+ ["bar", "bar", "baz", "baz", "baz"],
+ ["one", "two", "one", "two", "three"],
+ ]
+ )
+ ),
+ names=["first", "second"],
+ ),
+ ),
+ "dup": DataFrame(
+ np.arange(15).reshape(5, 3).astype(np.float64), columns=["A", "B", "A"]
+ ),
+ "cat_onecol": DataFrame({"A": Categorical(["foo", "bar"])}),
+ "cat_and_float": DataFrame(
+ {
+ "A": Categorical(["foo", "bar", "baz"]),
+ "B": np.arange(3).astype(np.int64),
+ }
+ ),
+ "mixed_dup": mixed_dup_df,
+ "dt_mixed_tzs": DataFrame(
+ {
+ "A": Timestamp("20130102", tz="US/Eastern"),
+ "B": Timestamp("20130603", tz="CET"),
+ },
+ index=range(5),
+ ),
+ "dt_mixed2_tzs": DataFrame(
+ {
+ "A": Timestamp("20130102", tz="US/Eastern"),
+ "B": Timestamp("20130603", tz="CET"),
+ "C": Timestamp("20130603", tz="UTC"),
+ },
+ index=range(5),
+ ),
+ "string": DataFrame(
+ {
+ "A": Series(["foo", "bar", "baz", "qux", "quux"], dtype="string"),
+ "B": Series(["one", "two", "one", "two", "three"], dtype="string"),
+ }
+ ),
+ }
+
+ cat = {
+ "int8": Categorical(list("abcdefg")),
+ "int16": Categorical(np.arange(1000)),
+ "int32": Categorical(np.arange(10000)),
+ }
+
+ timestamp = {
+ "normal": Timestamp("2011-01-01"),
+ "nat": NaT,
+ "tz": Timestamp("2011-01-01", tz="US/Eastern"),
+ }
+ if test:
+ # kept because those are present in the legacy pickles (<= 1.4)
+ timestamp["freq"] = Timestamp("2011-01-01")
+ timestamp["both"] = Timestamp("2011-01-01", tz="Asia/Tokyo")
+
+ off = {
+ "DateOffset": DateOffset(years=1),
+ "DateOffset_h_ns": DateOffset(hour=6, nanoseconds=5824),
+ "BusinessDay": BusinessDay(offset=timedelta(seconds=9)),
+ "BusinessHour": BusinessHour(normalize=True, n=6, end="15:14"),
+ "CustomBusinessDay": CustomBusinessDay(weekmask="Mon Fri"),
+ "SemiMonthBegin": SemiMonthBegin(day_of_month=9),
+ "SemiMonthEnd": SemiMonthEnd(day_of_month=24),
+ "MonthBegin": MonthBegin(1),
+ "MonthEnd": MonthEnd(1),
+ "QuarterBegin": QuarterBegin(1),
+ "QuarterEnd": QuarterEnd(1),
+ "Day": Day(1),
+ "YearBegin": YearBegin(1),
+ "YearEnd": YearEnd(1),
+ "Week": Week(1),
+ "Week_Tues": Week(2, normalize=False, weekday=1),
+ "WeekOfMonth": WeekOfMonth(week=3, weekday=4),
+ "LastWeekOfMonth": LastWeekOfMonth(n=1, weekday=3),
+ "FY5253": FY5253(n=2, weekday=6, startingMonth=7, variation="last"),
+ "Easter": Easter(),
+ "Hour": Hour(1),
+ "Minute": Minute(1),
+ }
+
+ return {
+ "series": series,
+ "frame": frame,
+ "index": index,
+ "scalars": scalars,
+ "mi": mi,
+ "sp_series": {"float": _create_sp_series(), "ts": _create_sp_tsseries()},
+ "sp_frame": {"float": _create_sp_frame()},
+ "cat": cat,
+ "timestamp": timestamp,
+ "offsets": off,
+ }
+
+
+def create_dataframe_all_types():
+ timestamps = Series(
+ [
+ Timestamp("2013-01-01"),
+ NaT,
+ Timestamp("2013-01-03"),
+ Timestamp("2013-01-04"),
+ Timestamp("2013-01-05"),
+ ]
+ )
+ timedeltas = timestamps - timestamps[0]
+
+ data = {
+ # "string": Series(
+ # ["a", "b", "c", None, "e"], dtype=StringDtype(na_value=np.nan)
+ # ),
+ # "object": Series(["a", "b", "c", None, "e"], dtype=object),
+ # "object_nan": Series(["a", "b", "c", np.nan, "e"], dtype=object),
+ "int": list(range(1, 6)),
+ "uint64": np.arange(3, 8).astype("uint64"),
+ "float": [0.1, 0.2, 0.3, 0.4, np.nan],
+ "float32": Series([0.1, 0.2, 0.3, 0.4, np.nan], dtype="float32"),
+ "bool": [True, False, True, False, True],
+ "datetime_ns": timestamps.dt.as_unit("ns"),
+ "datetime_us": timestamps.dt.as_unit("us"),
+ "datetime_ms": timestamps.dt.as_unit("ms"),
+ "datetime_s": timestamps.dt.as_unit("s"),
+ "datetimetz_ns": timestamps.dt.tz_localize("US/Eastern").dt.as_unit("ns"),
+ "datetimetz_us": timestamps.dt.tz_localize("US/Eastern").dt.as_unit("us"),
+ "timedelta_ns": timedeltas.dt.as_unit("ns"),
+ "timedelta_us": timedeltas.dt.as_unit("us"),
+ "timedelta_ms": timedeltas.dt.as_unit("ms"),
+ "timedelta_s": timedeltas.dt.as_unit("s"),
+ # "categorical": Categorical(
+ # Series(
+ # ["foo", "bar", "baz",np.nan,"foo"],dtype=StringDtype(na_value=np.nan)
+ # )
+ # ),
+ # "categorical_object": Categorical(
+ # Series(["foo", "bar", "baz", np.nan, "foo"], dtype=object)
+ # ),
+ "categorical_int": Categorical([1, 2, 3, np.nan, 1]),
+ }
+ return DataFrame(data)
+
+
+def platform_name():
+ return "_".join(
+ [
+ str(pandas.__version__),
+ str(pl.machine()),
+ str(pl.system().lower()),
+ str(pl.python_version()),
+ ]
+ )
+
+
+def write_legacy_pickles(output_dir):
+ pth = f"{platform_name()}.pickle"
+
+ with open(os.path.join(output_dir, pth), "wb") as fh:
+ pickle.dump(create_pickle_data(test=False), fh, pickle.DEFAULT_PROTOCOL)
+
+ print(f"created pickle file: {pth}")
+
+
+def write_legacy_hdf(output_dir, format):
+ import tables
+
+ pth = f"{platform_name()}_pytables-{tables.__version__}_{format}.h5"
+
+ df = create_dataframe_all_types()
+ if format == "fixed":
+ # df = df.drop(columns=["categorical", "categorical_object", "categorical_int"])
+ df = df.drop(columns=["categorical_int"])
+ complevel = 9 if format == "table" else None
+ df.to_hdf(
+ os.path.join(output_dir, pth),
+ key="df_alltypes",
+ format=format,
+ complevel=complevel,
+ )
+
+ print(f"created hdf file: {pth}")
+
+
+def write_legacy_file():
+ # force our cwd to be the first searched
+ sys.path.insert(0, "")
+
+ if not 3 <= len(sys.argv) <= 4:
+ sys.exit(
+ "Specify output directory and storage type: generate_legacy_"
+ "storage_files.py "
+ )
+
+ output_dir = str(sys.argv[1])
+ storage_type = str(sys.argv[2])
+
+ print(
+ "This script generates a storage file for the current arch, system, "
+ "and python version"
+ )
+ print(f" pandas version: {pandas.__version__}")
+ print(f" output dir : {output_dir}")
+ print(f" storage format: {storage_type}")
+
+ if not os.path.exists(output_dir):
+ os.mkdir(output_dir)
+
+ if storage_type == "pickle":
+ write_legacy_pickles(output_dir=output_dir)
+ elif storage_type == "hdf":
+ write_legacy_hdf(output_dir=output_dir, format="fixed")
+ write_legacy_hdf(output_dir=output_dir, format="table")
+ else:
+ sys.exit("storage_type must be one of {'pickle', 'hdf'}")
+
+
+if __name__ == "__main__":
+ write_legacy_file()
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/json/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/json/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/json/conftest.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/json/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..4e848cd48b42d70033af233bf71c6904b9d069f9
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/json/conftest.py
@@ -0,0 +1,9 @@
+import pytest
+
+
+@pytest.fixture(params=["split", "records", "index", "columns", "values"])
+def orient(request):
+ """
+ Fixture for orients excluding the table format.
+ """
+ return request.param
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_compression.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_compression.py
new file mode 100644
index 0000000000000000000000000000000000000000..d29f7d3c5ff2e87e482fa87814e75a5b88d56849
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_compression.py
@@ -0,0 +1,130 @@
+from io import (
+ BytesIO,
+ StringIO,
+)
+import uuid
+
+import pytest
+
+import pandas.util._test_decorators as td
+
+import pandas as pd
+import pandas._testing as tm
+
+
+def test_compression_roundtrip(compression, temp_file):
+ df = pd.DataFrame(
+ [[0.123456, 0.234567, 0.567567], [12.32112, 123123.2, 321321.2]],
+ index=["A", "B"],
+ columns=["X", "Y", "Z"],
+ )
+
+ df.to_json(temp_file, compression=compression)
+ tm.assert_frame_equal(df, pd.read_json(temp_file, compression=compression))
+
+ # explicitly ensure file was compressed.
+ with tm.decompress_file(temp_file, compression) as fh:
+ result = fh.read().decode("utf8")
+ data = StringIO(result)
+ tm.assert_frame_equal(df, pd.read_json(data))
+
+
+def test_read_zipped_json(datapath):
+ uncompressed_path = datapath("io", "json", "data", "tsframe_v012.json")
+ uncompressed_df = pd.read_json(uncompressed_path)
+
+ compressed_path = datapath("io", "json", "data", "tsframe_v012.json.zip")
+ compressed_df = pd.read_json(compressed_path, compression="zip")
+
+ tm.assert_frame_equal(uncompressed_df, compressed_df)
+
+
+@td.skip_if_not_us_locale
+@pytest.mark.single_cpu
+@pytest.mark.network
+def test_with_s3_url(compression, s3_bucket_public, s3so, temp_file):
+ # Bucket created in tests/io/conftest.py
+ df = pd.read_json(StringIO('{"a": [1, 2, 3], "b": [4, 5, 6]}'))
+
+ key = f"{uuid.uuid4()}.json"
+ df.to_json(temp_file, compression=compression)
+ with open(temp_file, "rb") as f:
+ s3_bucket_public.put_object(Key=key, Body=f)
+
+ roundtripped_df = pd.read_json(
+ f"s3://{s3_bucket_public.name}/{key}",
+ compression=compression,
+ storage_options=s3so,
+ )
+ tm.assert_frame_equal(df, roundtripped_df)
+
+
+def test_lines_with_compression(compression, temp_file):
+ df = pd.read_json(StringIO('{"a": [1, 2, 3], "b": [4, 5, 6]}'))
+ df.to_json(temp_file, orient="records", lines=True, compression=compression)
+ roundtripped_df = pd.read_json(temp_file, lines=True, compression=compression)
+ tm.assert_frame_equal(df, roundtripped_df)
+
+
+def test_chunksize_with_compression(compression, temp_file):
+ df = pd.read_json(StringIO('{"a": ["foo", "bar", "baz"], "b": [4, 5, 6]}'))
+ df.to_json(temp_file, orient="records", lines=True, compression=compression)
+
+ with pd.read_json(
+ temp_file, lines=True, chunksize=1, compression=compression
+ ) as res:
+ roundtripped_df = pd.concat(res)
+ tm.assert_frame_equal(df, roundtripped_df)
+
+
+def test_write_unsupported_compression_type(temp_file):
+ df = pd.read_json(StringIO('{"a": [1, 2, 3], "b": [4, 5, 6]}'))
+ msg = "Unrecognized compression type: unsupported"
+ with pytest.raises(ValueError, match=msg):
+ df.to_json(temp_file, compression="unsupported")
+
+
+def test_read_unsupported_compression_type(temp_file):
+ msg = "Unrecognized compression type: unsupported"
+ with pytest.raises(ValueError, match=msg):
+ pd.read_json(temp_file, compression="unsupported")
+
+
+@pytest.mark.parametrize(
+ "infer_string", [False, pytest.param(True, marks=td.skip_if_no("pyarrow"))]
+)
+@pytest.mark.parametrize("to_infer", [True, False])
+@pytest.mark.parametrize("read_infer", [True, False])
+def test_to_json_compression(
+ compression_only,
+ read_infer,
+ to_infer,
+ compression_to_extension,
+ infer_string,
+ tmp_path,
+):
+ with pd.option_context("future.infer_string", infer_string):
+ # see gh-15008
+ compression = compression_only
+
+ # We'll complete file extension subsequently.
+ filename = tmp_path / f"test.{compression_to_extension[compression]}"
+
+ df = pd.DataFrame({"A": [1]})
+
+ to_compression = "infer" if to_infer else compression
+ read_compression = "infer" if read_infer else compression
+
+ df.to_json(filename, compression=to_compression)
+ result = pd.read_json(filename, compression=read_compression)
+ tm.assert_frame_equal(result, df)
+
+
+def test_to_json_compression_mode(compression):
+ # GH 39985 (read_json does not support user-provided binary files)
+ expected = pd.DataFrame({"A": [1]})
+
+ with BytesIO() as buffer:
+ expected.to_json(buffer, compression=compression)
+ # df = pd.read_json(buffer, compression=compression)
+ # tm.assert_frame_equal(expected, df)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_deprecated_kwargs.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_deprecated_kwargs.py
new file mode 100644
index 0000000000000000000000000000000000000000..9da682c90a285e3b1e1aff9d02709522bfdac98f
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_deprecated_kwargs.py
@@ -0,0 +1,22 @@
+"""
+Tests for the deprecated keyword arguments for `read_json`.
+"""
+
+from io import StringIO
+
+import pandas as pd
+import pandas._testing as tm
+
+from pandas.io.json import read_json
+
+
+def test_good_kwargs():
+ df = pd.DataFrame({"A": [2, 4, 6], "B": [3, 6, 9]}, index=[0, 1, 2])
+
+ with tm.assert_produces_warning(None):
+ data1 = StringIO(df.to_json(orient="split"))
+ tm.assert_frame_equal(df, read_json(data1, orient="split"))
+ data2 = StringIO(df.to_json(orient="columns"))
+ tm.assert_frame_equal(df, read_json(data2, orient="columns"))
+ data3 = StringIO(df.to_json(orient="index"))
+ tm.assert_frame_equal(df, read_json(data3, orient="index"))
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_json_table_schema.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_json_table_schema.py
new file mode 100644
index 0000000000000000000000000000000000000000..bec53a0450b68898d5dc6c8199b5100d581a2a4e
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_json_table_schema.py
@@ -0,0 +1,898 @@
+"""Tests for Table Schema integration."""
+
+from collections import OrderedDict
+from io import StringIO
+import json
+
+import numpy as np
+import pytest
+
+from pandas.errors import Pandas4Warning
+
+from pandas.core.dtypes.dtypes import (
+ CategoricalDtype,
+ DatetimeTZDtype,
+ PeriodDtype,
+)
+
+import pandas as pd
+from pandas import DataFrame
+import pandas._testing as tm
+
+from pandas.io.json._table_schema import (
+ as_json_table_type,
+ build_table_schema,
+ convert_json_field_to_pandas_type,
+ convert_pandas_type_to_json_field,
+ set_default_names,
+)
+
+
+@pytest.fixture
+def df_schema():
+ return DataFrame(
+ {
+ "A": [1, 2, 3, 4],
+ "B": ["a", "b", "c", "c"],
+ "C": pd.date_range("2016-01-01", freq="D", periods=4),
+ "D": pd.timedelta_range("1h", periods=4, freq="min"),
+ },
+ index=pd.Index(range(4), name="idx"),
+ )
+
+
+@pytest.fixture
+def df_table():
+ return DataFrame(
+ {
+ "A": [1, 2, 3, 4],
+ "B": ["a", "b", "c", "c"],
+ "C": pd.date_range("2016-01-01", freq="D", periods=4),
+ "D": pd.timedelta_range("1h", periods=4, freq="min"),
+ "E": pd.Series(pd.Categorical(["a", "b", "c", "c"])),
+ "F": pd.Series(pd.Categorical(["a", "b", "c", "c"], ordered=True)),
+ "G": [1.0, 2.0, 3, 4.0],
+ "H": pd.date_range("2016-01-01", freq="D", periods=4, tz="US/Central"),
+ },
+ index=pd.Index(range(4), name="idx"),
+ )
+
+
+class TestBuildSchema:
+ def test_build_table_schema(self, df_schema, using_infer_string):
+ result = build_table_schema(df_schema, version=False)
+ expected = {
+ "fields": [
+ {"name": "idx", "type": "integer"},
+ {"name": "A", "type": "integer"},
+ {"name": "B", "type": "string"},
+ {"name": "C", "type": "datetime"},
+ {"name": "D", "type": "duration"},
+ ],
+ "primaryKey": ["idx"],
+ }
+ if using_infer_string:
+ expected["fields"][2] = {"name": "B", "type": "string", "extDtype": "str"}
+ assert result == expected
+ result = build_table_schema(df_schema)
+ assert "pandas_version" in result
+
+ def test_series(self):
+ s = pd.Series([1, 2, 3], name="foo")
+ result = build_table_schema(s, version=False)
+ expected = {
+ "fields": [
+ {"name": "index", "type": "integer"},
+ {"name": "foo", "type": "integer"},
+ ],
+ "primaryKey": ["index"],
+ }
+ assert result == expected
+ result = build_table_schema(s)
+ assert "pandas_version" in result
+
+ def test_series_unnamed(self):
+ result = build_table_schema(pd.Series([1, 2, 3]), version=False)
+ expected = {
+ "fields": [
+ {"name": "index", "type": "integer"},
+ {"name": "values", "type": "integer"},
+ ],
+ "primaryKey": ["index"],
+ }
+ assert result == expected
+
+ def test_multiindex(self, df_schema, using_infer_string):
+ df = df_schema
+ idx = pd.MultiIndex.from_product([("a", "b"), (1, 2)])
+ df.index = idx
+
+ result = build_table_schema(df, version=False)
+ expected = {
+ "fields": [
+ {"name": "level_0", "type": "string"},
+ {"name": "level_1", "type": "integer"},
+ {"name": "A", "type": "integer"},
+ {"name": "B", "type": "string"},
+ {"name": "C", "type": "datetime"},
+ {"name": "D", "type": "duration"},
+ ],
+ "primaryKey": ["level_0", "level_1"],
+ }
+ if using_infer_string:
+ expected["fields"][0] = {
+ "name": "level_0",
+ "type": "string",
+ "extDtype": "str",
+ }
+ expected["fields"][3] = {"name": "B", "type": "string", "extDtype": "str"}
+ assert result == expected
+
+ df.index.names = ["idx0", None]
+ expected["fields"][0]["name"] = "idx0"
+ expected["primaryKey"] = ["idx0", "level_1"]
+ result = build_table_schema(df, version=False)
+ assert result == expected
+
+
+class TestTableSchemaType:
+ @pytest.mark.parametrize("int_type", [int, np.int16, np.int32, np.int64])
+ def test_as_json_table_type_int_data(self, int_type):
+ int_data = [1, 2, 3]
+ assert as_json_table_type(np.array(int_data, dtype=int_type).dtype) == "integer"
+
+ @pytest.mark.parametrize("float_type", [float, np.float16, np.float32, np.float64])
+ def test_as_json_table_type_float_data(self, float_type):
+ float_data = [1.0, 2.0, 3.0]
+ assert (
+ as_json_table_type(np.array(float_data, dtype=float_type).dtype) == "number"
+ )
+
+ @pytest.mark.parametrize("bool_type", [bool, np.bool_])
+ def test_as_json_table_type_bool_data(self, bool_type):
+ bool_data = [True, False]
+ assert (
+ as_json_table_type(np.array(bool_data, dtype=bool_type).dtype) == "boolean"
+ )
+
+ @pytest.mark.parametrize(
+ "date_data",
+ [
+ pd.to_datetime(["2016"]),
+ pd.to_datetime(["2016"], utc=True),
+ pd.Series(pd.to_datetime(["2016"])),
+ pd.Series(pd.to_datetime(["2016"], utc=True)),
+ pd.period_range("2016", freq="Y", periods=3),
+ ],
+ )
+ def test_as_json_table_type_date_data(self, date_data):
+ assert as_json_table_type(date_data.dtype) == "datetime"
+
+ @pytest.mark.parametrize("klass", [pd.Series, pd.Index])
+ def test_as_json_table_type_string_data(self, klass):
+ str_data = klass(["a", "b"], dtype=object)
+ assert as_json_table_type(str_data.dtype) == "string"
+
+ @pytest.mark.parametrize(
+ "cat_data",
+ [
+ pd.Categorical(["a"]),
+ pd.Categorical([1]),
+ pd.Series(pd.Categorical([1])),
+ pd.CategoricalIndex([1]),
+ ],
+ )
+ def test_as_json_table_type_categorical_data(self, cat_data):
+ assert as_json_table_type(cat_data.dtype) == "any"
+
+ # ------
+ # dtypes
+ # ------
+ @pytest.mark.parametrize("int_dtype", [int, np.int16, np.int32, np.int64])
+ def test_as_json_table_type_int_dtypes(self, int_dtype):
+ assert as_json_table_type(int_dtype) == "integer"
+
+ @pytest.mark.parametrize("float_dtype", [float, np.float16, np.float32, np.float64])
+ def test_as_json_table_type_float_dtypes(self, float_dtype):
+ assert as_json_table_type(float_dtype) == "number"
+
+ @pytest.mark.parametrize("bool_dtype", [bool, np.bool_])
+ def test_as_json_table_type_bool_dtypes(self, bool_dtype):
+ assert as_json_table_type(bool_dtype) == "boolean"
+
+ @pytest.mark.parametrize(
+ "date_dtype",
+ [
+ np.dtype(" None:
+ self.hexed = hexed
+ self.binary = bytes.fromhex(hexed)
+
+ def __str__(self) -> str:
+ return self.hexed
+
+ hexed = "574b4454ba8c5eb4f98a8f45"
+ binthing = BinaryThing(hexed)
+
+ # verify the proper conversion of printable content
+ df_printable = DataFrame({"A": [binthing.hexed]})
+ assert df_printable.to_json() == f'{{"A":{{"0":"{hexed}"}}}}'
+
+ # check if non-printable content throws appropriate Exception
+ df_nonprintable = DataFrame({"A": [binthing]})
+ msg = "Unsupported UTF-8 sequence length when encoding string"
+ with pytest.raises(OverflowError, match=msg):
+ df_nonprintable.to_json()
+
+ # the same with multiple columns threw segfaults
+ df_mixed = DataFrame({"A": [binthing], "B": [1]}, columns=["A", "B"])
+ with pytest.raises(OverflowError, match=msg):
+ df_mixed.to_json()
+
+ # default_handler should resolve exceptions for non-string types
+ result = df_nonprintable.to_json(default_handler=str)
+ expected = f'{{"A":{{"0":"{hexed}"}}}}'
+ assert result == expected
+ assert (
+ df_mixed.to_json(default_handler=str)
+ == f'{{"A":{{"0":"{hexed}"}},"B":{{"0":1}}}}'
+ )
+
+ def test_label_overflow(self):
+ # GH14256: buffer length not checked when writing label
+ result = DataFrame({"bar" * 100000: [1], "foo": [1337]}).to_json()
+ expected = f'{{"{"bar" * 100000}":{{"0":1}},"foo":{{"0":1337}}}}'
+ assert result == expected
+
+ def test_series_non_unique_index(self):
+ s = Series(["a", "b"], index=[1, 1])
+
+ msg = "Series index must be unique for orient='index'"
+ with pytest.raises(ValueError, match=msg):
+ s.to_json(orient="index")
+
+ tm.assert_series_equal(
+ s,
+ read_json(
+ StringIO(s.to_json(orient="split")), orient="split", typ="series"
+ ),
+ )
+ unserialized = read_json(
+ StringIO(s.to_json(orient="records")), orient="records", typ="series"
+ )
+ tm.assert_equal(s.values, unserialized.values)
+
+ def test_series_default_orient(self, string_series):
+ assert string_series.to_json() == string_series.to_json(orient="index")
+
+ def test_series_roundtrip_simple(self, orient, string_series, using_infer_string):
+ data = StringIO(string_series.to_json(orient=orient))
+ result = read_json(data, typ="series", orient=orient)
+
+ expected = string_series
+ if using_infer_string and orient in ("split", "index", "columns"):
+ # These schemas don't contain dtypes, so we infer string
+ expected.index = expected.index.astype("str")
+ if orient in ("values", "records"):
+ expected = expected.reset_index(drop=True)
+ if orient != "split":
+ expected.name = None
+
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize("dtype", [False, None])
+ def test_series_roundtrip_object(self, orient, dtype, object_series):
+ data = StringIO(object_series.to_json(orient=orient))
+ result = read_json(data, typ="series", orient=orient, dtype=dtype)
+
+ expected = object_series
+ if orient in ("values", "records"):
+ expected = expected.reset_index(drop=True)
+ if orient != "split":
+ expected.name = None
+
+ if using_string_dtype():
+ expected = expected.astype("str")
+
+ tm.assert_series_equal(result, expected)
+
+ def test_series_roundtrip_empty(self, orient):
+ empty_series = Series([], index=[], dtype=np.float64)
+ data = StringIO(empty_series.to_json(orient=orient))
+ result = read_json(data, typ="series", orient=orient)
+
+ expected = empty_series.reset_index(drop=True)
+ if orient in ("split"):
+ expected.index = expected.index.astype(np.float64)
+
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "dtype_backend", [None, pytest.param("pyarrow", marks=td.skip_if_no("pyarrow"))]
+ )
+ def test_series_roundtrip_timeseries(self, dtype_backend, orient, datetime_series):
+ expected = datetime_series.copy()
+ if dtype_backend is not None:
+ datetime_series.index = Series(datetime_series.index).convert_dtypes(
+ dtype_backend=dtype_backend
+ )
+
+ data = StringIO(datetime_series.to_json(orient=orient))
+ result = read_json(data, typ="series", orient=orient)
+
+ if orient in ("values", "records"):
+ expected = expected.reset_index(drop=True)
+ else:
+ expected.index = expected.index.as_unit("ms")
+ if orient != "split":
+ expected.name = None
+
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize("dtype", [np.float64, int])
+ def test_series_roundtrip_numeric(self, orient, dtype):
+ s = Series(range(6), index=["a", "b", "c", "d", "e", "f"])
+ data = StringIO(s.to_json(orient=orient))
+ result = read_json(data, typ="series", orient=orient)
+
+ expected = s.copy()
+ if orient in ("values", "records"):
+ expected = expected.reset_index(drop=True)
+
+ tm.assert_series_equal(result, expected)
+
+ def test_series_to_json_except(self):
+ s = Series([1, 2, 3])
+ msg = "Invalid value 'garbage' for option 'orient'"
+ with pytest.raises(ValueError, match=msg):
+ s.to_json(orient="garbage")
+
+ def test_series_from_json_precise_float(self):
+ s = Series([4.56, 4.56, 4.56])
+ result = read_json(StringIO(s.to_json()), typ="series", precise_float=True)
+ tm.assert_series_equal(result, s, check_index_type=False)
+
+ def test_series_with_dtype(self):
+ # GH 21986
+ s = Series([4.56, 4.56, 4.56])
+ result = read_json(StringIO(s.to_json()), typ="series", dtype=np.int64)
+ expected = Series([4] * 3)
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "dtype_backend", [None, pytest.param("pyarrow", marks=td.skip_if_no("pyarrow"))]
+ )
+ @pytest.mark.parametrize(
+ "dtype,expected",
+ [
+ (True, Series(["2000-01-01"], dtype="datetime64[ms]")),
+ (False, Series([946684800000])),
+ ],
+ )
+ def test_series_with_dtype_datetime(self, dtype, expected, dtype_backend):
+ s = Series(["2000-01-01"], dtype="datetime64[ns]")
+ if dtype_backend is not None:
+ s = s.convert_dtypes(dtype_backend=dtype_backend)
+ msg = (
+ "The default 'epoch' date format is deprecated and will be removed "
+ "in a future version, please use 'iso' date format instead."
+ )
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ data = StringIO(s.to_json())
+ result = read_json(data, typ="series", dtype=dtype)
+ tm.assert_series_equal(result, expected)
+
+ def test_frame_from_json_precise_float(self):
+ df = DataFrame([[4.56, 4.56, 4.56], [4.56, 4.56, 4.56]])
+ result = read_json(StringIO(df.to_json()), precise_float=True)
+ tm.assert_frame_equal(result, df)
+
+ def test_typ(self):
+ s = Series(range(6), index=["a", "b", "c", "d", "e", "f"], dtype="int64")
+ result = read_json(StringIO(s.to_json()), typ="series")
+ tm.assert_series_equal(result, s)
+
+ def test_reconstruction_index(self):
+ df = DataFrame([[1, 2, 3], [4, 5, 6]])
+ result = read_json(StringIO(df.to_json()))
+ tm.assert_frame_equal(result, df)
+
+ df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, index=["A", "B", "C"])
+ result = read_json(StringIO(df.to_json()))
+ tm.assert_frame_equal(result, df)
+
+ def test_path(self, float_frame, int_frame, datetime_frame, temp_file):
+ for df in [float_frame, int_frame, datetime_frame]:
+ df.to_json(temp_file)
+ read_json(temp_file)
+
+ def test_axis_dates(self, datetime_series, datetime_frame):
+ # frame
+ json = StringIO(datetime_frame.to_json())
+ result = read_json(json)
+ expected = datetime_frame.copy()
+ expected.index = expected.index.as_unit("ms")
+ tm.assert_frame_equal(result, expected)
+
+ # series
+ json = StringIO(datetime_series.to_json())
+ result = read_json(json, typ="series")
+ expected = datetime_series.copy()
+ expected.index = expected.index.as_unit("ms")
+ tm.assert_series_equal(result, expected, check_names=False)
+ assert result.name is None
+
+ def test_convert_dates(self, datetime_series, datetime_frame):
+ # frame
+ df = datetime_frame
+ df["date"] = Timestamp("20130101").as_unit("ns")
+
+ msg = (
+ "The default 'epoch' date format is deprecated and will be removed "
+ "in a future version, please use 'iso' date format instead."
+ )
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ json = StringIO(df.to_json())
+ result = read_json(json)
+ expected = df.copy()
+ expected["date"] = expected["date"].dt.as_unit("ms")
+ expected.index = expected.index.as_unit("ms")
+ tm.assert_frame_equal(result, expected)
+
+ df["foo"] = 1.0
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ json = StringIO(df.to_json(date_unit="ns"))
+
+ result = read_json(json, convert_dates=False)
+ expected = df.copy()
+ expected["date"] = expected["date"].values.view("i8")
+ expected["foo"] = expected["foo"].astype("int64")
+ tm.assert_frame_equal(result, expected)
+
+ # series
+ ts = Series(Timestamp("20130101").as_unit("ns"), index=datetime_series.index)
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ json = StringIO(ts.to_json())
+ result = read_json(json, typ="series")
+ expected = ts.dt.as_unit("ms")
+ expected.index = expected.index.as_unit("ms")
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize("date_format", ["epoch", "iso"])
+ @pytest.mark.parametrize("as_object", [True, False])
+ @pytest.mark.parametrize("date_typ", [datetime.date, datetime.datetime, Timestamp])
+ def test_date_index_and_values(self, date_format, as_object, date_typ):
+ data = [date_typ(year=2020, month=1, day=1), pd.NaT]
+ if as_object:
+ data.append("a")
+
+ ser = Series(data, index=data)
+ if not as_object:
+ ser = ser.astype("M8[ns]")
+ if isinstance(ser.index, DatetimeIndex):
+ ser.index = ser.index.as_unit("ns")
+
+ expected_warning = None
+ if date_format == "epoch":
+ expected = '{"1577836800000":1577836800000,"null":null}'
+ expected_warning = Pandas4Warning
+ else:
+ expected = (
+ '{"2020-01-01T00:00:00.000":"2020-01-01T00:00:00.000","null":null}'
+ )
+
+ msg = (
+ "'epoch' date format is deprecated and will be removed in a future "
+ "version, please use 'iso' date format instead."
+ )
+ with tm.assert_produces_warning(expected_warning, match=msg):
+ result = ser.to_json(date_format=date_format)
+
+ if as_object:
+ expected = expected.replace("}", ',"a":"a"}')
+
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "infer_word",
+ [
+ "trade_time",
+ "date",
+ "datetime",
+ "sold_at",
+ "modified",
+ "timestamp",
+ "timestamps",
+ ],
+ )
+ def test_convert_dates_infer(self, infer_word):
+ # GH10747
+
+ data = [{"id": 1, infer_word: 1036713600000}, {"id": 2}]
+ expected = DataFrame(
+ [[1, Timestamp("2002-11-08")], [2, pd.NaT]], columns=["id", infer_word]
+ )
+ expected[infer_word] = expected[infer_word].astype("M8[ms]")
+
+ result = read_json(StringIO(ujson_dumps(data)))[["id", infer_word]]
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "date,date_unit",
+ [
+ ("20130101 20:43:42.123", None),
+ ("20130101 20:43:42", "s"),
+ ("20130101 20:43:42.123", "ms"),
+ ("20130101 20:43:42.123456", "us"),
+ ("20130101 20:43:42.123456789", "ns"),
+ ],
+ )
+ def test_date_format_frame(self, date, date_unit, datetime_frame):
+ df = datetime_frame
+
+ df["date"] = Timestamp(date).as_unit("ns")
+ df.iloc[1, df.columns.get_loc("date")] = pd.NaT
+ df.iloc[5, df.columns.get_loc("date")] = pd.NaT
+ if date_unit:
+ json = df.to_json(date_format="iso", date_unit=date_unit)
+ else:
+ json = df.to_json(date_format="iso")
+
+ result = read_json(StringIO(json))
+ expected = df.copy()
+ if date_unit != "ns":
+ expected["date"] = expected["date"].dt.as_unit("us")
+ expected.index = expected.index.as_unit("us")
+ tm.assert_frame_equal(result, expected)
+
+ def test_date_format_frame_raises(self, datetime_frame):
+ df = datetime_frame
+ msg = "Invalid value 'foo' for option 'date_unit'"
+ with pytest.raises(ValueError, match=msg):
+ df.to_json(date_format="iso", date_unit="foo")
+
+ @pytest.mark.parametrize(
+ "date,date_unit",
+ [
+ ("20130101 20:43:42.123", None),
+ ("20130101 20:43:42", "s"),
+ ("20130101 20:43:42.123", "ms"),
+ ("20130101 20:43:42.123456", "us"),
+ ("20130101 20:43:42.123456789", "ns"),
+ ],
+ )
+ def test_date_format_series(self, date, date_unit, datetime_series):
+ ts = Series(Timestamp(date), index=datetime_series.index)
+ ts.iloc[1] = pd.NaT
+ ts.iloc[5] = pd.NaT
+ if date_unit:
+ json = ts.to_json(date_format="iso", date_unit=date_unit)
+ else:
+ json = ts.to_json(date_format="iso")
+
+ result = read_json(StringIO(json), typ="series")
+ expected = ts.copy()
+ if date_unit != "ns":
+ expected.index = expected.index.as_unit("us")
+ tm.assert_series_equal(result, expected)
+
+ def test_date_format_series_raises(self, datetime_series):
+ ts = Series(Timestamp("20130101 20:43:42.123"), index=datetime_series.index)
+ msg = "Invalid value 'foo' for option 'date_unit'"
+ with pytest.raises(ValueError, match=msg):
+ ts.to_json(date_format="iso", date_unit="foo")
+
+ def test_date_unit(self, unit, datetime_frame):
+ df = datetime_frame
+ df["date"] = Timestamp("20130101 20:43:42").as_unit("ns")
+ dl = df.columns.get_loc("date")
+ df.iloc[1, dl] = Timestamp("19710101 20:43:42")
+ df.iloc[2, dl] = Timestamp("21460101 20:43:42")
+ df.iloc[4, dl] = pd.NaT
+
+ msg = (
+ "'epoch' date format is deprecated and will be removed in a future "
+ "version, please use 'iso' date format instead."
+ )
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ json = df.to_json(date_format="epoch", date_unit=unit)
+
+ # force date unit
+ # result = read_json(StringIO(json), date_unit=unit)
+ expected = df.copy()
+ expected["date"] = expected["date"].dt.as_unit(unit)
+ expected.index = expected.index.as_unit(unit)
+ # tm.assert_frame_equal(result, expected)
+
+ # detect date unit
+ result = read_json(StringIO(json), date_unit=None)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "df, warn",
+ [
+ (DataFrame({"A": ["a", "b", "c"], "B": np.arange(3)}), None),
+ (DataFrame({"A": [True, False, False]}), None),
+ (
+ DataFrame(
+ {"A": ["a", "b", "c"], "B": pd.to_timedelta(np.arange(3), unit="D")}
+ ),
+ Pandas4Warning,
+ ),
+ (
+ DataFrame(
+ {"A": pd.to_datetime(["2020-01-01", "2020-02-01", "2020-03-01"])}
+ ),
+ Pandas4Warning,
+ ),
+ ],
+ )
+ def test_default_epoch_date_format_deprecated(self, df, warn):
+ # GH 57063
+ msg = (
+ "The default 'epoch' date format is deprecated and will be removed "
+ "in a future version, please use 'iso' date format instead."
+ )
+ with tm.assert_produces_warning(warn, match=msg):
+ df.to_json()
+
+ @pytest.mark.parametrize(
+ "dtype_backend", [None, pytest.param("pyarrow", marks=td.skip_if_no("pyarrow"))]
+ )
+ @pytest.mark.parametrize("unit", ["s", "ms", "us"])
+ def test_iso_non_nano_datetimes(self, dtype_backend, unit):
+ # Test that numpy datetimes
+ # in an Index or a column with non-nano resolution can be serialized
+ # correctly
+ # GH53686
+ index = DatetimeIndex(
+ [np.datetime64("2023-01-01T11:22:33.123456", unit)],
+ dtype=f"datetime64[{unit}]",
+ )
+ df = DataFrame(
+ {
+ "date": Series(
+ [np.datetime64("2022-01-01T11:22:33.123456", unit)],
+ dtype=f"datetime64[{unit}]",
+ index=index,
+ ),
+ "date_obj": Series(
+ [np.datetime64("2023-01-01T11:22:33.123456", unit)],
+ dtype=object,
+ index=index,
+ ),
+ },
+ )
+ expected = df.copy()
+ expected.index = expected.index.as_unit("us")
+ expected["date"] = expected["date"].dt.as_unit("us")
+ expected["date_obj"] = expected["date_obj"].astype("datetime64[us]")
+
+ if dtype_backend is not None:
+ df = df.convert_dtypes(dtype_backend=dtype_backend)
+
+ json = df.to_json(date_format="iso", date_unit=unit)
+ result = read_json(StringIO(json), convert_dates=["date", "date_obj"])
+ tm.assert_frame_equal(result, expected)
+
+ def test_weird_nested_json(self):
+ # this used to core dump the parser
+ s = r"""{
+ "status": "success",
+ "data": {
+ "posts": [
+ {
+ "id": 1,
+ "title": "A blog post",
+ "body": "Some useful content"
+ },
+ {
+ "id": 2,
+ "title": "Another blog post",
+ "body": "More content"
+ }
+ ]
+ }
+ }"""
+ read_json(StringIO(s))
+
+ def test_doc_example(self):
+ dfj2 = DataFrame(
+ np.random.default_rng(2).standard_normal((5, 2)), columns=list("AB")
+ )
+ dfj2["date"] = Timestamp("20130101")
+ dfj2["ints"] = range(5)
+ dfj2["bools"] = True
+ dfj2.index = date_range("20130101", periods=5, unit="ns")
+
+ msg = (
+ "The default 'epoch' date format is deprecated and will be removed "
+ "in a future version, please use 'iso' date format instead."
+ )
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ json = StringIO(dfj2.to_json())
+ result = read_json(json, dtype={"ints": np.int64, "bools": np.bool_})
+ tm.assert_frame_equal(result, result)
+
+ def test_round_trip_exception(self, datapath):
+ # GH 3867
+ path = datapath("io", "json", "data", "teams.csv")
+ df = pd.read_csv(path)
+ s = df.to_json()
+
+ result = read_json(StringIO(s))
+ res = result.reindex(index=df.index, columns=df.columns)
+ res = res.fillna(np.nan)
+ tm.assert_frame_equal(res, df)
+
+ @pytest.mark.network
+ @pytest.mark.single_cpu
+ def test_url(self, httpserver):
+ data = '{"created_at": ["2023-06-23T18:21:36Z"], "closed_at": ["2023-06-23T18:21:36"], "updated_at": ["2023-06-23T18:21:36Z"]}\n' # noqa: E501
+ httpserver.serve_content(content=data)
+ result = read_json(httpserver.url, convert_dates=True)
+ for field, dtype in [
+ ["created_at", pd.DatetimeTZDtype("us", tz="UTC")],
+ ["closed_at", "datetime64[us]"],
+ ["updated_at", pd.DatetimeTZDtype("us", tz="UTC")],
+ ]:
+ assert result[field].dtype == dtype
+
+ def test_timedelta(self, unit):
+ converter = lambda x: pd.to_timedelta(x, unit="ms")
+
+ ser = Series([timedelta(23), timedelta(seconds=5)], dtype=f"m8[{unit}]")
+ assert ser.dtype == f"timedelta64[{unit}]"
+
+ msg = (
+ "The default 'epoch' date format is deprecated and will be removed "
+ "in a future version, please use 'iso' date format instead."
+ )
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ result = read_json(StringIO(ser.to_json()), typ="series").apply(converter)
+ tm.assert_series_equal(result, ser.astype("m8[ms]"))
+
+ ser = Series(
+ [timedelta(23), timedelta(seconds=5)], index=Index([0, 1]), dtype="m8[ns]"
+ )
+ assert ser.dtype == "timedelta64[ns]"
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ result = read_json(StringIO(ser.to_json()), typ="series").apply(converter)
+ tm.assert_series_equal(result, ser.astype("m8[ms]"))
+
+ frame = DataFrame([timedelta(23), timedelta(seconds=5)], dtype="m8[ns]")
+ assert frame[0].dtype == "timedelta64[ns]"
+
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ json = frame.to_json()
+ result = read_json(StringIO(json)).apply(converter)
+ tm.assert_frame_equal(frame.astype("m8[ms]"), result)
+
+ def test_timedelta2(self):
+ frame = DataFrame(
+ {
+ "a": [timedelta(days=23), timedelta(seconds=5)],
+ "b": [1, 2],
+ "c": date_range(start="20130101", periods=2, unit="ns"),
+ }
+ )
+ frame["a"] = frame["a"].astype("m8[ns]")
+ msg = (
+ "The default 'epoch' date format is deprecated and will be removed "
+ "in a future version, please use 'iso' date format instead."
+ )
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ data = StringIO(frame.to_json(date_unit="ns"))
+ result = read_json(data)
+ result["a"] = pd.to_timedelta(result.a, unit="ns")
+ result["c"] = pd.to_datetime(result.c)
+ tm.assert_frame_equal(frame, result)
+
+ def test_mixed_timedelta_datetime(self):
+ td = timedelta(23)
+ ts = Timestamp("20130101")
+ frame = DataFrame({"a": [td, ts]}, dtype=object)
+
+ expected = DataFrame(
+ {"a": [pd.Timedelta(td).as_unit("ns")._value, ts.as_unit("ns")._value]}
+ )
+ data = StringIO(frame.to_json(date_unit="ns"))
+ result = read_json(data, dtype={"a": "int64"})
+ tm.assert_frame_equal(result, expected, check_index_type=False)
+
+ @pytest.mark.parametrize("as_object", [True, False])
+ @pytest.mark.parametrize("date_format", ["iso", "epoch"])
+ @pytest.mark.parametrize("timedelta_typ", [pd.Timedelta, timedelta])
+ def test_timedelta_to_json(self, as_object, date_format, timedelta_typ):
+ # GH28156: to_json not correctly formatting Timedelta
+ data = [timedelta_typ(days=1), timedelta_typ(days=2), pd.NaT]
+ if as_object:
+ data.append("a")
+
+ ser = Series(data, index=data)
+ if not as_object:
+ ser = ser.astype("m8[ns]")
+ ser.index = ser.index.astype("m8[ns]")
+ expected_warning = None
+ if date_format == "iso":
+ expected = (
+ '{"P1DT0H0M0S":"P1DT0H0M0S","P2DT0H0M0S":"P2DT0H0M0S","null":null}'
+ )
+ else:
+ expected_warning = Pandas4Warning
+ expected = '{"86400000":86400000,"172800000":172800000,"null":null}'
+
+ if as_object:
+ expected = expected.replace("}", ',"a":"a"}')
+
+ msg = (
+ "'epoch' date format is deprecated and will be removed in a future "
+ "version, please use 'iso' date format instead."
+ )
+ with tm.assert_produces_warning(expected_warning, match=msg):
+ result = ser.to_json(date_format=date_format)
+ assert result == expected
+
+ @pytest.mark.parametrize("as_object", [True, False])
+ @pytest.mark.parametrize("timedelta_typ", [pd.Timedelta, timedelta])
+ def test_timedelta_to_json_fractional_precision(self, as_object, timedelta_typ):
+ data = [timedelta_typ(milliseconds=42)]
+ ser = Series(data, index=data).astype("m8[ns]")
+ ser.index = ser.index.astype("m8[ns]")
+ warn = Pandas4Warning
+ if as_object:
+ ser = ser.astype(object)
+ warn = None
+
+ msg = (
+ "The default 'epoch' date format is deprecated and will be removed "
+ "in a future version, please use 'iso' date format instead."
+ )
+ with tm.assert_produces_warning(warn, match=msg):
+ result = ser.to_json()
+ expected = '{"42":42}'
+ assert result == expected
+
+ def test_default_handler(self):
+ value = object()
+ frame = DataFrame({"a": [7, value]})
+ expected = DataFrame({"a": [7, str(value)]})
+ result = read_json(StringIO(frame.to_json(default_handler=str)))
+ tm.assert_frame_equal(expected, result, check_index_type=False)
+
+ def test_default_handler_indirect(self):
+ def default(obj):
+ if isinstance(obj, complex):
+ return [("mathjs", "Complex"), ("re", obj.real), ("im", obj.imag)]
+ return str(obj)
+
+ df_list = [
+ 9,
+ DataFrame(
+ {"a": [1, "STR", complex(4, -5)], "b": [float("nan"), None, "N/A"]},
+ columns=["a", "b"],
+ ),
+ ]
+ expected = (
+ '[9,[[1,null],["STR",null],[[["mathjs","Complex"],'
+ '["re",4.0],["im",-5.0]],"N\\/A"]]]'
+ )
+ assert (
+ ujson_dumps(df_list, default_handler=default, orient="values") == expected
+ )
+
+ def test_default_handler_numpy_unsupported_dtype(self):
+ # GH12554 to_json raises 'Unhandled numpy dtype 15'
+ df = DataFrame(
+ {"a": [1, 2.3, complex(4, -5)], "b": [float("nan"), None, complex(1.2, 0)]},
+ columns=["a", "b"],
+ )
+ expected = (
+ '[["(1+0j)","(nan+0j)"],["(2.3+0j)","(nan+0j)"],["(4-5j)","(1.2+0j)"]]'
+ )
+ assert df.to_json(default_handler=str, orient="values") == expected
+
+ def test_default_handler_raises(self):
+ msg = "raisin"
+
+ def my_handler_raises(obj):
+ raise TypeError(msg)
+
+ with pytest.raises(TypeError, match=msg):
+ DataFrame({"a": [1, 2, object()]}).to_json(
+ default_handler=my_handler_raises
+ )
+ with pytest.raises(TypeError, match=msg):
+ DataFrame({"a": [1, 2, complex(4, -5)]}).to_json(
+ default_handler=my_handler_raises
+ )
+
+ def test_categorical(self):
+ # GH4377 df.to_json segfaults with non-ndarray blocks
+ df = DataFrame({"A": ["a", "b", "c", "a", "b", "b", "a"]})
+ df["B"] = df["A"]
+ expected = df.to_json()
+
+ df["B"] = df["A"].astype("category")
+ assert expected == df.to_json()
+
+ s = df["A"]
+ sc = df["B"]
+ assert s.to_json() == sc.to_json()
+
+ def test_datetime_tz(self):
+ # GH4377 df.to_json segfaults with non-ndarray blocks
+ tz_range = date_range("20130101", periods=3, tz="US/Eastern", unit="ns")
+ tz_naive = tz_range.tz_convert("utc").tz_localize(None)
+
+ df = DataFrame(
+ {"A": tz_range, "B": date_range("20130101", periods=3, unit="ns")}
+ )
+
+ df_naive = df.copy()
+ df_naive["A"] = tz_naive
+ msg = (
+ "The default 'epoch' date format is deprecated and will be removed "
+ "in a future version, please use 'iso' date format instead."
+ )
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ expected = df_naive.to_json()
+ assert expected == df.to_json()
+
+ stz = Series(tz_range)
+ s_naive = Series(tz_naive)
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ assert stz.to_json() == s_naive.to_json()
+
+ def test_sparse(self):
+ # GH4377 df.to_json segfaults with non-ndarray blocks
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 4)))
+ df.loc[:8] = np.nan
+
+ sdf = df.astype("Sparse")
+ expected = df.to_json()
+ assert expected == sdf.to_json()
+
+ s = Series(np.random.default_rng(2).standard_normal(10))
+ s.loc[:8] = np.nan
+ ss = s.astype("Sparse")
+
+ expected = s.to_json()
+ assert expected == ss.to_json()
+
+ @pytest.mark.parametrize(
+ "ts",
+ [
+ Timestamp("2013-01-10 05:00:00Z"),
+ Timestamp("2013-01-10 00:00:00", tz="US/Eastern"),
+ Timestamp("2013-01-10 00:00:00-0500"),
+ ],
+ )
+ def test_tz_is_utc(self, ts):
+ exp = '"2013-01-10T05:00:00.000Z"'
+
+ assert ujson_dumps(ts, iso_dates=True) == exp
+ dt = ts.to_pydatetime()
+ assert ujson_dumps(dt, iso_dates=True) == exp
+
+ def test_tz_is_naive(self):
+ ts = Timestamp("2013-01-10 05:00:00")
+ exp = '"2013-01-10T05:00:00.000"'
+
+ assert ujson_dumps(ts, iso_dates=True) == exp
+ dt = ts.to_pydatetime()
+ assert ujson_dumps(dt, iso_dates=True) == exp
+
+ @pytest.mark.parametrize(
+ "tz_range",
+ [
+ date_range("2013-01-01 05:00:00Z", periods=2, unit="ns"),
+ date_range("2013-01-01 00:00:00", periods=2, tz="US/Eastern", unit="ns"),
+ date_range("2013-01-01 00:00:00-0500", periods=2, unit="ns"),
+ ],
+ )
+ def test_tz_range_is_utc(self, tz_range):
+ exp = '["2013-01-01T05:00:00.000Z","2013-01-02T05:00:00.000Z"]'
+ dfexp = '{"DT":{"0":"2013-01-01T05:00:00.000Z","1":"2013-01-02T05:00:00.000Z"}}'
+
+ assert ujson_dumps(tz_range, iso_dates=True) == exp
+ dti = DatetimeIndex(tz_range)
+ # Ensure datetimes in object array are serialized correctly
+ # in addition to the normal DTI case
+ assert ujson_dumps(dti, iso_dates=True) == exp
+ assert ujson_dumps(dti.astype(object), iso_dates=True) == exp
+ df = DataFrame({"DT": dti})
+ result = ujson_dumps(df, iso_dates=True)
+ assert result == dfexp
+ assert ujson_dumps(df.astype({"DT": object}), iso_dates=True)
+
+ def test_tz_range_is_naive(self):
+ dti = date_range("2013-01-01 05:00:00", periods=2, unit="ns")
+
+ exp = '["2013-01-01T05:00:00.000","2013-01-02T05:00:00.000"]'
+ dfexp = '{"DT":{"0":"2013-01-01T05:00:00.000","1":"2013-01-02T05:00:00.000"}}'
+
+ # Ensure datetimes in object array are serialized correctly
+ # in addition to the normal DTI case
+ assert ujson_dumps(dti, iso_dates=True) == exp
+ assert ujson_dumps(dti.astype(object), iso_dates=True) == exp
+ df = DataFrame({"DT": dti})
+ result = ujson_dumps(df, iso_dates=True)
+ assert result == dfexp
+ assert ujson_dumps(df.astype({"DT": object}), iso_dates=True)
+
+ def test_read_inline_jsonl(self):
+ # GH9180
+
+ result = read_json(StringIO('{"a": 1, "b": 2}\n{"b":2, "a" :1}\n'), lines=True)
+ expected = DataFrame([[1, 2], [1, 2]], columns=["a", "b"])
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.single_cpu
+ @pytest.mark.network
+ @td.skip_if_not_us_locale
+ def test_read_s3_jsonl(self, s3_bucket_public_with_data, s3so):
+ # GH17200
+ result = read_json(
+ f"s3n://{s3_bucket_public_with_data.name}/items.jsonl",
+ lines=True,
+ storage_options=s3so,
+ )
+ expected = DataFrame([[1, 2], [1, 2]], columns=["a", "b"])
+ tm.assert_frame_equal(result, expected)
+
+ def test_read_local_jsonl(self, temp_file):
+ # GH17200
+ with open(temp_file, "w", encoding="utf-8") as infile:
+ infile.write('{"a": 1, "b": 2}\n{"b":2, "a" :1}\n')
+ result = read_json(temp_file, lines=True)
+ expected = DataFrame([[1, 2], [1, 2]], columns=["a", "b"])
+ tm.assert_frame_equal(result, expected)
+
+ def test_read_jsonl_unicode_chars(self):
+ # GH15132: non-ascii unicode characters
+ # \u201d == RIGHT DOUBLE QUOTATION MARK
+
+ # simulate file handle
+ json = '{"a": "foo”", "b": "bar"}\n{"a": "foo", "b": "bar"}\n'
+ json = StringIO(json)
+ result = read_json(json, lines=True)
+ expected = DataFrame([["foo\u201d", "bar"], ["foo", "bar"]], columns=["a", "b"])
+ tm.assert_frame_equal(result, expected)
+
+ # simulate string
+ json = StringIO('{"a": "foo”", "b": "bar"}\n{"a": "foo", "b": "bar"}\n')
+ result = read_json(json, lines=True)
+ expected = DataFrame([["foo\u201d", "bar"], ["foo", "bar"]], columns=["a", "b"])
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("bigNum", [sys.maxsize + 1, -(sys.maxsize + 2)])
+ def test_to_json_large_numbers(self, bigNum):
+ # GH34473
+ series = Series(bigNum, dtype=object, index=["articleId"])
+ json = series.to_json()
+ expected = '{"articleId":' + str(bigNum) + "}"
+ assert json == expected
+
+ df = DataFrame(bigNum, dtype=object, index=["articleId"], columns=[0])
+ json = df.to_json()
+ expected = '{"0":{"articleId":' + str(bigNum) + "}}"
+ assert json == expected
+
+ @pytest.mark.parametrize("bigNum", [-(2**63) - 1, 2**64])
+ def test_read_json_large_numbers(self, bigNum):
+ # GH20599, 26068
+ json = StringIO('{"articleId":' + str(bigNum) + "}")
+ msg = r"Value is too small|Value is too big"
+ with pytest.raises(ValueError, match=msg):
+ read_json(json)
+
+ json = StringIO('{"0":{"articleId":' + str(bigNum) + "}}")
+ with pytest.raises(ValueError, match=msg):
+ read_json(json)
+
+ def test_read_json_large_numbers2(self):
+ # GH18842
+ json = '{"articleId": "1404366058080022500245"}'
+ json = StringIO(json)
+ result = read_json(json, typ="series")
+ expected = Series(1.404366e21, index=["articleId"])
+ tm.assert_series_equal(result, expected)
+
+ json = '{"0": {"articleId": "1404366058080022500245"}}'
+ json = StringIO(json)
+ result = read_json(json)
+ expected = DataFrame(1.404366e21, index=["articleId"], columns=[0])
+ tm.assert_frame_equal(result, expected)
+
+ def test_to_jsonl(self):
+ # GH9180
+ df = DataFrame([[1, 2], [1, 2]], columns=["a", "b"])
+ result = df.to_json(orient="records", lines=True)
+ expected = '{"a":1,"b":2}\n{"a":1,"b":2}\n'
+ assert result == expected
+
+ df = DataFrame([["foo}", "bar"], ['foo"', "bar"]], columns=["a", "b"])
+ result = df.to_json(orient="records", lines=True)
+ expected = '{"a":"foo}","b":"bar"}\n{"a":"foo\\"","b":"bar"}\n'
+ assert result == expected
+ tm.assert_frame_equal(read_json(StringIO(result), lines=True), df)
+
+ # GH15096: escaped characters in columns and data
+ df = DataFrame([["foo\\", "bar"], ['foo"', "bar"]], columns=["a\\", "b"])
+ result = df.to_json(orient="records", lines=True)
+ expected = '{"a\\\\":"foo\\\\","b":"bar"}\n{"a\\\\":"foo\\"","b":"bar"}\n'
+ assert result == expected
+
+ tm.assert_frame_equal(read_json(StringIO(result), lines=True), df)
+
+ # TODO: there is a near-identical test for pytables; can we share?
+ @pytest.mark.xfail(reason="GH#13774 encoding kwarg not supported", raises=TypeError)
+ @pytest.mark.parametrize(
+ "val",
+ [
+ [b"E\xc9, 17", b"", b"a", b"b", b"c"],
+ [b"E\xc9, 17", b"a", b"b", b"c"],
+ [b"EE, 17", b"", b"a", b"b", b"c"],
+ [b"E\xc9, 17", b"\xf8\xfc", b"a", b"b", b"c"],
+ [b"", b"a", b"b", b"c"],
+ [b"\xf8\xfc", b"a", b"b", b"c"],
+ [b"A\xf8\xfc", b"", b"a", b"b", b"c"],
+ [np.nan, b"", b"b", b"c"],
+ [b"A\xf8\xfc", np.nan, b"", b"b", b"c"],
+ ],
+ )
+ @pytest.mark.parametrize("dtype", ["category", object])
+ def test_latin_encoding(self, dtype, val, temp_file):
+ # GH 13774
+ ser = Series(
+ [x.decode("latin-1") if isinstance(x, bytes) else x for x in val],
+ dtype=dtype,
+ )
+ encoding = "latin-1"
+ ser.to_json(temp_file, encoding=encoding)
+ retr = read_json(StringIO(temp_file), encoding=encoding)
+ tm.assert_series_equal(ser, retr, check_categorical=False)
+
+ def test_data_frame_size_after_to_json(self):
+ # GH15344
+ df = DataFrame({"a": [str(1)]})
+
+ size_before = df.memory_usage(index=True, deep=True).sum()
+ df.to_json()
+ size_after = df.memory_usage(index=True, deep=True).sum()
+
+ assert size_before == size_after
+
+ @pytest.mark.parametrize(
+ "index", [None, [1, 2], [1.0, 2.0], ["a", "b"], ["1", "2"], ["1.", "2."]]
+ )
+ @pytest.mark.parametrize("columns", [["a", "b"], ["1", "2"], ["1.", "2."]])
+ def test_from_json_to_json_table_index_and_columns(self, index, columns):
+ # GH25433 GH25435
+ expected = DataFrame([[1, 2], [3, 4]], index=index, columns=columns)
+ dfjson = expected.to_json(orient="table")
+
+ result = read_json(StringIO(dfjson), orient="table")
+ tm.assert_frame_equal(result, expected)
+
+ def test_from_json_to_json_table_dtypes(self):
+ # GH21345
+ expected = DataFrame({"a": [1, 2], "b": [3.0, 4.0], "c": ["5", "6"]})
+ dfjson = expected.to_json(orient="table")
+ result = read_json(StringIO(dfjson), orient="table")
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("orient", ["split", "records", "index", "columns"])
+ def test_to_json_from_json_columns_dtypes(self, orient, using_infer_string):
+ # GH21892 GH33205
+ expected = DataFrame.from_dict(
+ {
+ "Integer": Series([1, 2, 3], dtype="int64"),
+ "Float": Series([None, 2.0, 3.0], dtype="float64"),
+ "Object": Series([None, "", "c"], dtype="object"),
+ "Bool": Series([True, False, True], dtype="bool"),
+ "Category": Series(["a", "b", None], dtype="category"),
+ "Datetime": Series(
+ ["2020-01-01", None, "2020-01-03"], dtype="datetime64[ms]"
+ ),
+ }
+ )
+ msg = (
+ "The default 'epoch' date format is deprecated and will be removed "
+ "in a future version, please use 'iso' date format instead."
+ )
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ dfjson = expected.to_json(orient=orient)
+
+ if using_infer_string:
+ # When this is read back in it is inferred to "str" dtype which
+ # uses NaN instead of None.
+ expected.loc[0, "Object"] = np.nan
+
+ result = read_json(
+ StringIO(dfjson),
+ orient=orient,
+ dtype={
+ "Integer": "int64",
+ "Float": "float64",
+ "Object": "object",
+ "Bool": "bool",
+ "Category": "category",
+ "Datetime": "datetime64[ns]",
+ },
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_to_json_with_index_as_a_column_name(self):
+ df = DataFrame(data={"index": [1, 2], "a": [2, 3]})
+ with pytest.raises(
+ ValueError, match="Overlapping names between the index and columns"
+ ):
+ df.to_json(orient="table")
+
+ @pytest.mark.parametrize("dtype", [True, {"b": int, "c": int}])
+ def test_read_json_table_dtype_raises(self, dtype):
+ # GH21345
+ df = DataFrame({"a": [1, 2], "b": [3.0, 4.0], "c": ["5", "6"]})
+ dfjson = df.to_json(orient="table")
+ msg = "cannot pass both dtype and orient='table'"
+ with pytest.raises(ValueError, match=msg):
+ read_json(dfjson, orient="table", dtype=dtype)
+
+ @pytest.mark.parametrize("orient", ["index", "columns", "records", "values"])
+ def test_read_json_table_empty_axes_dtype(self, orient):
+ # GH28558
+
+ expected = DataFrame()
+ result = read_json(StringIO("{}"), orient=orient, convert_axes=True)
+ tm.assert_index_equal(result.index, expected.index)
+ tm.assert_index_equal(result.columns, expected.columns)
+
+ def test_read_json_table_convert_axes_raises(self):
+ # GH25433 GH25435
+ df = DataFrame([[1, 2], [3, 4]], index=[1.0, 2.0], columns=["1.", "2."])
+ dfjson = df.to_json(orient="table")
+ msg = "cannot pass both convert_axes and orient='table'"
+ with pytest.raises(ValueError, match=msg):
+ read_json(dfjson, orient="table", convert_axes=True)
+
+ @pytest.mark.parametrize(
+ "data, expected",
+ [
+ (
+ DataFrame([[1, 2], [4, 5]], columns=["a", "b"]),
+ {"columns": ["a", "b"], "data": [[1, 2], [4, 5]]},
+ ),
+ (
+ DataFrame([[1, 2], [4, 5]], columns=["a", "b"]).rename_axis("foo"),
+ {"columns": ["a", "b"], "data": [[1, 2], [4, 5]]},
+ ),
+ (
+ DataFrame(
+ [[1, 2], [4, 5]], columns=["a", "b"], index=[["a", "b"], ["c", "d"]]
+ ),
+ {"columns": ["a", "b"], "data": [[1, 2], [4, 5]]},
+ ),
+ (Series([1, 2, 3], name="A"), {"name": "A", "data": [1, 2, 3]}),
+ (
+ Series([1, 2, 3], name="A").rename_axis("foo"),
+ {"name": "A", "data": [1, 2, 3]},
+ ),
+ (
+ Series([1, 2], name="A", index=[["a", "b"], ["c", "d"]]),
+ {"name": "A", "data": [1, 2]},
+ ),
+ ],
+ )
+ def test_index_false_to_json_split(self, data, expected):
+ # GH 17394
+ # Testing index=False in to_json with orient='split'
+
+ result = data.to_json(orient="split", index=False)
+ result = json.loads(result)
+
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "data",
+ [
+ (DataFrame([[1, 2], [4, 5]], columns=["a", "b"])),
+ (DataFrame([[1, 2], [4, 5]], columns=["a", "b"]).rename_axis("foo")),
+ (
+ DataFrame(
+ [[1, 2], [4, 5]], columns=["a", "b"], index=[["a", "b"], ["c", "d"]]
+ )
+ ),
+ (Series([1, 2, 3], name="A")),
+ (Series([1, 2, 3], name="A").rename_axis("foo")),
+ (Series([1, 2], name="A", index=[["a", "b"], ["c", "d"]])),
+ ],
+ )
+ def test_index_false_to_json_table(self, data):
+ # GH 17394
+ # Testing index=False in to_json with orient='table'
+
+ result = data.to_json(orient="table", index=False)
+ result = json.loads(result)
+
+ expected = {
+ "schema": pd.io.json.build_table_schema(data, index=False),
+ "data": DataFrame(data).to_dict(orient="records"),
+ }
+
+ assert result == expected
+
+ @pytest.mark.parametrize("orient", ["index", "columns"])
+ def test_index_false_error_to_json(self, orient):
+ # GH 17394, 25513
+ # Testing error message from to_json with index=False
+
+ df = DataFrame([[1, 2], [4, 5]], columns=["a", "b"])
+
+ msg = (
+ "'index=False' is only valid when 'orient' is 'split', "
+ "'table', 'records', or 'values'"
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.to_json(orient=orient, index=False)
+
+ @pytest.mark.parametrize("orient", ["records", "values"])
+ def test_index_true_error_to_json(self, orient):
+ # GH 25513
+ # Testing error message from to_json with index=True
+
+ df = DataFrame([[1, 2], [4, 5]], columns=["a", "b"])
+
+ msg = (
+ "'index=True' is only valid when 'orient' is 'split', "
+ "'table', 'index', or 'columns'"
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.to_json(orient=orient, index=True)
+
+ @pytest.mark.parametrize("orient", ["split", "table"])
+ @pytest.mark.parametrize("index", [True, False])
+ def test_index_false_from_json_to_json(self, orient, index):
+ # GH25170
+ # Test index=False in from_json to_json
+ expected = DataFrame({"a": [1, 2], "b": [3, 4]})
+ dfjson = expected.to_json(orient=orient, index=index)
+ result = read_json(StringIO(dfjson), orient=orient)
+ tm.assert_frame_equal(result, expected)
+
+ def test_read_timezone_information(self):
+ # GH 25546
+ result = read_json(
+ StringIO('{"2019-01-01T11:00:00.000Z":88}'), typ="series", orient="index"
+ )
+ exp_dti = DatetimeIndex(["2019-01-01 11:00:00"], dtype="M8[us, UTC]")
+ expected = Series([88], index=exp_dti)
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "url",
+ [
+ "s3://example-fsspec/",
+ "gcs://another-fsspec/file.json",
+ "filecache::s3://yet-another-fsspec/file.json",
+ "https://example-site.com/data",
+ "some-protocol://data.txt",
+ ],
+ )
+ def test_read_json_with_url_value(self, url):
+ # GH 36271
+ result = read_json(StringIO(f'{{"url":{{"0":"{url}"}}}}'))
+ expected = DataFrame({"url": [url]})
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "compression",
+ ["", ".gz", ".bz2", ".tar"],
+ )
+ def test_read_json_with_very_long_file_path(self, compression):
+ # GH 46718
+ long_json_path = f"{'a' * 1000}.json{compression}"
+ with pytest.raises(
+ FileNotFoundError, match=f"File {long_json_path} does not exist"
+ ):
+ # path too long for Windows is handled in file_exists() but raises in
+ # _get_data_from_filepath()
+ read_json(long_json_path)
+
+ @pytest.mark.parametrize(
+ "date_format,key", [("epoch", 86400000), ("iso", "1970-01-02T00:00:00.000")]
+ )
+ def test_datetime_as_label(self, date_format, key, unit):
+ df = DataFrame(
+ [[1]], columns=[(Timestamp(0) + pd.Timedelta("1D")).as_unit(unit)]
+ )
+ expected = f'{{"{key}":{{"0":1}}}}'
+
+ expected_warning = None
+ if date_format == "epoch":
+ expected_warning = Pandas4Warning
+
+ msg = (
+ "'epoch' date format is deprecated and will be removed in a future "
+ "version, please use 'iso' date format instead."
+ )
+ with tm.assert_produces_warning(expected_warning, match=msg):
+ result = df.to_json(date_format=date_format)
+
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "date_format,key", [("epoch", 86400000), ("iso", "P1DT0H0M0S")]
+ )
+ def test_timedelta_as_label(self, date_format, key, unit):
+ df = DataFrame([[1]], columns=[pd.Timedelta("1D").as_unit(unit)])
+ expected = f'{{"{key}":{{"0":1}}}}'
+
+ expected_warning = None
+ if date_format == "epoch":
+ expected_warning = Pandas4Warning
+
+ msg = (
+ "'epoch' date format is deprecated and will be removed in a future "
+ "version, please use 'iso' date format instead."
+ )
+ with tm.assert_produces_warning(expected_warning, match=msg):
+ result = df.to_json(date_format=date_format)
+
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "orient,expected",
+ [
+ ("index", "{\"('a', 'b')\":{\"('c', 'd')\":1}}"),
+ ("columns", "{\"('c', 'd')\":{\"('a', 'b')\":1}}"),
+ # TODO: the below have separate encoding procedures
+ pytest.param(
+ "split",
+ "",
+ marks=pytest.mark.xfail(
+ reason="Produces JSON but not in a consistent manner"
+ ),
+ ),
+ pytest.param(
+ "table",
+ "",
+ marks=pytest.mark.xfail(
+ reason="Produces JSON but not in a consistent manner"
+ ),
+ ),
+ ],
+ )
+ def test_tuple_labels(self, orient, expected):
+ # GH 20500
+ df = DataFrame([[1]], index=[("a", "b")], columns=[("c", "d")])
+ result = df.to_json(orient=orient)
+ assert result == expected
+
+ @pytest.mark.parametrize("indent", [1, 2, 4])
+ def test_to_json_indent(self, indent):
+ # GH 12004
+ df = DataFrame([["foo", "bar"], ["baz", "qux"]], columns=["a", "b"])
+
+ result = df.to_json(indent=indent)
+ spaces = " " * indent
+ expected = f"""{{
+{spaces}"a":{{
+{spaces}{spaces}"0":"foo",
+{spaces}{spaces}"1":"baz"
+{spaces}}},
+{spaces}"b":{{
+{spaces}{spaces}"0":"bar",
+{spaces}{spaces}"1":"qux"
+{spaces}}}
+}}"""
+
+ assert result == expected
+
+ @pytest.mark.skipif(
+ using_string_dtype(),
+ reason="Adjust expected when infer_string is default, no bug here, "
+ "just a complicated parametrization",
+ )
+ @pytest.mark.parametrize(
+ "orient,expected",
+ [
+ (
+ "split",
+ """{
+ "columns":[
+ "a",
+ "b"
+ ],
+ "index":[
+ 0,
+ 1
+ ],
+ "data":[
+ [
+ "foo",
+ "bar"
+ ],
+ [
+ "baz",
+ "qux"
+ ]
+ ]
+}""",
+ ),
+ (
+ "records",
+ """[
+ {
+ "a":"foo",
+ "b":"bar"
+ },
+ {
+ "a":"baz",
+ "b":"qux"
+ }
+]""",
+ ),
+ (
+ "index",
+ """{
+ "0":{
+ "a":"foo",
+ "b":"bar"
+ },
+ "1":{
+ "a":"baz",
+ "b":"qux"
+ }
+}""",
+ ),
+ (
+ "columns",
+ """{
+ "a":{
+ "0":"foo",
+ "1":"baz"
+ },
+ "b":{
+ "0":"bar",
+ "1":"qux"
+ }
+}""",
+ ),
+ (
+ "values",
+ """[
+ [
+ "foo",
+ "bar"
+ ],
+ [
+ "baz",
+ "qux"
+ ]
+]""",
+ ),
+ (
+ "table",
+ """{
+ "schema":{
+ "fields":[
+ {
+ "name":"index",
+ "type":"integer"
+ },
+ {
+ "name":"a",
+ "type":"string"
+ },
+ {
+ "name":"b",
+ "type":"string"
+ }
+ ],
+ "primaryKey":[
+ "index"
+ ],
+ "pandas_version":"1.4.0"
+ },
+ "data":[
+ {
+ "index":0,
+ "a":"foo",
+ "b":"bar"
+ },
+ {
+ "index":1,
+ "a":"baz",
+ "b":"qux"
+ }
+ ]
+}""",
+ ),
+ ],
+ )
+ def test_json_indent_all_orients(self, orient, expected):
+ # GH 12004
+ df = DataFrame([["foo", "bar"], ["baz", "qux"]], columns=["a", "b"])
+ result = df.to_json(orient=orient, indent=4)
+ assert result == expected
+
+ def test_json_negative_indent_raises(self):
+ with pytest.raises(ValueError, match="must be a nonnegative integer"):
+ DataFrame().to_json(indent=-1)
+
+ def test_emca_262_nan_inf_support(self):
+ # GH 12213
+ data = StringIO(
+ '["a", NaN, "NaN", Infinity, "Infinity", -Infinity, "-Infinity"]'
+ )
+ result = read_json(data)
+ expected = DataFrame(
+ ["a", None, "NaN", np.inf, "Infinity", -np.inf, "-Infinity"]
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_frame_int_overflow(self):
+ # GH 30320
+ encoded_json = json.dumps([{"col": "31900441201190696999"}, {"col": "Text"}])
+ expected = DataFrame({"col": ["31900441201190696999", "Text"]})
+ result = read_json(StringIO(encoded_json))
+ tm.assert_frame_equal(result, expected)
+
+ def test_json_multiindex(self):
+ dataframe = DataFrame({"x": [1, 2, 3], "y": ["a", "b", "c"]})
+ expected = (
+ '{"(0, \'x\')":1,"(0, \'y\')":"a","(1, \'x\')":2,'
+ '"(1, \'y\')":"b","(2, \'x\')":3,"(2, \'y\')":"c"}'
+ )
+ series = dataframe.stack()
+ result = series.to_json(orient="index")
+ assert result == expected
+
+ @pytest.mark.single_cpu
+ @pytest.mark.network
+ def test_to_s3(self, s3_bucket_public, s3so):
+ # GH 28375
+ mock_bucket_name = s3_bucket_public.name
+ target_file = f"{uuid.uuid4()}.json"
+ df = DataFrame({"x": [1, 2, 3], "y": [2, 4, 6]})
+ uri = f"s3://{mock_bucket_name}/{target_file}"
+ df.to_json(uri, storage_options=s3so)
+ result = read_json(uri, storage_options=s3so)
+ tm.assert_frame_equal(result, df)
+
+ def test_json_pandas_nulls(self, nulls_fixture):
+ # GH 31615
+ expected_warning = None
+ msg = (
+ "The default 'epoch' date format is deprecated and will be removed "
+ "in a future version, please use 'iso' date format instead."
+ )
+ if nulls_fixture is pd.NaT:
+ expected_warning = Pandas4Warning
+
+ with tm.assert_produces_warning(expected_warning, match=msg):
+ result = DataFrame([[nulls_fixture]]).to_json()
+ assert result == '{"0":{"0":null}}'
+
+ def test_readjson_bool_series(self):
+ # GH31464
+ result = read_json(StringIO("[true, true, false]"), typ="series")
+ expected = Series([True, True, False])
+ tm.assert_series_equal(result, expected)
+
+ def test_to_json_multiindex_escape(self):
+ # GH 15273
+ df = DataFrame(
+ True,
+ index=date_range("2017-01-20", "2017-01-23", unit="ns"),
+ columns=["foo", "bar"],
+ ).stack()
+ result = df.to_json()
+ expected = (
+ "{\"(Timestamp('2017-01-20 00:00:00'), 'foo')\":true,"
+ "\"(Timestamp('2017-01-20 00:00:00'), 'bar')\":true,"
+ "\"(Timestamp('2017-01-21 00:00:00'), 'foo')\":true,"
+ "\"(Timestamp('2017-01-21 00:00:00'), 'bar')\":true,"
+ "\"(Timestamp('2017-01-22 00:00:00'), 'foo')\":true,"
+ "\"(Timestamp('2017-01-22 00:00:00'), 'bar')\":true,"
+ "\"(Timestamp('2017-01-23 00:00:00'), 'foo')\":true,"
+ "\"(Timestamp('2017-01-23 00:00:00'), 'bar')\":true}"
+ )
+ assert result == expected
+
+ def test_to_json_series_of_objects(self):
+ class _TestObject:
+ def __init__(self, a, b, _c, d) -> None:
+ self.a = a
+ self.b = b
+ self._c = _c
+ self.d = d
+
+ def e(self):
+ return 5
+
+ # JSON keys should be all non-callable non-underscore attributes, see GH-42768
+ series = Series([_TestObject(a=1, b=2, _c=3, d=4)])
+ assert json.loads(series.to_json()) == {"0": {"a": 1, "b": 2, "d": 4}}
+
+ @pytest.mark.parametrize(
+ "data,expected",
+ [
+ (
+ Series({0: -6 + 8j, 1: 0 + 1j, 2: 9 - 5j}),
+ '{"0":{"imag":8.0,"real":-6.0},'
+ '"1":{"imag":1.0,"real":0.0},'
+ '"2":{"imag":-5.0,"real":9.0}}',
+ ),
+ (
+ Series({0: -9.39 + 0.66j, 1: 3.95 + 9.32j, 2: 4.03 - 0.17j}),
+ '{"0":{"imag":0.66,"real":-9.39},'
+ '"1":{"imag":9.32,"real":3.95},'
+ '"2":{"imag":-0.17,"real":4.03}}',
+ ),
+ (
+ DataFrame([[-2 + 3j, -1 - 0j], [4 - 3j, -0 - 10j]]),
+ '{"0":{"0":{"imag":3.0,"real":-2.0},'
+ '"1":{"imag":-3.0,"real":4.0}},'
+ '"1":{"0":{"imag":0.0,"real":-1.0},'
+ '"1":{"imag":-10.0,"real":0.0}}}',
+ ),
+ (
+ DataFrame(
+ [[-0.28 + 0.34j, -1.08 - 0.39j], [0.41 - 0.34j, -0.78 - 1.35j]]
+ ),
+ '{"0":{"0":{"imag":0.34,"real":-0.28},'
+ '"1":{"imag":-0.34,"real":0.41}},'
+ '"1":{"0":{"imag":-0.39,"real":-1.08},'
+ '"1":{"imag":-1.35,"real":-0.78}}}',
+ ),
+ ],
+ )
+ def test_complex_data_tojson(self, data, expected):
+ # GH41174
+ result = data.to_json()
+ assert result == expected
+
+ def test_json_uint64(self):
+ # GH21073
+ expected = (
+ '{"columns":["col1"],"index":[0,1],'
+ '"data":[[13342205958987758245],[12388075603347835679]]}'
+ )
+ df = DataFrame(data={"col1": [13342205958987758245, 12388075603347835679]})
+ result = df.to_json(orient="split")
+ assert result == expected
+
+ def test_read_json_dtype_backend(
+ self, string_storage, dtype_backend, orient, using_infer_string
+ ):
+ # GH#50750
+ df = DataFrame(
+ {
+ "a": Series([1, NA, 3], dtype="Int64"),
+ "b": Series([1, 2, 3], dtype="Int64"),
+ "c": Series([1.5, NA, 2.5], dtype="Float64"),
+ "d": Series([1.5, 2.0, 2.5], dtype="Float64"),
+ "e": [True, False, None],
+ "f": [True, False, True],
+ "g": ["a", "b", "c"],
+ "h": ["a", "b", None],
+ }
+ )
+
+ out = df.to_json(orient=orient)
+ with pd.option_context("mode.string_storage", string_storage):
+ result = read_json(
+ StringIO(out), dtype_backend=dtype_backend, orient=orient
+ )
+
+ if dtype_backend == "pyarrow":
+ pa = pytest.importorskip("pyarrow")
+ string_dtype = pd.ArrowDtype(pa.string())
+ else:
+ string_dtype = pd.StringDtype(string_storage)
+
+ expected = DataFrame(
+ {
+ "a": Series([1, NA, 3], dtype="Int64"),
+ "b": Series([1, 2, 3], dtype="Int64"),
+ "c": Series([1.5, NA, 2.5], dtype="Float64"),
+ "d": Series([1.5, 2.0, 2.5], dtype="Float64"),
+ "e": Series([True, False, NA], dtype="boolean"),
+ "f": Series([True, False, True], dtype="boolean"),
+ "g": Series(["a", "b", "c"], dtype=string_dtype),
+ "h": Series(["a", "b", None], dtype=string_dtype),
+ }
+ )
+
+ if dtype_backend == "pyarrow":
+ pa = pytest.importorskip("pyarrow")
+ from pandas.arrays import ArrowExtensionArray
+
+ expected = DataFrame(
+ {
+ col: ArrowExtensionArray(pa.array(expected[col], from_pandas=True))
+ for col in expected.columns
+ }
+ )
+
+ if orient == "values":
+ expected.columns = list(range(8))
+
+ # the storage of the str columns' Index is also affected by the
+ # string_storage setting -> ignore that for checking the result
+ tm.assert_frame_equal(result, expected, check_column_type=False)
+
+ @td.skip_if_no("pyarrow")
+ @pytest.mark.filterwarnings("ignore:Passing a BlockManager:DeprecationWarning")
+ def test_read_json_pyarrow_with_dtype(self):
+ dtype = {"a": "int32[pyarrow]", "b": "int64[pyarrow]"}
+ json = b'{"a": 1, "b": 2}\n'
+
+ df = read_json(
+ BytesIO(json),
+ dtype=dtype,
+ lines=True,
+ engine="pyarrow",
+ dtype_backend="pyarrow",
+ )
+
+ result = df.dtypes
+ expected = Series(
+ data=[
+ pd.ArrowDtype.construct_from_string("int32[pyarrow]"),
+ pd.ArrowDtype.construct_from_string("int64[pyarrow]"),
+ ],
+ index=["a", "b"],
+ )
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize("orient", ["split", "records", "index"])
+ def test_read_json_nullable_series(self, string_storage, dtype_backend, orient):
+ # GH#50750
+ pa = pytest.importorskip("pyarrow")
+ ser = Series([1, NA, 3], dtype="Int64")
+
+ out = ser.to_json(orient=orient)
+ with pd.option_context("mode.string_storage", string_storage):
+ result = read_json(
+ StringIO(out), dtype_backend=dtype_backend, orient=orient, typ="series"
+ )
+
+ expected = Series([1, NA, 3], dtype="Int64")
+
+ if dtype_backend == "pyarrow":
+ from pandas.arrays import ArrowExtensionArray
+
+ expected = Series(ArrowExtensionArray(pa.array(expected, from_pandas=True)))
+
+ tm.assert_series_equal(result, expected)
+
+ def test_invalid_dtype_backend(self):
+ msg = (
+ "dtype_backend numpy is invalid, only 'numpy_nullable' and "
+ "'pyarrow' are allowed."
+ )
+ with pytest.raises(ValueError, match=msg):
+ read_json("test", dtype_backend="numpy")
+
+
+def test_invalid_engine():
+ # GH 48893
+ ser = Series(range(1))
+ out = ser.to_json()
+ with pytest.raises(ValueError, match="The engine type foo"):
+ read_json(out, engine="foo")
+
+
+def test_pyarrow_engine_lines_false():
+ # GH 48893
+ ser = Series(range(1))
+ out = ser.to_json()
+ with pytest.raises(ValueError, match="currently pyarrow engine only supports"):
+ read_json(out, engine="pyarrow", lines=False)
+
+
+def test_json_roundtrip_string_inference(orient):
+ df = DataFrame(
+ [["a", "b"], ["c", "d"]], index=["row 1", "row 2"], columns=["col 1", "col 2"]
+ )
+ out = df.to_json()
+ with pd.option_context("future.infer_string", True):
+ result = read_json(StringIO(out))
+ dtype = pd.StringDtype(na_value=np.nan)
+ expected = DataFrame(
+ [["a", "b"], ["c", "d"]],
+ dtype=dtype,
+ index=Index(["row 1", "row 2"], dtype=dtype),
+ columns=Index(["col 1", "col 2"], dtype=dtype),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@td.skip_if_no("pyarrow")
+def test_to_json_ea_null():
+ # GH#57224
+ df = DataFrame(
+ {
+ "a": Series([1, NA], dtype="int64[pyarrow]"),
+ "b": Series([2, NA], dtype="Int64"),
+ }
+ )
+ result = df.to_json(orient="records", lines=True)
+ expected = """{"a":1,"b":2}
+{"a":null,"b":null}
+"""
+ assert result == expected
+
+
+def test_read_json_lines_rangeindex():
+ # GH 57429
+ data = """
+{"a": 1, "b": 2}
+{"a": 3, "b": 4}
+"""
+ result = read_json(StringIO(data), lines=True).index
+ expected = RangeIndex(2)
+ tm.assert_index_equal(result, expected, exact=True)
+
+
+def test_large_number():
+ # GH#20608
+ result = read_json(
+ StringIO('["9999999999999999"]'),
+ orient="values",
+ typ="series",
+ convert_dates=False,
+ )
+ expected = Series([9999999999999999])
+ tm.assert_series_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_readlines.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_readlines.py
new file mode 100644
index 0000000000000000000000000000000000000000..6e65a11792391655d5c1726b4c60f6cc114f67ff
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_readlines.py
@@ -0,0 +1,544 @@
+from collections.abc import Iterator
+from io import (
+ BytesIO,
+ StringIO,
+)
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ read_json,
+)
+import pandas._testing as tm
+
+from pandas.io.json._json import JsonReader
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+@pytest.fixture
+def lines_json_df():
+ df = DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
+ return df.to_json(lines=True, orient="records")
+
+
+@pytest.fixture(params=["ujson", "pyarrow"])
+def engine(request):
+ if request.param == "pyarrow":
+ pytest.importorskip("pyarrow.json")
+ return request.param
+
+
+def test_read_jsonl():
+ # GH9180
+ result = read_json(StringIO('{"a": 1, "b": 2}\n{"b":2, "a" :1}\n'), lines=True)
+ expected = DataFrame([[1, 2], [1, 2]], columns=["a", "b"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_jsonl_engine_pyarrow(datapath, engine):
+ result = read_json(
+ datapath("io", "json", "data", "line_delimited.json"),
+ lines=True,
+ engine=engine,
+ )
+ expected = DataFrame({"a": [1, 3, 5], "b": [2, 4, 6]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_datetime(request, engine):
+ # GH33787
+ if engine == "pyarrow":
+ # GH 48893
+ reason = "Pyarrow only supports a file path as an input and line delimited json"
+ request.applymarker(pytest.mark.xfail(reason=reason, raises=ValueError))
+
+ df = DataFrame(
+ [([1, 2], ["2020-03-05", "2020-04-08T09:58:49+00:00"], "hector")],
+ columns=["accounts", "date", "name"],
+ )
+ json_line = df.to_json(lines=True, orient="records")
+
+ if engine == "pyarrow":
+ result = read_json(StringIO(json_line), engine=engine)
+ else:
+ result = read_json(StringIO(json_line), engine=engine)
+ expected = DataFrame(
+ [[1, "2020-03-05", "hector"], [2, "2020-04-08T09:58:49+00:00", "hector"]],
+ columns=["accounts", "date", "name"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_jsonl_unicode_chars():
+ # GH15132: non-ascii unicode characters
+ # \u201d == RIGHT DOUBLE QUOTATION MARK
+
+ # simulate file handle
+ json = '{"a": "foo”", "b": "bar"}\n{"a": "foo", "b": "bar"}\n'
+ json = StringIO(json)
+ result = read_json(json, lines=True)
+ expected = DataFrame([["foo\u201d", "bar"], ["foo", "bar"]], columns=["a", "b"])
+ tm.assert_frame_equal(result, expected)
+
+ # simulate string
+ json = '{"a": "foo”", "b": "bar"}\n{"a": "foo", "b": "bar"}\n'
+ result = read_json(StringIO(json), lines=True)
+ expected = DataFrame([["foo\u201d", "bar"], ["foo", "bar"]], columns=["a", "b"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_to_jsonl():
+ # GH9180
+ df = DataFrame([[1, 2], [1, 2]], columns=["a", "b"])
+ result = df.to_json(orient="records", lines=True)
+ expected = '{"a":1,"b":2}\n{"a":1,"b":2}\n'
+ assert result == expected
+
+ df = DataFrame([["foo}", "bar"], ['foo"', "bar"]], columns=["a", "b"])
+ result = df.to_json(orient="records", lines=True)
+ expected = '{"a":"foo}","b":"bar"}\n{"a":"foo\\"","b":"bar"}\n'
+ assert result == expected
+ tm.assert_frame_equal(read_json(StringIO(result), lines=True), df)
+
+ # GH15096: escaped characters in columns and data
+ df = DataFrame([["foo\\", "bar"], ['foo"', "bar"]], columns=["a\\", "b"])
+ result = df.to_json(orient="records", lines=True)
+ expected = '{"a\\\\":"foo\\\\","b":"bar"}\n{"a\\\\":"foo\\"","b":"bar"}\n'
+ assert result == expected
+ tm.assert_frame_equal(read_json(StringIO(result), lines=True), df)
+
+
+def test_to_jsonl_count_new_lines():
+ # GH36888
+ df = DataFrame([[1, 2], [1, 2]], columns=["a", "b"])
+ actual_new_lines_count = df.to_json(orient="records", lines=True).count("\n")
+ expected_new_lines_count = 2
+ assert actual_new_lines_count == expected_new_lines_count
+
+
+@pytest.mark.parametrize("chunksize", [1, 1.0])
+@pytest.mark.parametrize("buffer", [BytesIO, StringIO])
+def test_readjson_chunks(request, lines_json_df, chunksize, buffer, engine):
+ # Basic test that read_json(chunks=True) gives the same result as
+ # read_json(chunks=False)
+ # GH17048: memory usage when lines=True
+ # GH#28906: read binary json lines in chunks
+
+ if buffer == BytesIO:
+ lines_json_df = lines_json_df.encode()
+
+ if engine == "pyarrow":
+ # GH 48893
+ reason = (
+ "Pyarrow only supports a file path as an input and line delimited json"
+ "and doesn't support chunksize parameter."
+ )
+ request.applymarker(pytest.mark.xfail(reason=reason, raises=ValueError))
+
+ unchunked = read_json(buffer(lines_json_df), lines=True)
+ with (
+ buffer(lines_json_df) as buf,
+ read_json(buf, lines=True, chunksize=chunksize, engine=engine) as reader,
+ ):
+ chunked = pd.concat(reader)
+
+ tm.assert_frame_equal(chunked, unchunked)
+
+
+def test_readjson_chunksize_requires_lines(lines_json_df, engine):
+ msg = "chunksize can only be passed if lines=True"
+ with pytest.raises(ValueError, match=msg):
+ with read_json(
+ StringIO(lines_json_df), lines=False, chunksize=2, engine=engine
+ ) as _:
+ pass
+
+
+def test_readjson_chunks_series(request, engine):
+ if engine == "pyarrow":
+ # GH 48893
+ reason = (
+ "Pyarrow only supports a file path as an input and line delimited json"
+ "and doesn't support chunksize parameter."
+ )
+ request.applymarker(pytest.mark.xfail(reason=reason))
+
+ # Test reading line-format JSON to Series with chunksize param
+ s = pd.Series({"A": 1, "B": 2})
+
+ strio = StringIO(s.to_json(lines=True, orient="records"))
+ unchunked = read_json(strio, lines=True, typ="series", engine=engine)
+
+ strio = StringIO(s.to_json(lines=True, orient="records"))
+ with read_json(
+ strio, lines=True, typ="series", chunksize=1, engine=engine
+ ) as reader:
+ chunked = pd.concat(reader)
+
+ tm.assert_series_equal(chunked, unchunked)
+
+
+def test_readjson_each_chunk(request, lines_json_df, engine):
+ if engine == "pyarrow":
+ # GH 48893
+ reason = (
+ "Pyarrow only supports a file path as an input and line delimited json"
+ "and doesn't support chunksize parameter."
+ )
+ request.applymarker(pytest.mark.xfail(reason=reason, raises=ValueError))
+
+ # Other tests check that the final result of read_json(chunksize=True)
+ # is correct. This checks the intermediate chunks.
+ with read_json(
+ StringIO(lines_json_df), lines=True, chunksize=2, engine=engine
+ ) as reader:
+ chunks = list(reader)
+ assert chunks[0].shape == (2, 2)
+ assert chunks[1].shape == (1, 2)
+
+
+def test_readjson_chunks_from_file(request, engine, temp_file):
+ if engine == "pyarrow":
+ # GH 48893
+ reason = (
+ "Pyarrow only supports a file path as an input and line delimited json"
+ "and doesn't support chunksize parameter."
+ )
+ request.applymarker(pytest.mark.xfail(reason=reason, raises=ValueError))
+
+ df = DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
+ df.to_json(temp_file, lines=True, orient="records")
+ with read_json(temp_file, lines=True, chunksize=1, engine=engine) as reader:
+ chunked = pd.concat(reader)
+ unchunked = read_json(temp_file, lines=True, engine=engine)
+ tm.assert_frame_equal(unchunked, chunked)
+
+
+@pytest.mark.parametrize("chunksize", [None, 1])
+def test_readjson_chunks_closes(chunksize, temp_file):
+ df = DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
+ df.to_json(temp_file, lines=True, orient="records")
+ reader = JsonReader(
+ temp_file,
+ orient=None,
+ typ="frame",
+ dtype=True,
+ convert_axes=True,
+ convert_dates=True,
+ keep_default_dates=True,
+ precise_float=False,
+ date_unit=None,
+ encoding=None,
+ lines=True,
+ chunksize=chunksize,
+ compression=None,
+ nrows=None,
+ )
+ with reader:
+ reader.read()
+ assert reader.handles.handle.closed, (
+ f"didn't close stream with chunksize = {chunksize}"
+ )
+
+
+@pytest.mark.parametrize("chunksize", [0, -1, 2.2, "foo"])
+def test_readjson_invalid_chunksize(lines_json_df, chunksize, engine):
+ msg = r"'chunksize' must be an integer >=1"
+
+ with pytest.raises(ValueError, match=msg):
+ with read_json(
+ StringIO(lines_json_df), lines=True, chunksize=chunksize, engine=engine
+ ) as _:
+ pass
+
+
+@pytest.mark.parametrize("chunksize", [None, 1, 2])
+def test_readjson_chunks_multiple_empty_lines(chunksize):
+ j = """
+
+ {"A":1,"B":4}
+
+
+
+ {"A":2,"B":5}
+
+
+
+
+
+
+
+ {"A":3,"B":6}
+ """
+ orig = DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
+ test = read_json(StringIO(j), lines=True, chunksize=chunksize)
+ if chunksize is not None:
+ with test:
+ test = pd.concat(test)
+ tm.assert_frame_equal(orig, test, obj=f"chunksize: {chunksize}")
+
+
+def test_readjson_unicode(request, monkeypatch, engine, temp_file):
+ if engine == "pyarrow":
+ # GH 48893
+ reason = (
+ "Pyarrow only supports a file path as an input and line delimited json"
+ "and doesn't support chunksize parameter."
+ )
+ request.applymarker(pytest.mark.xfail(reason=reason, raises=ValueError))
+
+ monkeypatch.setattr("locale.getpreferredencoding", lambda do_setlocale: "cp949")
+ with open(temp_file, "w", encoding="utf-8") as f:
+ f.write('{"£©µÀÆÖÞßéöÿ":["АБВГДабвгд가"]}')
+
+ result = read_json(temp_file, engine=engine)
+ expected = DataFrame({"£©µÀÆÖÞßéöÿ": ["АБВГДабвгд가"]})
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("nrows", [1, 2])
+def test_readjson_nrows(nrows, engine):
+ # GH 33916
+ # Test reading line-format JSON to Series with nrows param
+ jsonl = """{"a": 1, "b": 2}
+ {"a": 3, "b": 4}
+ {"a": 5, "b": 6}
+ {"a": 7, "b": 8}"""
+ result = read_json(StringIO(jsonl), lines=True, nrows=nrows)
+ expected = DataFrame({"a": [1, 3, 5, 7], "b": [2, 4, 6, 8]}).iloc[:nrows]
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("nrows,chunksize", [(2, 2), (4, 2)])
+def test_readjson_nrows_chunks(request, nrows, chunksize, engine):
+ # GH 33916
+ # Test reading line-format JSON to Series with nrows and chunksize param
+ if engine == "pyarrow":
+ # GH 48893
+ reason = (
+ "Pyarrow only supports a file path as an input and line delimited json"
+ "and doesn't support chunksize parameter."
+ )
+ request.applymarker(pytest.mark.xfail(reason=reason, raises=ValueError))
+
+ jsonl = """{"a": 1, "b": 2}
+ {"a": 3, "b": 4}
+ {"a": 5, "b": 6}
+ {"a": 7, "b": 8}"""
+
+ if engine != "pyarrow":
+ with read_json(
+ StringIO(jsonl), lines=True, nrows=nrows, chunksize=chunksize, engine=engine
+ ) as reader:
+ chunked = pd.concat(reader)
+ else:
+ with read_json(
+ jsonl, lines=True, nrows=nrows, chunksize=chunksize, engine=engine
+ ) as reader:
+ chunked = pd.concat(reader)
+ expected = DataFrame({"a": [1, 3, 5, 7], "b": [2, 4, 6, 8]}).iloc[:nrows]
+ tm.assert_frame_equal(chunked, expected)
+
+
+def test_readjson_nrows_requires_lines(engine):
+ # GH 33916
+ # Test ValueError raised if nrows is set without setting lines in read_json
+ jsonl = """{"a": 1, "b": 2}
+ {"a": 3, "b": 4}
+ {"a": 5, "b": 6}
+ {"a": 7, "b": 8}"""
+ msg = "nrows can only be passed if lines=True"
+ with pytest.raises(ValueError, match=msg):
+ read_json(jsonl, lines=False, nrows=2, engine=engine)
+
+
+def test_readjson_lines_chunks_fileurl(request, datapath, engine):
+ # GH 27135
+ # Test reading line-format JSON from file url
+ if engine == "pyarrow":
+ # GH 48893
+ reason = (
+ "Pyarrow only supports a file path as an input and line delimited json"
+ "and doesn't support chunksize parameter."
+ )
+ request.applymarker(pytest.mark.xfail(reason=reason, raises=ValueError))
+
+ df_list_expected = [
+ DataFrame([[1, 2]], columns=["a", "b"], index=[0]),
+ DataFrame([[3, 4]], columns=["a", "b"], index=[1]),
+ DataFrame([[5, 6]], columns=["a", "b"], index=[2]),
+ ]
+ os_path = datapath("io", "json", "data", "line_delimited.json")
+ file_url = Path(os_path).as_uri()
+ with read_json(file_url, lines=True, chunksize=1, engine=engine) as url_reader:
+ for index, chuck in enumerate(url_reader):
+ tm.assert_frame_equal(chuck, df_list_expected[index])
+
+
+def test_chunksize_is_incremental():
+ # See https://github.com/pandas-dev/pandas/issues/34548
+ jsonl = (
+ """{"a": 1, "b": 2}
+ {"a": 3, "b": 4}
+ {"a": 5, "b": 6}
+ {"a": 7, "b": 8}\n"""
+ * 1000
+ )
+
+ class MyReader:
+ def __init__(self, contents) -> None:
+ self.read_count = 0
+ self.stringio = StringIO(contents)
+
+ def read(self, *args):
+ self.read_count += 1
+ return self.stringio.read(*args)
+
+ def __iter__(self) -> Iterator:
+ self.read_count += 1
+ return iter(self.stringio)
+
+ reader = MyReader(jsonl)
+ assert len(list(read_json(reader, lines=True, chunksize=100))) > 1
+ assert reader.read_count > 10
+
+
+@pytest.mark.parametrize("orient_", ["split", "index", "table"])
+def test_to_json_append_orient(orient_):
+ # GH 35849
+ # Test ValueError when orient is not 'records'
+ df = DataFrame({"col1": [1, 2], "col2": ["a", "b"]})
+ msg = (
+ r"mode='a' \(append\) is only supported when "
+ "lines is True and orient is 'records'"
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.to_json(mode="a", orient=orient_)
+
+
+def test_to_json_append_lines():
+ # GH 35849
+ # Test ValueError when lines is not True
+ df = DataFrame({"col1": [1, 2], "col2": ["a", "b"]})
+ msg = (
+ r"mode='a' \(append\) is only supported when "
+ "lines is True and orient is 'records'"
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.to_json(mode="a", lines=False, orient="records")
+
+
+@pytest.mark.parametrize("mode_", ["r", "x"])
+def test_to_json_append_mode(mode_):
+ # GH 35849
+ # Test ValueError when mode is not supported option
+ df = DataFrame({"col1": [1, 2], "col2": ["a", "b"]})
+ msg = (
+ f"mode={mode_} is not a valid option.Only 'w' and 'a' are currently supported."
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.to_json(mode=mode_, lines=False, orient="records")
+
+
+def test_to_json_append_output_consistent_columns(temp_file):
+ # GH 35849
+ # Testing that resulting output reads in as expected.
+ # Testing same columns, new rows
+ df1 = DataFrame({"col1": [1, 2], "col2": ["a", "b"]})
+ df2 = DataFrame({"col1": [3, 4], "col2": ["c", "d"]})
+
+ expected = DataFrame({"col1": [1, 2, 3, 4], "col2": ["a", "b", "c", "d"]})
+ # Save dataframes to the same file
+ df1.to_json(temp_file, lines=True, orient="records")
+ df2.to_json(temp_file, mode="a", lines=True, orient="records")
+
+ # Read path file
+ result = read_json(temp_file, lines=True)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_to_json_append_output_inconsistent_columns(temp_file):
+ # GH 35849
+ # Testing that resulting output reads in as expected.
+ # Testing one new column, one old column, new rows
+ df1 = DataFrame({"col1": [1, 2], "col2": ["a", "b"]})
+ df3 = DataFrame({"col2": ["e", "f"], "col3": ["!", "#"]})
+
+ expected = DataFrame(
+ {
+ "col1": [1, 2, None, None],
+ "col2": ["a", "b", "e", "f"],
+ "col3": [np.nan, np.nan, "!", "#"],
+ }
+ )
+ # Save dataframes to the same file
+ df1.to_json(temp_file, mode="a", lines=True, orient="records")
+ df3.to_json(temp_file, mode="a", lines=True, orient="records")
+
+ # Read path file
+ result = read_json(temp_file, lines=True)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_to_json_append_output_different_columns(temp_file):
+ # GH 35849
+ # Testing that resulting output reads in as expected.
+ # Testing same, differing and new columns
+ df1 = DataFrame({"col1": [1, 2], "col2": ["a", "b"]})
+ df2 = DataFrame({"col1": [3, 4], "col2": ["c", "d"]})
+ df3 = DataFrame({"col2": ["e", "f"], "col3": ["!", "#"]})
+ df4 = DataFrame({"col4": [True, False]})
+
+ expected = DataFrame(
+ {
+ "col1": [1, 2, 3, 4, None, None, None, None],
+ "col2": ["a", "b", "c", "d", "e", "f", np.nan, np.nan],
+ "col3": [np.nan, np.nan, np.nan, np.nan, "!", "#", np.nan, np.nan],
+ "col4": [None, None, None, None, None, None, True, False],
+ }
+ ).astype({"col4": "float"})
+ # Save dataframes to the same file
+ df1.to_json(temp_file, mode="a", lines=True, orient="records")
+ df2.to_json(temp_file, mode="a", lines=True, orient="records")
+ df3.to_json(temp_file, mode="a", lines=True, orient="records")
+ df4.to_json(temp_file, mode="a", lines=True, orient="records")
+
+ # Read path file
+ result = read_json(temp_file, lines=True)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_to_json_append_output_different_columns_reordered(temp_file):
+ # GH 35849
+ # Testing that resulting output reads in as expected.
+ # Testing specific result column order.
+ df1 = DataFrame({"col1": [1, 2], "col2": ["a", "b"]})
+ df2 = DataFrame({"col1": [3, 4], "col2": ["c", "d"]})
+ df3 = DataFrame({"col2": ["e", "f"], "col3": ["!", "#"]})
+ df4 = DataFrame({"col4": [True, False]})
+
+ # df4, df3, df2, df1 (in that order)
+ expected = DataFrame(
+ {
+ "col4": [True, False, None, None, None, None, None, None],
+ "col2": [np.nan, np.nan, "e", "f", "c", "d", "a", "b"],
+ "col3": [np.nan, np.nan, "!", "#", np.nan, np.nan, np.nan, np.nan],
+ "col1": [None, None, None, None, 3, 4, 1, 2],
+ }
+ ).astype({"col4": "float"})
+ # Save dataframes to the same file
+ df4.to_json(temp_file, mode="a", lines=True, orient="records")
+ df3.to_json(temp_file, mode="a", lines=True, orient="records")
+ df2.to_json(temp_file, mode="a", lines=True, orient="records")
+ df1.to_json(temp_file, mode="a", lines=True, orient="records")
+
+ # Read path file
+ result = read_json(temp_file, lines=True)
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_ujson.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_ujson.py
new file mode 100644
index 0000000000000000000000000000000000000000..dd37504d3d811ebd16448ca8c5ecb8ade35682f4
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/json/test_ujson.py
@@ -0,0 +1,1045 @@
+import calendar
+import datetime
+import decimal
+import json
+import locale
+import math
+import re
+import time
+
+import dateutil
+import numpy as np
+import pytest
+
+import pandas._libs.json as ujson
+from pandas.compat import IS64
+
+from pandas import (
+ DataFrame,
+ DatetimeIndex,
+ Index,
+ NaT,
+ PeriodIndex,
+ Series,
+ Timedelta,
+ Timestamp,
+ date_range,
+)
+import pandas._testing as tm
+
+
+def _clean_dict(d):
+ """
+ Sanitize dictionary for JSON by converting all keys to strings.
+
+ Parameters
+ ----------
+ d : dict
+ The dictionary to convert.
+
+ Returns
+ -------
+ cleaned_dict : dict
+ """
+ return {str(k): v for k, v in d.items()}
+
+
+@pytest.fixture(
+ params=[None, "split", "records", "values", "index"] # Column indexed by default.
+)
+def orient(request):
+ return request.param
+
+
+class TestUltraJSONTests:
+ @pytest.mark.skipif(not IS64, reason="not compliant on 32-bit, xref #15865")
+ @pytest.mark.parametrize(
+ "value, double_precision",
+ [
+ ("1337.1337", 15),
+ ("0.95", 1),
+ ("0.94", 1),
+ ("1.95", 1),
+ ("-1.95", 1),
+ ("0.995", 2),
+ ("0.9995", 3),
+ ("0.99999999999999944", 15),
+ ],
+ )
+ def test_encode_decimal(self, value, double_precision):
+ sut = decimal.Decimal(value)
+ encoded = ujson.ujson_dumps(sut, double_precision=double_precision)
+ decoded = ujson.ujson_loads(encoded)
+ assert decoded == value
+
+ @pytest.mark.parametrize("ensure_ascii", [True, False])
+ def test_encode_string_conversion(self, ensure_ascii):
+ string_input = "A string \\ / \b \f \n \r \t &"
+ not_html_encoded = '"A string \\\\ \\/ \\b \\f \\n \\r \\t <\\/script> &"'
+ html_encoded = (
+ '"A string \\\\ \\/ \\b \\f \\n \\r \\t \\u003c\\/script\\u003e \\u0026"'
+ )
+
+ def helper(expected_output, **encode_kwargs):
+ output = ujson.ujson_dumps(
+ string_input, ensure_ascii=ensure_ascii, **encode_kwargs
+ )
+
+ assert output == expected_output
+ assert string_input == json.loads(output)
+ assert string_input == ujson.ujson_loads(output)
+
+ # Default behavior assumes encode_html_chars=False.
+ helper(not_html_encoded)
+
+ # Make sure explicit encode_html_chars=False works.
+ helper(not_html_encoded, encode_html_chars=False)
+
+ # Make sure explicit encode_html_chars=True does the encoding.
+ helper(html_encoded, encode_html_chars=True)
+
+ @pytest.mark.parametrize(
+ "long_number", [-4342969734183514, -12345678901234.56789012, -528656961.4399388]
+ )
+ def test_double_long_numbers(self, long_number):
+ sut = {"a": long_number}
+ encoded = ujson.ujson_dumps(sut, double_precision=15)
+
+ decoded = ujson.ujson_loads(encoded)
+ assert sut == decoded
+
+ def test_encode_non_c_locale(self):
+ lc_category = locale.LC_NUMERIC
+
+ # We just need one of these locales to work.
+ for new_locale in ("it_IT.UTF-8", "Italian_Italy"):
+ if tm.can_set_locale(new_locale, lc_category):
+ with tm.set_locale(new_locale, lc_category):
+ assert ujson.ujson_loads(ujson.ujson_dumps(4.78e60)) == 4.78e60
+ assert ujson.ujson_loads("4.78", precise_float=True) == 4.78
+ break
+
+ def test_decimal_decode_test_precise(self):
+ sut = {"a": 4.56}
+ encoded = ujson.ujson_dumps(sut)
+ decoded = ujson.ujson_loads(encoded, precise_float=True)
+ assert sut == decoded
+
+ def test_encode_double_tiny_exponential(self):
+ num = 1e-40
+ assert num == ujson.ujson_loads(ujson.ujson_dumps(num))
+ num = 1e-100
+ assert num == ujson.ujson_loads(ujson.ujson_dumps(num))
+ num = -1e-45
+ assert num == ujson.ujson_loads(ujson.ujson_dumps(num))
+ num = -1e-145
+ assert np.allclose(num, ujson.ujson_loads(ujson.ujson_dumps(num)))
+
+ @pytest.mark.parametrize("unicode_key", ["key1", "بن"])
+ def test_encode_dict_with_unicode_keys(self, unicode_key):
+ unicode_dict = {unicode_key: "value1"}
+ assert unicode_dict == ujson.ujson_loads(ujson.ujson_dumps(unicode_dict))
+
+ @pytest.mark.parametrize("double_input", [math.pi, -math.pi])
+ def test_encode_double_conversion(self, double_input):
+ output = ujson.ujson_dumps(double_input)
+ assert round(double_input, 5) == round(json.loads(output), 5)
+ assert round(double_input, 5) == round(ujson.ujson_loads(output), 5)
+
+ def test_encode_with_decimal(self):
+ decimal_input = 1.0
+ output = ujson.ujson_dumps(decimal_input)
+
+ assert output == "1.0"
+
+ def test_encode_array_of_nested_arrays(self):
+ nested_input = [[[[]]]] * 20
+ output = ujson.ujson_dumps(nested_input)
+
+ assert nested_input == json.loads(output)
+ assert nested_input == ujson.ujson_loads(output)
+
+ def test_encode_array_of_doubles(self):
+ doubles_input = [31337.31337, 31337.31337, 31337.31337, 31337.31337] * 10
+ output = ujson.ujson_dumps(doubles_input)
+
+ assert doubles_input == json.loads(output)
+ assert doubles_input == ujson.ujson_loads(output)
+
+ def test_double_precision(self):
+ double_input = 30.012345678901234
+ output = ujson.ujson_dumps(double_input, double_precision=15)
+
+ assert double_input == json.loads(output)
+ assert double_input == ujson.ujson_loads(output)
+
+ for double_precision in (3, 9):
+ output = ujson.ujson_dumps(double_input, double_precision=double_precision)
+ rounded_input = round(double_input, double_precision)
+
+ assert rounded_input == json.loads(output)
+ assert rounded_input == ujson.ujson_loads(output)
+
+ @pytest.mark.parametrize(
+ "invalid_val",
+ [
+ 20,
+ -1,
+ "9",
+ None,
+ ],
+ )
+ def test_invalid_double_precision(self, invalid_val):
+ double_input = 30.12345678901234567890
+ expected_exception = ValueError if isinstance(invalid_val, int) else TypeError
+ msg = (
+ r"Invalid value '.*' for option 'double_precision', max is '15'|"
+ r"an integer is required \(got type |"
+ r"object cannot be interpreted as an integer"
+ )
+ with pytest.raises(expected_exception, match=msg):
+ ujson.ujson_dumps(double_input, double_precision=invalid_val)
+
+ def test_encode_string_conversion2(self):
+ string_input = "A string \\ / \b \f \n \r \t"
+ output = ujson.ujson_dumps(string_input)
+
+ assert string_input == json.loads(output)
+ assert string_input == ujson.ujson_loads(output)
+ assert output == '"A string \\\\ \\/ \\b \\f \\n \\r \\t"'
+
+ @pytest.mark.parametrize(
+ "unicode_input",
+ ["Räksmörgås اسامة بن محمد بن عوض بن لادن", "\xe6\x97\xa5\xd1\x88"],
+ )
+ def test_encode_unicode_conversion(self, unicode_input):
+ enc = ujson.ujson_dumps(unicode_input)
+ dec = ujson.ujson_loads(enc)
+
+ assert enc == json.dumps(unicode_input)
+ assert dec == json.loads(enc)
+
+ def test_encode_control_escaping(self):
+ escaped_input = "\x19"
+ enc = ujson.ujson_dumps(escaped_input)
+ dec = ujson.ujson_loads(enc)
+
+ assert escaped_input == dec
+ assert enc == json.dumps(escaped_input)
+
+ def test_encode_unicode_surrogate_pair(self):
+ surrogate_input = "\xf0\x90\x8d\x86"
+ enc = ujson.ujson_dumps(surrogate_input)
+ dec = ujson.ujson_loads(enc)
+
+ assert enc == json.dumps(surrogate_input)
+ assert dec == json.loads(enc)
+
+ def test_encode_unicode_4bytes_utf8(self):
+ four_bytes_input = "\xf0\x91\x80\xb0TRAILINGNORMAL"
+ enc = ujson.ujson_dumps(four_bytes_input)
+ dec = ujson.ujson_loads(enc)
+
+ assert enc == json.dumps(four_bytes_input)
+ assert dec == json.loads(enc)
+
+ def test_encode_unicode_4bytes_utf8highest(self):
+ four_bytes_input = "\xf3\xbf\xbf\xbfTRAILINGNORMAL"
+ enc = ujson.ujson_dumps(four_bytes_input)
+
+ dec = ujson.ujson_loads(enc)
+
+ assert enc == json.dumps(four_bytes_input)
+ assert dec == json.loads(enc)
+
+ def test_encode_unicode_error(self):
+ string = "'\udac0'"
+ msg = (
+ r"'utf-8' codec can't encode character '\\udac0' "
+ r"in position 1: surrogates not allowed"
+ )
+ with pytest.raises(UnicodeEncodeError, match=msg):
+ ujson.ujson_dumps([string])
+
+ def test_encode_array_in_array(self):
+ arr_in_arr_input = [[[[]]]]
+ output = ujson.ujson_dumps(arr_in_arr_input)
+
+ assert arr_in_arr_input == json.loads(output)
+ assert output == json.dumps(arr_in_arr_input)
+ assert arr_in_arr_input == ujson.ujson_loads(output)
+
+ @pytest.mark.parametrize(
+ "num_input",
+ [
+ 31337,
+ -31337, # Negative number.
+ -9223372036854775808, # Large negative number.
+ ],
+ )
+ def test_encode_num_conversion(self, num_input):
+ output = ujson.ujson_dumps(num_input)
+ assert num_input == json.loads(output)
+ assert output == json.dumps(num_input)
+ assert num_input == ujson.ujson_loads(output)
+
+ def test_encode_list_conversion(self):
+ list_input = [1, 2, 3, 4]
+ output = ujson.ujson_dumps(list_input)
+
+ assert list_input == json.loads(output)
+ assert list_input == ujson.ujson_loads(output)
+
+ def test_encode_dict_conversion(self):
+ dict_input = {"k1": 1, "k2": 2, "k3": 3, "k4": 4}
+ output = ujson.ujson_dumps(dict_input)
+
+ assert dict_input == json.loads(output)
+ assert dict_input == ujson.ujson_loads(output)
+
+ @pytest.mark.parametrize("builtin_value", [None, True, False])
+ def test_encode_builtin_values_conversion(self, builtin_value):
+ output = ujson.ujson_dumps(builtin_value)
+ assert builtin_value == json.loads(output)
+ assert output == json.dumps(builtin_value)
+ assert builtin_value == ujson.ujson_loads(output)
+
+ def test_encode_datetime_conversion(self):
+ datetime_input = datetime.datetime.fromtimestamp(time.time())
+ output = ujson.ujson_dumps(datetime_input, date_unit="s")
+ expected = calendar.timegm(datetime_input.utctimetuple())
+
+ assert int(expected) == json.loads(output)
+ assert int(expected) == ujson.ujson_loads(output)
+
+ def test_encode_date_conversion(self):
+ date_input = datetime.date.fromtimestamp(time.time())
+ output = ujson.ujson_dumps(date_input, date_unit="s")
+
+ tup = (date_input.year, date_input.month, date_input.day, 0, 0, 0)
+ expected = calendar.timegm(tup)
+
+ assert int(expected) == json.loads(output)
+ assert int(expected) == ujson.ujson_loads(output)
+
+ @pytest.mark.parametrize(
+ "test",
+ [datetime.time(), datetime.time(1, 2, 3), datetime.time(10, 12, 15, 343243)],
+ )
+ def test_encode_time_conversion_basic(self, test):
+ output = ujson.ujson_dumps(test)
+ expected = f'"{test.isoformat()}"'
+ assert expected == output
+
+ def test_encode_time_conversion_pytz(self):
+ # see gh-11473: to_json segfaults with timezone-aware datetimes
+ pytz = pytest.importorskip("pytz")
+ test = datetime.time(10, 12, 15, 343243, pytz.utc)
+ output = ujson.ujson_dumps(test)
+ expected = f'"{test.isoformat()}"'
+ assert expected == output
+
+ def test_encode_time_conversion_dateutil(self):
+ # see gh-11473: to_json segfaults with timezone-aware datetimes
+ test = datetime.time(10, 12, 15, 343243, dateutil.tz.tzutc())
+ output = ujson.ujson_dumps(test)
+ expected = f'"{test.isoformat()}"'
+ assert expected == output
+
+ @pytest.mark.parametrize(
+ "decoded_input", [NaT, np.datetime64("NaT", "ns"), np.nan, np.inf, -np.inf]
+ )
+ def test_encode_as_null(self, decoded_input):
+ assert ujson.ujson_dumps(decoded_input) == "null", "Expected null"
+
+ def test_datetime_units(self):
+ val = datetime.datetime(2013, 8, 17, 21, 17, 12, 215504)
+ stamp = Timestamp(val).as_unit("ns")
+
+ roundtrip = ujson.ujson_loads(ujson.ujson_dumps(val, date_unit="s"))
+ assert roundtrip == stamp._value // 10**9
+
+ roundtrip = ujson.ujson_loads(ujson.ujson_dumps(val, date_unit="ms"))
+ assert roundtrip == stamp._value // 10**6
+
+ roundtrip = ujson.ujson_loads(ujson.ujson_dumps(val, date_unit="us"))
+ assert roundtrip == stamp._value // 10**3
+
+ roundtrip = ujson.ujson_loads(ujson.ujson_dumps(val, date_unit="ns"))
+ assert roundtrip == stamp._value
+
+ msg = "Invalid value 'foo' for option 'date_unit'"
+ with pytest.raises(ValueError, match=msg):
+ ujson.ujson_dumps(val, date_unit="foo")
+
+ def test_encode_to_utf8(self):
+ unencoded = "\xe6\x97\xa5\xd1\x88"
+
+ enc = ujson.ujson_dumps(unencoded, ensure_ascii=False)
+ dec = ujson.ujson_loads(enc)
+
+ assert enc == json.dumps(unencoded, ensure_ascii=False)
+ assert dec == json.loads(enc)
+
+ def test_decode_from_unicode(self):
+ unicode_input = '{"obj": 31337}'
+
+ dec1 = ujson.ujson_loads(unicode_input)
+ dec2 = ujson.ujson_loads(str(unicode_input))
+
+ assert dec1 == dec2
+
+ def test_encode_recursion_max(self):
+ # 8 is the max recursion depth
+
+ class O2:
+ member = 0
+
+ class O1:
+ member = 0
+
+ decoded_input = O1()
+ decoded_input.member = O2()
+ decoded_input.member.member = decoded_input
+
+ with pytest.raises(OverflowError, match="Maximum recursion level reached"):
+ ujson.ujson_dumps(decoded_input)
+
+ def test_decode_jibberish(self):
+ jibberish = "fdsa sda v9sa fdsa"
+ msg = "Unexpected character found when decoding 'false'"
+ with pytest.raises(ValueError, match=msg):
+ ujson.ujson_loads(jibberish)
+
+ @pytest.mark.parametrize(
+ "broken_json",
+ [
+ "[", # Broken array start.
+ "{", # Broken object start.
+ "]", # Broken array end.
+ "}", # Broken object end.
+ ],
+ )
+ def test_decode_broken_json(self, broken_json):
+ msg = "Expected object or value"
+ with pytest.raises(ValueError, match=msg):
+ ujson.ujson_loads(broken_json)
+
+ @pytest.mark.parametrize("too_big_char", ["[", "{"])
+ def test_decode_depth_too_big(self, too_big_char):
+ with pytest.raises(ValueError, match="Reached object decoding depth limit"):
+ ujson.ujson_loads(too_big_char * (1024 * 1024))
+
+ @pytest.mark.parametrize(
+ "bad_string",
+ [
+ '"TESTING', # Unterminated.
+ '"TESTING\\"', # Unterminated escape.
+ "tru", # Broken True.
+ "fa", # Broken False.
+ "n", # Broken None.
+ ],
+ )
+ def test_decode_bad_string(self, bad_string):
+ msg = (
+ "Unexpected character found when decoding|"
+ "Unmatched ''\"' when when decoding 'string'"
+ )
+ with pytest.raises(ValueError, match=msg):
+ ujson.ujson_loads(bad_string)
+
+ @pytest.mark.parametrize(
+ "broken_json, err_msg",
+ [
+ (
+ '{{1337:""}}',
+ "Key name of object must be 'string' when decoding 'object'",
+ ),
+ ('{{"key":"}', "Unmatched ''\"' when when decoding 'string'"),
+ ("[[[true", "Unexpected character found when decoding array value (2)"),
+ ],
+ )
+ def test_decode_broken_json_leak(self, broken_json, err_msg):
+ for _ in range(1000):
+ with pytest.raises(ValueError, match=re.escape(err_msg)):
+ ujson.ujson_loads(broken_json)
+
+ @pytest.mark.parametrize(
+ "invalid_dict",
+ [
+ "{{{{31337}}}}", # No key.
+ '{{{{"key":}}}}', # No value.
+ '{{{{"key"}}}}', # No colon or value.
+ ],
+ )
+ def test_decode_invalid_dict(self, invalid_dict):
+ msg = (
+ "Key name of object must be 'string' when decoding 'object'|"
+ "No ':' found when decoding object value|"
+ "Expected object or value"
+ )
+ with pytest.raises(ValueError, match=msg):
+ ujson.ujson_loads(invalid_dict)
+
+ @pytest.mark.parametrize("numeric_int_as_str", ["31337", "-31337"])
+ def test_decode_numeric_int(self, numeric_int_as_str):
+ assert int(numeric_int_as_str) == ujson.ujson_loads(numeric_int_as_str)
+
+ def test_encode_null_character(self):
+ wrapped_input = "31337 \x00 1337"
+ output = ujson.ujson_dumps(wrapped_input)
+
+ assert wrapped_input == json.loads(output)
+ assert output == json.dumps(wrapped_input)
+ assert wrapped_input == ujson.ujson_loads(output)
+
+ alone_input = "\x00"
+ output = ujson.ujson_dumps(alone_input)
+
+ assert alone_input == json.loads(output)
+ assert output == json.dumps(alone_input)
+ assert alone_input == ujson.ujson_loads(output)
+ assert '" \\u0000\\r\\n "' == ujson.ujson_dumps(" \u0000\r\n ")
+
+ def test_decode_null_character(self):
+ wrapped_input = '"31337 \\u0000 31337"'
+ assert ujson.ujson_loads(wrapped_input) == json.loads(wrapped_input)
+
+ def test_encode_list_long_conversion(self):
+ long_input = [
+ 9223372036854775807,
+ 9223372036854775807,
+ 9223372036854775807,
+ 9223372036854775807,
+ 9223372036854775807,
+ 9223372036854775807,
+ ]
+ output = ujson.ujson_dumps(long_input)
+
+ assert long_input == json.loads(output)
+ assert long_input == ujson.ujson_loads(output)
+
+ @pytest.mark.parametrize("long_input", [9223372036854775807, 18446744073709551615])
+ def test_encode_long_conversion(self, long_input):
+ output = ujson.ujson_dumps(long_input)
+
+ assert long_input == json.loads(output)
+ assert output == json.dumps(long_input)
+ assert long_input == ujson.ujson_loads(output)
+
+ @pytest.mark.parametrize("bigNum", [2**64, -(2**63) - 1])
+ def test_dumps_ints_larger_than_maxsize(self, bigNum):
+ encoding = ujson.ujson_dumps(bigNum)
+ assert str(bigNum) == encoding
+
+ with pytest.raises(
+ ValueError,
+ match="Value is too big|Value is too small",
+ ):
+ assert ujson.ujson_loads(encoding) == bigNum
+
+ @pytest.mark.parametrize(
+ "int_exp", ["1337E40", "1.337E40", "1337E+9", "1.337e+40", "1.337E-4"]
+ )
+ def test_decode_numeric_int_exp(self, int_exp):
+ assert ujson.ujson_loads(int_exp) == json.loads(int_exp)
+
+ def test_loads_non_str_bytes_raises(self):
+ msg = "a bytes-like object is required, not 'NoneType'"
+ with pytest.raises(TypeError, match=msg):
+ ujson.ujson_loads(None)
+
+ @pytest.mark.parametrize("val", [3590016419, 2**31, 2**32, (2**32) - 1])
+ def test_decode_number_with_32bit_sign_bit(self, val):
+ # Test that numbers that fit within 32 bits but would have the
+ # sign bit set (2**31 <= x < 2**32) are decoded properly.
+ doc = f'{{"id": {val}}}'
+ assert ujson.ujson_loads(doc)["id"] == val
+
+ def test_encode_big_escape(self):
+ # Make sure no Exception is raised.
+ for _ in range(10):
+ base = "\u00e5".encode()
+ escape_input = base * 1024 * 1024 * 2
+ ujson.ujson_dumps(escape_input)
+
+ def test_decode_big_escape(self):
+ # Make sure no Exception is raised.
+ for _ in range(10):
+ base = "\u00e5".encode()
+ quote = b'"'
+
+ escape_input = quote + (base * 1024 * 1024 * 2) + quote
+ ujson.ujson_loads(escape_input)
+
+ def test_to_dict(self):
+ d = {"key": 31337}
+
+ class DictTest:
+ def toDict(self):
+ return d
+
+ o = DictTest()
+ output = ujson.ujson_dumps(o)
+
+ dec = ujson.ujson_loads(output)
+ assert dec == d
+
+ def test_default_handler(self):
+ class _TestObject:
+ def __init__(self, val) -> None:
+ self.val = val
+
+ @property
+ def recursive_attr(self):
+ return _TestObject("recursive_attr")
+
+ def __str__(self) -> str:
+ return str(self.val)
+
+ msg = "Maximum recursion level reached"
+ with pytest.raises(OverflowError, match=msg):
+ ujson.ujson_dumps(_TestObject("foo"))
+ assert '"foo"' == ujson.ujson_dumps(_TestObject("foo"), default_handler=str)
+
+ def my_handler(_):
+ return "foobar"
+
+ assert '"foobar"' == ujson.ujson_dumps(
+ _TestObject("foo"), default_handler=my_handler
+ )
+
+ def my_handler_raises(_):
+ raise TypeError("I raise for anything")
+
+ with pytest.raises(TypeError, match="I raise for anything"):
+ ujson.ujson_dumps(_TestObject("foo"), default_handler=my_handler_raises)
+
+ def my_int_handler(_):
+ return 42
+
+ assert (
+ ujson.ujson_loads(
+ ujson.ujson_dumps(_TestObject("foo"), default_handler=my_int_handler)
+ )
+ == 42
+ )
+
+ def my_obj_handler(_):
+ return datetime.datetime(2013, 2, 3)
+
+ assert ujson.ujson_loads(
+ ujson.ujson_dumps(datetime.datetime(2013, 2, 3))
+ ) == ujson.ujson_loads(
+ ujson.ujson_dumps(_TestObject("foo"), default_handler=my_obj_handler)
+ )
+
+ obj_list = [_TestObject("foo"), _TestObject("bar")]
+ assert json.loads(json.dumps(obj_list, default=str)) == ujson.ujson_loads(
+ ujson.ujson_dumps(obj_list, default_handler=str)
+ )
+
+ def test_encode_object(self):
+ class _TestObject:
+ def __init__(self, a, b, _c, d) -> None:
+ self.a = a
+ self.b = b
+ self._c = _c
+ self.d = d
+
+ def e(self):
+ return 5
+
+ # JSON keys should be all non-callable non-underscore attributes, see GH-42768
+ test_object = _TestObject(a=1, b=2, _c=3, d=4)
+ assert ujson.ujson_loads(ujson.ujson_dumps(test_object)) == {
+ "a": 1,
+ "b": 2,
+ "d": 4,
+ }
+
+ def test_ujson__name__(self):
+ # GH 52898
+ assert ujson.__name__ == "pandas._libs.json"
+
+
+class TestNumpyJSONTests:
+ @pytest.mark.parametrize("bool_input", [True, False])
+ def test_bool(self, bool_input):
+ b = bool(bool_input)
+ assert ujson.ujson_loads(ujson.ujson_dumps(b)) == b
+
+ def test_bool_array(self):
+ bool_array = np.array(
+ [True, False, True, True, False, True, False, False], dtype=bool
+ )
+ output = np.array(ujson.ujson_loads(ujson.ujson_dumps(bool_array)), dtype=bool)
+ tm.assert_numpy_array_equal(bool_array, output)
+
+ def test_int(self, any_int_numpy_dtype):
+ klass = np.dtype(any_int_numpy_dtype).type
+ num = klass(1)
+
+ assert klass(ujson.ujson_loads(ujson.ujson_dumps(num))) == num
+
+ def test_int_array(self, any_int_numpy_dtype):
+ arr = np.arange(100, dtype=int)
+ arr_input = arr.astype(any_int_numpy_dtype)
+
+ arr_output = np.array(
+ ujson.ujson_loads(ujson.ujson_dumps(arr_input)), dtype=any_int_numpy_dtype
+ )
+ tm.assert_numpy_array_equal(arr_input, arr_output)
+
+ def test_int_max(self, any_int_numpy_dtype):
+ if any_int_numpy_dtype in ("int64", "uint64") and not IS64:
+ pytest.skip("Cannot test 64-bit integer on 32-bit platform")
+
+ klass = np.dtype(any_int_numpy_dtype).type
+
+ # uint64 max will always overflow,
+ # as it's encoded to signed.
+ if any_int_numpy_dtype == "uint64":
+ num = np.iinfo("int64").max
+ else:
+ num = np.iinfo(any_int_numpy_dtype).max
+
+ assert klass(ujson.ujson_loads(ujson.ujson_dumps(num))) == num
+
+ def test_float(self, float_numpy_dtype):
+ klass = np.dtype(float_numpy_dtype).type
+ num = klass(256.2013)
+
+ assert klass(ujson.ujson_loads(ujson.ujson_dumps(num))) == num
+
+ def test_float_array(self, float_numpy_dtype):
+ arr = np.arange(12.5, 185.72, 1.7322, dtype=float)
+ float_input = arr.astype(float_numpy_dtype)
+
+ float_output = np.array(
+ ujson.ujson_loads(ujson.ujson_dumps(float_input, double_precision=15)),
+ dtype=float_numpy_dtype,
+ )
+ tm.assert_almost_equal(float_input, float_output)
+
+ def test_float_max(self, float_numpy_dtype):
+ klass = np.dtype(float_numpy_dtype).type
+ num = klass(np.finfo(float_numpy_dtype).max / 10)
+
+ tm.assert_almost_equal(
+ klass(ujson.ujson_loads(ujson.ujson_dumps(num, double_precision=15))), num
+ )
+
+ def test_array_basic(self):
+ arr = np.arange(96)
+ arr = arr.reshape((2, 2, 2, 2, 3, 2))
+
+ tm.assert_numpy_array_equal(
+ np.array(ujson.ujson_loads(ujson.ujson_dumps(arr))), arr
+ )
+
+ @pytest.mark.parametrize("shape", [(10, 10), (5, 5, 4), (100, 1)])
+ def test_array_reshaped(self, shape):
+ arr = np.arange(100)
+ arr = arr.reshape(shape)
+
+ tm.assert_numpy_array_equal(
+ np.array(ujson.ujson_loads(ujson.ujson_dumps(arr))), arr
+ )
+
+ def test_array_list(self):
+ arr_list = [
+ "a",
+ [],
+ {},
+ {},
+ [],
+ 42,
+ 97.8,
+ ["a", "b"],
+ {"key": "val"},
+ ]
+ arr = np.array(arr_list, dtype=object)
+ result = np.array(ujson.ujson_loads(ujson.ujson_dumps(arr)), dtype=object)
+ tm.assert_numpy_array_equal(result, arr)
+
+ def test_array_float(self):
+ dtype = np.float32
+
+ arr = np.arange(100.202, 200.202, 1, dtype=dtype)
+ arr = arr.reshape((5, 5, 4))
+
+ arr_out = np.array(ujson.ujson_loads(ujson.ujson_dumps(arr)), dtype=dtype)
+ tm.assert_almost_equal(arr, arr_out)
+
+ def test_0d_array(self):
+ # gh-18878
+ msg = re.escape(
+ "array(1) (numpy-scalar) is not JSON serializable at the moment"
+ )
+ with pytest.raises(TypeError, match=msg):
+ ujson.ujson_dumps(np.array(1))
+
+ def test_array_long_double(self):
+ msg = re.compile(
+ "1234.5.* \\(numpy-scalar\\) is not JSON serializable at the moment"
+ )
+ with pytest.raises(TypeError, match=msg):
+ ujson.ujson_dumps(np.longdouble(1234.5))
+
+
+class TestPandasJSONTests:
+ def test_dataframe(self, orient):
+ dtype = np.int64
+
+ df = DataFrame(
+ [[1, 2, 3], [4, 5, 6]],
+ index=["a", "b"],
+ columns=["x", "y", "z"],
+ dtype=dtype,
+ )
+ encode_kwargs = {} if orient is None else {"orient": orient}
+ assert (df.dtypes == dtype).all()
+
+ output = ujson.ujson_loads(ujson.ujson_dumps(df, **encode_kwargs))
+ assert (df.dtypes == dtype).all()
+
+ # Ensure proper DataFrame initialization.
+ if orient == "split":
+ dec = _clean_dict(output)
+ output = DataFrame(**dec)
+ else:
+ output = DataFrame(output)
+
+ # Corrections to enable DataFrame comparison.
+ if orient == "values":
+ df.columns = [0, 1, 2]
+ df.index = [0, 1]
+ elif orient == "records":
+ df.index = [0, 1]
+ elif orient == "index":
+ df = df.transpose()
+
+ assert (df.dtypes == dtype).all()
+ tm.assert_frame_equal(output, df)
+
+ def test_dataframe_nested(self, orient):
+ df = DataFrame(
+ [[1, 2, 3], [4, 5, 6]], index=["a", "b"], columns=["x", "y", "z"]
+ )
+
+ nested = {"df1": df, "df2": df.copy()}
+ kwargs = {} if orient is None else {"orient": orient}
+
+ exp = {
+ "df1": ujson.ujson_loads(ujson.ujson_dumps(df, **kwargs)),
+ "df2": ujson.ujson_loads(ujson.ujson_dumps(df, **kwargs)),
+ }
+ assert ujson.ujson_loads(ujson.ujson_dumps(nested, **kwargs)) == exp
+
+ def test_series(self, orient):
+ dtype = np.int64
+ s = Series(
+ [10, 20, 30, 40, 50, 60],
+ name="series",
+ index=[6, 7, 8, 9, 10, 15],
+ dtype=dtype,
+ ).sort_values()
+ assert s.dtype == dtype
+
+ encode_kwargs = {} if orient is None else {"orient": orient}
+
+ output = ujson.ujson_loads(ujson.ujson_dumps(s, **encode_kwargs))
+ assert s.dtype == dtype
+
+ if orient == "split":
+ dec = _clean_dict(output)
+ output = Series(**dec)
+ else:
+ output = Series(output)
+
+ if orient in (None, "index"):
+ s.name = None
+ output = output.sort_values()
+ s.index = ["6", "7", "8", "9", "10", "15"]
+ elif orient in ("records", "values"):
+ s.name = None
+ s.index = [0, 1, 2, 3, 4, 5]
+
+ assert s.dtype == dtype
+ tm.assert_series_equal(output, s)
+
+ def test_series_nested(self, orient):
+ s = Series(
+ [10, 20, 30, 40, 50, 60], name="series", index=[6, 7, 8, 9, 10, 15]
+ ).sort_values()
+ nested = {"s1": s, "s2": s.copy()}
+ kwargs = {} if orient is None else {"orient": orient}
+
+ exp = {
+ "s1": ujson.ujson_loads(ujson.ujson_dumps(s, **kwargs)),
+ "s2": ujson.ujson_loads(ujson.ujson_dumps(s, **kwargs)),
+ }
+ assert ujson.ujson_loads(ujson.ujson_dumps(nested, **kwargs)) == exp
+
+ def test_index(self):
+ i = Index([23, 45, 18, 98, 43, 11], name="index")
+
+ # Column indexed.
+ output = Index(ujson.ujson_loads(ujson.ujson_dumps(i)), name="index")
+ tm.assert_index_equal(i, output)
+
+ dec = _clean_dict(ujson.ujson_loads(ujson.ujson_dumps(i, orient="split")))
+ output = Index(**dec)
+
+ tm.assert_index_equal(i, output)
+ assert i.name == output.name
+
+ tm.assert_index_equal(i, output)
+ assert i.name == output.name
+
+ output = Index(
+ ujson.ujson_loads(ujson.ujson_dumps(i, orient="values")), name="index"
+ )
+ tm.assert_index_equal(i, output)
+
+ output = Index(
+ ujson.ujson_loads(ujson.ujson_dumps(i, orient="records")), name="index"
+ )
+ tm.assert_index_equal(i, output)
+
+ output = Index(
+ ujson.ujson_loads(ujson.ujson_dumps(i, orient="index")), name="index"
+ )
+ tm.assert_index_equal(i, output)
+
+ def test_datetime_index(self):
+ date_unit = "ns"
+
+ # freq doesn't round-trip
+ rng = DatetimeIndex(
+ list(date_range("1/1/2000", periods=20, unit="ns")), freq=None
+ )
+ encoded = ujson.ujson_dumps(rng, date_unit=date_unit)
+
+ decoded = DatetimeIndex(np.array(ujson.ujson_loads(encoded)))
+ tm.assert_index_equal(rng, decoded)
+
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+ decoded = Series(ujson.ujson_loads(ujson.ujson_dumps(ts, date_unit=date_unit)))
+
+ idx_values = decoded.index.values.astype(np.int64)
+ decoded.index = DatetimeIndex(idx_values)
+ tm.assert_series_equal(ts, decoded)
+
+ @pytest.mark.parametrize(
+ "invalid_arr",
+ [
+ "[31337,]", # Trailing comma.
+ "[,31337]", # Leading comma.
+ "[]]", # Unmatched bracket.
+ "[,]", # Only comma.
+ ],
+ )
+ def test_decode_invalid_array(self, invalid_arr):
+ msg = (
+ "Expected object or value|Trailing data|"
+ "Unexpected character found when decoding array value"
+ )
+ with pytest.raises(ValueError, match=msg):
+ ujson.ujson_loads(invalid_arr)
+
+ @pytest.mark.parametrize("arr", [[], [31337]])
+ def test_decode_array(self, arr):
+ assert arr == ujson.ujson_loads(str(arr))
+
+ @pytest.mark.parametrize("extreme_num", [9223372036854775807, -9223372036854775808])
+ def test_decode_extreme_numbers(self, extreme_num):
+ assert extreme_num == ujson.ujson_loads(str(extreme_num))
+
+ @pytest.mark.parametrize("too_extreme_num", [f"{2**64}", f"{-(2**63) - 1}"])
+ def test_decode_too_extreme_numbers(self, too_extreme_num):
+ with pytest.raises(
+ ValueError,
+ match="Value is too big|Value is too small",
+ ):
+ ujson.ujson_loads(too_extreme_num)
+
+ def test_decode_with_trailing_whitespaces(self):
+ assert {} == ujson.ujson_loads("{}\n\t ")
+
+ def test_decode_with_trailing_non_whitespaces(self):
+ with pytest.raises(ValueError, match="Trailing data"):
+ ujson.ujson_loads("{}\n\t a")
+
+ @pytest.mark.parametrize("value", [f"{2**64}", f"{-(2**63) - 1}"])
+ def test_decode_array_with_big_int(self, value):
+ with pytest.raises(
+ ValueError,
+ match="Value is too big|Value is too small",
+ ):
+ ujson.ujson_loads(value)
+
+ @pytest.mark.parametrize(
+ "float_number",
+ [
+ 1.1234567893,
+ 1.234567893,
+ 1.34567893,
+ 1.4567893,
+ 1.567893,
+ 1.67893,
+ 1.7893,
+ 1.893,
+ 1.3,
+ ],
+ )
+ @pytest.mark.parametrize("sign", [-1, 1])
+ def test_decode_floating_point(self, sign, float_number):
+ float_number *= sign
+ tm.assert_almost_equal(
+ float_number, ujson.ujson_loads(str(float_number)), rtol=1e-15
+ )
+
+ def test_encode_big_set(self):
+ s = set(range(100000))
+ # Make sure no Exception is raised.
+ ujson.ujson_dumps(s)
+
+ def test_encode_empty_set(self):
+ assert "[]" == ujson.ujson_dumps(set())
+
+ def test_encode_set(self):
+ s = {1, 2, 3, 4, 5, 6, 7, 8, 9}
+ enc = ujson.ujson_dumps(s)
+ dec = ujson.ujson_loads(enc)
+
+ for v in dec:
+ assert v in s
+
+ @pytest.mark.parametrize(
+ "td",
+ [
+ Timedelta(days=366),
+ Timedelta(days=-1),
+ Timedelta(hours=13, minutes=5, seconds=5),
+ Timedelta(hours=13, minutes=20, seconds=30),
+ Timedelta(days=-1, nanoseconds=5),
+ Timedelta(nanoseconds=1),
+ Timedelta(microseconds=1, nanoseconds=1),
+ Timedelta(milliseconds=1, microseconds=1, nanoseconds=1),
+ Timedelta(milliseconds=999, microseconds=999, nanoseconds=999),
+ ],
+ )
+ def test_encode_timedelta_iso(self, td):
+ # GH 28256
+ result = ujson.ujson_dumps(td, iso_dates=True)
+ expected = f'"{td.isoformat()}"'
+
+ assert result == expected
+
+ def test_encode_periodindex(self):
+ # GH 46683
+ p = PeriodIndex(["2022-04-06", "2022-04-07"], freq="D")
+ df = DataFrame(index=p)
+ assert df.to_json() == "{}"
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_chunksize.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_chunksize.py
new file mode 100644
index 0000000000000000000000000000000000000000..98dbb7d45e9112c29263c6477e0d19b24be38959
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_chunksize.py
@@ -0,0 +1,364 @@
+"""
+Tests that work on both the Python and C engines but do not have a
+specific classification into the other test modules.
+"""
+
+from io import StringIO
+
+import numpy as np
+import pytest
+
+from pandas._libs import parsers as libparsers
+from pandas.errors import DtypeWarning
+
+from pandas import (
+ DataFrame,
+ concat,
+)
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+@pytest.mark.parametrize("index_col", [0, "index"])
+def test_read_chunksize_with_index(all_parsers, index_col):
+ parser = all_parsers
+ data = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+"""
+
+ expected = DataFrame(
+ [
+ ["foo", 2, 3, 4, 5],
+ ["bar", 7, 8, 9, 10],
+ ["baz", 12, 13, 14, 15],
+ ["qux", 12, 13, 14, 15],
+ ["foo2", 12, 13, 14, 15],
+ ["bar2", 12, 13, 14, 15],
+ ],
+ columns=["index", "A", "B", "C", "D"],
+ )
+ expected = expected.set_index("index")
+
+ if parser.engine == "pyarrow":
+ msg = "The 'chunksize' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ with parser.read_csv(StringIO(data), index_col=0, chunksize=2) as reader:
+ list(reader)
+ return
+
+ with parser.read_csv(StringIO(data), index_col=0, chunksize=2) as reader:
+ chunks = list(reader)
+ tm.assert_frame_equal(chunks[0], expected[:2])
+ tm.assert_frame_equal(chunks[1], expected[2:4])
+ tm.assert_frame_equal(chunks[2], expected[4:])
+
+
+@pytest.mark.parametrize("chunksize", [1.3, "foo", 0])
+def test_read_chunksize_bad(all_parsers, chunksize):
+ data = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+"""
+ parser = all_parsers
+ msg = r"'chunksize' must be an integer >=1"
+ if parser.engine == "pyarrow":
+ msg = "The 'chunksize' option is not supported with the 'pyarrow' engine"
+
+ with pytest.raises(ValueError, match=msg):
+ with parser.read_csv(StringIO(data), chunksize=chunksize) as _:
+ pass
+
+
+@pytest.mark.parametrize("chunksize", [2, 8])
+def test_read_chunksize_and_nrows(all_parsers, chunksize):
+ # see gh-15755
+ data = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+"""
+ parser = all_parsers
+ kwargs = {"index_col": 0, "nrows": 5}
+
+ if parser.engine == "pyarrow":
+ msg = "The 'nrows' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), **kwargs)
+ return
+
+ expected = parser.read_csv(StringIO(data), **kwargs)
+ with parser.read_csv(StringIO(data), chunksize=chunksize, **kwargs) as reader:
+ tm.assert_frame_equal(concat(reader), expected)
+
+
+def test_read_chunksize_and_nrows_changing_size(all_parsers):
+ data = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+"""
+ parser = all_parsers
+ kwargs = {"index_col": 0, "nrows": 5}
+
+ if parser.engine == "pyarrow":
+ msg = "The 'nrows' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), **kwargs)
+ return
+
+ expected = parser.read_csv(StringIO(data), **kwargs)
+ with parser.read_csv(StringIO(data), chunksize=8, **kwargs) as reader:
+ tm.assert_frame_equal(reader.get_chunk(size=2), expected.iloc[:2])
+ tm.assert_frame_equal(reader.get_chunk(size=4), expected.iloc[2:5])
+
+ with pytest.raises(StopIteration, match="^$"):
+ reader.get_chunk(size=3)
+
+
+def test_get_chunk_passed_chunksize(all_parsers):
+ parser = all_parsers
+ data = """A,B,C
+1,2,3
+4,5,6
+7,8,9
+1,2,3"""
+
+ if parser.engine == "pyarrow":
+ msg = "The 'chunksize' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ with parser.read_csv(StringIO(data), chunksize=2) as reader:
+ reader.get_chunk()
+ return
+
+ with parser.read_csv(StringIO(data), chunksize=2) as reader:
+ result = reader.get_chunk()
+
+ expected = DataFrame([[1, 2, 3], [4, 5, 6]], columns=["A", "B", "C"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("kwargs", [{}, {"index_col": 0}])
+def test_read_chunksize_compat(all_parsers, kwargs):
+ # see gh-12185
+ data = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+"""
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), **kwargs)
+
+ if parser.engine == "pyarrow":
+ msg = "The 'chunksize' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ with parser.read_csv(StringIO(data), chunksize=2, **kwargs) as reader:
+ concat(reader)
+ return
+
+ with parser.read_csv(StringIO(data), chunksize=2, **kwargs) as reader:
+ via_reader = concat(reader)
+ tm.assert_frame_equal(via_reader, result)
+
+
+def test_read_chunksize_jagged_names(all_parsers):
+ # see gh-23509
+ parser = all_parsers
+ data = "\n".join(["0"] * 7 + [",".join(["0"] * 10)])
+
+ expected = DataFrame([[0] + [np.nan] * 9] * 7 + [[0] * 10])
+
+ if parser.engine == "pyarrow":
+ msg = "The 'chunksize' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ with parser.read_csv(
+ StringIO(data), names=range(10), chunksize=4
+ ) as reader:
+ concat(reader)
+ return
+
+ with parser.read_csv(StringIO(data), names=range(10), chunksize=4) as reader:
+ result = concat(reader)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_chunk_begins_with_newline_whitespace(all_parsers):
+ # see gh-10022
+ parser = all_parsers
+ data = "\n hello\nworld\n"
+
+ result = parser.read_csv(StringIO(data), header=None)
+ expected = DataFrame([" hello", "world"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.slow
+def test_chunks_have_consistent_numerical_type(all_parsers, monkeypatch):
+ # mainly an issue with the C parser
+ heuristic = 2**3
+ parser = all_parsers
+ integers = [str(i) for i in range(heuristic - 1)]
+ data = "a\n" + "\n".join([*integers, "1.0", "2.0", *integers])
+
+ # Coercions should work without warnings.
+ with monkeypatch.context() as m:
+ m.setattr(libparsers, "DEFAULT_BUFFER_HEURISTIC", heuristic)
+ result = parser.read_csv(StringIO(data))
+
+ assert type(result.a[0]) is np.float64
+ assert result.a.dtype == float
+
+
+def test_warn_if_chunks_have_mismatched_type(
+ all_parsers, using_infer_string, monkeypatch
+):
+ warning_type = None
+ parser = all_parsers
+ heuristic = 2**3
+ size = 10
+
+ # see gh-3866: if chunks are different types and can't
+ # be coerced using numerical types, then issue warning.
+ if parser.engine == "c" and parser.low_memory:
+ warning_type = DtypeWarning
+ # Use a size to hit warning path dictated by DEFAULT_BUFFER_HEURISTIC
+ # monkeypatched below
+ size = heuristic - 1
+
+ integers = [str(i) for i in range(size)]
+ data = "a\n" + "\n".join([*integers, "a", "b", *integers])
+
+ buf = StringIO(data)
+
+ if parser.engine == "pyarrow":
+ df = parser.read_csv(
+ buf,
+ )
+ else:
+ with monkeypatch.context() as m:
+ m.setattr(libparsers, "DEFAULT_BUFFER_HEURISTIC", heuristic)
+ df = parser.read_csv_check_warnings(
+ warning_type,
+ r"Columns \(0: a\) have mixed types. "
+ "Specify dtype option on import or set low_memory=False.",
+ buf,
+ )
+ if parser.engine == "c" and parser.low_memory:
+ assert df.a.dtype == object
+ elif using_infer_string:
+ assert df.a.dtype == "str"
+ else:
+ assert df.a.dtype == object
+
+
+@pytest.mark.parametrize("iterator", [True, False])
+def test_empty_with_nrows_chunksize(all_parsers, iterator):
+ # see gh-9535
+ parser = all_parsers
+ expected = DataFrame(columns=["foo", "bar"])
+
+ nrows = 10
+ data = StringIO("foo,bar\n")
+
+ if parser.engine == "pyarrow":
+ msg = (
+ "The '(nrows|chunksize)' option is not supported with the 'pyarrow' engine"
+ )
+ with pytest.raises(ValueError, match=msg):
+ if iterator:
+ with parser.read_csv(data, chunksize=nrows) as reader:
+ next(iter(reader))
+ else:
+ parser.read_csv(data, nrows=nrows)
+ return
+
+ if iterator:
+ with parser.read_csv(data, chunksize=nrows) as reader:
+ result = next(iter(reader))
+ else:
+ result = parser.read_csv(data, nrows=nrows)
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_chunksize_with_usecols_second_block_shorter(all_parsers):
+ # GH#21211
+ parser = all_parsers
+ data = """1,2,3,4
+5,6,7,8
+9,10,11
+"""
+
+ if parser.engine == "pyarrow":
+ msg = "The 'chunksize' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data),
+ names=["a", "b"],
+ chunksize=2,
+ usecols=[0, 1],
+ header=None,
+ )
+ return
+
+ result_chunks = parser.read_csv(
+ StringIO(data),
+ names=["a", "b"],
+ chunksize=2,
+ usecols=[0, 1],
+ header=None,
+ )
+
+ expected_frames = [
+ DataFrame({"a": [1, 5], "b": [2, 6]}),
+ DataFrame({"a": [9], "b": [10]}, index=[2]),
+ ]
+
+ for i, result in enumerate(result_chunks):
+ tm.assert_frame_equal(result, expected_frames[i])
+
+
+def test_chunksize_second_block_shorter(all_parsers):
+ # GH#21211
+ parser = all_parsers
+ data = """a,b,c,d
+1,2,3,4
+5,6,7,8
+9,10,11
+"""
+
+ if parser.engine == "pyarrow":
+ msg = "The 'chunksize' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), chunksize=2)
+ return
+
+ result_chunks = parser.read_csv(StringIO(data), chunksize=2)
+
+ expected_frames = [
+ DataFrame({"a": [1, 5], "b": [2, 6], "c": [3, 7], "d": [4, 8]}),
+ DataFrame({"a": [9], "b": [10], "c": [11], "d": [np.nan]}, index=[2]),
+ ]
+
+ for i, result in enumerate(result_chunks):
+ tm.assert_frame_equal(result, expected_frames[i])
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_common_basic.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_common_basic.py
new file mode 100644
index 0000000000000000000000000000000000000000..b082b9cbf39159cd4a450755748f59aed774e879
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_common_basic.py
@@ -0,0 +1,868 @@
+"""
+Tests that work on both the Python and C engines but do not have a
+specific classification into the other test modules.
+"""
+
+from datetime import datetime
+from inspect import signature
+from io import StringIO
+import os
+import sys
+
+import numpy as np
+import pytest
+
+from pandas._config import using_string_dtype
+
+from pandas.compat import HAS_PYARROW
+from pandas.errors import (
+ EmptyDataError,
+ ParserError,
+ ParserWarning,
+)
+
+from pandas import (
+ DataFrame,
+ Index,
+ compat,
+)
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+
+def test_read_csv_local(all_parsers, csv1):
+ prefix = "file:///" if compat.is_platform_windows() else "file://"
+ parser = all_parsers
+
+ fname = prefix + str(os.path.abspath(csv1))
+ result = parser.read_csv(fname, index_col=0, parse_dates=True)
+
+ expected = DataFrame(
+ [
+ [0.980269, 3.685731, -0.364216805298, -1.159738],
+ [1.047916, -0.041232, -0.16181208307, 0.212549],
+ [0.498581, 0.731168, -0.537677223318, 1.346270],
+ [1.120202, 1.567621, 0.00364077397681, 0.675253],
+ [-0.487094, 0.571455, -1.6116394093, 0.103469],
+ [0.836649, 0.246462, 0.588542635376, 1.062782],
+ [-0.157161, 1.340307, 1.1957779562, -1.097007],
+ ],
+ columns=["A", "B", "C", "D"],
+ index=Index(
+ [
+ datetime(2000, 1, 3),
+ datetime(2000, 1, 4),
+ datetime(2000, 1, 5),
+ datetime(2000, 1, 6),
+ datetime(2000, 1, 7),
+ datetime(2000, 1, 10),
+ datetime(2000, 1, 11),
+ ],
+ dtype="M8[us]",
+ name="index",
+ ),
+ )
+ if parser.engine == "pyarrow":
+ expected.index = expected.index.astype("M8[s]")
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "number_csv, expected_number",
+ [
+ ("2,334", 2334),
+ ("-2,334", -2334),
+ ("-2,334,", -2334),
+ # Multiple consecutive thousand separators are allowed in C engine,
+ # but it's not necessarily intended behavior and may change in the future.
+ ("2,,,,,,,,,,,,,,,5", 25),
+ ("2,,3,4,,,,,,,,,,,,5", 2345),
+ ],
+)
+def test_1000_sep(all_parsers, number_csv, expected_number, request):
+ parser = all_parsers
+ data = f"""A|B|C
+1|{number_csv}|5
+10|13|10.
+"""
+ expected = DataFrame({"A": [1, 10], "B": [expected_number, 13], "C": [5, 10.0]})
+
+ if parser.engine == "pyarrow":
+ msg = "The 'thousands' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), sep="|", thousands=",")
+ return
+ elif parser.engine == "python" and ",," in number_csv:
+ mark = pytest.mark.xfail(
+ reason="Python engine doesn't allow consecutive thousands separators"
+ )
+ request.applymarker(mark)
+
+ result = parser.read_csv(StringIO(data), sep="|", thousands=",")
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("value", ["1 ,", ", 1", ",1"])
+def test_1000_sep_not_stripped_after_whitespace(all_parsers, value):
+ parser = all_parsers
+ data = f"a\n{value}\n"
+ expected = DataFrame({"a": [value]})
+
+ if parser.engine == "pyarrow":
+ msg = "The 'thousands' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), sep=";", thousands=",")
+ return
+
+ result = parser.read_csv(StringIO(data), sep=";", thousands=",")
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # ValueError: Found non-unique column index
+def test_unnamed_columns(all_parsers):
+ data = """A,B,C,,
+1,2,3,4,5
+6,7,8,9,10
+11,12,13,14,15
+"""
+ parser = all_parsers
+ expected = DataFrame(
+ [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15]],
+ dtype=np.int64,
+ columns=["A", "B", "C", "Unnamed: 3", "Unnamed: 4"],
+ )
+ result = parser.read_csv(StringIO(data))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_csv_mixed_type(all_parsers):
+ data = """A,B,C
+a,1,2
+b,3,4
+c,4,5
+"""
+ parser = all_parsers
+ expected = DataFrame({"A": ["a", "b", "c"], "B": [1, 3, 4], "C": [2, 4, 5]})
+ result = parser.read_csv(StringIO(data))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_csv_low_memory_no_rows_with_index(all_parsers):
+ # see gh-21141
+ parser = all_parsers
+
+ if not parser.low_memory:
+ pytest.skip("This is a low-memory specific test")
+
+ data = """A,B,C
+1,1,1,2
+2,2,3,4
+3,3,4,5
+"""
+
+ if parser.engine == "pyarrow":
+ msg = "The 'nrows' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), low_memory=True, index_col=0, nrows=0)
+ return
+
+ result = parser.read_csv(StringIO(data), low_memory=True, index_col=0, nrows=0)
+ expected = DataFrame(columns=["A", "B", "C"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_csv_dataframe(all_parsers, csv1):
+ parser = all_parsers
+ result = parser.read_csv(csv1, index_col=0, parse_dates=True)
+ expected = DataFrame(
+ [
+ [0.980269, 3.685731, -0.364216805298, -1.159738],
+ [1.047916, -0.041232, -0.16181208307, 0.212549],
+ [0.498581, 0.731168, -0.537677223318, 1.346270],
+ [1.120202, 1.567621, 0.00364077397681, 0.675253],
+ [-0.487094, 0.571455, -1.6116394093, 0.103469],
+ [0.836649, 0.246462, 0.588542635376, 1.062782],
+ [-0.157161, 1.340307, 1.1957779562, -1.097007],
+ ],
+ columns=["A", "B", "C", "D"],
+ index=Index(
+ [
+ datetime(2000, 1, 3),
+ datetime(2000, 1, 4),
+ datetime(2000, 1, 5),
+ datetime(2000, 1, 6),
+ datetime(2000, 1, 7),
+ datetime(2000, 1, 10),
+ datetime(2000, 1, 11),
+ ],
+ dtype="M8[us]",
+ name="index",
+ ),
+ )
+ if parser.engine == "pyarrow":
+ expected.index = expected.index.astype("M8[s]")
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("nrows", [3, 3.0])
+def test_read_nrows(all_parsers, nrows):
+ # see gh-10476
+ data = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+"""
+ expected = DataFrame(
+ [["foo", 2, 3, 4, 5], ["bar", 7, 8, 9, 10], ["baz", 12, 13, 14, 15]],
+ columns=["index", "A", "B", "C", "D"],
+ )
+ parser = all_parsers
+
+ if parser.engine == "pyarrow":
+ msg = "The 'nrows' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), nrows=nrows)
+ return
+
+ result = parser.read_csv(StringIO(data), nrows=nrows)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("nrows", [1.2, "foo", -1])
+def test_read_nrows_bad(all_parsers, nrows):
+ data = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+"""
+ msg = r"'nrows' must be an integer >=0"
+ parser = all_parsers
+ if parser.engine == "pyarrow":
+ msg = "The 'nrows' option is not supported with the 'pyarrow' engine"
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), nrows=nrows)
+
+
+def test_nrows_skipfooter_errors(all_parsers):
+ msg = "'skipfooter' not supported with 'nrows'"
+ data = "a\n1\n2\n3\n4\n5\n6"
+ parser = all_parsers
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), skipfooter=1, nrows=5)
+
+
+@skip_pyarrow
+def test_missing_trailing_delimiters(all_parsers):
+ parser = all_parsers
+ data = """A,B,C,D
+1,2,3,4
+1,3,3,
+1,4,5"""
+
+ result = parser.read_csv(StringIO(data))
+ expected = DataFrame(
+ [[1, 2, 3, 4], [1, 3, 3, np.nan], [1, 4, 5, np.nan]],
+ columns=["A", "B", "C", "D"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_skip_initial_space(all_parsers):
+ data = (
+ '"09-Apr-2012", "01:10:18.300", 2456026.548822908, 12849, '
+ "1.00361, 1.12551, 330.65659, 0355626618.16711, 73.48821, "
+ "314.11625, 1917.09447, 179.71425, 80.000, 240.000, -350, "
+ "70.06056, 344.98370, 1, 1, -0.689265, -0.692787, "
+ "0.212036, 14.7674, 41.605, -9999.0, -9999.0, "
+ "-9999.0, -9999.0, -9999.0, -9999.0, 000, 012, 128"
+ )
+ parser = all_parsers
+
+ if parser.engine == "pyarrow":
+ msg = "The 'skipinitialspace' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data),
+ names=list(range(33)),
+ header=None,
+ na_values=["-9999.0"],
+ skipinitialspace=True,
+ )
+ return
+
+ result = parser.read_csv(
+ StringIO(data),
+ names=list(range(33)),
+ header=None,
+ na_values=["-9999.0"],
+ skipinitialspace=True,
+ )
+ expected = DataFrame(
+ [
+ [
+ "09-Apr-2012",
+ "01:10:18.300",
+ 2456026.548822908,
+ 12849,
+ 1.00361,
+ 1.12551,
+ 330.65659,
+ 355626618.16711,
+ 73.48821,
+ 314.11625,
+ 1917.09447,
+ 179.71425,
+ 80.0,
+ 240.0,
+ -350,
+ 70.06056,
+ 344.9837,
+ 1,
+ 1,
+ -0.689265,
+ -0.692787,
+ 0.212036,
+ 14.7674,
+ 41.605,
+ np.nan,
+ np.nan,
+ np.nan,
+ np.nan,
+ np.nan,
+ np.nan,
+ 0,
+ 12,
+ 128,
+ ]
+ ]
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_trailing_delimiters(all_parsers):
+ # see gh-2442
+ data = """A,B,C
+1,2,3,
+4,5,6,
+7,8,9,"""
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), index_col=False)
+
+ expected = DataFrame({"A": [1, 4, 7], "B": [2, 5, 8], "C": [3, 6, 9]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_escapechar(all_parsers):
+ # https://stackoverflow.com/questions/13824840/feature-request-for-
+ # pandas-read-csv
+ data = '''SEARCH_TERM,ACTUAL_URL
+"bra tv board","http://www.ikea.com/se/sv/catalog/categories/departments/living_room/10475/?se%7cps%7cnonbranded%7cvardagsrum%7cgoogle%7ctv_bord"
+"tv p\xc3\xa5 hjul","http://www.ikea.com/se/sv/catalog/categories/departments/living_room/10475/?se%7cps%7cnonbranded%7cvardagsrum%7cgoogle%7ctv_bord"
+"SLAGBORD, \\"Bergslagen\\", IKEA:s 1700-tals series","http://www.ikea.com/se/sv/catalog/categories/departments/living_room/10475/?se%7cps%7cnonbranded%7cvardagsrum%7cgoogle%7ctv_bord"'''
+
+ parser = all_parsers
+ result = parser.read_csv(
+ StringIO(data), escapechar="\\", quotechar='"', encoding="utf-8"
+ )
+
+ assert result["SEARCH_TERM"][2] == 'SLAGBORD, "Bergslagen", IKEA:s 1700-tals series'
+
+ tm.assert_index_equal(result.columns, Index(["SEARCH_TERM", "ACTUAL_URL"]))
+
+
+def test_ignore_leading_whitespace(all_parsers):
+ # see gh-3374, gh-6607
+ parser = all_parsers
+ data = " a b c\n 1 2 3\n 4 5 6\n 7 8 9"
+
+ if parser.engine == "pyarrow":
+ msg = "the 'pyarrow' engine does not support regex separators"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), sep=r"\s+")
+ return
+ result = parser.read_csv(StringIO(data), sep=r"\s+")
+
+ expected = DataFrame({"a": [1, 4, 7], "b": [2, 5, 8], "c": [3, 6, 9]})
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+@pytest.mark.parametrize("usecols", [None, [0, 1], ["a", "b"]])
+def test_uneven_lines_with_usecols(all_parsers, usecols):
+ # see gh-12203
+ parser = all_parsers
+ data = r"""a,b,c
+0,1,2
+3,4,5,6,7
+8,9,10"""
+
+ if usecols is None:
+ # Make sure that an error is still raised
+ # when the "usecols" parameter is not provided.
+ msg = r"Expected \d+ fields in line \d+, saw \d+"
+ with pytest.raises(ParserError, match=msg):
+ parser.read_csv(StringIO(data))
+ else:
+ expected = DataFrame({"a": [0, 3, 8], "b": [1, 4, 9]})
+
+ result = parser.read_csv(StringIO(data), usecols=usecols)
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+@pytest.mark.parametrize(
+ "data,kwargs,expected",
+ [
+ # First, check to see that the response of parser when faced with no
+ # provided columns raises the correct error, with or without usecols.
+ ("", {}, None),
+ ("", {"usecols": ["X"]}, None),
+ (
+ ",,",
+ {"names": ["Dummy", "X", "Dummy_2"], "usecols": ["X"]},
+ DataFrame(columns=["X"], index=[0], dtype=np.float64),
+ ),
+ (
+ "",
+ {"names": ["Dummy", "X", "Dummy_2"], "usecols": ["X"]},
+ DataFrame(columns=["X"]),
+ ),
+ ],
+)
+def test_read_empty_with_usecols(all_parsers, data, kwargs, expected):
+ # see gh-12493
+ parser = all_parsers
+
+ if expected is None:
+ msg = "No columns to parse from file"
+ with pytest.raises(EmptyDataError, match=msg):
+ parser.read_csv(StringIO(data), **kwargs)
+ else:
+ result = parser.read_csv(StringIO(data), **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "kwargs,expected_data",
+ [
+ # gh-8661, gh-8679: this should ignore six lines, including
+ # lines with trailing whitespace and blank lines.
+ (
+ {
+ "header": None,
+ "sep": r"\s+",
+ "skiprows": [0, 1, 2, 3, 5, 6],
+ "skip_blank_lines": True,
+ },
+ [[1.0, 2.0, 4.0], [5.1, np.nan, 10.0]],
+ ),
+ # gh-8983: test skipping set of rows after a row with trailing spaces.
+ (
+ {
+ "sep": r"\s+",
+ "skiprows": [1, 2, 3, 5, 6],
+ "skip_blank_lines": True,
+ },
+ {"A": [1.0, 5.1], "B": [2.0, np.nan], "C": [4.0, 10]},
+ ),
+ ],
+)
+def test_trailing_spaces(all_parsers, kwargs, expected_data):
+ data = "A B C \nrandom line with trailing spaces \nskip\n1,2,3\n1,2.,4.\nrandom line with trailing tabs\t\t\t\n \n5.1,NaN,10.0\n" # noqa: E501
+ parser = all_parsers
+
+ if parser.engine == "pyarrow":
+ with pytest.raises(ValueError, match="the 'pyarrow' engine does not support"):
+ parser.read_csv(StringIO(data.replace(",", " ")), **kwargs)
+ return
+ expected = DataFrame(expected_data)
+ result = parser.read_csv(StringIO(data.replace(",", " ")), **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_filepath_or_buffer(all_parsers):
+ # see gh-43366
+ parser = all_parsers
+
+ with pytest.raises(TypeError, match="Expected file path name or file-like"):
+ parser.read_csv(filepath_or_buffer=b"input")
+
+
+def test_single_char_leading_whitespace(all_parsers):
+ # see gh-9710
+ parser = all_parsers
+ data = """\
+MyColumn
+a
+b
+a
+b\n"""
+
+ if parser.engine == "pyarrow":
+ msg = "The 'skipinitialspace' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data),
+ skipinitialspace=True,
+ )
+ return
+ expected = DataFrame({"MyColumn": list("abab")})
+ result = parser.read_csv(StringIO(data), skipinitialspace=True, sep=r"\s+")
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "sep,skip_blank_lines,exp_data",
+ [
+ (",", True, [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0], [-70.0, 0.4, 1.0]]),
+ (r"\s+", True, [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0], [-70.0, 0.4, 1.0]]),
+ (
+ ",",
+ False,
+ [
+ [1.0, 2.0, 4.0],
+ [np.nan, np.nan, np.nan],
+ [np.nan, np.nan, np.nan],
+ [5.0, np.nan, 10.0],
+ [np.nan, np.nan, np.nan],
+ [-70.0, 0.4, 1.0],
+ ],
+ ),
+ ],
+)
+def test_empty_lines(all_parsers, sep, skip_blank_lines, exp_data, request):
+ parser = all_parsers
+ data = """\
+A,B,C
+1,2.,4.
+
+
+5.,NaN,10.0
+
+-70,.4,1
+"""
+
+ if sep == r"\s+":
+ data = data.replace(",", " ")
+
+ if parser.engine == "pyarrow":
+ msg = "the 'pyarrow' engine does not support regex separators"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data), sep=sep, skip_blank_lines=skip_blank_lines
+ )
+ return
+
+ result = parser.read_csv(StringIO(data), sep=sep, skip_blank_lines=skip_blank_lines)
+ expected = DataFrame(exp_data, columns=["A", "B", "C"])
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_whitespace_lines(all_parsers):
+ parser = all_parsers
+ data = """
+
+\t \t\t
+\t
+A,B,C
+\t 1,2.,4.
+5.,NaN,10.0
+"""
+ expected = DataFrame([[1, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"])
+ result = parser.read_csv(StringIO(data))
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "data,expected",
+ [
+ (
+ """ A B C D
+a 1 2 3 4
+b 1 2 3 4
+c 1 2 3 4
+""",
+ DataFrame(
+ [[1, 2, 3, 4], [1, 2, 3, 4], [1, 2, 3, 4]],
+ columns=["A", "B", "C", "D"],
+ index=["a", "b", "c"],
+ ),
+ ),
+ (
+ " a b c\n1 2 3 \n4 5 6\n 7 8 9",
+ DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]], columns=["a", "b", "c"]),
+ ),
+ ],
+)
+def test_whitespace_regex_separator(all_parsers, data, expected):
+ # see gh-6607
+ parser = all_parsers
+ if parser.engine == "pyarrow":
+ msg = "the 'pyarrow' engine does not support regex separators"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), sep=r"\s+")
+ return
+
+ result = parser.read_csv(StringIO(data), sep=r"\s+")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_sub_character(all_parsers, csv_dir_path):
+ # see gh-16893
+ filename = os.path.join(csv_dir_path, "sub_char.csv")
+ expected = DataFrame([[1, 2, 3]], columns=["a", "\x1ab", "c"])
+
+ parser = all_parsers
+ result = parser.read_csv(filename)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("filename", ["sé-es-vé.csv", "ru-sй.csv", "中文文件名.csv"])
+def test_filename_with_special_chars(all_parsers, filename, tmp_path):
+ # see gh-15086.
+ parser = all_parsers
+ df = DataFrame({"a": [1, 2, 3]})
+
+ path = tmp_path / filename
+ df.to_csv(path, index=False)
+
+ result = parser.read_csv(path)
+ tm.assert_frame_equal(result, df)
+
+
+def test_read_table_same_signature_as_read_csv(all_parsers):
+ # GH-34976
+ parser = all_parsers
+
+ table_sign = signature(parser.read_table)
+ csv_sign = signature(parser.read_csv)
+
+ assert table_sign.parameters.keys() == csv_sign.parameters.keys()
+ assert table_sign.return_annotation == csv_sign.return_annotation
+
+ for key, csv_param in csv_sign.parameters.items():
+ table_param = table_sign.parameters[key]
+ if key == "sep":
+ assert csv_param.default == ","
+ assert table_param.default == "\t"
+ assert table_param.annotation == csv_param.annotation
+ assert table_param.kind == csv_param.kind
+ continue
+
+ assert table_param == csv_param
+
+
+def test_read_table_equivalency_to_read_csv(all_parsers):
+ # see gh-21948
+ # As of 0.25.0, read_table is undeprecated
+ parser = all_parsers
+ data = "a\tb\n1\t2\n3\t4"
+ expected = parser.read_csv(StringIO(data), sep="\t")
+ result = parser.read_table(StringIO(data))
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("read_func", ["read_csv", "read_table"])
+def test_read_csv_and_table_sys_setprofile(all_parsers, read_func):
+ # GH#41069
+ parser = all_parsers
+ data = "a b\n0 1"
+
+ sys.setprofile(lambda *a, **k: None)
+ result = getattr(parser, read_func)(StringIO(data))
+ sys.setprofile(None)
+
+ expected = DataFrame({"a b": ["0 1"]})
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_first_row_bom(all_parsers):
+ # see gh-26545
+ parser = all_parsers
+ data = '''\ufeff"Head1"\t"Head2"\t"Head3"'''
+
+ result = parser.read_csv(StringIO(data), delimiter="\t")
+ expected = DataFrame(columns=["Head1", "Head2", "Head3"])
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_first_row_bom_unquoted(all_parsers):
+ # see gh-36343
+ parser = all_parsers
+ data = """\ufeffHead1\tHead2\tHead3"""
+
+ result = parser.read_csv(StringIO(data), delimiter="\t")
+ expected = DataFrame(columns=["Head1", "Head2", "Head3"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("nrows", range(1, 6))
+def test_blank_lines_between_header_and_data_rows(all_parsers, nrows):
+ # GH 28071
+ ref = DataFrame(
+ [[np.nan, np.nan], [np.nan, np.nan], [1, 2], [np.nan, np.nan], [3, 4]],
+ columns=list("ab"),
+ )
+ csv = "\nheader\n\na,b\n\n\n1,2\n\n3,4"
+ parser = all_parsers
+
+ if parser.engine == "pyarrow":
+ msg = "The 'nrows' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(csv), header=3, nrows=nrows, skip_blank_lines=False
+ )
+ return
+
+ df = parser.read_csv(StringIO(csv), header=3, nrows=nrows, skip_blank_lines=False)
+ tm.assert_frame_equal(df, ref[:nrows])
+
+
+@skip_pyarrow
+def test_no_header_two_extra_columns(all_parsers):
+ # GH 26218
+ column_names = ["one", "two", "three"]
+ ref = DataFrame([["foo", "bar", "baz"]], columns=column_names)
+ stream = StringIO("foo,bar,baz,bam,blah")
+ parser = all_parsers
+ df = parser.read_csv_check_warnings(
+ ParserWarning,
+ "Length of header or names does not match length of data. "
+ "This leads to a loss of data with index_col=False.",
+ stream,
+ header=None,
+ names=column_names,
+ index_col=False,
+ )
+ tm.assert_frame_equal(df, ref)
+
+
+def test_read_csv_names_not_accepting_sets(all_parsers):
+ # GH 34946
+ data = """\
+ 1,2,3
+ 4,5,6\n"""
+ parser = all_parsers
+ with pytest.raises(ValueError, match="Names should be an ordered collection."):
+ parser.read_csv(StringIO(data), names=set("QAZ"))
+
+
+def test_read_csv_delimiter_and_sep_no_default(all_parsers):
+ # GH#39823
+ f = StringIO("a,b\n1,2")
+ parser = all_parsers
+ msg = "Specified a sep and a delimiter; you can only specify one."
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(f, sep=" ", delimiter=".")
+
+
+@pytest.mark.parametrize("kwargs", [{"delimiter": "\n"}, {"sep": "\n"}])
+def test_read_csv_line_break_as_separator(kwargs, all_parsers):
+ # GH#43528
+ parser = all_parsers
+ data = """a,b,c
+1,2,3
+ """
+ msg = (
+ r"Specified \\n as separator or delimiter. This forces the python engine "
+ r"which does not accept a line terminator. Hence it is not allowed to use "
+ r"the line terminator as separator."
+ )
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), **kwargs)
+
+
+@skip_pyarrow
+def test_dict_keys_as_names(all_parsers):
+ # GH: 36928
+ data = "1,2"
+
+ keys = {"a": int, "b": int}.keys()
+ parser = all_parsers
+
+ result = parser.read_csv(StringIO(data), names=keys)
+ expected = DataFrame({"a": [1], "b": [2]})
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.xfail(using_string_dtype() and HAS_PYARROW, reason="TODO(infer_string)")
+@xfail_pyarrow # UnicodeDecodeError: 'utf-8' codec can't decode byte 0xed in position 0
+def test_encoding_surrogatepass(all_parsers, tmp_path):
+ # GH39017
+ parser = all_parsers
+ content = b"\xed\xbd\xbf"
+ decoded = content.decode("utf-8", errors="surrogatepass")
+ expected = DataFrame({decoded: [decoded]}, index=[decoded * 2])
+ expected.index.name = decoded * 2
+
+ path = tmp_path / "test_encoding.csv"
+ path.write_bytes(
+ content * 2 + b"," + content + b"\n" + content * 2 + b"," + content
+ )
+ df = parser.read_csv(path, encoding_errors="surrogatepass", index_col=0)
+ tm.assert_frame_equal(df, expected)
+ with pytest.raises(UnicodeDecodeError, match="'utf-8' codec can't decode byte"):
+ parser.read_csv(path)
+
+
+def test_malformed_second_line(all_parsers):
+ # see GH14782
+ parser = all_parsers
+ data = "\na\nb\n"
+ result = parser.read_csv(StringIO(data), skip_blank_lines=False, header=1)
+ expected = DataFrame({"a": ["b"]})
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_short_single_line(all_parsers):
+ # GH 47566
+ parser = all_parsers
+ columns = ["a", "b", "c"]
+ data = "1,2"
+ result = parser.read_csv(StringIO(data), header=None, names=columns)
+ expected = DataFrame({"a": [1], "b": [2], "c": [np.nan]})
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # ValueError: Length mismatch: Expected axis has 2 elements
+def test_short_multi_line(all_parsers):
+ # GH 47566
+ parser = all_parsers
+ columns = ["a", "b", "c"]
+ data = "1,2\n1,2"
+ result = parser.read_csv(StringIO(data), header=None, names=columns)
+ expected = DataFrame({"a": [1, 1], "b": [2, 2], "c": [np.nan, np.nan]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_seek(all_parsers, tmp_path):
+ # GH48646
+ parser = all_parsers
+ prefix = "### DATA\n"
+ content = "nkey,value\ntables,rectangular\n"
+ path = tmp_path / "test_seek.csv"
+ path.write_text(prefix + content, encoding="utf-8")
+ with open(path, encoding="utf-8") as file:
+ file.readline()
+ actual = parser.read_csv(file)
+ expected = parser.read_csv(StringIO(content))
+ tm.assert_frame_equal(actual, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_data_list.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_data_list.py
new file mode 100644
index 0000000000000000000000000000000000000000..bf9293ddd841d63093ea112decab707e4a43c7ae
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_data_list.py
@@ -0,0 +1,92 @@
+"""
+Tests that work on both the Python and C engines but do not have a
+specific classification into the other test modules.
+"""
+
+import csv
+from io import StringIO
+
+import pytest
+
+from pandas import DataFrame
+import pandas._testing as tm
+
+from pandas.io.parsers import TextParser
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+
+
+@xfail_pyarrow
+def test_read_data_list(all_parsers):
+ parser = all_parsers
+ kwargs = {"index_col": 0}
+ data = "A,B,C\nfoo,1,2,3\nbar,4,5,6"
+
+ data_list = [["A", "B", "C"], ["foo", "1", "2", "3"], ["bar", "4", "5", "6"]]
+ expected = parser.read_csv(StringIO(data), **kwargs)
+
+ with TextParser(data_list, chunksize=2, **kwargs) as parser:
+ result = parser.read()
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_reader_list(all_parsers):
+ data = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+"""
+ parser = all_parsers
+ kwargs = {"index_col": 0}
+
+ lines = list(csv.reader(StringIO(data)))
+ with TextParser(lines, chunksize=2, **kwargs) as reader:
+ chunks = list(reader)
+
+ expected = parser.read_csv(StringIO(data), **kwargs)
+
+ tm.assert_frame_equal(chunks[0], expected[:2])
+ tm.assert_frame_equal(chunks[1], expected[2:4])
+ tm.assert_frame_equal(chunks[2], expected[4:])
+
+
+def test_reader_list_skiprows(all_parsers):
+ data = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+"""
+ parser = all_parsers
+ kwargs = {"index_col": 0}
+
+ lines = list(csv.reader(StringIO(data)))
+ with TextParser(lines, chunksize=2, skiprows=[1], **kwargs) as reader:
+ chunks = list(reader)
+
+ expected = parser.read_csv(StringIO(data), **kwargs)
+
+ tm.assert_frame_equal(chunks[0], expected[1:3])
+
+
+def test_read_csv_parse_simple_list(all_parsers):
+ parser = all_parsers
+ data = """foo
+bar baz
+qux foo
+foo
+bar"""
+
+ result = parser.read_csv(StringIO(data), header=None)
+ expected = DataFrame(["foo", "bar baz", "qux foo", "foo", "bar"])
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_decimal.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_decimal.py
new file mode 100644
index 0000000000000000000000000000000000000000..eb6c97097e5fb35bea06e2a3b6491a48ae1ded78
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_decimal.py
@@ -0,0 +1,73 @@
+"""
+Tests that work on both the Python and C engines but do not have a
+specific classification into the other test modules.
+"""
+
+from io import StringIO
+
+import pytest
+
+from pandas import DataFrame
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+@pytest.mark.parametrize(
+ "data,thousands,decimal",
+ [
+ (
+ """A|B|C
+1|2,334.01|5
+10|13|10.
+""",
+ ",",
+ ".",
+ ),
+ (
+ """A|B|C
+1|2.334,01|5
+10|13|10,
+""",
+ ".",
+ ",",
+ ),
+ ],
+)
+def test_1000_sep_with_decimal(all_parsers, data, thousands, decimal):
+ parser = all_parsers
+ expected = DataFrame({"A": [1, 10], "B": [2334.01, 13], "C": [5, 10.0]})
+
+ if parser.engine == "pyarrow":
+ msg = "The 'thousands' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data), sep="|", thousands=thousands, decimal=decimal
+ )
+ return
+
+ result = parser.read_csv(
+ StringIO(data), sep="|", thousands=thousands, decimal=decimal
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_euro_decimal_format(all_parsers):
+ parser = all_parsers
+ data = """Id;Number1;Number2;Text1;Text2;Number3
+1;1521,1541;187101,9543;ABC;poi;4,738797819
+2;121,12;14897,76;DEF;uyt;0,377320872
+3;878,158;108013,434;GHI;rez;2,735694704"""
+
+ result = parser.read_csv(StringIO(data), sep=";", decimal=",")
+ expected = DataFrame(
+ [
+ [1, 1521.1541, 187101.9543, "ABC", "poi", 4.738797819],
+ [2, 121.12, 14897.76, "DEF", "uyt", 0.377320872],
+ [3, 878.158, 108013.434, "GHI", "rez", 2.735694704],
+ ],
+ columns=["Id", "Number1", "Number2", "Text1", "Text2", "Number3"],
+ )
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_file_buffer_url.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_file_buffer_url.py
new file mode 100644
index 0000000000000000000000000000000000000000..8b2f6fc949256c9632816b56e86f67b89710adb7
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_file_buffer_url.py
@@ -0,0 +1,470 @@
+"""
+Tests that work on both the Python and C engines but do not have a
+specific classification into the other test modules.
+"""
+
+from io import (
+ BytesIO,
+ StringIO,
+)
+import os
+import platform
+from urllib.error import URLError
+import uuid
+
+import numpy as np
+import pytest
+
+from pandas.compat import WASM
+from pandas.errors import (
+ EmptyDataError,
+ ParserError,
+)
+import pandas.util._test_decorators as td
+
+from pandas import (
+ DataFrame,
+ Index,
+)
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+
+@pytest.mark.network
+@pytest.mark.single_cpu
+def test_url(all_parsers, csv_dir_path, httpserver):
+ parser = all_parsers
+ kwargs = {"sep": "\t"}
+
+ local_path = os.path.join(csv_dir_path, "salaries.csv")
+ with open(local_path, encoding="utf-8") as f:
+ httpserver.serve_content(content=f.read())
+
+ url_result = parser.read_csv(httpserver.url, **kwargs)
+
+ local_result = parser.read_csv(local_path, **kwargs)
+ tm.assert_frame_equal(url_result, local_result)
+
+
+@pytest.mark.slow
+def test_local_file(all_parsers, csv_dir_path):
+ parser = all_parsers
+ kwargs = {"sep": "\t"}
+
+ local_path = os.path.join(csv_dir_path, "salaries.csv")
+ local_result = parser.read_csv(local_path, **kwargs)
+ url = "file://localhost/" + local_path
+
+ try:
+ url_result = parser.read_csv(url, **kwargs)
+ tm.assert_frame_equal(url_result, local_result)
+ except URLError:
+ # Fails on some systems.
+ pytest.skip("Failing on: " + " ".join(platform.uname()))
+
+
+@xfail_pyarrow # AssertionError: DataFrame.index are different
+def test_path_path_lib(all_parsers, temp_file):
+ parser = all_parsers
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ result = tm.round_trip_pathlib(
+ df.to_csv, lambda p: parser.read_csv(p, index_col=0), temp_file
+ )
+ tm.assert_frame_equal(df, result)
+
+
+@pytest.mark.skipif(WASM, reason="limited file system access on WASM")
+def test_nonexistent_path(all_parsers):
+ # gh-2428: pls no segfault
+ # gh-14086: raise more helpful FileNotFoundError
+ # GH#29233 "File foo" instead of "File b'foo'"
+ parser = all_parsers
+ path = f"{uuid.uuid4()}.csv"
+
+ msg = r"\[Errno 2\]"
+ with pytest.raises(FileNotFoundError, match=msg) as e:
+ parser.read_csv(path)
+ assert path == e.value.filename
+
+
+@pytest.mark.skipif(WASM, reason="limited file system access on WASM")
+@td.skip_if_windows # os.chmod does not work in windows
+def test_no_permission(all_parsers, temp_file):
+ # GH 23784
+ parser = all_parsers
+
+ msg = r"\[Errno 13\]"
+ path = temp_file
+ os.chmod(path, 0) # make file unreadable
+
+ # verify that this process cannot open the file (not running as sudo)
+ try:
+ with open(path, encoding="utf-8"):
+ pass
+ pytest.skip("Running as sudo.")
+ except PermissionError:
+ pass
+
+ with pytest.raises(PermissionError, match=msg) as e:
+ parser.read_csv(path)
+ assert str(path.resolve()) == e.value.filename
+
+
+@pytest.mark.parametrize(
+ "data,kwargs,expected,msg",
+ [
+ # gh-10728: WHITESPACE_LINE
+ (
+ "a,b,c\n4,5,6\n ",
+ {},
+ DataFrame([[4, 5, 6]], columns=["a", "b", "c"]),
+ None,
+ ),
+ # gh-10548: EAT_LINE_COMMENT
+ (
+ "a,b,c\n4,5,6\n#comment",
+ {"comment": "#"},
+ DataFrame([[4, 5, 6]], columns=["a", "b", "c"]),
+ None,
+ ),
+ # EAT_CRNL_NOP
+ (
+ "a,b,c\n4,5,6\n\r",
+ {},
+ DataFrame([[4, 5, 6]], columns=["a", "b", "c"]),
+ None,
+ ),
+ # EAT_COMMENT
+ (
+ "a,b,c\n4,5,6#comment",
+ {"comment": "#"},
+ DataFrame([[4, 5, 6]], columns=["a", "b", "c"]),
+ None,
+ ),
+ # SKIP_LINE
+ (
+ "a,b,c\n4,5,6\nskipme",
+ {"skiprows": [2]},
+ DataFrame([[4, 5, 6]], columns=["a", "b", "c"]),
+ None,
+ ),
+ # EAT_LINE_COMMENT
+ (
+ "a,b,c\n4,5,6\n#comment",
+ {"comment": "#", "skip_blank_lines": False},
+ DataFrame([[4, 5, 6]], columns=["a", "b", "c"]),
+ None,
+ ),
+ # IN_FIELD
+ (
+ "a,b,c\n4,5,6\n ",
+ {"skip_blank_lines": False},
+ DataFrame([["4", 5, 6], [" ", None, None]], columns=["a", "b", "c"]),
+ None,
+ ),
+ # EAT_CRNL
+ (
+ "a,b,c\n4,5,6\n\r",
+ {"skip_blank_lines": False},
+ DataFrame([[4, 5, 6], [None, None, None]], columns=["a", "b", "c"]),
+ None,
+ ),
+ # ESCAPED_CHAR
+ (
+ "a,b,c\n4,5,6\n\\",
+ {"escapechar": "\\"},
+ None,
+ "(EOF following escape character)|(unexpected end of data)",
+ ),
+ # ESCAPE_IN_QUOTED_FIELD
+ (
+ 'a,b,c\n4,5,6\n"\\',
+ {"escapechar": "\\"},
+ None,
+ "(EOF inside string starting at row 2)|(unexpected end of data)",
+ ),
+ # IN_QUOTED_FIELD
+ (
+ 'a,b,c\n4,5,6\n"',
+ {"escapechar": "\\"},
+ None,
+ "(EOF inside string starting at row 2)|(unexpected end of data)",
+ ),
+ ],
+ ids=[
+ "whitespace-line",
+ "eat-line-comment",
+ "eat-crnl-nop",
+ "eat-comment",
+ "skip-line",
+ "eat-line-comment",
+ "in-field",
+ "eat-crnl",
+ "escaped-char",
+ "escape-in-quoted-field",
+ "in-quoted-field",
+ ],
+)
+def test_eof_states(all_parsers, data, kwargs, expected, msg, request):
+ # see gh-10728, gh-10548
+ parser = all_parsers
+
+ if parser.engine == "pyarrow" and "comment" in kwargs:
+ msg = "The 'comment' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), **kwargs)
+ return
+
+ if parser.engine == "pyarrow" and "\r" not in data:
+ # pandas.errors.ParserError: CSV parse error: Expected 3 columns, got 1:
+ # ValueError: skiprows argument must be an integer when using engine='pyarrow'
+ # AssertionError: Regex pattern did not match.
+ pytest.skip(reason="https://github.com/apache/arrow/issues/38676")
+
+ if expected is None:
+ with pytest.raises(ParserError, match=msg):
+ parser.read_csv(StringIO(data), **kwargs)
+ else:
+ result = parser.read_csv(StringIO(data), **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_temporary_file(all_parsers, temp_file):
+ # see gh-13398
+ parser = all_parsers
+ data = "0 0"
+
+ with temp_file.open(mode="w+", encoding="utf-8") as new_file:
+ new_file.write(data)
+ new_file.flush()
+ new_file.seek(0)
+
+ if parser.engine == "pyarrow":
+ msg = "the 'pyarrow' engine does not support regex separators"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(new_file, sep=r"\s+", header=None)
+ return
+
+ result = parser.read_csv(new_file, sep=r"\s+", header=None)
+
+ expected = DataFrame([[0, 0]])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_internal_eof_byte(all_parsers):
+ # see gh-5500
+ parser = all_parsers
+ data = "a,b\n1\x1a,2"
+
+ expected = DataFrame([["1\x1a", 2]], columns=["a", "b"])
+ result = parser.read_csv(StringIO(data))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_internal_eof_byte_to_file(all_parsers, temp_file):
+ # see gh-16559
+ parser = all_parsers
+ data = b'c1,c2\r\n"test \x1a test", test\r\n'
+ expected = DataFrame([["test \x1a test", " test"]], columns=["c1", "c2"])
+ path = f"__{uuid.uuid4()}__.csv"
+
+ path2 = temp_file.parent / path
+ with open(path2, "wb") as f:
+ f.write(data)
+
+ result = parser.read_csv(path2)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_file_handle_string_io(all_parsers):
+ # gh-14418
+ #
+ # Don't close user provided file handles.
+ parser = all_parsers
+ data = "a,b\n1,2"
+
+ fh = StringIO(data)
+ parser.read_csv(fh)
+ assert not fh.closed
+
+
+def test_file_handles_with_open(all_parsers, csv1):
+ # gh-14418
+ #
+ # Don't close user provided file handles.
+ parser = all_parsers
+
+ for mode in ["r", "rb"]:
+ with open(csv1, mode, encoding="utf-8" if mode == "r" else None) as f:
+ parser.read_csv(f)
+ assert not f.closed
+
+
+def test_invalid_file_buffer_class(all_parsers):
+ # see gh-15337
+ class InvalidBuffer:
+ pass
+
+ parser = all_parsers
+ msg = "Invalid file path or buffer object type"
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(InvalidBuffer())
+
+
+def test_invalid_file_buffer_mock(all_parsers):
+ # see gh-15337
+ parser = all_parsers
+ msg = "Invalid file path or buffer object type"
+
+ class Foo:
+ pass
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(Foo())
+
+
+def test_valid_file_buffer_seems_invalid(all_parsers):
+ # gh-16135: we want to ensure that "tell" and "seek"
+ # aren't actually being used when we call `read_csv`
+ #
+ # Thus, while the object may look "invalid" (these
+ # methods are attributes of the `StringIO` class),
+ # it is still a valid file-object for our purposes.
+ class NoSeekTellBuffer(StringIO):
+ def tell(self):
+ raise AttributeError("No tell method")
+
+ def seek(self, pos, whence=0):
+ raise AttributeError("No seek method")
+
+ data = "a\n1"
+ parser = all_parsers
+ expected = DataFrame({"a": [1]})
+
+ result = parser.read_csv(NoSeekTellBuffer(data))
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("io_class", [StringIO, BytesIO])
+@pytest.mark.parametrize("encoding", [None, "utf-8"])
+def test_read_csv_file_handle(all_parsers, io_class, encoding):
+ """
+ Test whether read_csv does not close user-provided file handles.
+
+ GH 36980
+ """
+ parser = all_parsers
+ expected = DataFrame({"a": [1], "b": [2]})
+
+ content = "a,b\n1,2"
+ handle = io_class(content.encode("utf-8") if io_class == BytesIO else content)
+
+ tm.assert_frame_equal(parser.read_csv(handle, encoding=encoding), expected)
+ assert not handle.closed
+
+
+def test_memory_map_compression(all_parsers, compression, temp_file):
+ """
+ Support memory map for compressed files.
+
+ GH 37621
+ """
+ parser = all_parsers
+ expected = DataFrame({"a": [1], "b": [2]})
+
+ path = temp_file
+ expected.to_csv(path, index=False, compression=compression)
+
+ if parser.engine == "pyarrow":
+ msg = "The 'memory_map' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(path, memory_map=True, compression=compression)
+ return
+
+ result = parser.read_csv(path, memory_map=True, compression=compression)
+
+ tm.assert_frame_equal(
+ result,
+ expected,
+ )
+
+
+def test_context_manager(all_parsers, datapath):
+ # make sure that opened files are closed
+ parser = all_parsers
+
+ path = datapath("io", "data", "csv", "iris.csv")
+
+ if parser.engine == "pyarrow":
+ msg = "The 'chunksize' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(path, chunksize=1)
+ return
+
+ reader = parser.read_csv(path, chunksize=1)
+ assert not reader.handles.handle.closed
+ try:
+ with reader:
+ next(reader)
+ raise AssertionError
+ except AssertionError:
+ assert reader.handles.handle.closed
+
+
+def test_context_manageri_user_provided(all_parsers, datapath):
+ # make sure that user-provided handles are not closed
+ parser = all_parsers
+
+ with open(datapath("io", "data", "csv", "iris.csv"), encoding="utf-8") as path:
+ if parser.engine == "pyarrow":
+ msg = "The 'chunksize' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(path, chunksize=1)
+ return
+
+ reader = parser.read_csv(path, chunksize=1)
+ assert not reader.handles.handle.closed
+ try:
+ with reader:
+ next(reader)
+ raise AssertionError
+ except AssertionError:
+ assert not reader.handles.handle.closed
+
+
+@skip_pyarrow # ParserError: Empty CSV file
+def test_file_descriptor_leak(all_parsers, temp_file):
+ # GH 31488
+ parser = all_parsers
+ path = temp_file
+ with pytest.raises(EmptyDataError, match="No columns to parse from file"):
+ parser.read_csv(path)
+
+
+def test_memory_map(all_parsers, csv_dir_path):
+ mmap_file = os.path.join(csv_dir_path, "test_mmap.csv")
+ parser = all_parsers
+
+ expected = DataFrame(
+ {"a": [1, 2, 3], "b": ["one", "two", "three"], "c": ["I", "II", "III"]}
+ )
+
+ if parser.engine == "pyarrow":
+ msg = "The 'memory_map' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(mmap_file, memory_map=True)
+ return
+
+ result = parser.read_csv(mmap_file, memory_map=True)
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_float.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_float.py
new file mode 100644
index 0000000000000000000000000000000000000000..d711ee6053b41b63fc6514fad6ff005aa4e5f170
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_float.py
@@ -0,0 +1,110 @@
+"""
+Tests that work on both the Python and C engines but do not have a
+specific classification into the other test modules.
+"""
+
+from io import StringIO
+
+import numpy as np
+import pytest
+
+from pandas import DataFrame
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+
+@skip_pyarrow # ParserError: CSV parse error: Empty CSV file or block
+def test_float_parser(all_parsers):
+ # see gh-9565
+ parser = all_parsers
+ data = "45e-1,4.5,45.,inf,-inf"
+ result = parser.read_csv(StringIO(data), header=None)
+
+ expected = DataFrame([[float(s) for s in data.split(",")]])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_scientific_no_exponent(all_parsers_all_precisions):
+ # see gh-12215
+ df = DataFrame.from_dict({"w": ["2e"], "x": ["3E"], "y": ["42e"], "z": ["632E"]})
+ data = df.to_csv(index=False)
+ parser, precision = all_parsers_all_precisions
+
+ df_roundtrip = parser.read_csv(StringIO(data), float_precision=precision)
+ tm.assert_frame_equal(df_roundtrip, df)
+
+
+@pytest.mark.parametrize(
+ "value, expected_value",
+ [
+ ("0E-617", 0.0),
+ ("0E99999999", 0.0),
+ ("-0E99999999", 0.0),
+ ("-0E-99999999", 0.0),
+ ("10E-617", 0.0),
+ ("10E-100000", 0.0),
+ ("-10E-100000", 0.0),
+ ("10e-99999999999", 0.0),
+ ("10e-999999999999", 0.0),
+ ("10e-9999999999999", 0.0),
+ ("10E999", np.inf),
+ ("-10e99999999999", -np.inf),
+ ("10e99999999999", np.inf),
+ ("10e999999999999", np.inf),
+ ("10e9999999999999", np.inf),
+ ("50060e8007123400", np.inf),
+ ("-50060e8007123400", -np.inf),
+ ],
+)
+def test_large_exponent(all_parsers_all_precisions, value, expected_value):
+ # GH#38753; GH#38794; GH#62740
+ parser, precision = all_parsers_all_precisions
+
+ data = f"data\n{value}"
+ result = parser.read_csv(StringIO(data), float_precision=precision)
+ expected = DataFrame({"data": [expected_value]})
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "value, expected_value",
+ [
+ ("32.0", 32.0),
+ ("32e0", 32.0),
+ ("3.2e1", 32.0),
+ ("3.2e80", 3.2e80),
+ ("3.2e-80", 3.2e-80),
+ ("18446744073709551616.0", float(1 << 64)), # loses precision
+ ("18446744073709551616.5", float(1 << 64)), # loses precision
+ ("36893488147419103232.3", float(1 << 65)), # loses precision
+ ],
+)
+def test_small_int_followed_by_float(
+ all_parsers_all_precisions, value, expected_value, request
+):
+ # GH#51295
+ parser, precision = all_parsers_all_precisions
+ data = f"""data
+ 42
+ {value}"""
+ result = parser.read_csv(StringIO(data), float_precision=precision)
+ expected = DataFrame({"data": [42.0, expected_value]})
+
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "value", ["81e31d04049863b72", "d81e31d04049863b72", "81e3104049863b72"]
+)
+def test_invalid_float_number(all_parsers_all_precisions, value):
+ # GH#62617
+ parser, precision = all_parsers_all_precisions
+ data = f"h1,h2,h3\ndata1,{value},data3"
+
+ result = parser.read_csv(StringIO(data), float_precision=precision)
+ expected = DataFrame({"h1": ["data1"], "h2": [value], "h3": "data3"})
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_index.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_index.py
new file mode 100644
index 0000000000000000000000000000000000000000..7570a4df1e89fa2a6ced35959e67daab6007e784
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_index.py
@@ -0,0 +1,307 @@
+"""
+Tests that work on both the Python and C engines but do not have a
+specific classification into the other test modules.
+"""
+
+from datetime import datetime
+from io import StringIO
+import os
+
+import pytest
+
+from pandas import (
+ DataFrame,
+ Index,
+ MultiIndex,
+)
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+
+@pytest.mark.parametrize(
+ "data,kwargs,expected",
+ [
+ (
+ """foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+""",
+ {"index_col": 0, "names": ["index", "A", "B", "C", "D"]},
+ DataFrame(
+ [
+ [2, 3, 4, 5],
+ [7, 8, 9, 10],
+ [12, 13, 14, 15],
+ [12, 13, 14, 15],
+ [12, 13, 14, 15],
+ [12, 13, 14, 15],
+ ],
+ index=Index(["foo", "bar", "baz", "qux", "foo2", "bar2"], name="index"),
+ columns=["A", "B", "C", "D"],
+ ),
+ ),
+ (
+ """foo,one,2,3,4,5
+foo,two,7,8,9,10
+foo,three,12,13,14,15
+bar,one,12,13,14,15
+bar,two,12,13,14,15
+""",
+ {"index_col": [0, 1], "names": ["index1", "index2", "A", "B", "C", "D"]},
+ DataFrame(
+ [
+ [2, 3, 4, 5],
+ [7, 8, 9, 10],
+ [12, 13, 14, 15],
+ [12, 13, 14, 15],
+ [12, 13, 14, 15],
+ ],
+ index=MultiIndex.from_tuples(
+ [
+ ("foo", "one"),
+ ("foo", "two"),
+ ("foo", "three"),
+ ("bar", "one"),
+ ("bar", "two"),
+ ],
+ names=["index1", "index2"],
+ ),
+ columns=["A", "B", "C", "D"],
+ ),
+ ),
+ ],
+)
+def test_pass_names_with_index(all_parsers, data, kwargs, expected):
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("index_col", [[0, 1], [1, 0]])
+def test_multi_index_no_level_names(
+ request, all_parsers, index_col, using_infer_string
+):
+ data = """index1,index2,A,B,C,D
+foo,one,2,3,4,5
+foo,two,7,8,9,10
+foo,three,12,13,14,15
+bar,one,12,13,14,15
+bar,two,12,13,14,15
+"""
+ headless_data = "\n".join(data.split("\n")[1:])
+
+ names = ["A", "B", "C", "D"]
+ parser = all_parsers
+
+ result = parser.read_csv(
+ StringIO(headless_data), index_col=index_col, header=None, names=names
+ )
+ expected = parser.read_csv(StringIO(data), index_col=index_col)
+
+ # No index names in headless data.
+ expected.index.names = [None] * 2
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_multi_index_no_level_names_implicit(all_parsers):
+ parser = all_parsers
+ data = """A,B,C,D
+foo,one,2,3,4,5
+foo,two,7,8,9,10
+foo,three,12,13,14,15
+bar,one,12,13,14,15
+bar,two,12,13,14,15
+"""
+
+ result = parser.read_csv(StringIO(data))
+ expected = DataFrame(
+ [
+ [2, 3, 4, 5],
+ [7, 8, 9, 10],
+ [12, 13, 14, 15],
+ [12, 13, 14, 15],
+ [12, 13, 14, 15],
+ ],
+ columns=["A", "B", "C", "D"],
+ index=MultiIndex.from_tuples(
+ [
+ ("foo", "one"),
+ ("foo", "two"),
+ ("foo", "three"),
+ ("bar", "one"),
+ ("bar", "two"),
+ ]
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+@pytest.mark.parametrize(
+ "data,columns,header",
+ [
+ ("a,b", ["a", "b"], [0]),
+ (
+ "a,b\nc,d",
+ MultiIndex.from_tuples([("a", "c"), ("b", "d")]),
+ [0, 1],
+ ),
+ ],
+)
+@pytest.mark.parametrize("round_trip", [True, False])
+def test_multi_index_blank_df(all_parsers, data, columns, header, round_trip):
+ # see gh-14545
+ parser = all_parsers
+ expected = DataFrame(columns=columns)
+ data = expected.to_csv(index=False) if round_trip else data
+
+ result = parser.read_csv(StringIO(data), header=header)
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # AssertionError: DataFrame.columns are different
+def test_no_unnamed_index(all_parsers):
+ parser = all_parsers
+ data = """ id c0 c1 c2
+0 1 0 a b
+1 2 0 c d
+2 2 2 e f
+"""
+ result = parser.read_csv(StringIO(data), sep=" ")
+ expected = DataFrame(
+ [[0, 1, 0, "a", "b"], [1, 2, 0, "c", "d"], [2, 2, 2, "e", "f"]],
+ columns=["Unnamed: 0", "id", "c0", "c1", "c2"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_duplicate_index_explicit(all_parsers):
+ data = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo,12,13,14,15
+bar,12,13,14,15
+"""
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), index_col=0)
+
+ expected = DataFrame(
+ [
+ [2, 3, 4, 5],
+ [7, 8, 9, 10],
+ [12, 13, 14, 15],
+ [12, 13, 14, 15],
+ [12, 13, 14, 15],
+ [12, 13, 14, 15],
+ ],
+ columns=["A", "B", "C", "D"],
+ index=Index(["foo", "bar", "baz", "qux", "foo", "bar"], name="index"),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_read_duplicate_index_implicit(all_parsers):
+ data = """A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo,12,13,14,15
+bar,12,13,14,15
+"""
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data))
+
+ expected = DataFrame(
+ [
+ [2, 3, 4, 5],
+ [7, 8, 9, 10],
+ [12, 13, 14, 15],
+ [12, 13, 14, 15],
+ [12, 13, 14, 15],
+ [12, 13, 14, 15],
+ ],
+ columns=["A", "B", "C", "D"],
+ index=Index(["foo", "bar", "baz", "qux", "foo", "bar"]),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_read_csv_no_index_name(all_parsers, csv_dir_path):
+ parser = all_parsers
+ csv2 = os.path.join(csv_dir_path, "test2.csv")
+ result = parser.read_csv(csv2, index_col=0, parse_dates=True)
+
+ expected = DataFrame(
+ [
+ [0.980269, 3.685731, -0.364216805298, -1.159738, "foo"],
+ [1.047916, -0.041232, -0.16181208307, 0.212549, "bar"],
+ [0.498581, 0.731168, -0.537677223318, 1.346270, "baz"],
+ [1.120202, 1.567621, 0.00364077397681, 0.675253, "qux"],
+ [-0.487094, 0.571455, -1.6116394093, 0.103469, "foo2"],
+ ],
+ columns=["A", "B", "C", "D", "E"],
+ index=Index(
+ [
+ datetime(2000, 1, 3),
+ datetime(2000, 1, 4),
+ datetime(2000, 1, 5),
+ datetime(2000, 1, 6),
+ datetime(2000, 1, 7),
+ ],
+ dtype="M8[us]",
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+def test_empty_with_index(all_parsers):
+ # see gh-10184
+ data = "x,y"
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), index_col=0)
+
+ expected = DataFrame(columns=["y"], index=Index([], name="x"))
+ tm.assert_frame_equal(result, expected)
+
+
+# CSV parse error: Empty CSV file or block: cannot infer number of columns
+@skip_pyarrow
+def test_empty_with_multi_index(all_parsers):
+ # see gh-10467
+ data = "x,y,z"
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), index_col=["x", "y"])
+
+ expected = DataFrame(
+ columns=["z"], index=MultiIndex.from_arrays([[]] * 2, names=["x", "y"])
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+# CSV parse error: Empty CSV file or block: cannot infer number of columns
+@skip_pyarrow
+def test_empty_with_reversed_multi_index(all_parsers):
+ data = "x,y,z"
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), index_col=[1, 0])
+
+ expected = DataFrame(
+ columns=["z"], index=MultiIndex.from_arrays([[]] * 2, names=["y", "x"])
+ )
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_inf.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_inf.py
new file mode 100644
index 0000000000000000000000000000000000000000..657aa3278a4424b08f062f2f36213d206d38099f
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_inf.py
@@ -0,0 +1,59 @@
+"""
+Tests that work on both the Python and C engines but do not have a
+specific classification into the other test modules.
+"""
+
+from io import StringIO
+
+import pytest
+
+from pandas import DataFrame
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+
+
+@xfail_pyarrow # AssertionError: DataFrame.index are different
+@pytest.mark.parametrize("na_filter", [True, False])
+def test_inf_parsing(all_parsers, na_filter):
+ parser = all_parsers
+ data = """\
+,A
+a,inf
+b,-inf
+c,+Inf
+d,-Inf
+e,INF
+f,-INF
+g,+INf
+h,-INf
+i,inF
+j,-inF"""
+ expected = DataFrame(
+ {"A": [float("inf"), float("-inf")] * 5},
+ index=["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"],
+ )
+ result = parser.read_csv(StringIO(data), index_col=0, na_filter=na_filter)
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # AssertionError: DataFrame.index are different
+@pytest.mark.parametrize("na_filter", [True, False])
+def test_infinity_parsing(all_parsers, na_filter):
+ parser = all_parsers
+ data = """\
+,A
+a,Infinity
+b,-Infinity
+c,+Infinity
+"""
+ expected = DataFrame(
+ {"A": [float("infinity"), float("-infinity"), float("+infinity")]},
+ index=["a", "b", "c"],
+ )
+ result = parser.read_csv(StringIO(data), index_col=0, na_filter=na_filter)
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_ints.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_ints.py
new file mode 100644
index 0000000000000000000000000000000000000000..9a98f99f80adc8d2eca31b895799ef95a90d005d
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_ints.py
@@ -0,0 +1,244 @@
+"""
+Tests that work on both the Python and C engines but do not have a
+specific classification into the other test modules.
+"""
+
+from io import StringIO
+
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ Series,
+)
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+
+def test_int_conversion(all_parsers):
+ data = """A,B
+1.0,1
+2.0,2
+3.0,3
+"""
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data))
+
+ expected = DataFrame([[1.0, 1], [2.0, 2], [3.0, 3]], columns=["A", "B"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "data,kwargs,expected",
+ [
+ (
+ "A,B\nTrue,1\nFalse,2\nTrue,3",
+ {},
+ [[True, 1], [False, 2], [True, 3]],
+ ),
+ (
+ "A,B\nYES,1\nno,2\nyes,3\nNo,3\nYes,3",
+ {"true_values": ["yes", "Yes", "YES"], "false_values": ["no", "NO", "No"]},
+ [[True, 1], [False, 2], [True, 3], [False, 3], [True, 3]],
+ ),
+ (
+ "A,B\nTRUE,1\nFALSE,2\nTRUE,3",
+ {},
+ [[True, 1], [False, 2], [True, 3]],
+ ),
+ (
+ "A,B\nfoo,bar\nbar,foo",
+ {"true_values": ["foo"], "false_values": ["bar"]},
+ [[True, False], [False, True]],
+ ),
+ ],
+)
+def test_parse_bool(all_parsers, data, kwargs, expected):
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), **kwargs)
+ expected = DataFrame(expected, columns=["A", "B"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_parse_integers_above_fp_precision(all_parsers):
+ data = """Numbers
+17007000002000191
+17007000002000191
+17007000002000191
+17007000002000191
+17007000002000192
+17007000002000192
+17007000002000192
+17007000002000192
+17007000002000192
+17007000002000194"""
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data))
+ expected = DataFrame(
+ {
+ "Numbers": [
+ 17007000002000191,
+ 17007000002000191,
+ 17007000002000191,
+ 17007000002000191,
+ 17007000002000192,
+ 17007000002000192,
+ 17007000002000192,
+ 17007000002000192,
+ 17007000002000192,
+ 17007000002000194,
+ ]
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("sep", [" ", r"\s+"])
+def test_integer_overflow_bug(all_parsers, sep):
+ # see gh-2601
+ data = "65248E10 11\n55555E55 22\n"
+ parser = all_parsers
+ if parser.engine == "pyarrow" and sep != " ":
+ msg = "the 'pyarrow' engine does not support regex separators"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), header=None, sep=sep)
+ return
+
+ result = parser.read_csv(StringIO(data), header=None, sep=sep)
+ expected = DataFrame([[6.5248e14, 11], [5.5555e59, 22]])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_int64_min_issues(all_parsers):
+ # see gh-2599
+ parser = all_parsers
+ data = "A,B\n0,0\n0,"
+ result = parser.read_csv(StringIO(data))
+
+ expected = DataFrame({"A": [0, 0], "B": [0, np.nan]})
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("conv", [None, np.int64, np.uint64])
+def test_int64_overflow(all_parsers, conv, request):
+ data = """ID
+00013007854817840016671868
+00013007854817840016749251
+00013007854817840016754630
+00013007854817840016781876
+00013007854817840017028824
+00013007854817840017963235
+00013007854817840018860166"""
+ parser = all_parsers
+
+ if conv is None:
+ # 13007854817840016671868 > UINT64_MAX, so this
+ # will overflow and return object as the dtype.
+ if parser.engine == "pyarrow":
+ mark = pytest.mark.xfail(reason="parses to float64")
+ request.applymarker(mark)
+
+ result = parser.read_csv(StringIO(data))
+ expected = DataFrame(
+ [
+ 13007854817840016671868,
+ 13007854817840016749251,
+ 13007854817840016754630,
+ 13007854817840016781876,
+ 13007854817840017028824,
+ 13007854817840017963235,
+ 13007854817840018860166,
+ ],
+ columns=["ID"],
+ )
+ tm.assert_frame_equal(result, expected)
+ else:
+ # 13007854817840016671868 > UINT64_MAX, so attempts
+ # to cast to either int64 or uint64 will result in
+ # an OverflowError being raised.
+ msg = "|".join(
+ [
+ "Python int too large to convert to C long",
+ "long too big to convert",
+ "int too big to convert",
+ ]
+ )
+ err = OverflowError
+ if parser.engine == "pyarrow":
+ err = ValueError
+ msg = "The 'converters' option is not supported with the 'pyarrow' engine"
+
+ with pytest.raises(err, match=msg):
+ parser.read_csv(StringIO(data), converters={"ID": conv})
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+@pytest.mark.parametrize(
+ "val", [np.iinfo(np.uint64).max, np.iinfo(np.int64).max, np.iinfo(np.int64).min]
+)
+def test_int64_uint64_range(all_parsers, val):
+ # These numbers fall right inside the int64-uint64
+ # range, so they should be parsed as integer.
+ parser = all_parsers
+ result = parser.read_csv(StringIO(str(val)), header=None)
+
+ expected = DataFrame([val])
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+@pytest.mark.parametrize(
+ "val", [np.iinfo(np.uint64).max + 1, np.iinfo(np.int64).min - 1]
+)
+def test_outside_int64_uint64_range(all_parsers, val, request):
+ # These numbers fall just outside the int64-uint64
+ # range, so they should be parsed as object.
+ parser = all_parsers
+
+ result = parser.read_csv(StringIO(str(val)), header=None)
+
+ expected = DataFrame([val])
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+@pytest.mark.parametrize(
+ "val", [np.iinfo(np.uint64).max + 1, np.iinfo(np.int64).min - 1]
+)
+def test_outside_int64_uint64_range_follow_str(all_parsers, val):
+ parser = all_parsers
+
+ result = parser.read_csv(StringIO(f"{val}\nabc"), header=None)
+
+ expected = DataFrame([str(val), "abc"])
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # gets float64 dtype instead of object
+@pytest.mark.parametrize("exp_data", [[str(-1), str(2**63)], [str(2**63), str(-1)]])
+def test_numeric_range_too_wide(all_parsers, exp_data):
+ # No numerical dtype can hold both negative and uint64
+ # values, so they should be cast as string.
+ parser = all_parsers
+ data = "\n".join(exp_data)
+ expected = DataFrame(exp_data)
+
+ result = parser.read_csv(StringIO(data), header=None)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_integer_precision(all_parsers):
+ # Gh 7072
+ s = """1,1;0;0;0;1;1;3844;3844;3844;1;1;1;1;1;1;0;0;1;1;0;0,,,4321583677327450765
+5,1;0;0;0;1;1;843;843;843;1;1;1;1;1;1;0;0;1;1;0;0,64.0,;,4321113141090630389"""
+ parser = all_parsers
+ result = parser.read_csv(StringIO(s), header=None)[4]
+ expected = Series([4321583677327450765, 4321113141090630389], name=4)
+ tm.assert_series_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_iterator.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_iterator.py
new file mode 100644
index 0000000000000000000000000000000000000000..5841fb7ad9594a2fb6aa7191d56829c462457a4b
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_iterator.py
@@ -0,0 +1,159 @@
+"""
+Tests that work on both the Python and C engines but do not have a
+specific classification into the other test modules.
+"""
+
+from io import StringIO
+
+import pytest
+
+from pandas import (
+ DataFrame,
+ concat,
+)
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+def test_iterator(all_parsers):
+ # see gh-6607
+ data = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+"""
+ parser = all_parsers
+ kwargs = {"index_col": 0}
+
+ expected = parser.read_csv(StringIO(data), **kwargs)
+
+ if parser.engine == "pyarrow":
+ msg = "The 'iterator' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), iterator=True, **kwargs)
+ return
+
+ with parser.read_csv(StringIO(data), iterator=True, **kwargs) as reader:
+ first_chunk = reader.read(3)
+ tm.assert_frame_equal(first_chunk, expected[:3])
+
+ last_chunk = reader.read(5)
+ tm.assert_frame_equal(last_chunk, expected[3:])
+
+
+def test_iterator2(all_parsers):
+ parser = all_parsers
+ data = """A,B,C
+foo,1,2,3
+bar,4,5,6
+baz,7,8,9
+"""
+
+ if parser.engine == "pyarrow":
+ msg = "The 'iterator' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), iterator=True)
+ return
+
+ with parser.read_csv(StringIO(data), iterator=True) as reader:
+ result = list(reader)
+
+ expected = DataFrame(
+ [[1, 2, 3], [4, 5, 6], [7, 8, 9]],
+ index=["foo", "bar", "baz"],
+ columns=["A", "B", "C"],
+ )
+ tm.assert_frame_equal(result[0], expected)
+
+
+def test_iterator_stop_on_chunksize(all_parsers):
+ # gh-3967: stopping iteration when chunksize is specified
+ parser = all_parsers
+ data = """A,B,C
+foo,1,2,3
+bar,4,5,6
+baz,7,8,9
+"""
+ if parser.engine == "pyarrow":
+ msg = "The 'chunksize' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), chunksize=1)
+ return
+
+ with parser.read_csv(StringIO(data), chunksize=1) as reader:
+ result = list(reader)
+
+ assert len(result) == 3
+ expected = DataFrame(
+ [[1, 2, 3], [4, 5, 6], [7, 8, 9]],
+ index=["foo", "bar", "baz"],
+ columns=["A", "B", "C"],
+ )
+ tm.assert_frame_equal(concat(result), expected)
+
+
+def test_nrows_iterator_without_chunksize(all_parsers):
+ # GH 59079
+ parser = all_parsers
+ data = """A,B,C
+foo,1,2,3
+bar,4,5,6
+baz,7,8,9
+"""
+ if parser.engine == "pyarrow":
+ msg = "The 'iterator' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), iterator=True, nrows=2)
+ return
+
+ with parser.read_csv(StringIO(data), iterator=True, nrows=2) as reader:
+ result = reader.get_chunk()
+
+ expected = DataFrame(
+ [[1, 2, 3], [4, 5, 6]],
+ index=["foo", "bar"],
+ columns=["A", "B", "C"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "kwargs", [{"iterator": True, "chunksize": 1}, {"iterator": True}, {"chunksize": 1}]
+)
+def test_iterator_skipfooter_errors(all_parsers, kwargs):
+ msg = "'skipfooter' not supported for iteration"
+ parser = all_parsers
+ data = "a\n1\n2"
+
+ if parser.engine == "pyarrow":
+ msg = (
+ "The '(chunksize|iterator)' option is not supported with the "
+ "'pyarrow' engine"
+ )
+
+ with pytest.raises(ValueError, match=msg):
+ with parser.read_csv(StringIO(data), skipfooter=1, **kwargs) as _:
+ pass
+
+
+def test_iteration_open_handle(temp_file, all_parsers):
+ parser = all_parsers
+ kwargs = {"header": None}
+
+ with open(temp_file, "w", encoding="utf-8") as f:
+ f.write("AAA\nBBB\nCCC\nDDD\nEEE\nFFF\nGGG")
+
+ with open(temp_file, encoding="utf-8") as f:
+ for line in f:
+ if "CCC" in line:
+ break
+
+ result = parser.read_csv(f, **kwargs)
+ expected = DataFrame({0: ["DDD", "EEE", "FFF", "GGG"]})
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_read_errors.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_read_errors.py
new file mode 100644
index 0000000000000000000000000000000000000000..b8cf435ef0443a17f71b48626579a7acd62b333e
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/common/test_read_errors.py
@@ -0,0 +1,315 @@
+"""
+Tests that work on the Python, C and PyArrow engines but do not have a
+specific classification into the other test modules.
+"""
+
+import codecs
+import csv
+from io import StringIO
+import os
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+from pandas.errors import (
+ EmptyDataError,
+ ParserError,
+ ParserWarning,
+)
+
+from pandas import DataFrame
+import pandas._testing as tm
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+
+def test_empty_decimal_marker(all_parsers):
+ data = """A|B|C
+1|2,334|5
+10|13|10.
+"""
+ # Parsers support only length-1 decimals
+ msg = "Only length-1 decimal markers supported"
+ parser = all_parsers
+
+ if parser.engine == "pyarrow":
+ msg = (
+ "only single character unicode strings can be "
+ "converted to Py_UCS4, got length 0"
+ )
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), decimal="")
+
+
+def test_bad_stream_exception(all_parsers, csv_dir_path):
+ # see gh-13652
+ #
+ # This test validates that both the Python engine and C engine will
+ # raise UnicodeDecodeError instead of C engine raising ParserError
+ # and swallowing the exception that caused read to fail.
+ path = os.path.join(csv_dir_path, "sauron.SHIFT_JIS.csv")
+ codec = codecs.lookup("utf-8")
+ utf8 = codecs.lookup("utf-8")
+ parser = all_parsers
+ msg = "'utf-8' codec can't decode byte"
+
+ # Stream must be binary UTF8.
+ with (
+ open(path, "rb") as handle,
+ codecs.StreamRecoder(
+ handle, utf8.encode, utf8.decode, codec.streamreader, codec.streamwriter
+ ) as stream,
+ ):
+ with pytest.raises(UnicodeDecodeError, match=msg):
+ parser.read_csv(stream)
+
+
+def test_malformed(all_parsers):
+ # see gh-6607
+ parser = all_parsers
+ data = """ignore
+A,B,C
+1,2,3 # comment
+1,2,3,4,5
+2,3,4
+"""
+ msg = "Expected 3 fields in line 4, saw 5"
+ err = ParserError
+ if parser.engine == "pyarrow":
+ msg = "The 'comment' option is not supported with the 'pyarrow' engine"
+ err = ValueError
+ with pytest.raises(err, match=msg):
+ parser.read_csv(StringIO(data), header=1, comment="#")
+
+
+@pytest.mark.parametrize("nrows", [5, 3, None])
+def test_malformed_chunks(all_parsers, nrows):
+ data = """ignore
+A,B,C
+skip
+1,2,3
+3,5,10 # comment
+1,2,3,4,5
+2,3,4
+"""
+ parser = all_parsers
+
+ if parser.engine == "pyarrow":
+ msg = "The 'iterator' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data),
+ header=1,
+ comment="#",
+ iterator=True,
+ chunksize=1,
+ skiprows=[2],
+ )
+ return
+
+ msg = "Expected 3 fields in line 6, saw 5"
+ with parser.read_csv(
+ StringIO(data), header=1, comment="#", iterator=True, chunksize=1, skiprows=[2]
+ ) as reader:
+ with pytest.raises(ParserError, match=msg):
+ reader.read(nrows)
+
+
+@xfail_pyarrow # does not raise
+def test_catch_too_many_names(all_parsers):
+ # see gh-5156
+ data = """\
+1,2,3
+4,,6
+7,8,9
+10,11,12\n"""
+ parser = all_parsers
+ msg = (
+ "Too many columns specified: expected 4 and found 3"
+ if parser.engine == "c"
+ else "Number of passed names did not match number of header fields in the file"
+ )
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), header=0, names=["a", "b", "c", "d"])
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+@pytest.mark.parametrize("nrows", [0, 1, 2, 3, 4, 5])
+def test_raise_on_no_columns(all_parsers, nrows):
+ parser = all_parsers
+ data = "\n" * nrows
+
+ msg = "No columns to parse from file"
+ with pytest.raises(EmptyDataError, match=msg):
+ parser.read_csv(StringIO(data))
+
+
+def test_unexpected_keyword_parameter_exception(all_parsers):
+ # GH-34976
+ parser = all_parsers
+
+ msg = "{}\\(\\) got an unexpected keyword argument 'foo'"
+ with pytest.raises(TypeError, match=msg.format("read_csv")):
+ parser.read_csv("foo.csv", foo=1)
+ with pytest.raises(TypeError, match=msg.format("read_table")):
+ parser.read_table("foo.tsv", foo=1)
+
+
+def test_suppress_error_output(all_parsers):
+ # see gh-15925
+ parser = all_parsers
+ data = "a\n1\n1,2,3\n4\n5,6,7"
+ expected = DataFrame({"a": [1, 4]})
+
+ result = parser.read_csv(StringIO(data), on_bad_lines="skip")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_error_bad_lines(all_parsers):
+ # see gh-15925
+ parser = all_parsers
+ data = "a\n1\n1,2,3\n4\n5,6,7"
+
+ msg = "Expected 1 fields in line 3, saw 3"
+
+ if parser.engine == "pyarrow":
+ # "CSV parse error: Expected 1 columns, got 3: 1,2,3"
+ pytest.skip(reason="https://github.com/apache/arrow/issues/38676")
+
+ with pytest.raises(ParserError, match=msg):
+ parser.read_csv(StringIO(data), on_bad_lines="error")
+
+
+def test_warn_bad_lines(all_parsers):
+ # see gh-15925
+ parser = all_parsers
+ data = "a\n1\n1,2,3\n4\n5,6,7"
+ expected = DataFrame({"a": [1, 4]})
+ match_msg = "Skipping line"
+
+ expected_warning = ParserWarning
+ if parser.engine == "pyarrow":
+ match_msg = "Expected 1 columns, but found 3: 1,2,3"
+
+ with tm.assert_produces_warning(
+ expected_warning, match=match_msg, check_stacklevel=False
+ ):
+ result = parser.read_csv(StringIO(data), on_bad_lines="warn")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_csv_wrong_num_columns(all_parsers):
+ # Too few columns.
+ data = """A,B,C,D,E,F
+1,2,3,4,5,6
+6,7,8,9,10,11,12
+11,12,13,14,15,16
+"""
+ parser = all_parsers
+ msg = "Expected 6 fields in line 3, saw 7"
+
+ if parser.engine == "pyarrow":
+ # Expected 6 columns, got 7: 6,7,8,9,10,11,12
+ pytest.skip(reason="https://github.com/apache/arrow/issues/38676")
+
+ with pytest.raises(ParserError, match=msg):
+ parser.read_csv(StringIO(data))
+
+
+def test_null_byte_char(request, all_parsers):
+ # see gh-2741
+ data = "\x00,foo"
+ names = ["a", "b"]
+ parser = all_parsers
+
+ if parser.engine in ["c", "python"]:
+ if parser.engine == "python":
+ request.applymarker(
+ pytest.mark.xfail(reason="This is read as an empty character not null")
+ )
+ expected = DataFrame([[np.nan, "foo"]], columns=names)
+ out = parser.read_csv(StringIO(data), names=names)
+ tm.assert_frame_equal(out, expected)
+ else:
+ if parser.engine == "pyarrow":
+ # CSV parse error: Empty CSV file or block: "
+ # cannot infer number of columns"
+ pytest.skip(reason="https://github.com/apache/arrow/issues/38676")
+ else:
+ msg = "NULL byte detected"
+ with pytest.raises(ParserError, match=msg):
+ parser.read_csv(StringIO(data), names=names)
+
+
+@pytest.mark.filterwarnings("always::ResourceWarning")
+def test_open_file(all_parsers, temp_file):
+ # GH 39024
+ parser = all_parsers
+
+ msg = "Could not determine delimiter"
+ err = csv.Error
+ if parser.engine == "c":
+ msg = "object of type 'NoneType' has no len"
+ err = TypeError
+ elif parser.engine == "pyarrow":
+ msg = "'utf-8' codec can't decode byte 0xe4"
+ err = ValueError
+
+ file = Path(temp_file)
+ file.write_bytes(b"\xe4\na\n1")
+
+ with tm.assert_produces_warning(None):
+ # should not trigger a ResourceWarning
+ with pytest.raises(err, match=msg):
+ parser.read_csv(file, sep=None, encoding_errors="replace")
+
+
+def test_invalid_on_bad_line(all_parsers):
+ parser = all_parsers
+ data = "a\n1\n1,2,3\n4\n5,6,7"
+ with pytest.raises(ValueError, match="Argument abc is invalid for on_bad_lines"):
+ parser.read_csv(StringIO(data), on_bad_lines="abc")
+
+
+def test_bad_header_uniform_error(all_parsers):
+ parser = all_parsers
+ data = "+++123456789...\ncol1,col2,col3,col4\n1,2,3,4\n"
+ msg = "Expected 2 fields in line 2, saw 4"
+ if parser.engine == "c":
+ msg = (
+ "Could not construct index. Requested to use 1 "
+ "number of columns, but 3 left to parse."
+ )
+ elif parser.engine == "pyarrow":
+ # "CSV parse error: Expected 1 columns, got 4: col1,col2,col3,col4"
+ pytest.skip(reason="https://github.com/apache/arrow/issues/38676")
+
+ with pytest.raises(ParserError, match=msg):
+ parser.read_csv(StringIO(data), index_col=0, on_bad_lines="error")
+
+
+def test_on_bad_lines_warn_correct_formatting(all_parsers):
+ # see gh-15925
+ parser = all_parsers
+ data = """1,2
+a,b
+a,b,c
+a,b,d
+a,b
+"""
+ expected = DataFrame({"1": "a", "2": ["b"] * 2})
+ match_msg = "Skipping line"
+
+ expected_warning = ParserWarning
+ if parser.engine == "pyarrow":
+ match_msg = "Expected 2 columns, but found 3: a,b,c"
+
+ with tm.assert_produces_warning(
+ expected_warning, match=match_msg, check_stacklevel=False
+ ):
+ result = parser.read_csv(StringIO(data), on_bad_lines="warn")
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/conftest.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..90f77a7024235f3458f96ba4e2938b7c28f966b4
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/conftest.py
@@ -0,0 +1,337 @@
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from pandas.compat import HAS_PYARROW
+from pandas.compat._optional import VERSIONS
+
+from pandas import (
+ read_csv,
+ read_table,
+)
+import pandas._testing as tm
+
+
+class BaseParser:
+ engine: str | None = None
+ low_memory = True
+ float_precision_choices: list[str | None] = []
+
+ def update_kwargs(self, kwargs):
+ kwargs = kwargs.copy()
+ kwargs.update({"engine": self.engine, "low_memory": self.low_memory})
+
+ return kwargs
+
+ def read_csv(self, *args, **kwargs):
+ kwargs = self.update_kwargs(kwargs)
+ return read_csv(*args, **kwargs)
+
+ def read_csv_check_warnings(
+ self,
+ warn_type: type[Warning],
+ warn_msg: str,
+ *args,
+ raise_on_extra_warnings=True,
+ check_stacklevel: bool = True,
+ **kwargs,
+ ):
+ # We need to check the stacklevel here instead of in the tests
+ # since this is where read_csv is called and where the warning
+ # should point to.
+ kwargs = self.update_kwargs(kwargs)
+ with tm.assert_produces_warning(
+ warn_type,
+ match=warn_msg,
+ raise_on_extra_warnings=raise_on_extra_warnings,
+ check_stacklevel=check_stacklevel,
+ ):
+ return read_csv(*args, **kwargs)
+
+ def read_table(self, *args, **kwargs):
+ kwargs = self.update_kwargs(kwargs)
+ return read_table(*args, **kwargs)
+
+ def read_table_check_warnings(
+ self,
+ warn_type: type[Warning],
+ warn_msg: str,
+ *args,
+ raise_on_extra_warnings=True,
+ **kwargs,
+ ):
+ # We need to check the stacklevel here instead of in the tests
+ # since this is where read_table is called and where the warning
+ # should point to.
+ kwargs = self.update_kwargs(kwargs)
+ with tm.assert_produces_warning(
+ warn_type, match=warn_msg, raise_on_extra_warnings=raise_on_extra_warnings
+ ):
+ return read_table(*args, **kwargs)
+
+
+class CParser(BaseParser):
+ engine = "c"
+ float_precision_choices = [None, "high", "round_trip"]
+
+
+class CParserHighMemory(CParser):
+ low_memory = False
+
+
+class CParserLowMemory(CParser):
+ low_memory = True
+
+
+class PythonParser(BaseParser):
+ engine = "python"
+ float_precision_choices = [None]
+
+
+class PyArrowParser(BaseParser):
+ engine = "pyarrow"
+ float_precision_choices = [None]
+
+
+@pytest.fixture
+def csv_dir_path(datapath):
+ """
+ The directory path to the data files needed for parser tests.
+ """
+ return datapath("io", "parser", "data")
+
+
+@pytest.fixture
+def csv1(datapath):
+ """
+ The path to the data file "test1.csv" needed for parser tests.
+ """
+ return os.path.join(datapath("io", "data", "csv"), "test1.csv")
+
+
+_cParserHighMemory = CParserHighMemory
+_cParserLowMemory = CParserLowMemory
+_pythonParser = PythonParser
+_pyarrowParser = PyArrowParser
+
+_py_parsers_only = [_pythonParser]
+_c_parsers_only = [_cParserHighMemory, _cParserLowMemory]
+_pyarrow_parsers_only = [
+ pytest.param(
+ _pyarrowParser,
+ marks=[
+ pytest.mark.single_cpu,
+ pytest.mark.skipif(not HAS_PYARROW, reason="pyarrow is not installed"),
+ ],
+ )
+]
+
+_all_parsers = [*_c_parsers_only, *_py_parsers_only, *_pyarrow_parsers_only]
+
+_py_parser_ids = ["python"]
+_c_parser_ids = ["c_high", "c_low"]
+_pyarrow_parsers_ids = ["pyarrow"]
+
+_all_parser_ids = [*_c_parser_ids, *_py_parser_ids, *_pyarrow_parsers_ids]
+
+
+@pytest.fixture(params=_all_parsers, ids=_all_parser_ids)
+def all_parsers(request):
+ """
+ Fixture all of the CSV parsers.
+ """
+ parser = request.param()
+ if parser.engine == "pyarrow":
+ pytest.importorskip("pyarrow", VERSIONS["pyarrow"])
+ # Try finding a way to disable threads all together
+ # for more stable CI runs
+ import pyarrow
+
+ pyarrow.set_cpu_count(1)
+ return parser
+
+
+@pytest.fixture(params=_c_parsers_only, ids=_c_parser_ids)
+def c_parser_only(request):
+ """
+ Fixture all of the CSV parsers using the C engine.
+ """
+ return request.param()
+
+
+@pytest.fixture(params=_py_parsers_only, ids=_py_parser_ids)
+def python_parser_only(request):
+ """
+ Fixture all of the CSV parsers using the Python engine.
+ """
+ return request.param()
+
+
+@pytest.fixture(params=_pyarrow_parsers_only, ids=_pyarrow_parsers_ids)
+def pyarrow_parser_only(request):
+ """
+ Fixture all of the CSV parsers using the Pyarrow engine.
+ """
+ return request.param()
+
+
+def _get_all_parser_float_precision_combinations():
+ """
+ Return all allowable parser and float precision
+ combinations and corresponding ids.
+ """
+ params = []
+ ids = []
+ for parser, parser_id in zip(_all_parsers, _all_parser_ids):
+ if hasattr(parser, "values"):
+ # Wrapped in pytest.param, get the actual parser back
+ parser = parser.values[0]
+ for precision in parser.float_precision_choices:
+ # Re-wrap in pytest.param for pyarrow
+ mark = (
+ [
+ pytest.mark.single_cpu,
+ pytest.mark.skipif(
+ not HAS_PYARROW, reason="pyarrow is not installed"
+ ),
+ ]
+ if parser.engine == "pyarrow"
+ else ()
+ )
+ param = pytest.param((parser(), precision), marks=mark)
+ params.append(param)
+ ids.append(f"{parser_id}-{precision}")
+
+ return {"params": params, "ids": ids}
+
+
+@pytest.fixture(
+ params=_get_all_parser_float_precision_combinations()["params"],
+ ids=_get_all_parser_float_precision_combinations()["ids"],
+)
+def all_parsers_all_precisions(request):
+ """
+ Fixture for all allowable combinations of parser
+ and float precision
+ """
+ return request.param
+
+
+_utf_values = [8, 16, 32]
+
+_encoding_seps = ["", "-", "_"]
+_encoding_prefixes = ["utf", "UTF"]
+
+_encoding_fmts = [
+ f"{prefix}{sep}{{0}}" for sep in _encoding_seps for prefix in _encoding_prefixes
+]
+
+
+@pytest.fixture(params=_utf_values)
+def utf_value(request):
+ """
+ Fixture for all possible integer values for a UTF encoding.
+ """
+ return request.param
+
+
+@pytest.fixture(params=_encoding_fmts)
+def encoding_fmt(request):
+ """
+ Fixture for all possible string formats of a UTF encoding.
+ """
+ return request.param
+
+
+@pytest.fixture(
+ params=[
+ ("-1,0", -1.0),
+ ("-1,2e0", -1.2),
+ ("-1e0", -1.0),
+ ("+1e0", 1.0),
+ ("+1e+0", 1.0),
+ ("+1e-1", 0.1),
+ ("+,1e1", 1.0),
+ ("+1,e0", 1.0),
+ ("-,1e1", -1.0),
+ ("-1,e0", -1.0),
+ ("0,1", 0.1),
+ ("1,", 1.0),
+ (",1", 0.1),
+ ("-,1", -0.1),
+ ("1_,", 1.0),
+ ("1_234,56", 1234.56),
+ ("1_234,56e0", 1234.56),
+ # negative cases; must not parse as float
+ ("_", "_"),
+ ("-_", "-_"),
+ ("-_1", "-_1"),
+ ("-_1e0", "-_1e0"),
+ ("_1", "_1"),
+ ("_1,", "_1,"),
+ ("_1,_", "_1,_"),
+ ("_1e0", "_1e0"),
+ ("1,2e_1", "1,2e_1"),
+ ("1,2e1_0", "1,2e1_0"),
+ ("1,_2", "1,_2"),
+ (",1__2", ",1__2"),
+ (",1e", ",1e"),
+ ("-,1e", "-,1e"),
+ ("1_000,000_000", "1_000,000_000"),
+ ("1,e1_2", "1,e1_2"),
+ ("e11,2", "e11,2"),
+ ("1e11,2", "1e11,2"),
+ ("1,2,2", "1,2,2"),
+ ("1,2_1", "1,2_1"),
+ ("1,2e-10e1", "1,2e-10e1"),
+ ("--1,2", "--1,2"),
+ ("1a_2,1", "1a_2,1"),
+ ("1,2E-1", 0.12),
+ ("1,2E1", 12.0),
+ ]
+)
+def numeric_decimal(request):
+ """
+ Fixture for all numeric formats which should get recognized. The first entry
+ represents the value to read while the second represents the expected result.
+ """
+ return request.param
+
+
+@pytest.fixture
+def pyarrow_xfail(request):
+ """
+ Fixture that xfails a test if the engine is pyarrow.
+
+ Use if failure is do to unsupported keywords or inconsistent results.
+ """
+ if "all_parsers" in request.fixturenames:
+ parser = request.getfixturevalue("all_parsers")
+ elif "all_parsers_all_precisions" in request.fixturenames:
+ # Return value is tuple of (engine, precision)
+ parser = request.getfixturevalue("all_parsers_all_precisions")[0]
+ else:
+ return
+ if parser.engine == "pyarrow":
+ mark = pytest.mark.xfail(reason="pyarrow doesn't support this.")
+ request.applymarker(mark)
+
+
+@pytest.fixture
+def pyarrow_skip(request):
+ """
+ Fixture that skips a test if the engine is pyarrow.
+
+ Use if failure is do a parsing failure from pyarrow.csv.read_csv
+ """
+ if "all_parsers" in request.fixturenames:
+ parser = request.getfixturevalue("all_parsers")
+ elif "all_parsers_all_precisions" in request.fixturenames:
+ # Return value is tuple of (engine, precision)
+ parser = request.getfixturevalue("all_parsers_all_precisions")[0]
+ else:
+ return
+ if parser.engine == "pyarrow":
+ pytest.skip(reason="https://github.com/apache/arrow/issues/38676")
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/dtypes/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/dtypes/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/dtypes/test_categorical.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/dtypes/test_categorical.py
new file mode 100644
index 0000000000000000000000000000000000000000..c157510e552b4ebe0f59b994b4c5c9dcda34be0b
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/dtypes/test_categorical.py
@@ -0,0 +1,338 @@
+"""
+Tests dtype specification during parsing
+for all of the parsers defined in parsers.py
+"""
+
+from io import StringIO
+import os
+
+import numpy as np
+import pytest
+
+from pandas._libs import parsers as libparsers
+from pandas.errors import Pandas4Warning
+
+from pandas.core.dtypes.dtypes import CategoricalDtype
+
+import pandas as pd
+from pandas import (
+ Categorical,
+ DataFrame,
+ Timestamp,
+)
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+
+
+@xfail_pyarrow # AssertionError: Attributes of DataFrame.iloc[:, 0] are different
+@pytest.mark.parametrize(
+ "dtype",
+ [
+ "category",
+ CategoricalDtype(),
+ {"a": "category", "b": "category", "c": CategoricalDtype()},
+ ],
+)
+def test_categorical_dtype(all_parsers, dtype):
+ # see gh-10153
+ parser = all_parsers
+ data = """a,b,c
+1,a,3.4
+1,a,3.4
+2,b,4.5"""
+ expected = DataFrame(
+ {
+ "a": Categorical(["1", "1", "2"]),
+ "b": Categorical(["a", "a", "b"]),
+ "c": Categorical(["3.4", "3.4", "4.5"]),
+ }
+ )
+ actual = parser.read_csv(StringIO(data), dtype=dtype)
+ tm.assert_frame_equal(actual, expected)
+
+
+@pytest.mark.parametrize("dtype", [{"b": "category"}, {1: "category"}])
+def test_categorical_dtype_single(all_parsers, dtype, request):
+ # see gh-10153
+ parser = all_parsers
+ data = """a,b,c
+1,a,3.4
+1,a,3.4
+2,b,4.5"""
+ expected = DataFrame(
+ {"a": [1, 1, 2], "b": Categorical(["a", "a", "b"]), "c": [3.4, 3.4, 4.5]}
+ )
+ if parser.engine == "pyarrow":
+ mark = pytest.mark.xfail(
+ strict=False,
+ reason="Flaky test sometimes gives object dtype instead of Categorical",
+ )
+ request.applymarker(mark)
+
+ actual = parser.read_csv(StringIO(data), dtype=dtype)
+ tm.assert_frame_equal(actual, expected)
+
+
+@xfail_pyarrow # AssertionError: Attributes of DataFrame.iloc[:, 0] are different
+def test_categorical_dtype_unsorted(all_parsers):
+ # see gh-10153
+ parser = all_parsers
+ data = """a,b,c
+1,b,3.4
+1,b,3.4
+2,a,4.5"""
+ expected = DataFrame(
+ {
+ "a": Categorical(["1", "1", "2"]),
+ "b": Categorical(["b", "b", "a"]),
+ "c": Categorical(["3.4", "3.4", "4.5"]),
+ }
+ )
+ actual = parser.read_csv(StringIO(data), dtype="category")
+ tm.assert_frame_equal(actual, expected)
+
+
+@xfail_pyarrow # AssertionError: Attributes of DataFrame.iloc[:, 0] are different
+def test_categorical_dtype_missing(all_parsers):
+ # see gh-10153
+ parser = all_parsers
+ data = """a,b,c
+1,b,3.4
+1,nan,3.4
+2,a,4.5"""
+ expected = DataFrame(
+ {
+ "a": Categorical(["1", "1", "2"]),
+ "b": Categorical(["b", np.nan, "a"]),
+ "c": Categorical(["3.4", "3.4", "4.5"]),
+ }
+ )
+ actual = parser.read_csv(StringIO(data), dtype="category")
+ tm.assert_frame_equal(actual, expected)
+
+
+@xfail_pyarrow # AssertionError: Attributes of DataFrame.iloc[:, 0] are different
+@pytest.mark.slow
+def test_categorical_dtype_high_cardinality_numeric(all_parsers, monkeypatch):
+ # see gh-18186
+ # was an issue with C parser, due to DEFAULT_BUFFER_HEURISTIC
+ parser = all_parsers
+ heuristic = 2**5
+ data = np.sort([str(i) for i in range(heuristic + 1)])
+ expected = DataFrame({"a": Categorical(data, ordered=True)})
+ with monkeypatch.context() as m:
+ m.setattr(libparsers, "DEFAULT_BUFFER_HEURISTIC", heuristic)
+ actual = parser.read_csv(StringIO("a\n" + "\n".join(data)), dtype="category")
+ actual["a"] = actual["a"].cat.reorder_categories(
+ np.sort(actual.a.cat.categories), ordered=True
+ )
+ tm.assert_frame_equal(actual, expected)
+
+
+def test_categorical_dtype_utf16(all_parsers, csv_dir_path):
+ # see gh-10153
+ pth = os.path.join(csv_dir_path, "utf16_ex.txt")
+ parser = all_parsers
+ encoding = "utf-16"
+ sep = "\t"
+
+ expected = parser.read_csv(pth, sep=sep, encoding=encoding)
+ expected = expected.apply(Categorical)
+
+ actual = parser.read_csv(pth, sep=sep, encoding=encoding, dtype="category")
+ tm.assert_frame_equal(actual, expected)
+
+
+def test_categorical_dtype_chunksize_infer_categories(all_parsers):
+ # see gh-10153
+ parser = all_parsers
+ data = """a,b
+1,a
+1,b
+1,b
+2,c"""
+ expecteds = [
+ DataFrame({"a": [1, 1], "b": Categorical(["a", "b"])}),
+ DataFrame({"a": [1, 2], "b": Categorical(["b", "c"])}, index=[2, 3]),
+ ]
+
+ if parser.engine == "pyarrow":
+ msg = "The 'chunksize' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), dtype={"b": "category"}, chunksize=2)
+ return
+
+ with parser.read_csv(
+ StringIO(data), dtype={"b": "category"}, chunksize=2
+ ) as actuals:
+ for actual, expected in zip(actuals, expecteds):
+ tm.assert_frame_equal(actual, expected)
+
+
+def test_categorical_dtype_chunksize_explicit_categories(all_parsers):
+ # see gh-10153
+ parser = all_parsers
+ data = """a,b
+1,a
+1,b
+1,b
+2,c"""
+ cats = ["a", "b", "c"]
+ expecteds = [
+ DataFrame({"a": [1, 1], "b": Categorical(["a", "b"], categories=cats)}),
+ DataFrame(
+ {"a": [1, 2], "b": Categorical(["b", "c"], categories=cats)},
+ index=[2, 3],
+ ),
+ ]
+ dtype = CategoricalDtype(cats)
+
+ if parser.engine == "pyarrow":
+ msg = "The 'chunksize' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), dtype={"b": dtype}, chunksize=2)
+ return
+
+ with parser.read_csv(StringIO(data), dtype={"b": dtype}, chunksize=2) as actuals:
+ for actual, expected in zip(actuals, expecteds):
+ tm.assert_frame_equal(actual, expected)
+
+
+def test_categorical_dtype_latin1(all_parsers, csv_dir_path):
+ # see gh-10153
+ pth = os.path.join(csv_dir_path, "unicode_series.csv")
+ parser = all_parsers
+ encoding = "latin-1"
+
+ expected = parser.read_csv(pth, header=None, encoding=encoding)
+ expected[1] = Categorical(expected[1])
+
+ actual = parser.read_csv(pth, header=None, encoding=encoding, dtype={1: "category"})
+ tm.assert_frame_equal(actual, expected)
+
+
+@pytest.mark.parametrize("ordered", [False, True])
+@pytest.mark.parametrize(
+ "categories",
+ [["a", "b", "c"], ["a", "c", "b"], ["a", "b", "c", "d"], ["c", "b", "a"]],
+)
+def test_categorical_category_dtype(all_parsers, categories, ordered):
+ parser = all_parsers
+ data = """a,b
+1,a
+1,b
+1,b
+2,c"""
+ expected = DataFrame(
+ {
+ "a": [1, 1, 1, 2],
+ "b": Categorical(
+ ["a", "b", "b", "c"], categories=categories, ordered=ordered
+ ),
+ }
+ )
+
+ dtype = {"b": CategoricalDtype(categories=categories, ordered=ordered)}
+ result = parser.read_csv(StringIO(data), dtype=dtype)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_categorical_category_dtype_unsorted(all_parsers):
+ parser = all_parsers
+ data = """a,b
+1,a
+1,b
+1,b
+2,c"""
+ dtype = CategoricalDtype(["c", "b", "a"])
+ expected = DataFrame(
+ {
+ "a": [1, 1, 1, 2],
+ "b": Categorical(["a", "b", "b", "c"], categories=["c", "b", "a"]),
+ }
+ )
+
+ result = parser.read_csv(StringIO(data), dtype={"b": dtype})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_categorical_coerces_numeric(all_parsers):
+ parser = all_parsers
+ dtype = {"b": CategoricalDtype([1, 2, 3])}
+
+ data = "b\n1\n1\n2\n3"
+ expected = DataFrame({"b": Categorical([1, 1, 2, 3])})
+
+ result = parser.read_csv(StringIO(data), dtype=dtype)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_categorical_coerces_datetime(all_parsers):
+ parser = all_parsers
+ dti = pd.DatetimeIndex(["2017-01-01", "2018-01-01", "2019-01-01"], freq=None)
+ dtype = {"b": CategoricalDtype(dti)}
+
+ data = "b\n2017-01-01\n2018-01-01\n2019-01-01"
+ expected = DataFrame({"b": Categorical(dtype["b"].categories)})
+
+ result = parser.read_csv(StringIO(data), dtype=dtype)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_categorical_coerces_timestamp(all_parsers):
+ parser = all_parsers
+ dtype = {"b": CategoricalDtype([Timestamp("2014")])}
+
+ data = "b\n2014-01-01\n2014-01-01"
+ expected = DataFrame({"b": Categorical([Timestamp("2014")] * 2)})
+
+ result = parser.read_csv(StringIO(data), dtype=dtype)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_categorical_coerces_timedelta(all_parsers):
+ parser = all_parsers
+ dtype = {"b": CategoricalDtype(pd.to_timedelta(["1h", "2h", "3h"]))}
+
+ data = "b\n1h\n2h\n3h"
+ expected = DataFrame({"b": Categorical(dtype["b"].categories)})
+
+ result = parser.read_csv(StringIO(data), dtype=dtype)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "data",
+ [
+ "b\nTrue\nFalse\nNA\nFalse",
+ "b\ntrue\nfalse\nNA\nfalse",
+ "b\nTRUE\nFALSE\nNA\nFALSE",
+ "b\nTrue\nFalse\nNA\nFALSE",
+ ],
+)
+def test_categorical_dtype_coerces_boolean(all_parsers, data):
+ # see gh-20498
+ parser = all_parsers
+ dtype = {"b": CategoricalDtype([False, True])}
+ expected = DataFrame({"b": Categorical([True, False, None, False])})
+
+ result = parser.read_csv(StringIO(data), dtype=dtype)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_categorical_unexpected_categories(all_parsers):
+ parser = all_parsers
+ dtype = {"b": CategoricalDtype(["a", "b", "d", "e"])}
+
+ data = "b\nd\na\nc\nd" # Unexpected c
+ expected = DataFrame({"b": Categorical(["d", "a", None, "d"], dtype=dtype["b"])})
+
+ msg = "Constructing a Categorical with a dtype and values containing"
+ with tm.assert_produces_warning(Pandas4Warning, match=msg, check_stacklevel=False):
+ result = parser.read_csv(StringIO(data), dtype=dtype)
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/dtypes/test_dtypes_basic.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/dtypes/test_dtypes_basic.py
new file mode 100644
index 0000000000000000000000000000000000000000..786fda332ad1fcbcf28941cb34e42e8ec0b1cc6c
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/dtypes/test_dtypes_basic.py
@@ -0,0 +1,639 @@
+"""
+Tests dtype specification during parsing
+for all of the parsers defined in parsers.py
+"""
+
+from collections import defaultdict
+from io import StringIO
+
+import numpy as np
+import pytest
+
+from pandas.errors import ParserWarning
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Timestamp,
+)
+import pandas._testing as tm
+from pandas.core.arrays import IntegerArray
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+
+
+@pytest.mark.parametrize("dtype", [str, object])
+@pytest.mark.parametrize("check_orig", [True, False])
+@pytest.mark.usefixtures("pyarrow_xfail")
+def test_dtype_all_columns(
+ all_parsers, dtype, check_orig, using_infer_string, temp_file
+):
+ # see gh-3795, gh-6607
+ parser = all_parsers
+
+ df = DataFrame(
+ np.random.default_rng(2).random((5, 2)).round(4),
+ columns=list("AB"),
+ index=["1A", "1B", "1C", "1D", "1E"],
+ )
+
+ df.to_csv(temp_file)
+
+ result = parser.read_csv(temp_file, dtype=dtype, index_col=0)
+
+ if check_orig:
+ expected = df.copy()
+ result = result.astype(float)
+ elif using_infer_string and dtype is str:
+ expected = df.astype(str)
+ else:
+ expected = df.astype(str).astype(object)
+
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.usefixtures("pyarrow_xfail")
+def test_dtype_per_column(all_parsers):
+ parser = all_parsers
+ data = """\
+one,two
+1,2.5
+2,3.5
+3,4.5
+4,5.5"""
+ expected = DataFrame(
+ [[1, "2.5"], [2, "3.5"], [3, "4.5"], [4, "5.5"]], columns=["one", "two"]
+ )
+ expected["one"] = expected["one"].astype(np.float64)
+
+ result = parser.read_csv(StringIO(data), dtype={"one": np.float64, 1: str})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_invalid_dtype_per_column(all_parsers):
+ parser = all_parsers
+ data = """\
+one,two
+1,2.5
+2,3.5
+3,4.5
+4,5.5"""
+
+ with pytest.raises(TypeError, match="data type [\"']foo[\"'] not understood"):
+ parser.read_csv(StringIO(data), dtype={"one": "foo", 1: "int"})
+
+
+def test_raise_on_passed_int_dtype_with_nas(all_parsers):
+ # see gh-2631
+ parser = all_parsers
+ data = """YEAR, DOY, a
+2001,106380451,10
+2001,,11
+2001,106380451,67"""
+
+ if parser.engine == "c":
+ msg = "Integer column has NA values"
+ elif parser.engine == "pyarrow":
+ msg = "The 'skipinitialspace' option is not supported with the 'pyarrow' engine"
+ else:
+ msg = "Unable to convert column DOY"
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), dtype={"DOY": np.int64}, skipinitialspace=True)
+
+
+def test_dtype_with_converters(all_parsers):
+ parser = all_parsers
+ data = """a,b
+1.1,2.2
+1.2,2.3"""
+
+ if parser.engine == "pyarrow":
+ msg = "The 'converters' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data), dtype={"a": "i8"}, converters={"a": lambda x: str(x)}
+ )
+ return
+
+ # Dtype spec ignored if converted specified.
+ result = parser.read_csv_check_warnings(
+ ParserWarning,
+ "Both a converter and dtype were specified for column a "
+ "- only the converter will be used.",
+ StringIO(data),
+ dtype={"a": "i8"},
+ converters={"a": lambda x: str(x)},
+ )
+ expected = DataFrame({"a": ["1.1", "1.2"], "b": [2.2, 2.3]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_numeric_dtype(all_parsers, any_real_numpy_dtype):
+ data = "0\n1"
+ parser = all_parsers
+ expected = DataFrame([0, 1], dtype=any_real_numpy_dtype)
+
+ result = parser.read_csv(StringIO(data), header=None, dtype=any_real_numpy_dtype)
+ tm.assert_frame_equal(expected, result, check_column_type=False)
+
+
+@pytest.mark.usefixtures("pyarrow_xfail")
+def test_boolean_dtype(all_parsers):
+ parser = all_parsers
+ data = "\n".join(
+ [
+ "a",
+ "True",
+ "TRUE",
+ "true",
+ "1",
+ "1.0",
+ "False",
+ "FALSE",
+ "false",
+ "0",
+ "0.0",
+ "NaN",
+ "nan",
+ "NA",
+ "null",
+ "NULL",
+ ]
+ )
+
+ result = parser.read_csv(StringIO(data), dtype="boolean")
+ expected = DataFrame(
+ {
+ "a": pd.array(
+ [
+ True,
+ True,
+ True,
+ True,
+ True,
+ False,
+ False,
+ False,
+ False,
+ False,
+ None,
+ None,
+ None,
+ None,
+ None,
+ ],
+ dtype="boolean",
+ )
+ }
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.usefixtures("pyarrow_xfail")
+def test_delimiter_with_usecols_and_parse_dates(all_parsers):
+ # GH#35873
+ result = all_parsers.read_csv(
+ StringIO('"dump","-9,1","-9,1",20101010'),
+ engine="python",
+ names=["col", "col1", "col2", "col3"],
+ usecols=["col1", "col2", "col3"],
+ parse_dates=["col3"],
+ decimal=",",
+ )
+ expected = DataFrame(
+ {"col1": [-9.1], "col2": [-9.1], "col3": [Timestamp("2010-10-10")]}
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("thousands", ["_", None])
+def test_decimal_and_exponential(
+ request, python_parser_only, numeric_decimal, thousands
+):
+ # GH#31920
+ decimal_number_check(request, python_parser_only, numeric_decimal, thousands, None)
+
+
+@pytest.mark.parametrize("thousands", ["_", None])
+@pytest.mark.parametrize("float_precision", [None, "legacy", "high", "round_trip"])
+def test_1000_sep_decimal_float_precision(
+ request, c_parser_only, numeric_decimal, float_precision, thousands
+):
+ # test decimal and thousand sep handling in across 'float_precision'
+ # parsers
+ decimal_number_check(
+ request, c_parser_only, numeric_decimal, thousands, float_precision
+ )
+ text, value = numeric_decimal
+ text = " " + text + " "
+ if isinstance(value, str): # the negative cases (parse as text)
+ value = " " + value + " "
+ decimal_number_check(
+ request, c_parser_only, (text, value), thousands, float_precision
+ )
+
+
+def decimal_number_check(request, parser, numeric_decimal, thousands, float_precision):
+ # GH#31920
+ value = numeric_decimal[0]
+ if thousands is None and value in ("1_,", "1_234,56", "1_234,56e0"):
+ request.applymarker(
+ pytest.mark.xfail(reason=f"thousands={thousands} and sep is in {value}")
+ )
+ df = parser.read_csv(
+ StringIO(value),
+ float_precision=float_precision,
+ sep="|",
+ thousands=thousands,
+ decimal=",",
+ header=None,
+ )
+ val = df.iloc[0, 0]
+ assert val == numeric_decimal[1]
+
+
+@pytest.mark.parametrize("float_precision", [None, "legacy", "high", "round_trip"])
+def test_skip_whitespace(c_parser_only, float_precision):
+ DATA = """id\tnum\t
+1\t1.2 \t
+1\t 2.1\t
+2\t 1\t
+2\t 1.2 \t
+"""
+ df = c_parser_only.read_csv(
+ StringIO(DATA),
+ float_precision=float_precision,
+ sep="\t",
+ header=0,
+ dtype={1: np.float64},
+ )
+ tm.assert_series_equal(df.iloc[:, 1], pd.Series([1.2, 2.1, 1.0, 1.2], name="num"))
+
+
+@pytest.mark.usefixtures("pyarrow_xfail")
+def test_true_values_cast_to_bool(all_parsers):
+ # GH#34655
+ text = """a,b
+yes,xxx
+no,yyy
+1,zzz
+0,aaa
+ """
+ parser = all_parsers
+ result = parser.read_csv(
+ StringIO(text),
+ true_values=["yes"],
+ false_values=["no"],
+ dtype={"a": "boolean"},
+ )
+ expected = DataFrame(
+ {"a": [True, False, True, False], "b": ["xxx", "yyy", "zzz", "aaa"]}
+ )
+ expected["a"] = expected["a"].astype("boolean")
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.usefixtures("pyarrow_xfail")
+@pytest.mark.parametrize("dtypes, exp_value", [({}, "1"), ({"a.1": "int64"}, 1)])
+def test_dtype_mangle_dup_cols(all_parsers, dtypes, exp_value):
+ # GH#35211
+ parser = all_parsers
+ data = """a,a\n1,1"""
+ dtype_dict = {"a": str, **dtypes}
+ # GH#42462
+ dtype_dict_copy = dtype_dict.copy()
+ result = parser.read_csv(StringIO(data), dtype=dtype_dict)
+ expected = DataFrame({"a": ["1"], "a.1": [exp_value]})
+ assert dtype_dict == dtype_dict_copy, "dtype dict changed"
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.usefixtures("pyarrow_xfail")
+def test_dtype_mangle_dup_cols_single_dtype(all_parsers):
+ # GH#42022
+ parser = all_parsers
+ data = """a,a\n1,1"""
+ result = parser.read_csv(StringIO(data), dtype=str)
+ expected = DataFrame({"a": ["1"], "a.1": ["1"]})
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.usefixtures("pyarrow_xfail")
+def test_dtype_multi_index(all_parsers):
+ # GH 42446
+ parser = all_parsers
+ data = "A,B,B\nX,Y,Z\n1,2,3"
+
+ result = parser.read_csv(
+ StringIO(data),
+ header=list(range(2)),
+ dtype={
+ ("A", "X"): np.int32,
+ ("B", "Y"): np.int32,
+ ("B", "Z"): np.float32,
+ },
+ )
+
+ expected = DataFrame(
+ {
+ ("A", "X"): np.int32([1]),
+ ("B", "Y"): np.int32([2]),
+ ("B", "Z"): np.float32([3]),
+ }
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_nullable_int_dtype(all_parsers, any_int_ea_dtype):
+ # GH 25472
+ parser = all_parsers
+ dtype = any_int_ea_dtype
+
+ data = """a,b,c
+,3,5
+1,,6
+2,4,"""
+ expected = DataFrame(
+ {
+ "a": pd.array([pd.NA, 1, 2], dtype=dtype),
+ "b": pd.array([3, pd.NA, 4], dtype=dtype),
+ "c": pd.array([5, 6, pd.NA], dtype=dtype),
+ }
+ )
+ actual = parser.read_csv(StringIO(data), dtype=dtype)
+ tm.assert_frame_equal(actual, expected)
+
+
+@pytest.mark.usefixtures("pyarrow_xfail")
+@pytest.mark.parametrize("default", ["float", "float64"])
+def test_dtypes_defaultdict(all_parsers, default):
+ # GH#41574
+ data = """a,b
+1,2
+"""
+ dtype = defaultdict(lambda: default, a="int64")
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), dtype=dtype)
+ expected = DataFrame({"a": [1], "b": 2.0})
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.usefixtures("pyarrow_xfail")
+def test_dtypes_defaultdict_mangle_dup_cols(all_parsers):
+ # GH#41574
+ data = """a,b,a,b,b.1
+1,2,3,4,5
+"""
+ dtype = defaultdict(lambda: "float64", a="int64")
+ dtype["b.1"] = "int64"
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), dtype=dtype)
+ expected = DataFrame({"a": [1], "b": [2.0], "a.1": [3], "b.2": [4.0], "b.1": [5]})
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.usefixtures("pyarrow_xfail")
+def test_dtypes_defaultdict_invalid(all_parsers):
+ # GH#41574
+ data = """a,b
+1,2
+"""
+ dtype = defaultdict(lambda: "invalid_dtype", a="int64")
+ parser = all_parsers
+ with pytest.raises(TypeError, match="not understood"):
+ parser.read_csv(StringIO(data), dtype=dtype)
+
+
+def test_dtype_backend(all_parsers):
+ # GH#36712
+
+ parser = all_parsers
+
+ data = """a,b,c,d,e,f,g,h,i,j
+1,2.5,True,a,,,,,12-31-2019,
+3,4.5,False,b,6,7.5,True,a,12-31-2019,
+"""
+ result = parser.read_csv(
+ StringIO(data), dtype_backend="numpy_nullable", parse_dates=["i"]
+ )
+ expected = DataFrame(
+ {
+ "a": pd.Series([1, 3], dtype="Int64"),
+ "b": pd.Series([2.5, 4.5], dtype="Float64"),
+ "c": pd.Series([True, False], dtype="boolean"),
+ "d": pd.Series(["a", "b"], dtype="string"),
+ "e": pd.Series([pd.NA, 6], dtype="Int64"),
+ "f": pd.Series([pd.NA, 7.5], dtype="Float64"),
+ "g": pd.Series([pd.NA, True], dtype="boolean"),
+ "h": pd.Series([pd.NA, "a"], dtype="string"),
+ "i": pd.Series([Timestamp("2019-12-31")] * 2),
+ "j": pd.Series([pd.NA, pd.NA], dtype="Int64"),
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_dtype_backend_and_dtype(all_parsers):
+ # GH#36712
+
+ parser = all_parsers
+
+ data = """a,b
+1,2.5
+,
+"""
+ result = parser.read_csv(
+ StringIO(data), dtype_backend="numpy_nullable", dtype="float64"
+ )
+ expected = DataFrame({"a": [1.0, np.nan], "b": [2.5, np.nan]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_dtype_backend_string(all_parsers, string_storage):
+ # GH#36712
+ with pd.option_context("mode.string_storage", string_storage):
+ parser = all_parsers
+
+ data = """a,b
+a,x
+b,
+"""
+ result = parser.read_csv(StringIO(data), dtype_backend="numpy_nullable")
+
+ expected = DataFrame(
+ {
+ "a": pd.array(["a", "b"], dtype=pd.StringDtype(string_storage)),
+ "b": pd.array(["x", pd.NA], dtype=pd.StringDtype(string_storage)),
+ },
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_dtype_backend_ea_dtype_specified(all_parsers):
+ # GH#491496
+ data = """a,b
+1,2
+"""
+ parser = all_parsers
+ result = parser.read_csv(
+ StringIO(data), dtype="Int64", dtype_backend="numpy_nullable"
+ )
+ expected = DataFrame({"a": [1], "b": 2}, dtype="Int64")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_dtype_backend_pyarrow(all_parsers, request):
+ # GH#36712
+ pa = pytest.importorskip("pyarrow")
+ parser = all_parsers
+
+ data = """a,b,c,d,e,f,g,h,i,j
+1,2.5,True,a,,,,,12-31-2019,
+3,4.5,False,b,6,7.5,True,a,12-31-2019,
+"""
+ result = parser.read_csv(StringIO(data), dtype_backend="pyarrow", parse_dates=["i"])
+ expected = DataFrame(
+ {
+ "a": pd.Series([1, 3], dtype="int64[pyarrow]"),
+ "b": pd.Series([2.5, 4.5], dtype="float64[pyarrow]"),
+ "c": pd.Series([True, False], dtype="bool[pyarrow]"),
+ "d": pd.Series(["a", "b"], dtype=pd.ArrowDtype(pa.string())),
+ "e": pd.Series([pd.NA, 6], dtype="int64[pyarrow]"),
+ "f": pd.Series([pd.NA, 7.5], dtype="float64[pyarrow]"),
+ "g": pd.Series([pd.NA, True], dtype="bool[pyarrow]"),
+ "h": pd.Series(
+ [pd.NA, "a"],
+ dtype=pd.ArrowDtype(pa.string()),
+ ),
+ "i": pd.Series([Timestamp("2019-12-31")] * 2),
+ "j": pd.Series([pd.NA, pd.NA], dtype="null[pyarrow]"),
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_ea_int_avoid_overflow(all_parsers):
+ # GH#32134
+ parser = all_parsers
+ data = """a,b
+1,1
+,1
+1582218195625938945,1
+"""
+ result = parser.read_csv(StringIO(data), dtype={"a": "Int64"})
+ expected = DataFrame(
+ {
+ "a": IntegerArray(
+ np.array([1, 1, 1582218195625938945]), np.array([False, True, False])
+ ),
+ "b": 1,
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_string_inference(all_parsers):
+ # GH#54430
+ dtype = pd.StringDtype(na_value=np.nan)
+
+ data = """a,b
+x,1
+y,2
+,3"""
+ parser = all_parsers
+ with pd.option_context("future.infer_string", True):
+ result = parser.read_csv(StringIO(data))
+
+ expected = DataFrame(
+ {"a": pd.Series(["x", "y", None], dtype=dtype), "b": [1, 2, 3]},
+ columns=pd.Index(["a", "b"], dtype=dtype),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("dtype", ["O", object, "object", np.object_, str, np.str_])
+def test_string_inference_object_dtype(all_parsers, dtype, using_infer_string):
+ # GH#56047
+ data = """a,b
+x,a
+y,a
+z,a"""
+ parser = all_parsers
+ with pd.option_context("future.infer_string", True):
+ result = parser.read_csv(StringIO(data), dtype=dtype)
+
+ expected_dtype = pd.StringDtype(na_value=np.nan) if dtype is str else object
+ expected = DataFrame(
+ {
+ "a": pd.Series(["x", "y", "z"], dtype=expected_dtype),
+ "b": pd.Series(["a", "a", "a"], dtype=expected_dtype),
+ },
+ columns=pd.Index(["a", "b"], dtype=pd.StringDtype(na_value=np.nan)),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ with pd.option_context("future.infer_string", True):
+ result = parser.read_csv(StringIO(data), dtype={"a": dtype})
+
+ expected = DataFrame(
+ {
+ "a": pd.Series(["x", "y", "z"], dtype=expected_dtype),
+ "b": pd.Series(["a", "a", "a"], dtype=pd.StringDtype(na_value=np.nan)),
+ },
+ columns=pd.Index(["a", "b"], dtype=pd.StringDtype(na_value=np.nan)),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_accurate_parsing_of_large_integers(all_parsers):
+ # GH#52505
+ data = """SYMBOL,MOMENT,ID,ID_DEAL
+AAPL,20230301181139587,1925036343869802844,
+AAPL,20230301181139587,2023552585717889863,2023552585717263358
+NVDA,20230301181139587,2023552585717889863,2023552585717263359
+AMC,20230301181139587,2023552585717889863,2023552585717263360
+AMZN,20230301181139587,2023552585717889759,2023552585717263360
+MSFT,20230301181139587,2023552585717889863,2023552585717263361
+NVDA,20230301181139587,2023552585717889827,2023552585717263361"""
+ orders = all_parsers.read_csv(StringIO(data), dtype={"ID_DEAL": pd.Int64Dtype()})
+ assert len(orders.loc[orders["ID_DEAL"] == 2023552585717263358, "ID_DEAL"]) == 1
+ assert len(orders.loc[orders["ID_DEAL"] == 2023552585717263359, "ID_DEAL"]) == 1
+ assert len(orders.loc[orders["ID_DEAL"] == 2023552585717263360, "ID_DEAL"]) == 2
+ assert len(orders.loc[orders["ID_DEAL"] == 2023552585717263361, "ID_DEAL"]) == 2
+
+
+def test_dtypes_with_usecols(all_parsers):
+ # GH#54868
+
+ parser = all_parsers
+ data = """a,b,c
+1,2,3
+4,5,6"""
+
+ result = parser.read_csv(StringIO(data), usecols=["a", "c"], dtype={"a": object})
+ if parser.engine == "pyarrow":
+ values = [1, 4]
+ else:
+ values = ["1", "4"]
+ expected = DataFrame({"a": pd.Series(values, dtype=object), "c": [3, 6]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_index_col_with_dtype_no_rangeindex(all_parsers):
+ data = StringIO("345.5,519.5,0\n519.5,726.5,1")
+ result = all_parsers.read_csv(
+ data,
+ header=None,
+ names=["start", "stop", "bin_id"],
+ dtype={"start": np.float32, "stop": np.float32, "bin_id": np.uint32},
+ index_col="bin_id",
+ ).index
+ expected = pd.Index([0, 1], dtype=np.uint32, name="bin_id")
+ tm.assert_index_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/dtypes/test_empty.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/dtypes/test_empty.py
new file mode 100644
index 0000000000000000000000000000000000000000..ebc61e7f0ca2bf3a587d9abee30627de83658310
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/dtypes/test_empty.py
@@ -0,0 +1,181 @@
+"""
+Tests dtype specification during parsing
+for all of the parsers defined in parsers.py
+"""
+
+from io import StringIO
+
+import numpy as np
+import pytest
+
+from pandas import (
+ Categorical,
+ DataFrame,
+ Index,
+ MultiIndex,
+ Series,
+ concat,
+)
+import pandas._testing as tm
+
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+def test_dtype_all_columns_empty(all_parsers):
+ # see gh-12048
+ parser = all_parsers
+ result = parser.read_csv(StringIO("A,B"), dtype=str)
+
+ expected = DataFrame({"A": [], "B": []}, dtype=str)
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+def test_empty_pass_dtype(all_parsers):
+ parser = all_parsers
+
+ data = "one,two"
+ result = parser.read_csv(StringIO(data), dtype={"one": "u1"})
+
+ expected = DataFrame(
+ {"one": np.empty(0, dtype="u1"), "two": np.empty(0, dtype=object)},
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+def test_empty_with_index_pass_dtype(all_parsers):
+ parser = all_parsers
+
+ data = "one,two"
+ result = parser.read_csv(
+ StringIO(data), index_col=["one"], dtype={"one": "u1", 1: "f"}
+ )
+
+ expected = DataFrame(
+ {"two": np.empty(0, dtype="f")}, index=Index([], dtype="u1", name="one")
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+def test_empty_with_multi_index_pass_dtype(all_parsers):
+ parser = all_parsers
+
+ data = "one,two,three"
+ result = parser.read_csv(
+ StringIO(data), index_col=["one", "two"], dtype={"one": "u1", 1: "f8"}
+ )
+
+ exp_idx = MultiIndex.from_arrays(
+ [np.empty(0, dtype="u1"), np.empty(0, dtype=np.float64)],
+ names=["one", "two"],
+ )
+ expected = DataFrame({"three": np.empty(0, dtype=object)}, index=exp_idx)
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+def test_empty_with_mangled_column_pass_dtype_by_names(all_parsers):
+ parser = all_parsers
+
+ data = "one,one"
+ result = parser.read_csv(StringIO(data), dtype={"one": "u1", "one.1": "f"})
+
+ expected = DataFrame(
+ {"one": np.empty(0, dtype="u1"), "one.1": np.empty(0, dtype="f")},
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+def test_empty_with_mangled_column_pass_dtype_by_indexes(all_parsers):
+ parser = all_parsers
+
+ data = "one,one"
+ result = parser.read_csv(StringIO(data), dtype={0: "u1", 1: "f"})
+
+ expected = DataFrame(
+ {"one": np.empty(0, dtype="u1"), "one.1": np.empty(0, dtype="f")},
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+def test_empty_with_dup_column_pass_dtype_by_indexes(all_parsers):
+ # see gh-9424
+ parser = all_parsers
+ expected = concat(
+ [Series([], name="one", dtype="u1"), Series([], name="one.1", dtype="f")],
+ axis=1,
+ )
+
+ data = "one,one"
+ result = parser.read_csv(StringIO(data), dtype={0: "u1", 1: "f"})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_empty_with_dup_column_pass_dtype_by_indexes_raises(all_parsers):
+ # see gh-9424
+ parser = all_parsers
+ expected = concat(
+ [Series([], name="one", dtype="u1"), Series([], name="one.1", dtype="f")],
+ axis=1,
+ )
+ expected.index = expected.index.astype(object)
+
+ with pytest.raises(ValueError, match="Duplicate names"):
+ parser.read_csv(StringIO(""), names=["one", "one"], dtype={0: "u1", 1: "f"})
+
+
+@pytest.mark.parametrize(
+ "dtype,expected",
+ [
+ (np.float64, DataFrame(columns=["a", "b"], dtype=np.float64)),
+ (
+ "category",
+ DataFrame({"a": Categorical([]), "b": Categorical([])}),
+ ),
+ (
+ {"a": "category", "b": "category"},
+ DataFrame({"a": Categorical([]), "b": Categorical([])}),
+ ),
+ ("datetime64[ns]", DataFrame(columns=["a", "b"], dtype="datetime64[ns]")),
+ (
+ "timedelta64[ns]",
+ DataFrame(
+ {
+ "a": Series([], dtype="timedelta64[ns]"),
+ "b": Series([], dtype="timedelta64[ns]"),
+ },
+ ),
+ ),
+ (
+ {"a": np.int64, "b": np.int32},
+ DataFrame(
+ {"a": Series([], dtype=np.int64), "b": Series([], dtype=np.int32)},
+ ),
+ ),
+ (
+ {0: np.int64, 1: np.int32},
+ DataFrame(
+ {"a": Series([], dtype=np.int64), "b": Series([], dtype=np.int32)},
+ ),
+ ),
+ (
+ {"a": np.int64, 1: np.int32},
+ DataFrame(
+ {"a": Series([], dtype=np.int64), "b": Series([], dtype=np.int32)},
+ ),
+ ),
+ ],
+)
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+def test_empty_dtype(all_parsers, dtype, expected):
+ # see gh-14712
+ parser = all_parsers
+ data = "a,b"
+
+ result = parser.read_csv(StringIO(data), header=0, dtype=dtype)
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_c_parser_only.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_c_parser_only.py
new file mode 100644
index 0000000000000000000000000000000000000000..08e9eae8a851e5e11f86fd4ca0ba942ec764233a
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_c_parser_only.py
@@ -0,0 +1,644 @@
+"""
+Tests that apply specifically to the CParser. Unless specifically stated
+as a CParser-specific issue, the goal is to eventually move as many of
+these tests out of this module as soon as the Python parser can accept
+further arguments when parsing.
+"""
+
+from decimal import Decimal
+from io import (
+ BytesIO,
+ StringIO,
+ TextIOWrapper,
+)
+import mmap
+import os
+import tarfile
+
+import numpy as np
+import pytest
+
+from pandas.compat import WASM
+from pandas.errors import (
+ ParserError,
+ ParserWarning,
+)
+import pandas.util._test_decorators as td
+
+from pandas import (
+ DataFrame,
+ concat,
+)
+import pandas._testing as tm
+
+
+@pytest.mark.parametrize(
+ "malformed",
+ ["1\r1\r1\r 1\r 1\r", "1\r1\r1\r 1\r 1\r11\r", "1\r1\r1\r 1\r 1\r11\r1\r"],
+ ids=["words pointer", "stream pointer", "lines pointer"],
+)
+def test_buffer_overflow(c_parser_only, malformed):
+ # see gh-9205: test certain malformed input files that cause
+ # buffer overflows in tokenizer.c
+ msg = "Buffer overflow caught - possible malformed input file."
+ parser = c_parser_only
+
+ with pytest.raises(ParserError, match=msg):
+ parser.read_csv(StringIO(malformed))
+
+
+def test_delim_whitespace_custom_terminator(c_parser_only):
+ # See gh-12912
+ data = "a b c~1 2 3~4 5 6~7 8 9"
+ parser = c_parser_only
+
+ df = parser.read_csv(StringIO(data), lineterminator="~", sep=r"\s+")
+ expected = DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]], columns=["a", "b", "c"])
+ tm.assert_frame_equal(df, expected)
+
+
+def test_dtype_and_names_error(c_parser_only):
+ # see gh-8833: passing both dtype and names
+ # resulting in an error reporting issue
+ parser = c_parser_only
+ data = """
+1.0 1
+2.0 2
+3.0 3
+"""
+ # base cases
+ result = parser.read_csv(StringIO(data), sep=r"\s+", header=None)
+ expected = DataFrame([[1.0, 1], [2.0, 2], [3.0, 3]])
+ tm.assert_frame_equal(result, expected)
+
+ result = parser.read_csv(StringIO(data), sep=r"\s+", header=None, names=["a", "b"])
+ expected = DataFrame([[1.0, 1], [2.0, 2], [3.0, 3]], columns=["a", "b"])
+ tm.assert_frame_equal(result, expected)
+
+ # fallback casting
+ result = parser.read_csv(
+ StringIO(data), sep=r"\s+", header=None, names=["a", "b"], dtype={"a": np.int32}
+ )
+ expected = DataFrame([[1, 1], [2, 2], [3, 3]], columns=["a", "b"])
+ expected["a"] = expected["a"].astype(np.int32)
+ tm.assert_frame_equal(result, expected)
+
+ data = """
+1.0 1
+nan 2
+3.0 3
+"""
+ # fallback casting, but not castable
+ if not WASM: # no fp exception support in wasm
+ with pytest.raises(ValueError, match="cannot safely convert"):
+ with tm.assert_produces_warning(RuntimeWarning, check_stacklevel=False):
+ parser.read_csv(
+ StringIO(data),
+ sep=r"\s+",
+ header=None,
+ names=["a", "b"],
+ dtype={"a": np.int32},
+ )
+
+
+@pytest.mark.parametrize(
+ "match,kwargs",
+ [
+ # For each of these cases, all of the dtypes are valid, just unsupported.
+ (
+ (
+ "the dtype datetime64 is not supported for parsing, "
+ "pass this column using parse_dates instead"
+ ),
+ {"dtype": {"A": "datetime64", "B": "float64"}},
+ ),
+ (
+ (
+ "the dtype datetime64 is not supported for parsing, "
+ "pass this column using parse_dates instead"
+ ),
+ {"dtype": {"A": "datetime64", "B": "float64"}, "parse_dates": ["B"]},
+ ),
+ (
+ "the dtype timedelta64 is not supported for parsing",
+ {"dtype": {"A": "timedelta64", "B": "float64"}},
+ ),
+ (
+ f"the dtype {tm.ENDIAN}U8 is not supported for parsing",
+ {"dtype": {"A": "U8"}},
+ ),
+ ],
+ ids=["dt64-0", "dt64-1", "td64", f"{tm.ENDIAN}U8"],
+)
+def test_unsupported_dtype(c_parser_only, match, kwargs, temp_file):
+ parser = c_parser_only
+ df = DataFrame(
+ np.random.default_rng(2).random((5, 2)),
+ columns=list("AB"),
+ index=["1A", "1B", "1C", "1D", "1E"],
+ )
+
+ df.to_csv(temp_file)
+
+ with pytest.raises(TypeError, match=match):
+ parser.read_csv(temp_file, index_col=0, **kwargs)
+
+
+@td.skip_if_32bit
+@pytest.mark.slow
+# test numbers between 1 and 2
+@pytest.mark.parametrize("num", np.linspace(1.0, 2.0, num=21))
+def test_precise_conversion(c_parser_only, num):
+ parser = c_parser_only
+
+ normal_errors = []
+ precise_errors = []
+
+ def error(val: float, actual_val: Decimal) -> Decimal:
+ return abs(Decimal(f"{val:.100}") - actual_val)
+
+ # 25 decimal digits of precision
+ text = f"a\n{num:.25}"
+
+ normal_val = float(
+ parser.read_csv(StringIO(text), float_precision="legacy")["a"][0]
+ )
+ precise_val = float(parser.read_csv(StringIO(text), float_precision="high")["a"][0])
+ roundtrip_val = float(
+ parser.read_csv(StringIO(text), float_precision="round_trip")["a"][0]
+ )
+ actual_val = Decimal(text[2:])
+
+ normal_errors.append(error(normal_val, actual_val))
+ precise_errors.append(error(precise_val, actual_val))
+
+ # round-trip should match float()
+ assert roundtrip_val == float(text[2:])
+
+ assert sum(precise_errors) <= sum(normal_errors)
+ assert max(precise_errors) <= max(normal_errors)
+
+
+def test_usecols_dtypes(c_parser_only, using_infer_string):
+ parser = c_parser_only
+ data = """\
+1,2,3
+4,5,6
+7,8,9
+10,11,12"""
+
+ result = parser.read_csv(
+ StringIO(data),
+ usecols=(0, 1, 2),
+ names=("a", "b", "c"),
+ header=None,
+ converters={"a": str},
+ dtype={"b": int, "c": float},
+ )
+ result2 = parser.read_csv(
+ StringIO(data),
+ usecols=(0, 2),
+ names=("a", "b", "c"),
+ header=None,
+ converters={"a": str},
+ dtype={"b": int, "c": float},
+ )
+
+ if using_infer_string:
+ assert (result.dtypes == ["string", int, float]).all()
+ assert (result2.dtypes == ["string", float]).all()
+ else:
+ assert (result.dtypes == [object, int, float]).all()
+ assert (result2.dtypes == [object, float]).all()
+
+
+def test_disable_bool_parsing(c_parser_only):
+ # see gh-2090
+
+ parser = c_parser_only
+ data = """A,B,C
+Yes,No,Yes
+No,Yes,Yes
+Yes,,Yes
+No,No,No"""
+
+ result = parser.read_csv(StringIO(data), dtype=object)
+ assert (result.dtypes == object).all()
+
+ result = parser.read_csv(StringIO(data), dtype=object, na_filter=False)
+ assert result["B"][2] == ""
+
+
+def test_custom_lineterminator(c_parser_only):
+ parser = c_parser_only
+ data = "a,b,c~1,2,3~4,5,6"
+
+ result = parser.read_csv(StringIO(data), lineterminator="~")
+ expected = parser.read_csv(StringIO(data.replace("~", "\n")))
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_parse_ragged_csv(c_parser_only):
+ parser = c_parser_only
+ data = """1,2,3
+1,2,3,4
+1,2,3,4,5
+1,2
+1,2,3,4"""
+
+ nice_data = """1,2,3,,
+1,2,3,4,
+1,2,3,4,5
+1,2,,,
+1,2,3,4,"""
+ result = parser.read_csv(
+ StringIO(data), header=None, names=["a", "b", "c", "d", "e"]
+ )
+
+ expected = parser.read_csv(
+ StringIO(nice_data), header=None, names=["a", "b", "c", "d", "e"]
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ # too many columns, cause segfault if not careful
+ data = "1,2\n3,4,5"
+
+ result = parser.read_csv(StringIO(data), header=None, names=range(50))
+ expected = parser.read_csv(StringIO(data), header=None, names=range(3)).reindex(
+ columns=range(50)
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_tokenize_CR_with_quoting(c_parser_only):
+ # see gh-3453
+ parser = c_parser_only
+ data = ' a,b,c\r"a,b","e,d","f,f"'
+
+ result = parser.read_csv(StringIO(data), header=None)
+ expected = parser.read_csv(StringIO(data.replace("\r", "\n")), header=None)
+ tm.assert_frame_equal(result, expected)
+
+ result = parser.read_csv(StringIO(data))
+ expected = parser.read_csv(StringIO(data.replace("\r", "\n")))
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.slow
+@pytest.mark.parametrize("count", [3 * 2**n for n in range(6)])
+def test_grow_boundary_at_cap(c_parser_only, count):
+ # See gh-12494
+ #
+ # Cause of error was that the C parser
+ # was not increasing the buffer size when
+ # the desired space would fill the buffer
+ # to capacity, which would later cause a
+ # buffer overflow error when checking the
+ # EOF terminator of the CSV stream.
+ # 3 * 2^n commas was observed to break the parser
+ parser = c_parser_only
+
+ with StringIO("," * count) as s:
+ expected = DataFrame(columns=[f"Unnamed: {i}" for i in range(count + 1)])
+ df = parser.read_csv(s)
+ tm.assert_frame_equal(df, expected)
+
+
+@pytest.mark.slow
+@pytest.mark.parametrize("encoding", [None, "utf-8"])
+def test_parse_trim_buffers(c_parser_only, encoding):
+ # This test is part of a bugfix for gh-13703. It attempts to
+ # to stress the system memory allocator, to cause it to move the
+ # stream buffer and either let the OS reclaim the region, or let
+ # other memory requests of parser otherwise modify the contents
+ # of memory space, where it was formally located.
+ # This test is designed to cause a `segfault` with unpatched
+ # `tokenizer.c`. Sometimes the test fails on `segfault`, other
+ # times it fails due to memory corruption, which causes the
+ # loaded DataFrame to differ from the expected one.
+
+ # Also force 'utf-8' encoding, so that `_string_convert` would take
+ # a different execution branch.
+
+ parser = c_parser_only
+
+ # Generate a large mixed-type CSV file on-the-fly (one record is
+ # approx 1.5KiB).
+ record_ = (
+ """9999-9,99:99,,,,ZZ,ZZ,,,ZZZ-ZZZZ,.Z-ZZZZ,-9.99,,,9.99,Z"""
+ """ZZZZ,,-99,9,ZZZ-ZZZZ,ZZ-ZZZZ,,9.99,ZZZ-ZZZZZ,ZZZ-ZZZZZ,"""
+ """ZZZ-ZZZZ,ZZZ-ZZZZ,ZZZ-ZZZZ,ZZZ-ZZZZ,ZZZ-ZZZZ,ZZZ-ZZZZ,9"""
+ """99,ZZZ-ZZZZ,,ZZ-ZZZZ,,,,,ZZZZ,ZZZ-ZZZZZ,ZZZ-ZZZZ,,,9,9,"""
+ """9,9,99,99,999,999,ZZZZZ,ZZZ-ZZZZZ,ZZZ-ZZZZ,9,ZZ-ZZZZ,9."""
+ """99,ZZ-ZZZZ,ZZ-ZZZZ,,,,ZZZZ,,,ZZ,ZZ,,,,,,,,,,,,,9,,,999."""
+ """99,999.99,,,ZZZZZ,,,Z9,,,,,,,ZZZ,ZZZ,,,,,,,,,,,ZZZZZ,ZZ"""
+ """ZZZ,ZZZ-ZZZZZZ,ZZZ-ZZZZZZ,ZZ-ZZZZ,ZZ-ZZZZ,ZZ-ZZZZ,ZZ-ZZ"""
+ """ZZ,,,999999,999999,ZZZ,ZZZ,,,ZZZ,ZZZ,999.99,999.99,,,,Z"""
+ """ZZ-ZZZ,ZZZ-ZZZ,-9.99,-9.99,9,9,,99,,9.99,9.99,9,9,9.99,"""
+ """9.99,,,,9.99,9.99,,99,,99,9.99,9.99,,,ZZZ,ZZZ,,999.99,,"""
+ """999.99,ZZZ,ZZZ-ZZZZ,ZZZ-ZZZZ,,,ZZZZZ,ZZZZZ,ZZZ,ZZZ,9,9,"""
+ """,,,,,ZZZ-ZZZZ,ZZZ999Z,,,999.99,,999.99,ZZZ-ZZZZ,,,9.999"""
+ """,9.999,9.999,9.999,-9.999,-9.999,-9.999,-9.999,9.999,9."""
+ """999,9.999,9.999,9.999,9.999,9.999,9.999,99999,ZZZ-ZZZZ,"""
+ """,9.99,ZZZ,,,,,,,,ZZZ,,,,,9,,,,9,,,,,,,,,,ZZZ-ZZZZ,ZZZ-Z"""
+ """ZZZ,,ZZZZZ,ZZZZZ,ZZZZZ,ZZZZZ,,,9.99,,ZZ-ZZZZ,ZZ-ZZZZ,ZZ"""
+ """,999,,,,ZZ-ZZZZ,ZZZ,ZZZ,ZZZ-ZZZZ,ZZZ-ZZZZ,,,99.99,99.99"""
+ """,,,9.99,9.99,9.99,9.99,ZZZ-ZZZZ,,,ZZZ-ZZZZZ,,,,,-9.99,-"""
+ """9.99,-9.99,-9.99,,,,,,,,,ZZZ-ZZZZ,,9,9.99,9.99,99ZZ,,-9"""
+ """.99,-9.99,ZZZ-ZZZZ,,,,,,,ZZZ-ZZZZ,9.99,9.99,9999,,,,,,,"""
+ """,,,-9.9,Z/Z-ZZZZ,999.99,9.99,,999.99,ZZ-ZZZZ,ZZ-ZZZZ,9."""
+ """99,9.99,9.99,9.99,9.99,9.99,,ZZZ-ZZZZZ,ZZZ-ZZZZZ,ZZZ-ZZ"""
+ """ZZZ,ZZZ-ZZZZZ,ZZZ-ZZZZZ,ZZZ,ZZZ,ZZZ,ZZZ,9.99,,,-9.99,ZZ"""
+ """-ZZZZ,-999.99,,-9999,,999.99,,,,999.99,99.99,,,ZZ-ZZZZZ"""
+ """ZZZ,ZZ-ZZZZ-ZZZZZZZ,,,,ZZ-ZZ-ZZZZZZZZ,ZZZZZZZZ,ZZZ-ZZZZ"""
+ """,9999,999.99,ZZZ-ZZZZ,-9.99,-9.99,ZZZ-ZZZZ,99:99:99,,99"""
+ """,99,,9.99,,-99.99,,,,,,9.99,ZZZ-ZZZZ,-9.99,-9.99,9.99,9"""
+ """.99,,ZZZ,,,,,,,ZZZ,ZZZ,,,,,"""
+ )
+
+ # Set the number of lines so that a call to `parser_trim_buffers`
+ # is triggered: after a couple of full chunks are consumed a
+ # relatively small 'residual' chunk would cause reallocation
+ # within the parser.
+ chunksize, n_lines = 128, 2 * 128 + 15
+ csv_data = "\n".join([record_] * n_lines) + "\n"
+
+ # We will use StringIO to load the CSV from this text buffer.
+ # pd.read_csv() will iterate over the file in chunks and will
+ # finally read a residual chunk of really small size.
+
+ # Generate the expected output: manually create the dataframe
+ # by splitting by comma and repeating the `n_lines` times.
+ row = tuple(val_ if val_ else np.nan for val_ in record_.split(","))
+ expected = DataFrame(
+ [row for _ in range(n_lines)], dtype=object, columns=None, index=None
+ )
+
+ # Iterate over the CSV file in chunks of `chunksize` lines
+ with parser.read_csv(
+ StringIO(csv_data),
+ header=None,
+ dtype=object,
+ chunksize=chunksize,
+ encoding=encoding,
+ ) as chunks_:
+ result = concat(chunks_, axis=0, ignore_index=True)
+
+ # Check for data corruption if there was no segfault
+ tm.assert_frame_equal(result, expected)
+
+
+def test_internal_null_byte(c_parser_only):
+ # see gh-14012
+ #
+ # The null byte ('\x00') should not be used as a
+ # true line terminator, escape character, or comment
+ # character, only as a placeholder to indicate that
+ # none was specified.
+ #
+ # This test should be moved to test_common.py ONLY when
+ # Python's csv class supports parsing '\x00'.
+ parser = c_parser_only
+
+ names = ["a", "b", "c"]
+ data = "1,2,3\n4,\x00,6\n7,8,9"
+ expected = DataFrame([[1, 2.0, 3], [4, np.nan, 6], [7, 8, 9]], columns=names)
+
+ result = parser.read_csv(StringIO(data), names=names)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_nrows_large(c_parser_only):
+ # gh-7626 - Read only nrows of data in for large inputs (>262144b)
+ parser = c_parser_only
+ header_narrow = "\t".join(["COL_HEADER_" + str(i) for i in range(10)]) + "\n"
+ data_narrow = "\t".join(["somedatasomedatasomedata1" for _ in range(10)]) + "\n"
+ header_wide = "\t".join(["COL_HEADER_" + str(i) for i in range(15)]) + "\n"
+ data_wide = "\t".join(["somedatasomedatasomedata2" for _ in range(15)]) + "\n"
+ test_input = header_narrow + data_narrow * 1050 + header_wide + data_wide * 2
+
+ df = parser.read_csv(StringIO(test_input), sep="\t", nrows=1010)
+
+ assert df.size == 1010 * 10
+
+
+def test_float_precision_round_trip_with_text(c_parser_only):
+ # see gh-15140
+ parser = c_parser_only
+ df = parser.read_csv(StringIO("a"), header=None, float_precision="round_trip")
+ tm.assert_frame_equal(df, DataFrame({0: ["a"]}))
+
+
+def test_large_difference_in_columns(c_parser_only):
+ # see gh-14125
+ parser = c_parser_only
+
+ count = 10000
+ large_row = ("X," * count)[:-1] + "\n"
+ normal_row = "XXXXXX XXXXXX,111111111111111\n"
+ test_input = (large_row + normal_row * 6)[:-1]
+
+ result = parser.read_csv(StringIO(test_input), header=None, usecols=[0])
+ rows = test_input.split("\n")
+
+ expected = DataFrame([row.split(",")[0] for row in rows])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_data_after_quote(c_parser_only):
+ # see gh-15910
+ parser = c_parser_only
+
+ data = 'a\n1\n"b"a'
+ result = parser.read_csv(StringIO(data))
+
+ expected = DataFrame({"a": ["1", "ba"]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_comment_whitespace_delimited(c_parser_only):
+ parser = c_parser_only
+ test_input = """\
+1 2
+2 2 3
+3 2 3 # 3 fields
+4 2 3# 3 fields
+5 2 # 2 fields
+6 2# 2 fields
+7 # 1 field, NaN
+8# 1 field, NaN
+9 2 3 # skipped line
+# comment"""
+ with tm.assert_produces_warning(
+ ParserWarning, match="Skipping line", check_stacklevel=False
+ ):
+ df = parser.read_csv(
+ StringIO(test_input),
+ comment="#",
+ header=None,
+ delimiter="\\s+",
+ skiprows=0,
+ on_bad_lines="warn",
+ )
+ expected = DataFrame([[1, 2], [5, 2], [6, 2], [7, np.nan], [8, np.nan]])
+ tm.assert_frame_equal(df, expected)
+
+
+def test_file_like_no_next(c_parser_only):
+ # gh-16530: the file-like need not have a "next" or "__next__"
+ # attribute despite having an "__iter__" attribute.
+ #
+ # NOTE: This is only true for the C engine, not Python engine.
+ class NoNextBuffer(StringIO):
+ def __next__(self):
+ raise AttributeError("No next method")
+
+ next = __next__
+
+ parser = c_parser_only
+ data = "a\n1"
+
+ expected = DataFrame({"a": [1]})
+ result = parser.read_csv(NoNextBuffer(data))
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_buffer_rd_bytes_bad_unicode(c_parser_only):
+ # see gh-22748
+ t = BytesIO(b"\xb0")
+ t = TextIOWrapper(t, encoding="UTF-8", errors="surrogateescape")
+ msg = "'utf-8' codec can't encode character"
+ with pytest.raises(UnicodeError, match=msg):
+ c_parser_only.read_csv(t, encoding="UTF-8")
+
+
+@pytest.mark.parametrize("tar_suffix", [".tar", ".tar.gz"])
+def test_read_tarfile(c_parser_only, csv_dir_path, tar_suffix):
+ # see gh-16530
+ #
+ # Unfortunately, Python's CSV library can't handle
+ # tarfile objects (expects string, not bytes when
+ # iterating through a file-like).
+ parser = c_parser_only
+ tar_path = os.path.join(csv_dir_path, "tar_csv" + tar_suffix)
+
+ with tarfile.open(tar_path, "r") as tar:
+ data_file = tar.extractfile("tar_data.csv")
+
+ out = parser.read_csv(data_file)
+ expected = DataFrame({"a": [1]})
+ tm.assert_frame_equal(out, expected)
+
+
+def test_chunk_whitespace_on_boundary(c_parser_only):
+ # see gh-9735: this issue is C parser-specific (bug when
+ # parsing whitespace and characters at chunk boundary)
+ #
+ # This test case has a field too large for the Python parser / CSV library.
+ parser = c_parser_only
+
+ chunk1 = "a" * (1024 * 256 - 2) + "\na"
+ chunk2 = "\n a"
+ result = parser.read_csv(StringIO(chunk1 + chunk2), header=None)
+
+ expected = DataFrame(["a" * (1024 * 256 - 2), "a", " a"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.skipif(WASM, reason="limited file system access on WASM")
+def test_file_handles_mmap(c_parser_only, csv1):
+ # gh-14418
+ #
+ # Don't close user provided file handles.
+ parser = c_parser_only
+
+ with open(csv1, encoding="utf-8") as f:
+ with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as m:
+ parser.read_csv(m)
+ assert not m.closed
+
+
+def test_file_binary_mode(c_parser_only, temp_file):
+ # see gh-23779
+ parser = c_parser_only
+ expected = DataFrame([[1, 2, 3], [4, 5, 6]])
+
+ path = temp_file
+ with open(path, "w", encoding="utf-8") as f:
+ f.write("1,2,3\n4,5,6")
+
+ with open(path, "rb") as f:
+ result = parser.read_csv(f, header=None)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_unix_style_breaks(c_parser_only, temp_file):
+ # GH 11020
+ parser = c_parser_only
+ path = temp_file
+ with open(path, "w", newline="\n", encoding="utf-8") as f:
+ f.write("blah\n\ncol_1,col_2,col_3\n\n")
+ result = parser.read_csv(path, skiprows=2, encoding="utf-8", engine="c")
+ expected = DataFrame(columns=["col_1", "col_2", "col_3"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("float_precision", [None, "legacy", "high", "round_trip"])
+@pytest.mark.parametrize(
+ "data,thousands,decimal",
+ [
+ (
+ """A|B|C
+1|2,334.01|5
+10|13|10.
+""",
+ ",",
+ ".",
+ ),
+ (
+ """A|B|C
+1|2.334,01|5
+10|13|10,
+""",
+ ".",
+ ",",
+ ),
+ ],
+)
+def test_1000_sep_with_decimal(
+ c_parser_only, data, thousands, decimal, float_precision
+):
+ parser = c_parser_only
+ expected = DataFrame({"A": [1, 10], "B": [2334.01, 13], "C": [5, 10.0]})
+
+ result = parser.read_csv(
+ StringIO(data),
+ sep="|",
+ thousands=thousands,
+ decimal=decimal,
+ float_precision=float_precision,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_float_precision_options(c_parser_only):
+ # GH 17154, 36228
+ parser = c_parser_only
+ s = "foo\n243.164\n"
+ df = parser.read_csv(StringIO(s))
+ df2 = parser.read_csv(StringIO(s), float_precision="high")
+
+ tm.assert_frame_equal(df, df2)
+
+ df3 = parser.read_csv(StringIO(s), float_precision="legacy")
+
+ assert not df.iloc[0, 0] == df3.iloc[0, 0]
+
+ msg = "Unrecognized float_precision option: junk"
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(s), float_precision="junk")
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_comment.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_comment.py
new file mode 100644
index 0000000000000000000000000000000000000000..ba27b170aecdcb0e7492764c025aea219b390559
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_comment.py
@@ -0,0 +1,214 @@
+"""
+Tests that comments are properly handled during parsing
+for all of the parsers defined in parsers.py
+"""
+
+from io import StringIO
+
+import numpy as np
+import pytest
+
+from pandas import DataFrame
+import pandas._testing as tm
+
+
+@pytest.mark.parametrize("na_values", [None, ["NaN"]])
+def test_comment(all_parsers, na_values):
+ parser = all_parsers
+ data = """A,B,C
+1,2.,4.#hello world
+5.,NaN,10.0
+"""
+ expected = DataFrame(
+ [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"]
+ )
+ if parser.engine == "pyarrow":
+ msg = "The 'comment' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), comment="#", na_values=na_values)
+ return
+ result = parser.read_csv(StringIO(data), comment="#", na_values=na_values)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("read_kwargs", [{}, {"lineterminator": "*"}, {"sep": r"\s+"}])
+def test_line_comment(all_parsers, read_kwargs):
+ parser = all_parsers
+ data = """# empty
+A,B,C
+1,2.,4.#hello world
+#ignore this line
+5.,NaN,10.0
+"""
+ if read_kwargs.get("sep"):
+ data = data.replace(",", " ")
+ elif read_kwargs.get("lineterminator"):
+ data = data.replace("\n", read_kwargs.get("lineterminator"))
+
+ read_kwargs["comment"] = "#"
+ if parser.engine == "pyarrow":
+ if "lineterminator" in read_kwargs:
+ msg = (
+ "The 'lineterminator' option is not supported with the 'pyarrow' engine"
+ )
+ else:
+ msg = "The 'comment' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), **read_kwargs)
+ return
+ elif parser.engine == "python" and read_kwargs.get("lineterminator"):
+ msg = r"Custom line terminators not supported in python parser \(yet\)"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), **read_kwargs)
+ return
+
+ result = parser.read_csv(StringIO(data), **read_kwargs)
+ expected = DataFrame(
+ [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"]
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_comment_skiprows(all_parsers):
+ parser = all_parsers
+ data = """# empty
+random line
+# second empty line
+1,2,3
+A,B,C
+1,2.,4.
+5.,NaN,10.0
+"""
+ # This should ignore the first four lines (including comments).
+ expected = DataFrame(
+ [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"]
+ )
+ if parser.engine == "pyarrow":
+ msg = "The 'comment' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), comment="#", skiprows=4)
+ return
+
+ result = parser.read_csv(StringIO(data), comment="#", skiprows=4)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_comment_header(all_parsers):
+ parser = all_parsers
+ data = """# empty
+# second empty line
+1,2,3
+A,B,C
+1,2.,4.
+5.,NaN,10.0
+"""
+ # Header should begin at the second non-comment line.
+ expected = DataFrame(
+ [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"]
+ )
+ if parser.engine == "pyarrow":
+ msg = "The 'comment' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), comment="#", header=1)
+ return
+ result = parser.read_csv(StringIO(data), comment="#", header=1)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_comment_skiprows_header(all_parsers):
+ parser = all_parsers
+ data = """# empty
+# second empty line
+# third empty line
+X,Y,Z
+1,2,3
+A,B,C
+1,2.,4.
+5.,NaN,10.0
+"""
+ # Skiprows should skip the first 4 lines (including comments),
+ # while header should start from the second non-commented line,
+ # starting with line 5.
+ expected = DataFrame(
+ [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"]
+ )
+ if parser.engine == "pyarrow":
+ msg = "The 'comment' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), comment="#", skiprows=4, header=1)
+ return
+
+ result = parser.read_csv(StringIO(data), comment="#", skiprows=4, header=1)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("comment_char", ["#", "~", "&", "^", "*", "@"])
+def test_custom_comment_char(all_parsers, comment_char):
+ parser = all_parsers
+ data = "a,b,c\n1,2,3#ignore this!\n4,5,6#ignorethistoo"
+
+ if parser.engine == "pyarrow":
+ msg = "The 'comment' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data.replace("#", comment_char)), comment=comment_char
+ )
+ return
+ result = parser.read_csv(
+ StringIO(data.replace("#", comment_char)), comment=comment_char
+ )
+
+ expected = DataFrame([[1, 2, 3], [4, 5, 6]], columns=["a", "b", "c"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("header", ["infer", None])
+def test_comment_first_line(all_parsers, header):
+ # see gh-4623
+ parser = all_parsers
+ data = "# notes\na,b,c\n# more notes\n1,2,3"
+
+ if header is None:
+ expected = DataFrame({0: ["a", "1"], 1: ["b", "2"], 2: ["c", "3"]})
+ else:
+ expected = DataFrame([[1, 2, 3]], columns=["a", "b", "c"])
+
+ if parser.engine == "pyarrow":
+ msg = "The 'comment' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), comment="#", header=header)
+ return
+ result = parser.read_csv(StringIO(data), comment="#", header=header)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_comment_char_in_default_value(all_parsers, request):
+ # GH#34002
+ if all_parsers.engine == "c":
+ reason = "see gh-34002: works on the python engine but not the c engine"
+ # NA value containing comment char is interpreted as comment
+ request.applymarker(pytest.mark.xfail(reason=reason, raises=AssertionError))
+ parser = all_parsers
+
+ data = (
+ "# this is a comment\n"
+ "col1,col2,col3,col4\n"
+ "1,2,3,4#inline comment\n"
+ "4,5#,6,10\n"
+ "7,8,#N/A,11\n"
+ )
+ if parser.engine == "pyarrow":
+ msg = "The 'comment' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), comment="#", na_values="#N/A")
+ return
+ result = parser.read_csv(StringIO(data), comment="#", na_values="#N/A")
+ expected = DataFrame(
+ {
+ "col1": [1, 4, 7],
+ "col2": [2, 5, 8],
+ "col3": [3.0, np.nan, np.nan],
+ "col4": [4.0, np.nan, 11.0],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_compression.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_compression.py
new file mode 100644
index 0000000000000000000000000000000000000000..9db820cc1566c78d9069f0d8719e65380a2b9e52
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_compression.py
@@ -0,0 +1,214 @@
+"""
+Tests compressed data parsing functionality for all
+of the parsers defined in parsers.py
+"""
+
+import os
+from pathlib import Path
+import tarfile
+import zipfile
+
+import pytest
+
+from pandas import DataFrame
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+@pytest.fixture(params=[True, False])
+def buffer(request):
+ return request.param
+
+
+@pytest.fixture
+def parser_and_data(all_parsers, csv1):
+ parser = all_parsers
+
+ with open(csv1, "rb") as f:
+ data = f.read()
+ expected = parser.read_csv(csv1)
+
+ return parser, data, expected
+
+
+@pytest.mark.parametrize("compression", ["zip", "infer", "zip2"])
+def test_zip(tmp_path, parser_and_data, compression):
+ parser, data, expected = parser_and_data
+
+ path = tmp_path / "test_file.zip"
+ with zipfile.ZipFile(path, mode="w") as tmp:
+ tmp.writestr("test_file", data)
+
+ if compression == "zip2":
+ with open(path, "rb") as f:
+ result = parser.read_csv(f, compression="zip")
+ else:
+ result = parser.read_csv(path, compression=compression)
+
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("compression", ["zip", "infer"])
+def test_zip_error_multiple_files(tmp_path, parser_and_data, compression):
+ parser, data, expected = parser_and_data
+
+ path = tmp_path / "combined_zip.zip"
+ inner_file_names = ["test_file", "second_file"]
+
+ with zipfile.ZipFile(path, mode="w") as tmp:
+ for file_name in inner_file_names:
+ tmp.writestr(file_name, data)
+
+ with pytest.raises(ValueError, match="Multiple files"):
+ parser.read_csv(path, compression=compression)
+
+
+def test_zip_error_no_files(tmp_path, parser_and_data):
+ parser, _, _ = parser_and_data
+
+ path = tmp_path / "test_file.zip"
+ with zipfile.ZipFile(path, mode="w"):
+ pass
+
+ with pytest.raises(ValueError, match="Zero files"):
+ parser.read_csv(path, compression="zip")
+
+
+def test_zip_error_invalid_zip(tmp_path, parser_and_data):
+ parser, _, _ = parser_and_data
+
+ path = tmp_path / "invalid_file.zip"
+ path.touch()
+ with open(path, "rb") as f:
+ with pytest.raises(zipfile.BadZipFile, match="File is not a zip file"):
+ parser.read_csv(f, compression="zip")
+
+
+@pytest.mark.parametrize("filename", [None, "test.{ext}"])
+def test_compression(
+ tmp_path,
+ request,
+ parser_and_data,
+ compression_only,
+ buffer,
+ filename,
+ compression_to_extension,
+):
+ parser, data, expected = parser_and_data
+ compress_type = compression_only
+
+ ext = compression_to_extension[compress_type]
+ filename = filename if filename is None else filename.format(ext=ext)
+
+ if filename and buffer:
+ request.applymarker(
+ pytest.mark.xfail(
+ reason="Cannot deduce compression from buffer of compressed data."
+ )
+ )
+
+ path = tmp_path / filename if filename else tmp_path / "test_file"
+ tm.write_to_compressed(compress_type, path, data)
+ compression = "infer" if filename else compress_type
+
+ if buffer:
+ with open(path, "rb") as f:
+ result = parser.read_csv(f, compression=compression)
+ else:
+ result = parser.read_csv(path, compression=compression)
+
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("ext", [None, "gz", "bz2"])
+def test_infer_compression(all_parsers, csv1, buffer, ext):
+ # see gh-9770
+ parser = all_parsers
+ kwargs = {"index_col": 0, "parse_dates": True}
+
+ expected = parser.read_csv(csv1, **kwargs)
+ kwargs["compression"] = "infer"
+
+ if buffer:
+ with open(csv1, encoding="utf-8") as f:
+ result = parser.read_csv(f, **kwargs)
+ else:
+ ext = "." + ext if ext else ""
+ result = parser.read_csv(csv1 + ext, **kwargs)
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_compression_utf_encoding(all_parsers, csv_dir_path, utf_value, encoding_fmt):
+ # see gh-18071, gh-24130
+ parser = all_parsers
+ encoding = encoding_fmt.format(utf_value)
+ path = os.path.join(csv_dir_path, f"utf{utf_value}_ex_small.zip")
+
+ result = parser.read_csv(path, encoding=encoding, compression="zip", sep="\t")
+ expected = DataFrame(
+ {
+ "Country": ["Venezuela", "Venezuela"],
+ "Twitter": ["Hugo Chávez Frías", "Henrique Capriles R."],
+ }
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("invalid_compression", ["sfark", "bz3", "zipper"])
+def test_invalid_compression(all_parsers, invalid_compression):
+ parser = all_parsers
+ compress_kwargs = {"compression": invalid_compression}
+
+ msg = f"Unrecognized compression type: {invalid_compression}"
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv("test_file.zip", **compress_kwargs)
+
+
+def test_compression_tar_archive(all_parsers, csv_dir_path):
+ parser = all_parsers
+ path = os.path.join(csv_dir_path, "tar_csv.tar.gz")
+ df = parser.read_csv(path)
+ assert list(df.columns) == ["a"]
+
+
+def test_ignore_compression_extension(tmp_path, all_parsers):
+ parser = all_parsers
+ df = DataFrame({"a": [0, 1]})
+
+ path_csv = tmp_path / "test.csv"
+ path_zip = tmp_path / "test.csv.zip"
+ # make sure to create un-compressed file with zip extension
+ df.to_csv(path_csv, index=False)
+ Path(path_zip).write_text(
+ Path(path_csv).read_text(encoding="utf-8"), encoding="utf-8"
+ )
+
+ tm.assert_frame_equal(parser.read_csv(path_zip, compression=None), df)
+
+
+def test_writes_tar_gz(tmp_path, all_parsers):
+ parser = all_parsers
+ data = DataFrame(
+ {
+ "Country": ["Venezuela", "Venezuela"],
+ "Twitter": ["Hugo Chávez Frías", "Henrique Capriles R."],
+ }
+ )
+ tar_path = tmp_path / "test.tar.gz"
+ data.to_csv(tar_path, index=False)
+
+ # test that read_csv infers .tar.gz to gzip:
+ tm.assert_frame_equal(parser.read_csv(tar_path), data)
+
+ # test that file is indeed gzipped:
+ with tarfile.open(tar_path, "r:gz") as tar:
+ result = parser.read_csv(
+ tar.extractfile(tar.getnames()[0]), compression="infer"
+ )
+ tm.assert_frame_equal(result, data)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_concatenate_chunks.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_concatenate_chunks.py
new file mode 100644
index 0000000000000000000000000000000000000000..3e418faeff8e03ff5b524085a78cc94a23518ae7
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_concatenate_chunks.py
@@ -0,0 +1,38 @@
+import numpy as np
+import pytest
+
+from pandas.errors import DtypeWarning
+
+import pandas._testing as tm
+from pandas.core.arrays import ArrowExtensionArray
+
+from pandas.io.parsers.c_parser_wrapper import _concatenate_chunks
+
+
+def test_concatenate_chunks_pyarrow():
+ # GH#51876
+ pa = pytest.importorskip("pyarrow")
+ chunks = [
+ {0: ArrowExtensionArray(pa.array([1.5, 2.5]))},
+ {0: ArrowExtensionArray(pa.array([1, 2]))},
+ ]
+ result = _concatenate_chunks(chunks, ["column_0", "column_1"])
+ expected = ArrowExtensionArray(pa.array([1.5, 2.5, 1.0, 2.0]))
+ tm.assert_extension_array_equal(result[0], expected)
+
+
+def test_concatenate_chunks_pyarrow_strings():
+ # GH#51876
+ pa = pytest.importorskip("pyarrow")
+ chunks = [
+ {0: ArrowExtensionArray(pa.array([1.5, 2.5]))},
+ {0: ArrowExtensionArray(pa.array(["a", "b"]))},
+ ]
+ with tm.assert_produces_warning(
+ DtypeWarning, match="Columns \\(0: column_0\\) have mixed types"
+ ):
+ result = _concatenate_chunks(chunks, ["column_0", "column_1"])
+ expected = np.concatenate(
+ [np.array([1.5, 2.5], dtype=object), np.array(["a", "b"])]
+ )
+ tm.assert_numpy_array_equal(result[0], expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_converters.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_converters.py
new file mode 100644
index 0000000000000000000000000000000000000000..87d95e77d30d7f495762ee33a474e811345af374
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_converters.py
@@ -0,0 +1,265 @@
+"""
+Tests column conversion functionality during parsing
+for all of the parsers defined in parsers.py
+"""
+
+from io import StringIO
+
+from dateutil.parser import parse
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+)
+import pandas._testing as tm
+
+
+def test_converters_type_must_be_dict(all_parsers):
+ parser = all_parsers
+ data = """index,A,B,C,D
+foo,2,3,4,5
+"""
+ if parser.engine == "pyarrow":
+ msg = "The 'converters' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), converters=0)
+ return
+ with pytest.raises(TypeError, match="Type converters.+"):
+ parser.read_csv(StringIO(data), converters=0)
+
+
+@pytest.mark.parametrize("column", [3, "D"])
+@pytest.mark.parametrize(
+ "converter",
+ [parse, lambda x: int(x.split("/")[2])], # Produce integer.
+)
+def test_converters(all_parsers, column, converter):
+ parser = all_parsers
+ data = """A,B,C,D
+a,1,2,01/01/2009
+b,3,4,01/02/2009
+c,4,5,01/03/2009
+"""
+ if parser.engine == "pyarrow":
+ msg = "The 'converters' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), converters={column: converter})
+ return
+
+ result = parser.read_csv(StringIO(data), converters={column: converter})
+
+ expected = parser.read_csv(StringIO(data))
+ expected["D"] = expected["D"].map(converter)
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_converters_no_implicit_conv(all_parsers):
+ # see gh-2184
+ parser = all_parsers
+ data = """000102,1.2,A\n001245,2,B"""
+
+ converters = {0: lambda x: x.strip()}
+
+ if parser.engine == "pyarrow":
+ msg = "The 'converters' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), header=None, converters=converters)
+ return
+
+ result = parser.read_csv(StringIO(data), header=None, converters=converters)
+
+ # Column 0 should not be casted to numeric and should remain as object.
+ expected = DataFrame([["000102", 1.2, "A"], ["001245", 2, "B"]])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_converters_euro_decimal_format(all_parsers):
+ # see gh-583
+ converters = {}
+ parser = all_parsers
+
+ data = """Id;Number1;Number2;Text1;Text2;Number3
+1;1521,1541;187101,9543;ABC;poi;4,7387
+2;121,12;14897,76;DEF;uyt;0,3773
+3;878,158;108013,434;GHI;rez;2,7356"""
+ converters["Number1"] = converters["Number2"] = converters["Number3"] = lambda x: (
+ float(x.replace(",", "."))
+ )
+
+ if parser.engine == "pyarrow":
+ msg = "The 'converters' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), sep=";", converters=converters)
+ return
+
+ result = parser.read_csv(StringIO(data), sep=";", converters=converters)
+ expected = DataFrame(
+ [
+ [1, 1521.1541, 187101.9543, "ABC", "poi", 4.7387],
+ [2, 121.12, 14897.76, "DEF", "uyt", 0.3773],
+ [3, 878.158, 108013.434, "GHI", "rez", 2.7356],
+ ],
+ columns=["Id", "Number1", "Number2", "Text1", "Text2", "Number3"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_converters_corner_with_nans(all_parsers):
+ parser = all_parsers
+ data = """id,score,days
+1,2,12
+2,2-5,
+3,,14+
+4,6-12,2"""
+
+ # Example converters.
+ def convert_days(x):
+ x = x.strip()
+
+ if not x:
+ return np.nan
+
+ is_plus = x.endswith("+")
+
+ if is_plus:
+ x = int(x[:-1]) + 1
+ else:
+ x = int(x)
+
+ return x
+
+ def convert_days_sentinel(x):
+ x = x.strip()
+
+ if not x:
+ return np.nan
+
+ is_plus = x.endswith("+")
+
+ if is_plus:
+ x = int(x[:-1]) + 1
+ else:
+ x = int(x)
+
+ return x
+
+ def convert_score(x):
+ x = x.strip()
+
+ if not x:
+ return np.nan
+
+ if x.find("-") > 0:
+ val_min, val_max = map(int, x.split("-"))
+ val = 0.5 * (val_min + val_max)
+ else:
+ val = float(x)
+
+ return val
+
+ results = []
+
+ for day_converter in [convert_days, convert_days_sentinel]:
+ if parser.engine == "pyarrow":
+ msg = "The 'converters' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data),
+ converters={"score": convert_score, "days": day_converter},
+ na_values=["", None],
+ )
+ continue
+
+ result = parser.read_csv(
+ StringIO(data),
+ converters={"score": convert_score, "days": day_converter},
+ na_values=["", None],
+ )
+ assert pd.isna(result["days"][1])
+ results.append(result)
+
+ if parser.engine != "pyarrow":
+ tm.assert_frame_equal(results[0], results[1])
+
+
+@pytest.mark.parametrize("conv_f", [lambda x: x, str])
+def test_converter_index_col_bug(all_parsers, conv_f):
+ # see gh-1835 , GH#40589
+ parser = all_parsers
+ data = "A;B\n1;2\n3;4"
+
+ if parser.engine == "pyarrow":
+ msg = "The 'converters' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data), sep=";", index_col="A", converters={"A": conv_f}
+ )
+ return
+
+ rs = parser.read_csv(
+ StringIO(data), sep=";", index_col="A", converters={"A": conv_f}
+ )
+
+ xp = DataFrame({"B": [2, 4]}, index=Index(["1", "3"], name="A"))
+ tm.assert_frame_equal(rs, xp)
+
+
+def test_converter_identity_object(all_parsers):
+ # GH#40589
+ parser = all_parsers
+ data = "A,B\n1,2\n3,4"
+
+ if parser.engine == "pyarrow":
+ msg = "The 'converters' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), converters={"A": lambda x: x})
+ return
+
+ rs = parser.read_csv(StringIO(data), converters={"A": lambda x: x})
+
+ xp = DataFrame({"A": ["1", "3"], "B": [2, 4]})
+ tm.assert_frame_equal(rs, xp)
+
+
+def test_converter_multi_index(all_parsers):
+ # GH 42446
+ parser = all_parsers
+ data = "A,B,B\nX,Y,Z\n1,2,3"
+
+ if parser.engine == "pyarrow":
+ msg = "The 'converters' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data),
+ header=list(range(2)),
+ converters={
+ ("A", "X"): np.int32,
+ ("B", "Y"): np.int32,
+ ("B", "Z"): np.float32,
+ },
+ )
+ return
+
+ result = parser.read_csv(
+ StringIO(data),
+ header=list(range(2)),
+ converters={
+ ("A", "X"): np.int32,
+ ("B", "Y"): np.int32,
+ ("B", "Z"): np.float32,
+ },
+ )
+
+ expected = DataFrame(
+ {
+ ("A", "X"): np.int32([1]),
+ ("B", "Y"): np.int32([2]),
+ ("B", "Z"): np.float32([3]),
+ }
+ )
+
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_dialect.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_dialect.py
new file mode 100644
index 0000000000000000000000000000000000000000..803114723bc7403942d9182f3f94d4edc7fd941a
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_dialect.py
@@ -0,0 +1,195 @@
+"""
+Tests that dialects are properly handled during parsing
+for all of the parsers defined in parsers.py
+"""
+
+import csv
+from io import StringIO
+
+import pytest
+
+from pandas.errors import ParserWarning
+
+from pandas import DataFrame
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+@pytest.fixture
+def custom_dialect():
+ dialect_name = "weird"
+ dialect_kwargs = {
+ "doublequote": False,
+ "escapechar": "~",
+ "delimiter": ":",
+ "skipinitialspace": False,
+ "quotechar": "`",
+ "quoting": 3,
+ }
+ return dialect_name, dialect_kwargs
+
+
+def test_dialect(all_parsers):
+ parser = all_parsers
+ data = """\
+label1,label2,label3
+index1,"a,c,e
+index2,b,d,f
+"""
+
+ dia = csv.excel()
+ dia.quoting = csv.QUOTE_NONE
+
+ if parser.engine == "pyarrow":
+ msg = "The 'dialect' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), dialect=dia)
+ return
+
+ df = parser.read_csv(StringIO(data), dialect=dia)
+
+ data = """\
+label1,label2,label3
+index1,a,c,e
+index2,b,d,f
+"""
+ exp = parser.read_csv(StringIO(data))
+ exp.replace("a", '"a', inplace=True)
+ tm.assert_frame_equal(df, exp)
+
+
+def test_dialect_str(all_parsers):
+ dialect_name = "mydialect"
+ parser = all_parsers
+ data = """\
+fruit:vegetable
+apple:broccoli
+pear:tomato
+"""
+ exp = DataFrame({"fruit": ["apple", "pear"], "vegetable": ["broccoli", "tomato"]})
+
+ with tm.with_csv_dialect(dialect_name, delimiter=":"):
+ if parser.engine == "pyarrow":
+ msg = "The 'dialect' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), dialect=dialect_name)
+ return
+
+ df = parser.read_csv(StringIO(data), dialect=dialect_name)
+ tm.assert_frame_equal(df, exp)
+
+
+def test_invalid_dialect(all_parsers):
+ class InvalidDialect:
+ pass
+
+ data = "a\n1"
+ parser = all_parsers
+ msg = "Invalid dialect"
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), dialect=InvalidDialect)
+
+
+@pytest.mark.parametrize(
+ "arg",
+ [None, "doublequote", "escapechar", "skipinitialspace", "quotechar", "quoting"],
+)
+@pytest.mark.parametrize("value", ["dialect", "default", "other"])
+def test_dialect_conflict_except_delimiter(all_parsers, custom_dialect, arg, value):
+ # see gh-23761.
+ dialect_name, dialect_kwargs = custom_dialect
+ parser = all_parsers
+
+ expected = DataFrame({"a": [1], "b": [2]})
+ data = "a:b\n1:2"
+
+ warning_klass = None
+ kwds = {}
+
+ # arg=None tests when we pass in the dialect without any other arguments.
+ if arg is not None:
+ if value == "dialect": # No conflict --> no warning.
+ kwds[arg] = dialect_kwargs[arg]
+ elif value == "default": # Default --> no warning.
+ from pandas.io.parsers.base_parser import parser_defaults
+
+ kwds[arg] = parser_defaults[arg]
+ else: # Non-default + conflict with dialect --> warning.
+ warning_klass = ParserWarning
+ kwds[arg] = "blah"
+
+ with tm.with_csv_dialect(dialect_name, **dialect_kwargs):
+ if parser.engine == "pyarrow":
+ msg = "The 'dialect' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv_check_warnings(
+ # No warning bc we raise
+ None,
+ "Conflicting values for",
+ StringIO(data),
+ dialect=dialect_name,
+ **kwds,
+ )
+ return
+ result = parser.read_csv_check_warnings(
+ warning_klass,
+ "Conflicting values for",
+ StringIO(data),
+ dialect=dialect_name,
+ **kwds,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "kwargs,warning_klass",
+ [
+ ({"sep": ","}, None), # sep is default --> sep_override=True
+ ({"sep": "."}, ParserWarning), # sep isn't default --> sep_override=False
+ ({"delimiter": ":"}, None), # No conflict
+ ({"delimiter": None}, None), # Default arguments --> sep_override=True
+ ({"delimiter": ","}, ParserWarning), # Conflict
+ ({"delimiter": "."}, ParserWarning), # Conflict
+ ],
+ ids=[
+ "sep-override-true",
+ "sep-override-false",
+ "delimiter-no-conflict",
+ "delimiter-default-arg",
+ "delimiter-conflict",
+ "delimiter-conflict2",
+ ],
+)
+def test_dialect_conflict_delimiter(all_parsers, custom_dialect, kwargs, warning_klass):
+ # see gh-23761.
+ dialect_name, dialect_kwargs = custom_dialect
+ parser = all_parsers
+
+ expected = DataFrame({"a": [1], "b": [2]})
+ data = "a:b\n1:2"
+
+ with tm.with_csv_dialect(dialect_name, **dialect_kwargs):
+ if parser.engine == "pyarrow":
+ msg = "The 'dialect' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv_check_warnings(
+ # no warning bc we raise
+ None,
+ "Conflicting values for 'delimiter'",
+ StringIO(data),
+ dialect=dialect_name,
+ **kwargs,
+ )
+ return
+ result = parser.read_csv_check_warnings(
+ warning_klass,
+ "Conflicting values for 'delimiter'",
+ StringIO(data),
+ dialect=dialect_name,
+ **kwargs,
+ )
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_encoding.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_encoding.py
new file mode 100644
index 0000000000000000000000000000000000000000..50d42f15cc0f82c8d6055dd1ae68b4e4c1543f80
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_encoding.py
@@ -0,0 +1,333 @@
+"""
+Tests encoding functionality during parsing
+for all of the parsers defined in parsers.py
+"""
+
+from io import (
+ BytesIO,
+ TextIOWrapper,
+)
+import os
+import tempfile
+
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ read_csv,
+)
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+
+def test_bytes_io_input(all_parsers):
+ encoding = "cp1255"
+ parser = all_parsers
+
+ data = BytesIO("שלום:1234\n562:123".encode(encoding))
+ result = parser.read_csv(data, sep=":", encoding=encoding)
+
+ expected = DataFrame([[562, 123]], columns=["שלום", "1234"])
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+def test_read_csv_unicode(all_parsers):
+ parser = all_parsers
+ data = BytesIO("\u0141aski, Jan;1".encode())
+
+ result = parser.read_csv(data, sep=";", encoding="utf-8", header=None)
+ expected = DataFrame([["\u0141aski, Jan", 1]])
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow
+@pytest.mark.parametrize("sep", [",", "\t"])
+@pytest.mark.parametrize("encoding", ["utf-16", "utf-16le", "utf-16be"])
+def test_utf16_bom_skiprows(all_parsers, sep, encoding, temp_file):
+ # see gh-2298
+ parser = all_parsers
+ data = """skip this
+skip this too
+A,B,C
+1,2,3
+4,5,6""".replace(",", sep)
+ kwargs = {"sep": sep, "skiprows": 2}
+ utf8 = "utf-8"
+
+ bytes_data = data.encode(encoding)
+
+ with open(temp_file, "wb") as f:
+ f.write(bytes_data)
+
+ with TextIOWrapper(BytesIO(data.encode(utf8)), encoding=utf8) as bytes_buffer:
+ result = parser.read_csv(temp_file, encoding=encoding, **kwargs)
+ expected = parser.read_csv(bytes_buffer, encoding=utf8, **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_utf16_example(all_parsers, csv_dir_path):
+ path = os.path.join(csv_dir_path, "utf16_ex.txt")
+ parser = all_parsers
+ result = parser.read_csv(path, encoding="utf-16", sep="\t")
+ assert len(result) == 50
+
+
+def test_unicode_encoding(all_parsers, csv_dir_path):
+ path = os.path.join(csv_dir_path, "unicode_series.csv")
+ parser = all_parsers
+
+ result = parser.read_csv(path, header=None, encoding="latin-1")
+ result = result.set_index(0)
+ got = result[1][1632]
+
+ expected = "\xc1 k\xf6ldum klaka (Cold Fever) (1994)"
+ assert got == expected
+
+
+@pytest.mark.parametrize(
+ "data,kwargs,expected",
+ [
+ # Basic test
+ ("a\n1", {}, [1]),
+ # "Regular" quoting
+ ('"a"\n1', {"quotechar": '"'}, [1]),
+ # Test in a data row instead of header
+ ("b\n1", {"names": ["a"]}, ["b", "1"]),
+ # Test in empty data row with skipping
+ ("\n1", {"names": ["a"], "skip_blank_lines": True}, [1]),
+ # Test in empty data row without skipping
+ (
+ "\n1",
+ {"names": ["a"], "skip_blank_lines": False},
+ [np.nan, 1],
+ ),
+ ],
+)
+def test_utf8_bom(all_parsers, data, kwargs, expected):
+ # see gh-4793
+ parser = all_parsers
+ bom = "\ufeff"
+ utf8 = "utf-8"
+
+ def _encode_data_with_bom(_data):
+ bom_data = (bom + _data).encode(utf8)
+ return BytesIO(bom_data)
+
+ if (
+ parser.engine == "pyarrow"
+ and data == "\n1"
+ and kwargs.get("skip_blank_lines", True)
+ ):
+ # CSV parse error: Empty CSV file or block: cannot infer number of columns
+ pytest.skip(reason="https://github.com/apache/arrow/issues/38676")
+
+ result = parser.read_csv(_encode_data_with_bom(data), encoding=utf8, **kwargs)
+ expected = DataFrame({"a": expected})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_csv_utf_aliases(all_parsers, utf_value, encoding_fmt):
+ # see gh-13549
+ expected = DataFrame({"mb_num": [4.8], "multibyte": ["test"]})
+ parser = all_parsers
+
+ encoding = encoding_fmt.format(utf_value)
+ data = "mb_num,multibyte\n4.8,test".encode(encoding)
+
+ result = parser.read_csv(BytesIO(data), encoding=encoding)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "file_path,encoding",
+ [
+ (("io", "data", "csv", "test1.csv"), "utf-8"),
+ (("io", "parser", "data", "unicode_series.csv"), "latin-1"),
+ (("io", "parser", "data", "sauron.SHIFT_JIS.csv"), "shiftjis"),
+ ],
+)
+def test_binary_mode_file_buffers(all_parsers, file_path, encoding, datapath):
+ # gh-23779: Python csv engine shouldn't error on files opened in binary.
+ # gh-31575: Python csv engine shouldn't error on files opened in raw binary.
+ parser = all_parsers
+
+ fpath = datapath(*file_path)
+ expected = parser.read_csv(fpath, encoding=encoding)
+
+ with open(fpath, encoding=encoding) as fa:
+ result = parser.read_csv(fa)
+ assert not fa.closed
+ tm.assert_frame_equal(expected, result)
+
+ with open(fpath, mode="rb") as fb:
+ result = parser.read_csv(fb, encoding=encoding)
+ assert not fb.closed
+ tm.assert_frame_equal(expected, result)
+
+ with open(fpath, mode="rb", buffering=0) as fb:
+ result = parser.read_csv(fb, encoding=encoding)
+ assert not fb.closed
+ tm.assert_frame_equal(expected, result)
+
+
+@pytest.mark.parametrize("pass_encoding", [True, False])
+def test_encoding_temp_file(
+ all_parsers, utf_value, encoding_fmt, pass_encoding, temp_file
+):
+ # see gh-24130
+ parser = all_parsers
+ encoding = encoding_fmt.format(utf_value)
+
+ if parser.engine == "pyarrow" and pass_encoding is True and utf_value in [16, 32]:
+ # FIXME: this is bad!
+ pytest.skip("These cases freeze")
+
+ expected = DataFrame({"foo": ["bar"]})
+
+ with temp_file.open(mode="w+", encoding=encoding) as f:
+ f.write("foo\nbar")
+ f.seek(0)
+
+ result = parser.read_csv(f, encoding=encoding if pass_encoding else None)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_encoding_named_temp_file(all_parsers):
+ # see gh-31819
+ parser = all_parsers
+ encoding = "shift-jis"
+
+ title = "てすと"
+ data = "こむ"
+
+ expected = DataFrame({title: [data]})
+
+ with tempfile.NamedTemporaryFile() as f:
+ f.write(f"{title}\n{data}".encode(encoding))
+
+ f.seek(0)
+
+ result = parser.read_csv(f, encoding=encoding)
+ tm.assert_frame_equal(result, expected)
+ assert not f.closed
+
+
+@pytest.mark.parametrize(
+ "encoding", ["utf-8", "utf-16", "utf-16-be", "utf-16-le", "utf-32"]
+)
+def test_parse_encoded_special_characters(encoding):
+ # GH16218 Verify parsing of data with encoded special characters
+ # Data contains a Unicode 'FULLWIDTH COLON' (U+FF1A) at position (0,"a")
+ data = "a\tb\n:foo\t0\nbar\t1\nbaz\t2" # noqa: RUF001
+ encoded_data = BytesIO(data.encode(encoding))
+ result = read_csv(encoded_data, delimiter="\t", encoding=encoding)
+
+ expected = DataFrame(
+ data=[[":foo", 0], ["bar", 1], ["baz", 2]], # noqa: RUF001
+ columns=["a", "b"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("encoding", ["utf-8", None, "utf-16", "cp1255", "latin-1"])
+def test_encoding_memory_map(all_parsers, encoding, temp_file):
+ # GH40986
+ parser = all_parsers
+ expected = DataFrame(
+ {
+ "name": ["Raphael", "Donatello", "Miguel Angel", "Leonardo"],
+ "mask": ["red", "purple", "orange", "blue"],
+ "weapon": ["sai", "bo staff", "nunchunk", "katana"],
+ }
+ )
+ expected.to_csv(temp_file, index=False, encoding=encoding)
+
+ if parser.engine == "pyarrow":
+ msg = "The 'memory_map' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(temp_file, encoding=encoding, memory_map=True)
+ return
+
+ df = parser.read_csv(temp_file, encoding=encoding, memory_map=True)
+ tm.assert_frame_equal(df, expected)
+
+
+def test_chunk_splits_multibyte_char(all_parsers, temp_file):
+ """
+ Chunk splits a multibyte character with memory_map=True
+
+ GH 43540
+ """
+ parser = all_parsers
+ # DEFAULT_CHUNKSIZE = 262144, defined in parsers.pyx
+ df = DataFrame(data=["a" * 127] * 2048)
+
+ # Put two-bytes utf-8 encoded character "ą" at the end of chunk
+ # utf-8 encoding of "ą" is b'\xc4\x85'
+ df.iloc[2047] = "a" * 127 + "ą"
+ df.to_csv(temp_file, index=False, header=False, encoding="utf-8")
+
+ if parser.engine == "pyarrow":
+ msg = "The 'memory_map' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(temp_file, header=None, memory_map=True)
+ return
+
+ dfr = parser.read_csv(temp_file, header=None, memory_map=True)
+ tm.assert_frame_equal(dfr, df)
+
+
+def test_readcsv_memmap_utf8(all_parsers, temp_file):
+ """
+ GH 43787
+
+ Test correct handling of UTF-8 chars when memory_map=True and encoding is UTF-8
+ """
+ lines = []
+ line_length = 128
+ start_char = " "
+ end_char = "\U00010080"
+ # This for loop creates a list of 128-char strings
+ # consisting of consecutive Unicode chars
+ for lnum in range(ord(start_char), ord(end_char), line_length):
+ line = "".join([chr(c) for c in range(lnum, lnum + 0x80)]) + "\n"
+ try:
+ line.encode("utf-8")
+ except UnicodeEncodeError:
+ continue
+ lines.append(line)
+ parser = all_parsers
+ df = DataFrame(lines)
+ df.to_csv(temp_file, index=False, header=False, encoding="utf-8")
+
+ if parser.engine == "pyarrow":
+ msg = "The 'memory_map' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(temp_file, header=None, memory_map=True, encoding="utf-8")
+ return
+
+ dfr = parser.read_csv(temp_file, header=None, memory_map=True, encoding="utf-8")
+ tm.assert_frame_equal(df, dfr)
+
+
+@pytest.mark.usefixtures("pyarrow_xfail")
+@pytest.mark.parametrize("mode", ["w+b", "w+t"])
+def test_not_readable(all_parsers, mode):
+ # GH43439
+ parser = all_parsers
+ content = b"abcd"
+ if "t" in mode:
+ content = "abcd"
+ with tempfile.SpooledTemporaryFile(mode=mode, encoding="utf-8") as handle:
+ handle.write(content)
+ handle.seek(0)
+ df = parser.read_csv(handle)
+ expected = DataFrame([], columns=["abcd"])
+ tm.assert_frame_equal(df, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_header.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_header.py
new file mode 100644
index 0000000000000000000000000000000000000000..d333aef723de29411ac15e606d0f13de792b8b01
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_header.py
@@ -0,0 +1,727 @@
+"""
+Tests that the file header is properly handled or inferred
+during parsing for all of the parsers defined in parsers.py
+"""
+
+from collections import namedtuple
+from io import StringIO
+
+import numpy as np
+import pytest
+
+from pandas.errors import ParserError
+
+from pandas import (
+ DataFrame,
+ Index,
+ MultiIndex,
+)
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_read_with_bad_header(all_parsers):
+ parser = all_parsers
+ msg = r"but only \d+ lines in file"
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(",,"), header=[10])
+
+
+def test_negative_header(all_parsers):
+ # see gh-27779
+ parser = all_parsers
+ data = """1,2,3,4,5
+6,7,8,9,10
+11,12,13,14,15
+"""
+ with pytest.raises(
+ ValueError,
+ match="Passing negative integer to header is invalid. "
+ "For no header, use header=None instead",
+ ):
+ parser.read_csv(StringIO(data), header=-1)
+
+
+@pytest.mark.parametrize("header", [([-1, 2, 4]), ([-5, 0])])
+def test_negative_multi_index_header(all_parsers, header):
+ # see gh-27779
+ parser = all_parsers
+ data = """1,2,3,4,5
+ 6,7,8,9,10
+ 11,12,13,14,15
+ """
+ with pytest.raises(
+ ValueError, match="cannot specify multi-index header with negative integers"
+ ):
+ parser.read_csv(StringIO(data), header=header)
+
+
+@pytest.mark.parametrize("header", [True, False])
+def test_bool_header_arg(all_parsers, header):
+ # see gh-6114
+ parser = all_parsers
+ data = """\
+MyColumn
+a
+b
+a
+b"""
+ msg = "Passing a bool to header is invalid"
+ with pytest.raises(TypeError, match=msg):
+ parser.read_csv(StringIO(data), header=header)
+
+
+@xfail_pyarrow # AssertionError: DataFrame are different
+def test_header_with_index_col(all_parsers):
+ parser = all_parsers
+ data = """foo,1,2,3
+bar,4,5,6
+baz,7,8,9
+"""
+ names = ["A", "B", "C"]
+ result = parser.read_csv(StringIO(data), names=names)
+
+ expected = DataFrame(
+ [[1, 2, 3], [4, 5, 6], [7, 8, 9]],
+ index=["foo", "bar", "baz"],
+ columns=["A", "B", "C"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_header_not_first_line(all_parsers):
+ parser = all_parsers
+ data = """got,to,ignore,this,line
+got,to,ignore,this,line
+index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+"""
+ data2 = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+"""
+
+ result = parser.read_csv(StringIO(data), header=2, index_col=0)
+ expected = parser.read_csv(StringIO(data2), header=0, index_col=0)
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_header_multi_index(all_parsers):
+ parser = all_parsers
+
+ data = """\
+C0,,C_l0_g0,C_l0_g1,C_l0_g2
+
+C1,,C_l1_g0,C_l1_g1,C_l1_g2
+C2,,C_l2_g0,C_l2_g1,C_l2_g2
+C3,,C_l3_g0,C_l3_g1,C_l3_g2
+R0,R1,,,
+R_l0_g0,R_l1_g0,R0C0,R0C1,R0C2
+R_l0_g1,R_l1_g1,R1C0,R1C1,R1C2
+R_l0_g2,R_l1_g2,R2C0,R2C1,R2C2
+R_l0_g3,R_l1_g3,R3C0,R3C1,R3C2
+R_l0_g4,R_l1_g4,R4C0,R4C1,R4C2
+"""
+ result = parser.read_csv(StringIO(data), header=[0, 1, 2, 3], index_col=[0, 1])
+ data_gen_f = lambda r, c: f"R{r}C{c}"
+
+ data = [[data_gen_f(r, c) for c in range(3)] for r in range(5)]
+ index = MultiIndex.from_arrays(
+ [[f"R_l0_g{i}" for i in range(5)], [f"R_l1_g{i}" for i in range(5)]],
+ names=["R0", "R1"],
+ )
+ columns = MultiIndex.from_arrays(
+ [
+ [f"C_l0_g{i}" for i in range(3)],
+ [f"C_l1_g{i}" for i in range(3)],
+ [f"C_l2_g{i}" for i in range(3)],
+ [f"C_l3_g{i}" for i in range(3)],
+ ],
+ names=["C0", "C1", "C2", "C3"],
+ )
+ expected = DataFrame(data, columns=columns, index=index)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "kwargs,msg",
+ [
+ (
+ {"index_col": ["foo", "bar"]},
+ (
+ "index_col must only contain "
+ "integers of column positions when specifying "
+ "a multi-index header"
+ ),
+ ),
+ (
+ {"index_col": [0, 1], "names": ["foo", "bar"]},
+ ("cannot specify names when specifying a multi-index header"),
+ ),
+ (
+ {"index_col": [0, 1], "usecols": ["foo", "bar"]},
+ ("cannot specify usecols when specifying a multi-index header"),
+ ),
+ ],
+)
+def test_header_multi_index_invalid(all_parsers, kwargs, msg):
+ data = """\
+C0,,C_l0_g0,C_l0_g1,C_l0_g2
+
+C1,,C_l1_g0,C_l1_g1,C_l1_g2
+C2,,C_l2_g0,C_l2_g1,C_l2_g2
+C3,,C_l3_g0,C_l3_g1,C_l3_g2
+R0,R1,,,
+R_l0_g0,R_l1_g0,R0C0,R0C1,R0C2
+R_l0_g1,R_l1_g1,R1C0,R1C1,R1C2
+R_l0_g2,R_l1_g2,R2C0,R2C1,R2C2
+R_l0_g3,R_l1_g3,R3C0,R3C1,R3C2
+R_l0_g4,R_l1_g4,R4C0,R4C1,R4C2
+"""
+ parser = all_parsers
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), header=[0, 1, 2, 3], **kwargs)
+
+
+_TestTuple = namedtuple("_TestTuple", ["first", "second"])
+
+
+@xfail_pyarrow # TypeError: an integer is required
+@pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"header": [0, 1]},
+ {
+ "skiprows": 3,
+ "names": [
+ ("a", "q"),
+ ("a", "r"),
+ ("a", "s"),
+ ("b", "t"),
+ ("c", "u"),
+ ("c", "v"),
+ ],
+ },
+ {
+ "skiprows": 3,
+ "names": [
+ _TestTuple("a", "q"),
+ _TestTuple("a", "r"),
+ _TestTuple("a", "s"),
+ _TestTuple("b", "t"),
+ _TestTuple("c", "u"),
+ _TestTuple("c", "v"),
+ ],
+ },
+ ],
+)
+def test_header_multi_index_common_format1(all_parsers, kwargs):
+ parser = all_parsers
+ expected = DataFrame(
+ [[1, 2, 3, 4, 5, 6], [7, 8, 9, 10, 11, 12]],
+ index=["one", "two"],
+ columns=MultiIndex.from_tuples(
+ [("a", "q"), ("a", "r"), ("a", "s"), ("b", "t"), ("c", "u"), ("c", "v")]
+ ),
+ )
+ data = """,a,a,a,b,c,c
+,q,r,s,t,u,v
+,,,,,,
+one,1,2,3,4,5,6
+two,7,8,9,10,11,12"""
+
+ result = parser.read_csv(StringIO(data), index_col=0, **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+@pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"header": [0, 1]},
+ {
+ "skiprows": 2,
+ "names": [
+ ("a", "q"),
+ ("a", "r"),
+ ("a", "s"),
+ ("b", "t"),
+ ("c", "u"),
+ ("c", "v"),
+ ],
+ },
+ {
+ "skiprows": 2,
+ "names": [
+ _TestTuple("a", "q"),
+ _TestTuple("a", "r"),
+ _TestTuple("a", "s"),
+ _TestTuple("b", "t"),
+ _TestTuple("c", "u"),
+ _TestTuple("c", "v"),
+ ],
+ },
+ ],
+)
+def test_header_multi_index_common_format2(all_parsers, kwargs):
+ parser = all_parsers
+ expected = DataFrame(
+ [[1, 2, 3, 4, 5, 6], [7, 8, 9, 10, 11, 12]],
+ index=["one", "two"],
+ columns=MultiIndex.from_tuples(
+ [("a", "q"), ("a", "r"), ("a", "s"), ("b", "t"), ("c", "u"), ("c", "v")]
+ ),
+ )
+ data = """,a,a,a,b,c,c
+,q,r,s,t,u,v
+one,1,2,3,4,5,6
+two,7,8,9,10,11,12"""
+
+ result = parser.read_csv(StringIO(data), index_col=0, **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+@pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"header": [0, 1]},
+ {
+ "skiprows": 2,
+ "names": [
+ ("a", "q"),
+ ("a", "r"),
+ ("a", "s"),
+ ("b", "t"),
+ ("c", "u"),
+ ("c", "v"),
+ ],
+ },
+ {
+ "skiprows": 2,
+ "names": [
+ _TestTuple("a", "q"),
+ _TestTuple("a", "r"),
+ _TestTuple("a", "s"),
+ _TestTuple("b", "t"),
+ _TestTuple("c", "u"),
+ _TestTuple("c", "v"),
+ ],
+ },
+ ],
+)
+def test_header_multi_index_common_format3(all_parsers, kwargs):
+ parser = all_parsers
+ expected = DataFrame(
+ [[1, 2, 3, 4, 5, 6], [7, 8, 9, 10, 11, 12]],
+ index=["one", "two"],
+ columns=MultiIndex.from_tuples(
+ [("a", "q"), ("a", "r"), ("a", "s"), ("b", "t"), ("c", "u"), ("c", "v")]
+ ),
+ )
+ expected = expected.reset_index(drop=True)
+ data = """a,a,a,b,c,c
+q,r,s,t,u,v
+1,2,3,4,5,6
+7,8,9,10,11,12"""
+
+ result = parser.read_csv(StringIO(data), index_col=None, **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_header_multi_index_common_format_malformed1(all_parsers):
+ parser = all_parsers
+ expected = DataFrame(
+ np.array([[2, 3, 4, 5, 6], [8, 9, 10, 11, 12]], dtype="int64"),
+ index=Index([1, 7]),
+ columns=MultiIndex(
+ levels=[["a", "b", "c"], ["r", "s", "t", "u", "v"]],
+ codes=[[0, 0, 1, 2, 2], [0, 1, 2, 3, 4]],
+ names=["a", "q"],
+ ),
+ )
+ data = """a,a,a,b,c,c
+q,r,s,t,u,v
+1,2,3,4,5,6
+7,8,9,10,11,12"""
+
+ result = parser.read_csv(StringIO(data), header=[0, 1], index_col=0)
+ tm.assert_frame_equal(expected, result)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_header_multi_index_common_format_malformed2(all_parsers):
+ parser = all_parsers
+ expected = DataFrame(
+ np.array([[2, 3, 4, 5, 6], [8, 9, 10, 11, 12]], dtype="int64"),
+ index=range(1, 13, 6),
+ columns=MultiIndex(
+ levels=[["a", "b", "c"], ["r", "s", "t", "u", "v"]],
+ codes=[[0, 0, 1, 2, 2], [0, 1, 2, 3, 4]],
+ names=[None, "q"],
+ ),
+ )
+
+ data = """,a,a,b,c,c
+q,r,s,t,u,v
+1,2,3,4,5,6
+7,8,9,10,11,12"""
+
+ result = parser.read_csv(StringIO(data), header=[0, 1], index_col=0)
+ tm.assert_frame_equal(expected, result)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_header_multi_index_common_format_malformed3(all_parsers):
+ parser = all_parsers
+ expected = DataFrame(
+ np.array([[3, 4, 5, 6], [9, 10, 11, 12]], dtype="int64"),
+ index=MultiIndex(levels=[[1, 7], [2, 8]], codes=[[0, 1], [0, 1]]),
+ columns=MultiIndex(
+ levels=[["a", "b", "c"], ["s", "t", "u", "v"]],
+ codes=[[0, 1, 2, 2], [0, 1, 2, 3]],
+ names=[None, "q"],
+ ),
+ )
+ data = """,a,a,b,c,c
+q,r,s,t,u,v
+1,2,3,4,5,6
+7,8,9,10,11,12"""
+
+ result = parser.read_csv(StringIO(data), header=[0, 1], index_col=[0, 1])
+ tm.assert_frame_equal(expected, result)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_header_multi_index_blank_line(all_parsers):
+ # GH 40442
+ parser = all_parsers
+ data = [[None, None], [1, 2], [3, 4]]
+ columns = MultiIndex.from_tuples([("a", "A"), ("b", "B")])
+ expected = DataFrame(data, columns=columns)
+ data = "a,b\nA,B\n,\n1,2\n3,4"
+ result = parser.read_csv(StringIO(data), header=[0, 1])
+ tm.assert_frame_equal(expected, result)
+
+
+@pytest.mark.parametrize(
+ "data,header", [("1,2,3\n4,5,6", None), ("foo,bar,baz\n1,2,3\n4,5,6", 0)]
+)
+def test_header_names_backward_compat(all_parsers, data, header, request):
+ # see gh-2539
+ parser = all_parsers
+
+ if parser.engine == "pyarrow" and header is not None:
+ mark = pytest.mark.xfail(reason="DataFrame.columns are different")
+ request.applymarker(mark)
+
+ expected = parser.read_csv(StringIO("1,2,3\n4,5,6"), names=["a", "b", "c"])
+
+ result = parser.read_csv(StringIO(data), names=["a", "b", "c"], header=header)
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block: cannot infer
+@pytest.mark.parametrize("kwargs", [{}, {"index_col": False}])
+def test_read_only_header_no_rows(all_parsers, kwargs):
+ # See gh-7773
+ parser = all_parsers
+ expected = DataFrame(columns=["a", "b", "c"])
+
+ result = parser.read_csv(StringIO("a,b,c"), **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "kwargs,names",
+ [
+ ({}, [0, 1, 2, 3, 4]),
+ (
+ {"names": ["foo", "bar", "baz", "quux", "panda"]},
+ ["foo", "bar", "baz", "quux", "panda"],
+ ),
+ ],
+)
+def test_no_header(all_parsers, kwargs, names):
+ parser = all_parsers
+ data = """1,2,3,4,5
+6,7,8,9,10
+11,12,13,14,15
+"""
+ expected = DataFrame(
+ [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10], [11, 12, 13, 14, 15]], columns=names
+ )
+ result = parser.read_csv(StringIO(data), header=None, **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("header", [["a", "b"], "string_header"])
+def test_non_int_header(all_parsers, header):
+ # see gh-16338
+ msg = "header must be integer or list of integers"
+ data = """1,2\n3,4"""
+ parser = all_parsers
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), header=header)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_singleton_header(all_parsers):
+ # see gh-7757
+ data = """a,b,c\n0,1,2\n1,2,3"""
+ parser = all_parsers
+
+ expected = DataFrame({"a": [0, 1], "b": [1, 2], "c": [2, 3]})
+ result = parser.read_csv(StringIO(data), header=[0])
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+@pytest.mark.parametrize(
+ "data,expected",
+ [
+ (
+ "A,A,A,B\none,one,one,two\n0,40,34,0.1",
+ DataFrame(
+ [[0, 40, 34, 0.1]],
+ columns=MultiIndex.from_tuples(
+ [("A", "one"), ("A", "one.1"), ("A", "one.2"), ("B", "two")]
+ ),
+ ),
+ ),
+ (
+ "A,A,A,B\none,one,one.1,two\n0,40,34,0.1",
+ DataFrame(
+ [[0, 40, 34, 0.1]],
+ columns=MultiIndex.from_tuples(
+ [("A", "one"), ("A", "one.1"), ("A", "one.1.1"), ("B", "two")]
+ ),
+ ),
+ ),
+ (
+ "A,A,A,B,B\none,one,one.1,two,two\n0,40,34,0.1,0.1",
+ DataFrame(
+ [[0, 40, 34, 0.1, 0.1]],
+ columns=MultiIndex.from_tuples(
+ [
+ ("A", "one"),
+ ("A", "one.1"),
+ ("A", "one.1.1"),
+ ("B", "two"),
+ ("B", "two.1"),
+ ]
+ ),
+ ),
+ ),
+ ],
+)
+def test_mangles_multi_index(all_parsers, data, expected):
+ # see gh-18062
+ parser = all_parsers
+
+ result = parser.read_csv(StringIO(data), header=[0, 1])
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+@pytest.mark.parametrize("index_col", [None, [0]])
+@pytest.mark.parametrize(
+ "columns", [None, (["", "Unnamed"]), (["Unnamed", ""]), (["Unnamed", "NotUnnamed"])]
+)
+def test_multi_index_unnamed(all_parsers, index_col, columns):
+ # see gh-23687
+ #
+ # When specifying a multi-index header, make sure that
+ # we don't error just because one of the rows in our header
+ # has ALL column names containing the string "Unnamed". The
+ # correct condition to check is whether the row contains
+ # ALL columns that did not have names (and instead were given
+ # placeholder ones).
+ parser = all_parsers
+ header = [0, 1]
+
+ if index_col is None:
+ data = ",".join(columns or ["", ""]) + "\n0,1\n2,3\n4,5\n"
+ else:
+ data = ",".join([""] + (columns or ["", ""])) + "\n,0,1\n0,2,3\n1,4,5\n"
+
+ result = parser.read_csv(StringIO(data), header=header, index_col=index_col)
+ exp_columns = []
+
+ if columns is None:
+ columns = ["", "", ""]
+
+ for i, col in enumerate(columns):
+ if not col: # Unnamed.
+ col = f"Unnamed: {i if index_col is None else i + 1}_level_0"
+
+ exp_columns.append(col)
+
+ columns = MultiIndex.from_tuples(zip(exp_columns, ["0", "1"]))
+ expected = DataFrame([[2, 3], [4, 5]], columns=columns)
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Expected 2 columns, got 3
+def test_names_longer_than_header_but_equal_with_data_rows(all_parsers):
+ # GH#38453
+ parser = all_parsers
+ data = """a, b
+1,2,3
+5,6,4
+"""
+ result = parser.read_csv(StringIO(data), header=0, names=["A", "B", "C"])
+ expected = DataFrame({"A": [1, 5], "B": [2, 6], "C": [3, 4]})
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_read_csv_multiindex_columns(all_parsers):
+ # GH#6051
+ parser = all_parsers
+
+ s1 = "Male, Male, Male, Female, Female\nR, R, L, R, R\n.86, .67, .88, .78, .81"
+ s2 = (
+ "Male, Male, Male, Female, Female\n"
+ "R, R, L, R, R\n"
+ ".86, .67, .88, .78, .81\n"
+ ".86, .67, .88, .78, .82"
+ )
+
+ mi = MultiIndex.from_tuples(
+ [
+ ("Male", "R"),
+ (" Male", " R"),
+ (" Male", " L"),
+ (" Female", " R"),
+ (" Female", " R.1"),
+ ]
+ )
+ expected = DataFrame(
+ [[0.86, 0.67, 0.88, 0.78, 0.81], [0.86, 0.67, 0.88, 0.78, 0.82]], columns=mi
+ )
+
+ df1 = parser.read_csv(StringIO(s1), header=[0, 1])
+ tm.assert_frame_equal(df1, expected.iloc[:1])
+ df2 = parser.read_csv(StringIO(s2), header=[0, 1])
+ tm.assert_frame_equal(df2, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_read_csv_multi_header_length_check(all_parsers):
+ # GH#43102
+ parser = all_parsers
+
+ case = """row11,row12,row13
+row21,row22, row23
+row31,row32
+"""
+
+ with pytest.raises(
+ ParserError, match="Header rows must have an equal number of columns."
+ ):
+ parser.read_csv(StringIO(case), header=[0, 2])
+
+
+@skip_pyarrow # CSV parse error: Expected 3 columns, got 2
+def test_header_none_and_implicit_index(all_parsers):
+ # GH#22144
+ parser = all_parsers
+ data = "x,1,5\ny,2\nz,3\n"
+ result = parser.read_csv(StringIO(data), names=["a", "b"], header=None)
+ expected = DataFrame(
+ {"a": [1, 2, 3], "b": [5, np.nan, np.nan]}, index=["x", "y", "z"]
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # regex mismatch "CSV parse error: Expected 2 columns, got "
+def test_header_none_and_implicit_index_in_second_row(all_parsers):
+ # GH#22144
+ parser = all_parsers
+ data = "x,1\ny,2,5\nz,3\n"
+ with pytest.raises(ParserError, match="Expected 2 fields in line 2, saw 3"):
+ parser.read_csv(StringIO(data), names=["a", "b"], header=None)
+
+
+def test_header_none_and_on_bad_lines_skip(all_parsers):
+ # GH#22144
+ parser = all_parsers
+ data = "x,1\ny,2,5\nz,3\n"
+ result = parser.read_csv(
+ StringIO(data), names=["a", "b"], header=None, on_bad_lines="skip"
+ )
+ expected = DataFrame({"a": ["x", "z"], "b": [1, 3]})
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_header_missing_rows(all_parsers):
+ # GH#47400
+ parser = all_parsers
+ data = """a,b
+1,2
+"""
+ msg = r"Passed header=\[0,1,2\], len of 3, but only 2 lines in file"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), header=[0, 1, 2])
+
+
+# ValueError: the 'pyarrow' engine does not support regex separators
+@xfail_pyarrow
+def test_header_multiple_whitespaces(all_parsers):
+ # GH#54931
+ parser = all_parsers
+ data = """aa bb(1,1) cc(1,1)
+ 0 2 3.5"""
+
+ result = parser.read_csv(StringIO(data), sep=r"\s+")
+ expected = DataFrame({"aa": [0], "bb(1,1)": 2, "cc(1,1)": 3.5})
+ tm.assert_frame_equal(result, expected)
+
+
+# ValueError: the 'pyarrow' engine does not support regex separators
+@xfail_pyarrow
+def test_header_delim_whitespace(all_parsers):
+ # GH#54918
+ parser = all_parsers
+ data = """a,b
+1,2
+3,4
+ """
+ result = parser.read_csv(StringIO(data), sep=r"\s+")
+ expected = DataFrame({"a,b": ["1,2", "3,4"]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_usecols_no_header_pyarrow(pyarrow_parser_only):
+ parser = pyarrow_parser_only
+ data = """
+a,i,x
+b,j,y
+"""
+ result = parser.read_csv(
+ StringIO(data),
+ header=None,
+ usecols=[0, 1],
+ dtype="string[pyarrow]",
+ dtype_backend="pyarrow",
+ engine="pyarrow",
+ )
+ expected = DataFrame([["a", "i"], ["b", "j"]], dtype="string[pyarrow]")
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_index_col.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_index_col.py
new file mode 100644
index 0000000000000000000000000000000000000000..73875468f45fced28b0a7403f4d8966daeedec66
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_index_col.py
@@ -0,0 +1,375 @@
+"""
+Tests that the specified index column (a.k.a "index_col")
+is properly handled or inferred during parsing for all of
+the parsers defined in parsers.py
+"""
+
+from io import StringIO
+
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ Index,
+ MultiIndex,
+)
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+
+@pytest.mark.parametrize("with_header", [True, False])
+def test_index_col_named(all_parsers, with_header):
+ parser = all_parsers
+ no_header = """\
+KORD1,19990127, 19:00:00, 18:56:00, 0.8100, 2.8100, 7.2000, 0.0000, 280.0000
+KORD2,19990127, 20:00:00, 19:56:00, 0.0100, 2.2100, 7.2000, 0.0000, 260.0000
+KORD3,19990127, 21:00:00, 20:56:00, -0.5900, 2.2100, 5.7000, 0.0000, 280.0000
+KORD4,19990127, 21:00:00, 21:18:00, -0.9900, 2.0100, 3.6000, 0.0000, 270.0000
+KORD5,19990127, 22:00:00, 21:56:00, -0.5900, 1.7100, 5.1000, 0.0000, 290.0000
+KORD6,19990127, 23:00:00, 22:56:00, -0.5900, 1.7100, 4.6000, 0.0000, 280.0000"""
+ header = "ID,date,NominalTime,ActualTime,TDew,TAir,Windspeed,Precip,WindDir\n"
+
+ if with_header:
+ data = header + no_header
+
+ result = parser.read_csv(StringIO(data), index_col="ID")
+ expected = parser.read_csv(StringIO(data), header=0).set_index("ID")
+ tm.assert_frame_equal(result, expected)
+ else:
+ data = no_header
+ msg = "Index ID invalid"
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), index_col="ID")
+
+
+def test_index_col_named2(all_parsers):
+ parser = all_parsers
+ data = """\
+1,2,3,4,hello
+5,6,7,8,world
+9,10,11,12,foo
+"""
+
+ expected = DataFrame(
+ {"a": [1, 5, 9], "b": [2, 6, 10], "c": [3, 7, 11], "d": [4, 8, 12]},
+ index=Index(["hello", "world", "foo"], name="message"),
+ )
+ names = ["a", "b", "c", "d", "message"]
+
+ result = parser.read_csv(StringIO(data), names=names, index_col=["message"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_index_col_is_true(all_parsers):
+ # see gh-9798
+ data = "a,b\n1,2"
+ parser = all_parsers
+
+ msg = "The value of index_col couldn't be 'True'"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), index_col=True)
+
+
+@skip_pyarrow # CSV parse error: Expected 3 columns, got 4
+def test_infer_index_col(all_parsers):
+ data = """A,B,C
+foo,1,2,3
+bar,4,5,6
+baz,7,8,9
+"""
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data))
+
+ expected = DataFrame(
+ [[1, 2, 3], [4, 5, 6], [7, 8, 9]],
+ index=["foo", "bar", "baz"],
+ columns=["A", "B", "C"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+@pytest.mark.parametrize(
+ "index_col,kwargs",
+ [
+ (None, {"columns": ["x", "y", "z"]}),
+ (False, {"columns": ["x", "y", "z"]}),
+ (0, {"columns": ["y", "z"], "index": Index([], name="x")}),
+ (1, {"columns": ["x", "z"], "index": Index([], name="y")}),
+ ("x", {"columns": ["y", "z"], "index": Index([], name="x")}),
+ ("y", {"columns": ["x", "z"], "index": Index([], name="y")}),
+ (
+ [0, 1],
+ {
+ "columns": ["z"],
+ "index": MultiIndex.from_arrays([[]] * 2, names=["x", "y"]),
+ },
+ ),
+ (
+ ["x", "y"],
+ {
+ "columns": ["z"],
+ "index": MultiIndex.from_arrays([[]] * 2, names=["x", "y"]),
+ },
+ ),
+ (
+ [1, 0],
+ {
+ "columns": ["z"],
+ "index": MultiIndex.from_arrays([[]] * 2, names=["y", "x"]),
+ },
+ ),
+ (
+ ["y", "x"],
+ {
+ "columns": ["z"],
+ "index": MultiIndex.from_arrays([[]] * 2, names=["y", "x"]),
+ },
+ ),
+ ],
+)
+def test_index_col_empty_data(all_parsers, index_col, kwargs):
+ data = "x,y,z"
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), index_col=index_col)
+
+ expected = DataFrame(**kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+def test_empty_with_index_col_false(all_parsers):
+ # see gh-10413
+ data = "x,y"
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), index_col=False)
+
+ expected = DataFrame(columns=["x", "y"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "index_names",
+ [
+ ["", ""],
+ ["foo", ""],
+ ["", "bar"],
+ ["foo", "bar"],
+ ["NotReallyUnnamed", "Unnamed: 0"],
+ ],
+)
+def test_multi_index_naming(all_parsers, index_names, request):
+ parser = all_parsers
+
+ if parser.engine == "pyarrow" and "" in index_names:
+ mark = pytest.mark.xfail(reason="One case raises, others are wrong")
+ request.applymarker(mark)
+
+ # We don't want empty index names being replaced with "Unnamed: 0"
+ data = ",".join([*index_names, "col\na,c,1\na,d,2\nb,c,3\nb,d,4"])
+ result = parser.read_csv(StringIO(data), index_col=[0, 1])
+
+ expected = DataFrame(
+ {"col": [1, 2, 3, 4]}, index=MultiIndex.from_product([["a", "b"], ["c", "d"]])
+ )
+ expected.index.names = [name if name else None for name in index_names]
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # ValueError: Found non-unique column index
+def test_multi_index_naming_not_all_at_beginning(all_parsers):
+ parser = all_parsers
+ data = ",Unnamed: 2,\na,c,1\na,d,2\nb,c,3\nb,d,4"
+ result = parser.read_csv(StringIO(data), index_col=[0, 2])
+
+ expected = DataFrame(
+ {"Unnamed: 2": ["c", "d", "c", "d"]},
+ index=MultiIndex(
+ levels=[["a", "b"], [1, 2, 3, 4]], codes=[[0, 0, 1, 1], [0, 1, 2, 3]]
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # ValueError: Found non-unique column index
+def test_no_multi_index_level_names_empty(temp_file, all_parsers):
+ # GH 10984
+ parser = all_parsers
+ midx = MultiIndex.from_tuples([("A", 1, 2), ("A", 1, 2), ("B", 1, 2)])
+ expected = DataFrame(
+ np.random.default_rng(2).standard_normal((3, 3)),
+ index=midx,
+ columns=["x", "y", "z"],
+ )
+ expected.to_csv(temp_file)
+ result = parser.read_csv(temp_file, index_col=[0, 1, 2])
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_header_with_index_col(all_parsers):
+ # GH 33476
+ parser = all_parsers
+ data = """
+I11,A,A
+I12,B,B
+I2,1,3
+"""
+ midx = MultiIndex.from_tuples([("A", "B"), ("A", "B.1")], names=["I11", "I12"])
+ idx = Index(["I2"])
+ expected = DataFrame([[1, 3]], index=idx, columns=midx)
+
+ result = parser.read_csv(StringIO(data), index_col=0, header=[0, 1])
+ tm.assert_frame_equal(result, expected)
+
+ col_idx = Index(["A", "A.1"])
+ idx = Index(["I12", "I2"], name="I11")
+ expected = DataFrame([["B", "B"], ["1", "3"]], index=idx, columns=col_idx)
+
+ result = parser.read_csv(StringIO(data), index_col="I11", header=0)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.slow
+def test_index_col_large_csv(temp_file, all_parsers, monkeypatch):
+ # https://github.com/pandas-dev/pandas/issues/37094
+ parser = all_parsers
+
+ ARR_LEN = 100
+ df = DataFrame(
+ {
+ "a": range(ARR_LEN + 1),
+ "b": np.random.default_rng(2).standard_normal(ARR_LEN + 1),
+ }
+ )
+
+ df.to_csv(temp_file, index=False)
+ with monkeypatch.context() as m:
+ m.setattr("pandas.core.algorithms._MINIMUM_COMP_ARR_LEN", ARR_LEN)
+ result = parser.read_csv(temp_file, index_col=[0])
+
+ tm.assert_frame_equal(result, df.set_index("a"))
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_index_col_multiindex_columns_no_data(all_parsers):
+ # GH#38292
+ parser = all_parsers
+ result = parser.read_csv(
+ StringIO("a0,a1,a2\nb0,b1,b2\n"), header=[0, 1], index_col=0
+ )
+ expected = DataFrame(
+ [],
+ index=Index([]),
+ columns=MultiIndex.from_arrays(
+ [["a1", "a2"], ["b1", "b2"]], names=["a0", "b0"]
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_index_col_header_no_data(all_parsers):
+ # GH#38292
+ parser = all_parsers
+ result = parser.read_csv(StringIO("a0,a1,a2\n"), header=[0], index_col=0)
+ expected = DataFrame(
+ [],
+ columns=["a1", "a2"],
+ index=Index([], name="a0"),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_multiindex_columns_no_data(all_parsers):
+ # GH#38292
+ parser = all_parsers
+ result = parser.read_csv(StringIO("a0,a1,a2\nb0,b1,b2\n"), header=[0, 1])
+ expected = DataFrame(
+ [], columns=MultiIndex.from_arrays([["a0", "a1", "a2"], ["b0", "b1", "b2"]])
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_multiindex_columns_index_col_with_data(all_parsers):
+ # GH#38292
+ parser = all_parsers
+ result = parser.read_csv(
+ StringIO("a0,a1,a2\nb0,b1,b2\ndata,data,data"), header=[0, 1], index_col=0
+ )
+ expected = DataFrame(
+ [["data", "data"]],
+ columns=MultiIndex.from_arrays(
+ [["a1", "a2"], ["b1", "b2"]], names=["a0", "b0"]
+ ),
+ index=Index(["data"]),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Empty CSV file or block
+def test_infer_types_boolean_sum(all_parsers):
+ # GH#44079
+ parser = all_parsers
+ result = parser.read_csv(
+ StringIO("0,1"),
+ names=["a", "b"],
+ index_col=["a"],
+ dtype={"a": "UInt8"},
+ )
+ expected = DataFrame(
+ data={
+ "a": [
+ 0,
+ ],
+ "b": [1],
+ }
+ ).set_index("a")
+ # Not checking index type now, because the C parser will return a
+ # index column of dtype 'object', and the Python parser will return a
+ # index column of dtype 'int64'.
+ tm.assert_frame_equal(result, expected, check_index_type=False)
+
+
+@pytest.mark.parametrize("dtype, val", [(object, "01"), ("int64", 1)])
+def test_specify_dtype_for_index_col(all_parsers, dtype, val, request):
+ # GH#9435
+ data = "a,b\n01,2"
+ parser = all_parsers
+ if dtype == object and parser.engine == "pyarrow":
+ request.applymarker(
+ pytest.mark.xfail(reason="Cannot disable type-inference for pyarrow engine")
+ )
+ result = parser.read_csv(StringIO(data), index_col="a", dtype={"a": dtype})
+ expected = DataFrame({"b": [2]}, index=Index([val], name="a", dtype=dtype))
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_multiindex_columns_not_leading_index_col(all_parsers):
+ # GH#38549
+ parser = all_parsers
+ data = """a,b,c,d
+e,f,g,h
+x,y,1,2
+"""
+ result = parser.read_csv(
+ StringIO(data),
+ header=[0, 1],
+ index_col=1,
+ )
+ cols = MultiIndex.from_tuples(
+ [("a", "e"), ("c", "g"), ("d", "h")], names=["b", "f"]
+ )
+ expected = DataFrame([["x", 1, 2]], columns=cols, index=["y"])
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_mangle_dupes.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_mangle_dupes.py
new file mode 100644
index 0000000000000000000000000000000000000000..55c8bbc4bb9e1f79ff8cea22b85a7e6104c2d9ae
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_mangle_dupes.py
@@ -0,0 +1,183 @@
+"""
+Tests that duplicate columns are handled appropriately when parsed by the
+CSV engine. In general, the expected result is that they are either thoroughly
+de-duplicated (if mangling requested) or ignored otherwise.
+"""
+
+from io import StringIO
+
+import pytest
+
+from pandas import (
+ DataFrame,
+ Index,
+)
+import pandas._testing as tm
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+@xfail_pyarrow # ValueError: Found non-unique column index
+def test_basic(all_parsers):
+ parser = all_parsers
+
+ data = "a,a,b,b,b\n1,2,3,4,5"
+ result = parser.read_csv(StringIO(data), sep=",")
+
+ expected = DataFrame([[1, 2, 3, 4, 5]], columns=["a", "a.1", "b", "b.1", "b.2"])
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # ValueError: Found non-unique column index
+def test_basic_names(all_parsers):
+ # See gh-7160
+ parser = all_parsers
+
+ data = "a,b,a\n0,1,2\n3,4,5"
+ expected = DataFrame([[0, 1, 2], [3, 4, 5]], columns=["a", "b", "a.1"])
+
+ result = parser.read_csv(StringIO(data))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_basic_names_raise(all_parsers):
+ # See gh-7160
+ parser = all_parsers
+
+ data = "0,1,2\n3,4,5"
+ with pytest.raises(ValueError, match="Duplicate names"):
+ parser.read_csv(StringIO(data), names=["a", "b", "a"])
+
+
+@xfail_pyarrow # ValueError: Found non-unique column index
+@pytest.mark.parametrize(
+ "data,expected",
+ [
+ ("a,a,a.1\n1,2,3", DataFrame([[1, 2, 3]], columns=["a", "a.2", "a.1"])),
+ (
+ "a,a,a.1,a.1.1,a.1.1.1,a.1.1.1.1\n1,2,3,4,5,6",
+ DataFrame(
+ [[1, 2, 3, 4, 5, 6]],
+ columns=["a", "a.2", "a.1", "a.1.1", "a.1.1.1", "a.1.1.1.1"],
+ ),
+ ),
+ (
+ "a,a,a.3,a.1,a.2,a,a\n1,2,3,4,5,6,7",
+ DataFrame(
+ [[1, 2, 3, 4, 5, 6, 7]],
+ columns=["a", "a.4", "a.3", "a.1", "a.2", "a.5", "a.6"],
+ ),
+ ),
+ ],
+)
+def test_thorough_mangle_columns(all_parsers, data, expected):
+ # see gh-17060
+ parser = all_parsers
+
+ result = parser.read_csv(StringIO(data))
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "data,names,expected",
+ [
+ (
+ "a,b,b\n1,2,3",
+ ["a.1", "a.1", "a.1.1"],
+ DataFrame(
+ [["a", "b", "b"], ["1", "2", "3"]], columns=["a.1", "a.1.1", "a.1.1.1"]
+ ),
+ ),
+ (
+ "a,b,c,d,e,f\n1,2,3,4,5,6",
+ ["a", "a", "a.1", "a.1.1", "a.1.1.1", "a.1.1.1.1"],
+ DataFrame(
+ [["a", "b", "c", "d", "e", "f"], ["1", "2", "3", "4", "5", "6"]],
+ columns=["a", "a.1", "a.1.1", "a.1.1.1", "a.1.1.1.1", "a.1.1.1.1.1"],
+ ),
+ ),
+ (
+ "a,b,c,d,e,f,g\n1,2,3,4,5,6,7",
+ ["a", "a", "a.3", "a.1", "a.2", "a", "a"],
+ DataFrame(
+ [
+ ["a", "b", "c", "d", "e", "f", "g"],
+ ["1", "2", "3", "4", "5", "6", "7"],
+ ],
+ columns=["a", "a.1", "a.3", "a.1.1", "a.2", "a.2.1", "a.3.1"],
+ ),
+ ),
+ ],
+)
+def test_thorough_mangle_names(all_parsers, data, names, expected):
+ # see gh-17095
+ parser = all_parsers
+
+ with pytest.raises(ValueError, match="Duplicate names"):
+ parser.read_csv(StringIO(data), names=names)
+
+
+@xfail_pyarrow # AssertionError: DataFrame.columns are different
+def test_mangled_unnamed_placeholders(all_parsers):
+ # xref gh-13017
+ orig_key = "0"
+ parser = all_parsers
+
+ orig_value = [1, 2, 3]
+ df = DataFrame({orig_key: orig_value})
+
+ # This test recursively updates `df`.
+ for i in range(3):
+ expected = DataFrame(columns=Index([], dtype="str"))
+
+ for j in range(i + 1):
+ col_name = "Unnamed: 0" + f".{1 * j}" * min(j, 1)
+ expected.insert(loc=0, column=col_name, value=[0, 1, 2])
+
+ expected[orig_key] = orig_value
+ df = parser.read_csv(StringIO(df.to_csv()))
+
+ tm.assert_frame_equal(df, expected)
+
+
+@xfail_pyarrow # ValueError: Found non-unique column index
+def test_mangle_dupe_cols_already_exists(all_parsers):
+ # GH#14704
+ parser = all_parsers
+
+ data = "a,a,a.1,a,a.3,a.1,a.1.1\n1,2,3,4,5,6,7"
+ result = parser.read_csv(StringIO(data))
+ expected = DataFrame(
+ [[1, 2, 3, 4, 5, 6, 7]],
+ columns=["a", "a.2", "a.1", "a.4", "a.3", "a.1.2", "a.1.1"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # ValueError: Found non-unique column index
+def test_mangle_dupe_cols_already_exists_unnamed_col(all_parsers):
+ # GH#14704
+ parser = all_parsers
+
+ data = ",Unnamed: 0,,Unnamed: 2\n1,2,3,4"
+ result = parser.read_csv(StringIO(data))
+ expected = DataFrame(
+ [[1, 2, 3, 4]],
+ columns=["Unnamed: 0.1", "Unnamed: 0", "Unnamed: 2.1", "Unnamed: 2"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("usecol, engine", [([0, 1, 1], "python"), ([0, 1, 1], "c")])
+def test_mangle_cols_names(all_parsers, usecol, engine):
+ # GH 11823
+ parser = all_parsers
+ data = "1,2,3"
+ names = ["A", "A", "B"]
+ with pytest.raises(ValueError, match="Duplicate names"):
+ parser.read_csv(StringIO(data), names=names, usecols=usecol, engine=engine)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_multi_thread.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_multi_thread.py
new file mode 100644
index 0000000000000000000000000000000000000000..fa9465c9a66af200311def2dca612fd28739c6e1
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_multi_thread.py
@@ -0,0 +1,159 @@
+"""
+Tests multithreading behaviour for reading and
+parsing files for each parser defined in parsers.py
+"""
+
+from contextlib import ExitStack
+from io import BytesIO
+from multiprocessing.pool import ThreadPool
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import DataFrame
+import pandas._testing as tm
+from pandas.util.version import Version
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+
+# We'll probably always skip these for pyarrow
+# Maybe we'll add our own tests for pyarrow too
+pytestmark = [
+ pytest.mark.single_cpu,
+ pytest.mark.slow,
+]
+
+
+@pytest.mark.filterwarnings("ignore:Passing a BlockManager:DeprecationWarning")
+def test_multi_thread_string_io_read_csv(all_parsers, request):
+ # see gh-11786
+ parser = all_parsers
+ if parser.engine == "pyarrow":
+ pa = pytest.importorskip("pyarrow")
+ if Version(pa.__version__) < Version("16.0"):
+ request.applymarker(
+ pytest.mark.xfail(reason="# ValueError: Found non-unique column index")
+ )
+ max_row_range = 100
+ num_files = 10
+
+ bytes_to_df = (
+ "\n".join([f"{i:d},{i:d},{i:d}" for i in range(max_row_range)]).encode()
+ for _ in range(num_files)
+ )
+
+ # Read all files in many threads.
+ with ExitStack() as stack:
+ files = [stack.enter_context(BytesIO(b)) for b in bytes_to_df]
+
+ pool = stack.enter_context(ThreadPool(8))
+
+ results = pool.map(parser.read_csv, files)
+ first_result = results[0]
+
+ for result in results:
+ tm.assert_frame_equal(first_result, result)
+
+
+def _generate_multi_thread_dataframe(parser, path, num_rows, num_tasks):
+ """
+ Generate a DataFrame via multi-thread.
+
+ Parameters
+ ----------
+ parser : BaseParser
+ The parser object to use for reading the data.
+ path : str
+ The location of the CSV file to read.
+ num_rows : int
+ The number of rows to read per task.
+ num_tasks : int
+ The number of tasks to use for reading this DataFrame.
+
+ Returns
+ -------
+ df : DataFrame
+ """
+
+ def reader(arg):
+ """
+ Create a reader for part of the CSV.
+
+ Parameters
+ ----------
+ arg : tuple
+ A tuple of the following:
+
+ * start : int
+ The starting row to start for parsing CSV
+ * nrows : int
+ The number of rows to read.
+
+ Returns
+ -------
+ df : DataFrame
+ """
+ start, nrows = arg
+
+ if not start:
+ return parser.read_csv(
+ path, index_col=0, header=0, nrows=nrows, parse_dates=["date"]
+ )
+
+ return parser.read_csv(
+ path,
+ index_col=0,
+ header=None,
+ skiprows=int(start) + 1,
+ nrows=nrows,
+ parse_dates=[9],
+ )
+
+ tasks = [
+ (num_rows * i // num_tasks, num_rows // num_tasks) for i in range(num_tasks)
+ ]
+
+ with ThreadPool(processes=num_tasks) as pool:
+ results = pool.map(reader, tasks)
+
+ header = results[0].columns
+
+ for r in results[1:]:
+ r.columns = header
+
+ final_dataframe = pd.concat(results)
+ return final_dataframe
+
+
+@xfail_pyarrow # ValueError: The 'nrows' option is not supported
+def test_multi_thread_path_multipart_read_csv(tmp_path, all_parsers):
+ # see gh-11786
+ num_tasks = 4
+ num_rows = 48
+
+ parser = all_parsers
+ file_name = "__thread_pool_reader__.csv"
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).random(num_rows),
+ "b": np.random.default_rng(2).random(num_rows),
+ "c": np.random.default_rng(2).random(num_rows),
+ "d": np.random.default_rng(2).random(num_rows),
+ "e": np.random.default_rng(2).random(num_rows),
+ "foo": ["foo"] * num_rows,
+ "bar": ["bar"] * num_rows,
+ "baz": ["baz"] * num_rows,
+ "date": pd.date_range("20000101 09:00:00", periods=num_rows, freq="s"),
+ "int": np.arange(num_rows, dtype="int64"),
+ }
+ )
+
+ path = tmp_path / file_name
+ df.to_csv(path)
+
+ result = _generate_multi_thread_dataframe(parser, path, num_rows, num_tasks)
+
+ expected = df[:]
+ expected["date"] = expected["date"].astype("M8[us]")
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_na_values.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_na_values.py
new file mode 100644
index 0000000000000000000000000000000000000000..9998ba1b49bddc6778996fa4c106c5630c32c76c
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_na_values.py
@@ -0,0 +1,844 @@
+"""
+Tests that NA values are properly handled during
+parsing for all of the parsers defined in parsers.py
+"""
+
+from io import StringIO
+
+import numpy as np
+import pytest
+
+from pandas._libs.parsers import STR_NA_VALUES
+
+from pandas import (
+ DataFrame,
+ Index,
+ MultiIndex,
+)
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+
+def test_string_nas(all_parsers):
+ parser = all_parsers
+ data = """A,B,C
+a,b,c
+d,,f
+,g,h
+"""
+ result = parser.read_csv(StringIO(data))
+ expected = DataFrame(
+ [["a", "b", "c"], ["d", np.nan, "f"], [np.nan, "g", "h"]],
+ columns=["A", "B", "C"],
+ )
+ if parser.engine == "pyarrow":
+ expected.loc[2, "A"] = None
+ expected.loc[1, "B"] = None
+ tm.assert_frame_equal(result, expected)
+
+
+def test_detect_string_na(all_parsers):
+ parser = all_parsers
+ data = """A,B
+foo,bar
+NA,baz
+NaN,nan
+"""
+ expected = DataFrame(
+ [["foo", "bar"], [np.nan, "baz"], [np.nan, np.nan]], columns=["A", "B"]
+ )
+ if parser.engine == "pyarrow":
+ expected.loc[[1, 2], "A"] = None
+ expected.loc[2, "B"] = None
+ result = parser.read_csv(StringIO(data))
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "na_values",
+ [
+ ["-999.0", "-999"],
+ [-999, -999.0],
+ [-999.0, -999],
+ ["-999.0"],
+ ["-999"],
+ [-999.0],
+ [-999],
+ ],
+)
+@pytest.mark.parametrize(
+ "data",
+ [
+ """A,B
+-999,1.2
+2,-999
+3,4.5
+""",
+ """A,B
+-999,1.200
+2,-999.000
+3,4.500
+""",
+ ],
+)
+def test_non_string_na_values(all_parsers, data, na_values, request):
+ # see gh-3611: with an odd float format, we can't match
+ # the string "999.0" exactly but still need float matching
+ parser = all_parsers
+ expected = DataFrame([[np.nan, 1.2], [2.0, np.nan], [3.0, 4.5]], columns=["A", "B"])
+
+ if parser.engine == "pyarrow" and not all(isinstance(x, str) for x in na_values):
+ msg = "The 'pyarrow' engine requires all na_values to be strings"
+ with pytest.raises(TypeError, match=msg):
+ parser.read_csv(StringIO(data), na_values=na_values)
+ return
+ elif parser.engine == "pyarrow" and "-999.000" in data:
+ # bc the pyarrow engine does not include the float-ified version
+ # of "-999" -> -999, it does not match the entry with the trailing
+ # zeros, so "-999.000" is not treated as null.
+ mark = pytest.mark.xfail(
+ reason="pyarrow engined does not recognize equivalent floats"
+ )
+ request.applymarker(mark)
+
+ result = parser.read_csv(StringIO(data), na_values=na_values)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_default_na_values(all_parsers):
+ _NA_VALUES = {
+ "-1.#IND",
+ "1.#QNAN",
+ "1.#IND",
+ "-1.#QNAN",
+ "#N/A",
+ "N/A",
+ "n/a",
+ "NA",
+ "",
+ "#NA",
+ "NULL",
+ "null",
+ "NaN",
+ "nan",
+ "-NaN",
+ "-nan",
+ "#N/A N/A",
+ "",
+ "None",
+ }
+ assert _NA_VALUES == STR_NA_VALUES
+
+ parser = all_parsers
+ nv = len(_NA_VALUES)
+
+ def f(i, v):
+ if i == 0:
+ buf = ""
+ elif i > 0:
+ buf = "".join([","] * i)
+
+ buf = f"{buf}{v}"
+
+ if i < nv - 1:
+ joined = "".join([","] * (nv - i - 1))
+ buf = f"{buf}{joined}"
+
+ return buf
+
+ data = StringIO("\n".join([f(i, v) for i, v in enumerate(_NA_VALUES)]))
+ expected = DataFrame(np.nan, columns=range(nv), index=range(nv))
+
+ result = parser.read_csv(data, header=None)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("na_values", ["baz", ["baz"]])
+def test_custom_na_values(all_parsers, na_values):
+ parser = all_parsers
+ data = """A,B,C
+ignore,this,row
+1,NA,3
+-1.#IND,5,baz
+7,8,NaN
+"""
+ expected = DataFrame(
+ [[1.0, np.nan, 3], [np.nan, 5, np.nan], [7, 8, np.nan]], columns=["A", "B", "C"]
+ )
+ if parser.engine == "pyarrow":
+ msg = "skiprows argument must be an integer when using engine='pyarrow'"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), na_values=na_values, skiprows=[1])
+ return
+
+ result = parser.read_csv(StringIO(data), na_values=na_values, skiprows=[1])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_bool_na_values(all_parsers):
+ data = """A,B,C
+True,False,True
+NA,True,False
+False,NA,True"""
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data))
+ expected = DataFrame(
+ {
+ "A": np.array([True, np.nan, False], dtype=object),
+ "B": np.array([False, True, np.nan], dtype=object),
+ "C": [True, False, True],
+ }
+ )
+ if parser.engine == "pyarrow":
+ expected.loc[1, "A"] = None
+ expected.loc[2, "B"] = None
+ tm.assert_frame_equal(result, expected)
+
+
+def test_na_value_dict(all_parsers):
+ data = """A,B,C
+foo,bar,NA
+bar,foo,foo
+foo,bar,NA
+bar,foo,foo"""
+ parser = all_parsers
+
+ if parser.engine == "pyarrow":
+ msg = "pyarrow engine doesn't support passing a dict for na_values"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), na_values={"A": ["foo"], "B": ["bar"]})
+ return
+
+ df = parser.read_csv(StringIO(data), na_values={"A": ["foo"], "B": ["bar"]})
+ expected = DataFrame(
+ {
+ "A": [np.nan, "bar", np.nan, "bar"],
+ "B": [np.nan, "foo", np.nan, "foo"],
+ "C": [np.nan, "foo", np.nan, "foo"],
+ }
+ )
+ tm.assert_frame_equal(df, expected)
+
+
+@pytest.mark.parametrize(
+ "index_col,expected",
+ [
+ (
+ [0],
+ DataFrame({"b": [np.nan], "c": [1], "d": [5]}, index=Index([0], name="a")),
+ ),
+ (
+ [0, 2],
+ DataFrame(
+ {"b": [np.nan], "d": [5]},
+ index=MultiIndex.from_tuples([(0, 1)], names=["a", "c"]),
+ ),
+ ),
+ (
+ ["a", "c"],
+ DataFrame(
+ {"b": [np.nan], "d": [5]},
+ index=MultiIndex.from_tuples([(0, 1)], names=["a", "c"]),
+ ),
+ ),
+ ],
+)
+def test_na_value_dict_multi_index(all_parsers, index_col, expected):
+ data = """\
+a,b,c,d
+0,NA,1,5
+"""
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), na_values=set(), index_col=index_col)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "kwargs,expected",
+ [
+ (
+ {},
+ {
+ "A": ["a", "b", np.nan, "d", "e", np.nan, "g"],
+ "B": [1, 2, 3, 4, 5, 6, 7],
+ "C": ["one", "two", "three", np.nan, "five", np.nan, "seven"],
+ },
+ ),
+ (
+ {"na_values": {"A": [], "C": []}, "keep_default_na": False},
+ {
+ "A": ["a", "b", "", "d", "e", "nan", "g"],
+ "B": [1, 2, 3, 4, 5, 6, 7],
+ "C": ["one", "two", "three", "nan", "five", "", "seven"],
+ },
+ ),
+ (
+ {"na_values": ["a"], "keep_default_na": False},
+ {
+ "A": [np.nan, "b", "", "d", "e", "nan", "g"],
+ "B": [1, 2, 3, 4, 5, 6, 7],
+ "C": ["one", "two", "three", "nan", "five", "", "seven"],
+ },
+ ),
+ (
+ {"na_values": {"A": [], "C": []}},
+ {
+ "A": ["a", "b", np.nan, "d", "e", np.nan, "g"],
+ "B": [1, 2, 3, 4, 5, 6, 7],
+ "C": ["one", "two", "three", np.nan, "five", np.nan, "seven"],
+ },
+ ),
+ ],
+)
+def test_na_values_keep_default(
+ all_parsers, kwargs, expected, request, using_infer_string
+):
+ data = """\
+A,B,C
+a,1,one
+b,2,two
+,3,three
+d,4,nan
+e,5,five
+nan,6,
+g,7,seven
+"""
+ parser = all_parsers
+ if parser.engine == "pyarrow":
+ if "na_values" in kwargs and isinstance(kwargs["na_values"], dict):
+ msg = "The pyarrow engine doesn't support passing a dict for na_values"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), **kwargs)
+ return
+ if not using_infer_string or "na_values" in kwargs:
+ mark = pytest.mark.xfail()
+ request.applymarker(mark)
+
+ result = parser.read_csv(StringIO(data), **kwargs)
+ expected = DataFrame(expected)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_no_na_values_no_keep_default(all_parsers):
+ # see gh-4318: passing na_values=None and
+ # keep_default_na=False yields 'None" as a na_value
+ data = """\
+A,B,C
+a,1,None
+b,2,two
+,3,None
+d,4,nan
+e,5,five
+nan,6,
+g,7,seven
+"""
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), keep_default_na=False)
+
+ expected = DataFrame(
+ {
+ "A": ["a", "b", "", "d", "e", "nan", "g"],
+ "B": [1, 2, 3, 4, 5, 6, 7],
+ "C": ["None", "two", "None", "nan", "five", "", "seven"],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_no_keep_default_na_dict_na_values(all_parsers):
+ # see gh-19227
+ data = "a,b\n,2"
+ parser = all_parsers
+
+ if parser.engine == "pyarrow":
+ msg = "The pyarrow engine doesn't support passing a dict for na_values"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data), na_values={"b": ["2"]}, keep_default_na=False
+ )
+ return
+
+ result = parser.read_csv(
+ StringIO(data), na_values={"b": ["2"]}, keep_default_na=False
+ )
+ expected = DataFrame({"a": [""], "b": [np.nan]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_no_keep_default_na_dict_na_scalar_values(all_parsers):
+ # see gh-19227
+ #
+ # Scalar values shouldn't cause the parsing to crash or fail.
+ data = "a,b\n1,2"
+ parser = all_parsers
+
+ if parser.engine == "pyarrow":
+ msg = "The pyarrow engine doesn't support passing a dict for na_values"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), na_values={"b": 2}, keep_default_na=False)
+ return
+
+ df = parser.read_csv(StringIO(data), na_values={"b": 2}, keep_default_na=False)
+ expected = DataFrame({"a": [1], "b": [np.nan]})
+ tm.assert_frame_equal(df, expected)
+
+
+@pytest.mark.parametrize("col_zero_na_values", [113125, "113125"])
+def test_no_keep_default_na_dict_na_values_diff_reprs(all_parsers, col_zero_na_values):
+ # see gh-19227
+ data = """\
+113125,"blah","/blaha",kjsdkj,412.166,225.874,214.008
+729639,"qwer","",asdfkj,466.681,,252.373
+"""
+ parser = all_parsers
+ expected = DataFrame(
+ {
+ 0: [np.nan, 729639.0],
+ 1: [np.nan, "qwer"],
+ 2: ["/blaha", np.nan],
+ 3: ["kjsdkj", "asdfkj"],
+ 4: [412.166, 466.681],
+ 5: ["225.874", ""],
+ 6: [np.nan, 252.373],
+ }
+ )
+
+ if parser.engine == "pyarrow":
+ msg = "The pyarrow engine doesn't support passing a dict for na_values"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data),
+ header=None,
+ keep_default_na=False,
+ na_values={2: "", 6: "214.008", 1: "blah", 0: col_zero_na_values},
+ )
+ return
+
+ result = parser.read_csv(
+ StringIO(data),
+ header=None,
+ keep_default_na=False,
+ na_values={2: "", 6: "214.008", 1: "blah", 0: col_zero_na_values},
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "na_filter,row_data",
+ [
+ (True, [[1, "A"], [np.nan, np.nan], [3, "C"]]),
+ (False, [["1", "A"], ["nan", "B"], ["3", "C"]]),
+ ],
+)
+def test_na_values_na_filter_override(
+ request, all_parsers, na_filter, row_data, using_infer_string
+):
+ parser = all_parsers
+ if parser.engine == "pyarrow":
+ # mismatched dtypes in both cases, FutureWarning in the True case
+ if not (using_infer_string and na_filter):
+ mark = pytest.mark.xfail(reason="pyarrow doesn't support this.")
+ request.applymarker(mark)
+ data = """\
+A,B
+1,A
+nan,B
+3,C
+"""
+ result = parser.read_csv(StringIO(data), na_values=["B"], na_filter=na_filter)
+
+ expected = DataFrame(row_data, columns=["A", "B"])
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Expected 8 columns, got 5:
+def test_na_trailing_columns(all_parsers):
+ parser = all_parsers
+ data = """Date,Currency,Symbol,Type,Units,UnitPrice,Cost,Tax
+2012-03-14,USD,AAPL,BUY,1000
+2012-05-12,USD,SBUX,SELL,500"""
+
+ # Trailing columns should be all NaN.
+ result = parser.read_csv(StringIO(data))
+ expected = DataFrame(
+ [
+ ["2012-03-14", "USD", "AAPL", "BUY", 1000, np.nan, np.nan, np.nan],
+ ["2012-05-12", "USD", "SBUX", "SELL", 500, np.nan, np.nan, np.nan],
+ ],
+ columns=[
+ "Date",
+ "Currency",
+ "Symbol",
+ "Type",
+ "Units",
+ "UnitPrice",
+ "Cost",
+ "Tax",
+ ],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "na_values,row_data",
+ [
+ (1, [[np.nan, 2.0], [2.0, np.nan]]),
+ ({"a": 2, "b": 1}, [[1.0, 2.0], [np.nan, np.nan]]),
+ ],
+)
+def test_na_values_scalar(all_parsers, na_values, row_data):
+ # see gh-12224
+ parser = all_parsers
+ names = ["a", "b"]
+ data = "1,2\n2,1"
+
+ if parser.engine == "pyarrow" and isinstance(na_values, dict):
+ msg = "The pyarrow engine doesn't support passing a dict for na_values"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), names=names, na_values=na_values)
+ return
+ elif parser.engine == "pyarrow":
+ msg = "The 'pyarrow' engine requires all na_values to be strings"
+ with pytest.raises(TypeError, match=msg):
+ parser.read_csv(StringIO(data), names=names, na_values=na_values)
+ return
+
+ result = parser.read_csv(StringIO(data), names=names, na_values=na_values)
+ expected = DataFrame(row_data, columns=names)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_na_values_dict_aliasing(all_parsers):
+ parser = all_parsers
+ na_values = {"a": 2, "b": 1}
+ na_values_copy = na_values.copy()
+
+ names = ["a", "b"]
+ data = "1,2\n2,1"
+
+ expected = DataFrame([[1.0, 2.0], [np.nan, np.nan]], columns=names)
+
+ if parser.engine == "pyarrow":
+ msg = "The pyarrow engine doesn't support passing a dict for na_values"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), names=names, na_values=na_values)
+ return
+
+ result = parser.read_csv(StringIO(data), names=names, na_values=na_values)
+
+ tm.assert_frame_equal(result, expected)
+ tm.assert_dict_equal(na_values, na_values_copy)
+
+
+def test_na_values_dict_null_column_name(all_parsers):
+ # see gh-57547
+ parser = all_parsers
+ data = ",x,y\n\nMA,1,2\nNA,2,1\nOA,,3"
+ names = [None, "x", "y"]
+ na_values = dict.fromkeys(names, STR_NA_VALUES)
+ dtype = {None: "object", "x": "float64", "y": "float64"}
+
+ if parser.engine == "pyarrow":
+ msg = "The pyarrow engine doesn't support passing a dict for na_values"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data),
+ index_col=0,
+ header=0,
+ dtype=dtype,
+ names=names,
+ na_values=na_values,
+ keep_default_na=False,
+ )
+ return
+
+ expected = DataFrame(
+ {"x": [1.0, 2.0, np.nan], "y": [2.0, 1.0, 3.0]},
+ index=Index(["MA", "NA", "OA"], dtype=object),
+ )
+
+ result = parser.read_csv(
+ StringIO(data),
+ index_col=0,
+ header=0,
+ dtype=dtype,
+ names=names,
+ na_values=na_values,
+ keep_default_na=False,
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_na_values_dict_col_index(all_parsers):
+ # see gh-14203
+ data = "a\nfoo\n1"
+ parser = all_parsers
+ na_values = {0: "foo"}
+
+ if parser.engine == "pyarrow":
+ msg = "The pyarrow engine doesn't support passing a dict for na_values"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), na_values=na_values)
+ return
+
+ result = parser.read_csv(StringIO(data), na_values=na_values)
+ expected = DataFrame({"a": [np.nan, 1]})
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "data,kwargs,expected",
+ [
+ (
+ str(2**63) + "\n" + str(2**63 + 1),
+ {"na_values": [2**63]},
+ [str(2**63), str(2**63 + 1)],
+ ),
+ (str(2**63) + ",1" + "\n,2", {}, [[str(2**63), 1], ["", 2]]),
+ (str(2**63) + "\n1", {"na_values": [2**63]}, [np.nan, 1]),
+ ],
+)
+def test_na_values_uint64(all_parsers, data, kwargs, expected, request):
+ # see gh-14983
+ parser = all_parsers
+
+ if parser.engine == "pyarrow" and "na_values" in kwargs:
+ msg = "The 'pyarrow' engine requires all na_values to be strings"
+ with pytest.raises(TypeError, match=msg):
+ parser.read_csv(StringIO(data), header=None, **kwargs)
+ return
+ elif parser.engine == "pyarrow":
+ mark = pytest.mark.xfail(reason="Returns float64 instead of object")
+ request.applymarker(mark)
+
+ result = parser.read_csv(StringIO(data), header=None, **kwargs)
+ expected = DataFrame(expected)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_empty_na_values_no_default_with_index(all_parsers):
+ # see gh-15835
+ data = "a,1\nb,2"
+ parser = all_parsers
+ expected = DataFrame({"1": [2]}, index=Index(["b"], name="a"))
+
+ result = parser.read_csv(StringIO(data), index_col=0, keep_default_na=False)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "na_filter,index_data", [(False, ["", "5"]), (True, [np.nan, 5.0])]
+)
+def test_no_na_filter_on_index(all_parsers, na_filter, index_data, request):
+ # see gh-5239
+ #
+ # Don't parse NA-values in index unless na_filter=True
+ parser = all_parsers
+ data = "a,b,c\n1,,3\n4,5,6"
+
+ if parser.engine == "pyarrow" and na_filter is False:
+ mark = pytest.mark.xfail(reason="mismatched index result")
+ request.applymarker(mark)
+
+ expected = DataFrame({"a": [1, 4], "c": [3, 6]}, index=Index(index_data, name="b"))
+ result = parser.read_csv(StringIO(data), index_col=[1], na_filter=na_filter)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_inf_na_values_with_int_index(all_parsers):
+ # see gh-17128
+ parser = all_parsers
+ data = "idx,col1,col2\n1,3,4\n2,inf,-inf"
+
+ # Don't fail with OverflowError with inf's and integer index column.
+ out = parser.read_csv(StringIO(data), index_col=[0], na_values=["inf", "-inf"])
+ expected = DataFrame(
+ {"col1": [3, np.nan], "col2": [4, np.nan]}, index=Index([1, 2], name="idx")
+ )
+ tm.assert_frame_equal(out, expected)
+
+
+@pytest.mark.parametrize("na_filter", [True, False])
+def test_na_values_with_dtype_str_and_na_filter(
+ all_parsers, na_filter, using_infer_string, request
+):
+ # see gh-20377
+ parser = all_parsers
+ if parser.engine == "pyarrow" and (na_filter is False or not using_infer_string):
+ mark = pytest.mark.xfail(reason="mismatched shape")
+ request.applymarker(mark)
+
+ data = "a,b,c\n1,,3\n4,5,6"
+
+ # na_filter=True --> missing value becomes NaN.
+ # na_filter=False --> missing value remains empty string.
+ empty = np.nan if na_filter else ""
+ expected = DataFrame({"a": ["1", "4"], "b": [empty, "5"], "c": ["3", "6"]})
+
+ result = parser.read_csv(StringIO(data), na_filter=na_filter, dtype=str)
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # mismatched exception message
+@pytest.mark.parametrize(
+ "data, na_values",
+ [
+ ("false,1\n,1\ntrue", None),
+ ("false,1\nnull,1\ntrue", None),
+ ("false,1\nnan,1\ntrue", None),
+ ("false,1\nfoo,1\ntrue", "foo"),
+ ("false,1\nfoo,1\ntrue", ["foo"]),
+ ("false,1\nfoo,1\ntrue", {"a": "foo"}),
+ ],
+)
+def test_cast_NA_to_bool_raises_error(all_parsers, data, na_values):
+ parser = all_parsers
+ msg = "|".join(
+ [
+ "Bool column has NA values in column [0a]",
+ "cannot safely convert passed user dtype of "
+ "bool for object dtyped data in column 0",
+ ]
+ )
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data),
+ header=None,
+ names=["a", "b"],
+ dtype={"a": "bool"},
+ na_values=na_values,
+ )
+
+
+# TODO: this test isn't about the na_values keyword, it is about the empty entries
+# being returned with NaN entries, whereas the pyarrow engine returns "nan"
+@xfail_pyarrow # mismatched shapes
+def test_str_nan_dropped(all_parsers):
+ # see gh-21131
+ parser = all_parsers
+
+ data = """File: small.csv,,
+10010010233,0123,654
+foo,,bar
+01001000155,4530,898"""
+
+ result = parser.read_csv(
+ StringIO(data),
+ header=None,
+ names=["col1", "col2", "col3"],
+ dtype={"col1": str, "col2": str, "col3": str},
+ ).dropna()
+
+ expected = DataFrame(
+ {
+ "col1": ["10010010233", "01001000155"],
+ "col2": ["0123", "4530"],
+ "col3": ["654", "898"],
+ },
+ index=[1, 3],
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_nan_multi_index(all_parsers):
+ # GH 42446
+ parser = all_parsers
+ data = "A,B,B\nX,Y,Z\n1,2,inf"
+
+ if parser.engine == "pyarrow":
+ msg = "The pyarrow engine doesn't support passing a dict for na_values"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data), header=list(range(2)), na_values={("B", "Z"): "inf"}
+ )
+ return
+
+ result = parser.read_csv(
+ StringIO(data), header=list(range(2)), na_values={("B", "Z"): "inf"}
+ )
+
+ expected = DataFrame(
+ {
+ ("A", "X"): [1],
+ ("B", "Y"): [2],
+ ("B", "Z"): [np.nan],
+ }
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # Failed: DID NOT RAISE ; it casts the NaN to False
+def test_bool_and_nan_to_bool(all_parsers):
+ # GH#42808
+ parser = all_parsers
+ data = """0
+NaN
+True
+False
+"""
+ with pytest.raises(ValueError, match="NA values"):
+ parser.read_csv(StringIO(data), dtype="bool")
+
+
+def test_bool_and_nan_to_int(all_parsers):
+ # GH#42808
+ parser = all_parsers
+ data = """0
+NaN
+True
+False
+"""
+ msg = (
+ "cannot safely convert passed user dtype of int(64|32) for "
+ " dtyped data in column 0 due to NA values"
+ )
+ if parser.engine == "python":
+ msg = "Unable to convert column 0 to type int(64|32)"
+ elif parser.engine == "pyarrow":
+ msg = (
+ r"int\(\) argument must be a string, a bytes-like object or a "
+ "real number, not 'NoneType"
+ )
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), dtype="int")
+
+
+def test_bool_and_nan_to_float(all_parsers):
+ # GH#42808
+ parser = all_parsers
+ data = """0
+NaN
+True
+False
+"""
+ result = parser.read_csv(StringIO(data), dtype="float")
+ expected = DataFrame.from_dict({"0": [np.nan, 1.0, 0.0]})
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow
+@pytest.mark.parametrize(
+ "na_values",
+ [[-99.0, -99], [-99, -99.0]],
+)
+def test_na_values_dict_without_dtype(all_parsers, na_values):
+ parser = all_parsers
+ data = """A
+-99
+-99
+-99.0
+-99.0"""
+
+ result = parser.read_csv(StringIO(data), na_values=na_values)
+ expected = DataFrame({"A": [np.nan, np.nan, np.nan, np.nan]})
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_network.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_network.py
new file mode 100644
index 0000000000000000000000000000000000000000..d6bdff5be9b4e06a28a1df2eabbd38cc45a3a772
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_network.py
@@ -0,0 +1,244 @@
+"""
+Tests parsers ability to read and parse non-local files
+and hence require a network connection to be read.
+"""
+
+from io import BytesIO
+import logging
+import re
+
+import numpy as np
+import pytest
+
+import pandas.util._test_decorators as td
+
+from pandas import DataFrame
+import pandas._testing as tm
+
+from pandas.io.feather_format import read_feather
+from pandas.io.parsers import read_csv
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+@pytest.mark.network
+@pytest.mark.single_cpu
+@pytest.mark.parametrize("mode", ["explicit", "infer"])
+@pytest.mark.parametrize("engine", ["python", "c"])
+def test_compressed_urls(
+ httpserver,
+ datapath,
+ salaries_table,
+ mode,
+ engine,
+ compression_only,
+ compression_to_extension,
+):
+ # test reading compressed urls with various engines and
+ # extension inference
+ if compression_only == "tar":
+ pytest.skip("TODO: Add tar salaries.csv to pandas/io/parsers/data")
+
+ extension = compression_to_extension[compression_only]
+ with open(datapath("io", "parser", "data", "salaries.csv" + extension), "rb") as f:
+ httpserver.serve_content(content=f.read())
+
+ url = httpserver.url + "/salaries.csv" + extension
+
+ if mode != "explicit":
+ compression_only = mode
+
+ url_table = read_csv(url, sep="\t", compression=compression_only, engine=engine)
+ tm.assert_frame_equal(url_table, salaries_table)
+
+
+@pytest.mark.network
+@pytest.mark.single_cpu
+def test_url_encoding_csv(httpserver, datapath):
+ """
+ read_csv should honor the requested encoding for URLs.
+
+ GH 10424
+ """
+ with open(datapath("io", "parser", "data", "unicode_series.csv"), "rb") as f:
+ httpserver.serve_content(content=f.read())
+ df = read_csv(httpserver.url, encoding="latin-1", header=None)
+ assert df.loc[15, 1] == "Á köldum klaka (Cold Fever) (1994)"
+
+
+@pytest.fixture
+def tips_df(datapath):
+ """DataFrame with the tips dataset."""
+ return read_csv(datapath("io", "data", "csv", "tips.csv"))
+
+
+@pytest.mark.single_cpu
+@pytest.mark.network
+@td.skip_if_not_us_locale()
+class TestS3:
+ @pytest.mark.parametrize(
+ "suffix, compression",
+ [
+ ("", None),
+ (".gz", "gzip"),
+ (".bz2", "bz2"),
+ ],
+ )
+ @pytest.mark.parametrize("nrows", [None, 10])
+ @pytest.mark.parametrize("engine", ["c", "python"])
+ def test_parse_public_s3_bucket(
+ self,
+ s3_bucket_public_with_data,
+ s3so,
+ tips_df,
+ suffix,
+ compression,
+ nrows,
+ engine,
+ ):
+ # more of an integration test due to the not-public contents portion
+ # can probably mock this though.
+ pytest.importorskip("s3fs")
+ df = read_csv(
+ f"s3://{s3_bucket_public_with_data.name}/tips.csv{suffix}",
+ nrows=nrows,
+ compression=compression,
+ storage_options=s3so,
+ engine=engine,
+ )
+ tm.assert_frame_equal(df, tips_df.iloc[:nrows])
+
+ def test_parse_private_s3_bucket(self, s3_bucket_private_with_data, s3so, tips_df):
+ # Read public file from bucket with not-public contents
+ pytest.importorskip("s3fs")
+ df = read_csv(
+ f"s3://{s3_bucket_private_with_data.name}/tips.csv", storage_options=s3so
+ )
+ tm.assert_frame_equal(df, tips_df)
+
+ @pytest.mark.parametrize("scheme", ["s3n", "s3a"])
+ def test_parse_public_bucket_s3n_s3a(
+ self, s3_bucket_public_with_data, s3so, tips_df, scheme
+ ):
+ nrows = 10
+ df = read_csv(
+ f"{scheme}://{s3_bucket_public_with_data.name}/tips.csv",
+ nrows=nrows,
+ storage_options=s3so,
+ )
+ tm.assert_frame_equal(df, tips_df.iloc[:nrows])
+
+ @pytest.mark.parametrize(
+ "suffix, compression",
+ [
+ ("", None),
+ (".gz", "gzip"),
+ (".bz2", "bz2"),
+ ],
+ )
+ @pytest.mark.parametrize("engine", ["c", "python"])
+ def test_parse_public_s3_bucket_chunked(
+ self, s3_bucket_public_with_data, s3so, tips_df, suffix, compression, engine
+ ):
+ # Read with a chunksize
+ chunksize = 5
+ with read_csv(
+ f"s3://{s3_bucket_public_with_data.name}/tips.csv{suffix}",
+ chunksize=chunksize,
+ compression=compression,
+ storage_options=s3so,
+ engine=engine,
+ ) as df_reader:
+ assert df_reader.chunksize == chunksize
+ for i_chunk in [0, 1, 2]:
+ # Read a couple of chunks and make sure we see them
+ # properly.
+ df = df_reader.get_chunk()
+ assert isinstance(df, DataFrame)
+ assert not df.empty
+ true_df = tips_df.iloc[chunksize * i_chunk : chunksize * (i_chunk + 1)]
+ tm.assert_frame_equal(true_df, df)
+
+ @pytest.mark.parametrize("suffix", ["", ".gz", ".bz2"])
+ def test_infer_s3_compression(
+ self, s3_bucket_public_with_data, s3so, tips_df, suffix
+ ):
+ df = read_csv(
+ f"s3://{s3_bucket_public_with_data.name}/tips.csv{suffix}",
+ engine="python",
+ compression="infer",
+ storage_options=s3so,
+ )
+ tm.assert_frame_equal(df, tips_df)
+
+ def test_read_s3_fails(self, s3_bucket_public_with_data, s3so):
+ msg = "The specified bucket does not exist"
+ with pytest.raises(OSError, match=msg):
+ read_csv("s3://nyqpug/asdf.csv", storage_options=s3so)
+
+ def test_read_s3_fails_private(self, s3_bucket_private_with_data, s3so):
+ s3_url = f"{s3_bucket_private_with_data.name}/file.csv"
+ msg = rf"{s3_url}"
+ # Receive a permission error when trying to read a private bucket.
+ # It's irrelevant here that this isn't actually a table.
+ with pytest.raises(FileNotFoundError, match=msg):
+ read_csv(
+ f"s3://{s3_url}",
+ storage_options=s3so,
+ )
+
+ @pytest.mark.single_cpu
+ def test_read_csv_handles_boto_s3_object(
+ self, s3_bucket_public_with_data, tips_file
+ ):
+ # see gh-16135
+
+ s3_object = s3_bucket_public_with_data.Object("tips.csv")
+
+ with BytesIO(s3_object.get()["Body"].read()) as buffer:
+ result = read_csv(buffer, encoding="utf8")
+ assert isinstance(result, DataFrame)
+ assert not result.empty
+
+ expected = read_csv(tips_file)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.single_cpu
+ def test_read_csv_chunked_download(self, s3_bucket_public, s3so, caplog):
+ # 8 MB, S3FS uses 5MB chunks
+ df = DataFrame(np.zeros((100000, 4)), columns=list("abcd"))
+ with BytesIO(df.to_csv().encode("utf-8")) as buf:
+ s3_bucket_public.put_object(Key="large-file.csv", Body=buf)
+ uri = f"{s3_bucket_public.name}/large-file.csv"
+ match_re = re.compile(rf"^Fetch: {uri}, 0-(?P\d+)$")
+ with caplog.at_level(logging.DEBUG, logger="s3fs"):
+ read_csv(
+ f"s3://{uri}",
+ nrows=5,
+ storage_options=s3so,
+ )
+ for log in caplog.messages:
+ if match := re.match(match_re, log):
+ # Less than 8 MB
+ assert int(match.group("stop")) < 8000000
+
+ def test_read_s3_with_hash_in_key(self, s3_bucket_public_with_data, s3so, tips_df):
+ # GH 25945
+ result = read_csv(
+ f"s3://{s3_bucket_public_with_data.name}/tips#1.csv", storage_options=s3so
+ )
+ tm.assert_frame_equal(tips_df, result)
+
+ def test_read_feather_s3_file_path(
+ self, s3_bucket_public_with_data, s3so, feather_file
+ ):
+ # GH 29055
+ pytest.importorskip("pyarrow")
+ expected = read_feather(feather_file)
+ res = read_feather(
+ f"s3://{s3_bucket_public_with_data.name}/simple_dataset.feather",
+ storage_options=s3so,
+ )
+ tm.assert_frame_equal(expected, res)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_parse_dates.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_parse_dates.py
new file mode 100644
index 0000000000000000000000000000000000000000..41effd4c2896e1990302626346b0e8cb18ef2c7d
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_parse_dates.py
@@ -0,0 +1,857 @@
+"""
+Tests date parsing functionality for all of the
+parsers defined in parsers.py
+"""
+
+from datetime import (
+ datetime,
+ timedelta,
+ timezone,
+)
+from io import StringIO
+
+import numpy as np
+import pytest
+
+from pandas.errors import Pandas4Warning
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ DatetimeIndex,
+ Index,
+ MultiIndex,
+ Series,
+ Timestamp,
+)
+import pandas._testing as tm
+from pandas.core.indexes.datetimes import date_range
+from pandas.core.tools.datetimes import start_caching_at
+
+from pandas.io.parsers import read_csv
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+
+def test_date_col_as_index_col(all_parsers):
+ data = """\
+KORD,19990127 19:00:00, 18:56:00, 0.8100, 2.8100, 7.2000, 0.0000, 280.0000
+KORD,19990127 20:00:00, 19:56:00, 0.0100, 2.2100, 7.2000, 0.0000, 260.0000
+KORD,19990127 21:00:00, 20:56:00, -0.5900, 2.2100, 5.7000, 0.0000, 280.0000
+KORD,19990127 21:00:00, 21:18:00, -0.9900, 2.0100, 3.6000, 0.0000, 270.0000
+KORD,19990127 22:00:00, 21:56:00, -0.5900, 1.7100, 5.1000, 0.0000, 290.0000
+"""
+ parser = all_parsers
+ kwds = {
+ "header": None,
+ "parse_dates": [1],
+ "index_col": 1,
+ "names": ["X0", "X1", "X2", "X3", "X4", "X5", "X6", "X7"],
+ }
+ result = parser.read_csv(StringIO(data), **kwds)
+
+ index = Index(
+ [
+ datetime(1999, 1, 27, 19, 0),
+ datetime(1999, 1, 27, 20, 0),
+ datetime(1999, 1, 27, 21, 0),
+ datetime(1999, 1, 27, 21, 0),
+ datetime(1999, 1, 27, 22, 0),
+ ],
+ dtype="M8[us]",
+ name="X1",
+ )
+ expected = DataFrame(
+ [
+ ["KORD", " 18:56:00", 0.81, 2.81, 7.2, 0.0, 280.0],
+ ["KORD", " 19:56:00", 0.01, 2.21, 7.2, 0.0, 260.0],
+ ["KORD", " 20:56:00", -0.59, 2.21, 5.7, 0.0, 280.0],
+ ["KORD", " 21:18:00", -0.99, 2.01, 3.6, 0.0, 270.0],
+ ["KORD", " 21:56:00", -0.59, 1.71, 5.1, 0.0, 290.0],
+ ],
+ columns=["X0", "X2", "X3", "X4", "X5", "X6", "X7"],
+ index=index,
+ )
+ if parser.engine == "pyarrow":
+ # https://github.com/pandas-dev/pandas/issues/44231
+ # pyarrow 6.0 starts to infer time type
+ expected["X2"] = pd.to_datetime("1970-01-01" + expected["X2"]).dt.time
+
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow
+def test_nat_parse(all_parsers, temp_file):
+ # see gh-3062
+ parser = all_parsers
+ df = DataFrame(
+ {
+ "A": np.arange(10, dtype="float64"),
+ "B": Timestamp("20010101"),
+ }
+ )
+ df.iloc[3:6, :] = np.nan
+
+ path = temp_file
+ df.to_csv(path)
+
+ result = parser.read_csv(path, index_col=0, parse_dates=["B"])
+ tm.assert_frame_equal(result, df)
+
+
+@skip_pyarrow
+def test_parse_dates_implicit_first_col(all_parsers):
+ data = """A,B,C
+20090101,a,1,2
+20090102,b,3,4
+20090103,c,4,5
+"""
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), parse_dates=True)
+
+ expected = parser.read_csv(StringIO(data), index_col=0, parse_dates=True)
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow
+def test_parse_dates_string(all_parsers):
+ data = """date,A,B,C
+20090101,a,1,2
+20090102,b,3,4
+20090103,c,4,5
+"""
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), index_col="date", parse_dates=["date"])
+ # freq doesn't round-trip
+ index = date_range("1/1/2009", periods=3, name="date", unit="us")._with_freq(None)
+
+ expected = DataFrame(
+ {"A": ["a", "b", "c"], "B": [1, 3, 4], "C": [2, 4, 5]}, index=index
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow
+@pytest.mark.parametrize("parse_dates", [[0, 2], ["a", "c"]])
+def test_parse_dates_column_list(all_parsers, parse_dates):
+ data = "a,b,c\n01/01/2010,1,15/02/2010"
+ parser = all_parsers
+
+ expected = DataFrame(
+ {"a": [datetime(2010, 1, 1)], "b": [1], "c": [datetime(2010, 2, 15)]}
+ )
+ expected["a"] = expected["a"].astype("M8[us]")
+ expected["c"] = expected["c"].astype("M8[us]")
+ expected = expected.set_index(["a", "b"])
+
+ result = parser.read_csv(
+ StringIO(data), index_col=[0, 1], parse_dates=parse_dates, dayfirst=True
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow
+@pytest.mark.parametrize("index_col", [[0, 1], [1, 0]])
+def test_multi_index_parse_dates(all_parsers, index_col):
+ data = """index1,index2,A,B,C
+20090101,one,a,1,2
+20090101,two,b,3,4
+20090101,three,c,4,5
+20090102,one,a,1,2
+20090102,two,b,3,4
+20090102,three,c,4,5
+20090103,one,a,1,2
+20090103,two,b,3,4
+20090103,three,c,4,5
+"""
+ parser = all_parsers
+ dti = date_range("2009-01-01", periods=3, freq="D", unit="us")
+ index = MultiIndex.from_product(
+ [
+ dti,
+ ("one", "two", "three"),
+ ],
+ names=["index1", "index2"],
+ )
+
+ # Out of order.
+ if index_col == [1, 0]:
+ index = index.swaplevel(0, 1)
+
+ expected = DataFrame(
+ [
+ ["a", 1, 2],
+ ["b", 3, 4],
+ ["c", 4, 5],
+ ["a", 1, 2],
+ ["b", 3, 4],
+ ["c", 4, 5],
+ ["a", 1, 2],
+ ["b", 3, 4],
+ ["c", 4, 5],
+ ],
+ columns=["A", "B", "C"],
+ index=index,
+ )
+ result = parser.read_csv_check_warnings(
+ UserWarning,
+ "Could not infer format",
+ StringIO(data),
+ index_col=index_col,
+ parse_dates=True,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_parse_tz_aware(all_parsers):
+ # See gh-1693
+ parser = all_parsers
+ data = "Date,x\n2012-06-13T01:39:00Z,0.5"
+
+ result = parser.read_csv(StringIO(data), index_col=0, parse_dates=True)
+ expected = DataFrame(
+ {"x": [0.5]}, index=Index([Timestamp("2012-06-13 01:39:00+00:00")], name="Date")
+ )
+ if parser.engine == "pyarrow":
+ expected.index = expected.index.as_unit("s")
+ tm.assert_frame_equal(result, expected)
+ assert result.index.tz is timezone.utc
+
+
+@pytest.mark.parametrize("kwargs", [{}, {"index_col": "C"}])
+def test_read_with_parse_dates_scalar_non_bool(all_parsers, kwargs):
+ # see gh-5636
+ parser = all_parsers
+ msg = "Only booleans and lists are accepted for the 'parse_dates' parameter"
+ data = """A,B,C
+ 1,2,2003-11-1"""
+
+ with pytest.raises(TypeError, match=msg):
+ parser.read_csv(StringIO(data), parse_dates="C", **kwargs)
+
+
+@pytest.mark.parametrize("parse_dates", [(1,), np.array([4, 5]), {1, 3}])
+def test_read_with_parse_dates_invalid_type(all_parsers, parse_dates):
+ parser = all_parsers
+ msg = "Only booleans and lists are accepted for the 'parse_dates' parameter"
+ data = """A,B,C
+ 1,2,2003-11-1"""
+
+ with pytest.raises(TypeError, match=msg):
+ parser.read_csv(StringIO(data), parse_dates=parse_dates)
+
+
+@pytest.mark.parametrize("value", ["nan", ""])
+def test_bad_date_parse(all_parsers, cache, value):
+ # if we have an invalid date make sure that we handle this with
+ # and w/o the cache properly
+ parser = all_parsers
+ s = StringIO((f"{value},\n") * (start_caching_at + 1))
+
+ parser.read_csv(
+ s,
+ header=None,
+ names=["foo", "bar"],
+ parse_dates=["foo"],
+ cache_dates=cache,
+ )
+
+
+def test_bad_date_parse_with_warning(all_parsers, cache):
+ # if we have an invalid date make sure that we handle this with
+ # and w/o the cache properly.
+ parser = all_parsers
+ s = StringIO(("0,\n") * (start_caching_at + 1))
+
+ if parser.engine == "pyarrow":
+ # pyarrow reads "0" as 0 (of type int64), and so
+ # pandas doesn't try to guess the datetime format
+ # TODO: parse dates directly in pyarrow, see
+ # https://github.com/pandas-dev/pandas/issues/48017
+ warn = None
+ elif cache:
+ # Note: warning is not raised if 'cache_dates', because here there is only a
+ # single unique date and hence no risk of inconsistent parsing.
+ warn = None
+ else:
+ warn = UserWarning
+ parser.read_csv_check_warnings(
+ warn,
+ "Could not infer format",
+ s,
+ header=None,
+ names=["foo", "bar"],
+ parse_dates=["foo"],
+ cache_dates=cache,
+ raise_on_extra_warnings=False,
+ )
+
+
+def test_parse_dates_empty_string(all_parsers):
+ # see gh-2263
+ parser = all_parsers
+ data = "Date,test\n2012-01-01,1\n,2"
+ result = parser.read_csv(StringIO(data), parse_dates=["Date"], na_filter=False)
+
+ expected = DataFrame(
+ [[datetime(2012, 1, 1), 1], [pd.NaT, 2]], columns=["Date", "test"]
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow
+@pytest.mark.parametrize(
+ "data,kwargs,expected",
+ [
+ (
+ "a\n04.15.2016",
+ {"parse_dates": ["a"]},
+ DataFrame([datetime(2016, 4, 15)], columns=["a"], dtype="M8[us]"),
+ ),
+ (
+ "a\n04.15.2016",
+ {"parse_dates": True, "index_col": 0},
+ DataFrame(
+ index=DatetimeIndex(["2016-04-15"], dtype="M8[us]", name="a"),
+ columns=[],
+ ),
+ ),
+ (
+ "a,b\n04.15.2016,09.16.2013",
+ {"parse_dates": ["a", "b"]},
+ DataFrame(
+ [[datetime(2016, 4, 15), datetime(2013, 9, 16)]],
+ dtype="M8[us]",
+ columns=["a", "b"],
+ ),
+ ),
+ (
+ "a,b\n04.15.2016,09.16.2013",
+ {"parse_dates": True, "index_col": [0, 1]},
+ DataFrame(
+ index=MultiIndex.from_tuples(
+ [
+ (
+ Timestamp(2016, 4, 15),
+ Timestamp(2013, 9, 16),
+ )
+ ],
+ names=["a", "b"],
+ ),
+ columns=[],
+ ),
+ ),
+ ],
+)
+def test_parse_dates_no_convert_thousands(all_parsers, data, kwargs, expected):
+ # see gh-14066
+ parser = all_parsers
+
+ result = parser.read_csv(StringIO(data), thousands=".", **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_parse_date_column_with_empty_string(all_parsers):
+ # see gh-6428
+ parser = all_parsers
+ data = "case,opdate\n7,10/18/2006\n7,10/18/2008\n621, "
+ result = parser.read_csv(StringIO(data), parse_dates=["opdate"])
+
+ expected_data = [[7, "10/18/2006"], [7, "10/18/2008"], [621, " "]]
+ expected = DataFrame(expected_data, columns=["case", "opdate"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "data,expected",
+ [
+ (
+ "a\n135217135789158401\n1352171357E+5",
+ [135217135789158401, 135217135700000],
+ ),
+ (
+ "a\n99999999999\n123456789012345\n1234E+0",
+ [99999999999, 123456789012345, 1234],
+ ),
+ ],
+)
+@pytest.mark.parametrize("parse_dates", [True, False])
+def test_parse_date_float(all_parsers, data, expected, parse_dates):
+ # see gh-2697
+ #
+ # Date parsing should fail, so we leave the data untouched
+ # (i.e. float precision should remain unchanged).
+ parser = all_parsers
+
+ result = parser.read_csv(StringIO(data), parse_dates=parse_dates)
+ expected = DataFrame({"a": expected}, dtype="float64")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_parse_timezone(all_parsers):
+ # see gh-22256
+ parser = all_parsers
+ data = """dt,val
+ 2018-01-04 09:01:00+09:00,23350
+ 2018-01-04 09:02:00+09:00,23400
+ 2018-01-04 09:03:00+09:00,23400
+ 2018-01-04 09:04:00+09:00,23400
+ 2018-01-04 09:05:00+09:00,23400"""
+ result = parser.read_csv(StringIO(data), parse_dates=["dt"])
+
+ dti = date_range(
+ start="2018-01-04 09:01:00",
+ end="2018-01-04 09:05:00",
+ freq="1min",
+ tz=timezone(timedelta(minutes=540)),
+ unit="us",
+ )._with_freq(None)
+ expected_data = {"dt": dti, "val": [23350, 23400, 23400, 23400, 23400]}
+
+ expected = DataFrame(expected_data)
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # pandas.errors.ParserError: CSV parse error
+@pytest.mark.parametrize(
+ "date_string",
+ ["32/32/2019", "02/30/2019", "13/13/2019", "13/2019", "a3/11/2018", "10/11/2o17"],
+)
+def test_invalid_parse_delimited_date(all_parsers, date_string):
+ parser = all_parsers
+ expected = DataFrame({0: [date_string]}, dtype="str")
+ result = parser.read_csv(
+ StringIO(date_string),
+ header=None,
+ parse_dates=[0],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "date_string,dayfirst,expected",
+ [
+ # %d/%m/%Y; month > 12 thus replacement
+ ("13/02/2019", True, datetime(2019, 2, 13)),
+ # %m/%d/%Y; day > 12 thus there will be no replacement
+ ("02/13/2019", False, datetime(2019, 2, 13)),
+ # %d/%m/%Y; dayfirst==True thus replacement
+ ("04/02/2019", True, datetime(2019, 2, 4)),
+ ],
+)
+def test_parse_delimited_date_swap_no_warning(
+ all_parsers, date_string, dayfirst, expected, request
+):
+ parser = all_parsers
+ expected = DataFrame({0: [expected]}, dtype="datetime64[us]")
+ if parser.engine == "pyarrow":
+ if not dayfirst:
+ # "CSV parse error: Empty CSV file or block"
+ pytest.skip(reason="https://github.com/apache/arrow/issues/38676")
+ msg = "The 'dayfirst' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(date_string), header=None, dayfirst=dayfirst, parse_dates=[0]
+ )
+ return
+
+ result = parser.read_csv(
+ StringIO(date_string), header=None, dayfirst=dayfirst, parse_dates=[0]
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+# ArrowInvalid: CSV parse error: Empty CSV file or block: cannot infer number of columns
+@skip_pyarrow
+@pytest.mark.parametrize(
+ "date_string,dayfirst,expected",
+ [
+ # %d/%m/%Y; month > 12
+ ("13/02/2019", False, datetime(2019, 2, 13)),
+ # %m/%d/%Y; day > 12
+ ("02/13/2019", True, datetime(2019, 2, 13)),
+ ],
+)
+def test_parse_delimited_date_swap_with_warning(
+ all_parsers, date_string, dayfirst, expected
+):
+ parser = all_parsers
+ expected = DataFrame({0: [expected]}, dtype="datetime64[us]")
+ warning_msg = (
+ "Parsing dates in .* format when dayfirst=.* was specified. "
+ "Pass `dayfirst=.*` or specify a format to silence this warning."
+ )
+ result = parser.read_csv_check_warnings(
+ UserWarning,
+ warning_msg,
+ StringIO(date_string),
+ header=None,
+ dayfirst=dayfirst,
+ parse_dates=[0],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_parse_multiple_delimited_dates_with_swap_warnings():
+ # GH46210
+ with pytest.raises(
+ ValueError,
+ match=(
+ r'^time data "31/05/2000" doesn\'t match format "%m/%d/%Y". '
+ r"You might want to try:"
+ ),
+ ):
+ pd.to_datetime(["01/01/2000", "31/05/2000", "31/05/2001", "01/02/2000"])
+
+
+# ArrowKeyError: Column 'fdate1' in include_columns does not exist in CSV file
+@skip_pyarrow
+@pytest.mark.parametrize(
+ "names, usecols, parse_dates, missing_cols",
+ [
+ (None, ["val"], ["date", "time"], "date, time"),
+ (None, ["val"], [0, "time"], "time"),
+ (["date1", "time1", "temperature"], None, ["date", "time"], "date, time"),
+ (
+ ["date1", "time1", "temperature"],
+ ["date1", "temperature"],
+ ["date1", "time"],
+ "time",
+ ),
+ ],
+)
+def test_missing_parse_dates_column_raises(
+ all_parsers, names, usecols, parse_dates, missing_cols
+):
+ # gh-31251 column names provided in parse_dates could be missing.
+ parser = all_parsers
+ content = StringIO("date,time,val\n2020-01-31,04:20:32,32\n")
+ msg = f"Missing column provided to 'parse_dates': '{missing_cols}'"
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ content, sep=",", names=names, usecols=usecols, parse_dates=parse_dates
+ )
+
+
+@xfail_pyarrow # mismatched shape
+def test_date_parser_and_names(all_parsers):
+ # GH#33699
+ parser = all_parsers
+ data = StringIO("""x,y\n1,2""")
+ warn = UserWarning
+ if parser.engine == "pyarrow":
+ # Pandas4Warning for passing a Manager object
+ warn = (UserWarning, Pandas4Warning)
+ result = parser.read_csv_check_warnings(
+ warn,
+ "Could not infer format",
+ data,
+ parse_dates=["B"],
+ names=["B"],
+ )
+ expected = DataFrame({"B": ["y", "2"]}, index=["x", "1"])
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # TypeError: an integer is required
+def test_date_parser_multiindex_columns(all_parsers):
+ parser = all_parsers
+ data = """a,b
+1,2
+2019-12-31,6"""
+ result = parser.read_csv(StringIO(data), parse_dates=[("a", "1")], header=[0, 1])
+ expected = DataFrame({("a", "1"): Timestamp("2019-12-31"), ("b", "2"): [6]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_date_parser_usecols_thousands(all_parsers):
+ # GH#39365
+ data = """A,B,C
+ 1,3,20-09-01-01
+ 2,4,20-09-01-01
+ """
+
+ parser = all_parsers
+
+ if parser.engine == "pyarrow":
+ # DeprecationWarning for passing a Manager object
+ msg = "The 'thousands' option is not supported with the 'pyarrow' engine"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(
+ StringIO(data),
+ parse_dates=[1],
+ usecols=[1, 2],
+ thousands="-",
+ )
+ return
+
+ result = parser.read_csv_check_warnings(
+ UserWarning,
+ "Could not infer format",
+ StringIO(data),
+ parse_dates=[1],
+ usecols=[1, 2],
+ thousands="-",
+ )
+ expected = DataFrame({"B": [3, 4], "C": [Timestamp("20-09-2001 01:00:00")] * 2})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_dayfirst_warnings():
+ # GH 12585
+
+ # CASE 1: valid input
+ input = "date\n31/12/2014\n10/03/2011"
+ expected = DatetimeIndex(
+ ["2014-12-31", "2011-03-10"], dtype="datetime64[us]", freq=None, name="date"
+ )
+ warning_msg = (
+ "Parsing dates in .* format when dayfirst=.* was specified. "
+ "Pass `dayfirst=.*` or specify a format to silence this warning."
+ )
+
+ # A. dayfirst arg correct, no warning
+ res1 = read_csv(
+ StringIO(input), parse_dates=["date"], dayfirst=True, index_col="date"
+ ).index
+ tm.assert_index_equal(expected, res1)
+
+ # B. dayfirst arg incorrect, warning
+ with tm.assert_produces_warning(UserWarning, match=warning_msg):
+ res2 = read_csv(
+ StringIO(input), parse_dates=["date"], dayfirst=False, index_col="date"
+ ).index
+ tm.assert_index_equal(expected, res2)
+
+ # CASE 2: invalid input
+ # cannot consistently process with single format
+ # return to user unaltered
+
+ # first in DD/MM/YYYY, second in MM/DD/YYYY
+ input = "date\n31/12/2014\n03/30/2011"
+ expected = Index(["31/12/2014", "03/30/2011"], dtype="str", name="date")
+
+ # A. use dayfirst=True
+ res5 = read_csv(
+ StringIO(input), parse_dates=["date"], dayfirst=True, index_col="date"
+ ).index
+ tm.assert_index_equal(expected, res5)
+
+ # B. use dayfirst=False
+ with tm.assert_produces_warning(UserWarning, match=warning_msg):
+ res6 = read_csv(
+ StringIO(input), parse_dates=["date"], dayfirst=False, index_col="date"
+ ).index
+ tm.assert_index_equal(expected, res6)
+
+
+@pytest.mark.parametrize(
+ "date_string, dayfirst",
+ [
+ pytest.param(
+ "31/1/2014",
+ False,
+ id="second date is single-digit",
+ ),
+ pytest.param(
+ "1/31/2014",
+ True,
+ id="first date is single-digit",
+ ),
+ ],
+)
+def test_dayfirst_warnings_no_leading_zero(date_string, dayfirst):
+ # GH47880
+ initial_value = f"date\n{date_string}"
+ expected = DatetimeIndex(
+ ["2014-01-31"], dtype="datetime64[us]", freq=None, name="date"
+ )
+ warning_msg = (
+ "Parsing dates in .* format when dayfirst=.* was specified. "
+ "Pass `dayfirst=.*` or specify a format to silence this warning."
+ )
+ with tm.assert_produces_warning(UserWarning, match=warning_msg):
+ res = read_csv(
+ StringIO(initial_value),
+ parse_dates=["date"],
+ index_col="date",
+ dayfirst=dayfirst,
+ ).index
+ tm.assert_index_equal(expected, res)
+
+
+@skip_pyarrow # CSV parse error: Expected 3 columns, got 4
+def test_infer_first_column_as_index(all_parsers):
+ # GH#11019
+ parser = all_parsers
+ data = "a,b,c\n1970-01-01,2,3,4"
+ result = parser.read_csv(
+ StringIO(data),
+ parse_dates=["a"],
+ )
+ expected = DataFrame({"a": "2", "b": 3, "c": 4}, index=["1970-01-01"])
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # pyarrow engine doesn't support passing a dict for na_values
+def test_replace_nans_before_parsing_dates(all_parsers):
+ # GH#26203
+ parser = all_parsers
+ data = """Test
+2012-10-01
+0
+2015-05-15
+#
+2017-09-09
+"""
+ result = parser.read_csv(
+ StringIO(data),
+ na_values={"Test": ["#", "0"]},
+ parse_dates=["Test"],
+ date_format="%Y-%m-%d",
+ )
+ expected = DataFrame(
+ {
+ "Test": [
+ Timestamp("2012-10-01"),
+ pd.NaT,
+ Timestamp("2015-05-15"),
+ pd.NaT,
+ Timestamp("2017-09-09"),
+ ]
+ },
+ dtype="M8[us]",
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # string[python] instead of dt64[ns]
+def test_parse_dates_and_string_dtype(all_parsers):
+ # GH#34066
+ parser = all_parsers
+ data = """a,b
+1,2019-12-31
+"""
+ result = parser.read_csv(StringIO(data), dtype="string", parse_dates=["b"])
+ expected = DataFrame({"a": ["1"], "b": [Timestamp("2019-12-31")]})
+ expected["a"] = expected["a"].astype("string")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_parse_dot_separated_dates(all_parsers):
+ # https://github.com/pandas-dev/pandas/issues/2586
+ parser = all_parsers
+ data = """a,b
+27.03.2003 14:55:00.000,1
+03.08.2003 15:20:00.000,2"""
+ if parser.engine == "pyarrow":
+ expected_index = Index(
+ ["27.03.2003 14:55:00.000", "03.08.2003 15:20:00.000"],
+ dtype="str",
+ name="a",
+ )
+ warn = None
+ else:
+ expected_index = DatetimeIndex(
+ ["2003-03-27 14:55:00", "2003-08-03 15:20:00"],
+ dtype="datetime64[us]",
+ name="a",
+ )
+ warn = UserWarning
+ msg = r"when dayfirst=False \(the default\) was specified"
+ result = parser.read_csv_check_warnings(
+ warn,
+ msg,
+ StringIO(data),
+ parse_dates=True,
+ index_col=0,
+ raise_on_extra_warnings=False,
+ )
+ expected = DataFrame({"b": [1, 2]}, index=expected_index)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_parse_dates_dict_format(all_parsers):
+ # GH#51240
+ parser = all_parsers
+ data = """a,b
+2019-12-31,31-12-2019
+2020-12-31,31-12-2020"""
+
+ result = parser.read_csv(
+ StringIO(data),
+ date_format={"a": "%Y-%m-%d", "b": "%d-%m-%Y"},
+ parse_dates=["a", "b"],
+ )
+ expected = DataFrame(
+ {
+ "a": [Timestamp("2019-12-31"), Timestamp("2020-12-31")],
+ "b": [Timestamp("2019-12-31"), Timestamp("2020-12-31")],
+ },
+ dtype="M8[us]",
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # object dtype index
+def test_parse_dates_dict_format_index(all_parsers):
+ # GH#51240
+ parser = all_parsers
+ data = """a,b
+2019-12-31,31-12-2019
+2020-12-31,31-12-2020"""
+
+ result = parser.read_csv(
+ StringIO(data), date_format={"a": "%Y-%m-%d"}, parse_dates=True, index_col=0
+ )
+ expected = DataFrame(
+ {
+ "b": ["31-12-2019", "31-12-2020"],
+ },
+ index=Index([Timestamp("2019-12-31"), Timestamp("2020-12-31")], name="a"),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_parse_dates_arrow_engine(all_parsers):
+ # GH#53295
+ parser = all_parsers
+ data = """a,b
+2000-01-01 00:00:00,1
+2000-01-01 00:00:01,1"""
+
+ result = parser.read_csv(StringIO(data), parse_dates=["a"])
+ expected = DataFrame(
+ {
+ "a": [
+ Timestamp("2000-01-01 00:00:00"),
+ Timestamp("2000-01-01 00:00:01"),
+ ],
+ "b": 1,
+ }
+ )
+ if parser.engine == "pyarrow":
+ expected["a"] = expected["a"].astype("M8[s]")
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # object dtype index
+def test_from_csv_with_mixed_offsets(all_parsers):
+ parser = all_parsers
+ data = "a\n2020-01-01T00:00:00+01:00\n2020-01-01T00:00:00+00:00"
+ result = parser.read_csv(StringIO(data), parse_dates=["a"])["a"]
+ expected = Series(
+ [
+ "2020-01-01T00:00:00+01:00",
+ "2020-01-01T00:00:00+00:00",
+ ],
+ name="a",
+ index=[0, 1],
+ )
+ tm.assert_series_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_python_parser_only.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_python_parser_only.py
new file mode 100644
index 0000000000000000000000000000000000000000..17f0187f4377e1210f3f915675e16bdfe9c66774
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_python_parser_only.py
@@ -0,0 +1,643 @@
+"""
+Tests that apply specifically to the Python parser. Unless specifically
+stated as a Python-specific issue, the goal is to eventually move as many of
+these tests out of this module as soon as the C parser can accept further
+arguments when parsing.
+"""
+
+from __future__ import annotations
+
+import csv
+from io import (
+ BytesIO,
+ StringIO,
+ TextIOWrapper,
+)
+from typing import TYPE_CHECKING
+
+import numpy as np
+import pytest
+
+from pandas.errors import (
+ ParserError,
+ ParserWarning,
+)
+
+from pandas import (
+ DataFrame,
+ Index,
+ MultiIndex,
+)
+import pandas._testing as tm
+
+if TYPE_CHECKING:
+ from collections.abc import Iterator
+
+
+def test_default_separator(python_parser_only):
+ # see gh-17333
+ #
+ # csv.Sniffer in Python treats "o" as separator.
+ data = "aob\n1o2\n3o4"
+ parser = python_parser_only
+ expected = DataFrame({"a": [1, 3], "b": [2, 4]})
+
+ result = parser.read_csv(StringIO(data), sep=None)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("skipfooter", ["foo", 1.5, True])
+def test_invalid_skipfooter_non_int(python_parser_only, skipfooter):
+ # see gh-15925 (comment)
+ data = "a\n1\n2"
+ parser = python_parser_only
+ msg = "skipfooter must be an integer"
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), skipfooter=skipfooter)
+
+
+def test_invalid_skipfooter_negative(python_parser_only):
+ # see gh-15925 (comment)
+ data = "a\n1\n2"
+ parser = python_parser_only
+ msg = "skipfooter cannot be negative"
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), skipfooter=-1)
+
+
+@pytest.mark.parametrize("kwargs", [{"sep": None}, {"delimiter": "|"}])
+def test_sniff_delimiter(python_parser_only, kwargs):
+ data = """index|A|B|C
+foo|1|2|3
+bar|4|5|6
+baz|7|8|9
+"""
+ parser = python_parser_only
+ result = parser.read_csv(StringIO(data), index_col=0, **kwargs)
+ expected = DataFrame(
+ [[1, 2, 3], [4, 5, 6], [7, 8, 9]],
+ columns=["A", "B", "C"],
+ index=Index(["foo", "bar", "baz"], name="index"),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_sniff_delimiter_comment(python_parser_only):
+ data = """# comment line
+index|A|B|C
+# comment line
+foo|1|2|3 # ignore | this
+bar|4|5|6
+baz|7|8|9
+"""
+ parser = python_parser_only
+ result = parser.read_csv(StringIO(data), index_col=0, sep=None, comment="#")
+ expected = DataFrame(
+ [[1, 2, 3], [4, 5, 6], [7, 8, 9]],
+ columns=["A", "B", "C"],
+ index=Index(["foo", "bar", "baz"], name="index"),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("encoding", [None, "utf-8"])
+def test_sniff_delimiter_encoding(python_parser_only, encoding):
+ parser = python_parser_only
+ data = """ignore this
+ignore this too
+index|A|B|C
+foo|1|2|3
+bar|4|5|6
+baz|7|8|9
+"""
+
+ if encoding is not None:
+ data = data.encode(encoding)
+ data = BytesIO(data)
+ data = TextIOWrapper(data, encoding=encoding)
+ else:
+ data = StringIO(data)
+
+ result = parser.read_csv(data, index_col=0, sep=None, skiprows=2, encoding=encoding)
+ expected = DataFrame(
+ [[1, 2, 3], [4, 5, 6], [7, 8, 9]],
+ columns=["A", "B", "C"],
+ index=Index(["foo", "bar", "baz"], name="index"),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_single_line(python_parser_only):
+ # see gh-6607: sniff separator
+ parser = python_parser_only
+ result = parser.read_csv(StringIO("1,2"), names=["a", "b"], header=None, sep=None)
+
+ expected = DataFrame({"a": [1], "b": [2]})
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("kwargs", [{"skipfooter": 2}, {"nrows": 3}])
+def test_skipfooter(python_parser_only, kwargs):
+ # see gh-6607
+ data = """A,B,C
+1,2,3
+4,5,6
+7,8,9
+want to skip this
+also also skip this
+"""
+ parser = python_parser_only
+ result = parser.read_csv(StringIO(data), **kwargs)
+
+ expected = DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]], columns=["A", "B", "C"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "compression,klass", [("gzip", "GzipFile"), ("bz2", "BZ2File")]
+)
+def test_decompression_regex_sep(
+ temp_file, python_parser_only, csv1, compression, klass
+):
+ # see gh-6607
+ parser = python_parser_only
+
+ with open(csv1, "rb") as f:
+ data = f.read()
+
+ data = data.replace(b",", b"::")
+ expected = parser.read_csv(csv1)
+
+ module = pytest.importorskip(compression)
+ klass = getattr(module, klass)
+
+ with klass(temp_file, mode="wb") as tmp:
+ tmp.write(data)
+
+ result = parser.read_csv(temp_file, sep="::", compression=compression)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_csv_buglet_4x_multi_index(python_parser_only):
+ # see gh-6607
+ data = """ A B C D E
+one two three four
+a b 10.0032 5 -0.5109 -2.3358 -0.4645 0.05076 0.3640
+a q 20 4 0.4473 1.4152 0.2834 1.00661 0.1744
+x q 30 3 -0.6662 -0.5243 -0.3580 0.89145 2.5838"""
+ parser = python_parser_only
+
+ expected = DataFrame(
+ [
+ [-0.5109, -2.3358, -0.4645, 0.05076, 0.3640],
+ [0.4473, 1.4152, 0.2834, 1.00661, 0.1744],
+ [-0.6662, -0.5243, -0.3580, 0.89145, 2.5838],
+ ],
+ columns=["A", "B", "C", "D", "E"],
+ index=MultiIndex.from_tuples(
+ [("a", "b", 10.0032, 5), ("a", "q", 20, 4), ("x", "q", 30, 3)],
+ names=["one", "two", "three", "four"],
+ ),
+ )
+ result = parser.read_csv(StringIO(data), sep=r"\s+")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_csv_buglet_4x_multi_index2(python_parser_only):
+ # see gh-6893
+ data = " A B C\na b c\n1 3 7 0 3 6\n3 1 4 1 5 9"
+ parser = python_parser_only
+
+ expected = DataFrame.from_records(
+ [(1, 3, 7, 0, 3, 6), (3, 1, 4, 1, 5, 9)],
+ columns=list("abcABC"),
+ index=list("abc"),
+ )
+ result = parser.read_csv(StringIO(data), sep=r"\s+")
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("add_footer", [True, False])
+def test_skipfooter_with_decimal(python_parser_only, add_footer):
+ # see gh-6971
+ data = "1#2\n3#4"
+ parser = python_parser_only
+ expected = DataFrame({"a": [1.2, 3.4]})
+
+ if add_footer:
+ # The stray footer line should not mess with the
+ # casting of the first two lines if we skip it.
+ kwargs = {"skipfooter": 1}
+ data += "\nFooter"
+ else:
+ kwargs = {}
+
+ result = parser.read_csv(StringIO(data), names=["a"], decimal="#", **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "sep", ["::", "#####", "!!!", "123", "#1!c5", "%!c!d", "@@#4:2", "_!pd#_"]
+)
+@pytest.mark.parametrize(
+ "encoding", ["utf-16", "utf-16-be", "utf-16-le", "utf-32", "cp037"]
+)
+def test_encoding_non_utf8_multichar_sep(python_parser_only, sep, encoding):
+ # see gh-3404
+ expected = DataFrame({"a": [1], "b": [2]})
+ parser = python_parser_only
+
+ data = "1" + sep + "2"
+ encoded_data = data.encode(encoding)
+
+ result = parser.read_csv(
+ BytesIO(encoded_data), sep=sep, names=["a", "b"], encoding=encoding
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("quoting", [csv.QUOTE_MINIMAL, csv.QUOTE_NONE])
+def test_multi_char_sep_quotes(python_parser_only, quoting):
+ # see gh-13374
+ kwargs = {"sep": ",,"}
+ parser = python_parser_only
+
+ data = 'a,,b\n1,,a\n2,,"2,,b"'
+
+ if quoting == csv.QUOTE_NONE:
+ msg = "Expected 2 fields in line 3, saw 3"
+ with pytest.raises(ParserError, match=msg):
+ parser.read_csv(StringIO(data), quoting=quoting, **kwargs)
+ else:
+ msg = "ignored when a multi-char delimiter is used"
+ with pytest.raises(ParserError, match=msg):
+ parser.read_csv(StringIO(data), quoting=quoting, **kwargs)
+
+
+def test_none_delimiter(python_parser_only):
+ # see gh-13374 and gh-17465
+ parser = python_parser_only
+ data = "a,b,c\n0,1,2\n3,4,5,6\n7,8,9"
+ expected = DataFrame({"a": [0, 7], "b": [1, 8], "c": [2, 9]})
+
+ # We expect the third line in the data to be
+ # skipped because it is malformed, but we do
+ # not expect any errors to occur.
+ with tm.assert_produces_warning(
+ ParserWarning, match="Skipping line 3", check_stacklevel=False
+ ):
+ result = parser.read_csv(
+ StringIO(data), header=0, sep=None, on_bad_lines="warn"
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("data", ['a\n1\n"b"a', 'a,b,c\ncat,foo,bar\ndog,foo,"baz'])
+@pytest.mark.parametrize("skipfooter", [0, 1])
+def test_skipfooter_bad_row(python_parser_only, data, skipfooter):
+ # see gh-13879 and gh-15910
+ parser = python_parser_only
+ if skipfooter:
+ msg = "parsing errors in the skipped footer rows"
+ with pytest.raises(ParserError, match=msg):
+ parser.read_csv(StringIO(data), skipfooter=skipfooter)
+ else:
+ msg = "unexpected end of data|expected after"
+ with pytest.raises(ParserError, match=msg):
+ parser.read_csv(StringIO(data), skipfooter=skipfooter)
+
+
+def test_malformed_skipfooter(python_parser_only):
+ parser = python_parser_only
+ data = """ignore
+A,B,C
+1,2,3 # comment
+1,2,3,4,5
+2,3,4
+footer
+"""
+ msg = "Expected 3 fields in line 4, saw 5"
+ with pytest.raises(ParserError, match=msg):
+ parser.read_csv(StringIO(data), header=1, comment="#", skipfooter=1)
+
+
+def test_python_engine_file_no_next(python_parser_only):
+ parser = python_parser_only
+
+ class NoNextBuffer:
+ def __init__(self, csv_data) -> None:
+ self.data = csv_data
+
+ def __iter__(self) -> Iterator:
+ return self.data.__iter__()
+
+ def read(self):
+ return self.data
+
+ def readline(self):
+ return self.data
+
+ parser.read_csv(NoNextBuffer("a\n1"))
+
+
+@pytest.mark.parametrize("bad_line_func", [lambda x: ["2", "3"], lambda x: x[:2]])
+def test_on_bad_lines_callable(python_parser_only, bad_line_func):
+ # GH 5686
+ parser = python_parser_only
+ data = """a,b
+1,2
+2,3,4,5,6
+3,4
+"""
+ bad_sio = StringIO(data)
+ result = parser.read_csv(bad_sio, on_bad_lines=bad_line_func)
+ expected = DataFrame({"a": [1, 2, 3], "b": [2, 3, 4]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_on_bad_lines_callable_write_to_external_list(python_parser_only):
+ # GH 5686
+ parser = python_parser_only
+ data = """a,b
+1,2
+2,3,4,5,6
+3,4
+"""
+ bad_sio = StringIO(data)
+ lst = []
+
+ def bad_line_func(bad_line: list[str]) -> list[str]:
+ lst.append(bad_line)
+ return ["2", "3"]
+
+ result = parser.read_csv(bad_sio, on_bad_lines=bad_line_func)
+ expected = DataFrame({"a": [1, 2, 3], "b": [2, 3, 4]})
+ tm.assert_frame_equal(result, expected)
+ assert lst == [["2", "3", "4", "5", "6"]]
+
+
+@pytest.mark.parametrize("bad_line_func", [lambda x: ["foo", "bar"], lambda x: x[:2]])
+@pytest.mark.parametrize("sep", [",", "111"])
+def test_on_bad_lines_callable_iterator_true(python_parser_only, bad_line_func, sep):
+ # GH 5686
+ # iterator=True has a separate code path than iterator=False
+ parser = python_parser_only
+ data = f"""
+0{sep}1
+hi{sep}there
+foo{sep}bar{sep}baz
+good{sep}bye
+"""
+ bad_sio = StringIO(data)
+ result_iter = parser.read_csv(
+ bad_sio, on_bad_lines=bad_line_func, chunksize=1, iterator=True, sep=sep
+ )
+ expecteds = [
+ {"0": "hi", "1": "there"},
+ {"0": "foo", "1": "bar"},
+ {"0": "good", "1": "bye"},
+ ]
+ for i, (result, expected) in enumerate(zip(result_iter, expecteds)):
+ expected = DataFrame(expected, index=range(i, i + 1))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_on_bad_lines_callable_dont_swallow_errors(python_parser_only):
+ # GH 5686
+ parser = python_parser_only
+ data = """a,b
+1,2
+2,3,4,5,6
+3,4
+"""
+ bad_sio = StringIO(data)
+ msg = "This function is buggy."
+
+ def bad_line_func(bad_line):
+ raise ValueError(msg)
+
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(bad_sio, on_bad_lines=bad_line_func)
+
+
+def test_on_bad_lines_callable_not_expected_length(python_parser_only):
+ # GH 5686
+ parser = python_parser_only
+ data = """a,b
+1,2
+2,3,4,5,6
+3,4
+"""
+ bad_sio = StringIO(data)
+
+ result = parser.read_csv_check_warnings(
+ ParserWarning, "from bad_lines callable", bad_sio, on_bad_lines=lambda x: x
+ )
+ expected = DataFrame({"a": [1, 2, 3], "b": [2, 3, 4]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_on_bad_lines_callable_returns_none(python_parser_only):
+ # GH 5686
+ parser = python_parser_only
+ data = """a,b
+1,2
+2,3,4,5,6
+3,4
+"""
+ bad_sio = StringIO(data)
+
+ result = parser.read_csv(bad_sio, on_bad_lines=lambda x: None)
+ expected = DataFrame({"a": [1, 3], "b": [2, 4]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_on_bad_lines_index_col_inferred(python_parser_only):
+ # GH 5686
+ parser = python_parser_only
+ data = """a,b
+1,2,3
+4,5,6
+"""
+ bad_sio = StringIO(data)
+
+ result = parser.read_csv(bad_sio, on_bad_lines=lambda x: ["99", "99"])
+ expected = DataFrame({"a": [2, 5], "b": [3, 6]}, index=[1, 4])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_index_col_false_and_header_none(python_parser_only):
+ # GH#46955
+ parser = python_parser_only
+ data = """
+0.5,0.03
+0.1,0.2,0.3,2
+"""
+ result = parser.read_csv_check_warnings(
+ ParserWarning,
+ "Length of header",
+ StringIO(data),
+ sep=",",
+ header=None,
+ index_col=False,
+ )
+ expected = DataFrame({0: [0.5, 0.1], 1: [0.03, 0.2]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_header_int_do_not_infer_multiindex_names_on_different_line(python_parser_only):
+ # GH#46569
+ parser = python_parser_only
+ data = StringIO("a\na,b\nc,d,e\nf,g,h")
+ result = parser.read_csv_check_warnings(
+ ParserWarning, "Length of header", data, engine="python", index_col=False
+ )
+ expected = DataFrame({"a": ["a", "c", "f"]})
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "dtype", [{"a": object}, {"a": str, "b": np.int64, "c": np.int64}]
+)
+def test_no_thousand_convert_with_dot_for_non_numeric_cols(python_parser_only, dtype):
+ # GH#50270
+ parser = python_parser_only
+ data = """\
+a;b;c
+0000.7995;16.000;0
+3.03.001.00514;0;4.000
+4923.600.041;23.000;131"""
+ result = parser.read_csv(
+ StringIO(data),
+ sep=";",
+ dtype=dtype,
+ thousands=".",
+ )
+ expected = DataFrame(
+ {
+ "a": ["0000.7995", "3.03.001.00514", "4923.600.041"],
+ "b": [16000, 0, 23000],
+ "c": [0, 4000, 131],
+ }
+ )
+ if dtype["a"] == object:
+ expected["a"] = expected["a"].astype(object)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "dtype,expected",
+ [
+ (
+ {"a": str, "b": np.float64, "c": np.int64},
+ {
+ "b": [16000.1, 0, 23000],
+ "c": [0, 4001, 131],
+ },
+ ),
+ (
+ str,
+ {
+ "b": ["16,000.1", "0", "23,000"],
+ "c": ["0", "4,001", "131"],
+ },
+ ),
+ ],
+)
+def test_no_thousand_convert_for_non_numeric_cols(python_parser_only, dtype, expected):
+ # GH#50270
+ parser = python_parser_only
+ data = """a;b;c
+0000,7995;16,000.1;0
+3,03,001,00514;0;4,001
+4923,600,041;23,000;131
+"""
+ result = parser.read_csv(
+ StringIO(data),
+ sep=";",
+ dtype=dtype,
+ thousands=",",
+ )
+ expected = DataFrame(expected)
+ expected.insert(0, "a", ["0000,7995", "3,03,001,00514", "4923,600,041"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("index_col", [None, 0])
+def test_on_bad_lines_callable_warns_and_truncates_with_index_col(
+ python_parser_only, index_col
+):
+ # GH#61837
+ parser = python_parser_only
+ data = "id,field_1,field_2\n101,A,B\n102,C,D,E\n103,F,G\n"
+
+ def fixer(bad_line):
+ return [*list(bad_line), "EXTRA1", "EXTRA2"]
+
+ result = parser.read_csv_check_warnings(
+ ParserWarning,
+ "from bad_lines callable",
+ StringIO(data),
+ on_bad_lines=fixer,
+ index_col=index_col,
+ )
+
+ if index_col is None:
+ expected = DataFrame(
+ {
+ "id": [101, 102, 103],
+ "field_1": ["A", "C", "F"],
+ "field_2": ["B", "D", "G"],
+ }
+ )
+ else:
+ expected = DataFrame(
+ {"field_1": ["A", "C", "F"], "field_2": ["B", "D", "G"]},
+ index=Index([101, 102, 103], name="id"),
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_csv_leading_quote_skip(python_parser_only):
+ # GH 62739
+ tbl = """\
+ "
+a b
+1 3
+"""
+ parser = python_parser_only
+ result = parser.read_csv(
+ StringIO(tbl),
+ delimiter=" ",
+ skiprows=1,
+ )
+ expected = DataFrame({"a": [1], "b": [3]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_csv_unclosed_double_quote_in_data_still_errors(python_parser_only):
+ # GH 62739
+ tbl = """\
+a b
+"
+1 3
+"""
+ parser = python_parser_only
+ with pytest.raises(ParserError, match="unexpected end of data"):
+ parser.read_csv(StringIO(tbl), delimiter=" ", skiprows=1)
+
+
+def test_read_csv_skiprows_zero(python_parser_only):
+ # GH 62739
+ tbl = """\
+"
+a b
+1 3
+"""
+ parser = python_parser_only
+ # don't skip anything
+ with pytest.raises(ParserError, match="unexpected end of data"):
+ parser.read_csv(StringIO(tbl), delimiter=" ", skiprows=0, engine="python")
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_quoting.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_quoting.py
new file mode 100644
index 0000000000000000000000000000000000000000..63c459d0f8bdee3b3292d8e0d9419caaea4c81b2
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_quoting.py
@@ -0,0 +1,196 @@
+"""
+Tests that quoting specifications are properly handled
+during parsing for all of the parsers defined in parsers.py
+"""
+
+import csv
+from io import StringIO
+
+import pytest
+
+from pandas.compat import PY314
+from pandas.errors import ParserError
+
+from pandas import DataFrame
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+
+if PY314:
+ # TODO: write a regex that works with all new possitibilities here
+ MSG1 = ""
+ MSG2 = r"[\s\S]*"
+else:
+ MSG1 = "a(n)? 1-character string"
+ MSG2 = "string( or None)?"
+
+
+@pytest.mark.parametrize(
+ "kwargs,msg",
+ [
+ ({"quotechar": "foo"}, f'"quotechar" must be {MSG1}'),
+ (
+ {"quotechar": None, "quoting": csv.QUOTE_MINIMAL},
+ "quotechar must be set if quoting enabled",
+ ),
+ ({"quotechar": 2}, f'"quotechar" must be {MSG2}, not int'),
+ ],
+)
+@skip_pyarrow # ParserError: CSV parse error: Empty CSV file or block
+def test_bad_quote_char(all_parsers, kwargs, msg):
+ data = "1,2,3"
+ parser = all_parsers
+
+ with pytest.raises(TypeError, match=msg):
+ parser.read_csv(StringIO(data), **kwargs)
+
+
+@pytest.mark.parametrize(
+ "quoting,msg",
+ [
+ ("foo", '"quoting" must be an integer|Argument'),
+ (10, 'bad "quoting" value'), # quoting must be in the range [0, 3]
+ ],
+)
+@xfail_pyarrow # ValueError: The 'quoting' option is not supported
+def test_bad_quoting(all_parsers, quoting, msg):
+ data = "1,2,3"
+ parser = all_parsers
+
+ with pytest.raises(TypeError, match=msg):
+ parser.read_csv(StringIO(data), quoting=quoting)
+
+
+def test_quote_char_basic(all_parsers):
+ parser = all_parsers
+ data = 'a,b,c\n1,2,"cat"'
+ expected = DataFrame([[1, 2, "cat"]], columns=["a", "b", "c"])
+
+ result = parser.read_csv(StringIO(data), quotechar='"')
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("quote_char", ["~", "*", "%", "$", "@", "P"])
+def test_quote_char_various(all_parsers, quote_char):
+ parser = all_parsers
+ expected = DataFrame([[1, 2, "cat"]], columns=["a", "b", "c"])
+
+ data = 'a,b,c\n1,2,"cat"'
+ new_data = data.replace('"', quote_char)
+
+ result = parser.read_csv(StringIO(new_data), quotechar=quote_char)
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # ValueError: The 'quoting' option is not supported
+@pytest.mark.parametrize("quoting", [csv.QUOTE_MINIMAL, csv.QUOTE_NONE])
+@pytest.mark.parametrize("quote_char", ["", None])
+def test_null_quote_char(all_parsers, quoting, quote_char):
+ kwargs = {"quotechar": quote_char, "quoting": quoting}
+ data = "a,b,c\n1,2,3"
+ parser = all_parsers
+
+ if quoting != csv.QUOTE_NONE:
+ # Sanity checking.
+ if not PY314:
+ msg = "1-character string"
+ else:
+ msg = "unicode character or None"
+ msg = (
+ f'"quotechar" must be a {msg}'
+ if all_parsers.engine == "python" and quote_char == ""
+ else "quotechar must be set if quoting enabled"
+ )
+
+ with pytest.raises(TypeError, match=msg):
+ parser.read_csv(StringIO(data), **kwargs)
+ elif all_parsers.engine != "python":
+ # Python doesn't support null/blank quote chars in their csv parsers
+ expected = DataFrame([[1, 2, 3]], columns=["a", "b", "c"])
+ result = parser.read_csv(StringIO(data), **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "kwargs,exp_data",
+ [
+ ({}, [[1, 2, "foo"]]), # Test default.
+ # QUOTE_MINIMAL only applies to CSV writing, so no effect on reading.
+ ({"quotechar": '"', "quoting": csv.QUOTE_MINIMAL}, [[1, 2, "foo"]]),
+ # QUOTE_MINIMAL only applies to CSV writing, so no effect on reading.
+ ({"quotechar": '"', "quoting": csv.QUOTE_ALL}, [[1, 2, "foo"]]),
+ # QUOTE_NONE tells the reader to do no special handling
+ # of quote characters and leave them alone.
+ ({"quotechar": '"', "quoting": csv.QUOTE_NONE}, [[1, 2, '"foo"']]),
+ # QUOTE_NONNUMERIC tells the reader to cast
+ # all non-quoted fields to float
+ ({"quotechar": '"', "quoting": csv.QUOTE_NONNUMERIC}, [[1.0, 2.0, "foo"]]),
+ ],
+)
+@xfail_pyarrow # ValueError: The 'quoting' option is not supported
+def test_quoting_various(all_parsers, kwargs, exp_data):
+ data = '1,2,"foo"'
+ parser = all_parsers
+ columns = ["a", "b", "c"]
+
+ result = parser.read_csv(StringIO(data), names=columns, **kwargs)
+ expected = DataFrame(exp_data, columns=columns)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "doublequote,exp_data", [(True, [[3, '4 " 5']]), (False, [[3, '4 " 5"']])]
+)
+def test_double_quote(all_parsers, doublequote, exp_data, request):
+ parser = all_parsers
+ data = 'a,b\n3,"4 "" 5"'
+
+ if parser.engine == "pyarrow" and not doublequote:
+ mark = pytest.mark.xfail(reason="Mismatched result")
+ request.applymarker(mark)
+
+ result = parser.read_csv(StringIO(data), quotechar='"', doublequote=doublequote)
+ expected = DataFrame(exp_data, columns=["a", "b"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("quotechar", ['"', "\u0001"])
+def test_quotechar_unicode(all_parsers, quotechar):
+ # see gh-14477
+ data = "a\n1"
+ parser = all_parsers
+ expected = DataFrame({"a": [1]})
+
+ result = parser.read_csv(StringIO(data), quotechar=quotechar)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("balanced", [True, False])
+def test_unbalanced_quoting(all_parsers, balanced, request):
+ # see gh-22789.
+ parser = all_parsers
+ data = 'a,b,c\n1,2,"3'
+
+ if parser.engine == "pyarrow" and not balanced:
+ mark = pytest.mark.xfail(reason="Mismatched result")
+ request.applymarker(mark)
+
+ if balanced:
+ # Re-balance the quoting and read in without errors.
+ expected = DataFrame([[1, 2, 3]], columns=["a", "b", "c"])
+ result = parser.read_csv(StringIO(data + '"'))
+ tm.assert_frame_equal(result, expected)
+ else:
+ msg = (
+ "EOF inside string starting at row 1"
+ if parser.engine == "c"
+ else "unexpected end of data"
+ )
+
+ with pytest.raises(ParserError, match=msg):
+ parser.read_csv(StringIO(data))
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_read_fwf.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_read_fwf.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae6eef3ba0d486e287eadaab4f91411ee073ea90
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_read_fwf.py
@@ -0,0 +1,1005 @@
+"""
+Tests the 'read_fwf' function in parsers.py. This
+test suite is independent of the others because the
+engine is set to 'python-fwf' internally.
+"""
+
+from io import (
+ BytesIO,
+ StringIO,
+)
+
+import numpy as np
+import pytest
+
+from pandas.errors import EmptyDataError
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ DatetimeIndex,
+)
+import pandas._testing as tm
+
+from pandas.io.common import urlopen
+from pandas.io.parsers import (
+ read_csv,
+ read_fwf,
+)
+
+
+def test_basic():
+ data = """\
+A B C D
+201158 360.242940 149.910199 11950.7
+201159 444.953632 166.985655 11788.4
+201160 364.136849 183.628767 11806.2
+201161 413.836124 184.375703 11916.8
+201162 502.953953 173.237159 12468.3
+"""
+ result = read_fwf(StringIO(data))
+ expected = DataFrame(
+ [
+ [201158, 360.242940, 149.910199, 11950.7],
+ [201159, 444.953632, 166.985655, 11788.4],
+ [201160, 364.136849, 183.628767, 11806.2],
+ [201161, 413.836124, 184.375703, 11916.8],
+ [201162, 502.953953, 173.237159, 12468.3],
+ ],
+ columns=["A", "B", "C", "D"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_colspecs():
+ data = """\
+A B C D E
+201158 360.242940 149.910199 11950.7
+201159 444.953632 166.985655 11788.4
+201160 364.136849 183.628767 11806.2
+201161 413.836124 184.375703 11916.8
+201162 502.953953 173.237159 12468.3
+"""
+ colspecs = [(0, 4), (4, 8), (8, 20), (21, 33), (34, 43)]
+ result = read_fwf(StringIO(data), colspecs=colspecs)
+
+ expected = DataFrame(
+ [
+ [2011, 58, 360.242940, 149.910199, 11950.7],
+ [2011, 59, 444.953632, 166.985655, 11788.4],
+ [2011, 60, 364.136849, 183.628767, 11806.2],
+ [2011, 61, 413.836124, 184.375703, 11916.8],
+ [2011, 62, 502.953953, 173.237159, 12468.3],
+ ],
+ columns=["A", "B", "C", "D", "E"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_widths():
+ data = """\
+A B C D E
+2011 58 360.242940 149.910199 11950.7
+2011 59 444.953632 166.985655 11788.4
+2011 60 364.136849 183.628767 11806.2
+2011 61 413.836124 184.375703 11916.8
+2011 62 502.953953 173.237159 12468.3
+"""
+ result = read_fwf(StringIO(data), widths=[5, 5, 13, 13, 7])
+
+ expected = DataFrame(
+ [
+ [2011, 58, 360.242940, 149.910199, 11950.7],
+ [2011, 59, 444.953632, 166.985655, 11788.4],
+ [2011, 60, 364.136849, 183.628767, 11806.2],
+ [2011, 61, 413.836124, 184.375703, 11916.8],
+ [2011, 62, 502.953953, 173.237159, 12468.3],
+ ],
+ columns=["A", "B", "C", "D", "E"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_non_space_filler():
+ # From Thomas Kluyver:
+ #
+ # Apparently, some non-space filler characters can be seen, this is
+ # supported by specifying the 'delimiter' character:
+ #
+ # http://publib.boulder.ibm.com/infocenter/dmndhelp/v6r1mx/index.jsp?topic=/com.ibm.wbit.612.help.config.doc/topics/rfixwidth.html
+ data = """\
+A~~~~B~~~~C~~~~~~~~~~~~D~~~~~~~~~~~~E
+201158~~~~360.242940~~~149.910199~~~11950.7
+201159~~~~444.953632~~~166.985655~~~11788.4
+201160~~~~364.136849~~~183.628767~~~11806.2
+201161~~~~413.836124~~~184.375703~~~11916.8
+201162~~~~502.953953~~~173.237159~~~12468.3
+"""
+ colspecs = [(0, 4), (4, 8), (8, 20), (21, 33), (34, 43)]
+ result = read_fwf(StringIO(data), colspecs=colspecs, delimiter="~")
+
+ expected = DataFrame(
+ [
+ [2011, 58, 360.242940, 149.910199, 11950.7],
+ [2011, 59, 444.953632, 166.985655, 11788.4],
+ [2011, 60, 364.136849, 183.628767, 11806.2],
+ [2011, 61, 413.836124, 184.375703, 11916.8],
+ [2011, 62, 502.953953, 173.237159, 12468.3],
+ ],
+ columns=["A", "B", "C", "D", "E"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_over_specified():
+ data = """\
+A B C D E
+201158 360.242940 149.910199 11950.7
+201159 444.953632 166.985655 11788.4
+201160 364.136849 183.628767 11806.2
+201161 413.836124 184.375703 11916.8
+201162 502.953953 173.237159 12468.3
+"""
+ colspecs = [(0, 4), (4, 8), (8, 20), (21, 33), (34, 43)]
+
+ with pytest.raises(ValueError, match="must specify only one of"):
+ read_fwf(StringIO(data), colspecs=colspecs, widths=[6, 10, 10, 7])
+
+
+def test_under_specified():
+ data = """\
+A B C D E
+201158 360.242940 149.910199 11950.7
+201159 444.953632 166.985655 11788.4
+201160 364.136849 183.628767 11806.2
+201161 413.836124 184.375703 11916.8
+201162 502.953953 173.237159 12468.3
+"""
+ with pytest.raises(ValueError, match="Must specify either"):
+ read_fwf(StringIO(data), colspecs=None, widths=None)
+
+
+def test_read_csv_compat():
+ csv_data = """\
+A,B,C,D,E
+2011,58,360.242940,149.910199,11950.7
+2011,59,444.953632,166.985655,11788.4
+2011,60,364.136849,183.628767,11806.2
+2011,61,413.836124,184.375703,11916.8
+2011,62,502.953953,173.237159,12468.3
+"""
+ expected = read_csv(StringIO(csv_data), engine="python")
+
+ fwf_data = """\
+A B C D E
+201158 360.242940 149.910199 11950.7
+201159 444.953632 166.985655 11788.4
+201160 364.136849 183.628767 11806.2
+201161 413.836124 184.375703 11916.8
+201162 502.953953 173.237159 12468.3
+"""
+ colspecs = [(0, 4), (4, 8), (8, 20), (21, 33), (34, 43)]
+ result = read_fwf(StringIO(fwf_data), colspecs=colspecs)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_bytes_io_input():
+ data = BytesIO("שלום\nשלום".encode()) # noqa: RUF001
+ result = read_fwf(data, widths=[2, 2], encoding="utf8")
+ expected = DataFrame([["של", "ום"]], columns=["של", "ום"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_fwf_colspecs_is_list_or_tuple():
+ data = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+"""
+
+ msg = "column specifications must be a list or tuple.+"
+
+ with pytest.raises(TypeError, match=msg):
+ read_fwf(StringIO(data), colspecs={"a": 1}, delimiter=",")
+
+
+def test_fwf_colspecs_is_list_or_tuple_of_two_element_tuples():
+ data = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+"""
+
+ msg = "Each column specification must be.+"
+
+ with pytest.raises(TypeError, match=msg):
+ read_fwf(StringIO(data), colspecs=[("a", 1)])
+
+
+@pytest.mark.parametrize(
+ "colspecs,exp_data",
+ [
+ ([(0, 3), (3, None)], [[123, 456], [456, 789]]),
+ ([(None, 3), (3, 6)], [[123, 456], [456, 789]]),
+ ([(0, None), (3, None)], [[123456, 456], [456789, 789]]),
+ ([(None, None), (3, 6)], [[123456, 456], [456789, 789]]),
+ ],
+)
+def test_fwf_colspecs_none(colspecs, exp_data):
+ # see gh-7079
+ data = """\
+123456
+456789
+"""
+ expected = DataFrame(exp_data)
+
+ result = read_fwf(StringIO(data), colspecs=colspecs, header=None)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "infer_nrows,exp_data",
+ [
+ # infer_nrows --> colspec == [(2, 3), (5, 6)]
+ (1, [[1, 2], [3, 8]]),
+ # infer_nrows > number of rows
+ (10, [[1, 2], [123, 98]]),
+ ],
+)
+def test_fwf_colspecs_infer_nrows(infer_nrows, exp_data):
+ # see gh-15138
+ data = """\
+ 1 2
+123 98
+"""
+ expected = DataFrame(exp_data)
+
+ result = read_fwf(StringIO(data), infer_nrows=infer_nrows, header=None)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_fwf_regression():
+ # see gh-3594
+ #
+ # Turns out "T060" is parsable as a datetime slice!
+ tz_list = [1, 10, 20, 30, 60, 80, 100]
+ widths = [16] + [8] * len(tz_list)
+ names = ["SST"] + [f"T{z:03d}" for z in tz_list[1:]]
+
+ data = """ 2009164202000 9.5403 9.4105 8.6571 7.8372 6.0612 5.8843 5.5192
+2009164203000 9.5435 9.2010 8.6167 7.8176 6.0804 5.8728 5.4869
+2009164204000 9.5873 9.1326 8.4694 7.5889 6.0422 5.8526 5.4657
+2009164205000 9.5810 9.0896 8.4009 7.4652 6.0322 5.8189 5.4379
+2009164210000 9.6034 9.0897 8.3822 7.4905 6.0908 5.7904 5.4039
+"""
+ expected = DataFrame(
+ [
+ [9.5403, 9.4105, 8.6571, 7.8372, 6.0612, 5.8843, 5.5192],
+ [9.5435, 9.2010, 8.6167, 7.8176, 6.0804, 5.8728, 5.4869],
+ [9.5873, 9.1326, 8.4694, 7.5889, 6.0422, 5.8526, 5.4657],
+ [9.5810, 9.0896, 8.4009, 7.4652, 6.0322, 5.8189, 5.4379],
+ [9.6034, 9.0897, 8.3822, 7.4905, 6.0908, 5.7904, 5.4039],
+ ],
+ index=DatetimeIndex(
+ [
+ "2009-06-13 20:20:00",
+ "2009-06-13 20:30:00",
+ "2009-06-13 20:40:00",
+ "2009-06-13 20:50:00",
+ "2009-06-13 21:00:00",
+ ],
+ dtype="M8[us]",
+ ),
+ columns=["SST", "T010", "T020", "T030", "T060", "T080", "T100"],
+ )
+ result = read_fwf(
+ StringIO(data),
+ index_col=0,
+ header=None,
+ names=names,
+ widths=widths,
+ parse_dates=True,
+ date_format="%Y%j%H%M%S",
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_fwf_for_uint8():
+ data = """1421302965.213420 PRI=3 PGN=0xef00 DST=0x17 SRC=0x28 04 154 00 00 00 00 00 127
+1421302964.226776 PRI=6 PGN=0xf002 SRC=0x47 243 00 00 255 247 00 00 71""" # noqa: E501
+ df = read_fwf(
+ StringIO(data),
+ colspecs=[(0, 17), (25, 26), (33, 37), (49, 51), (58, 62), (63, 1000)],
+ names=["time", "pri", "pgn", "dst", "src", "data"],
+ converters={
+ "pgn": lambda x: int(x, 16),
+ "src": lambda x: int(x, 16),
+ "dst": lambda x: int(x, 16),
+ "data": lambda x: len(x.split(" ")),
+ },
+ )
+
+ expected = DataFrame(
+ [
+ [1421302965.213420, 3, 61184, 23, 40, 8],
+ [1421302964.226776, 6, 61442, None, 71, 8],
+ ],
+ columns=["time", "pri", "pgn", "dst", "src", "data"],
+ )
+ expected["dst"] = expected["dst"].astype(object)
+ tm.assert_frame_equal(df, expected)
+
+
+@pytest.mark.parametrize("comment", ["#", "~", "!"])
+def test_fwf_comment(comment):
+ data = """\
+ 1 2. 4 #hello world
+ 5 NaN 10.0
+"""
+ data = data.replace("#", comment)
+
+ colspecs = [(0, 3), (4, 9), (9, 25)]
+ expected = DataFrame([[1, 2.0, 4], [5, np.nan, 10.0]])
+
+ result = read_fwf(StringIO(data), colspecs=colspecs, header=None, comment=comment)
+ tm.assert_almost_equal(result, expected)
+
+
+def test_fwf_skip_blank_lines():
+ data = """
+
+A B C D
+
+201158 360.242940 149.910199 11950.7
+201159 444.953632 166.985655 11788.4
+
+
+201162 502.953953 173.237159 12468.3
+
+"""
+ result = read_fwf(StringIO(data), skip_blank_lines=True)
+ expected = DataFrame(
+ [
+ [201158, 360.242940, 149.910199, 11950.7],
+ [201159, 444.953632, 166.985655, 11788.4],
+ [201162, 502.953953, 173.237159, 12468.3],
+ ],
+ columns=["A", "B", "C", "D"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+ data = """\
+A B C D
+201158 360.242940 149.910199 11950.7
+201159 444.953632 166.985655 11788.4
+
+
+201162 502.953953 173.237159 12468.3
+"""
+ result = read_fwf(StringIO(data), skip_blank_lines=False)
+ expected = DataFrame(
+ [
+ [201158, 360.242940, 149.910199, 11950.7],
+ [201159, 444.953632, 166.985655, 11788.4],
+ [np.nan, np.nan, np.nan, np.nan],
+ [np.nan, np.nan, np.nan, np.nan],
+ [201162, 502.953953, 173.237159, 12468.3],
+ ],
+ columns=["A", "B", "C", "D"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("thousands", [",", "#", "~"])
+def test_fwf_thousands(thousands):
+ data = """\
+ 1 2,334.0 5
+10 13 10.
+"""
+ data = data.replace(",", thousands)
+
+ colspecs = [(0, 3), (3, 11), (12, 16)]
+ expected = DataFrame([[1, 2334.0, 5], [10, 13, 10.0]])
+
+ result = read_fwf(
+ StringIO(data), header=None, colspecs=colspecs, thousands=thousands
+ )
+ tm.assert_almost_equal(result, expected)
+
+
+@pytest.mark.parametrize("header", [True, False])
+def test_bool_header_arg(header):
+ # see gh-6114
+ data = """\
+MyColumn
+ a
+ b
+ a
+ b"""
+
+ msg = "Passing a bool to header is invalid"
+ with pytest.raises(TypeError, match=msg):
+ read_fwf(StringIO(data), header=header)
+
+
+def test_full_file():
+ # File with all values.
+ test = """index A B C
+2000-01-03T00:00:00 0.980268513777 3 foo
+2000-01-04T00:00:00 1.04791624281 -4 bar
+2000-01-05T00:00:00 0.498580885705 73 baz
+2000-01-06T00:00:00 1.12020151869 1 foo
+2000-01-07T00:00:00 0.487094399463 0 bar
+2000-01-10T00:00:00 0.836648671666 2 baz
+2000-01-11T00:00:00 0.157160753327 34 foo"""
+ colspecs = ((0, 19), (21, 35), (38, 40), (42, 45))
+ expected = read_fwf(StringIO(test), colspecs=colspecs)
+
+ result = read_fwf(StringIO(test))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_full_file_with_missing():
+ # File with missing values.
+ test = """index A B C
+2000-01-03T00:00:00 0.980268513777 3 foo
+2000-01-04T00:00:00 1.04791624281 -4 bar
+ 0.498580885705 73 baz
+2000-01-06T00:00:00 1.12020151869 1 foo
+2000-01-07T00:00:00 0 bar
+2000-01-10T00:00:00 0.836648671666 2 baz
+ 34"""
+ colspecs = ((0, 19), (21, 35), (38, 40), (42, 45))
+ expected = read_fwf(StringIO(test), colspecs=colspecs)
+
+ result = read_fwf(StringIO(test))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_full_file_with_spaces():
+ # File with spaces in columns.
+ test = """
+Account Name Balance CreditLimit AccountCreated
+101 Keanu Reeves 9315.45 10000.00 1/17/1998
+312 Gerard Butler 90.00 1000.00 8/6/2003
+868 Jennifer Love Hewitt 0 17000.00 5/25/1985
+761 Jada Pinkett-Smith 49654.87 100000.00 12/5/2006
+317 Bill Murray 789.65 5000.00 2/5/2007
+""".strip("\r\n")
+ colspecs = ((0, 7), (8, 28), (30, 38), (42, 53), (56, 70))
+ expected = read_fwf(StringIO(test), colspecs=colspecs)
+
+ result = read_fwf(StringIO(test))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_full_file_with_spaces_and_missing():
+ # File with spaces and missing values in columns.
+ test = """
+Account Name Balance CreditLimit AccountCreated
+101 10000.00 1/17/1998
+312 Gerard Butler 90.00 1000.00 8/6/2003
+868 5/25/1985
+761 Jada Pinkett-Smith 49654.87 100000.00 12/5/2006
+317 Bill Murray 789.65
+""".strip("\r\n")
+ colspecs = ((0, 7), (8, 28), (30, 38), (42, 53), (56, 70))
+ expected = read_fwf(StringIO(test), colspecs=colspecs)
+
+ result = read_fwf(StringIO(test))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_messed_up_data():
+ # Completely messed up file.
+ test = """
+ Account Name Balance Credit Limit Account Created
+ 101 10000.00 1/17/1998
+ 312 Gerard Butler 90.00 1000.00
+
+ 761 Jada Pinkett-Smith 49654.87 100000.00 12/5/2006
+ 317 Bill Murray 789.65
+""".strip("\r\n")
+ colspecs = ((2, 10), (15, 33), (37, 45), (49, 61), (64, 79))
+ expected = read_fwf(StringIO(test), colspecs=colspecs)
+
+ result = read_fwf(StringIO(test))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_multiple_delimiters():
+ test = r"""
+col1~~~~~col2 col3++++++++++++++++++col4
+~~22.....11.0+++foo~~~~~~~~~~Keanu Reeves
+ 33+++122.33\\\bar.........Gerard Butler
+++44~~~~12.01 baz~~Jennifer Love Hewitt
+~~55 11+++foo++++Jada Pinkett-Smith
+..66++++++.03~~~bar Bill Murray
+""".strip("\r\n")
+ delimiter = " +~.\\"
+ colspecs = ((0, 4), (7, 13), (15, 19), (21, 41))
+ expected = read_fwf(StringIO(test), colspecs=colspecs, delimiter=delimiter)
+
+ result = read_fwf(StringIO(test), delimiter=delimiter)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_variable_width_unicode():
+ data = """
+שלום שלום
+ום שלל
+של ום
+""".strip("\r\n")
+ encoding = "utf8"
+ kwargs = {"header": None, "encoding": encoding}
+
+ expected = read_fwf(
+ BytesIO(data.encode(encoding)), colspecs=[(0, 4), (5, 9)], **kwargs
+ )
+ result = read_fwf(BytesIO(data.encode(encoding)), **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("dtype", [{}, {"a": "float64", "b": str, "c": "int32"}])
+def test_dtype(dtype):
+ data = """ a b c
+1 2 3.2
+3 4 5.2
+"""
+ colspecs = [(0, 5), (5, 10), (10, None)]
+ result = read_fwf(StringIO(data), colspecs=colspecs, dtype=dtype)
+
+ expected = DataFrame(
+ {"a": [1, 3], "b": [2, 4], "c": [3.2, 5.2]}, columns=["a", "b", "c"]
+ )
+
+ for col, dt in dtype.items():
+ expected[col] = expected[col].astype(dt)
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_skiprows_inference():
+ # see gh-11256
+ data = """
+Text contained in the file header
+
+DataCol1 DataCol2
+ 0.0 1.0
+ 101.6 956.1
+""".strip()
+ skiprows = 2
+
+ expected = read_csv(StringIO(data), skiprows=skiprows, sep=r"\s+")
+ result = read_fwf(StringIO(data), skiprows=skiprows)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_skiprows_by_index_inference():
+ data = """
+To be skipped
+Not To Be Skipped
+Once more to be skipped
+123 34 8 123
+456 78 9 456
+""".strip()
+ skiprows = [0, 2]
+
+ expected = read_csv(StringIO(data), skiprows=skiprows, sep=r"\s+")
+ result = read_fwf(StringIO(data), skiprows=skiprows)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_skiprows_inference_empty():
+ data = """
+AA BBB C
+12 345 6
+78 901 2
+""".strip()
+
+ msg = "No rows from which to infer column width"
+ with pytest.raises(EmptyDataError, match=msg):
+ read_fwf(StringIO(data), skiprows=3)
+
+
+def test_whitespace_preservation():
+ # see gh-16772
+ header = None
+ csv_data = """
+ a ,bbb
+ cc,dd """
+
+ fwf_data = """
+ a bbb
+ ccdd """
+ result = read_fwf(
+ StringIO(fwf_data), widths=[3, 3], header=header, skiprows=[0], delimiter="\n\t"
+ )
+ expected = read_csv(StringIO(csv_data), header=header)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_default_delimiter():
+ header = None
+ csv_data = """
+a,bbb
+cc,dd"""
+
+ fwf_data = """
+a \tbbb
+cc\tdd """
+ result = read_fwf(StringIO(fwf_data), widths=[3, 3], header=header, skiprows=[0])
+ expected = read_csv(StringIO(csv_data), header=header)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("infer", [True, False])
+def test_fwf_compression(compression_only, infer, compression_to_extension, temp_file):
+ data = """1111111111
+ 2222222222
+ 3333333333""".strip()
+
+ compression = compression_only
+ extension = compression_to_extension[compression]
+
+ kwargs = {"widths": [5, 5], "names": ["one", "two"]}
+ expected = read_fwf(StringIO(data), **kwargs)
+
+ data = bytes(data, encoding="utf-8")
+
+ path = temp_file.parent / f"tmp.{extension}"
+ tm.write_to_compressed(compression, path, data)
+
+ if infer is not None:
+ kwargs["compression"] = "infer" if infer else compression
+
+ result = read_fwf(path, **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_binary_mode(temp_file):
+ """
+ read_fwf supports opening files in binary mode.
+
+ GH 18035.
+ """
+ data = """aaa aaa aaa
+bba bab b a"""
+ df_reference = DataFrame(
+ [["bba", "bab", "b a"]], columns=["aaa", "aaa.1", "aaa.2"], index=[0]
+ )
+ path = temp_file
+ path.write_text(data, encoding="utf-8")
+ with open(path, "rb") as file:
+ df = read_fwf(file)
+ file.seek(0)
+ tm.assert_frame_equal(df, df_reference)
+
+
+@pytest.mark.parametrize("memory_map", [True, False])
+def test_encoding_mmap(memory_map, temp_file):
+ """
+ encoding should be working, even when using a memory-mapped file.
+
+ GH 23254.
+ """
+ encoding = "iso8859_1"
+ path = temp_file
+ path.write_bytes(" 1 A Ä 2\n".encode(encoding))
+ df = read_fwf(
+ path,
+ header=None,
+ widths=[2, 2, 2, 2],
+ encoding=encoding,
+ memory_map=memory_map,
+ )
+ df_reference = DataFrame([[1, "A", "Ä", 2]])
+ tm.assert_frame_equal(df, df_reference)
+
+
+@pytest.mark.parametrize(
+ "colspecs, names, widths, index_col",
+ [
+ (
+ [(0, 6), (6, 12), (12, 18), (18, None)],
+ list("abcde"),
+ None,
+ None,
+ ),
+ (
+ None,
+ list("abcde"),
+ [6] * 4,
+ None,
+ ),
+ (
+ [(0, 6), (6, 12), (12, 18), (18, None)],
+ list("abcde"),
+ None,
+ True,
+ ),
+ (
+ None,
+ list("abcde"),
+ [6] * 4,
+ False,
+ ),
+ (
+ None,
+ list("abcde"),
+ [6] * 4,
+ True,
+ ),
+ (
+ [(0, 6), (6, 12), (12, 18), (18, None)],
+ list("abcde"),
+ None,
+ False,
+ ),
+ ],
+)
+def test_len_colspecs_len_names(colspecs, names, widths, index_col):
+ # GH#40830
+ data = """col1 col2 col3 col4
+ bab ba 2"""
+ msg = "Length of colspecs must match length of names"
+ with pytest.raises(ValueError, match=msg):
+ read_fwf(
+ StringIO(data),
+ colspecs=colspecs,
+ names=names,
+ widths=widths,
+ index_col=index_col,
+ )
+
+
+@pytest.mark.parametrize(
+ "colspecs, names, widths, index_col, expected",
+ [
+ (
+ [(0, 6), (6, 12), (12, 18), (18, None)],
+ list("abc"),
+ None,
+ 0,
+ DataFrame(
+ index=["col1", "ba"],
+ columns=["a", "b", "c"],
+ data=[["col2", "col3", "col4"], ["b ba", "2", np.nan]],
+ ),
+ ),
+ (
+ [(0, 6), (6, 12), (12, 18), (18, None)],
+ list("ab"),
+ None,
+ [0, 1],
+ DataFrame(
+ index=[["col1", "ba"], ["col2", "b ba"]],
+ columns=["a", "b"],
+ data=[["col3", "col4"], ["2", np.nan]],
+ ),
+ ),
+ (
+ [(0, 6), (6, 12), (12, 18), (18, None)],
+ list("a"),
+ None,
+ [0, 1, 2],
+ DataFrame(
+ index=[["col1", "ba"], ["col2", "b ba"], ["col3", "2"]],
+ columns=["a"],
+ data=[["col4"], [np.nan]],
+ ),
+ ),
+ (
+ None,
+ list("abc"),
+ [6] * 4,
+ 0,
+ DataFrame(
+ index=["col1", "ba"],
+ columns=["a", "b", "c"],
+ data=[["col2", "col3", "col4"], ["b ba", "2", np.nan]],
+ ),
+ ),
+ (
+ None,
+ list("ab"),
+ [6] * 4,
+ [0, 1],
+ DataFrame(
+ index=[["col1", "ba"], ["col2", "b ba"]],
+ columns=["a", "b"],
+ data=[["col3", "col4"], ["2", np.nan]],
+ ),
+ ),
+ (
+ None,
+ list("a"),
+ [6] * 4,
+ [0, 1, 2],
+ DataFrame(
+ index=[["col1", "ba"], ["col2", "b ba"], ["col3", "2"]],
+ columns=["a"],
+ data=[["col4"], [np.nan]],
+ ),
+ ),
+ ],
+)
+def test_len_colspecs_len_names_with_index_col(
+ colspecs, names, widths, index_col, expected
+):
+ # GH#40830
+ data = """col1 col2 col3 col4
+ bab ba 2"""
+ result = read_fwf(
+ StringIO(data),
+ colspecs=colspecs,
+ names=names,
+ widths=widths,
+ index_col=index_col,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_colspecs_with_comment():
+ # GH 14135
+ result = read_fwf(
+ StringIO("#\nA1K\n"), colspecs=[(1, 2), (2, 3)], comment="#", header=None
+ )
+ expected = DataFrame([[1, "K"]], columns=[0, 1])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_skip_rows_and_n_rows():
+ # GH#44021
+ data = """a\tb
+1\t a
+2\t b
+3\t c
+4\t d
+5\t e
+6\t f
+ """
+ result = read_fwf(StringIO(data), nrows=4, skiprows=[2, 4])
+ expected = DataFrame({"a": [1, 3, 5, 6], "b": ["a", "c", "e", "f"]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_skiprows_with_iterator():
+ # GH#10261, GH#56323
+ data = """0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+ """
+ df_iter = read_fwf(
+ StringIO(data),
+ colspecs=[(0, 2)],
+ names=["a"],
+ iterator=True,
+ chunksize=2,
+ skiprows=[0, 1, 2, 6, 9],
+ )
+ expected_frames = [
+ DataFrame({"a": [3, 4]}),
+ DataFrame({"a": [5, 7]}, index=[2, 3]),
+ DataFrame({"a": [8]}, index=[4]),
+ ]
+ for i, result in enumerate(df_iter):
+ tm.assert_frame_equal(result, expected_frames[i])
+
+
+def test_names_and_infer_colspecs():
+ # GH#45337
+ data = """X Y Z
+ 959.0 345 22.2
+ """
+ result = read_fwf(StringIO(data), skiprows=1, usecols=[0, 2], names=["a", "b"])
+ expected = DataFrame({"a": [959.0], "b": 22.2})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_widths_and_usecols():
+ # GH#46580
+ data = """0 1 n -0.4100.1
+0 2 p 0.2 90.1
+0 3 n -0.3140.4"""
+ result = read_fwf(
+ StringIO(data),
+ header=None,
+ usecols=(0, 1, 3),
+ widths=(3, 5, 1, 5, 5),
+ index_col=False,
+ names=("c0", "c1", "c3"),
+ )
+ expected = DataFrame(
+ {
+ "c0": 0,
+ "c1": [1, 2, 3],
+ "c3": [-0.4, 0.2, -0.3],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_dtype_backend(string_storage, dtype_backend):
+ # GH#50289
+ data = """a b c d e f g h i
+1 2.5 True a
+3 4.5 False b True 6 7.5 a"""
+ with pd.option_context("mode.string_storage", string_storage):
+ result = read_fwf(StringIO(data), dtype_backend=dtype_backend)
+
+ if dtype_backend == "pyarrow":
+ pa = pytest.importorskip("pyarrow")
+ string_dtype = pd.ArrowDtype(pa.string())
+ else:
+ string_dtype = pd.StringDtype(string_storage)
+
+ expected = DataFrame(
+ {
+ "a": pd.Series([1, 3], dtype="Int64"),
+ "b": pd.Series([2.5, 4.5], dtype="Float64"),
+ "c": pd.Series([True, False], dtype="boolean"),
+ "d": pd.Series(["a", "b"], dtype=string_dtype),
+ "e": pd.Series([pd.NA, True], dtype="boolean"),
+ "f": pd.Series([pd.NA, 6], dtype="Int64"),
+ "g": pd.Series([pd.NA, 7.5], dtype="Float64"),
+ "h": pd.Series([None, "a"], dtype=string_dtype),
+ "i": pd.Series([pd.NA, pd.NA], dtype="Int64"),
+ }
+ )
+ if dtype_backend == "pyarrow":
+ pa = pytest.importorskip("pyarrow")
+ from pandas.arrays import ArrowExtensionArray
+
+ expected = DataFrame(
+ {
+ col: ArrowExtensionArray(pa.array(expected[col], from_pandas=True))
+ for col in expected.columns
+ }
+ )
+ expected["i"] = ArrowExtensionArray(pa.array([None, None]))
+
+ # the storage of the str columns' Index is also affected by the
+ # string_storage setting -> ignore that for checking the result
+ tm.assert_frame_equal(result, expected, check_column_type=False)
+
+
+def test_invalid_dtype_backend():
+ msg = (
+ "dtype_backend numpy is invalid, only 'numpy_nullable' and "
+ "'pyarrow' are allowed."
+ )
+ with pytest.raises(ValueError, match=msg):
+ read_fwf("test", dtype_backend="numpy")
+
+
+@pytest.mark.network
+@pytest.mark.single_cpu
+def test_url_urlopen(httpserver):
+ data = """\
+A B C D
+201158 360.242940 149.910199 11950.7
+201159 444.953632 166.985655 11788.4
+201160 364.136849 183.628767 11806.2
+201161 413.836124 184.375703 11916.8
+201162 502.953953 173.237159 12468.3
+"""
+ httpserver.serve_content(content=data)
+ expected = pd.Index(list("ABCD"))
+ with urlopen(httpserver.url) as f:
+ result = read_fwf(f).columns
+
+ tm.assert_index_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_skiprows.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_skiprows.py
new file mode 100644
index 0000000000000000000000000000000000000000..619b53abec8c711297ddf4a2469a8fce77507ca3
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_skiprows.py
@@ -0,0 +1,336 @@
+"""
+Tests that skipped rows are properly handled during
+parsing for all of the parsers defined in parsers.py
+"""
+
+from datetime import datetime
+from io import StringIO
+
+import numpy as np
+import pytest
+
+from pandas.errors import EmptyDataError
+
+from pandas import (
+ DataFrame,
+ Index,
+)
+import pandas._testing as tm
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+@xfail_pyarrow # ValueError: skiprows argument must be an integer
+@pytest.mark.parametrize("skiprows", [list(range(6)), 6])
+def test_skip_rows_bug(all_parsers, skiprows):
+ # see gh-505
+ parser = all_parsers
+ text = """#foo,a,b,c
+#foo,a,b,c
+#foo,a,b,c
+#foo,a,b,c
+#foo,a,b,c
+#foo,a,b,c
+1/1/2000,1.,2.,3.
+1/2/2000,4,5,6
+1/3/2000,7,8,9
+"""
+ result = parser.read_csv(
+ StringIO(text), skiprows=skiprows, header=None, index_col=0, parse_dates=True
+ )
+ index = Index(
+ [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)],
+ dtype="M8[us]",
+ name=0,
+ )
+
+ expected = DataFrame(
+ np.arange(1.0, 10.0).reshape((3, 3)), columns=[1, 2, 3], index=index
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # ValueError: skiprows argument must be an integer
+def test_deep_skip_rows(all_parsers):
+ # see gh-4382
+ parser = all_parsers
+ data = "a,b,c\n" + "\n".join(
+ [",".join([str(i), str(i + 1), str(i + 2)]) for i in range(10)]
+ )
+ condensed_data = "a,b,c\n" + "\n".join(
+ [",".join([str(i), str(i + 1), str(i + 2)]) for i in [0, 1, 2, 3, 4, 6, 8, 9]]
+ )
+
+ result = parser.read_csv(StringIO(data), skiprows=[6, 8])
+ condensed_result = parser.read_csv(StringIO(condensed_data))
+ tm.assert_frame_equal(result, condensed_result)
+
+
+@xfail_pyarrow # AssertionError: DataFrame are different
+def test_skip_rows_blank(all_parsers):
+ # see gh-9832
+ parser = all_parsers
+ text = """#foo,a,b,c
+#foo,a,b,c
+
+#foo,a,b,c
+#foo,a,b,c
+
+1/1/2000,1.,2.,3.
+1/2/2000,4,5,6
+1/3/2000,7,8,9
+"""
+ data = parser.read_csv(
+ StringIO(text), skiprows=6, header=None, index_col=0, parse_dates=True
+ )
+ index = Index(
+ [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)],
+ dtype="M8[us]",
+ name=0,
+ )
+
+ expected = DataFrame(
+ np.arange(1.0, 10.0).reshape((3, 3)), columns=[1, 2, 3], index=index
+ )
+ tm.assert_frame_equal(data, expected)
+
+
+@pytest.mark.parametrize(
+ "data,kwargs,expected",
+ [
+ (
+ """id,text,num_lines
+1,"line 11
+line 12",2
+2,"line 21
+line 22",2
+3,"line 31",1""",
+ {"skiprows": [1]},
+ DataFrame(
+ [[2, "line 21\nline 22", 2], [3, "line 31", 1]],
+ columns=["id", "text", "num_lines"],
+ ),
+ ),
+ (
+ "a,b,c\n~a\n b~,~e\n d~,~f\n f~\n1,2,~12\n 13\n 14~",
+ {"quotechar": "~", "skiprows": [2]},
+ DataFrame([["a\n b", "e\n d", "f\n f"]], columns=["a", "b", "c"]),
+ ),
+ (
+ (
+ "Text,url\n~example\n "
+ "sentence\n one~,url1\n~"
+ "example\n sentence\n two~,url2\n~"
+ "example\n sentence\n three~,url3"
+ ),
+ {"quotechar": "~", "skiprows": [1, 3]},
+ DataFrame([["example\n sentence\n two", "url2"]], columns=["Text", "url"]),
+ ),
+ ],
+)
+@xfail_pyarrow # ValueError: skiprows argument must be an integer
+def test_skip_row_with_newline(all_parsers, data, kwargs, expected):
+ # see gh-12775 and gh-10911
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # ValueError: skiprows argument must be an integer
+def test_skip_row_with_quote(all_parsers):
+ # see gh-12775 and gh-10911
+ parser = all_parsers
+ data = """id,text,num_lines
+1,"line '11' line 12",2
+2,"line '21' line 22",2
+3,"line '31' line 32",1"""
+
+ exp_data = [[2, "line '21' line 22", 2], [3, "line '31' line 32", 1]]
+ expected = DataFrame(exp_data, columns=["id", "text", "num_lines"])
+
+ result = parser.read_csv(StringIO(data), skiprows=[1])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "data,exp_data",
+ [
+ (
+ """id,text,num_lines
+1,"line \n'11' line 12",2
+2,"line \n'21' line 22",2
+3,"line \n'31' line 32",1""",
+ [[2, "line \n'21' line 22", 2], [3, "line \n'31' line 32", 1]],
+ ),
+ (
+ """id,text,num_lines
+1,"line '11\n' line 12",2
+2,"line '21\n' line 22",2
+3,"line '31\n' line 32",1""",
+ [[2, "line '21\n' line 22", 2], [3, "line '31\n' line 32", 1]],
+ ),
+ (
+ """id,text,num_lines
+1,"line '11\n' \r\tline 12",2
+2,"line '21\n' \r\tline 22",2
+3,"line '31\n' \r\tline 32",1""",
+ [[2, "line '21\n' \r\tline 22", 2], [3, "line '31\n' \r\tline 32", 1]],
+ ),
+ ],
+)
+@xfail_pyarrow # ValueError: skiprows argument must be an integer
+def test_skip_row_with_newline_and_quote(all_parsers, data, exp_data):
+ # see gh-12775 and gh-10911
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), skiprows=[1])
+
+ expected = DataFrame(exp_data, columns=["id", "text", "num_lines"])
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # ValueError: the 'pyarrow' engine does not support regex separators
+@pytest.mark.parametrize(
+ "lineterminator",
+ ["\n", "\r\n", "\r"], # "LF" # "CRLF" # "CR"
+)
+def test_skiprows_lineterminator(all_parsers, lineterminator, request):
+ # see gh-9079
+ parser = all_parsers
+ data = "\n".join(
+ [
+ "SMOSMANIA ThetaProbe-ML2X ",
+ "2007/01/01 01:00 0.2140 U M ",
+ "2007/01/01 02:00 0.2141 M O ",
+ "2007/01/01 04:00 0.2142 D M ",
+ ]
+ )
+ expected = DataFrame(
+ [
+ ["2007/01/01", "01:00", 0.2140, "U", "M"],
+ ["2007/01/01", "02:00", 0.2141, "M", "O"],
+ ["2007/01/01", "04:00", 0.2142, "D", "M"],
+ ],
+ columns=["date", "time", "var", "flag", "oflag"],
+ )
+
+ if parser.engine == "python" and lineterminator == "\r":
+ mark = pytest.mark.xfail(reason="'CR' not respect with the Python parser yet")
+ request.applymarker(mark)
+
+ data = data.replace("\n", lineterminator)
+
+ result = parser.read_csv(
+ StringIO(data),
+ skiprows=1,
+ sep=r"\s+",
+ names=["date", "time", "var", "flag", "oflag"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # AssertionError: DataFrame are different
+def test_skiprows_infield_quote(all_parsers):
+ # see gh-14459
+ parser = all_parsers
+ data = 'a"\nb"\na\n1'
+ expected = DataFrame({"a": [1]})
+
+ result = parser.read_csv(StringIO(data), skiprows=2)
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # ValueError: skiprows argument must be an integer
+@pytest.mark.parametrize(
+ "kwargs,expected",
+ [
+ ({}, "1"),
+ ({"header": 0, "names": ["foo"]}, "foo"),
+ ],
+)
+def test_skip_rows_callable(all_parsers, kwargs, expected):
+ parser = all_parsers
+ data = "a\n1\n2\n3\n4\n5"
+
+ result = parser.read_csv(StringIO(data), skiprows=lambda x: x % 2 == 0, **kwargs)
+ expected = DataFrame({expected: [3, 5]})
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # ValueError: skiprows argument must be an integer
+def test_skip_rows_callable_not_in(all_parsers):
+ parser = all_parsers
+ data = "0,a\n1,b\n2,c\n3,d\n4,e"
+ expected = DataFrame([[1, "b"], [3, "d"]])
+
+ result = parser.read_csv(
+ StringIO(data), header=None, skiprows=lambda x: x not in [1, 3]
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # ValueError: skiprows argument must be an integer
+def test_skip_rows_skip_all(all_parsers):
+ parser = all_parsers
+ data = "a\n1\n2\n3\n4\n5"
+ msg = "No columns to parse from file"
+
+ with pytest.raises(EmptyDataError, match=msg):
+ parser.read_csv(StringIO(data), skiprows=lambda x: True)
+
+
+@xfail_pyarrow # ValueError: skiprows argument must be an integer
+def test_skip_rows_bad_callable(all_parsers):
+ msg = "by zero"
+ parser = all_parsers
+ data = "a\n1\n2\n3\n4\n5"
+
+ with pytest.raises(ZeroDivisionError, match=msg):
+ parser.read_csv(StringIO(data), skiprows=lambda x: 1 / 0)
+
+
+@xfail_pyarrow # ValueError: skiprows argument must be an integer
+def test_skip_rows_and_n_rows(all_parsers):
+ # GH#44021
+ data = """a,b
+1,a
+2,b
+3,c
+4,d
+5,e
+6,f
+7,g
+8,h
+"""
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), nrows=5, skiprows=[2, 4, 6])
+ expected = DataFrame({"a": [1, 3, 5, 7, 8], "b": ["a", "c", "e", "g", "h"]})
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow
+def test_skip_rows_with_chunks(all_parsers):
+ # GH 55677
+ data = """col_a
+10
+20
+30
+40
+50
+60
+70
+80
+90
+100
+"""
+ parser = all_parsers
+ reader = parser.read_csv(
+ StringIO(data), engine=parser, skiprows=lambda x: x in [1, 4, 5], chunksize=4
+ )
+ df1 = next(reader)
+ df2 = next(reader)
+
+ tm.assert_frame_equal(df1, DataFrame({"col_a": [20, 30, 60, 70]}))
+ tm.assert_frame_equal(df2, DataFrame({"col_a": [80, 90, 100]}, index=[4, 5, 6]))
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_textreader.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_textreader.py
new file mode 100644
index 0000000000000000000000000000000000000000..ea44564e3f3e14e691ab682a2be6975b18c75c10
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_textreader.py
@@ -0,0 +1,386 @@
+"""
+Tests the TextReader class in parsers.pyx, which
+is integral to the C engine in parsers.py
+"""
+
+from io import (
+ BytesIO,
+ StringIO,
+)
+
+import numpy as np
+import pytest
+
+import pandas._libs.parsers as parser
+from pandas._libs.parsers import TextReader
+from pandas.errors import ParserWarning
+
+from pandas import DataFrame
+import pandas._testing as tm
+
+from pandas.io.parsers import (
+ TextFileReader,
+ read_csv,
+)
+from pandas.io.parsers.c_parser_wrapper import ensure_dtype_objs
+
+# The only non-test way that TextReader gets called has na_valuess and na_fvalues
+# either both-sets or both dicts, and the code assumes this is the case.
+# But the default argument in its __init__ is None, so we have to pass these
+# explicitly in tests.
+_na_value_kwargs: dict[str, set] = {"na_values": set(), "na_fvalues": set()}
+
+
+class TestTextReader:
+ @pytest.fixture
+ def csv_path(self, datapath):
+ return datapath("io", "data", "csv", "test1.csv")
+
+ def test_file_handle(self, csv_path):
+ with open(csv_path, "rb") as f:
+ reader = TextReader(f, **_na_value_kwargs)
+ reader.read()
+
+ def test_file_handle_mmap(self, csv_path):
+ # this was never using memory_map=True
+ with open(csv_path, "rb") as f:
+ reader = TextReader(f, header=None, **_na_value_kwargs)
+ reader.read()
+
+ def test_StringIO(self, csv_path):
+ with open(csv_path, "rb") as f:
+ text = f.read()
+ src = BytesIO(text)
+ reader = TextReader(src, header=None, **_na_value_kwargs)
+ reader.read()
+
+ def test_encoding_mismatch_warning(self, csv_path):
+ # GH-57954
+ with open(csv_path, encoding="UTF-8") as f:
+ msg = "latin1 is different from the encoding"
+ with pytest.raises(ValueError, match=msg):
+ read_csv(f, encoding="latin1")
+
+ def test_string_factorize(self):
+ # should this be optional?
+ data = "a\nb\na\nb\na"
+ reader = TextReader(StringIO(data), header=None, **_na_value_kwargs)
+ result = reader.read()
+ assert len(set(map(id, result[0]))) == 2
+
+ def test_skipinitialspace(self):
+ data = "a, b\na, b\na, b\na, b"
+
+ reader = TextReader(
+ StringIO(data), skipinitialspace=True, header=None, **_na_value_kwargs
+ )
+ result = reader.read()
+
+ tm.assert_numpy_array_equal(
+ result[0], np.array(["a", "a", "a", "a"], dtype=np.object_)
+ )
+ tm.assert_numpy_array_equal(
+ result[1], np.array(["b", "b", "b", "b"], dtype=np.object_)
+ )
+
+ def test_parse_booleans(self):
+ data = "True\nFalse\nTrue\nTrue"
+
+ reader = TextReader(StringIO(data), header=None, **_na_value_kwargs)
+ result = reader.read()
+
+ assert result[0].dtype == np.bool_
+
+ def test_delimit_whitespace(self):
+ data = 'a b\na\t\t "b"\n"a"\t \t b'
+
+ reader = TextReader(
+ StringIO(data), delim_whitespace=True, header=None, **_na_value_kwargs
+ )
+ result = reader.read()
+
+ tm.assert_numpy_array_equal(
+ result[0], np.array(["a", "a", "a"], dtype=np.object_)
+ )
+ tm.assert_numpy_array_equal(
+ result[1], np.array(["b", "b", "b"], dtype=np.object_)
+ )
+
+ def test_embedded_newline(self):
+ data = 'a\n"hello\nthere"\nthis'
+
+ reader = TextReader(StringIO(data), header=None, **_na_value_kwargs)
+ result = reader.read()
+
+ expected = np.array(["a", "hello\nthere", "this"], dtype=np.object_)
+ tm.assert_numpy_array_equal(result[0], expected)
+
+ def test_euro_decimal(self):
+ data = "12345,67\n345,678"
+
+ reader = TextReader(
+ StringIO(data), delimiter=":", decimal=",", header=None, **_na_value_kwargs
+ )
+ result = reader.read()
+
+ expected = np.array([12345.67, 345.678])
+ tm.assert_almost_equal(result[0], expected)
+
+ def test_integer_thousands(self):
+ data = "123,456\n12,500"
+
+ reader = TextReader(
+ StringIO(data),
+ delimiter=":",
+ thousands=",",
+ header=None,
+ **_na_value_kwargs,
+ )
+ result = reader.read()
+
+ expected = np.array([123456, 12500], dtype=np.int64)
+ tm.assert_almost_equal(result[0], expected)
+
+ def test_integer_thousands_alt(self):
+ data = "123.456\n12.500"
+
+ reader = TextFileReader(
+ StringIO(data), delimiter=":", thousands=".", header=None
+ )
+ result = reader.read()
+
+ expected = DataFrame([123456, 12500])
+ tm.assert_frame_equal(result, expected)
+
+ def test_skip_bad_lines(self):
+ # too many lines, see #2430 for why
+ data = "a:b:c\nd:e:f\ng:h:i\nj:k:l:m\nl:m:n\no:p:q:r"
+
+ reader = TextReader(
+ StringIO(data), delimiter=":", header=None, **_na_value_kwargs
+ )
+ msg = r"Error tokenizing data\. C error: Expected 3 fields in line 4, saw 4"
+ with pytest.raises(parser.ParserError, match=msg):
+ reader.read()
+
+ reader = TextReader(
+ StringIO(data),
+ delimiter=":",
+ header=None,
+ on_bad_lines=2, # Skip
+ **_na_value_kwargs,
+ )
+ result = reader.read()
+ expected = {
+ 0: np.array(["a", "d", "g", "l"], dtype=object),
+ 1: np.array(["b", "e", "h", "m"], dtype=object),
+ 2: np.array(["c", "f", "i", "n"], dtype=object),
+ }
+ assert_array_dicts_equal(result, expected)
+
+ with tm.assert_produces_warning(ParserWarning, match="Skipping line"):
+ reader = TextReader(
+ StringIO(data),
+ delimiter=":",
+ header=None,
+ on_bad_lines=1, # Warn
+ **_na_value_kwargs,
+ )
+ reader.read()
+
+ def test_header_not_enough_lines(self):
+ data = "skip this\nskip this\na,b,c\n1,2,3\n4,5,6"
+
+ reader = TextReader(StringIO(data), delimiter=",", header=2, **_na_value_kwargs)
+ header = reader.header
+ expected = [["a", "b", "c"]]
+ assert header == expected
+
+ recs = reader.read()
+ expected = {
+ 0: np.array([1, 4], dtype=np.int64),
+ 1: np.array([2, 5], dtype=np.int64),
+ 2: np.array([3, 6], dtype=np.int64),
+ }
+ assert_array_dicts_equal(recs, expected)
+
+ def test_escapechar(self):
+ data = '\\"hello world"\n\\"hello world"\n\\"hello world"'
+
+ reader = TextReader(
+ StringIO(data),
+ delimiter=",",
+ header=None,
+ escapechar="\\",
+ **_na_value_kwargs,
+ )
+ result = reader.read()
+ expected = {0: np.array(['"hello world"'] * 3, dtype=object)}
+ assert_array_dicts_equal(result, expected)
+
+ def test_eof_has_eol(self):
+ # handling of new line at EOF
+ pass
+
+ def test_na_substitution(self):
+ pass
+
+ def test_numpy_string_dtype(self):
+ data = """\
+a,1
+aa,2
+aaa,3
+aaaa,4
+aaaaa,5"""
+
+ def _make_reader(**kwds):
+ if "dtype" in kwds:
+ kwds["dtype"] = ensure_dtype_objs(kwds["dtype"])
+ return TextReader(
+ StringIO(data), delimiter=",", header=None, **kwds, **_na_value_kwargs
+ )
+
+ reader = _make_reader(dtype="S5,i4")
+ result = reader.read()
+
+ assert result[0].dtype == "S5"
+
+ ex_values = np.array(["a", "aa", "aaa", "aaaa", "aaaaa"], dtype="S5")
+ assert (result[0] == ex_values).all()
+ assert result[1].dtype == "i4"
+
+ reader = _make_reader(dtype="S4")
+ result = reader.read()
+ assert result[0].dtype == "S4"
+ ex_values = np.array(["a", "aa", "aaa", "aaaa", "aaaa"], dtype="S4")
+ assert (result[0] == ex_values).all()
+ assert result[1].dtype == "S4"
+
+ def test_pass_dtype(self):
+ data = """\
+one,two
+1,a
+2,b
+3,c
+4,d"""
+
+ def _make_reader(**kwds):
+ if "dtype" in kwds:
+ kwds["dtype"] = ensure_dtype_objs(kwds["dtype"])
+ return TextReader(StringIO(data), delimiter=",", **kwds, **_na_value_kwargs)
+
+ reader = _make_reader(dtype={"one": "u1", 1: "S1"})
+ result = reader.read()
+ assert result[0].dtype == "u1"
+ assert result[1].dtype == "S1"
+
+ reader = _make_reader(dtype={"one": np.uint8, 1: object})
+ result = reader.read()
+ assert result[0].dtype == "u1"
+ assert result[1].dtype == "O"
+
+ reader = _make_reader(dtype={"one": np.dtype("u1"), 1: np.dtype("O")})
+ result = reader.read()
+ assert result[0].dtype == "u1"
+ assert result[1].dtype == "O"
+
+ def test_usecols(self):
+ data = """\
+a,b,c
+1,2,3
+4,5,6
+7,8,9
+10,11,12"""
+
+ def _make_reader(**kwds):
+ return TextReader(StringIO(data), delimiter=",", **kwds, **_na_value_kwargs)
+
+ reader = _make_reader(usecols=(1, 2))
+ result = reader.read()
+
+ exp = _make_reader().read()
+ assert len(result) == 2
+ assert (result[1] == exp[1]).all()
+ assert (result[2] == exp[2]).all()
+
+ @pytest.mark.parametrize(
+ "text, kwargs",
+ [
+ ("a,b,c\r1,2,3\r4,5,6\r7,8,9\r10,11,12", {"delimiter": ","}),
+ (
+ "a b c\r1 2 3\r4 5 6\r7 8 9\r10 11 12",
+ {"delim_whitespace": True},
+ ),
+ ("a,b,c\r1,2,3\r4,5,6\r,88,9\r10,11,12", {"delimiter": ","}),
+ (
+ (
+ "A,B,C,D,E,F,G,H,I,J,K,L,M,N,O\r"
+ "AAAAA,BBBBB,0,0,0,0,0,0,0,0,0,0,0,0,0\r"
+ ",BBBBB,0,0,0,0,0,0,0,0,0,0,0,0,0"
+ ),
+ {"delimiter": ","},
+ ),
+ ("A B C\r 2 3\r4 5 6", {"delim_whitespace": True}),
+ ("A B C\r2 3\r4 5 6", {"delim_whitespace": True}),
+ ],
+ )
+ def test_cr_delimited(self, text, kwargs):
+ nice_text = text.replace("\r", "\r\n")
+ result = TextReader(StringIO(text), **kwargs, **_na_value_kwargs).read()
+ expected = TextReader(StringIO(nice_text), **kwargs, **_na_value_kwargs).read()
+ assert_array_dicts_equal(result, expected)
+
+ def test_empty_field_eof(self):
+ data = "a,b,c\n1,2,3\n4,,"
+
+ result = TextReader(StringIO(data), delimiter=",", **_na_value_kwargs).read()
+
+ expected = {
+ 0: np.array([1, 4], dtype=np.int64),
+ 1: np.array(["2", ""], dtype=object),
+ 2: np.array(["3", ""], dtype=object),
+ }
+ assert_array_dicts_equal(result, expected)
+
+ @pytest.mark.parametrize("repeat", range(10))
+ def test_empty_field_eof_mem_access_bug(self, repeat):
+ # GH5664
+ a = DataFrame([["b"], [np.nan]], columns=["a"], index=["a", "c"])
+ b = DataFrame([[1, 1, 1, 0], [1, 1, 1, 0]], columns=list("abcd"), index=[1, 1])
+ c = DataFrame(
+ [
+ [1, 2, 3, 4],
+ [6, np.nan, np.nan, np.nan],
+ [8, 9, 10, 11],
+ [13, 14, np.nan, np.nan],
+ ],
+ columns=list("abcd"),
+ index=[0, 5, 7, 12],
+ )
+
+ df = read_csv(StringIO("a,b\nc\n"), skiprows=0, names=["a"], engine="c")
+ tm.assert_frame_equal(df, a)
+
+ df = read_csv(
+ StringIO("1,1,1,1,0\n" * 2 + "\n" * 2), names=list("abcd"), engine="c"
+ )
+ tm.assert_frame_equal(df, b)
+
+ df = read_csv(
+ StringIO("0,1,2,3,4\n5,6\n7,8,9,10,11\n12,13,14"),
+ names=list("abcd"),
+ engine="c",
+ )
+ tm.assert_frame_equal(df, c)
+
+ def test_empty_csv_input(self):
+ # GH14867
+ with read_csv(
+ StringIO(), chunksize=20, header=None, names=["a", "b", "c"]
+ ) as df:
+ assert isinstance(df, TextFileReader)
+
+
+def assert_array_dicts_equal(left, right):
+ for k, v in left.items():
+ tm.assert_numpy_array_equal(np.asarray(v), np.asarray(right[k]))
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_unsupported.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_unsupported.py
new file mode 100644
index 0000000000000000000000000000000000000000..8812a65ee4e7d1fb2d19399dd5ce6ba409182249
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_unsupported.py
@@ -0,0 +1,207 @@
+"""
+Tests that features that are currently unsupported in
+either the Python or C parser are actually enforced
+and are clearly communicated to the user.
+
+Ultimately, the goal is to remove test cases from this
+test suite as new feature support is added to the parsers.
+"""
+
+from io import StringIO
+import os
+from pathlib import Path
+
+import pytest
+
+from pandas.errors import ParserError
+
+import pandas._testing as tm
+
+from pandas.io.parsers import read_csv
+import pandas.io.parsers.readers as parsers
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+@pytest.fixture(params=["python", "python-fwf"], ids=lambda val: val)
+def python_engine(request):
+ return request.param
+
+
+class TestUnsupportedFeatures:
+ def test_mangle_dupe_cols_false(self):
+ # see gh-12935
+ data = "a b c\n1 2 3"
+
+ for engine in ("c", "python"):
+ with pytest.raises(TypeError, match="unexpected keyword"):
+ read_csv(StringIO(data), engine=engine, mangle_dupe_cols=True)
+
+ def test_c_engine(self):
+ # see gh-6607
+ data = "a b c\n1 2 3"
+ msg = "does not support"
+
+ # specify C engine with unsupported options (raise)
+ with pytest.raises(ValueError, match=msg):
+ read_csv(StringIO(data), engine="c", sep=r"\s")
+ with pytest.raises(ValueError, match=msg):
+ read_csv(StringIO(data), engine="c", sep="\t", quotechar=chr(128))
+ with pytest.raises(ValueError, match=msg):
+ read_csv(StringIO(data), engine="c", skipfooter=1)
+
+ # specify C-unsupported options without python-unsupported options
+ with tm.assert_produces_warning(parsers.ParserWarning):
+ read_csv(StringIO(data), sep=r"\s")
+ with tm.assert_produces_warning(parsers.ParserWarning):
+ read_csv(StringIO(data), sep="\t", quotechar=chr(128))
+ with tm.assert_produces_warning(parsers.ParserWarning):
+ read_csv(StringIO(data), skipfooter=1)
+
+ text = """ A B C D E
+one two three four
+a b 10.0032 5 -0.5109 -2.3358 -0.4645 0.05076 0.3640
+a q 20 4 0.4473 1.4152 0.2834 1.00661 0.1744
+x q 30 3 -0.6662 -0.5243 -0.3580 0.89145 2.5838"""
+ msg = "Error tokenizing data"
+
+ with pytest.raises(ParserError, match=msg):
+ read_csv(StringIO(text), sep="\\s+")
+ with pytest.raises(ParserError, match=msg):
+ read_csv(StringIO(text), engine="c", sep="\\s+")
+
+ msg = "Only length-1 thousands markers supported"
+ data = """A|B|C
+1|2,334|5
+10|13|10.
+"""
+ with pytest.raises(ValueError, match=msg):
+ read_csv(StringIO(data), thousands=",,")
+ with pytest.raises(ValueError, match=msg):
+ read_csv(StringIO(data), thousands="")
+
+ msg = "Only length-1 line terminators supported"
+ data = "a,b,c~~1,2,3~~4,5,6"
+ with pytest.raises(ValueError, match=msg):
+ read_csv(StringIO(data), lineterminator="~~")
+
+ def test_python_engine(self, python_engine):
+ from pandas.io.parsers.readers import _python_unsupported as py_unsupported
+
+ data = """1,2,3,,
+1,2,3,4,
+1,2,3,4,5
+1,2,,,
+1,2,3,4,"""
+
+ for default in py_unsupported:
+ msg = (
+ f"The {default!r} option is not "
+ f"supported with the {python_engine!r} engine"
+ )
+
+ kwargs = {default: object()}
+ with pytest.raises(ValueError, match=msg):
+ read_csv(StringIO(data), engine=python_engine, **kwargs)
+
+ def test_python_engine_file_no_iter(self, python_engine):
+ # see gh-16530
+ class NoNextBuffer:
+ def __init__(self, csv_data) -> None:
+ self.data = csv_data
+
+ def __next__(self):
+ return self.data.__next__()
+
+ def read(self):
+ return self.data
+
+ def readline(self):
+ return self.data
+
+ data = "a\n1"
+ msg = "'NoNextBuffer' object is not iterable|argument 1 must be an iterator"
+
+ with pytest.raises(TypeError, match=msg):
+ read_csv(NoNextBuffer(data), engine=python_engine)
+
+ def test_pyarrow_engine(self):
+ from pandas.io.parsers.readers import _pyarrow_unsupported as pa_unsupported
+
+ data = """1,2,3,,
+ 1,2,3,4,
+ 1,2,3,4,5
+ 1,2,,,
+ 1,2,3,4,"""
+
+ for default in pa_unsupported:
+ msg = f"The {default!r} option is not supported with the 'pyarrow' engine"
+ kwargs = {default: object()}
+ default_needs_bool = {"warn_bad_lines", "error_bad_lines"}
+ if default == "dialect":
+ kwargs[default] = "excel" # test a random dialect
+ elif default in default_needs_bool:
+ kwargs[default] = True
+ elif default == "on_bad_lines":
+ kwargs[default] = "warn"
+
+ with pytest.raises(ValueError, match=msg):
+ read_csv(StringIO(data), engine="pyarrow", **kwargs)
+
+ def test_on_bad_lines_callable_python_or_pyarrow(self, all_parsers):
+ # GH 5686
+ # GH 54643
+ sio = StringIO("a,b\n1,2")
+ bad_lines_func = lambda x: x
+ parser = all_parsers
+ if all_parsers.engine not in ["python", "pyarrow"]:
+ msg = (
+ "on_bad_line can only be a callable "
+ "function if engine='python' or 'pyarrow'"
+ )
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(sio, on_bad_lines=bad_lines_func)
+ else:
+ parser.read_csv(sio, on_bad_lines=bad_lines_func)
+
+
+def test_close_file_handle_on_invalid_usecols(all_parsers, temp_file):
+ # GH 45384
+ parser = all_parsers
+
+ error = ValueError
+ if parser.engine == "pyarrow":
+ # Raises pyarrow.lib.ArrowKeyError
+ pytest.skip(reason="https://github.com/apache/arrow/issues/38676")
+
+ fname = temp_file
+ Path(fname).write_text("col1,col2\na,b\n1,2", encoding="utf-8")
+ with tm.assert_produces_warning(False):
+ with pytest.raises(error, match="col3"):
+ parser.read_csv(fname, usecols=["col1", "col2", "col3"])
+ # unlink fails on windows if file handles still point to it
+ os.unlink(fname)
+
+
+def test_invalid_file_inputs(request, all_parsers):
+ # GH#45957
+ parser = all_parsers
+ if parser.engine == "python":
+ request.applymarker(
+ pytest.mark.xfail(reason=f"{parser.engine} engine supports lists.")
+ )
+
+ with pytest.raises(ValueError, match="Invalid"):
+ parser.read_csv([])
+
+
+def test_invalid_dtype_backend(all_parsers):
+ parser = all_parsers
+ msg = (
+ "dtype_backend numpy is invalid, only 'numpy_nullable' and "
+ "'pyarrow' are allowed."
+ )
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv("test", dtype_backend="numpy")
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_upcast.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_upcast.py
new file mode 100644
index 0000000000000000000000000000000000000000..c17b7b6871945534f835003f46e53646e86bd8ae
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/test_upcast.py
@@ -0,0 +1,101 @@
+import numpy as np
+import pytest
+
+from pandas._libs.parsers import (
+ _maybe_upcast,
+ na_values,
+)
+
+import pandas as pd
+from pandas import NA
+import pandas._testing as tm
+from pandas.core.arrays import (
+ ArrowStringArray,
+ BooleanArray,
+ FloatingArray,
+ IntegerArray,
+)
+
+
+def test_maybe_upcast(any_real_numpy_dtype):
+ # GH#36712
+
+ dtype = np.dtype(any_real_numpy_dtype)
+ na_value = na_values[dtype]
+ arr = np.array([1, 2, na_value], dtype=dtype)
+ result = _maybe_upcast(arr, use_dtype_backend=True)
+
+ expected_mask = np.array([False, False, True])
+ if issubclass(dtype.type, np.integer):
+ expected = IntegerArray(arr, mask=expected_mask)
+ else:
+ expected = FloatingArray(arr, mask=expected_mask)
+
+ tm.assert_extension_array_equal(result, expected)
+
+
+def test_maybe_upcast_no_na(any_real_numpy_dtype):
+ # GH#36712
+ arr = np.array([1, 2, 3], dtype=any_real_numpy_dtype)
+ result = _maybe_upcast(arr, use_dtype_backend=True)
+
+ expected_mask = np.array([False, False, False])
+ if issubclass(np.dtype(any_real_numpy_dtype).type, np.integer):
+ expected = IntegerArray(arr, mask=expected_mask)
+ else:
+ expected = FloatingArray(arr, mask=expected_mask)
+
+ tm.assert_extension_array_equal(result, expected)
+
+
+def test_maybe_upcaste_bool():
+ # GH#36712
+ dtype = np.bool_
+ na_value = na_values[dtype]
+ arr = np.array([True, False, na_value], dtype="uint8").view(dtype)
+ result = _maybe_upcast(arr, use_dtype_backend=True)
+
+ expected_mask = np.array([False, False, True])
+ expected = BooleanArray(arr, mask=expected_mask)
+ tm.assert_extension_array_equal(result, expected)
+
+
+def test_maybe_upcaste_bool_no_nan():
+ # GH#36712
+ dtype = np.bool_
+ arr = np.array([True, False, False], dtype="uint8").view(dtype)
+ result = _maybe_upcast(arr, use_dtype_backend=True)
+
+ expected_mask = np.array([False, False, False])
+ expected = BooleanArray(arr, mask=expected_mask)
+ tm.assert_extension_array_equal(result, expected)
+
+
+def test_maybe_upcaste_all_nan():
+ # GH#36712
+ dtype = np.int64
+ na_value = na_values[dtype]
+ arr = np.array([na_value, na_value], dtype=dtype)
+ result = _maybe_upcast(arr, use_dtype_backend=True)
+
+ expected_mask = np.array([True, True])
+ expected = IntegerArray(arr, mask=expected_mask)
+ tm.assert_extension_array_equal(result, expected)
+
+
+@pytest.mark.parametrize("val", [na_values[np.object_], "c"])
+def test_maybe_upcast_object(val, string_storage):
+ # GH#36712
+ pa = pytest.importorskip("pyarrow")
+
+ with pd.option_context("mode.string_storage", string_storage):
+ arr = np.array(["a", "b", val], dtype=np.object_)
+ result = _maybe_upcast(arr, use_dtype_backend=True)
+
+ if string_storage == "python":
+ exp_val = "c" if val == "c" else NA
+ expected = pd.array(["a", "b", exp_val], dtype=pd.StringDtype())
+ else:
+ exp_val = "c" if val == "c" else None
+ expected = ArrowStringArray(pa.array(["a", "b", exp_val]))
+ tm.assert_extension_array_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/usecols/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/usecols/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/usecols/test_parse_dates.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/usecols/test_parse_dates.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc54f2487aa60230c091bbce1fc5f52a34e79a4f
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/usecols/test_parse_dates.py
@@ -0,0 +1,87 @@
+"""
+Tests the usecols functionality during parsing
+for all of the parsers defined in parsers.py
+"""
+
+from io import StringIO
+
+import pytest
+
+from pandas import (
+ DataFrame,
+ Index,
+ Timestamp,
+)
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+_msg_pyarrow_requires_names = (
+ "The pyarrow engine does not allow 'usecols' to be integer column "
+ "positions. Pass a list of string column names instead."
+)
+
+
+@skip_pyarrow # pyarrow.lib.ArrowKeyError: Column 'fdate' in include_columns
+def test_usecols_with_parse_dates2(all_parsers):
+ # see gh-13604
+ parser = all_parsers
+ data = """2008-02-07 09:40,1032.43
+2008-02-07 09:50,1042.54
+2008-02-07 10:00,1051.65"""
+
+ names = ["date", "values"]
+ usecols = names[:]
+ parse_dates = [0]
+
+ index = Index(
+ [
+ Timestamp("2008-02-07 09:40"),
+ Timestamp("2008-02-07 09:50"),
+ Timestamp("2008-02-07 10:00"),
+ ],
+ name="date",
+ )
+ cols = {"values": [1032.43, 1042.54, 1051.65]}
+ expected = DataFrame(cols, index=index)
+
+ result = parser.read_csv(
+ StringIO(data),
+ parse_dates=parse_dates,
+ index_col=0,
+ usecols=usecols,
+ header=None,
+ names=names,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_usecols_with_parse_dates3(all_parsers):
+ # see gh-14792
+ parser = all_parsers
+ data = """a,b,c,d,e,f,g,h,i,j
+2016/09/21,1,1,2,3,4,5,6,7,8"""
+
+ usecols = list("abcdefghij")
+ parse_dates = [0]
+
+ cols = {
+ "a": Timestamp("2016-09-21"),
+ "b": [1],
+ "c": [1],
+ "d": [2],
+ "e": [3],
+ "f": [4],
+ "g": [5],
+ "h": [6],
+ "i": [7],
+ "j": [8],
+ }
+ expected = DataFrame(cols, columns=usecols)
+
+ result = parser.read_csv(StringIO(data), usecols=usecols, parse_dates=parse_dates)
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/usecols/test_strings.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/usecols/test_strings.py
new file mode 100644
index 0000000000000000000000000000000000000000..1538bd4e805f7b2d1e0d495a5286198eae063e60
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/usecols/test_strings.py
@@ -0,0 +1,96 @@
+"""
+Tests the usecols functionality during parsing
+for all of the parsers defined in parsers.py
+"""
+
+from io import StringIO
+
+import pytest
+
+from pandas import DataFrame
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+def test_usecols_with_unicode_strings(all_parsers):
+ # see gh-13219
+ data = """AAA,BBB,CCC,DDD
+0.056674973,8,True,a
+2.613230982,2,False,b
+3.568935038,7,False,a"""
+ parser = all_parsers
+
+ exp_data = {
+ "AAA": {
+ 0: 0.056674972999999997,
+ 1: 2.6132309819999997,
+ 2: 3.5689350380000002,
+ },
+ "BBB": {0: 8, 1: 2, 2: 7},
+ }
+ expected = DataFrame(exp_data)
+
+ result = parser.read_csv(StringIO(data), usecols=["AAA", "BBB"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_usecols_with_single_byte_unicode_strings(all_parsers):
+ # see gh-13219
+ data = """A,B,C,D
+0.056674973,8,True,a
+2.613230982,2,False,b
+3.568935038,7,False,a"""
+ parser = all_parsers
+
+ exp_data = {
+ "A": {
+ 0: 0.056674972999999997,
+ 1: 2.6132309819999997,
+ 2: 3.5689350380000002,
+ },
+ "B": {0: 8, 1: 2, 2: 7},
+ }
+ expected = DataFrame(exp_data)
+
+ result = parser.read_csv(StringIO(data), usecols=["A", "B"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("usecols", [["AAA", b"BBB"], [b"AAA", "BBB"]])
+def test_usecols_with_mixed_encoding_strings(all_parsers, usecols):
+ data = """AAA,BBB,CCC,DDD
+0.056674973,8,True,a
+2.613230982,2,False,b
+3.568935038,7,False,a"""
+ parser = all_parsers
+ _msg_validate_usecols_arg = (
+ "'usecols' must either be list-like "
+ "of all strings, all unicode, all "
+ "integers or a callable."
+ )
+ with pytest.raises(ValueError, match=_msg_validate_usecols_arg):
+ parser.read_csv(StringIO(data), usecols=usecols)
+
+
+def test_usecols_with_multi_byte_characters(all_parsers):
+ data = """あああ,いい,ううう,ええええ
+0.056674973,8,True,a
+2.613230982,2,False,b
+3.568935038,7,False,a"""
+ parser = all_parsers
+
+ exp_data = {
+ "あああ": {
+ 0: 0.056674972999999997,
+ 1: 2.6132309819999997,
+ 2: 3.5689350380000002,
+ },
+ "いい": {0: 8, 1: 2, 2: 7},
+ }
+ expected = DataFrame(exp_data)
+
+ result = parser.read_csv(StringIO(data), usecols=["あああ", "いい"])
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/usecols/test_usecols_basic.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/usecols/test_usecols_basic.py
new file mode 100644
index 0000000000000000000000000000000000000000..82b42beb38ae00c6ede967222f0d6729e74d43b9
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/parser/usecols/test_usecols_basic.py
@@ -0,0 +1,547 @@
+"""
+Tests the usecols functionality during parsing
+for all of the parsers defined in parsers.py
+"""
+
+from io import StringIO
+
+import numpy as np
+import pytest
+
+from pandas.errors import ParserError
+
+from pandas import (
+ DataFrame,
+ Index,
+ array,
+)
+import pandas._testing as tm
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+_msg_validate_usecols_arg = (
+ "'usecols' must either be list-like "
+ "of all strings, all unicode, all "
+ "integers or a callable."
+)
+_msg_validate_usecols_names = (
+ "Usecols do not match columns, columns expected but not found: {0}"
+)
+_msg_pyarrow_requires_names = (
+ "The pyarrow engine does not allow 'usecols' to be integer column "
+ "positions. Pass a list of string column names instead."
+)
+
+xfail_pyarrow = pytest.mark.usefixtures("pyarrow_xfail")
+skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame is deprecated:DeprecationWarning"
+)
+
+
+def test_raise_on_mixed_dtype_usecols(all_parsers):
+ # See gh-12678
+ data = """a,b,c
+ 1000,2000,3000
+ 4000,5000,6000
+ """
+ usecols = [0, "b", 2]
+ parser = all_parsers
+
+ with pytest.raises(ValueError, match=_msg_validate_usecols_arg):
+ parser.read_csv(StringIO(data), usecols=usecols)
+
+
+@pytest.mark.parametrize("usecols", [(1, 2), ("b", "c")])
+def test_usecols(all_parsers, usecols, request):
+ data = """\
+a,b,c
+1,2,3
+4,5,6
+7,8,9
+10,11,12"""
+ parser = all_parsers
+ if parser.engine == "pyarrow" and isinstance(usecols[0], int):
+ with pytest.raises(ValueError, match=_msg_pyarrow_requires_names):
+ parser.read_csv(StringIO(data), usecols=usecols)
+ return
+
+ result = parser.read_csv(StringIO(data), usecols=usecols)
+
+ expected = DataFrame([[2, 3], [5, 6], [8, 9], [11, 12]], columns=["b", "c"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_usecols_with_names(all_parsers):
+ data = """\
+a,b,c
+1,2,3
+4,5,6
+7,8,9
+10,11,12"""
+ parser = all_parsers
+ names = ["foo", "bar"]
+
+ if parser.engine == "pyarrow":
+ with pytest.raises(ValueError, match=_msg_pyarrow_requires_names):
+ parser.read_csv(StringIO(data), names=names, usecols=[1, 2], header=0)
+ return
+
+ result = parser.read_csv(StringIO(data), names=names, usecols=[1, 2], header=0)
+
+ expected = DataFrame([[2, 3], [5, 6], [8, 9], [11, 12]], columns=names)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "names,usecols", [(["b", "c"], [1, 2]), (["a", "b", "c"], ["b", "c"])]
+)
+def test_usecols_relative_to_names(all_parsers, names, usecols):
+ data = """\
+1,2,3
+4,5,6
+7,8,9
+10,11,12"""
+ parser = all_parsers
+ if parser.engine == "pyarrow" and not isinstance(usecols[0], int):
+ # ArrowKeyError: Column 'fb' in include_columns does not exist
+ pytest.skip(reason="https://github.com/apache/arrow/issues/38676")
+
+ result = parser.read_csv(StringIO(data), names=names, header=None, usecols=usecols)
+
+ expected = DataFrame([[2, 3], [5, 6], [8, 9], [11, 12]], columns=["b", "c"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_usecols_relative_to_names2(all_parsers):
+ # see gh-5766
+ data = """\
+1,2,3
+4,5,6
+7,8,9
+10,11,12"""
+ parser = all_parsers
+
+ result = parser.read_csv(
+ StringIO(data), names=["a", "b"], header=None, usecols=[0, 1]
+ )
+
+ expected = DataFrame([[1, 2], [4, 5], [7, 8], [10, 11]], columns=["a", "b"])
+ tm.assert_frame_equal(result, expected)
+
+
+# regex mismatch: "Length mismatch: Expected axis has 1 elements"
+@xfail_pyarrow
+def test_usecols_name_length_conflict(all_parsers):
+ data = """\
+1,2,3
+4,5,6
+7,8,9
+10,11,12"""
+ parser = all_parsers
+ msg = "Number of passed names did not match number of header fields in the file"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), names=["a", "b"], header=None, usecols=[1])
+
+
+def test_usecols_single_string(all_parsers):
+ # see gh-20558
+ parser = all_parsers
+ data = """foo, bar, baz
+1000, 2000, 3000
+4000, 5000, 6000"""
+
+ with pytest.raises(ValueError, match=_msg_validate_usecols_arg):
+ parser.read_csv(StringIO(data), usecols="foo")
+
+
+# ArrowKeyError: Column 'a' in include_columns does not exist in CSV file
+@skip_pyarrow
+@pytest.mark.parametrize(
+ "data", ["a,b,c,d\n1,2,3,4\n5,6,7,8", "a,b,c,d\n1,2,3,4,\n5,6,7,8,"]
+)
+def test_usecols_index_col_false(all_parsers, data):
+ # see gh-9082
+ parser = all_parsers
+ usecols = ["a", "c", "d"]
+ expected = DataFrame({"a": [1, 5], "c": [3, 7], "d": [4, 8]})
+
+ result = parser.read_csv(StringIO(data), usecols=usecols, index_col=False)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("index_col", ["b", 0])
+@pytest.mark.parametrize("usecols", [["b", "c"], [1, 2]])
+def test_usecols_index_col_conflict(all_parsers, usecols, index_col, request):
+ # see gh-4201: test that index_col as integer reflects usecols
+ parser = all_parsers
+ data = "a,b,c,d\nA,a,1,one\nB,b,2,two"
+
+ if parser.engine == "pyarrow" and isinstance(usecols[0], int):
+ with pytest.raises(ValueError, match=_msg_pyarrow_requires_names):
+ parser.read_csv(StringIO(data), usecols=usecols, index_col=index_col)
+ return
+
+ expected = DataFrame({"c": [1, 2]}, index=Index(["a", "b"], name="b"))
+
+ result = parser.read_csv(StringIO(data), usecols=usecols, index_col=index_col)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_usecols_index_col_conflict2(all_parsers):
+ # see gh-4201: test that index_col as integer reflects usecols
+ parser = all_parsers
+ data = "a,b,c,d\nA,a,1,one\nB,b,2,two"
+
+ expected = DataFrame({"b": ["a", "b"], "c": [1, 2], "d": ("one", "two")})
+ expected = expected.set_index(["b", "c"])
+
+ result = parser.read_csv(
+ StringIO(data), usecols=["b", "c", "d"], index_col=["b", "c"]
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Expected 3 columns, got 4
+def test_usecols_implicit_index_col(all_parsers):
+ # see gh-2654
+ parser = all_parsers
+ data = "a,b,c\n4,apple,bat,5.7\n8,orange,cow,10"
+
+ result = parser.read_csv(StringIO(data), usecols=["a", "b"])
+ expected = DataFrame({"a": ["apple", "orange"], "b": ["bat", "cow"]}, index=[4, 8])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_usecols_index_col_middle(all_parsers):
+ # GH#9098
+ parser = all_parsers
+ data = """a,b,c,d
+1,2,3,4
+"""
+ result = parser.read_csv(StringIO(data), usecols=["b", "c", "d"], index_col="c")
+ expected = DataFrame({"b": [2], "d": [4]}, index=Index([3], name="c"))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_usecols_index_col_end(all_parsers):
+ # GH#9098
+ parser = all_parsers
+ data = """a,b,c,d
+1,2,3,4
+"""
+ result = parser.read_csv(StringIO(data), usecols=["b", "c", "d"], index_col="d")
+ expected = DataFrame({"b": [2], "c": [3]}, index=Index([4], name="d"))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_usecols_regex_sep(all_parsers):
+ # see gh-2733
+ parser = all_parsers
+ data = "a b c\n4 apple bat 5.7\n8 orange cow 10"
+
+ if parser.engine == "pyarrow":
+ msg = "the 'pyarrow' engine does not support regex separators"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), sep=r"\s+", usecols=("a", "b"))
+ return
+
+ result = parser.read_csv(StringIO(data), sep=r"\s+", usecols=("a", "b"))
+
+ expected = DataFrame({"a": ["apple", "orange"], "b": ["bat", "cow"]}, index=[4, 8])
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # Column 'a' in include_columns does not exist in CSV file
+def test_usecols_with_whitespace(all_parsers):
+ parser = all_parsers
+ data = "a b c\n4 apple bat 5.7\n8 orange cow 10"
+
+ result = parser.read_csv(StringIO(data), sep=r"\s+", usecols=("a", "b"))
+ expected = DataFrame({"a": ["apple", "orange"], "b": ["bat", "cow"]}, index=[4, 8])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "usecols,expected",
+ [
+ # Column selection by index.
+ ([0, 1], DataFrame(data=[[1000, 2000], [4000, 5000]], columns=["2", "0"])),
+ # Column selection by name.
+ (
+ ["0", "1"],
+ DataFrame(data=[[2000, 3000], [5000, 6000]], columns=["0", "1"]),
+ ),
+ ],
+)
+def test_usecols_with_integer_like_header(all_parsers, usecols, expected, request):
+ parser = all_parsers
+ data = """2,0,1
+1000,2000,3000
+4000,5000,6000"""
+
+ if parser.engine == "pyarrow" and isinstance(usecols[0], int):
+ with pytest.raises(ValueError, match=_msg_pyarrow_requires_names):
+ parser.read_csv(StringIO(data), usecols=usecols)
+ return
+
+ result = parser.read_csv(StringIO(data), usecols=usecols)
+ tm.assert_frame_equal(result, expected)
+
+
+@xfail_pyarrow # mismatched shape
+def test_empty_usecols(all_parsers):
+ data = "a,b,c\n1,2,3\n4,5,6"
+ expected = DataFrame(columns=Index([]))
+ parser = all_parsers
+
+ result = parser.read_csv(StringIO(data), usecols=set())
+ tm.assert_frame_equal(result, expected)
+
+
+def test_np_array_usecols(all_parsers):
+ # see gh-12546
+ parser = all_parsers
+ data = "a,b,c\n1,2,3"
+ usecols = np.array(["a", "b"])
+
+ expected = DataFrame([[1, 2]], columns=usecols)
+ result = parser.read_csv(StringIO(data), usecols=usecols)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "usecols,expected",
+ [
+ (
+ lambda x: x.upper() in ["AAA", "BBB", "DDD"],
+ DataFrame(
+ {
+ "AaA": {
+ 0: 0.056674972999999997,
+ 1: 2.6132309819999997,
+ 2: 3.5689350380000002,
+ },
+ "bBb": {0: 8, 1: 2, 2: 7},
+ "ddd": {0: "a", 1: "b", 2: "a"},
+ }
+ ),
+ ),
+ (lambda x: False, DataFrame(columns=Index([]))),
+ ],
+)
+def test_callable_usecols(all_parsers, usecols, expected):
+ # see gh-14154
+ data = """AaA,bBb,CCC,ddd
+0.056674973,8,True,a
+2.613230982,2,False,b
+3.568935038,7,False,a"""
+ parser = all_parsers
+
+ if parser.engine == "pyarrow":
+ msg = "The pyarrow engine does not allow 'usecols' to be a callable"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), usecols=usecols)
+ return
+
+ result = parser.read_csv(StringIO(data), usecols=usecols)
+ tm.assert_frame_equal(result, expected)
+
+
+# ArrowKeyError: Column 'fa' in include_columns does not exist in CSV file
+@skip_pyarrow
+@pytest.mark.parametrize("usecols", [["a", "c"], lambda x: x in ["a", "c"]])
+def test_incomplete_first_row(all_parsers, usecols):
+ # see gh-6710
+ data = "1,2\n1,2,3"
+ parser = all_parsers
+ names = ["a", "b", "c"]
+ expected = DataFrame({"a": [1, 1], "c": [np.nan, 3]})
+
+ result = parser.read_csv(StringIO(data), names=names, usecols=usecols)
+ tm.assert_frame_equal(result, expected)
+
+
+@skip_pyarrow # CSV parse error: Expected 3 columns, got 4
+@pytest.mark.parametrize(
+ "data,usecols,kwargs,expected",
+ [
+ # see gh-8985
+ (
+ "19,29,39\n" * 2 + "10,20,30,40",
+ [0, 1, 2],
+ {"header": None},
+ [[19, 29, 39], [19, 29, 39], [10, 20, 30]],
+ ),
+ # see gh-9549
+ (
+ ("A,B,C\n1,2,3\n3,4,5\n1,2,4,5,1,6\n1,2,3,,,1,\n1,2,3\n5,6,7"),
+ ["A", "B", "C"],
+ {},
+ {
+ "A": [1, 3, 1, 1, 1, 5],
+ "B": [2, 4, 2, 2, 2, 6],
+ "C": [3, 5, 4, 3, 3, 7],
+ },
+ ),
+ ],
+)
+def test_uneven_length_cols(all_parsers, data, usecols, kwargs, expected):
+ # see gh-8985
+ parser = all_parsers
+ result = parser.read_csv(StringIO(data), usecols=usecols, **kwargs)
+ expected = DataFrame(expected)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "usecols,kwargs,expected,msg",
+ [
+ (
+ ["a", "b", "c", "d"],
+ {},
+ DataFrame({"a": [1, 5], "b": [2, 6], "c": [3, 7], "d": [4, 8]}),
+ None,
+ ),
+ (
+ ["a", "b", "c", "f"],
+ {},
+ None,
+ _msg_validate_usecols_names.format(r"\['f'\]"),
+ ),
+ (["a", "b", "f"], {}, None, _msg_validate_usecols_names.format(r"\['f'\]")),
+ (
+ ["a", "b", "f", "g"],
+ {},
+ None,
+ _msg_validate_usecols_names.format(r"\[('f', 'g'|'g', 'f')\]"),
+ ),
+ # see gh-14671
+ (
+ None,
+ {"header": 0, "names": ["A", "B", "C", "D"]},
+ DataFrame({"A": [1, 5], "B": [2, 6], "C": [3, 7], "D": [4, 8]}),
+ None,
+ ),
+ (
+ ["A", "B", "C", "f"],
+ {"header": 0, "names": ["A", "B", "C", "D"]},
+ None,
+ _msg_validate_usecols_names.format(r"\['f'\]"),
+ ),
+ (
+ ["A", "B", "f"],
+ {"names": ["A", "B", "C", "D"]},
+ None,
+ _msg_validate_usecols_names.format(r"\['f'\]"),
+ ),
+ ],
+)
+def test_raises_on_usecols_names_mismatch(
+ all_parsers, usecols, kwargs, expected, msg, request
+):
+ data = "a,b,c,d\n1,2,3,4\n5,6,7,8"
+ kwargs.update(usecols=usecols)
+ parser = all_parsers
+
+ if parser.engine == "pyarrow" and not (
+ usecols is not None and expected is not None
+ ):
+ # everything but the first case
+ # ArrowKeyError: Column 'f' in include_columns does not exist in CSV file
+ pytest.skip(reason="https://github.com/apache/arrow/issues/38676")
+
+ if expected is None:
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO(data), **kwargs)
+ else:
+ result = parser.read_csv(StringIO(data), **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("usecols", [["A", "C"], [0, 2]])
+def test_usecols_subset_names_mismatch_orig_columns(all_parsers, usecols, request):
+ data = "a,b,c,d\n1,2,3,4\n5,6,7,8"
+ names = ["A", "B", "C", "D"]
+ parser = all_parsers
+
+ if parser.engine == "pyarrow":
+ if isinstance(usecols[0], int):
+ with pytest.raises(ValueError, match=_msg_pyarrow_requires_names):
+ parser.read_csv(StringIO(data), header=0, names=names, usecols=usecols)
+ return
+ # "pyarrow.lib.ArrowKeyError: Column 'A' in include_columns does not exist"
+ pytest.skip(reason="https://github.com/apache/arrow/issues/38676")
+
+ result = parser.read_csv(StringIO(data), header=0, names=names, usecols=usecols)
+ expected = DataFrame({"A": [1, 5], "C": [3, 7]})
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("names", [None, ["a", "b"]])
+def test_usecols_indices_out_of_bounds(all_parsers, names):
+ # GH#25623 & GH 41130; enforced in 2.0
+ parser = all_parsers
+ data = """
+a,b
+1,2
+ """
+
+ err = ParserError
+ msg = "Defining usecols with out-of-bounds"
+ if parser.engine == "pyarrow":
+ err = ValueError
+ msg = _msg_pyarrow_requires_names
+
+ with pytest.raises(err, match=msg):
+ parser.read_csv(StringIO(data), usecols=[0, 2], names=names, header=0)
+
+
+def test_usecols_additional_columns(all_parsers):
+ # GH#46997
+ parser = all_parsers
+ usecols = lambda header: header.strip() in ["a", "b", "c"]
+
+ if parser.engine == "pyarrow":
+ msg = "The pyarrow engine does not allow 'usecols' to be a callable"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO("a,b\nx,y,z"), index_col=False, usecols=usecols)
+ return
+ result = parser.read_csv(StringIO("a,b\nx,y,z"), index_col=False, usecols=usecols)
+ expected = DataFrame({"a": ["x"], "b": "y"})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_usecols_additional_columns_integer_columns(all_parsers):
+ # GH#46997
+ parser = all_parsers
+ usecols = lambda header: header.strip() in ["0", "1"]
+ if parser.engine == "pyarrow":
+ msg = "The pyarrow engine does not allow 'usecols' to be a callable"
+ with pytest.raises(ValueError, match=msg):
+ parser.read_csv(StringIO("0,1\nx,y,z"), index_col=False, usecols=usecols)
+ return
+ result = parser.read_csv(StringIO("0,1\nx,y,z"), index_col=False, usecols=usecols)
+ expected = DataFrame({"0": ["x"], "1": "y"})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_usecols_dtype(all_parsers):
+ parser = all_parsers
+ data = """
+col1,col2,col3
+a,1,x
+b,2,y
+"""
+ result = parser.read_csv(
+ StringIO(data),
+ usecols=["col1", "col2"],
+ dtype={"col1": "string", "col2": "uint8", "col3": "string"},
+ )
+ expected = DataFrame(
+ {"col1": array(["a", "b"]), "col2": np.array([1, 2], dtype="uint8")}
+ )
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/common.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/common.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/conftest.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d0a7951ce82222094fbdab78a788d845dada1be
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/conftest.py
@@ -0,0 +1,25 @@
+import uuid
+
+import pytest
+
+from pandas.io.pytables import HDFStore
+
+tables = pytest.importorskip("tables")
+# set these parameters so we don't have file sharing
+tables.parameters.MAX_NUMEXPR_THREADS = 1
+tables.parameters.MAX_BLOSC_THREADS = 1
+tables.parameters.MAX_THREADS = 1
+
+
+@pytest.fixture
+def temp_h5_path(tmp_path):
+ """Fixture for HDF5 path"""
+ file_path = tmp_path / f"{uuid.uuid4()}.h5"
+ file_path.touch()
+ return file_path
+
+
+@pytest.fixture
+def temp_hdfstore(temp_h5_path):
+ with HDFStore(temp_h5_path, mode="a") as store:
+ yield store
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_append.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_append.py
new file mode 100644
index 0000000000000000000000000000000000000000..d19f6dda5ffc51c7a5309a7db4f828afcc693a5d
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_append.py
@@ -0,0 +1,989 @@
+import datetime
+from datetime import timedelta
+import re
+
+import numpy as np
+import pytest
+
+from pandas._libs.tslibs import Timestamp
+from pandas.compat import PY312
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ Series,
+ _testing as tm,
+ concat,
+ date_range,
+ read_hdf,
+)
+
+pytestmark = [pytest.mark.single_cpu]
+
+tables = pytest.importorskip("tables")
+
+
+@pytest.mark.filterwarnings("ignore::tables.NaturalNameWarning")
+def test_append(temp_hdfstore):
+ # this is allowed by almost always don't want to do it
+ # tables.NaturalNameWarning):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((20, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=20, freq="B"),
+ )
+ temp_hdfstore.append("df1", df[:10])
+ temp_hdfstore.append("df1", df[10:])
+ tm.assert_frame_equal(temp_hdfstore["df1"], df)
+
+ temp_hdfstore.put("df2", df[:10], format="table")
+ temp_hdfstore.append("df2", df[10:])
+ tm.assert_frame_equal(temp_hdfstore["df2"], df)
+
+ temp_hdfstore.append("/df3", df[:10])
+ temp_hdfstore.append("/df3", df[10:])
+ tm.assert_frame_equal(temp_hdfstore["df3"], df)
+
+ # this is allowed by almost always don't want to do it
+ # tables.NaturalNameWarning
+ temp_hdfstore.append("/df3 foo", df[:10])
+ temp_hdfstore.append("/df3 foo", df[10:])
+ tm.assert_frame_equal(temp_hdfstore["df3 foo"], df)
+
+ # dtype issues - mizxed type in a single object column
+ df = DataFrame(data=[[1, 2], [0, 1], [1, 2], [0, 0]])
+ df["mixed_column"] = "testing"
+ df.loc[2, "mixed_column"] = np.nan
+ temp_hdfstore.append("df", df)
+ tm.assert_frame_equal(temp_hdfstore["df"], df)
+
+ # uints - test storage of uints
+ uint_data = DataFrame(
+ {
+ "u08": Series(
+ np.random.default_rng(2).integers(0, high=255, size=5),
+ dtype=np.uint8,
+ ),
+ "u16": Series(
+ np.random.default_rng(2).integers(0, high=65535, size=5),
+ dtype=np.uint16,
+ ),
+ "u32": Series(
+ np.random.default_rng(2).integers(0, high=2**30, size=5),
+ dtype=np.uint32,
+ ),
+ "u64": Series(
+ [2**58, 2**59, 2**60, 2**61, 2**62],
+ dtype=np.uint64,
+ ),
+ },
+ index=np.arange(5),
+ )
+ temp_hdfstore.append("uints", uint_data)
+ tm.assert_frame_equal(temp_hdfstore["uints"], uint_data, check_index_type=True)
+
+ # uints - test storage of uints in indexable columns
+ temp_hdfstore.remove("uints")
+ # 64-bit indices not yet supported
+ temp_hdfstore.append("uints", uint_data, data_columns=["u08", "u16", "u32"])
+ tm.assert_frame_equal(temp_hdfstore["uints"], uint_data, check_index_type=True)
+
+
+def test_append_series(temp_hdfstore):
+ # basic
+ ss = Series(range(20), dtype=np.float64, index=[f"i_{i}" for i in range(20)])
+ ts = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ ns = Series(np.arange(100))
+
+ temp_hdfstore.append("ss", ss)
+ result = temp_hdfstore["ss"]
+ tm.assert_series_equal(result, ss)
+ assert result.name is None
+
+ temp_hdfstore.append("ts", ts)
+ result = temp_hdfstore["ts"]
+ tm.assert_series_equal(result, ts)
+ assert result.name is None
+
+ ns.name = "foo"
+ temp_hdfstore.append("ns", ns)
+ result = temp_hdfstore["ns"]
+ tm.assert_series_equal(result, ns)
+ assert result.name == ns.name
+
+ # select on the values
+ expected = ns[ns > 60]
+ result = temp_hdfstore.select("ns", "foo>60")
+ tm.assert_series_equal(result, expected)
+
+ # select on the index and values
+ expected = ns[(ns > 70) & (ns.index < 90)]
+ # Reading/writing RangeIndex info is not supported yet
+ expected.index = Index(expected.index._data)
+ result = temp_hdfstore.select("ns", "foo>70 and index<90")
+ tm.assert_series_equal(result, expected, check_index_type=True)
+
+ # multi-index
+ mi = DataFrame(np.random.default_rng(2).standard_normal((5, 1)), columns=["A"])
+ mi["B"] = np.arange(len(mi))
+ mi["C"] = "foo"
+ mi.loc[3:5, "C"] = "bar"
+ mi.set_index(["C", "B"], inplace=True)
+ s = mi.stack()
+ s.index = s.index.droplevel(2)
+ temp_hdfstore.append("mi", s)
+ tm.assert_series_equal(temp_hdfstore["mi"], s, check_index_type=True)
+
+
+def test_append_some_nans(temp_hdfstore):
+ df = DataFrame(
+ {
+ "A": Series(np.random.default_rng(2).standard_normal(20)).astype("int32"),
+ "A1": np.random.default_rng(2).standard_normal(20),
+ "A2": np.random.default_rng(2).standard_normal(20),
+ "B": "foo",
+ "C": "bar",
+ "D": Timestamp("2001-01-01").as_unit("ns"),
+ "E": Timestamp("2001-01-02").as_unit("ns"),
+ },
+ index=np.arange(20),
+ )
+ # some nans
+ df.loc[0:15, ["A1", "B", "D", "E"]] = np.nan
+ temp_hdfstore.append("df1", df[:10])
+ temp_hdfstore.append("df1", df[10:])
+ tm.assert_frame_equal(temp_hdfstore["df1"], df, check_index_type=True)
+
+ # first column
+ df1 = df.copy()
+ df1["A1"] = np.nan
+ temp_hdfstore.remove("df1")
+ temp_hdfstore.append("df1", df1[:10])
+ temp_hdfstore.append("df1", df1[10:])
+ tm.assert_frame_equal(temp_hdfstore["df1"], df1, check_index_type=True)
+
+ # 2nd column
+ df2 = df.copy()
+ df2["A2"] = np.nan
+ temp_hdfstore.append("df2", df2[:10])
+ temp_hdfstore.append("df2", df2[10:])
+ tm.assert_frame_equal(temp_hdfstore["df2"], df2, check_index_type=True)
+
+ # datetimes
+ df3 = df.copy()
+ df3["E"] = np.nan
+ temp_hdfstore.append("df3", df3[:10])
+ temp_hdfstore.append("df3", df3[10:])
+ tm.assert_frame_equal(temp_hdfstore["df3"], df3, check_index_type=True)
+
+
+def test_append_all_nans(temp_hdfstore, using_infer_string):
+ df = DataFrame(
+ {
+ "A1": np.random.default_rng(2).standard_normal(20),
+ "A2": np.random.default_rng(2).standard_normal(20),
+ },
+ index=np.arange(20),
+ )
+ df.loc[0:15, :] = np.nan
+
+ # nan some entire rows (dropna=True)
+ temp_hdfstore.append("df", df[:10], dropna=True)
+ temp_hdfstore.append("df", df[10:], dropna=True)
+ tm.assert_frame_equal(temp_hdfstore["df"], df[-4:], check_index_type=True)
+
+ # nan some entire rows (dropna=False)
+ temp_hdfstore.append("df2", df[:10], dropna=False)
+ temp_hdfstore.append("df2", df[10:], dropna=False)
+ tm.assert_frame_equal(temp_hdfstore["df2"], df, check_index_type=True)
+
+ # tests the option io.hdf.dropna_table
+ with pd.option_context("io.hdf.dropna_table", False):
+ temp_hdfstore.append("df3", df[:10])
+ temp_hdfstore.append("df3", df[10:])
+ tm.assert_frame_equal(temp_hdfstore["df3"], df)
+
+ with pd.option_context("io.hdf.dropna_table", True):
+ temp_hdfstore.append("df4", df[:10])
+ temp_hdfstore.append("df4", df[10:])
+ tm.assert_frame_equal(temp_hdfstore["df4"], df[-4:])
+
+ # nan some entire rows (string are still written!)
+ df = DataFrame(
+ {
+ "A1": np.random.default_rng(2).standard_normal(20),
+ "A2": np.random.default_rng(2).standard_normal(20),
+ "B": "foo",
+ "C": "bar",
+ },
+ index=np.arange(20),
+ )
+
+ df.loc[0:15, :] = np.nan
+
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df[:10], dropna=True)
+ temp_hdfstore.append("df", df[10:], dropna=True)
+ result = temp_hdfstore["df"]
+ expected = df
+ if using_infer_string:
+ # TODO: Test is incorrect when not using_infer_string.
+ # Should take the last 4 rows uncondiationally.
+ expected = expected[-4:]
+ tm.assert_frame_equal(result, expected, check_index_type=True)
+
+ temp_hdfstore.remove("df2")
+ temp_hdfstore.append("df2", df[:10], dropna=False)
+ temp_hdfstore.append("df2", df[10:], dropna=False)
+ tm.assert_frame_equal(temp_hdfstore["df2"], df, check_index_type=True)
+
+ # nan some entire rows (but since we have dates they are still
+ # written!)
+ df = DataFrame(
+ {
+ "A1": np.random.default_rng(2).standard_normal(20),
+ "A2": np.random.default_rng(2).standard_normal(20),
+ "B": "foo",
+ "C": "bar",
+ "D": Timestamp("2001-01-01").as_unit("ns"),
+ "E": Timestamp("2001-01-02").as_unit("ns"),
+ },
+ index=np.arange(20),
+ )
+
+ df.loc[0:15, :] = np.nan
+
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df[:10], dropna=True)
+ temp_hdfstore.append("df", df[10:], dropna=True)
+ tm.assert_frame_equal(temp_hdfstore["df"], df, check_index_type=True)
+
+ temp_hdfstore.remove("df2")
+ temp_hdfstore.append("df2", df[:10], dropna=False)
+ temp_hdfstore.append("df2", df[10:], dropna=False)
+ tm.assert_frame_equal(temp_hdfstore["df2"], df, check_index_type=True)
+
+
+def test_append_frame_column_oriented(temp_hdfstore, request):
+ # column oriented
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ df.index = df.index._with_freq(None) # freq doesn't round-trip
+
+ temp_hdfstore.append("df1", df.iloc[:, :2], axes=["columns"])
+ temp_hdfstore.append("df1", df.iloc[:, 2:])
+ tm.assert_frame_equal(temp_hdfstore["df1"], df)
+
+ result = temp_hdfstore.select("df1", "columns=A")
+ expected = df.reindex(columns=["A"])
+ tm.assert_frame_equal(expected, result)
+
+ # selection on the non-indexable
+ request.applymarker(
+ pytest.mark.xfail(
+ PY312,
+ reason="AST change in PY312",
+ raises=ValueError,
+ )
+ )
+ result = temp_hdfstore.select("df1", ("columns=A", "index=df.index[0:4]"))
+ expected = df.reindex(columns=["A"], index=df.index[0:4])
+ tm.assert_frame_equal(expected, result)
+
+ # this isn't supported
+ msg = re.escape(
+ "passing a filterable condition to a non-table indexer "
+ "[Filter: Not Initialized]"
+ )
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.select("df1", "columns=A and index>df.index[4]")
+
+
+def test_append_with_different_block_ordering(temp_hdfstore):
+ # GH 4096; using same frames, but different block orderings
+ for i in range(10):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)), columns=list("AB")
+ )
+ df["index"] = range(10)
+ df["index"] += i * 10
+ df["int64"] = Series([1] * len(df), dtype="int64")
+ df["int16"] = Series([1] * len(df), dtype="int16")
+
+ if i % 2 == 0:
+ del df["int64"]
+ df["int64"] = Series([1] * len(df), dtype="int64")
+ if i % 3 == 0:
+ a = df.pop("A")
+ df["A"] = a
+
+ df.set_index("index", inplace=True)
+
+ temp_hdfstore.append("df", df)
+
+ # test a different ordering but with more fields (like invalid
+ # combinations)
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)),
+ columns=list("AB"),
+ dtype="float64",
+ )
+ df["int64"] = Series([1] * len(df), dtype="int64")
+ df["int16"] = Series([1] * len(df), dtype="int16")
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df)
+
+ # store additional fields in different blocks
+ df["int16_2"] = Series([1] * len(df), dtype="int16")
+ msg = re.escape(
+ "cannot match existing table structure for [int16] on appending data"
+ )
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("df", df)
+
+ # store multiple additional fields in different blocks
+ df["float_3"] = Series([1.0] * len(df), dtype="float64")
+ msg = re.escape("cannot match existing table structure for [A,B] on appending data")
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("df", df)
+
+
+def test_append_with_strings(temp_hdfstore):
+ def check_col(key, name, size):
+ assert (
+ getattr(temp_hdfstore.get_storer(key).table.description, name).itemsize
+ == size
+ )
+
+ # avoid truncation on elements
+ df = DataFrame([[123, "asdqwerty"], [345, "dggnhebbsdfbdfb"]])
+ temp_hdfstore.append("df_big", df)
+ tm.assert_frame_equal(temp_hdfstore.select("df_big"), df)
+ check_col("df_big", "values_block_1", 15)
+
+ # appending smaller string ok
+ df2 = DataFrame([[124, "asdqy"], [346, "dggnhefbdfb"]])
+ temp_hdfstore.append("df_big", df2)
+ expected = concat([df, df2])
+ tm.assert_frame_equal(temp_hdfstore.select("df_big"), expected)
+ check_col("df_big", "values_block_1", 15)
+
+ # avoid truncation on elements
+ df = DataFrame([[123, "asdqwerty"], [345, "dggnhebbsdfbdfb"]])
+ temp_hdfstore.append("df_big2", df, min_itemsize={"values": 50})
+ tm.assert_frame_equal(temp_hdfstore.select("df_big2"), df)
+ check_col("df_big2", "values_block_1", 50)
+
+ # bigger string on next append
+ temp_hdfstore.append("df_new", df)
+ df_new = DataFrame([[124, "abcdefqhij"], [346, "abcdefghijklmnopqrtsuvwxyz"]])
+ msg = (
+ r"Trying to store a string with len \[26\] in "
+ r"\[values_block_1\] column but\n"
+ r"this column has a limit of \[15\]!\n"
+ "Consider using min_itemsize to preset the sizes on these "
+ "columns"
+ )
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("df_new", df_new)
+
+ # min_itemsize on Series index (GH 11412)
+ df = DataFrame(
+ {
+ "A": [0.0, 1.0, 2.0, 3.0, 4.0],
+ "B": [0.0, 1.0, 0.0, 1.0, 0.0],
+ "C": Index(["foo1", "foo2", "foo3", "foo4", "foo5"]),
+ "D": date_range("20130101", periods=5),
+ }
+ ).set_index("C")
+ temp_hdfstore.append("ss", df["B"], min_itemsize={"index": 4})
+ tm.assert_series_equal(temp_hdfstore.select("ss"), df["B"])
+
+ # same as above, with data_columns=True
+ temp_hdfstore.append("ss2", df["B"], data_columns=True, min_itemsize={"index": 4})
+ tm.assert_series_equal(temp_hdfstore.select("ss2"), df["B"])
+
+ # min_itemsize in index without appending (GH 10381)
+ temp_hdfstore.put("ss3", df, format="table", min_itemsize={"index": 6})
+ # just make sure there is a longer string:
+ df2 = df.copy().reset_index().assign(C="longer").set_index("C")
+ temp_hdfstore.append("ss3", df2)
+ tm.assert_frame_equal(temp_hdfstore.select("ss3"), concat([df, df2]))
+
+ # same as above, with a Series
+ temp_hdfstore.put("ss4", df["B"], format="table", min_itemsize={"index": 6})
+ temp_hdfstore.append("ss4", df2["B"])
+ tm.assert_series_equal(temp_hdfstore.select("ss4"), concat([df["B"], df2["B"]]))
+
+ # with nans
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ df["string"] = "foo"
+ df.loc[df.index[1:4], "string"] = np.nan
+ df["string2"] = "bar"
+ df.loc[df.index[4:8], "string2"] = np.nan
+ df["string3"] = "bah"
+ df.loc[df.index[1:], "string3"] = np.nan
+ temp_hdfstore.append("df", df)
+ result = temp_hdfstore.select("df")
+ tm.assert_frame_equal(result, df)
+
+
+def test_append_with_strings2(temp_hdfstore):
+ def check_col(key, name, size):
+ assert (
+ getattr(temp_hdfstore.get_storer(key).table.description, name).itemsize
+ == size
+ )
+
+ df = DataFrame({"A": "foo", "B": "bar"}, index=range(10))
+
+ # a min_itemsize that creates a data_column
+ temp_hdfstore.append("df", df, min_itemsize={"A": 200})
+ check_col("df", "A", 200)
+ assert temp_hdfstore.get_storer("df").data_columns == ["A"]
+
+ # a min_itemsize that creates a data_column2
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df, data_columns=["B"], min_itemsize={"A": 200})
+ check_col("df", "A", 200)
+ assert temp_hdfstore.get_storer("df").data_columns == ["B", "A"]
+
+ # a min_itemsize that creates a data_column2
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df, data_columns=["B"], min_itemsize={"values": 200})
+ check_col("df", "B", 200)
+ check_col("df", "values_block_0", 200)
+ assert temp_hdfstore.get_storer("df").data_columns == ["B"]
+
+ # infer the .typ on subsequent appends
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df[:5], min_itemsize=200)
+ temp_hdfstore.append("df", df[5:], min_itemsize=200)
+ tm.assert_frame_equal(temp_hdfstore["df"], df)
+
+ # invalid min_itemsize keys
+ df = DataFrame(["foo", "foo", "foo", "barh", "barh", "barh"], columns=["A"])
+ temp_hdfstore.remove("df")
+ msg = re.escape(
+ "min_itemsize has the key [foo] which is not an axis or data_column"
+ )
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("df", df, min_itemsize={"foo": 20, "foobar": 20})
+
+
+def test_append_with_empty_string(temp_hdfstore):
+ # with all empty strings (GH 12242)
+ df = DataFrame({"x": ["a", "b", "c", "d", "e", "f", ""]})
+ temp_hdfstore.append("df", df[:-1], min_itemsize={"x": 1})
+ temp_hdfstore.append("df", df[-1:], min_itemsize={"x": 1})
+ tm.assert_frame_equal(temp_hdfstore.select("df"), df)
+
+
+def test_append_with_data_columns(temp_hdfstore):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B", unit="ns"),
+ )
+ df.iloc[0, df.columns.get_loc("B")] = 1.0
+ temp_hdfstore.append("df", df[:2], data_columns=["B"])
+ temp_hdfstore.append("df", df[2:])
+ tm.assert_frame_equal(temp_hdfstore["df"], df)
+
+ # check that we have indices created
+ assert temp_hdfstore._handle.root.df.table.cols.index.is_indexed is True
+ assert temp_hdfstore._handle.root.df.table.cols.B.is_indexed is True
+
+ # data column searching
+ result = temp_hdfstore.select("df", "B>0")
+ expected = df[df.B > 0]
+ tm.assert_frame_equal(result, expected)
+
+ # data column searching (with an indexable and a data_columns)
+ result = temp_hdfstore.select("df", "B>0 and index>df.index[3]")
+ df_new = df.reindex(index=df.index[4:])
+ expected = df_new[df_new.B > 0]
+ tm.assert_frame_equal(result, expected)
+
+ # data column selection with a string data_column
+ df_new = df.copy()
+ df_new["string"] = "foo"
+ df_new.loc[df_new.index[1:4], "string"] = np.nan
+ df_new.loc[df_new.index[5:6], "string"] = "bar"
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df_new, data_columns=["string"])
+ result = temp_hdfstore.select("df", "string='foo'")
+ expected = df_new[df_new.string == "foo"]
+ tm.assert_frame_equal(result, expected)
+
+ # using min_itemsize and a data column
+ def check_col(key, name, size):
+ assert (
+ getattr(temp_hdfstore.get_storer(key).table.description, name).itemsize
+ == size
+ )
+
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append(
+ "df", df_new, data_columns=["string"], min_itemsize={"string": 30}
+ )
+ check_col("df", "string", 30)
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df_new, data_columns=["string"], min_itemsize=30)
+ check_col("df", "string", 30)
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append(
+ "df", df_new, data_columns=["string"], min_itemsize={"values": 30}
+ )
+ check_col("df", "string", 30)
+
+ df_new["string2"] = "foobarbah"
+ df_new["string_block1"] = "foobarbah1"
+ df_new["string_block2"] = "foobarbah2"
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append(
+ "df",
+ df_new,
+ data_columns=["string", "string2"],
+ min_itemsize={"string": 30, "string2": 40, "values": 50},
+ )
+ check_col("df", "string", 30)
+ check_col("df", "string2", 40)
+ check_col("df", "values_block_1", 50)
+
+ # multiple data columns
+ df_new = df.copy()
+ df_new.iloc[0, df_new.columns.get_loc("A")] = 1.0
+ df_new.iloc[0, df_new.columns.get_loc("B")] = -1.0
+ df_new["string"] = "foo"
+
+ sl = df_new.columns.get_loc("string")
+ df_new.iloc[1:4, sl] = np.nan
+ df_new.iloc[5:6, sl] = "bar"
+
+ df_new["string2"] = "foo"
+ sl = df_new.columns.get_loc("string2")
+ df_new.iloc[2:5, sl] = np.nan
+ df_new.iloc[7:8, sl] = "bar"
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df_new, data_columns=["A", "B", "string", "string2"])
+ result = temp_hdfstore.select(
+ "df", "string='foo' and string2='foo' and A>0 and B<0"
+ )
+ expected = df_new[
+ (df_new.string == "foo")
+ & (df_new.string2 == "foo")
+ & (df_new.A > 0)
+ & (df_new.B < 0)
+ ]
+ tm.assert_frame_equal(result, expected, check_freq=False)
+ # FIXME: 2020-05-07 freq check randomly fails in the CI
+
+ # yield an empty frame
+ result = temp_hdfstore.select("df", "string='foo' and string2='cool'")
+ expected = df_new[(df_new.string == "foo") & (df_new.string2 == "cool")]
+ tm.assert_frame_equal(result, expected)
+
+ # doc example
+ df_dc = df.copy()
+ df_dc["string"] = "foo"
+ df_dc.loc[df_dc.index[4:6], "string"] = np.nan
+ df_dc.loc[df_dc.index[7:9], "string"] = "bar"
+ df_dc["string2"] = "cool"
+ df_dc["datetime"] = Timestamp("20010102").as_unit("ns")
+ df_dc.loc[df_dc.index[3:5], ["A", "B", "datetime"]] = np.nan
+
+ temp_hdfstore.append(
+ "df_dc", df_dc, data_columns=["B", "C", "string", "string2", "datetime"]
+ )
+ result = temp_hdfstore.select("df_dc", "B>0")
+
+ expected = df_dc[df_dc.B > 0]
+ tm.assert_frame_equal(result, expected)
+
+ result = temp_hdfstore.select("df_dc", ["B > 0", "C > 0", "string == foo"])
+ expected = df_dc[(df_dc.B > 0) & (df_dc.C > 0) & (df_dc.string == "foo")]
+ tm.assert_frame_equal(result, expected, check_freq=False)
+ # FIXME: 2020-12-07 intermittent build failures here with freq of
+ # None instead of BDay(4)
+
+ # doc example part 2
+
+ index = date_range("1/1/2000", periods=8)
+ df_dc = DataFrame(
+ np.random.default_rng(2).standard_normal((8, 3)),
+ index=index,
+ columns=["A", "B", "C"],
+ )
+ df_dc["string"] = "foo"
+ df_dc.loc[df_dc.index[4:6], "string"] = np.nan
+ df_dc.loc[df_dc.index[7:9], "string"] = "bar"
+ df_dc[["B", "C"]] = df_dc[["B", "C"]].abs()
+ df_dc["string2"] = "cool"
+
+ # on-disk operations
+ temp_hdfstore.remove("df_dc")
+ temp_hdfstore.append("df_dc", df_dc, data_columns=["B", "C", "string", "string2"])
+
+ result = temp_hdfstore.select("df_dc", "B>0")
+ expected = df_dc[df_dc.B > 0]
+ tm.assert_frame_equal(result, expected)
+
+ result = temp_hdfstore.select("df_dc", ["B > 0", "C > 0", 'string == "foo"'])
+ expected = df_dc[(df_dc.B > 0) & (df_dc.C > 0) & (df_dc.string == "foo")]
+ tm.assert_frame_equal(result, expected)
+
+
+def test_append_hierarchical(temp_hdfstore, multiindex_dataframe_random_data):
+ df = multiindex_dataframe_random_data
+ df.columns.name = None
+
+ temp_hdfstore.append("mi", df)
+ result = temp_hdfstore.select("mi")
+ tm.assert_frame_equal(result, df)
+
+ # GH 3748
+ result = temp_hdfstore.select("mi", columns=["A", "B"])
+ expected = df.reindex(columns=["A", "B"])
+ tm.assert_frame_equal(result, expected)
+
+ df.to_hdf(temp_hdfstore, key="df", format="table")
+ result = read_hdf(temp_hdfstore, "df", columns=["A", "B"])
+ expected = df.reindex(columns=["A", "B"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_append_misc(temp_hdfstore):
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ temp_hdfstore.append("df", df, chunksize=1)
+ result = temp_hdfstore.select("df")
+ tm.assert_frame_equal(result, df)
+
+ temp_hdfstore.append("df1", df, expectedrows=10)
+ result = temp_hdfstore.select("df1")
+ tm.assert_frame_equal(result, df)
+
+
+@pytest.mark.parametrize("chunksize", [10, 200, 1000])
+def test_append_misc_chunksize(temp_hdfstore, chunksize):
+ # more chunksize in append tests
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ df["string"] = "foo"
+ df["float322"] = 1.0
+ df["float322"] = df["float322"].astype("float32")
+ df["bool"] = df["float322"] > 0
+ df["time1"] = Timestamp("20130101").as_unit("ns")
+ df["time2"] = Timestamp("20130102").as_unit("ns")
+ temp_hdfstore.append("obj", df, chunksize=chunksize)
+ result = temp_hdfstore.select("obj")
+ tm.assert_frame_equal(result, df)
+
+
+def test_append_misc_empty_frame(temp_hdfstore):
+ # empty frame, GH4273
+ # 0 len
+ df_empty = DataFrame(columns=list("ABC"))
+ temp_hdfstore.append("df", df_empty)
+ with pytest.raises(KeyError, match="'No object named df in the file'"):
+ temp_hdfstore.select("df")
+
+ # repeated append of 0/non-zero frames
+ df = DataFrame(np.random.default_rng(2).random((10, 3)), columns=list("ABC"))
+ temp_hdfstore.append("df", df)
+ tm.assert_frame_equal(temp_hdfstore.select("df"), df)
+ temp_hdfstore.append("df", df_empty)
+ tm.assert_frame_equal(temp_hdfstore.select("df"), df)
+
+ # store
+ df = DataFrame(columns=list("ABC"))
+ temp_hdfstore.put("df2", df)
+ tm.assert_frame_equal(temp_hdfstore.select("df2"), df)
+
+
+def test_append_raise(temp_hdfstore, using_infer_string):
+ # test append with invalid input to get good error messages
+
+ # list in column
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ df["invalid"] = [["a"]] * len(df)
+ assert df.dtypes["invalid"] == np.object_
+ msg = re.escape(
+ """Cannot serialize the column [invalid]
+because its data contents are not [string] but [mixed] object dtype"""
+ )
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.append("df", df)
+
+ # multiple invalid columns
+ df["invalid2"] = [["a"]] * len(df)
+ df["invalid3"] = [["a"]] * len(df)
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.append("df", df)
+
+ # datetime with embedded nans as object
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ s = Series(datetime.datetime(2001, 1, 2), index=df.index)
+ s = s.astype(object)
+ s[0:5] = np.nan
+ df["invalid"] = s
+ assert df.dtypes["invalid"] == np.object_
+ msg = "too many timezones in this block, create separate data columns"
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.append("df", df)
+
+ # directly ndarray
+ msg = "value must be None, Series, or DataFrame"
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.append("df", np.arange(10))
+
+ # series directly
+ msg = re.escape(
+ "cannot properly create the storer for: "
+ "[group->df,value->]"
+ )
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.append("df", Series(np.arange(10)))
+
+ # appending an incompatible table
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ temp_hdfstore.append("df", df)
+
+ df["foo"] = "foo"
+ msg = re.escape(
+ "invalid combination of [non_index_axes] on appending data "
+ "[(1, ['A', 'B', 'C', 'D', 'foo'])] vs current table "
+ "[(1, ['A', 'B', 'C', 'D'])]"
+ )
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("df", df)
+
+ # incompatible type (GH 41897)
+ df["foo"] = Timestamp("20130101")
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df)
+ df["foo"] = "bar"
+ msg = re.escape(
+ "Cannot serialize the column [foo] "
+ "because its data contents are not [string] "
+ "but [datetime64[us]] object dtype"
+ )
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("df", df)
+
+
+def test_append_with_timedelta(temp_hdfstore, unit):
+ # GH 3577
+ # append timedelta
+
+ ts = Timestamp("20130101").as_unit("ns")
+ df = DataFrame(
+ {
+ "A": ts,
+ "B": [ts + timedelta(days=i, seconds=10) for i in range(10)],
+ }
+ )
+ df["C"] = df["A"] - df["B"]
+ df["C"] = df["C"].astype(f"m8[{unit}]")
+ df.loc[3:5, "C"] = np.nan
+
+ # table
+ temp_hdfstore.append("df", df, data_columns=True)
+ result = temp_hdfstore.select("df")
+ tm.assert_frame_equal(result, df)
+
+ result = temp_hdfstore.select("df", where="C<100000")
+ tm.assert_frame_equal(result, df)
+
+ result = temp_hdfstore.select("df", where="C0", "B>0"], selector="df1"
+ )
+ expected = df[(df.A > 0) & (df.B > 0)]
+ tm.assert_frame_equal(result, expected)
+
+
+def test_append_to_multiple_dropna(temp_hdfstore):
+ df1 = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ df2 = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ ).rename(columns="{}_2".format)
+ df1.iloc[1, df1.columns.get_indexer(["A", "B"])] = np.nan
+ df = concat([df1, df2], axis=1)
+
+ # dropna=True should guarantee rows are synchronized
+ temp_hdfstore.append_to_multiple(
+ {"df1": ["A", "B"], "df2": None}, df, selector="df1", dropna=True
+ )
+ result = temp_hdfstore.select_as_multiple(["df1", "df2"])
+ expected = df.dropna()
+ tm.assert_frame_equal(result, expected, check_index_type=True)
+ tm.assert_index_equal(
+ temp_hdfstore.select("df1").index, temp_hdfstore.select("df2").index
+ )
+
+
+def test_append_to_multiple_dropna_false(temp_hdfstore):
+ df1 = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ df2 = df1.copy().rename(columns="{}_2".format)
+ df1.iloc[1, df1.columns.get_indexer(["A", "B"])] = np.nan
+ df = concat([df1, df2], axis=1)
+
+ with pd.option_context("io.hdf.dropna_table", True):
+ # dropna=False shouldn't synchronize row indexes
+ temp_hdfstore.append_to_multiple(
+ {"df1a": ["A", "B"], "df2a": None}, df, selector="df1a", dropna=False
+ )
+
+ msg = "all tables must have exactly the same nrows!"
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.select_as_multiple(["df1a", "df2a"])
+
+ assert not temp_hdfstore.select("df1a").index.equals(
+ temp_hdfstore.select("df2a").index
+ )
+
+
+def test_append_to_multiple_min_itemsize(temp_hdfstore):
+ # GH 11238
+ df = DataFrame(
+ {
+ "IX": np.arange(1, 21),
+ "Num": np.arange(1, 21),
+ "BigNum": np.arange(1, 21) * 88,
+ "Str": ["a" for _ in range(20)],
+ "LongStr": ["abcde" for _ in range(20)],
+ }
+ )
+ expected = df.iloc[[0]]
+ # Reading/writing RangeIndex info is not supported yet
+ expected.index = Index(list(range(len(expected.index))))
+
+ temp_hdfstore.append_to_multiple(
+ {
+ "index": ["IX"],
+ "nums": ["Num", "BigNum"],
+ "strs": ["Str", "LongStr"],
+ },
+ df.iloc[[0]],
+ "index",
+ min_itemsize={"Str": 10, "LongStr": 100, "Num": 2},
+ )
+ result = temp_hdfstore.select_as_multiple(["index", "nums", "strs"])
+ tm.assert_frame_equal(result, expected, check_index_type=True)
+
+
+def test_append_string_nan_rep(temp_hdfstore):
+ # GH 16300
+ df = DataFrame({"A": "a", "B": "foo"}, index=np.arange(10))
+ df_nan = df.copy()
+ df_nan.loc[0:4, :] = np.nan
+ msg = "NaN representation is too large for existing column size"
+
+ # string column too small
+ temp_hdfstore.append("sa", df["A"])
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("sa", df_nan["A"])
+
+ # nan_rep too big
+ temp_hdfstore.append("sb", df["B"], nan_rep="bars")
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("sb", df_nan["B"])
+
+ # smaller modified nan_rep
+ temp_hdfstore.append("sc", df["A"], nan_rep="n")
+ temp_hdfstore.append("sc", df_nan["A"])
+ result = temp_hdfstore["sc"]
+ expected = concat([df["A"], df_nan["A"]])
+ tm.assert_series_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_categorical.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_categorical.py
new file mode 100644
index 0000000000000000000000000000000000000000..ce39f21fb3f8c74064f1cbe69bd6b887fd43b408
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_categorical.py
@@ -0,0 +1,189 @@
+import numpy as np
+import pytest
+
+from pandas import (
+ Categorical,
+ DataFrame,
+ Series,
+ _testing as tm,
+ concat,
+ read_hdf,
+)
+
+pytestmark = [pytest.mark.single_cpu]
+
+
+def test_categorical(temp_hdfstore):
+ # Basic
+ s = Series(
+ Categorical(
+ ["a", "b", "b", "a", "a", "c"],
+ categories=["a", "b", "c", "d"],
+ ordered=False,
+ )
+ )
+ temp_hdfstore.append("s", s, format="table")
+ result = temp_hdfstore.select("s")
+ tm.assert_series_equal(s, result)
+
+ s = Series(
+ Categorical(
+ ["a", "b", "b", "a", "a", "c"],
+ categories=["a", "b", "c", "d"],
+ ordered=True,
+ )
+ )
+ temp_hdfstore.append("s_ordered", s, format="table")
+ result = temp_hdfstore.select("s_ordered")
+ tm.assert_series_equal(s, result)
+
+ df = DataFrame({"s": s, "vals": [1, 2, 3, 4, 5, 6]})
+ temp_hdfstore.append("df", df, format="table")
+ result = temp_hdfstore.select("df")
+ tm.assert_frame_equal(result, df)
+
+ # Dtypes
+ s = Series([1, 1, 2, 2, 3, 4, 5]).astype("category")
+ temp_hdfstore.append("si", s)
+ result = temp_hdfstore.select("si")
+ tm.assert_series_equal(result, s)
+
+ s = Series([1, 1, np.nan, 2, 3, 4, 5]).astype("category")
+ temp_hdfstore.append("si2", s)
+ result = temp_hdfstore.select("si2")
+ tm.assert_series_equal(result, s)
+
+ # Multiple
+ df2 = df.copy()
+ df2["s2"] = Series(list("abcdefg")).astype("category")
+ temp_hdfstore.append("df2", df2)
+ result = temp_hdfstore.select("df2")
+ tm.assert_frame_equal(result, df2)
+
+ # Make sure the metadata is OK
+ info = temp_hdfstore.info()
+ assert "/df2 " in info
+ # df2._mgr.blocks[0] and df2._mgr.blocks[2] are Categorical
+ assert "/df2/meta/values_block_0/meta" in info
+ assert "/df2/meta/values_block_2/meta" in info
+
+ # unordered
+ s = Series(
+ Categorical(
+ ["a", "b", "b", "a", "a", "c"],
+ categories=["a", "b", "c", "d"],
+ ordered=False,
+ )
+ )
+ temp_hdfstore.append("s2", s, format="table")
+ result = temp_hdfstore.select("s2")
+ tm.assert_series_equal(result, s)
+
+ # Query
+ temp_hdfstore.append("df3", df, data_columns=["s"])
+ expected = df[df.s.isin(["b", "c"])]
+ result = temp_hdfstore.select("df3", where=['s in ["b","c"]'])
+ tm.assert_frame_equal(result, expected)
+
+ expected = df[df.s.isin(["b", "c"])]
+ result = temp_hdfstore.select("df3", where=['s = ["b","c"]'])
+ tm.assert_frame_equal(result, expected)
+
+ expected = df[df.s.isin(["d"])]
+ result = temp_hdfstore.select("df3", where=['s in ["d"]'])
+ tm.assert_frame_equal(result, expected)
+
+ expected = df[df.s.isin(["f"])]
+ result = temp_hdfstore.select("df3", where=['s in ["f"]'])
+ tm.assert_frame_equal(result, expected)
+
+ # Appending with same categories is ok
+ temp_hdfstore.append("df3", df)
+
+ df = concat([df, df])
+ expected = df[df.s.isin(["b", "c"])]
+ result = temp_hdfstore.select("df3", where=['s in ["b","c"]'])
+ tm.assert_frame_equal(result, expected)
+
+ # Appending must have the same categories
+ df3 = df.copy()
+ df3["s"] = df3["s"].cat.remove_unused_categories()
+
+ msg = "cannot append a categorical with different categories to the existing"
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("df3", df3)
+
+ # Remove, and make sure meta data is removed (its a recursive
+ # removal so should be).
+ result = temp_hdfstore.select("df3/meta/s/meta")
+ assert result is not None
+ temp_hdfstore.remove("df3")
+
+ with pytest.raises(KeyError, match="'No object named df3/meta/s/meta in the file'"):
+ temp_hdfstore.select("df3/meta/s/meta")
+
+
+def test_categorical_conversion(temp_h5_path):
+ # GH13322
+ # Check that read_hdf with categorical columns doesn't return rows if
+ # where criteria isn't met.
+ obsids = ["ESP_012345_6789", "ESP_987654_3210"]
+ imgids = ["APF00006np", "APF0001imm"]
+ data = [4.3, 9.8]
+
+ # Test without categories
+ df = DataFrame({"obsids": obsids, "imgids": imgids, "data": data})
+
+ # We are expecting an empty DataFrame matching types of df
+ expected = df.iloc[[], :]
+ df.to_hdf(temp_h5_path, key="df", format="table", data_columns=True)
+ result = read_hdf(temp_h5_path, "df", where="obsids=B")
+ tm.assert_frame_equal(result, expected)
+
+ # Test with categories
+ df.obsids = df.obsids.astype("category")
+ df.imgids = df.imgids.astype("category")
+
+ # We are expecting an empty DataFrame matching types of df
+ expected = df.iloc[[], :]
+ df.to_hdf(temp_h5_path, key="df", format="table", data_columns=True)
+ result = read_hdf(temp_h5_path, "df", where="obsids=B")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_categorical_nan_only_columns(temp_h5_path):
+ # GH18413
+ # Check that read_hdf with categorical columns with NaN-only values can
+ # be read back.
+ df = DataFrame(
+ {
+ "a": ["a", "b", "c", np.nan],
+ "b": [np.nan, np.nan, np.nan, np.nan],
+ "c": [1, 2, 3, 4],
+ "d": Series([None] * 4, dtype=object),
+ }
+ )
+ df["a"] = df.a.astype("category")
+ df["b"] = df.b.astype("category")
+ df["d"] = df.b.astype("category")
+ expected = df
+ df.to_hdf(temp_h5_path, key="df", format="table", data_columns=True)
+ result = read_hdf(temp_h5_path, "df")
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("where, expected", [["q", []], ["a", ["a"]]])
+def test_convert_value(temp_h5_path, where: str, expected):
+ # GH39420
+ # Check that read_hdf with categorical columns can filter by where condition.
+ df = DataFrame({"col": ["a", "b", "s"]})
+ df.col = df.col.astype("category")
+ max_widths = {"col": 1}
+ categorical_values = sorted(df.col.unique())
+ expected = DataFrame({"col": expected})
+ expected.col = expected.col.astype("category")
+ expected.col = expected.col.cat.set_categories(categorical_values)
+
+ df.to_hdf(temp_h5_path, key="df", format="table", min_itemsize=max_widths)
+ result = read_hdf(temp_h5_path, where=f'col=="{where}"')
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_compat.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_compat.py
new file mode 100644
index 0000000000000000000000000000000000000000..4262b1031c657c7a9a976bed0cd7935694517622
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_compat.py
@@ -0,0 +1,156 @@
+from pathlib import Path
+
+import pytest
+
+from pandas.compat.numpy import np_version_gt2
+
+import pandas as pd
+import pandas._testing as tm
+from pandas.tests.io.generate_legacy_storage_files import create_dataframe_all_types
+from pandas.util.version import Version
+
+tables = pytest.importorskip("tables")
+
+
+@pytest.fixture
+def pytables_hdf5_file(temp_h5_path):
+ """
+ Use PyTables to create a simple HDF5 file.
+ """
+ table_schema = {
+ "c0": tables.Time64Col(pos=0),
+ "c1": tables.StringCol(5, pos=1),
+ "c2": tables.Int64Col(pos=2),
+ }
+
+ t0 = 1_561_105_000.0
+
+ testsamples = [
+ {"c0": t0, "c1": "aaaaa", "c2": 1},
+ {"c0": t0 + 1, "c1": "bbbbb", "c2": 2},
+ {"c0": t0 + 2, "c1": "ccccc", "c2": 10**5},
+ {"c0": t0 + 3, "c1": "ddddd", "c2": 4_294_967_295},
+ ]
+
+ objname = "pandas_test_timeseries"
+
+ with tables.open_file(temp_h5_path, mode="w") as f:
+ t = f.create_table("/", name=objname, description=table_schema)
+ for sample in testsamples:
+ for key, value in sample.items():
+ t.row[key] = value
+ t.row.append()
+
+ return temp_h5_path, objname, pd.DataFrame(testsamples)
+
+
+class TestReadPyTablesHDF5:
+ """
+ A group of tests which covers reading HDF5 files written by plain PyTables
+ (not written by pandas).
+
+ Was introduced for regression-testing issue 11188.
+ """
+
+ def test_read_complete(self, pytables_hdf5_file):
+ path, objname, df = pytables_hdf5_file
+ result = pd.read_hdf(path, key=objname)
+ expected = df
+ tm.assert_frame_equal(result, expected, check_index_type=True)
+
+ def test_read_with_start(self, pytables_hdf5_file):
+ path, objname, df = pytables_hdf5_file
+ # This is a regression test for pandas-dev/pandas/issues/11188
+ result = pd.read_hdf(path, key=objname, start=1)
+ expected = df[1:].reset_index(drop=True)
+ tm.assert_frame_equal(result, expected, check_index_type=True)
+
+ def test_read_with_stop(self, pytables_hdf5_file):
+ path, objname, df = pytables_hdf5_file
+ # This is a regression test for pandas-dev/pandas/issues/11188
+ result = pd.read_hdf(path, key=objname, stop=1)
+ expected = df[:1].reset_index(drop=True)
+ tm.assert_frame_equal(result, expected, check_index_type=True)
+
+ def test_read_with_startstop(self, pytables_hdf5_file):
+ path, objname, df = pytables_hdf5_file
+ # This is a regression test for pandas-dev/pandas/issues/11188
+ result = pd.read_hdf(path, key=objname, start=1, stop=2)
+ expected = df[1:2].reset_index(drop=True)
+ tm.assert_frame_equal(result, expected, check_index_type=True)
+
+
+_legacy_files = list(Path(__file__).parent.parent.glob("data/legacy_hdf/*/*.h5"))
+
+
+@pytest.mark.parametrize("legacy_file", _legacy_files, ids=lambda x: x.name)
+def test_legacy_files(datapath, legacy_file, using_infer_string, request):
+ legacy_version = Version(legacy_file.parent.name)
+ legacy_file = datapath(legacy_file)
+
+ if not np_version_gt2 and legacy_file.endswith("fixed.h5"):
+ # Files created for versions 2.0-3.0 used a numpy version >= 2.0, and
+ # unpickling the object dtype column fails with older numpy
+ pytest.skip("Fixed format pickle objects don't deserialize with numpy < 2.0")
+
+ result = pd.read_hdf(legacy_file)
+
+ expected = create_dataframe_all_types()
+
+ # the fixed format doesn't include categorical columns (not supported)
+ if legacy_file.endswith("fixed.h5"):
+ expected = expected.drop(
+ # columns=["categorical", "categorical_object", "categorical_int"]
+ columns=["categorical_int"]
+ )
+
+ # # object dtype columns with strings get read as `str`
+ # if using_infer_string:
+ # expected["object"] = expected["object"].astype("str")
+ # expected["object_nan"] = expected["object_nan"].astype("str")
+ # if legacy_file.endswith("table.h5"):
+ # expected["categorical_object"] = expected["categorical_object"].astype(
+ # pd.CategoricalDtype(
+ # expected["categorical_object"].cat.categories.astype("str")
+ # )
+ # )
+ # else:
+ # expected["string"] = expected["string"].astype("object")
+ # if legacy_file.endswith("table.h5"):
+ # expected["object"] = expected["object"].fillna(np.nan)
+ # expected["categorical"] = expected["categorical"].astype(
+ # pd.CategoricalDtype(
+ # expected["categorical"].cat.categories.astype(object)
+ # )
+ # )
+ # else:
+ # expected["string"] = expected["string"].fillna("nan")
+
+ if legacy_version < Version("2.2.0") or (
+ legacy_version < Version("3.0.0") and legacy_file.endswith("fixed.h5")
+ ):
+ # timedelta columns gets read as nanoseconds, resulting in buggy values
+ # (this also happened for direct roundtrips with those versions)
+ assert not result["timedelta_us"].equals(expected["timedelta_us"])
+ assert not result["timedelta_ms"].equals(expected["timedelta_ms"])
+ assert not result["timedelta_s"].equals(expected["timedelta_s"])
+ result = result.drop(columns=["timedelta_us", "timedelta_ms", "timedelta_s"])
+ expected = expected.drop(
+ columns=["timedelta_us", "timedelta_ms", "timedelta_s"]
+ )
+
+ if legacy_version < Version("2.2.0"):
+ # datetime columns gets read as nanoseconds, resulting in buggy values
+ # (this also happened for direct roundtrips with those versions)
+ assert not result["datetime_us"].equals(expected["datetime_us"])
+ assert not result["datetime_ms"].equals(expected["datetime_ms"])
+ assert not result["datetime_s"].equals(expected["datetime_s"])
+ assert not result["datetimetz_us"].equals(expected["datetimetz_us"])
+ result = result.drop(
+ columns=["datetime_us", "datetime_ms", "datetime_s", "datetimetz_us"]
+ )
+ expected = expected.drop(
+ columns=["datetime_us", "datetime_ms", "datetime_s", "datetimetz_us"]
+ )
+
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_complex.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_complex.py
new file mode 100644
index 0000000000000000000000000000000000000000..046477c6fc8ab45ae7a828d2b2cb7f601ddd8ca8
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_complex.py
@@ -0,0 +1,206 @@
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Series,
+)
+import pandas._testing as tm
+
+from pandas.io.pytables import read_hdf
+
+
+def test_complex_fixed(temp_h5_path):
+ df = DataFrame(
+ np.random.default_rng(2).random((4, 5)).astype(np.complex64),
+ index=list("abcd"),
+ columns=list("ABCDE"),
+ )
+
+ df.to_hdf(temp_h5_path, key="df")
+ reread = read_hdf(temp_h5_path, "df")
+ tm.assert_frame_equal(df, reread)
+
+ df = DataFrame(
+ np.random.default_rng(2).random((4, 5)).astype(np.complex128),
+ index=list("abcd"),
+ columns=list("ABCDE"),
+ )
+ df.to_hdf(temp_h5_path, key="df")
+ reread = read_hdf(temp_h5_path, "df")
+ tm.assert_frame_equal(df, reread)
+
+
+def test_complex_table(temp_h5_path):
+ df = DataFrame(
+ np.random.default_rng(2).random((4, 5)).astype(np.complex64),
+ index=list("abcd"),
+ columns=list("ABCDE"),
+ )
+
+ df.to_hdf(temp_h5_path, key="df", format="table")
+ reread = read_hdf(temp_h5_path, key="df")
+ tm.assert_frame_equal(df, reread)
+
+ df = DataFrame(
+ np.random.default_rng(2).random((4, 5)).astype(np.complex128),
+ index=list("abcd"),
+ columns=list("ABCDE"),
+ )
+
+ df.to_hdf(temp_h5_path, key="df", format="table", mode="w")
+ reread = read_hdf(temp_h5_path, "df")
+ tm.assert_frame_equal(df, reread)
+
+
+def test_complex_mixed_fixed(temp_h5_path):
+ complex64 = np.array(
+ [1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j], dtype=np.complex64
+ )
+ complex128 = np.array(
+ [1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j], dtype=np.complex128
+ )
+ df = DataFrame(
+ {
+ "A": [1, 2, 3, 4],
+ "B": ["a", "b", "c", "d"],
+ "C": complex64,
+ "D": complex128,
+ "E": [1.0, 2.0, 3.0, 4.0],
+ },
+ index=list("abcd"),
+ )
+ df.to_hdf(temp_h5_path, key="df")
+ reread = read_hdf(temp_h5_path, "df")
+ tm.assert_frame_equal(df, reread)
+
+
+@pytest.mark.filterwarnings(
+ "ignore:`alltrue` is deprecated as of NumPy 1.25.0:DeprecationWarning"
+)
+def test_complex_mixed_table_store_select(temp_hdfstore):
+ complex64 = np.array(
+ [1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j], dtype=np.complex64
+ )
+ complex128 = np.array(
+ [1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j], dtype=np.complex128
+ )
+ df = DataFrame(
+ {
+ "A": [1, 2, 3, 4],
+ "B": ["a", "b", "c", "d"],
+ "C": complex64,
+ "D": complex128,
+ "E": [1.0, 2.0, 3.0, 4.0],
+ },
+ index=list("abcd"),
+ )
+
+ temp_hdfstore.append("df", df, data_columns=["A", "B"])
+ result = temp_hdfstore.select("df", where="A>2")
+ tm.assert_frame_equal(df.loc[df.A > 2], result)
+
+
+@pytest.mark.filterwarnings(
+ "ignore:`alltrue` is deprecated as of NumPy 1.25.0:DeprecationWarning"
+)
+def test_complex_mixed_table_store_to_path(temp_h5_path):
+ complex64 = np.array(
+ [1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j], dtype=np.complex64
+ )
+ complex128 = np.array(
+ [1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j], dtype=np.complex128
+ )
+ df = DataFrame(
+ {
+ "A": [1, 2, 3, 4],
+ "B": ["a", "b", "c", "d"],
+ "C": complex64,
+ "D": complex128,
+ "E": [1.0, 2.0, 3.0, 4.0],
+ },
+ index=list("abcd"),
+ )
+
+ df.to_hdf(temp_h5_path, key="df", format="table")
+ reread = read_hdf(temp_h5_path, "df")
+ tm.assert_frame_equal(df, reread)
+
+
+def test_complex_across_dimensions_fixed(temp_h5_path):
+ complex128 = np.array([1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j])
+ s = Series(complex128, index=list("abcd"))
+ df = DataFrame({"A": s, "B": s})
+
+ objs = [s, df]
+ comps = [tm.assert_series_equal, tm.assert_frame_equal]
+ for obj, comp in zip(objs, comps):
+ obj.to_hdf(temp_h5_path, key="obj", format="fixed")
+ reread = read_hdf(temp_h5_path, "obj")
+ comp(obj, reread)
+
+
+def test_complex_across_dimensions(temp_h5_path):
+ complex128 = np.array([1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j])
+ s = Series(complex128, index=list("abcd"))
+ df = DataFrame({"A": s, "B": s})
+
+ df.to_hdf(temp_h5_path, key="obj", format="table")
+ reread = read_hdf(temp_h5_path, "obj")
+ tm.assert_frame_equal(df, reread)
+
+
+def test_complex_indexing_error(temp_hdfstore):
+ complex128 = np.array(
+ [1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j], dtype=np.complex128
+ )
+ df = DataFrame(
+ {"A": [1, 2, 3, 4], "B": ["a", "b", "c", "d"], "C": complex128},
+ index=list("abcd"),
+ )
+
+ msg = (
+ "Columns containing complex values can be stored "
+ "but cannot be indexed when using table format. "
+ "Either use fixed format, set index=False, "
+ "or do not include the columns containing complex "
+ "values to data_columns when initializing the table."
+ )
+
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.append("df", df, data_columns=["C"])
+
+
+def test_complex_series_error(temp_h5_path):
+ complex128 = np.array([1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j, 1.0 + 1.0j])
+ s = Series(complex128, index=list("abcd"))
+
+ msg = (
+ "Columns containing complex values can be stored "
+ "but cannot be indexed when using table format. "
+ "Either use fixed format, set index=False, "
+ "or do not include the columns containing complex "
+ "values to data_columns when initializing the table."
+ )
+
+ with pytest.raises(TypeError, match=msg):
+ s.to_hdf(temp_h5_path, key="obj", format="t")
+
+ s.to_hdf(temp_h5_path, key="obj", format="t", index=False)
+ reread = read_hdf(temp_h5_path, "obj")
+ tm.assert_series_equal(s, reread)
+
+
+def test_complex_append(temp_hdfstore):
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).standard_normal(100).astype(np.complex128),
+ "b": np.random.default_rng(2).standard_normal(100),
+ }
+ )
+
+ temp_hdfstore.append("df", df, data_columns=["b"])
+ temp_hdfstore.append("df", df)
+ result = temp_hdfstore.select("df")
+ tm.assert_frame_equal(pd.concat([df, df], axis=0), result)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_errors.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_errors.py
new file mode 100644
index 0000000000000000000000000000000000000000..7898508cde385822bf071a92c02975c6881e9fa0
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_errors.py
@@ -0,0 +1,254 @@
+import datetime
+from io import BytesIO
+import re
+import uuid
+
+import numpy as np
+import pytest
+
+from pandas import (
+ CategoricalIndex,
+ DataFrame,
+ HDFStore,
+ Index,
+ MultiIndex,
+ date_range,
+ read_hdf,
+)
+
+from pandas.io.pytables import (
+ Term,
+ _maybe_adjust_name,
+)
+
+pytestmark = [pytest.mark.single_cpu]
+
+
+def test_pass_spec_to_storer(temp_hdfstore):
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+
+ temp_hdfstore.put("df", df)
+ msg = (
+ "cannot pass a column specification when reading a Fixed format "
+ "store. this store must be selected in its entirety"
+ )
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.select("df", columns=["A"])
+ msg = (
+ "cannot pass a where specification when reading from a Fixed "
+ "format store. this store must be selected in its entirety"
+ )
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.select("df", where=["columns=A"])
+
+
+def test_table_index_incompatible_dtypes(temp_hdfstore):
+ df1 = DataFrame({"a": [1, 2, 3]})
+ df2 = DataFrame(
+ {"a": [4, 5, 6]}, index=date_range("1/1/2000", periods=3, unit="ns")
+ )
+
+ temp_hdfstore.put("frame", df1, format="table")
+ msg = re.escape("incompatible kind in col [integer - datetime64[ns]]")
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.put("frame", df2, format="table", append=True)
+
+
+def test_unimplemented_dtypes_table_columns(temp_hdfstore):
+ dtypes = [("date", datetime.date(2001, 1, 2))]
+
+ # currently not supported dtypes ####
+ for n, f in dtypes:
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ df[n] = f
+ msg = re.escape(f"[{n}] is not implemented as a table column")
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.append(f"df1_{n}", df)
+
+
+def test_unimplemented_dtypes_table_columns2(temp_hdfstore):
+ # frame
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ df["obj1"] = "foo"
+ df["obj2"] = "bar"
+ df["datetime1"] = datetime.date(2001, 1, 2)
+ df = df._consolidate()
+
+ # this fails because we have a date in the object block......
+ msg = "|".join(
+ [
+ re.escape(
+ "Cannot serialize the column [datetime1]\nbecause its data "
+ "contents are not [string] but [date] object dtype"
+ ),
+ re.escape("[date] is not implemented as a table column"),
+ ]
+ )
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.append("df_unimplemented", df)
+
+
+def test_invalid_terms(temp_hdfstore):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B", unit="ns"),
+ )
+ df["string"] = "foo"
+ df.loc[df.index[0:4], "string"] = "bar"
+
+ temp_hdfstore.put("df", df, format="table")
+
+ # some invalid terms
+ msg = re.escape("__init__() missing 1 required positional argument: 'where'")
+ with pytest.raises(TypeError, match=msg):
+ Term()
+
+ # more invalid
+ msg = re.escape(
+ "cannot process expression [df.index[3]], "
+ "[2000-01-06 00:00:00] is not a valid condition"
+ )
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.select("df", "df.index[3]")
+
+ msg = "invalid syntax"
+ with pytest.raises(SyntaxError, match=msg):
+ temp_hdfstore.select("df", "index>")
+
+
+def test_invalid_terms_from_docs(temp_h5_path):
+ # from the docs
+ dfq = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=list("ABCD"),
+ index=date_range("20130101", periods=10, unit="ns"),
+ )
+ dfq.to_hdf(temp_h5_path, key="dfq", format="table", data_columns=True)
+
+ # check ok
+ read_hdf(
+ temp_h5_path, "dfq", where="index>Timestamp('20130104') & columns=['A', 'B']"
+ )
+ read_hdf(temp_h5_path, "dfq", where="A>0 or C>0")
+
+
+def test_invalid_terms_reference(temp_h5_path):
+ # catch the invalid reference
+ dfq = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=list("ABCD"),
+ index=date_range("20130101", periods=10, unit="ns"),
+ )
+ dfq.to_hdf(temp_h5_path, key="dfq", format="table")
+
+ msg = (
+ r"The passed where expression: A>0 or C>0\n\s*"
+ r"contains an invalid variable reference\n\s*"
+ r"all of the variable references must be a reference to\n\s*"
+ r"an axis \(e.g. 'index' or 'columns'\), or a data_column\n\s*"
+ r"The currently defined references are: index,columns\n"
+ )
+ with pytest.raises(ValueError, match=msg):
+ read_hdf(temp_h5_path, "dfq", where="A>0 or C>0")
+
+
+def test_append_with_diff_col_name_types_raises_value_error(temp_hdfstore):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 1)))
+ df2 = DataFrame({"a": np.random.default_rng(2).standard_normal(10)})
+ df3 = DataFrame({(1, 2): np.random.default_rng(2).standard_normal(10)})
+ df4 = DataFrame({("1", 2): np.random.default_rng(2).standard_normal(10)})
+ df5 = DataFrame({("1", 2, object): np.random.default_rng(2).standard_normal(10)})
+
+ name = "df_diff_valerror"
+ temp_hdfstore.append(name, df)
+
+ for d in (df2, df3, df4, df5):
+ msg = re.escape(
+ "cannot match existing table structure for [0] on appending data"
+ )
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append(name, d)
+
+
+def test_invalid_complib(temp_h5_path):
+ df = DataFrame(
+ np.random.default_rng(2).random((4, 5)),
+ index=list("abcd"),
+ columns=list("ABCDE"),
+ )
+ msg = r"complib only supports \[.*\] compression."
+ with pytest.raises(ValueError, match=msg):
+ df.to_hdf(temp_h5_path, key="df", complib="foolib")
+
+
+@pytest.mark.parametrize(
+ "idx",
+ [
+ date_range("2019", freq="D", periods=3, tz="UTC", unit="ns"),
+ CategoricalIndex(list("abc")),
+ ],
+)
+def test_to_hdf_multiindex_extension_dtype(idx, temp_h5_path):
+ # GH 7775
+ mi = MultiIndex.from_arrays([idx, idx])
+ df = DataFrame(0, index=mi, columns=["a"])
+ with pytest.raises(NotImplementedError, match="Saving a MultiIndex"):
+ df.to_hdf(temp_h5_path, key="df")
+
+
+def test_unsuppored_hdf_file_error(datapath):
+ # GH 9539
+ data_path = datapath("io", "data", "legacy_hdf/incompatible_dataset.h5")
+ message = (
+ r"Dataset\(s\) incompatible with Pandas data types, "
+ "not table, or no datasets found in HDF5 file."
+ )
+
+ with pytest.raises(ValueError, match=message):
+ read_hdf(data_path)
+
+
+def test_read_hdf_errors(temp_h5_path):
+ df = DataFrame(
+ np.random.default_rng(2).random((4, 5)),
+ index=list("abcd"),
+ columns=list("ABCDE"),
+ )
+
+ msg = r"File [\S]* does not exist"
+ with pytest.raises(OSError, match=msg):
+ read_hdf(f"{uuid.uuid4()}.h5", "key")
+
+ df.to_hdf(temp_h5_path, key="df")
+ store = HDFStore(temp_h5_path, mode="r")
+ store.close()
+
+ msg = "The HDFStore must be open for reading."
+ with pytest.raises(OSError, match=msg):
+ read_hdf(store, "df")
+
+
+def test_read_hdf_generic_buffer_errors():
+ msg = "Support for generic buffers has not been implemented."
+ with pytest.raises(NotImplementedError, match=msg):
+ read_hdf(BytesIO(b""), "df")
+
+
+@pytest.mark.parametrize("bad_version", [(1, 2), (1,), [], "12", "123"])
+def test_maybe_adjust_name_bad_version_raises(bad_version):
+ msg = "Version is incorrect, expected sequence of 3 integers"
+ with pytest.raises(ValueError, match=msg):
+ _maybe_adjust_name("values_block_0", version=bad_version)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_file_handling.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_file_handling.py
new file mode 100644
index 0000000000000000000000000000000000000000..d81c545de262286832a27acf547e2d7c3aca811c
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_file_handling.py
@@ -0,0 +1,489 @@
+import os
+import uuid
+
+import numpy as np
+import pytest
+
+from pandas.compat import (
+ is_platform_linux,
+ is_platform_little_endian,
+ is_platform_mac,
+)
+from pandas.errors import (
+ ClosedFileError,
+ PossibleDataLossError,
+)
+
+from pandas import (
+ DataFrame,
+ HDFStore,
+ Index,
+ Series,
+ _testing as tm,
+ date_range,
+ read_hdf,
+)
+
+from pandas.io import pytables
+from pandas.io.pytables import Term
+
+tables = pytest.importorskip("tables")
+pytestmark = [pytest.mark.single_cpu]
+
+
+@pytest.mark.parametrize("mode", ["r", "r+", "a", "w"])
+def test_mode(temp_h5_path, mode, using_infer_string):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ msg = r"[\S]* does not exist"
+ doesnt_exist = f"{uuid.uuid4()}.h5"
+
+ # constructor
+ if mode in ["r", "r+"]:
+ with pytest.raises(OSError, match=msg):
+ HDFStore(doesnt_exist, mode=mode)
+
+ else:
+ with HDFStore(temp_h5_path, mode=mode) as store:
+ assert store._handle.mode == mode
+
+ # context
+ if mode in ["r", "r+"]:
+ with pytest.raises(OSError, match=msg):
+ with HDFStore(doesnt_exist, mode=mode) as store:
+ pass
+ else:
+ with HDFStore(temp_h5_path, mode=mode) as store:
+ assert store._handle.mode == mode
+
+ # conv write
+ if mode in ["r", "r+"]:
+ with pytest.raises(OSError, match=msg):
+ df.to_hdf(doesnt_exist, key="df", mode=mode)
+ df.to_hdf(temp_h5_path, key="df", mode="w")
+ else:
+ df.to_hdf(temp_h5_path, key="df", mode=mode)
+
+ # conv read
+ if mode in ["w"]:
+ msg = (
+ "mode w is not allowed while performing a read. "
+ r"Allowed modes are r, r\+ and a."
+ )
+ with pytest.raises(ValueError, match=msg):
+ read_hdf(temp_h5_path, "df", mode=mode)
+ else:
+ result = read_hdf(temp_h5_path, "df", mode=mode)
+ if using_infer_string:
+ df.columns = df.columns.astype("str")
+ tm.assert_frame_equal(result, df)
+
+
+def test_default_mode(temp_h5_path, using_infer_string):
+ # read_hdf uses default mode
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ df.to_hdf(temp_h5_path, key="df", mode="w")
+ result = read_hdf(temp_h5_path, "df")
+ expected = df.copy()
+ if using_infer_string:
+ expected.columns = expected.columns.astype("str")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_reopen_handle(temp_h5_path):
+ store = HDFStore(temp_h5_path, mode="a")
+ store["a"] = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+
+ msg = (
+ r"Re-opening the file \[[\S]*\] with mode \[a\] will delete the "
+ "current file!"
+ )
+ # invalid mode change
+ with pytest.raises(PossibleDataLossError, match=msg):
+ store.open("w")
+
+ store.close()
+ assert not store.is_open
+
+ # truncation ok here
+ store.open("w")
+ assert store.is_open
+ assert len(store) == 0
+ store.close()
+ assert not store.is_open
+
+ store = HDFStore(temp_h5_path, mode="a")
+ store["a"] = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+
+ # reopen as read
+ store.open("r")
+ assert store.is_open
+ assert len(store) == 1
+ assert store._mode == "r"
+ store.close()
+ assert not store.is_open
+
+ # reopen as append
+ store.open("a")
+ assert store.is_open
+ assert len(store) == 1
+ assert store._mode == "a"
+ store.close()
+ assert not store.is_open
+
+ # reopen as append (again)
+ store.open("a")
+ assert store.is_open
+ assert len(store) == 1
+ assert store._mode == "a"
+ store.close()
+ assert not store.is_open
+
+
+def test_open_args(using_infer_string):
+ not_written = f"{uuid.uuid4()}.h5"
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+
+ # create an in memory store
+ store = HDFStore(
+ not_written, mode="a", driver="H5FD_CORE", driver_core_backing_store=0
+ )
+ store["df"] = df
+ store.append("df2", df)
+
+ expected = df.copy()
+ if using_infer_string:
+ expected.index = expected.index.astype("str")
+ expected.columns = expected.columns.astype("str")
+
+ tm.assert_frame_equal(store["df"], expected)
+ tm.assert_frame_equal(store["df2"], expected)
+
+ store.close()
+
+ # the file should not have actually been written
+ assert not os.path.exists(not_written)
+
+
+def test_flush(temp_h5_path):
+ with HDFStore(temp_h5_path, mode="w") as store:
+ store["a"] = Series(range(5))
+ store.flush()
+ store.flush(fsync=True)
+
+
+def test_complibs_default_settings(temp_h5_path, using_infer_string):
+ # GH15943
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+
+ # Set complevel and check if complib is automatically set to
+ # default value
+ df.to_hdf(temp_h5_path, key="df", complevel=9)
+ result = read_hdf(temp_h5_path, "df")
+ expected = df.copy()
+ if using_infer_string:
+ expected.index = expected.index.astype("str")
+ expected.columns = expected.columns.astype("str")
+ tm.assert_frame_equal(result, expected)
+
+ with tables.open_file(temp_h5_path, mode="r") as h5file:
+ for node in h5file.walk_nodes(where="/df", classname="Leaf"):
+ assert node.filters.complevel == 9
+ assert node.filters.complib == "zlib"
+
+ # Set complib and check to see if compression is disabled
+ df.to_hdf(temp_h5_path, key="df", complib="zlib")
+ result = read_hdf(temp_h5_path, "df")
+ expected = df.copy()
+ if using_infer_string:
+ expected.index = expected.index.astype("str")
+ expected.columns = expected.columns.astype("str")
+ tm.assert_frame_equal(result, expected)
+
+ with tables.open_file(temp_h5_path, mode="r") as h5file:
+ for node in h5file.walk_nodes(where="/df", classname="Leaf"):
+ assert node.filters.complevel == 0
+ assert node.filters.complib is None
+
+ # Check if not setting complib or complevel results in no compression
+ df.to_hdf(temp_h5_path, key="df")
+ result = read_hdf(temp_h5_path, "df")
+ expected = df.copy()
+ if using_infer_string:
+ expected.index = expected.index.astype("str")
+ expected.columns = expected.columns.astype("str")
+ tm.assert_frame_equal(result, expected)
+
+ with tables.open_file(temp_h5_path, mode="r") as h5file:
+ for node in h5file.walk_nodes(where="/df", classname="Leaf"):
+ assert node.filters.complevel == 0
+ assert node.filters.complib is None
+
+
+def test_complibs_default_settings_override(temp_h5_path):
+ # Check if file-defaults can be overridden on a per table basis
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ store = HDFStore(temp_h5_path)
+ store.append("dfc", df, complevel=9, complib="blosc")
+ store.append("df", df)
+ store.close()
+
+ with tables.open_file(temp_h5_path, mode="r") as h5file:
+ for node in h5file.walk_nodes(where="/df", classname="Leaf"):
+ assert node.filters.complevel == 0
+ assert node.filters.complib is None
+ for node in h5file.walk_nodes(where="/dfc", classname="Leaf"):
+ assert node.filters.complevel == 9
+ assert node.filters.complib == "blosc"
+
+
+@pytest.mark.parametrize("lvl", range(10))
+@pytest.mark.parametrize("lib", tables.filters.all_complibs)
+@pytest.mark.filterwarnings("ignore:object name is not a valid")
+def test_complibs(tmp_path, lvl, lib, request):
+ # GH14478
+ if is_platform_linux() and lib == "blosc2" and lvl != 0:
+ request.applymarker(pytest.mark.xfail(reason=f"Fails for {lib} on Linux"))
+ df = DataFrame(
+ np.ones((30, 4)), columns=list("ABCD"), index=np.arange(30).astype(np.str_)
+ )
+
+ # Remove lzo if its not available on this platform
+ if not tables.which_lib_version("lzo"):
+ pytest.skip("lzo not available")
+ # Remove bzip2 if its not available on this platform
+ if not tables.which_lib_version("bzip2"):
+ pytest.skip("bzip2 not available")
+
+ tmpfile = tmp_path / f"{lvl}_{lib}.h5"
+ gname = f"{lvl}_{lib}"
+
+ # Write and read file to see if data is consistent
+ df.to_hdf(tmpfile, key=gname, complib=lib, complevel=lvl)
+ result = read_hdf(tmpfile, gname)
+ tm.assert_frame_equal(result, df)
+
+ is_mac = is_platform_mac()
+
+ # Open file and check metadata for correct amount of compression
+ with tables.open_file(tmpfile, mode="r") as h5table:
+ for node in h5table.walk_nodes(where="/" + gname, classname="Leaf"):
+ assert node.filters.complevel == lvl
+ if lvl == 0:
+ assert node.filters.complib is None
+ elif is_mac and lib == "blosc2":
+ res = node.filters.complib
+ assert res in [lib, "blosc2:blosclz"], res
+ else:
+ assert node.filters.complib == lib
+
+
+@pytest.mark.skipif(
+ not is_platform_little_endian(), reason="reason platform is not little endian"
+)
+def test_encoding(temp_hdfstore):
+ df = DataFrame({"A": "foo", "B": "bar"}, index=range(5))
+ df.loc[2, "A"] = np.nan
+ df.loc[3, "B"] = np.nan
+ temp_hdfstore.append("df", df, encoding="ascii")
+ tm.assert_frame_equal(temp_hdfstore["df"], df)
+
+ expected = df.reindex(columns=["A"])
+ result = temp_hdfstore.select("df", Term("columns=A", encoding="ascii"))
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "val",
+ [
+ [b"E\xc9, 17", b"", b"a", b"b", b"c"],
+ [b"E\xc9, 17", b"a", b"b", b"c"],
+ [b"EE, 17", b"", b"a", b"b", b"c"],
+ [b"E\xc9, 17", b"\xf8\xfc", b"a", b"b", b"c"],
+ [b"", b"a", b"b", b"c"],
+ [b"\xf8\xfc", b"a", b"b", b"c"],
+ [b"A\xf8\xfc", b"", b"a", b"b", b"c"],
+ [np.nan, b"", b"b", b"c"],
+ [b"A\xf8\xfc", np.nan, b"", b"b", b"c"],
+ ],
+)
+@pytest.mark.parametrize("dtype", ["category", None])
+def test_latin_encoding(temp_h5_path, dtype, val):
+ enc = "latin-1"
+ nan_rep = ""
+ key = "data"
+
+ val = [x.decode(enc) if isinstance(x, bytes) else x for x in val]
+ ser = Series(val, dtype=dtype)
+
+ ser.to_hdf(temp_h5_path, key=key, format="table", encoding=enc, nan_rep=nan_rep)
+ retr = read_hdf(temp_h5_path, key)
+
+ # TODO:(3.0): once Categorical replace deprecation is enforced,
+ # we may be able to re-simplify the construction of s_nan
+ if dtype == "category":
+ if nan_rep in ser.cat.categories:
+ s_nan = ser.cat.remove_categories([nan_rep])
+ else:
+ s_nan = ser
+ else:
+ s_nan = ser.replace(nan_rep, np.nan)
+
+ tm.assert_series_equal(s_nan, retr)
+
+
+def test_multiple_open_close(temp_h5_path):
+ # gh-4409: open & close multiple times
+
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ df.to_hdf(temp_h5_path, key="df", mode="w", format="table")
+
+ # single
+ store = HDFStore(temp_h5_path)
+ assert "CLOSED" not in store.info()
+ assert store.is_open
+
+ store.close()
+ assert "CLOSED" in store.info()
+ assert not store.is_open
+
+ if pytables._table_file_open_policy_is_strict:
+ # multiples
+ store1 = HDFStore(temp_h5_path)
+ msg = (
+ r"The file [\S]* is already opened\. Please close it before "
+ r"reopening in write mode\."
+ )
+ with pytest.raises(ValueError, match=msg):
+ HDFStore(temp_h5_path)
+
+ store1.close()
+ else:
+ # multiples
+ store1 = HDFStore(temp_h5_path)
+ store2 = HDFStore(temp_h5_path)
+
+ assert "CLOSED" not in store1.info()
+ assert "CLOSED" not in store2.info()
+ assert store1.is_open
+ assert store2.is_open
+
+ store1.close()
+ assert "CLOSED" in store1.info()
+ assert not store1.is_open
+ assert "CLOSED" not in store2.info()
+ assert store2.is_open
+
+ store2.close()
+ assert "CLOSED" in store1.info()
+ assert "CLOSED" in store2.info()
+ assert not store1.is_open
+ assert not store2.is_open
+
+ # nested close
+ store = HDFStore(temp_h5_path, mode="w")
+ store.append("df", df)
+
+ store2 = HDFStore(temp_h5_path)
+ store2.append("df2", df)
+ store2.close()
+ assert "CLOSED" in store2.info()
+ assert not store2.is_open
+
+ store.close()
+ assert "CLOSED" in store.info()
+ assert not store.is_open
+
+ # double closing
+ store = HDFStore(temp_h5_path, mode="w")
+ store.append("df", df)
+
+ store2 = HDFStore(temp_h5_path)
+ store.close()
+ assert "CLOSED" in store.info()
+ assert not store.is_open
+
+ store2.close()
+ assert "CLOSED" in store2.info()
+ assert not store2.is_open
+
+ # ops on a closed store
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ df.to_hdf(temp_h5_path, key="df", mode="w", format="table")
+
+ store = HDFStore(temp_h5_path)
+ store.close()
+
+ msg = r"[\S]* file is not open!"
+ with pytest.raises(ClosedFileError, match=msg):
+ store.keys()
+
+ with pytest.raises(ClosedFileError, match=msg):
+ "df" in store
+
+ with pytest.raises(ClosedFileError, match=msg):
+ len(store)
+
+ with pytest.raises(ClosedFileError, match=msg):
+ store["df"]
+
+ with pytest.raises(ClosedFileError, match=msg):
+ store.select("df")
+
+ with pytest.raises(ClosedFileError, match=msg):
+ store.get("df")
+
+ with pytest.raises(ClosedFileError, match=msg):
+ store.append("df2", df)
+
+ with pytest.raises(ClosedFileError, match=msg):
+ store.put("df3", df)
+
+ with pytest.raises(ClosedFileError, match=msg):
+ store.get_storer("df2")
+
+ with pytest.raises(ClosedFileError, match=msg):
+ store.remove("df2")
+
+ with pytest.raises(ClosedFileError, match=msg):
+ store.select("df")
+
+ msg = "'HDFStore' object has no attribute 'df'"
+ with pytest.raises(AttributeError, match=msg):
+ store.df
+
+
+def test_fspath(temp_h5_path):
+ with HDFStore(temp_h5_path) as store:
+ assert os.fspath(store) == str(temp_h5_path)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_keys.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_keys.py
new file mode 100644
index 0000000000000000000000000000000000000000..d0f22c332d0b555eca78ef3337768967c2e38477
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_keys.py
@@ -0,0 +1,80 @@
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ HDFStore,
+ Index,
+ Series,
+ date_range,
+)
+
+tables = pytest.importorskip("tables")
+
+pytestmark = [pytest.mark.single_cpu]
+
+
+def test_keys(temp_hdfstore):
+ temp_hdfstore["a"] = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ temp_hdfstore["b"] = Series(
+ range(10), dtype="float64", index=[f"i_{i}" for i in range(10)]
+ )
+ temp_hdfstore["c"] = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+
+ assert len(temp_hdfstore) == 3
+ expected = {"/a", "/b", "/c"}
+ assert set(temp_hdfstore.keys()) == expected
+ assert set(temp_hdfstore) == expected
+
+
+def test_non_pandas_keys(temp_h5_path):
+ class Table1(tables.IsDescription):
+ value1 = tables.Float32Col()
+
+ class Table2(tables.IsDescription):
+ value2 = tables.Float32Col()
+
+ class Table3(tables.IsDescription):
+ value3 = tables.Float32Col()
+
+ with tables.open_file(temp_h5_path, mode="w") as h5file:
+ group = h5file.create_group("/", "group")
+ h5file.create_table(group, "table1", Table1, "Table 1")
+ h5file.create_table(group, "table2", Table2, "Table 2")
+ h5file.create_table(group, "table3", Table3, "Table 3")
+ with HDFStore(temp_h5_path) as store:
+ assert len(store.keys(include="native")) == 3
+ expected = {"/group/table1", "/group/table2", "/group/table3"}
+ assert set(store.keys(include="native")) == expected
+ assert set(store.keys(include="pandas")) == set()
+ for name in expected:
+ df = store.get(name)
+ assert len(df.columns) == 1
+
+
+def test_keys_illegal_include_keyword_value(temp_hdfstore):
+ with pytest.raises(
+ ValueError,
+ match="`include` should be either 'pandas' or 'native' but is 'illegal'",
+ ):
+ temp_hdfstore.keys(include="illegal")
+
+
+def test_keys_ignore_hdf_softlink(temp_hdfstore):
+ # GH 20523
+ # Puts a softlink into HDF file and rereads
+ df = DataFrame({"A": range(5), "B": range(5)})
+ temp_hdfstore.put("df", df)
+
+ assert temp_hdfstore.keys() == ["/df"]
+
+ temp_hdfstore._handle.create_soft_link(temp_hdfstore._handle.root, "symlink", "df")
+
+ # Should ignore the softlink
+ assert temp_hdfstore.keys() == ["/df"]
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_put.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_put.py
new file mode 100644
index 0000000000000000000000000000000000000000..125a81abd33b83a7c2cf1fbf61028079fd2f063e
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_put.py
@@ -0,0 +1,405 @@
+import re
+
+import numpy as np
+import pytest
+
+from pandas._libs.tslibs import Timestamp
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ HDFStore,
+ Index,
+ MultiIndex,
+ Series,
+ _testing as tm,
+ concat,
+ date_range,
+)
+from pandas.util import _test_decorators as td
+
+pytestmark = [pytest.mark.single_cpu]
+
+
+def test_format_type(temp_hdfstore):
+ df = DataFrame({"A": [1, 2]})
+ temp_hdfstore.put("a", df, format="fixed")
+ temp_hdfstore.put("b", df, format="table")
+
+ assert temp_hdfstore.get_storer("a").format_type == "fixed"
+ assert temp_hdfstore.get_storer("b").format_type == "table"
+
+
+def test_format_kwarg_in_constructor(temp_h5_path):
+ # GH 13291
+
+ msg = "format is not a defined argument for HDFStore"
+
+ with pytest.raises(ValueError, match=msg):
+ HDFStore(temp_h5_path, format="table")
+
+
+def test_api_default_format(temp_hdfstore):
+ # default_format option
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+
+ with pd.option_context("io.hdf.default_format", "fixed"):
+ temp_hdfstore.put("df", df)
+ assert not temp_hdfstore.get_storer("df").is_table
+
+ msg = "Can only append to Tables"
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("df2", df)
+
+ with pd.option_context("io.hdf.default_format", "table"):
+ temp_hdfstore.remove("df")
+ temp_hdfstore.put("df", df)
+ assert temp_hdfstore.get_storer("df").is_table
+
+ temp_hdfstore.append("df2", df)
+ assert temp_hdfstore.get_storer("df").is_table
+
+
+def test_api_default_format_path(temp_h5_path):
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+
+ with pd.option_context("io.hdf.default_format", "fixed"):
+ df.to_hdf(temp_h5_path, key="df")
+ with HDFStore(temp_h5_path) as store:
+ assert not store.get_storer("df").is_table
+ msg = "Can only append to Tables"
+ with pytest.raises(ValueError, match=msg):
+ df.to_hdf(temp_h5_path, key="df2", append=True)
+
+ with pd.option_context("io.hdf.default_format", "table"):
+ df.to_hdf(temp_h5_path, key="df3")
+ with HDFStore(temp_h5_path) as store:
+ assert store.get_storer("df3").is_table
+ df.to_hdf(temp_h5_path, key="df4", append=True)
+ with HDFStore(temp_h5_path) as store:
+ assert store.get_storer("df4").is_table
+
+
+def test_put(temp_hdfstore):
+ store = temp_hdfstore
+ ts = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((20, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=20, freq="B"),
+ )
+ store["a"] = ts
+ store["b"] = df[:10]
+ store["foo/bar/bah"] = df[:10]
+ store["foo"] = df[:10]
+ store["/foo"] = df[:10]
+ store.put("c", df[:10], format="table")
+
+ # not OK, not a table
+ msg = "Can only append to Tables"
+ with pytest.raises(ValueError, match=msg):
+ store.put("b", df[10:], append=True)
+
+ # node does not currently exist, test _is_table_type returns False
+ # in this case
+ with pytest.raises(ValueError, match=msg):
+ store.put("f", df[10:], append=True)
+
+ # can't put to a table (use append instead)
+ with pytest.raises(ValueError, match=msg):
+ store.put("c", df[10:], append=True)
+
+ # overwrite table
+ store.put("c", df[:10], format="table", append=False)
+ tm.assert_frame_equal(df[:10], store["c"])
+
+
+def test_put_string_index(temp_hdfstore):
+ store = temp_hdfstore
+ index = Index([f"I am a very long string index: {i}" for i in range(20)])
+ s = Series(np.arange(20), index=index)
+ df = DataFrame({"A": s, "B": s})
+
+ store["a"] = s
+ tm.assert_series_equal(store["a"], s)
+
+ store["b"] = df
+ tm.assert_frame_equal(store["b"], df)
+
+ # mixed length
+ index = Index(
+ ["abcdefghijklmnopqrstuvwxyz1234567890"]
+ + [f"I am a very long string index: {i}" for i in range(20)]
+ )
+ s = Series(np.arange(21), index=index)
+ df = DataFrame({"A": s, "B": s})
+ store["a"] = s
+ tm.assert_series_equal(store["a"], s)
+
+ store["b"] = df
+ tm.assert_frame_equal(store["b"], df)
+
+
+def test_put_compression(temp_hdfstore):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+
+ temp_hdfstore.put("c", df, format="table", complib="zlib")
+ tm.assert_frame_equal(temp_hdfstore["c"], df)
+
+ # can't compress if format='fixed'
+ msg = "Compression not supported on Fixed format stores"
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.put("b", df, format="fixed", complib="zlib")
+
+
+@td.skip_if_windows
+def test_put_compression_blosc(temp_hdfstore):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ # can't compress if format='fixed'
+ msg = "Compression not supported on Fixed format stores"
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.put("b", df, format="fixed", complib="blosc")
+
+ temp_hdfstore.put("c", df, format="table", complib="blosc")
+ tm.assert_frame_equal(temp_hdfstore["c"], df)
+
+
+def test_put_datetime_ser(temp_hdfstore, performance_warning, using_infer_string):
+ # https://github.com/pandas-dev/pandas/pull/60663
+ ser = Series(3 * [Timestamp("20010102").as_unit("ns")])
+ temp_hdfstore.put("ser", ser)
+ expected = ser.copy()
+ result = temp_hdfstore.get("ser")
+ tm.assert_series_equal(result, expected)
+
+
+def test_put_mixed_type(temp_hdfstore, performance_warning, using_infer_string):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ df["obj1"] = "foo"
+ df["obj2"] = "bar"
+ df["bool1"] = df["A"] > 0
+ df["bool2"] = df["B"] > 0
+ df["bool3"] = True
+ df["int1"] = 1
+ df["int2"] = 2
+ df["timestamp1"] = Timestamp("20010102").as_unit("ns")
+ df["timestamp2"] = Timestamp("20010103").as_unit("ns")
+ df["datetime1"] = Timestamp("20010102").as_unit("ns")
+ df["datetime2"] = Timestamp("20010103").as_unit("ns")
+ df.loc[df.index[3:6], ["obj1"]] = np.nan
+ df = df._consolidate()
+
+ warning = None if using_infer_string else performance_warning
+ with tm.assert_produces_warning(warning):
+ temp_hdfstore.put("df", df)
+
+ expected = temp_hdfstore.get("df")
+ tm.assert_frame_equal(expected, df)
+
+
+def test_put_str_frame(temp_hdfstore, performance_warning, string_dtype_arguments):
+ # https://github.com/pandas-dev/pandas/pull/60663
+ dtype = pd.StringDtype(*string_dtype_arguments)
+ df = DataFrame({"a": pd.array(["x", pd.NA, "y"], dtype=dtype)})
+
+ temp_hdfstore.put("df", df)
+ expected_dtype = "str" if dtype.na_value is np.nan else "string"
+ expected = df.astype(expected_dtype)
+ result = temp_hdfstore.get("df")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_put_str_series(temp_hdfstore, performance_warning, string_dtype_arguments):
+ # https://github.com/pandas-dev/pandas/pull/60663
+ dtype = pd.StringDtype(*string_dtype_arguments)
+ ser = Series(["x", pd.NA, "y"], dtype=dtype)
+
+ temp_hdfstore.put("ser", ser)
+ expected_dtype = "str" if dtype.na_value is np.nan else "string"
+ expected = ser.astype(expected_dtype)
+ result = temp_hdfstore.get("ser")
+ tm.assert_series_equal(result, expected)
+
+
+def test_put_str_frame_complevel(temp_hdfstore, string_dtype_arguments):
+ # GH#64180 - writing StringDtype columns to HDFStore with fixed format + complevel
+ dtype = pd.StringDtype(*string_dtype_arguments)
+ df = DataFrame({"a": pd.array(["x", pd.NA, "y"], dtype=dtype), "b": [1, 2, 3]})
+ temp_hdfstore.put("df", df, complevel=1)
+ expected_dtype = "str" if dtype.na_value is np.nan else "string"
+ expected = df.copy()
+ expected["a"] = expected["a"].astype(expected_dtype)
+ result = temp_hdfstore.get("df")
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("format", ["table", "fixed"])
+@pytest.mark.parametrize(
+ "index",
+ [
+ Index([str(i) for i in range(10)]),
+ Index(np.arange(10, dtype=float)),
+ Index(np.arange(10)),
+ date_range("2020-01-01", periods=10),
+ pd.period_range("2020-01-01", periods=10),
+ ],
+)
+def test_store_index_types(temp_hdfstore, format, index):
+ # GH5386
+ # test storing various index types
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)),
+ columns=list("AB"),
+ index=index,
+ )
+ temp_hdfstore.put("df", df, format=format)
+ tm.assert_frame_equal(df, temp_hdfstore["df"])
+
+
+def test_column_multiindex(temp_hdfstore, using_infer_string):
+ # GH 4710
+ # recreate multi-indexes properly
+
+ index = MultiIndex.from_tuples(
+ [("A", "a"), ("A", "b"), ("B", "a"), ("B", "b")], names=["first", "second"]
+ )
+ df = DataFrame(np.arange(12).reshape(3, 4), columns=index)
+ expected = df.set_axis(df.index.to_numpy())
+
+ temp_hdfstore.put("df", df)
+ tm.assert_frame_equal(
+ temp_hdfstore["df"], expected, check_index_type=True, check_column_type=True
+ )
+
+ temp_hdfstore.put("df1", df, format="table")
+ tm.assert_frame_equal(
+ temp_hdfstore["df1"], expected, check_index_type=True, check_column_type=True
+ )
+
+ msg = re.escape("cannot use a multi-index on axis [1] with data_columns ['A']")
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.put("df2", df, format="table", data_columns=["A"])
+ msg = re.escape("cannot use a multi-index on axis [1] with data_columns True")
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.put("df3", df, format="table", data_columns=True)
+
+
+def test_column_multiindex_existing(temp_hdfstore, using_infer_string):
+ # appending multi-column on existing table (see GH 6167)
+
+ index = MultiIndex.from_tuples(
+ [("A", "a"), ("A", "b"), ("B", "a"), ("B", "b")], names=["first", "second"]
+ )
+ df = DataFrame(np.arange(12).reshape(3, 4), columns=index)
+ temp_hdfstore.append("df2", df)
+ temp_hdfstore.append("df2", df)
+
+ tm.assert_frame_equal(temp_hdfstore["df2"], concat((df, df)))
+
+
+def test_column_multiindex_non_index_axes(temp_hdfstore, using_infer_string):
+ df = DataFrame(np.arange(12).reshape(3, 4), columns=Index(list("ABCD"), name="foo"))
+ expected = df.set_axis(df.index.to_numpy())
+
+ temp_hdfstore.put("df1", df, format="table")
+ tm.assert_frame_equal(
+ temp_hdfstore["df1"], expected, check_index_type=True, check_column_type=True
+ )
+
+
+def test_store_multiindex(temp_hdfstore):
+ # validate multi-index names
+ # GH 5527
+
+ def make_index(names=None):
+ dti = date_range("2013-12-01", "2013-12-02")
+ mi = MultiIndex.from_product([dti, range(2), range(3)], names=names)
+ return mi
+
+ # no names
+ df = DataFrame(np.zeros((12, 2)), columns=["a", "b"], index=make_index())
+ temp_hdfstore.append("df", df)
+ tm.assert_frame_equal(temp_hdfstore.select("df"), df)
+
+ # partial names
+ temp_hdfstore.remove("df")
+ df = DataFrame(
+ np.zeros((12, 2)),
+ columns=["a", "b"],
+ index=make_index(["date", None, None]),
+ )
+ temp_hdfstore.append("df", df)
+ tm.assert_frame_equal(temp_hdfstore.select("df"), df)
+
+ # series
+ ser = Series(np.zeros(12), index=make_index(["date", None, None]))
+ temp_hdfstore.append("ser", ser)
+ xp = Series(np.zeros(12), index=make_index(["date", "level_1", "level_2"]))
+ tm.assert_series_equal(temp_hdfstore.select("ser"), xp)
+
+ # dup with column
+ temp_hdfstore.remove("df")
+ df = DataFrame(
+ np.zeros((12, 2)),
+ columns=["a", "b"],
+ index=make_index(["date", "a", "t"]),
+ )
+ msg = "duplicate names/columns in the multi-index when storing as a table"
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("df", df)
+
+ # dup within level
+ temp_hdfstore.remove("df")
+ df = DataFrame(
+ np.zeros((12, 2)),
+ columns=["a", "b"],
+ index=make_index(["date", "date", "date"]),
+ )
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("df", df)
+
+ # fully names
+ temp_hdfstore.remove("df")
+ df = DataFrame(
+ np.zeros((12, 2)),
+ columns=["a", "b"],
+ index=make_index(["date", "s", "t"]),
+ )
+ temp_hdfstore.append("df", df)
+ tm.assert_frame_equal(temp_hdfstore.select("df"), df)
+
+
+@pytest.mark.parametrize("format", ["fixed", "table"])
+def test_store_periodindex(temp_h5_path, format):
+ # GH 7796
+ # test of PeriodIndex in HDFStore
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((5, 1)),
+ index=pd.period_range("20220101", freq="M", periods=5),
+ )
+
+ df.to_hdf(temp_h5_path, key="df", mode="w", format=format)
+ expected = pd.read_hdf(temp_h5_path, "df")
+ tm.assert_frame_equal(df, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_pytables_missing.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_pytables_missing.py
new file mode 100644
index 0000000000000000000000000000000000000000..5594ef5fa626130e354fae9425d9cd988d2fd7fc
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_pytables_missing.py
@@ -0,0 +1,12 @@
+import pytest
+
+import pandas.util._test_decorators as td
+
+import pandas as pd
+
+
+@td.skip_if_installed("tables")
+def test_pytables_raises(temp_h5_path):
+ df = pd.DataFrame({"A": [1, 2]})
+ with pytest.raises(ImportError, match="tables"):
+ df.to_hdf(temp_h5_path, key="df")
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_read.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_read.py
new file mode 100644
index 0000000000000000000000000000000000000000..e8b01ead9c17b0aec12ffedf03e1f46d1f9c58ff
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_read.py
@@ -0,0 +1,305 @@
+from contextlib import closing
+import re
+
+import numpy as np
+import pytest
+
+from pandas.compat import is_platform_windows
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ HDFStore,
+ Index,
+ Series,
+ _testing as tm,
+ date_range,
+ read_hdf,
+)
+
+from pandas.io.pytables import TableIterator
+
+pytestmark = [pytest.mark.single_cpu]
+
+
+def test_read_missing_key_close_store(temp_h5_path):
+ # GH 25766
+ df = DataFrame({"a": range(2), "b": range(2)})
+ df.to_hdf(temp_h5_path, key="k1")
+
+ with pytest.raises(KeyError, match="'No object named k2 in the file'"):
+ read_hdf(temp_h5_path, "k2")
+
+ # smoke test to test that file is properly closed after
+ # read with KeyError before another write
+ df.to_hdf(temp_h5_path, key="k2")
+
+
+def test_read_index_error_close_store(temp_h5_path):
+ # GH 25766
+ df = DataFrame({"A": [], "B": []}, index=[])
+ df.to_hdf(temp_h5_path, key="k1")
+
+ with pytest.raises(IndexError, match=r"list index out of range"):
+ read_hdf(temp_h5_path, "k1", stop=0)
+
+ # smoke test to test that file is properly closed after
+ # read with IndexError before another write
+ df.to_hdf(temp_h5_path, key="k1")
+
+
+def test_read_missing_key_opened_store(temp_h5_path):
+ # GH 28699
+ df = DataFrame({"a": range(2), "b": range(2)})
+ df.to_hdf(temp_h5_path, key="k1")
+
+ with HDFStore(temp_h5_path, "r") as store:
+ with pytest.raises(KeyError, match="'No object named k2 in the file'"):
+ read_hdf(store, "k2")
+
+ # Test that the file is still open after a KeyError and that we can
+ # still read from it.
+ read_hdf(store, "k1")
+
+
+def test_read_column(temp_hdfstore):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+
+ # GH 17912
+ # HDFStore.select_column should raise a KeyError
+ # exception if the key is not a valid store
+ with pytest.raises(KeyError, match="No object named df in the file"):
+ temp_hdfstore.select_column("df", "index")
+
+ temp_hdfstore.append("df", df)
+ # error
+ with pytest.raises(
+ KeyError, match=re.escape("'column [foo] not found in the table'")
+ ):
+ temp_hdfstore.select_column("df", "foo")
+
+ msg = re.escape("select_column() got an unexpected keyword argument 'where'")
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.select_column("df", "index", where=["index>5"])
+
+ # valid
+ result = temp_hdfstore.select_column("df", "index")
+ tm.assert_almost_equal(result.values, Series(df.index).values)
+ assert isinstance(result, Series)
+
+ # not a data indexable column
+ msg = re.escape(
+ "column [values_block_0] can not be extracted individually; "
+ "it is not data indexable"
+ )
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.select_column("df", "values_block_0")
+
+ # a data column
+ df2 = df.copy()
+ df2["string"] = "foo"
+ temp_hdfstore.append("df2", df2, data_columns=["string"])
+ result = temp_hdfstore.select_column("df2", "string")
+ tm.assert_almost_equal(result.values, df2["string"].values)
+
+ # a data column with NaNs, result excludes the NaNs
+ df3 = df.copy()
+ df3["string"] = "foo"
+ df3.loc[df3.index[4:6], "string"] = np.nan
+ temp_hdfstore.append("df3", df3, data_columns=["string"])
+ result = temp_hdfstore.select_column("df3", "string")
+ tm.assert_almost_equal(result.values, df3["string"].values)
+
+ # start/stop
+ result = temp_hdfstore.select_column("df3", "string", start=2)
+ tm.assert_almost_equal(result.values, df3["string"].values[2:])
+
+ result = temp_hdfstore.select_column("df3", "string", start=-2)
+ tm.assert_almost_equal(result.values, df3["string"].values[-2:])
+
+ result = temp_hdfstore.select_column("df3", "string", stop=2)
+ tm.assert_almost_equal(result.values, df3["string"].values[:2])
+
+ result = temp_hdfstore.select_column("df3", "string", stop=-2)
+ tm.assert_almost_equal(result.values, df3["string"].values[:-2])
+
+ result = temp_hdfstore.select_column("df3", "string", start=2, stop=-2)
+ tm.assert_almost_equal(result.values, df3["string"].values[2:-2])
+
+ result = temp_hdfstore.select_column("df3", "string", start=-2, stop=2)
+ tm.assert_almost_equal(result.values, df3["string"].values[-2:2])
+
+ # GH 10392 - make sure column name is preserved
+ df4 = DataFrame({"A": np.random.default_rng(2).standard_normal(10), "B": "foo"})
+ temp_hdfstore.append("df4", df4, data_columns=True)
+ expected = df4["B"]
+ result = temp_hdfstore.select_column("df4", "B")
+ tm.assert_series_equal(result, expected)
+
+
+def test_pytables_native_read(datapath):
+ with HDFStore(
+ datapath("io", "data", "legacy_hdf/pytables_native.h5"), mode="r"
+ ) as store:
+ d2 = store["detector/readout"]
+ assert isinstance(d2, DataFrame)
+
+
+@pytest.mark.skipif(is_platform_windows(), reason="native2 read fails oddly on windows")
+def test_pytables_native2_read(datapath):
+ with HDFStore(
+ datapath("io", "data", "legacy_hdf", "pytables_native2.h5"), mode="r"
+ ) as store:
+ str(store)
+ d1 = store["detector"]
+ assert isinstance(d1, DataFrame)
+
+
+def test_read_hdf_open_store(temp_h5_path, using_infer_string):
+ # GH10330
+ # No check for non-string path_or-buf, and no test of open store
+ df = DataFrame(
+ np.random.default_rng(2).random((4, 5)),
+ index=list("abcd"),
+ columns=list("ABCDE"),
+ )
+ df.index.name = "letters"
+ df = df.set_index(keys="E", append=True)
+
+ df.to_hdf(temp_h5_path, key="df", mode="w")
+ direct = read_hdf(temp_h5_path, "df")
+ with HDFStore(temp_h5_path, mode="r") as store:
+ indirect = read_hdf(store, "df")
+ tm.assert_frame_equal(direct, indirect)
+ assert store.is_open
+
+
+def test_read_hdf_index_not_view(temp_h5_path):
+ # GH 37441
+ # Ensure that the index of the DataFrame is not a view
+ # into the original recarray that pytables reads in
+ df = DataFrame(
+ np.random.default_rng(2).random((4, 5)),
+ index=[0, 1, 2, 3],
+ columns=list("ABCDE"),
+ )
+
+ df.to_hdf(temp_h5_path, key="df", mode="w", format="table")
+
+ df2 = read_hdf(temp_h5_path, "df")
+ assert df2.index._data.base is None
+ tm.assert_frame_equal(df, df2)
+
+
+def test_read_hdf_iterator(temp_h5_path):
+ df = DataFrame(
+ np.random.default_rng(2).random((4, 5)),
+ index=list("abcd"),
+ columns=list("ABCDE"),
+ )
+ df.index.name = "letters"
+ df = df.set_index(keys="E", append=True)
+
+ df.to_hdf(temp_h5_path, key="df", mode="w", format="t")
+ direct = read_hdf(temp_h5_path, "df")
+ iterator = read_hdf(temp_h5_path, "df", iterator=True)
+ with closing(iterator.store):
+ assert isinstance(iterator, TableIterator)
+ indirect = next(iterator.__iter__())
+ tm.assert_frame_equal(direct, indirect)
+
+
+def test_read_nokey(temp_h5_path):
+ # GH10443
+ df = DataFrame(
+ np.random.default_rng(2).random((4, 5)),
+ index=list("abcd"),
+ columns=list("ABCDE"),
+ )
+
+ # Categorical dtype not supported for "fixed" format. So no need
+ # to test with that dtype in the dataframe here.
+ df.to_hdf(temp_h5_path, key="df", mode="a")
+ reread = read_hdf(temp_h5_path)
+ tm.assert_frame_equal(df, reread)
+ df.to_hdf(temp_h5_path, key="df2", mode="a")
+
+ msg = "key must be provided when HDF5 file contains multiple datasets."
+ with pytest.raises(ValueError, match=msg):
+ read_hdf(temp_h5_path)
+
+
+def test_read_nokey_table(temp_h5_path):
+ # GH13231
+ df = DataFrame({"i": range(5), "c": Series(list("abacd"), dtype="category")})
+
+ df.to_hdf(temp_h5_path, key="df", mode="a", format="table")
+ reread = read_hdf(temp_h5_path)
+ tm.assert_frame_equal(df, reread)
+ df.to_hdf(temp_h5_path, key="df2", mode="a", format="table")
+
+ msg = "key must be provided when HDF5 file contains multiple datasets."
+ with pytest.raises(ValueError, match=msg):
+ read_hdf(temp_h5_path)
+
+
+def test_read_nokey_empty(temp_h5_path):
+ store = HDFStore(temp_h5_path)
+ store.close()
+ msg = re.escape(
+ "Dataset(s) incompatible with Pandas data types, not table, or no "
+ "datasets found in HDF5 file."
+ )
+ with pytest.raises(ValueError, match=msg):
+ read_hdf(temp_h5_path)
+
+
+def test_read_from_pathlib_path(temp_h5_path):
+ # GH11773
+ expected = DataFrame(
+ np.random.default_rng(2).random((4, 5)),
+ index=list("abcd"),
+ columns=list("ABCDE"),
+ )
+
+ expected.to_hdf(temp_h5_path, key="df", mode="a")
+ actual = read_hdf(temp_h5_path, key="df")
+
+ tm.assert_frame_equal(expected, actual)
+
+
+@pytest.mark.parametrize("format", ["fixed", "table"])
+def test_read_hdf_series_mode_r(temp_h5_path, format):
+ # GH 16583
+ # Tests that reading a Series saved to an HDF file
+ # still works if a mode='r' argument is supplied
+ series = Series(range(10), dtype=np.float64)
+ series.to_hdf(temp_h5_path, key="data", format=format)
+ result = read_hdf(temp_h5_path, key="data", mode="r")
+ tm.assert_series_equal(result, series)
+
+
+def test_read_infer_string(temp_h5_path):
+ # GH#54431
+ df = DataFrame({"a": ["a", "b", None]})
+ df.to_hdf(temp_h5_path, key="data", format="table")
+ with pd.option_context("future.infer_string", True):
+ result = read_hdf(temp_h5_path, key="data", mode="r")
+ expected = DataFrame(
+ {"a": ["a", "b", None]},
+ dtype=pd.StringDtype(na_value=np.nan),
+ columns=Index(["a"], dtype=pd.StringDtype(na_value=np.nan)),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_hdfstore_read_datetime64_unit_s(temp_hdfstore):
+ # GH 59004
+ df_s = DataFrame(["2001-01-01", "2002-02-02"], dtype="datetime64[s]")
+ temp_hdfstore.put("df_s", df_s)
+ df_fromstore = temp_hdfstore.get("df_s")
+ tm.assert_frame_equal(df_s, df_fromstore)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_retain_attributes.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_retain_attributes.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a88018e2c7ad04ddffb45c88a24b2e188688f96
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_retain_attributes.py
@@ -0,0 +1,95 @@
+import pytest
+
+from pandas import (
+ DataFrame,
+ DatetimeIndex,
+ Series,
+ _testing as tm,
+ date_range,
+ errors,
+ read_hdf,
+)
+
+pytestmark = pytest.mark.single_cpu
+
+
+def test_retain_index_attributes(temp_hdfstore, unit):
+ # GH 3499, losing frequency info on index recreation
+ dti = date_range("2000-1-1", periods=3, freq="h", unit=unit)
+ df = DataFrame({"A": Series(range(3), index=dti)})
+
+ temp_hdfstore.put("data", df, format="table")
+
+ result = temp_hdfstore.get("data")
+ tm.assert_frame_equal(df, result)
+
+ for attr in ["freq", "tz", "name"]:
+ for idx in ["index", "columns"]:
+ assert getattr(getattr(df, idx), attr, None) == getattr(
+ getattr(result, idx), attr, None
+ )
+
+ dti2 = date_range("2002-1-1", periods=3, freq="D", unit=unit)
+ # try to append a table with a different frequency
+ with tm.assert_produces_warning(errors.AttributeConflictWarning):
+ df2 = DataFrame({"A": Series(range(3), index=dti2)})
+ temp_hdfstore.append("data", df2)
+
+ assert temp_hdfstore.get_storer("data").info["index"]["freq"] is None
+
+ # this is ok
+ dti3 = DatetimeIndex(
+ ["2001-01-01", "2001-01-02", "2002-01-01"], dtype=f"M8[{unit}]"
+ )
+ df2 = DataFrame(
+ {
+ "A": Series(
+ range(3),
+ index=dti3,
+ )
+ }
+ )
+ temp_hdfstore.append("df2", df2)
+ dti4 = date_range("2002-1-1", periods=3, freq="D", unit=unit)
+ df3 = DataFrame({"A": Series(range(3), index=dti4)})
+ temp_hdfstore.append("df2", df3)
+
+
+def test_retain_index_attributes2(temp_h5_path):
+ with tm.assert_produces_warning(errors.AttributeConflictWarning):
+ df = DataFrame(
+ {"A": Series(range(3), index=date_range("2000-1-1", periods=3, freq="h"))}
+ )
+ df.to_hdf(temp_h5_path, key="data", mode="w", append=True)
+ df2 = DataFrame(
+ {"A": Series(range(3), index=date_range("2002-1-1", periods=3, freq="D"))}
+ )
+
+ df2.to_hdf(temp_h5_path, key="data", append=True)
+
+ idx = date_range("2000-1-1", periods=3, freq="h")
+ idx.name = "foo"
+ df = DataFrame({"A": Series(range(3), index=idx)})
+ df.to_hdf(temp_h5_path, key="data", mode="w", append=True)
+
+ assert read_hdf(temp_h5_path, key="data").index.name == "foo"
+
+ with tm.assert_produces_warning(errors.AttributeConflictWarning):
+ idx2 = date_range("2001-1-1", periods=3, freq="h")
+ idx2.name = "bar"
+ df2 = DataFrame({"A": Series(range(3), index=idx2)})
+ df2.to_hdf(temp_h5_path, key="data", append=True)
+
+ assert read_hdf(temp_h5_path, "data").index.name is None
+
+
+def test_retain_datetime_attribute(temp_h5_path):
+ ser = Series(
+ ["2024-08-26 15:13:14", "2024-08-26 15:14:14"],
+ dtype="datetime64[us, UTC]",
+ )
+ dataframe = DataFrame(ser)
+ dataframe.to_hdf(temp_h5_path, key="Annotations", mode="w")
+
+ recovered_dataframe = read_hdf(temp_h5_path, key="Annotations")
+ tm.assert_frame_equal(dataframe, recovered_dataframe)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_round_trip.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_round_trip.py
new file mode 100644
index 0000000000000000000000000000000000000000..62814d7c778d382d5f4a9f879736665c1bfea0e0
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_round_trip.py
@@ -0,0 +1,572 @@
+import datetime
+import re
+
+import numpy as np
+import pytest
+
+from pandas._libs.tslibs import Timestamp
+from pandas.compat import is_platform_windows
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ DatetimeIndex,
+ HDFStore,
+ Index,
+ Series,
+ _testing as tm,
+ bdate_range,
+ date_range,
+ read_hdf,
+)
+from pandas.util import _test_decorators as td
+
+pytestmark = [pytest.mark.single_cpu]
+
+
+def test_conv_read_write(temp_h5_path):
+ def roundtrip(key, obj, **kwargs):
+ obj.to_hdf(temp_h5_path, key=key, **kwargs)
+ return read_hdf(temp_h5_path, key)
+
+ o = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ tm.assert_series_equal(o, roundtrip("series", o))
+
+ o = Series(range(10), dtype="float64", index=[f"i_{i}" for i in range(10)])
+ tm.assert_series_equal(o, roundtrip("string_series", o))
+
+ o = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ tm.assert_frame_equal(o, roundtrip("frame", o))
+
+ # table
+ df = DataFrame({"A": range(5), "B": range(5)})
+ df.to_hdf(temp_h5_path, key="table", append=True)
+ result = read_hdf(temp_h5_path, "table", where=["index>2"])
+ tm.assert_frame_equal(df[df.index > 2], result)
+
+
+def test_long_strings(temp_hdfstore):
+ # GH6166
+ data = ["a" * 50] * 10
+ df = DataFrame({"a": data}, index=data)
+
+ temp_hdfstore.append("df", df, data_columns=["a"])
+
+ result = temp_hdfstore.select("df")
+ tm.assert_frame_equal(df, result)
+
+
+def test_api(temp_h5_path):
+ # GH4584
+ # API issue when to_hdf doesn't accept append AND format args
+ path = temp_h5_path
+
+ df = DataFrame(range(20))
+ df.iloc[:10].to_hdf(path, key="df", append=True, format="table")
+ df.iloc[10:].to_hdf(path, key="df", append=True, format="table")
+ tm.assert_frame_equal(read_hdf(path, "df"), df)
+
+ # append to False
+ df.iloc[:10].to_hdf(path, key="df", append=False, format="table")
+ df.iloc[10:].to_hdf(path, key="df", append=True, format="table")
+ tm.assert_frame_equal(read_hdf(path, "df"), df)
+
+
+def test_api_append(temp_h5_path):
+ path = temp_h5_path
+
+ df = DataFrame(range(20))
+ df.iloc[:10].to_hdf(path, key="df", append=True)
+ df.iloc[10:].to_hdf(path, key="df", append=True, format="table")
+ tm.assert_frame_equal(read_hdf(path, "df"), df)
+
+ # append to False
+ df.iloc[:10].to_hdf(path, key="df", append=False, format="table")
+ df.iloc[10:].to_hdf(path, key="df", append=True)
+ tm.assert_frame_equal(read_hdf(path, "df"), df)
+
+
+def test_api_2(temp_h5_path):
+ df = DataFrame(range(20))
+ df.to_hdf(temp_h5_path, key="df", append=False, format="fixed")
+ tm.assert_frame_equal(read_hdf(temp_h5_path, "df"), df)
+
+ df.to_hdf(temp_h5_path, key="df", append=False, format="f")
+ tm.assert_frame_equal(read_hdf(temp_h5_path, "df"), df)
+
+ df.to_hdf(temp_h5_path, key="df", append=False)
+ tm.assert_frame_equal(read_hdf(temp_h5_path, "df"), df)
+
+ df.to_hdf(temp_h5_path, key="df")
+ tm.assert_frame_equal(read_hdf(temp_h5_path, "df"), df)
+
+
+def test_api_3(temp_hdfstore):
+ df = DataFrame(range(20))
+
+ temp_hdfstore.append("df", df.iloc[:10], append=True, format="table")
+ temp_hdfstore.append("df", df.iloc[10:], append=True, format="table")
+ tm.assert_frame_equal(temp_hdfstore.select("df"), df)
+
+ # append to False
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df.iloc[:10], append=False, format="table")
+ temp_hdfstore.append("df", df.iloc[10:], append=True, format="table")
+ tm.assert_frame_equal(temp_hdfstore.select("df"), df)
+
+ # formats
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df.iloc[:10], append=False, format="table")
+ temp_hdfstore.append("df", df.iloc[10:], append=True, format="table")
+ tm.assert_frame_equal(temp_hdfstore.select("df"), df)
+
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df.iloc[:10], append=False, format="table")
+ temp_hdfstore.append("df", df.iloc[10:], append=True, format=None)
+ tm.assert_frame_equal(temp_hdfstore.select("df"), df)
+
+
+def test_api_invalid(temp_h5_path):
+ path = temp_h5_path
+ # Invalid.
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+
+ msg = "Can only append to Tables"
+
+ with pytest.raises(ValueError, match=msg):
+ df.to_hdf(path, key="df", append=True, format="f")
+
+ with pytest.raises(ValueError, match=msg):
+ df.to_hdf(path, key="df", append=True, format="fixed")
+
+ msg = r"invalid HDFStore format specified \[foo\]"
+
+ with pytest.raises(TypeError, match=msg):
+ df.to_hdf(path, key="df", append=True, format="foo")
+
+ with pytest.raises(TypeError, match=msg):
+ df.to_hdf(path, key="df", append=False, format="foo")
+
+ # File path doesn't exist
+ path = ""
+ msg = f"File {path} does not exist"
+
+ with pytest.raises(FileNotFoundError, match=msg):
+ read_hdf(path, "df")
+
+
+def test_get(temp_hdfstore):
+ temp_hdfstore["a"] = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ left = temp_hdfstore.get("a")
+ right = temp_hdfstore["a"]
+ tm.assert_series_equal(left, right)
+
+ left = temp_hdfstore.get("/a")
+ right = temp_hdfstore["/a"]
+ tm.assert_series_equal(left, right)
+
+ with pytest.raises(KeyError, match="'No object named b in the file'"):
+ temp_hdfstore.get("b")
+
+
+def test_put_integer(temp_h5_path):
+ # non-date, non-string index
+ df = DataFrame(np.random.default_rng(2).standard_normal((50, 100)))
+ _check_roundtrip(df, tm.assert_frame_equal, temp_h5_path)
+
+
+def test_table_values_dtypes_roundtrip(temp_hdfstore, using_infer_string):
+ df1 = DataFrame({"a": [1, 2, 3]}, dtype="f8")
+ temp_hdfstore.append("df_f8", df1)
+ tm.assert_series_equal(df1.dtypes, temp_hdfstore["df_f8"].dtypes)
+
+ df2 = DataFrame({"a": [1, 2, 3]}, dtype="i8")
+ temp_hdfstore.append("df_i8", df2)
+ tm.assert_series_equal(df2.dtypes, temp_hdfstore["df_i8"].dtypes)
+
+ # incompatible dtype
+ msg = re.escape(
+ "Cannot serialize the column [a] "
+ "because its data contents are not [float] "
+ "but [integer] object dtype"
+ )
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("df_i8", df1)
+
+ # check creation/storage/retrieval of float32 (a bit hacky to
+ # actually create them thought)
+ df1 = DataFrame(np.array([[1], [2], [3]], dtype="f4"), columns=["A"])
+ temp_hdfstore.append("df_f4", df1)
+ tm.assert_series_equal(df1.dtypes, temp_hdfstore["df_f4"].dtypes)
+ assert df1.dtypes.iloc[0] == "float32"
+
+ # check with mixed dtypes
+ df1 = DataFrame(
+ {
+ c: Series(np.random.default_rng(2).integers(5), dtype=c)
+ for c in ["float32", "float64", "int32", "int64", "int16", "int8"]
+ }
+ )
+ df1["string"] = "foo"
+ df1["float322"] = 1.0
+ df1["float322"] = df1["float322"].astype("float32")
+ df1["bool"] = df1["float32"] > 0
+ df1["time_s_1"] = Timestamp("20130101").as_unit("s")
+ df1["time_s_2"] = Timestamp("20130101 00:00:00").as_unit("s")
+ df1["time_ms"] = Timestamp("20130101 00:00:00.000").as_unit("ms")
+ df1["time_ns"] = Timestamp("20130102 00:00:00.000000000")
+
+ temp_hdfstore.append("df_mixed_dtypes1", df1)
+ result = temp_hdfstore.select("df_mixed_dtypes1").dtypes.value_counts()
+ result.index = [str(i) for i in result.index]
+ str_dtype = "str" if using_infer_string else "object"
+ expected = Series(
+ {
+ "float32": 2,
+ "float64": 1,
+ "int32": 1,
+ "bool": 1,
+ "int16": 1,
+ "int8": 1,
+ "int64": 1,
+ str_dtype: 1,
+ "datetime64[s]": 2,
+ "datetime64[ms]": 1,
+ "datetime64[ns]": 1,
+ },
+ name="count",
+ )
+ result = result.sort_index()
+ expected = expected.sort_index()
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.filterwarnings("ignore::pandas.errors.PerformanceWarning")
+def test_series(temp_h5_path):
+ s = Series(range(10), dtype="float64", index=[f"i_{i}" for i in range(10)])
+ _check_roundtrip(s, tm.assert_series_equal, path=temp_h5_path)
+
+ ts = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ _check_roundtrip(ts, tm.assert_series_equal, path=temp_h5_path)
+
+ ts2 = Series(ts.index, Index(ts.index))
+ _check_roundtrip(ts2, tm.assert_series_equal, path=temp_h5_path)
+
+ ts3 = Series(ts.values, Index(np.asarray(ts.index)))
+ _check_roundtrip(
+ ts3, tm.assert_series_equal, path=temp_h5_path, check_index_type=False
+ )
+
+
+def test_float_index(temp_h5_path):
+ # GH #454
+ index = np.random.default_rng(2).standard_normal(10)
+ s = Series(np.random.default_rng(2).standard_normal(10), index=index)
+ _check_roundtrip(s, tm.assert_series_equal, path=temp_h5_path)
+
+
+def test_tuple_index(temp_h5_path, performance_warning):
+ # GH #492
+ col = np.arange(10)
+ idx = [(0.0, 1.0), (2.0, 3.0), (4.0, 5.0)]
+ data = np.random.default_rng(2).standard_normal(30).reshape((3, 10))
+ DF = DataFrame(data, index=idx, columns=col)
+
+ with tm.assert_produces_warning(performance_warning):
+ _check_roundtrip(DF, tm.assert_frame_equal, path=temp_h5_path)
+
+
+@pytest.mark.filterwarnings("ignore::pandas.errors.PerformanceWarning")
+def test_index_types(temp_h5_path):
+ values = np.random.default_rng(2).standard_normal(2)
+
+ func = lambda lhs, rhs: tm.assert_series_equal(lhs, rhs, check_index_type=True)
+
+ ser = Series(values, [0, "y"])
+ _check_roundtrip(ser, func, path=temp_h5_path)
+
+ ser = Series(values, [datetime.datetime.today(), 0])
+ _check_roundtrip(ser, func, path=temp_h5_path)
+
+ ser = Series(values, ["y", 0])
+ _check_roundtrip(ser, func, path=temp_h5_path)
+
+ ser = Series(values, [datetime.date.today(), "a"])
+ _check_roundtrip(ser, func, path=temp_h5_path)
+
+ ser = Series(values, [0, "y"])
+ _check_roundtrip(ser, func, path=temp_h5_path)
+
+ ser = Series(values, [datetime.datetime.today(), 0])
+ _check_roundtrip(ser, func, path=temp_h5_path)
+
+ ser = Series(values, ["y", 0])
+ _check_roundtrip(ser, func, path=temp_h5_path)
+
+ ser = Series(values, [datetime.date.today(), "a"])
+ _check_roundtrip(ser, func, path=temp_h5_path)
+
+ ser = Series(values, [1.23, "b"])
+ _check_roundtrip(ser, func, path=temp_h5_path)
+
+ ser = Series(values, [1, 1.53])
+ _check_roundtrip(ser, func, path=temp_h5_path)
+
+ ser = Series(values, [1, 5])
+ _check_roundtrip(ser, func, path=temp_h5_path)
+
+ dti = DatetimeIndex(["2012-01-01", "2012-01-02"], dtype="M8[ns]")
+ ser = Series(values, index=dti)
+ _check_roundtrip(ser, func, path=temp_h5_path)
+
+ ser.index = ser.index.as_unit("s")
+ _check_roundtrip(ser, func, path=temp_h5_path)
+
+
+def test_timeseries_preepoch(temp_h5_path, request):
+ dr = bdate_range("1/1/1940", "1/1/1960")
+ ts = Series(np.random.default_rng(2).standard_normal(len(dr)), index=dr)
+ try:
+ _check_roundtrip(ts, tm.assert_series_equal, path=temp_h5_path)
+ except OverflowError:
+ if is_platform_windows():
+ request.applymarker(
+ pytest.mark.xfail("known failure on some windows platforms")
+ )
+ raise
+
+
+@pytest.mark.parametrize(
+ "compression", [False, pytest.param(True, marks=td.skip_if_windows)]
+)
+def test_frame(compression, temp_h5_path):
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+
+ # put in some random NAs
+ df.iloc[0, 0] = np.nan
+ df.iloc[5, 3] = np.nan
+
+ _check_roundtrip_table(
+ df, tm.assert_frame_equal, path=temp_h5_path, compression=compression
+ )
+ _check_roundtrip(
+ df, tm.assert_frame_equal, path=temp_h5_path, compression=compression
+ )
+
+ tdf = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ _check_roundtrip(
+ tdf, tm.assert_frame_equal, path=temp_h5_path, compression=compression
+ )
+
+ with HDFStore(temp_h5_path) as store:
+ # not consolidated
+ df["foo"] = np.random.default_rng(2).standard_normal(len(df))
+ store["df"] = df
+ recons = store["df"]
+ assert recons._mgr.is_consolidated()
+
+ # empty
+ df2 = df[:0]
+ # Prevent df2 from having index with inferred_type as string
+ df2.index = Index([])
+ _check_roundtrip(df2[:0], tm.assert_frame_equal, path=temp_h5_path)
+
+
+def test_empty_series_frame(temp_h5_path):
+ s0 = Series(dtype=object)
+ s1 = Series(name="myseries", dtype=object)
+ df0 = DataFrame()
+ df1 = DataFrame(index=["a", "b", "c"])
+ df2 = DataFrame(columns=["d", "e", "f"])
+
+ _check_roundtrip(s0, tm.assert_series_equal, path=temp_h5_path)
+ _check_roundtrip(s1, tm.assert_series_equal, path=temp_h5_path)
+ _check_roundtrip(df0, tm.assert_frame_equal, path=temp_h5_path)
+ _check_roundtrip(df1, tm.assert_frame_equal, path=temp_h5_path)
+ _check_roundtrip(df2, tm.assert_frame_equal, path=temp_h5_path)
+
+
+@pytest.mark.parametrize("dtype", [np.int64, np.float64, object, "m8[ns]", "M8[ns]"])
+def test_empty_series(dtype, temp_h5_path):
+ s = Series(dtype=dtype)
+ _check_roundtrip(s, tm.assert_series_equal, path=temp_h5_path)
+
+
+def test_can_serialize_dates(temp_h5_path):
+ rng = [x.date() for x in bdate_range("1/1/2000", "1/30/2000")]
+ frame = DataFrame(
+ np.random.default_rng(2).standard_normal((len(rng), 4)), index=rng
+ )
+
+ _check_roundtrip(frame, tm.assert_frame_equal, path=temp_h5_path)
+
+
+def test_store_hierarchical(
+ temp_h5_path, using_infer_string, multiindex_dataframe_random_data
+):
+ frame = multiindex_dataframe_random_data
+
+ _check_roundtrip(frame, tm.assert_frame_equal, path=temp_h5_path)
+ _check_roundtrip(frame.T, tm.assert_frame_equal, path=temp_h5_path)
+ _check_roundtrip(frame["A"], tm.assert_series_equal, path=temp_h5_path)
+
+ # check that the names are stored
+ with HDFStore(temp_h5_path) as store:
+ store["frame"] = frame
+ recons = store["frame"]
+ tm.assert_frame_equal(recons, frame)
+
+
+@pytest.mark.parametrize(
+ "compression", [False, pytest.param(True, marks=td.skip_if_windows)]
+)
+def test_store_mixed(compression, temp_h5_path):
+ def _make_one():
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ df["obj1"] = "foo"
+ df["obj2"] = "bar"
+ df["bool1"] = df["A"] > 0
+ df["bool2"] = df["B"] > 0
+ df["int1"] = 1
+ df["int2"] = 2
+ return df._consolidate()
+
+ df1 = _make_one()
+ df2 = _make_one()
+
+ _check_roundtrip(df1, tm.assert_frame_equal, path=temp_h5_path)
+ _check_roundtrip(df2, tm.assert_frame_equal, path=temp_h5_path)
+
+ with HDFStore(temp_h5_path) as store:
+ store["obj"] = df1
+ tm.assert_frame_equal(store["obj"], df1)
+ store["obj"] = df2
+ tm.assert_frame_equal(store["obj"], df2)
+
+ # check that can store Series of all of these types
+ _check_roundtrip(
+ df1["obj1"],
+ tm.assert_series_equal,
+ path=temp_h5_path,
+ compression=compression,
+ )
+ _check_roundtrip(
+ df1["bool1"],
+ tm.assert_series_equal,
+ path=temp_h5_path,
+ compression=compression,
+ )
+ _check_roundtrip(
+ df1["int1"],
+ tm.assert_series_equal,
+ path=temp_h5_path,
+ compression=compression,
+ )
+
+
+def _check_roundtrip(obj, comparator, path, compression=False, **kwargs):
+ options = {}
+ if compression:
+ options["complib"] = "blosc"
+
+ with HDFStore(path, "w", **options) as store:
+ store["obj"] = obj
+ retrieved = store["obj"]
+ comparator(retrieved, obj, **kwargs)
+
+
+def _check_roundtrip_table(obj, comparator, path, compression=False):
+ options = {}
+ if compression:
+ options["complib"] = "blosc"
+
+ with HDFStore(path, "w", **options) as store:
+ store.put("obj", obj, format="table")
+ retrieved = store["obj"]
+
+ comparator(retrieved, obj)
+
+
+def test_unicode_index(temp_h5_path):
+ unicode_values = ["\u03c3", "\u03c3\u03c3"]
+
+ s = Series(
+ np.random.default_rng(2).standard_normal(len(unicode_values)),
+ unicode_values,
+ )
+ _check_roundtrip(s, tm.assert_series_equal, path=temp_h5_path)
+
+
+def test_unicode_longer_encoded(temp_hdfstore):
+ # GH 11234
+ char = "\u0394"
+ df = DataFrame({"A": [char]})
+ temp_hdfstore.put("df", df, format="table", encoding="utf-8")
+ result = temp_hdfstore.get("df")
+ tm.assert_frame_equal(result, df)
+
+ df = DataFrame({"A": ["a", char], "B": ["b", "b"]})
+ temp_hdfstore.remove("df")
+ temp_hdfstore.put("df", df, format="table", encoding="utf-8")
+ result = temp_hdfstore.get("df")
+ tm.assert_frame_equal(result, df)
+
+
+def test_store_datetime_mixed(temp_h5_path):
+ df = DataFrame({"a": [1, 2, 3], "b": [1.0, 2.0, 3.0], "c": ["a", "b", "c"]})
+ ts = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ df["d"] = ts.index[:3]
+ _check_roundtrip(df, tm.assert_frame_equal, path=temp_h5_path)
+
+
+def test_round_trip_equals(temp_h5_path):
+ # GH 9330
+ df = DataFrame({"B": [1, 2], "A": ["x", "y"]})
+
+ df.to_hdf(temp_h5_path, key="df", format="table")
+ other = read_hdf(temp_h5_path, "df")
+ tm.assert_frame_equal(df, other)
+ assert df.equals(other)
+ assert other.equals(df)
+
+
+def test_infer_string_columns(temp_h5_path):
+ # GH#
+ pytest.importorskip("pyarrow")
+ with pd.option_context("future.infer_string", True):
+ df = DataFrame(1, columns=list("ABCD"), index=list(range(10))).set_index(
+ ["A", "B"]
+ )
+ expected = df.copy()
+ df.to_hdf(temp_h5_path, key="df", format="table")
+
+ result = read_hdf(temp_h5_path, "df")
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_select.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_select.py
new file mode 100644
index 0000000000000000000000000000000000000000..ca98dcb2520851cffad2fefecd16e6c6da96cc5f
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_select.py
@@ -0,0 +1,1050 @@
+import numpy as np
+import pytest
+
+from pandas._libs.tslibs import Timestamp
+from pandas.compat import PY312
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ HDFStore,
+ Index,
+ MultiIndex,
+ Series,
+ _testing as tm,
+ bdate_range,
+ concat,
+ date_range,
+ isna,
+ read_hdf,
+)
+
+from pandas.io.pytables import Term
+
+pytestmark = [pytest.mark.single_cpu]
+
+
+def test_select_columns_in_where(temp_hdfstore):
+ # GH 6169
+ # recreate multi-indexes when columns is passed
+ # in the `where` argument
+ index = MultiIndex(
+ levels=[["foo", "bar", "baz", "qux"], ["one", "two", "three"]],
+ codes=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3], [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
+ names=["foo_name", "bar_name"],
+ )
+
+ # With a DataFrame
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 3)),
+ index=index,
+ columns=["A", "B", "C"],
+ )
+
+ temp_hdfstore.put("df", df, format="table")
+ expected = df[["A"]]
+
+ tm.assert_frame_equal(temp_hdfstore.select("df", columns=["A"]), expected)
+
+ tm.assert_frame_equal(temp_hdfstore.select("df", where="columns=['A']"), expected)
+
+ # With a Series
+ s = Series(np.random.default_rng(2).standard_normal(10), index=index, name="A")
+ temp_hdfstore.put("s", s, format="table")
+ tm.assert_series_equal(temp_hdfstore.select("s", where="columns=['A']"), s)
+
+
+def test_select_with_dups(temp_hdfstore):
+ # single dtypes
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)), columns=["A", "A", "B", "B"]
+ )
+ df.index = date_range("20130101 9:30", periods=10, freq="min", unit="ns")
+
+ temp_hdfstore.append("df", df)
+
+ result = temp_hdfstore.select("df")
+ expected = df
+ tm.assert_frame_equal(result, expected, by_blocks=True)
+
+ result = temp_hdfstore.select("df", columns=df.columns)
+ expected = df
+ tm.assert_frame_equal(result, expected, by_blocks=True)
+
+ result = temp_hdfstore.select("df", columns=["A"])
+ expected = df.loc[:, ["A"]]
+ tm.assert_frame_equal(result, expected)
+
+
+def test_select_with_dups_across_dtypes(temp_hdfstore):
+ df = concat(
+ [
+ DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=["A", "A", "B", "B"],
+ ),
+ DataFrame(
+ np.random.default_rng(2).integers(0, 10, size=20).reshape(10, 2),
+ columns=["A", "C"],
+ ),
+ ],
+ axis=1,
+ )
+ df.index = date_range("20130101 9:30", periods=10, freq="min", unit="ns")
+
+ temp_hdfstore.append("df", df)
+
+ result = temp_hdfstore.select("df")
+ expected = df
+ tm.assert_frame_equal(result, expected, by_blocks=True)
+
+ result = temp_hdfstore.select("df", columns=df.columns)
+ expected = df
+ tm.assert_frame_equal(result, expected, by_blocks=True)
+
+ expected = df.loc[:, ["A"]]
+ result = temp_hdfstore.select("df", columns=["A"])
+ tm.assert_frame_equal(result, expected, by_blocks=True)
+
+ expected = df.loc[:, ["B", "A"]]
+ result = temp_hdfstore.select("df", columns=["B", "A"])
+ tm.assert_frame_equal(result, expected, by_blocks=True)
+
+
+def test_select_with_dups_across_index_and_columns(temp_hdfstore):
+ df = concat(
+ [
+ DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=["A", "A", "B", "B"],
+ ),
+ DataFrame(
+ np.random.default_rng(2).integers(0, 10, size=20).reshape(10, 2),
+ columns=["A", "C"],
+ ),
+ ],
+ axis=1,
+ )
+ df.index = date_range("20130101 9:30", periods=10, freq="min", unit="ns")
+ temp_hdfstore.append("df", df)
+ temp_hdfstore.append("df", df)
+
+ expected = df.loc[:, ["B", "A"]]
+ expected = concat([expected, expected])
+ result = temp_hdfstore.select("df", columns=["B", "A"])
+ tm.assert_frame_equal(result, expected, by_blocks=True)
+
+
+def test_select(temp_hdfstore):
+ # select with columns=
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B", unit="ns"),
+ )
+ temp_hdfstore.append("df", df)
+ result = temp_hdfstore.select("df", columns=["A", "B"])
+ expected = df.reindex(columns=["A", "B"])
+ tm.assert_frame_equal(expected, result)
+
+ # equivalently
+ result = temp_hdfstore.select("df", ["columns=['A', 'B']"])
+ expected = df.reindex(columns=["A", "B"])
+ tm.assert_frame_equal(expected, result)
+
+ # with a data column
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df, data_columns=["A"])
+ result = temp_hdfstore.select("df", ["A > 0"], columns=["A", "B"])
+ expected = df[df.A > 0].reindex(columns=["A", "B"])
+ tm.assert_frame_equal(expected, result)
+
+ # all a data columns
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df, data_columns=True)
+ result = temp_hdfstore.select("df", ["A > 0"], columns=["A", "B"])
+ expected = df[df.A > 0].reindex(columns=["A", "B"])
+ tm.assert_frame_equal(expected, result)
+
+ # with a data column, but different columns
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df, data_columns=["A"])
+ result = temp_hdfstore.select("df", ["A > 0"], columns=["C", "D"])
+ expected = df[df.A > 0].reindex(columns=["C", "D"])
+ tm.assert_frame_equal(expected, result)
+
+
+def test_select_dtypes_timestamp(temp_hdfstore):
+ # with a Timestamp data column (GH #2637)
+ df = DataFrame(
+ {
+ "ts": bdate_range("2012-01-01", periods=300, unit="ns"),
+ "A": np.random.default_rng(2).standard_normal(300),
+ }
+ )
+ temp_hdfstore.append("df", df, data_columns=["ts", "A"])
+
+ result = temp_hdfstore.select("df", "ts>=Timestamp('2012-02-01')")
+ expected = df[df.ts >= Timestamp("2012-02-01")]
+ tm.assert_frame_equal(expected, result)
+
+
+def test_select_dtypes_bools(temp_hdfstore):
+ # bool columns (GH #2849)
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 2)), columns=["A", "B"])
+ df["object"] = "foo"
+ df.loc[4:5, "object"] = "bar"
+ df["boolv"] = df["A"] > 0
+ temp_hdfstore.append("df", df, data_columns=True)
+
+ expected = df[df.boolv == True].reindex(columns=["A", "boolv"]) # noqa: E712
+ for v in [True, "true", 1]:
+ result = temp_hdfstore.select("df", f"boolv == {v}", columns=["A", "boolv"])
+ tm.assert_frame_equal(expected, result)
+
+ expected = df[df.boolv == False].reindex(columns=["A", "boolv"]) # noqa: E712
+ for v in [False, "false", 0]:
+ result = temp_hdfstore.select("df", f"boolv == {v}", columns=["A", "boolv"])
+ tm.assert_frame_equal(expected, result)
+
+
+def test_select_dtypes_integer_index(temp_hdfstore):
+ df = DataFrame(
+ {
+ "A": np.random.default_rng(2).random(20),
+ "B": np.random.default_rng(2).random(20),
+ }
+ )
+ temp_hdfstore.append("df_int", df)
+ result = temp_hdfstore.select("df_int", "index<10 and columns=['A']")
+ expected = df.reindex(index=list(df.index)[0:10], columns=["A"])
+ tm.assert_frame_equal(expected, result)
+
+
+def test_select_dtypes_float_index(temp_hdfstore):
+ df = DataFrame(
+ {
+ "A": np.random.default_rng(2).random(20),
+ "B": np.random.default_rng(2).random(20),
+ "index": np.arange(20, dtype="f8"),
+ }
+ )
+ temp_hdfstore.append("df_float", df)
+ result = temp_hdfstore.select("df_float", "index<10.0 and columns=['A']")
+ expected = df.reindex(index=list(df.index)[0:10], columns=["A"])
+ tm.assert_frame_equal(expected, result)
+
+
+def test_select_dtypes_floats_without_nan(temp_hdfstore):
+ df = DataFrame({"cols": range(11), "values": range(11)}, dtype="float64")
+ df["cols"] = (df["cols"] + 10).apply(str)
+
+ temp_hdfstore.append("df1", df, data_columns=True)
+ result = temp_hdfstore.select("df1", where="values>2.0")
+ expected = df[df["values"] > 2.0]
+ tm.assert_frame_equal(expected, result)
+
+
+def test_select_dtypes_floats_with_nan(temp_hdfstore):
+ df = DataFrame({"cols": range(11), "values": range(11)}, dtype="float64")
+ df["cols"] = (df["cols"] + 10).apply(str)
+ df.iloc[0] = np.nan
+ expected = df[df["values"] > 2.0]
+
+ temp_hdfstore.append("df2", df, data_columns=True, index=False)
+ result = temp_hdfstore.select("df2", where="values>2.0")
+ tm.assert_frame_equal(expected, result)
+
+
+def test_select_dtypes_floats_with_nan_not_first_position(temp_hdfstore):
+ df = DataFrame({"cols": range(11), "values": range(11)}, dtype="float64")
+ df["cols"] = (df["cols"] + 10).apply(str)
+
+ df.iloc[1] = np.nan
+ expected = df[df["values"] > 2.0]
+
+ temp_hdfstore.append("df4", df, data_columns=True)
+ result = temp_hdfstore.select("df4", where="values>2.0")
+ tm.assert_frame_equal(expected, result)
+
+
+def test_select_dtypes_comparison_with_numpy_scalar(temp_hdfstore, request):
+ # GH 11283
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+
+ expected = df[df["A"] > 0]
+
+ temp_hdfstore.append("df", df, data_columns=True)
+ request.applymarker(
+ pytest.mark.xfail(
+ PY312,
+ reason="AST change in PY312",
+ raises=ValueError,
+ )
+ )
+ np_zero = np.float64(0) # noqa: F841
+ result = temp_hdfstore.select("df", where=["A>np_zero"])
+ tm.assert_frame_equal(expected, result)
+
+
+def test_select_with_many_inputs(temp_hdfstore):
+ df = DataFrame(
+ {
+ "ts": bdate_range("2012-01-01", periods=300, unit="ns"),
+ "A": np.random.default_rng(2).standard_normal(300),
+ "B": range(300),
+ "users": ["a"] * 50
+ + ["b"] * 50
+ + ["c"] * 100
+ + [f"a{i:03d}" for i in range(100)],
+ }
+ )
+ temp_hdfstore.append("df", df, data_columns=["ts", "A", "B", "users"])
+
+ # regular select
+ result = temp_hdfstore.select("df", "ts>=Timestamp('2012-02-01')")
+ expected = df[df.ts >= Timestamp("2012-02-01")]
+ tm.assert_frame_equal(expected, result)
+
+ # small selector
+ result = temp_hdfstore.select(
+ "df", "ts>=Timestamp('2012-02-01') & users=['a','b','c']"
+ )
+ expected = df[(df.ts >= Timestamp("2012-02-01")) & df.users.isin(["a", "b", "c"])]
+ tm.assert_frame_equal(expected, result)
+
+ # big selector along the columns
+ selector = ["a", "b", "c"] + [f"a{i:03d}" for i in range(60)]
+ result = temp_hdfstore.select(
+ "df", "ts>=Timestamp('2012-02-01') and users=selector"
+ )
+ expected = df[(df.ts >= Timestamp("2012-02-01")) & df.users.isin(selector)]
+ tm.assert_frame_equal(expected, result)
+
+ selector = range(100, 200)
+ result = temp_hdfstore.select("df", "B=selector")
+ expected = df[df.B.isin(selector)]
+ tm.assert_frame_equal(expected, result)
+ assert len(result) == 100
+
+ # big selector along the index
+ selector = Index(df.ts[0:100].values)
+ result = temp_hdfstore.select("df", "ts=selector")
+ expected = df[df.ts.isin(selector.values)]
+ tm.assert_frame_equal(expected, result)
+ assert len(result) == 100
+
+
+def test_select_iterator(temp_hdfstore):
+ # single table
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B", unit="ns"),
+ )
+ temp_hdfstore.append("df", df)
+
+ expected = temp_hdfstore.select("df")
+
+ results = list(temp_hdfstore.select("df", iterator=True))
+ result = concat(results)
+ tm.assert_frame_equal(expected, result)
+
+ results = list(temp_hdfstore.select("df", chunksize=2))
+ assert len(results) == 5
+ result = concat(results)
+ tm.assert_frame_equal(expected, result)
+
+ results = list(temp_hdfstore.select("df", chunksize=2))
+ result = concat(results)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_select_iterator2(temp_h5_path):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B", unit="ns"),
+ )
+ df.to_hdf(temp_h5_path, key="df_non_table")
+
+ msg = "can only use an iterator or chunksize on a table"
+ with pytest.raises(TypeError, match=msg):
+ read_hdf(temp_h5_path, "df_non_table", chunksize=2)
+
+ with pytest.raises(TypeError, match=msg):
+ read_hdf(temp_h5_path, "df_non_table", iterator=True)
+
+
+def test_select_iterator3(temp_h5_path):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B", unit="ns"),
+ )
+ df.to_hdf(temp_h5_path, key="df", format="table")
+
+ results = list(read_hdf(temp_h5_path, "df", chunksize=2))
+ result = concat(results)
+
+ assert len(results) == 5
+ tm.assert_frame_equal(result, df)
+ tm.assert_frame_equal(result, read_hdf(temp_h5_path, "df"))
+
+
+def test_select_iterator_multiple(temp_hdfstore):
+ df1 = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B", unit="ns"),
+ )
+ temp_hdfstore.append("df1", df1, data_columns=True)
+ df2 = df1.copy().rename(columns="{}_2".format)
+ df2["foo"] = "bar"
+ temp_hdfstore.append("df2", df2)
+
+ # full selection
+ expected = temp_hdfstore.select_as_multiple(["df1", "df2"], selector="df1")
+ results = list(
+ temp_hdfstore.select_as_multiple(["df1", "df2"], selector="df1", chunksize=2)
+ )
+ result = concat(results)
+ tm.assert_frame_equal(expected, result)
+
+
+def test_select_iterator_complete_8014(temp_hdfstore):
+ # GH 8014
+ # using iterator and where clause
+ # no iterator
+ expected = DataFrame(
+ np.random.default_rng(2).standard_normal((100064, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=100064, freq="s", unit="ns"),
+ )
+ temp_hdfstore.append("df", expected)
+
+ beg_dt = expected.index[0]
+ end_dt = expected.index[-1]
+
+ # select w/o iteration and no where clause works
+ result = temp_hdfstore.select("df")
+ tm.assert_frame_equal(expected, result)
+
+ # select w/o iterator and where clause, single term, begin
+ # of range, works
+ where = f"index >= '{beg_dt}'"
+ result = temp_hdfstore.select("df", where=where)
+ tm.assert_frame_equal(expected, result)
+
+ # select w/o iterator and where clause, single term, end
+ # of range, works
+ where = f"index <= '{end_dt}'"
+ result = temp_hdfstore.select("df", where=where)
+ tm.assert_frame_equal(expected, result)
+
+ # select w/o iterator and where clause, inclusive range,
+ # works
+ where = f"index >= '{beg_dt}' & index <= '{end_dt}'"
+ result = temp_hdfstore.select("df", where=where)
+ tm.assert_frame_equal(expected, result)
+
+
+def test_select_iterator_complete_8014_full_range(temp_hdfstore):
+ # GH 8014
+ chunksize = 1e4
+ expected = DataFrame(
+ np.random.default_rng(2).standard_normal((100064, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=100064, freq="s", unit="ns"),
+ )
+ temp_hdfstore.append("df", expected)
+
+ beg_dt = expected.index[0]
+ end_dt = expected.index[-1]
+
+ # select w/iterator and no where clause works
+ results = list(temp_hdfstore.select("df", chunksize=chunksize))
+ result = concat(results)
+ tm.assert_frame_equal(expected, result)
+
+ # select w/iterator and where clause, single term, begin of range
+ where = f"index >= '{beg_dt}'"
+ results = list(temp_hdfstore.select("df", where=where, chunksize=chunksize))
+ result = concat(results)
+ tm.assert_frame_equal(expected, result)
+
+ # select w/iterator and where clause, single term, end of range
+ where = f"index <= '{end_dt}'"
+ results = list(temp_hdfstore.select("df", where=where, chunksize=chunksize))
+ result = concat(results)
+ tm.assert_frame_equal(expected, result)
+
+ # select w/iterator and where clause, inclusive range
+ where = f"index >= '{beg_dt}' & index <= '{end_dt}'"
+ results = list(temp_hdfstore.select("df", where=where, chunksize=chunksize))
+ result = concat(results)
+ tm.assert_frame_equal(expected, result)
+
+
+def test_select_iterator_non_complete_8014(temp_hdfstore):
+ # GH 8014
+ # using iterator and where clause
+ chunksize = 1e4
+
+ # with iterator, non complete range
+ expected = DataFrame(
+ np.random.default_rng(2).standard_normal((100064, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=100064, freq="s", unit="ns"),
+ )
+ temp_hdfstore.append("df", expected)
+
+ beg_dt = expected.index[1]
+ end_dt = expected.index[-2]
+
+ # select w/iterator and where clause, single term, begin of range
+ where = f"index >= '{beg_dt}'"
+ results = list(temp_hdfstore.select("df", where=where, chunksize=chunksize))
+ result = concat(results)
+ rexpected = expected[expected.index >= beg_dt]
+ tm.assert_frame_equal(rexpected, result)
+
+ # select w/iterator and where clause, single term, end of range
+ where = f"index <= '{end_dt}'"
+ results = list(temp_hdfstore.select("df", where=where, chunksize=chunksize))
+ result = concat(results)
+ rexpected = expected[expected.index <= end_dt]
+ tm.assert_frame_equal(rexpected, result)
+
+ # select w/iterator and where clause, inclusive range
+ where = f"index >= '{beg_dt}' & index <= '{end_dt}'"
+ results = list(temp_hdfstore.select("df", where=where, chunksize=chunksize))
+ result = concat(results)
+ rexpected = expected[(expected.index >= beg_dt) & (expected.index <= end_dt)]
+ tm.assert_frame_equal(rexpected, result)
+
+
+def test_select_iterator_non_complete_8014_empty_where(temp_hdfstore):
+ chunksize = 1e4
+ expected = DataFrame(
+ np.random.default_rng(2).standard_normal((100064, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=100064, freq="s", unit="ns"),
+ )
+ temp_hdfstore.append("df", expected)
+
+ end_dt = expected.index[-1]
+
+ # select w/iterator and where clause, single term, begin of range
+ where = f"index > '{end_dt}'"
+ results = list(temp_hdfstore.select("df", where=where, chunksize=chunksize))
+ assert 0 == len(results)
+
+
+def test_select_iterator_many_empty_frames(temp_hdfstore):
+ # GH 8014
+ # using iterator and where clause can return many empty
+ # frames.
+ chunksize = 10_000
+
+ # with iterator, range limited to the first chunk
+ expected = DataFrame(
+ np.random.default_rng(2).standard_normal((100064, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=100064, freq="s", unit="ns"),
+ )
+ temp_hdfstore.append("df", expected)
+
+ beg_dt = expected.index[0]
+ end_dt = expected.index[chunksize - 1]
+
+ # select w/iterator and where clause, single term, begin of range
+ where = f"index >= '{beg_dt}'"
+ results = list(temp_hdfstore.select("df", where=where, chunksize=chunksize))
+ result = concat(results)
+ rexpected = expected[expected.index >= beg_dt]
+ tm.assert_frame_equal(rexpected, result)
+
+ # select w/iterator and where clause, single term, end of range
+ where = f"index <= '{end_dt}'"
+ results = list(temp_hdfstore.select("df", where=where, chunksize=chunksize))
+
+ assert len(results) == 1
+ result = concat(results)
+ rexpected = expected[expected.index <= end_dt]
+ tm.assert_frame_equal(rexpected, result)
+
+ # select w/iterator and where clause, inclusive range
+ where = f"index >= '{beg_dt}' & index <= '{end_dt}'"
+ results = list(temp_hdfstore.select("df", where=where, chunksize=chunksize))
+
+ # should be 1, is 10
+ assert len(results) == 1
+ result = concat(results)
+ rexpected = expected[(expected.index >= beg_dt) & (expected.index <= end_dt)]
+ tm.assert_frame_equal(rexpected, result)
+
+ # select w/iterator and where clause which selects
+ # *nothing*.
+ #
+ # To be consistent with Python idiom I suggest this should
+ # return [] e.g. `for e in []: print True` never prints
+ # True.
+
+ where = f"index <= '{beg_dt}' & index >= '{end_dt}'"
+ results = list(temp_hdfstore.select("df", where=where, chunksize=chunksize))
+
+ # should be []
+ assert len(results) == 0
+
+
+def test_frame_select(temp_hdfstore, request):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B", unit="ns"),
+ )
+
+ temp_hdfstore.put("frame", df, format="table")
+ date = df.index[len(df) // 2]
+
+ crit1 = Term("index>=date")
+ assert crit1.env.scope["date"] == date
+
+ crit2 = "columns=['A', 'D']"
+ crit3 = "columns=A"
+
+ request.applymarker(
+ pytest.mark.xfail(
+ PY312,
+ reason="AST change in PY312",
+ raises=TypeError,
+ )
+ )
+ result = temp_hdfstore.select("frame", [crit1, crit2])
+ expected = df.loc[date:, ["A", "D"]]
+ tm.assert_frame_equal(result, expected)
+
+ result = temp_hdfstore.select("frame", [crit3])
+ expected = df.loc[:, ["A"]]
+ tm.assert_frame_equal(result, expected)
+
+ # invalid terms
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B", unit="ns"),
+ )
+ temp_hdfstore.append("df_time", df)
+ msg = "day is out of range for month: 0"
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.select("df_time", "index>0")
+
+ # can't select if not written as table
+ # temp_hdfstore['frame'] = df
+ # with pytest.raises(ValueError):
+ # temp_hdfstore.select('frame', [crit1, crit2])
+
+
+def test_frame_select_complex(temp_hdfstore):
+ # select via complex criteria
+
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B", unit="ns"),
+ )
+ df["string"] = "foo"
+ df.loc[df.index[0:4], "string"] = "bar"
+
+ temp_hdfstore.put("df", df, format="table", data_columns=["string"])
+
+ # empty
+ result = temp_hdfstore.select("df", 'index>df.index[3] & string="bar"')
+ expected = df.loc[(df.index > df.index[3]) & (df.string == "bar")]
+ tm.assert_frame_equal(result, expected)
+
+ result = temp_hdfstore.select("df", 'index>df.index[3] & string="foo"')
+ expected = df.loc[(df.index > df.index[3]) & (df.string == "foo")]
+ tm.assert_frame_equal(result, expected)
+
+ # or
+ result = temp_hdfstore.select("df", 'index>df.index[3] | string="bar"')
+ expected = df.loc[(df.index > df.index[3]) | (df.string == "bar")]
+ tm.assert_frame_equal(result, expected)
+
+ result = temp_hdfstore.select(
+ "df", '(index>df.index[3] & index<=df.index[6]) | string="bar"'
+ )
+ expected = df.loc[
+ ((df.index > df.index[3]) & (df.index <= df.index[6])) | (df.string == "bar")
+ ]
+ tm.assert_frame_equal(result, expected)
+
+ # invert
+ result = temp_hdfstore.select("df", 'string!="bar"')
+ expected = df.loc[df.string != "bar"]
+ tm.assert_frame_equal(result, expected)
+
+ # invert not implemented in numexpr :(
+ msg = "cannot use an invert condition when passing to numexpr"
+ with pytest.raises(NotImplementedError, match=msg):
+ temp_hdfstore.select("df", '~(string="bar")')
+
+ # invert ok for filters
+ result = temp_hdfstore.select("df", "~(columns=['A','B'])")
+ expected = df.loc[:, df.columns.difference(["A", "B"])]
+ tm.assert_frame_equal(result, expected)
+
+ # in
+ result = temp_hdfstore.select("df", "index>df.index[3] & columns in ['A','B']")
+ expected = df.loc[df.index > df.index[3]].reindex(columns=["A", "B"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_frame_select_complex2(tmp_path):
+ pp = tmp_path / "params.hdf"
+ hh = tmp_path / "hist.hdf"
+
+ # use non-trivial selection criteria
+ params = DataFrame({"A": [1, 1, 2, 2, 3]})
+ params.to_hdf(pp, key="df", mode="w", format="table", data_columns=["A"])
+
+ selection = read_hdf(pp, "df", where="A=[2,3]")
+ hist = DataFrame(
+ np.random.default_rng(2).standard_normal((25, 1)),
+ columns=["data"],
+ index=MultiIndex.from_tuples(
+ [(i, j) for i in range(5) for j in range(5)], names=["l1", "l2"]
+ ),
+ )
+
+ hist.to_hdf(hh, key="df", mode="w", format="table")
+
+ expected = read_hdf(hh, "df", where="l1=[2, 3, 4]")
+
+ # scope with list like
+ l0 = selection.index.tolist() # noqa: F841
+ with HDFStore(hh) as store:
+ result = store.select("df", where="l1=l0")
+ tm.assert_frame_equal(result, expected)
+
+ result = read_hdf(hh, "df", where="l1=l0")
+ tm.assert_frame_equal(result, expected)
+
+ # index
+ index = selection.index # noqa: F841
+ result = read_hdf(hh, "df", where="l1=index")
+ tm.assert_frame_equal(result, expected)
+
+ result = read_hdf(hh, "df", where="l1=selection.index")
+ tm.assert_frame_equal(result, expected)
+
+ result = read_hdf(hh, "df", where="l1=selection.index.tolist()")
+ tm.assert_frame_equal(result, expected)
+
+ result = read_hdf(hh, "df", where="l1=list(selection.index)")
+ tm.assert_frame_equal(result, expected)
+
+ # scope with index
+ with HDFStore(hh) as store:
+ result = store.select("df", where="l1=index")
+ tm.assert_frame_equal(result, expected)
+
+ result = store.select("df", where="l1=selection.index")
+ tm.assert_frame_equal(result, expected)
+
+ result = store.select("df", where="l1=selection.index.tolist()")
+ tm.assert_frame_equal(result, expected)
+
+ result = store.select("df", where="l1=list(selection.index)")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_invalid_filtering(temp_hdfstore):
+ # can't use more than one filter (atm)
+
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B", unit="ns"),
+ )
+
+ temp_hdfstore.put("df", df, format="table")
+
+ msg = "unable to collapse Joint Filters"
+ # not implemented
+ with pytest.raises(NotImplementedError, match=msg):
+ temp_hdfstore.select("df", "columns=['A'] | columns=['B']")
+
+ # in theory we could deal with this
+ with pytest.raises(NotImplementedError, match=msg):
+ temp_hdfstore.select("df", "columns=['A','B'] & columns=['C']")
+
+
+def test_string_select(temp_hdfstore):
+ # GH 2973
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B", unit="ns"),
+ )
+
+ # test string ==/!=
+ df["x"] = "none"
+ df.loc[df.index[2:7], "x"] = ""
+
+ temp_hdfstore.append("df", df, data_columns=["x"])
+
+ result = temp_hdfstore.select("df", "x=none")
+ expected = df[df.x == "none"]
+ tm.assert_frame_equal(result, expected)
+
+ result = temp_hdfstore.select("df", "x!=none")
+ expected = df[df.x != "none"]
+ tm.assert_frame_equal(result, expected)
+
+ df2 = df.copy()
+ df2.loc[df2.x == "", "x"] = np.nan
+
+ temp_hdfstore.append("df2", df2, data_columns=["x"])
+ result = temp_hdfstore.select("df2", "x!=none")
+ expected = df2[isna(df2.x)]
+ tm.assert_frame_equal(result, expected)
+
+ # int ==/!=
+ df["int"] = 1
+ df.loc[df.index[2:7], "int"] = 2
+
+ temp_hdfstore.append("df3", df, data_columns=["int"])
+
+ result = temp_hdfstore.select("df3", "int=2")
+ expected = df[df.int == 2]
+ tm.assert_frame_equal(result, expected)
+
+ result = temp_hdfstore.select("df3", "int!=2")
+ expected = df[df.int != 2]
+ tm.assert_frame_equal(result, expected)
+
+
+def test_select_as_multiple(temp_hdfstore):
+ df1 = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B", unit="ns"),
+ )
+ df2 = df1.copy().rename(columns="{}_2".format)
+ df2["foo"] = "bar"
+
+ msg = "keys must be a list/tuple"
+ # no tables stored
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.select_as_multiple(None, where=["A>0", "B>0"], selector="df1")
+
+ temp_hdfstore.append("df1", df1, data_columns=["A", "B"])
+ temp_hdfstore.append("df2", df2)
+
+ # exceptions
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.select_as_multiple(None, where=["A>0", "B>0"], selector="df1")
+
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.select_as_multiple([None], where=["A>0", "B>0"], selector="df1")
+
+ msg = "'No object named df3 in the file'"
+ with pytest.raises(KeyError, match=msg):
+ temp_hdfstore.select_as_multiple(
+ ["df1", "df3"], where=["A>0", "B>0"], selector="df1"
+ )
+
+ with pytest.raises(KeyError, match=msg):
+ temp_hdfstore.select_as_multiple(["df3"], where=["A>0", "B>0"], selector="df1")
+
+ with pytest.raises(KeyError, match="'No object named df4 in the file'"):
+ temp_hdfstore.select_as_multiple(
+ ["df1", "df2"], where=["A>0", "B>0"], selector="df4"
+ )
+
+ # default select
+ result = temp_hdfstore.select("df1", ["A>0", "B>0"])
+ expected = temp_hdfstore.select_as_multiple(
+ ["df1"], where=["A>0", "B>0"], selector="df1"
+ )
+ tm.assert_frame_equal(result, expected)
+ expected = temp_hdfstore.select_as_multiple(
+ "df1", where=["A>0", "B>0"], selector="df1"
+ )
+ tm.assert_frame_equal(result, expected)
+
+ # multiple
+ result = temp_hdfstore.select_as_multiple(
+ ["df1", "df2"], where=["A>0", "B>0"], selector="df1"
+ )
+ expected = concat([df1, df2], axis=1)
+ expected = expected[(expected.A > 0) & (expected.B > 0)]
+ tm.assert_frame_equal(result, expected, check_freq=False)
+ # FIXME: 2021-01-20 this is failing with freq None vs 4B on some builds
+
+ # multiple (diff selector)
+ result = temp_hdfstore.select_as_multiple(
+ ["df1", "df2"], where="index>df2.index[4]", selector="df2"
+ )
+ expected = concat([df1, df2], axis=1)
+ expected = expected[5:]
+ tm.assert_frame_equal(result, expected)
+
+ # test exception for diff rows
+ df3 = df1.copy().head(2)
+ temp_hdfstore.append("df3", df3)
+ msg = "all tables must have exactly the same nrows!"
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.select_as_multiple(
+ ["df1", "df3"], where=["A>0", "B>0"], selector="df1"
+ )
+
+
+def test_nan_selection_bug_4858(temp_hdfstore):
+ df = DataFrame({"cols": range(6), "values": range(6)}, dtype="float64")
+ df["cols"] = (df["cols"] + 10).apply(str)
+ df.iloc[0] = np.nan
+
+ expected = DataFrame(
+ {"cols": ["13.0", "14.0", "15.0"], "values": [3.0, 4.0, 5.0]},
+ index=[3, 4, 5],
+ )
+
+ # write w/o the index on that particular column
+ temp_hdfstore.append("df", df, data_columns=True, index=["cols"])
+ result = temp_hdfstore.select("df", where="values>2.0")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_query_with_nested_special_character(temp_hdfstore):
+ df = DataFrame(
+ {
+ "a": ["a", "a", "c", "b", "test & test", "c", "b", "e"],
+ "b": [1, 2, 3, 4, 5, 6, 7, 8],
+ }
+ )
+ expected = df[df.a == "test & test"]
+ temp_hdfstore.append("test", df, format="table", data_columns=True)
+ result = temp_hdfstore.select("test", 'a = "test & test"')
+ tm.assert_frame_equal(expected, result)
+
+
+def test_query_long_float_literal(temp_hdfstore):
+ # GH 14241
+ df = DataFrame({"A": [1000000000.0009, 1000000000.0011, 1000000000.0015]})
+
+ temp_hdfstore.append("test", df, format="table", data_columns=True)
+
+ cutoff = 1000000000.0006
+ result = temp_hdfstore.select("test", f"A < {cutoff:.4f}")
+ assert result.empty
+
+ cutoff = 1000000000.0010
+ result = temp_hdfstore.select("test", f"A > {cutoff:.4f}")
+ expected = df.loc[[1, 2], :]
+ tm.assert_frame_equal(expected, result)
+
+ exact = 1000000000.0011
+ result = temp_hdfstore.select("test", f"A == {exact:.4f}")
+ expected = df.loc[[1], :]
+ tm.assert_frame_equal(expected, result)
+
+
+def test_query_compare_column_type(temp_hdfstore):
+ # GH 15492
+ df = DataFrame(
+ {
+ "date": ["2014-01-01", "2014-01-02"],
+ "real_date": date_range("2014-01-01", periods=2, unit="ns"),
+ "float": [1.1, 1.2],
+ "int": [1, 2],
+ },
+ columns=["date", "real_date", "float", "int"],
+ )
+
+ temp_hdfstore.append("test", df, format="table", data_columns=True)
+
+ ts = Timestamp("2014-01-01") # noqa: F841
+ result = temp_hdfstore.select("test", where="real_date > ts")
+ expected = df.loc[[1], :]
+ tm.assert_frame_equal(expected, result)
+
+ for op in ["<", ">", "=="]:
+ # non strings to string column always fail
+ for v in [2.1, True, Timestamp("2014-01-01"), pd.Timedelta(1, "s")]:
+ query = f"date {op} v"
+ msg = f"Cannot compare {v} of type {type(v)} to string column"
+ with pytest.raises(TypeError, match=msg):
+ temp_hdfstore.select("test", where=query)
+
+ # strings to other columns must be convertible to type
+ v = "a"
+ for col in ["int", "float", "real_date"]:
+ query = f"{col} {op} v"
+ if col == "real_date":
+ msg = 'Given date string "a" not likely a datetime'
+ else:
+ msg = "could not convert string to"
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.select("test", where=query)
+
+ for v, col in zip(["1", "1.1", "2014-01-01"], ["int", "float", "real_date"]):
+ query = f"{col} {op} v"
+ result = temp_hdfstore.select("test", where=query)
+
+ if op == "==":
+ expected = df.loc[[0], :]
+ elif op == ">":
+ expected = df.loc[[1], :]
+ else:
+ expected = df.loc[[], :]
+ tm.assert_frame_equal(expected, result)
+
+
+@pytest.mark.parametrize("where", ["", (), (None,), [], [None]])
+def test_select_empty_where(temp_hdfstore, where):
+ # GH26610
+
+ df = DataFrame([1, 2, 3])
+ temp_hdfstore.put("df", df, "t")
+ result = read_hdf(temp_hdfstore, "df", where=where)
+ tm.assert_frame_equal(result, df)
+
+
+def test_select_large_integer(temp_hdfstore):
+ df = DataFrame(
+ zip(
+ ["a", "b", "c", "d"],
+ [-9223372036854775801, -9223372036854775802, -9223372036854775803, 123],
+ ),
+ columns=["x", "y"],
+ )
+ temp_hdfstore.append("data", df, data_columns=True, index=False)
+ result = (
+ temp_hdfstore.select("data", where="y==-9223372036854775801").get("y").get(0)
+ )
+ expected = df["y"][0]
+
+ assert expected == result
+
+
+@pytest.mark.parametrize("unit", ["us", "ns", "ms", "s"])
+def test_select_where_datetime_index_with_non_ns_resolution(temp_hdfstore, unit):
+ # GH#64310
+ idx = date_range("2020-01-01", periods=10, freq="h", unit=unit)
+ df = DataFrame({"value": range(10)}, index=idx)
+
+ temp_hdfstore.append("df", df)
+
+ result = temp_hdfstore.select(
+ "df", where="index >= '2020-01-02' & index <= '2020-01-03'"
+ )
+ expected = df[(df.index >= "2020-01-02") & (df.index <= "2020-01-03")]
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_store.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_store.py
new file mode 100644
index 0000000000000000000000000000000000000000..7274acc7b4cd84b312bdbd86beab634fc29679e1
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_store.py
@@ -0,0 +1,1102 @@
+import contextlib
+import datetime as dt
+import hashlib
+import tempfile
+import time
+
+import numpy as np
+import pytest
+
+from pandas.compat import PY312
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ DatetimeIndex,
+ Index,
+ MultiIndex,
+ Series,
+ Timestamp,
+ concat,
+ date_range,
+ period_range,
+ timedelta_range,
+)
+import pandas._testing as tm
+from pandas.api.types import (
+ CategoricalDtype,
+)
+from pandas.util.version import Version
+
+from pandas.io.pytables import (
+ HDFStore,
+ read_hdf,
+)
+
+pytestmark = [pytest.mark.single_cpu]
+
+tables = pytest.importorskip("tables")
+
+
+def test_context(temp_h5_path):
+ try:
+ with HDFStore(temp_h5_path) as tbl:
+ raise ValueError("blah")
+ except ValueError:
+ pass
+ with HDFStore(temp_h5_path) as tbl:
+ tbl["a"] = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ assert len(tbl) == 1
+ assert type(tbl["a"]) == DataFrame
+
+
+@pytest.mark.xfail(
+ Version(tables.hdf5_version) >= Version("2"),
+ reason="track_times=False produces non-deterministic files with HDF5 >= 2",
+)
+def test_no_track_times(temp_h5_path):
+ # GH 32682
+ # enables to set track_times (see `pytables` `create_table` documentation)
+
+ def checksum(filename, hash_factory=hashlib.md5, chunk_num_blocks=128):
+ h = hash_factory()
+ with open(filename, "rb") as f:
+ for chunk in iter(lambda: f.read(chunk_num_blocks * h.block_size), b""):
+ h.update(chunk)
+ return h.digest()
+
+ def create_h5_and_return_checksum(temp_h5_path, track_times):
+ df = DataFrame({"a": [1]})
+
+ with HDFStore(temp_h5_path, mode="w") as hdf:
+ hdf.put(
+ "table",
+ df,
+ format="table",
+ data_columns=True,
+ index=None,
+ track_times=track_times,
+ )
+
+ return checksum(temp_h5_path)
+
+ checksum_0_tt_false = create_h5_and_return_checksum(temp_h5_path, track_times=False)
+ checksum_0_tt_true = create_h5_and_return_checksum(temp_h5_path, track_times=True)
+
+ # sleep is necessary to create h5 with different creation time
+ time.sleep(1)
+
+ checksum_1_tt_false = create_h5_and_return_checksum(temp_h5_path, track_times=False)
+ checksum_1_tt_true = create_h5_and_return_checksum(temp_h5_path, track_times=True)
+
+ # checksums are the same if track_time = False
+ assert checksum_0_tt_false == checksum_1_tt_false
+
+ # checksums are NOT same if track_time = True
+ assert checksum_0_tt_true != checksum_1_tt_true
+
+
+def test_iter_empty(temp_hdfstore):
+ # GH 12221
+ assert list(temp_hdfstore) == []
+
+
+def test_repr(temp_hdfstore, performance_warning, using_infer_string):
+ store = temp_hdfstore
+ repr(store)
+ store.info()
+ store["a"] = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ store["b"] = Series(range(10), dtype="float64", index=[f"i_{i}" for i in range(10)])
+ store["c"] = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ df["obj1"] = "foo"
+ df["obj2"] = "bar"
+ df["bool1"] = df["A"] > 0
+ df["bool2"] = df["B"] > 0
+ df["bool3"] = True
+ df["int1"] = 1
+ df["int2"] = 2
+ df["timestamp1"] = Timestamp("20010102")
+ df["timestamp2"] = Timestamp("20010103")
+ df["datetime1"] = dt.datetime(2001, 1, 2, 0, 0)
+ df["datetime2"] = dt.datetime(2001, 1, 3, 0, 0)
+ df.loc[df.index[3:6], ["obj1"]] = np.nan
+ df = df._consolidate()
+
+ warning = None if using_infer_string else performance_warning
+ msg = "cannot\nmap directly to c-types .* dtype='object'"
+ with tm.assert_produces_warning(warning, match=msg):
+ store["df"] = df
+
+ # make a random group in hdf space
+ store._handle.create_group(store._handle.root, "bah")
+
+ assert store.filename in repr(store)
+ assert store.filename in str(store)
+ store.info()
+
+
+def test_repr_get_storer(temp_hdfstore):
+ # storers
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ temp_hdfstore.append("df", df)
+
+ s = temp_hdfstore.get_storer("df")
+ repr(s)
+ str(s)
+
+
+def test_contains(temp_hdfstore):
+ store = temp_hdfstore
+ store["a"] = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ store["b"] = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ store["foo/bar"] = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ assert "a" in store
+ assert "b" in store
+ assert "c" not in store
+ assert "foo/bar" in store
+ assert "/foo/bar" in store
+ assert "/foo/b" not in store
+ assert "bar" not in store
+
+ # gh-2694: tables.NaturalNameWarning
+ with tm.assert_produces_warning(tables.NaturalNameWarning, check_stacklevel=False):
+ store["node())"] = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ assert "node())" in store
+
+
+def test_versioning(temp_hdfstore):
+ store = temp_hdfstore
+ store["a"] = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ store["b"] = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((20, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=20, freq="B"),
+ )
+ store.append("df1", df[:10])
+ store.append("df1", df[10:])
+ assert store.root.a._v_attrs.pandas_version == "0.15.2"
+ assert store.root.b._v_attrs.pandas_version == "0.15.2"
+ assert store.root.df1._v_attrs.pandas_version == "0.15.2"
+
+ # write a file and wipe its versioning
+ store.append("df2", df)
+
+ # this is an error because its table_type is appendable, but no
+ # version info
+ store.get_node("df2")._v_attrs.pandas_version = None
+
+ msg = "'NoneType' object has no attribute 'startswith'"
+
+ with pytest.raises(Exception, match=msg):
+ store.select("df2")
+
+
+@pytest.mark.parametrize(
+ "where, expected",
+ [
+ (
+ "/",
+ {
+ "": ({"first_group", "second_group"}, set()),
+ "/first_group": (set(), {"df1", "df2"}),
+ "/second_group": ({"third_group"}, {"df3", "s1"}),
+ "/second_group/third_group": (set(), {"df4"}),
+ },
+ ),
+ (
+ "/second_group",
+ {
+ "/second_group": ({"third_group"}, {"df3", "s1"}),
+ "/second_group/third_group": (set(), {"df4"}),
+ },
+ ),
+ ],
+)
+def test_walk(where, expected, temp_hdfstore):
+ # GH10143
+ objs = {
+ "df1": DataFrame([1, 2, 3]),
+ "df2": DataFrame([4, 5, 6]),
+ "df3": DataFrame([6, 7, 8]),
+ "df4": DataFrame([9, 10, 11]),
+ "s1": Series([10, 9, 8]),
+ # Next 3 items aren't pandas objects and should be ignored
+ "a1": np.array([[1, 2, 3], [4, 5, 6]]),
+ "tb1": np.array([(1, 2, 3), (4, 5, 6)], dtype="i,i,i"),
+ "tb2": np.array([(7, 8, 9), (10, 11, 12)], dtype="i,i,i"),
+ }
+
+ store = temp_hdfstore
+ store.put("/first_group/df1", objs["df1"])
+ store.put("/first_group/df2", objs["df2"])
+ store.put("/second_group/df3", objs["df3"])
+ store.put("/second_group/s1", objs["s1"])
+ store.put("/second_group/third_group/df4", objs["df4"])
+ # Create non-pandas objects
+ store._handle.create_array("/first_group", "a1", objs["a1"])
+ store._handle.create_table("/first_group", "tb1", obj=objs["tb1"])
+ store._handle.create_table("/second_group", "tb2", obj=objs["tb2"])
+
+ assert len(list(store.walk(where=where))) == len(expected)
+ for path, groups, leaves in store.walk(where=where):
+ assert path in expected
+ expected_groups, expected_frames = expected[path]
+ assert expected_groups == set(groups)
+ assert expected_frames == set(leaves)
+ for leaf in leaves:
+ frame_path = "/".join([path, leaf])
+ obj = store.get(frame_path)
+ if "df" in leaf:
+ tm.assert_frame_equal(obj, objs[leaf])
+ else:
+ tm.assert_series_equal(obj, objs[leaf])
+
+
+def test_getattr(temp_hdfstore):
+ store = temp_hdfstore
+ s = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ store["a"] = s
+
+ # test attribute access
+ result = store.a
+ tm.assert_series_equal(result, s)
+ result = store.a
+ tm.assert_series_equal(result, s)
+
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ store["df"] = df
+ result = store.df
+ tm.assert_frame_equal(result, df)
+
+ # errors
+ for x in ["d", "mode", "path", "handle", "complib"]:
+ msg = f"'HDFStore' object has no attribute '{x}'"
+ with pytest.raises(AttributeError, match=msg):
+ getattr(store, x)
+
+ # not stores
+ for x in ["mode", "path", "handle", "complib"]:
+ getattr(store, f"_{x}")
+
+
+def test_store_dropna(temp_h5_path):
+ df_with_missing = DataFrame(
+ {"col1": [0.0, np.nan, 2.0], "col2": [1.0, np.nan, np.nan]},
+ index=list("abc"),
+ )
+ df_without_missing = DataFrame(
+ {"col1": [0.0, 2.0], "col2": [1.0, np.nan]}, index=list("ac")
+ )
+
+ # # Test to make sure defaults are to not drop.
+ # # Corresponding to Issue 9382
+ df_with_missing.to_hdf(temp_h5_path, key="df", format="table")
+ reloaded = read_hdf(temp_h5_path, "df")
+ tm.assert_frame_equal(df_with_missing, reloaded)
+
+ df_with_missing.to_hdf(temp_h5_path, key="df", format="table", dropna=False)
+ reloaded = read_hdf(temp_h5_path, "df")
+ tm.assert_frame_equal(df_with_missing, reloaded)
+
+ df_with_missing.to_hdf(temp_h5_path, key="df", format="table", dropna=True)
+ reloaded = read_hdf(temp_h5_path, "df")
+ tm.assert_frame_equal(df_without_missing, reloaded)
+
+
+def test_to_hdf_with_min_itemsize(temp_h5_path):
+ # min_itemsize in index with to_hdf (GH 10381)
+ df = DataFrame(
+ {
+ "A": [0.0, 1.0, 2.0, 3.0, 4.0],
+ "B": [0.0, 1.0, 0.0, 1.0, 0.0],
+ "C": Index(["foo1", "foo2", "foo3", "foo4", "foo5"]),
+ "D": date_range("20130101", periods=5),
+ }
+ ).set_index("C")
+ df.to_hdf(temp_h5_path, key="ss3", format="table", min_itemsize={"index": 6})
+ # just make sure there is a longer string:
+ df2 = df.copy().reset_index().assign(C="longer").set_index("C")
+ df2.to_hdf(temp_h5_path, key="ss3", append=True, format="table")
+ tm.assert_frame_equal(read_hdf(temp_h5_path, "ss3"), concat([df, df2]))
+
+ # same as above, with a Series
+ df["B"].to_hdf(temp_h5_path, key="ss4", format="table", min_itemsize={"index": 6})
+ df2["B"].to_hdf(temp_h5_path, key="ss4", append=True, format="table")
+ tm.assert_series_equal(read_hdf(temp_h5_path, "ss4"), concat([df["B"], df2["B"]]))
+
+
+@pytest.mark.parametrize("format", ["fixed", "table"])
+def test_to_hdf_errors(format, temp_h5_path, using_infer_string):
+ data = ["\ud800foo"]
+ ser = Series(data, index=Index(data, dtype="object"), dtype="object")
+ # GH 20835
+ ser.to_hdf(temp_h5_path, key="table", format=format, errors="surrogatepass")
+
+ result = read_hdf(temp_h5_path, "table", errors="surrogatepass")
+
+ if using_infer_string:
+ # https://github.com/pandas-dev/pandas/pull/60993
+ # Surrogates fallback to python storage.
+ dtype = pd.StringDtype(storage="python", na_value=np.nan)
+ else:
+ dtype = "object"
+ expected = Series(data, index=Index(data, dtype=dtype), dtype=dtype)
+ tm.assert_series_equal(result, expected)
+
+
+def test_create_table_index(temp_hdfstore):
+ store = temp_hdfstore
+
+ def col(t, column):
+ return getattr(store.get_storer(t).table.cols, column)
+
+ # data columns
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ df["string"] = "foo"
+ df["string2"] = "bar"
+ store.append("f", df, data_columns=["string", "string2"])
+ assert col("f", "index").is_indexed is True
+ assert col("f", "string").is_indexed is True
+ assert col("f", "string2").is_indexed is True
+
+ # specify index=columns
+ store.append("f2", df, index=["string"], data_columns=["string", "string2"])
+ assert col("f2", "index").is_indexed is False
+ assert col("f2", "string").is_indexed is True
+ assert col("f2", "string2").is_indexed is False
+
+ # try to index a non-table
+ store.put("f2", df)
+ msg = "cannot create table index on a Fixed format store"
+ with pytest.raises(TypeError, match=msg):
+ store.create_table_index("f2")
+
+
+def test_create_table_index_data_columns_argument(temp_hdfstore):
+ # GH 28156
+
+ store = temp_hdfstore
+
+ def col(t, column):
+ return getattr(store.get_storer(t).table.cols, column)
+
+ # data columns
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ df["string"] = "foo"
+ df["string2"] = "bar"
+ store.append("f", df, data_columns=["string"])
+ assert col("f", "index").is_indexed is True
+ assert col("f", "string").is_indexed is True
+
+ msg = "'Cols' object has no attribute 'string2'"
+ with pytest.raises(AttributeError, match=msg):
+ col("f", "string2").is_indexed
+
+ # try to index a col which isn't a data_column
+ msg = (
+ "column string2 is not a data_column.\n"
+ "In order to read column string2 you must reload the dataframe \n"
+ "into HDFStore and include string2 with the data_columns argument."
+ )
+ with pytest.raises(AttributeError, match=msg):
+ store.create_table_index("f", columns=["string2"])
+
+
+def test_mi_data_columns(temp_hdfstore):
+ # GH 14435
+ idx = MultiIndex.from_arrays(
+ [date_range("2000-01-01", periods=5), range(5)], names=["date", "id"]
+ )
+ df = DataFrame({"a": [1.1, 1.2, 1.3, 1.4, 1.5]}, index=idx)
+
+ temp_hdfstore.append("df", df, data_columns=True)
+
+ actual = temp_hdfstore.select("df", where="id == 1")
+ expected = df.iloc[[1], :]
+ tm.assert_frame_equal(actual, expected)
+
+
+def test_table_mixed_dtypes(temp_hdfstore):
+ # frame
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ df["obj1"] = "foo"
+ df["obj2"] = "bar"
+ df["bool1"] = df["A"] > 0
+ df["bool2"] = df["B"] > 0
+ df["bool3"] = True
+ df["int1"] = 1
+ df["int2"] = 2
+ df["timestamp1"] = Timestamp("20010102").as_unit("ns")
+ df["timestamp2"] = Timestamp("20010103").as_unit("ns")
+ df["datetime1"] = Timestamp("20010102").as_unit("ns")
+ df["datetime2"] = Timestamp("20010103").as_unit("ns")
+ df.loc[df.index[3:6], ["obj1"]] = np.nan
+ df = df._consolidate()
+
+ temp_hdfstore.append("df1_mixed", df)
+ tm.assert_frame_equal(temp_hdfstore.select("df1_mixed"), df)
+
+
+def test_calendar_roundtrip_issue(temp_hdfstore):
+ # 8591
+ # doc example from tseries holiday section
+ weekmask_egypt = "Sun Mon Tue Wed Thu"
+ holidays = [
+ "2012-05-01",
+ dt.datetime(2013, 5, 1),
+ np.datetime64("2014-05-01"),
+ ]
+ bday_egypt = pd.offsets.CustomBusinessDay(
+ holidays=holidays, weekmask=weekmask_egypt
+ )
+ mydt = dt.datetime(2013, 4, 30)
+ dts = date_range(mydt, periods=5, freq=bday_egypt)
+
+ s = Series(dts.weekday, dts).map(Series("Mon Tue Wed Thu Fri Sat Sun".split()))
+
+ temp_hdfstore.put("fixed", s)
+ result = temp_hdfstore.select("fixed")
+ tm.assert_series_equal(result, s)
+
+ temp_hdfstore.append("table", s)
+ result = temp_hdfstore.select("table")
+ tm.assert_series_equal(result, s)
+
+
+def test_remove(temp_hdfstore):
+ store = temp_hdfstore
+ ts = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ store["a"] = ts
+ store["b"] = df
+ store.remove("a")
+ assert len(store) == 1
+ tm.assert_frame_equal(df, store["b"])
+
+ store.remove("b")
+ assert len(store) == 0
+
+ # nonexistence
+ with pytest.raises(
+ KeyError, match="'No object named a_nonexistent_store in the file'"
+ ):
+ store.remove("a_nonexistent_store")
+
+ # pathing
+ store["a"] = ts
+ store["b/foo"] = df
+ store.remove("b/foo")
+ assert len(store) == 1
+
+ store["a"] = ts
+ store["b/foo"] = df
+ store.remove("b")
+ assert len(store) == 1
+
+ # __delitem__
+ store["a"] = ts
+ store["b"] = df
+ del store["a"]
+ del store["b"]
+ assert len(store) == 0
+
+
+def test_same_name_scoping(temp_hdfstore):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((20, 2)),
+ index=date_range("20130101", periods=20, unit="ns"),
+ )
+ temp_hdfstore.put("df", df, format="table")
+ expected = df[df.index > Timestamp("20130105")]
+
+ result = temp_hdfstore.select("df", "index>datetime.datetime(2013,1,5)")
+ tm.assert_frame_equal(result, expected)
+
+ # changes what 'datetime' points to in the namespace where
+ # 'select' does the lookup
+
+ # technically an error, but allow it
+ result = temp_hdfstore.select("df", "index>datetime.datetime(2013,1,5)")
+ tm.assert_frame_equal(result, expected)
+
+ result = temp_hdfstore.select("df", "index>datetime(2013,1,5)")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_store_index_name(temp_hdfstore):
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ df.index.name = "foo"
+
+ temp_hdfstore["frame"] = df
+ recons = temp_hdfstore["frame"]
+ tm.assert_frame_equal(recons, df)
+
+
+@pytest.mark.parametrize("tz", [None, "US/Pacific"])
+@pytest.mark.parametrize("table_format", ["table", "fixed"])
+def test_store_index_name_numpy_str(table_format, temp_h5_path, unit, tz):
+ # GH #13492
+ idx = (
+ DatetimeIndex(
+ [dt.date(2000, 1, 1), dt.date(2000, 1, 2)],
+ name="cols\u05d2",
+ )
+ .tz_localize(tz)
+ .as_unit(unit)
+ )
+ idx1 = (
+ DatetimeIndex(
+ [dt.date(2010, 1, 1), dt.date(2010, 1, 2)],
+ name="rows\u05d0",
+ )
+ .as_unit(unit)
+ .tz_localize(tz)
+ )
+ df = DataFrame(np.arange(4).reshape(2, 2), columns=idx, index=idx1)
+
+ # This used to fail, returning numpy strings instead of python strings.
+ df.to_hdf(temp_h5_path, key="df", format=table_format)
+ df2 = read_hdf(temp_h5_path, "df")
+
+ tm.assert_frame_equal(df, df2, check_names=True)
+
+ assert isinstance(df2.index.name, str)
+ assert isinstance(df2.columns.name, str)
+
+
+def test_store_series_name(temp_hdfstore):
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ series = df["A"]
+
+ temp_hdfstore["series"] = series
+ recons = temp_hdfstore["series"]
+ tm.assert_series_equal(recons, series)
+
+
+def test_overwrite_node(temp_hdfstore):
+ temp_hdfstore["a"] = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ ts = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ temp_hdfstore["a"] = ts
+
+ tm.assert_series_equal(temp_hdfstore["a"], ts)
+
+
+def test_coordinates(temp_hdfstore):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+
+ store = temp_hdfstore
+ store.append("df", df)
+
+ # all
+ c = store.select_as_coordinates("df")
+ assert (c.values == np.arange(len(df.index))).all()
+
+
+def test_coordinates2(temp_hdfstore):
+ # get coordinates back & test vs frame
+ store = temp_hdfstore
+
+ df = DataFrame({"A": range(5), "B": range(5)})
+ store.append("df", df)
+ c = store.select_as_coordinates("df", ["index<3"])
+ assert (c.values == np.arange(3)).all()
+ result = store.select("df", where=c)
+ expected = df.loc[0:2, :]
+ tm.assert_frame_equal(result, expected)
+
+ c = store.select_as_coordinates("df", ["index>=3", "index<=4"])
+ assert (c.values == np.arange(2) + 3).all()
+ result = store.select("df", where=c)
+ expected = df.loc[3:4, :]
+ tm.assert_frame_equal(result, expected)
+ assert isinstance(c, Index)
+
+
+def test_coordinates_multiple_tables(temp_hdfstore):
+ store = temp_hdfstore
+ df1 = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ df2 = df1.copy().rename(columns="{}_2".format)
+ store.append("df1", df1, data_columns=["A", "B"])
+ store.append("df2", df2)
+
+ c = store.select_as_coordinates("df1", ["A>0", "B>0"])
+ df1_result = store.select("df1", c)
+ df2_result = store.select("df2", c)
+ result = concat([df1_result, df2_result], axis=1)
+
+ expected = concat([df1, df2], axis=1)
+ expected = expected[(expected.A > 0) & (expected.B > 0)]
+ tm.assert_frame_equal(result, expected, check_freq=False)
+ # FIXME: 2021-01-18 on some (mostly windows) builds we get freq=None
+ # but expect freq="18B"
+
+
+def test_coordinates_array_mask(temp_hdfstore):
+ store = temp_hdfstore
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((1000, 2)),
+ index=date_range("20000101", periods=1000),
+ )
+ store.append("df", df)
+ c = store.select_column("df", "index")
+ where = c[DatetimeIndex(c).month == 5].index
+ expected = df.iloc[where]
+
+ # locations
+ result = store.select("df", where=where)
+ tm.assert_frame_equal(result, expected)
+
+ # boolean
+ result = store.select("df", where=where)
+ tm.assert_frame_equal(result, expected)
+
+ # invalid
+ msg = "where must be passed as a string, PyTablesExpr, or list-like of PyTablesExpr"
+ with pytest.raises(TypeError, match=msg):
+ store.select("df", where=np.arange(len(df), dtype="float64"))
+
+ with pytest.raises(TypeError, match=msg):
+ store.select("df", where=np.arange(len(df) + 1))
+
+ with pytest.raises(TypeError, match=msg):
+ store.select("df", where=np.arange(len(df)), start=5)
+
+ with pytest.raises(TypeError, match=msg):
+ store.select("df", where=np.arange(len(df)), start=5, stop=10)
+
+ # selection with filter
+ selection = date_range("20000101", periods=500)
+ result = store.select("df", where="index in selection")
+ expected = df[df.index.isin(selection)]
+ tm.assert_frame_equal(result, expected)
+
+ # list
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 2)))
+ store.append("df2", df)
+ result = store.select("df2", where=[0, 3, 5])
+ expected = df.iloc[[0, 3, 5]]
+ tm.assert_frame_equal(result, expected)
+
+ # boolean
+ where = [True] * 10
+ where[-2] = False
+ result = store.select("df2", where=where)
+ expected = df.loc[where]
+ tm.assert_frame_equal(result, expected)
+
+ # start/stop
+ result = store.select("df2", start=5, stop=10)
+ expected = df[5:10]
+ tm.assert_frame_equal(result, expected)
+
+
+def test_start_stop_table(temp_hdfstore):
+ # table
+ df = DataFrame(
+ {
+ "A": np.random.default_rng(2).random(20),
+ "B": np.random.default_rng(2).random(20),
+ }
+ )
+ temp_hdfstore.append("df", df)
+
+ result = temp_hdfstore.select("df", "columns=['A']", start=0, stop=5)
+ expected = df.loc[0:4, ["A"]]
+ tm.assert_frame_equal(result, expected)
+
+ # out of range
+ result = temp_hdfstore.select("df", "columns=['A']", start=30, stop=40)
+ assert len(result) == 0
+ expected = df.loc[30:40, ["A"]]
+ tm.assert_frame_equal(result, expected)
+
+
+def test_start_stop_multiple(temp_hdfstore):
+ # GH 16209
+ df = DataFrame({"foo": [1, 2], "bar": [1, 2]})
+
+ temp_hdfstore.append_to_multiple(
+ {"selector": ["foo"], "data": None}, df, selector="selector"
+ )
+ result = temp_hdfstore.select_as_multiple(
+ ["selector", "data"], selector="selector", start=0, stop=1
+ )
+ expected = df.loc[[0], ["foo", "bar"]]
+ tm.assert_frame_equal(result, expected)
+
+
+def test_start_stop_fixed(temp_hdfstore):
+ # fixed, GH 8287
+ store = temp_hdfstore
+ df = DataFrame(
+ {
+ "A": np.random.default_rng(2).random(20),
+ "B": np.random.default_rng(2).random(20),
+ },
+ index=date_range("20130101", periods=20),
+ )
+ store.put("df", df)
+
+ result = store.select("df", start=0, stop=5)
+ expected = df.iloc[0:5, :]
+ tm.assert_frame_equal(result, expected)
+
+ result = store.select("df", start=5, stop=10)
+ expected = df.iloc[5:10, :]
+ tm.assert_frame_equal(result, expected)
+
+ # out of range
+ result = store.select("df", start=30, stop=40)
+ expected = df.iloc[30:40, :]
+ tm.assert_frame_equal(result, expected)
+
+ # series
+ s = df.A
+ store.put("s", s)
+ result = store.select("s", start=0, stop=5)
+ expected = s.iloc[0:5]
+ tm.assert_series_equal(result, expected)
+
+ result = store.select("s", start=5, stop=10)
+ expected = s.iloc[5:10]
+ tm.assert_series_equal(result, expected)
+
+ # sparse; not implemented
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ df.iloc[3:5, 1:3] = np.nan
+ df.iloc[8:10, -2] = np.nan
+
+
+def test_select_filter_corner(temp_hdfstore, request):
+ df = DataFrame(np.random.default_rng(2).standard_normal((50, 100)))
+ df.index = [f"{c:3d}" for c in df.index]
+ df.columns = [f"{c:3d}" for c in df.columns]
+
+ temp_hdfstore.put("frame", df, format="table")
+
+ request.applymarker(
+ pytest.mark.xfail(
+ PY312,
+ reason="AST change in PY312",
+ raises=ValueError,
+ )
+ )
+ crit = "columns=df.columns[:75]"
+ result = temp_hdfstore.select("frame", [crit])
+ tm.assert_frame_equal(result, df.loc[:, df.columns[:75]])
+
+ crit = "columns=df.columns[:75:2]"
+ result = temp_hdfstore.select("frame", [crit])
+ tm.assert_frame_equal(result, df.loc[:, df.columns[:75:2]])
+
+
+def test_path_pathlib(temp_h5_path):
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+
+ result = tm.round_trip_pathlib(
+ lambda p: df.to_hdf(p, key="df"), lambda p: read_hdf(p, "df"), temp_h5_path
+ )
+ tm.assert_frame_equal(df, result)
+
+
+@pytest.mark.parametrize("start, stop", [(0, 2), (1, 2), (None, None)])
+def test_contiguous_mixed_data_table(start, stop, temp_hdfstore):
+ # GH 17021
+ df = DataFrame(
+ {
+ "a": Series([20111010, 20111011, 20111012]),
+ "b": Series(["ab", "cd", "ab"]),
+ }
+ )
+
+ temp_hdfstore.append("test_dataset", df)
+
+ result = temp_hdfstore.select("test_dataset", start=start, stop=stop)
+ tm.assert_frame_equal(df[start:stop], result)
+
+
+def test_path_pathlib_hdfstore(temp_h5_path):
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+
+ def writer(path):
+ with HDFStore(path) as store:
+ df.to_hdf(store, key="df")
+
+ def reader(path):
+ with HDFStore(path) as store:
+ return read_hdf(store, "df")
+
+ result = tm.round_trip_pathlib(writer, reader, temp_h5_path)
+ tm.assert_frame_equal(df, result)
+
+
+def test_pickle_path_localpath(temp_h5_path):
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+ result = tm.round_trip_pathlib(
+ lambda p: df.to_hdf(p, key="df"), lambda p: read_hdf(p, "df"), temp_h5_path
+ )
+ tm.assert_frame_equal(df, result)
+
+
+@pytest.mark.parametrize("propindexes", [True, False])
+def test_copy(propindexes, temp_file):
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+
+ with HDFStore(temp_file) as st:
+ st.append("df", df, data_columns=["A"])
+ with tempfile.NamedTemporaryFile() as new_f:
+ with HDFStore(temp_file) as store:
+ with contextlib.closing(
+ store.copy(new_f.name, keys=None, propindexes=propindexes)
+ ) as tstore:
+ # check keys
+ keys = store.keys()
+ assert set(keys) == set(tstore.keys())
+ # check indices & nrows
+ for k in tstore.keys():
+ if tstore.get_storer(k).is_table:
+ new_t = tstore.get_storer(k)
+ orig_t = store.get_storer(k)
+
+ assert orig_t.nrows == new_t.nrows
+
+ # check propindixes
+ if propindexes:
+ for a in orig_t.axes:
+ if a.is_indexed:
+ assert new_t[a.name].is_indexed
+
+
+def test_duplicate_column_name(temp_h5_path):
+ df = DataFrame(columns=["a", "a"], data=[[0, 0]])
+
+ msg = "Columns index has to be unique for fixed format"
+ with pytest.raises(ValueError, match=msg):
+ df.to_hdf(temp_h5_path, key="df", format="fixed")
+
+ df.to_hdf(temp_h5_path, key="df", format="table")
+ other = read_hdf(temp_h5_path, "df")
+
+ tm.assert_frame_equal(df, other)
+ assert df.equals(other)
+ assert other.equals(df)
+
+
+def test_preserve_timedeltaindex_type(temp_hdfstore, unit):
+ # GH9635
+ df = DataFrame(np.random.default_rng(2).normal(size=(10, 5)))
+ df.index = timedelta_range(
+ start="0s", periods=10, freq="1s", name="example", unit=unit
+ )
+
+ temp_hdfstore["df"] = df
+ tm.assert_frame_equal(temp_hdfstore["df"], df)
+
+
+def test_columns_multiindex_modified(temp_h5_path):
+ # BUG: 7212
+
+ df = DataFrame(
+ np.random.default_rng(2).random((4, 5)),
+ index=list("abcd"),
+ columns=list("ABCDE"),
+ )
+ df.index.name = "letters"
+ df = df.set_index(keys="E", append=True)
+
+ data_columns = df.index.names + df.columns.tolist()
+ df.to_hdf(
+ temp_h5_path,
+ key="df",
+ mode="a",
+ append=True,
+ data_columns=data_columns,
+ index=False,
+ )
+ cols2load = list("BCD")
+ cols2load_original = list(cols2load)
+ # GH#10055 make sure read_hdf call does not alter cols2load inplace
+ read_hdf(temp_h5_path, "df", columns=cols2load)
+ assert cols2load_original == cols2load
+
+
+@pytest.mark.filterwarnings(r"ignore:PeriodDtype\[B\] is deprecated:FutureWarning")
+@pytest.mark.parametrize(
+ "columns",
+ [
+ Index([0, 1], dtype=np.int64),
+ Index([0.0, 1.0], dtype=np.float64),
+ date_range("2020-01-01", periods=2),
+ timedelta_range("1 day", periods=2),
+ period_range("2020-01-01", periods=2, freq="D"),
+ ],
+)
+def test_to_hdf_with_object_column_names_should_fail(temp_h5_path, columns):
+ # GH9057
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 2)), columns=columns)
+ msg = "cannot have non-object label DataIndexableCol"
+ with pytest.raises(ValueError, match=msg):
+ df.to_hdf(temp_h5_path, key="df", format="table", data_columns=True)
+
+
+@pytest.mark.parametrize("dtype", [None, "category"])
+def test_to_hdf_with_object_column_names_should_run(temp_h5_path, dtype):
+ # GH9057
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)),
+ columns=Index(["a", "b"], dtype=dtype),
+ )
+ df.to_hdf(temp_h5_path, key="df", format="table", data_columns=True)
+ result = read_hdf(temp_h5_path, "df", where=f"index = [{df.index[0]}]")
+ assert len(result)
+
+
+def test_hdfstore_strides(temp_hdfstore):
+ # GH22073
+ df = DataFrame({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]})
+ temp_hdfstore.put("df", df)
+ assert df["a"].values.strides == temp_hdfstore["df"]["a"].values.strides
+
+
+def test_store_bool_index(temp_h5_path):
+ # GH#48667
+ df = DataFrame([[1]], columns=[True], index=Index([False], dtype="bool"))
+ expected = df.copy()
+
+ # # Test to make sure defaults are to not drop.
+ # # Corresponding to Issue 9382
+ df.to_hdf(temp_h5_path, key="a")
+ result = read_hdf(temp_h5_path, "a")
+ tm.assert_frame_equal(expected, result)
+
+
+@pytest.mark.parametrize("model", ["name", "longname", "verylongname"])
+def test_select_categorical_string_columns(temp_hdfstore, model):
+ # Corresponding to BUG: 57608
+
+ models = CategoricalDtype(categories=["name", "longname", "verylongname"])
+ df = DataFrame(
+ {"modelId": ["name", "longname", "longname"], "value": [1, 2, 3]}
+ ).astype({"modelId": models, "value": int})
+
+ temp_hdfstore.append("df", df, data_columns=["modelId"])
+
+ result = temp_hdfstore.select("df", "modelId == model")
+ expected = df[df["modelId"] == model]
+ tm.assert_frame_equal(result, expected)
+
+
+def test_to_hdf_multiindex_string_dtype_crash(temp_h5_path):
+ # GH#63412
+ index = MultiIndex.from_tuples([("a", "x"), ("b", "y")], names=["level1", "level2"])
+ df = DataFrame({"value": [1, 2]}, index=index)
+ df.to_hdf(temp_h5_path, key="test")
+ result = read_hdf(temp_h5_path, key="test")
+ tm.assert_frame_equal(df, result, check_dtype=False)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_subclass.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_subclass.py
new file mode 100644
index 0000000000000000000000000000000000000000..fa50575ece840ac227ec7cfae5ba8e6c348827f6
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_subclass.py
@@ -0,0 +1,48 @@
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ Series,
+)
+import pandas._testing as tm
+
+from pandas.io.pytables import (
+ HDFStore,
+ read_hdf,
+)
+
+pytest.importorskip("tables")
+
+
+class TestHDFStoreSubclass:
+ # GH 33748
+ def test_supported_for_subclass_dataframe(self, temp_h5_path):
+ data = {"a": [1, 2], "b": [3, 4]}
+ sdf = tm.SubclassedDataFrame(data, dtype=np.intp)
+
+ expected = DataFrame(data, dtype=np.intp)
+
+ sdf.to_hdf(temp_h5_path, key="df")
+ result = read_hdf(temp_h5_path, "df")
+ tm.assert_frame_equal(result, expected)
+
+ with HDFStore(temp_h5_path) as store:
+ store.put("df", sdf)
+ result = read_hdf(temp_h5_path, "df")
+ tm.assert_frame_equal(result, expected)
+
+ def test_supported_for_subclass_series(self, temp_h5_path):
+ data = [1, 2, 3]
+ sser = tm.SubclassedSeries(data, dtype=np.intp)
+
+ expected = Series(data, dtype=np.intp)
+
+ sser.to_hdf(temp_h5_path, key="ser")
+ result = read_hdf(temp_h5_path, "ser")
+ tm.assert_series_equal(result, expected)
+
+ with HDFStore(temp_h5_path) as store:
+ store.put("ser", sser)
+ result = read_hdf(temp_h5_path, "ser")
+ tm.assert_series_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_time_series.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_time_series.py
new file mode 100644
index 0000000000000000000000000000000000000000..42d2f4ddb4db917f4b2da6780b660ee82d5af1ef
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_time_series.py
@@ -0,0 +1,66 @@
+import datetime
+
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ DatetimeIndex,
+ Series,
+ _testing as tm,
+ date_range,
+ period_range,
+)
+
+pytestmark = pytest.mark.single_cpu
+
+
+@pytest.mark.parametrize("unit", ["us", "ns"])
+def test_store_datetime_fractional_secs(temp_hdfstore, unit):
+ dt = datetime.datetime(2012, 1, 2, 3, 4, 5, 123456)
+ dti = DatetimeIndex([dt], dtype=f"M8[{unit}]")
+ series = Series([0], index=dti)
+ temp_hdfstore["a"] = series
+ assert temp_hdfstore["a"].index[0] == dt
+
+
+@pytest.mark.filterwarnings(r"ignore:PeriodDtype\[B\] is deprecated:FutureWarning")
+def test_tseries_indices_series(temp_hdfstore):
+ idx = date_range("2020-01-01", periods=10)
+ ser = Series(np.random.default_rng(2).standard_normal(len(idx)), idx)
+ temp_hdfstore["a"] = ser
+ result = temp_hdfstore["a"]
+
+ tm.assert_series_equal(result, ser)
+ assert result.index.freq == ser.index.freq
+ tm.assert_class_equal(result.index, ser.index, obj="series index")
+
+ idx = period_range("2020-01-01", periods=10, freq="D")
+ ser = Series(np.random.default_rng(2).standard_normal(len(idx)), idx)
+ temp_hdfstore["a"] = ser
+ result = temp_hdfstore["a"]
+
+ tm.assert_series_equal(result, ser)
+ assert result.index.freq == ser.index.freq
+ tm.assert_class_equal(result.index, ser.index, obj="series index")
+
+
+@pytest.mark.filterwarnings(r"ignore:PeriodDtype\[B\] is deprecated:FutureWarning")
+def test_tseries_indices_frame(temp_hdfstore):
+ idx = date_range("2020-01-01", periods=10)
+ df = DataFrame(np.random.default_rng(2).standard_normal((len(idx), 3)), index=idx)
+ temp_hdfstore["a"] = df
+ result = temp_hdfstore["a"]
+
+ tm.assert_frame_equal(result, df)
+ assert result.index.freq == df.index.freq
+ tm.assert_class_equal(result.index, df.index, obj="dataframe index")
+
+ idx = period_range("2020-01-01", periods=10, freq="D")
+ df = DataFrame(np.random.default_rng(2).standard_normal((len(idx), 3)), idx)
+ temp_hdfstore["a"] = df
+ result = temp_hdfstore["a"]
+
+ tm.assert_frame_equal(result, df)
+ assert result.index.freq == df.index.freq
+ tm.assert_class_equal(result.index, df.index, obj="dataframe index")
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_timezones.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_timezones.py
new file mode 100644
index 0000000000000000000000000000000000000000..8f7b635a5ff633bff75d30fbb3484e623c2740b0
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/pytables/test_timezones.py
@@ -0,0 +1,334 @@
+from datetime import (
+ date,
+ timedelta,
+)
+
+import numpy as np
+import pytest
+
+from pandas._libs.tslibs.timezones import maybe_get_tz
+import pandas.util._test_decorators as td
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ DatetimeIndex,
+ Series,
+ Timestamp,
+ date_range,
+)
+import pandas._testing as tm
+
+
+def _compare_with_tz(a, b):
+ tm.assert_frame_equal(a, b)
+
+ # compare the zones on each element
+ for c in a.columns:
+ for i in a.index:
+ a_e = a.loc[i, c]
+ b_e = b.loc[i, c]
+ if not (a_e == b_e and a_e.tz == b_e.tz):
+ raise AssertionError(f"invalid tz comparison [{a_e}] [{b_e}]")
+
+
+# use maybe_get_tz instead of dateutil.tz.gettz to handle the windows
+# filename issues.
+gettz_dateutil = lambda x: maybe_get_tz("dateutil/" + x)
+gettz_pytz = lambda x: x
+
+
+@pytest.mark.filterwarnings(
+ "ignore:`alltrue` is deprecated as of NumPy 1.25.0:DeprecationWarning"
+)
+@pytest.mark.parametrize("gettz", [gettz_dateutil, gettz_pytz])
+def test_append_with_timezones(temp_hdfstore, gettz):
+ # as columns
+
+ # Single-tzinfo, no DST transition
+ df_est = DataFrame(
+ {
+ "A": [
+ Timestamp("20130102 2:00:00", tz=gettz("US/Eastern")).as_unit("ns")
+ + timedelta(hours=1) * i
+ for i in range(5)
+ ]
+ }
+ )
+
+ # frame with all columns having same tzinfo, but different sides
+ # of DST transition
+ df_crosses_dst = DataFrame(
+ {
+ "A": Timestamp("20130102", tz=gettz("US/Eastern")).as_unit("ns"),
+ "B": Timestamp("20130603", tz=gettz("US/Eastern")).as_unit("ns"),
+ },
+ index=range(5),
+ )
+
+ df_mixed_tz = DataFrame(
+ {
+ "A": Timestamp("20130102", tz=gettz("US/Eastern")).as_unit("ns"),
+ "B": Timestamp("20130102", tz=gettz("EET")).as_unit("ns"),
+ },
+ index=range(5),
+ )
+
+ df_different_tz = DataFrame(
+ {
+ "A": Timestamp("20130102", tz=gettz("US/Eastern")).as_unit("ns"),
+ "B": Timestamp("20130102", tz=gettz("CET")).as_unit("ns"),
+ },
+ index=range(5),
+ )
+
+ temp_hdfstore.append("df_tz", df_est, data_columns=["A"])
+ result = temp_hdfstore["df_tz"]
+ _compare_with_tz(result, df_est)
+ tm.assert_frame_equal(result, df_est)
+
+ # select with tz aware
+ expected = df_est[df_est.A >= df_est.A[3]]
+ result = temp_hdfstore.select("df_tz", where="A>=df_est.A[3]")
+ _compare_with_tz(result, expected)
+
+ # ensure we include dates in DST and STD time here.
+ temp_hdfstore.remove("df_tz")
+ temp_hdfstore.append("df_tz", df_crosses_dst)
+ result = temp_hdfstore["df_tz"]
+ _compare_with_tz(result, df_crosses_dst)
+ tm.assert_frame_equal(result, df_crosses_dst)
+
+ msg = (
+ r"invalid info for \[values_block_1\] for \[tz\], "
+ r"existing_value \[(dateutil/.*)?(US/Eastern|America/New_York)\] "
+ r"conflicts with new value \[(dateutil/.*)?EET\]"
+ )
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("df_tz", df_mixed_tz)
+
+ # this is ok
+ temp_hdfstore.remove("df_tz")
+ temp_hdfstore.append("df_tz", df_mixed_tz, data_columns=["A", "B"])
+ result = temp_hdfstore["df_tz"]
+ _compare_with_tz(result, df_mixed_tz)
+ tm.assert_frame_equal(result, df_mixed_tz)
+
+ # can't append with diff timezone
+ msg = (
+ r"invalid info for \[B\] for \[tz\], "
+ r"existing_value \[(dateutil/.*)?EET\] "
+ r"conflicts with new value \[(dateutil/.*)?CET\]"
+ )
+ with pytest.raises(ValueError, match=msg):
+ temp_hdfstore.append("df_tz", df_different_tz)
+
+
+@pytest.mark.parametrize("gettz", [gettz_dateutil, gettz_pytz])
+def test_append_with_timezones_as_index(temp_hdfstore, gettz):
+ # GH#4098 example
+
+ dti = date_range("2000-1-1", periods=3, freq="h", tz=gettz("US/Eastern"))
+ dti = dti._with_freq(None) # freq doesn't round-trip
+
+ df = DataFrame({"A": Series(range(3), index=dti)})
+
+ temp_hdfstore.put("df", df)
+ result = temp_hdfstore.select("df")
+ tm.assert_frame_equal(result, df)
+
+ temp_hdfstore.remove("df")
+ temp_hdfstore.append("df", df)
+ result = temp_hdfstore.select("df")
+ tm.assert_frame_equal(result, df)
+
+
+def test_roundtrip_tz_aware_index(temp_hdfstore, unit):
+ # GH 17618
+ ts = Timestamp("2000-01-01 01:00:00", tz="US/Eastern")
+ dti = DatetimeIndex([ts]).as_unit(unit)
+ df = DataFrame(data=[0], index=dti)
+
+ temp_hdfstore.put("frame", df, format="fixed")
+ recons = temp_hdfstore["frame"]
+ tm.assert_frame_equal(recons, df)
+
+ value = recons.index[0]._value
+ denom = {"ns": 1, "us": 1000, "ms": 10**6, "s": 10**9}[unit]
+ assert value == 946706400000000000 // denom
+
+
+def test_store_index_name_with_tz(temp_hdfstore):
+ # GH 13884
+ df = DataFrame({"A": [1, 2]})
+ df.index = DatetimeIndex([1234567890123456787, 1234567890123456788])
+ df.index = df.index.tz_localize("UTC")
+ df.index.name = "foo"
+
+ temp_hdfstore.put("frame", df, format="table")
+ recons = temp_hdfstore["frame"]
+ tm.assert_frame_equal(recons, df)
+
+
+def test_tseries_select_index_column(temp_hdfstore):
+ # GH7777
+ # selecting a UTC datetimeindex column did
+ # not preserve UTC tzinfo set before storing
+
+ # check that no tz still works
+ rng = date_range("1/1/2000", "1/30/2000")
+ frame = DataFrame(
+ np.random.default_rng(2).standard_normal((len(rng), 4)), index=rng
+ )
+
+ temp_hdfstore.append("frame", frame)
+ result = temp_hdfstore.select_column("frame", "index")
+ assert rng.tz == DatetimeIndex(result.values).tz
+
+ # check utc
+ rng = date_range("1/1/2000", "1/30/2000", tz="UTC")
+ frame = DataFrame(
+ np.random.default_rng(2).standard_normal((len(rng), 4)), index=rng
+ )
+
+ temp_hdfstore.remove("frame")
+ temp_hdfstore.append("frame", frame)
+ result = temp_hdfstore.select_column("frame", "index")
+ assert rng.tz == result.dt.tz
+
+ # double check non-utc
+ rng = date_range("1/1/2000", "1/30/2000", tz="US/Eastern")
+ frame = DataFrame(
+ np.random.default_rng(2).standard_normal((len(rng), 4)), index=rng
+ )
+
+ temp_hdfstore.remove("frame")
+ temp_hdfstore.append("frame", frame)
+ result = temp_hdfstore.select_column("frame", "index")
+ assert rng.tz == result.dt.tz
+
+
+def test_timezones_fixed_format_frame_non_empty(temp_hdfstore):
+ # index
+ rng = date_range("1/1/2000", "1/30/2000", tz="US/Eastern")
+ rng = rng._with_freq(None) # freq doesn't round-trip
+ df = DataFrame(np.random.default_rng(2).standard_normal((len(rng), 4)), index=rng)
+ temp_hdfstore["df"] = df
+ result = temp_hdfstore["df"]
+ tm.assert_frame_equal(result, df)
+
+
+def test_timezones_fixed_format_frame_non_empty_as_data(temp_hdfstore):
+ # GH11411
+ rng = date_range("1/1/2000", "1/30/2000", tz="US/Eastern")
+ rng = rng._with_freq(None) # freq doesn't round-trip
+ df = DataFrame(
+ {
+ "A": rng,
+ "B": rng.tz_convert("UTC").tz_localize(None),
+ "C": rng.tz_convert("CET"),
+ "D": range(len(rng)),
+ },
+ index=rng,
+ )
+ temp_hdfstore["df"] = df
+ result = temp_hdfstore["df"]
+ tm.assert_frame_equal(result, df)
+
+
+def test_timezones_fixed_format_empty(temp_hdfstore, tz_aware_fixture, frame_or_series):
+ # GH 20594
+
+ dtype = pd.DatetimeTZDtype(tz=tz_aware_fixture)
+
+ obj = Series(dtype=dtype, name="A")
+ if frame_or_series is DataFrame:
+ obj = obj.to_frame()
+
+ temp_hdfstore["obj"] = obj
+ result = temp_hdfstore["obj"]
+ tm.assert_equal(result, obj)
+
+
+def test_timezones_fixed_format_series_nonempty(temp_hdfstore, tz_aware_fixture):
+ # GH 20594
+
+ dtype = pd.DatetimeTZDtype(tz=tz_aware_fixture)
+
+ s = Series([0], dtype=dtype)
+ temp_hdfstore["s"] = s
+ result = temp_hdfstore["s"]
+ tm.assert_series_equal(result, s)
+
+
+def test_fixed_offset_tz(temp_hdfstore):
+ rng = date_range("1/1/2000 00:00:00-07:00", "1/30/2000 00:00:00-07:00")
+ frame = DataFrame(
+ np.random.default_rng(2).standard_normal((len(rng), 4)), index=rng
+ )
+
+ temp_hdfstore["frame"] = frame
+ recons = temp_hdfstore["frame"]
+ tm.assert_index_equal(recons.index, rng)
+ assert rng.tz == recons.index.tz
+
+
+@td.skip_if_windows
+def test_store_timezone(temp_hdfstore):
+ # GH2852
+ # issue storing datetime.date with a timezone as it resets when read
+ # back in a new timezone
+
+ # original method
+ today = date(2013, 9, 10)
+ df = DataFrame([1, 2, 3], index=[today, today, today])
+ temp_hdfstore["obj1"] = df
+ result = temp_hdfstore["obj1"]
+ tm.assert_frame_equal(result, df)
+
+ # with tz setting
+ with tm.set_timezone("EST5EDT"):
+ today = date(2013, 9, 10)
+ df = DataFrame([1, 2, 3], index=[today, today, today])
+ temp_hdfstore["obj2"] = df
+
+ with tm.set_timezone("CST6CDT"):
+ result = temp_hdfstore["obj2"]
+
+ tm.assert_frame_equal(result, df)
+
+
+def test_dst_transitions(temp_hdfstore):
+ # make sure we are not failing on transitions
+ times = date_range(
+ "2013-10-26 23:00",
+ "2013-10-27 01:00",
+ tz="Europe/London",
+ freq="h",
+ ambiguous="infer",
+ )
+ times = times._with_freq(None) # freq doesn't round-trip
+
+ for i in [times, times + pd.Timedelta("10min")]:
+ df = DataFrame({"A": range(len(i)), "B": i}, index=i)
+ temp_hdfstore.append("df", df)
+ result = temp_hdfstore.select("df")
+ tm.assert_frame_equal(result, df)
+ temp_hdfstore.remove("df")
+
+
+@pytest.mark.filterwarnings(
+ "ignore:`alltrue` is deprecated as of NumPy 1.25.0:DeprecationWarning"
+)
+def test_read_with_where_tz_aware_index(temp_hdfstore):
+ # GH 11926
+ periods = 10
+ dts = date_range("20151201", periods=periods, freq="D", tz="UTC", unit="ns")
+ mi = pd.MultiIndex.from_arrays([dts, range(periods)], names=["DATE", "NO"])
+ expected = DataFrame({"MYCOL": 0}, index=mi)
+
+ key = "mykey"
+ with pd.HDFStore(temp_hdfstore) as store:
+ store.append(key, expected, format="table", append=True)
+ result = pd.read_hdf(temp_hdfstore, key, where="DATE > 20151130")
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/sas/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/sas/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/sas/test_byteswap.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/sas/test_byteswap.py
new file mode 100644
index 0000000000000000000000000000000000000000..78f0d2232c4ba08b27ea294cc60136acb0890e56
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/sas/test_byteswap.py
@@ -0,0 +1,57 @@
+from hypothesis import (
+ assume,
+ example,
+ given,
+ strategies as st,
+)
+import numpy as np
+import pytest
+
+from pandas._libs.byteswap import (
+ read_double_with_byteswap,
+ read_float_with_byteswap,
+ read_uint16_with_byteswap,
+ read_uint32_with_byteswap,
+ read_uint64_with_byteswap,
+)
+
+import pandas._testing as tm
+
+
+@pytest.mark.slow
+@given(read_offset=st.integers(0, 11), number=st.integers(min_value=0))
+@example(number=2**16, read_offset=0)
+@example(number=2**32, read_offset=0)
+@example(number=2**64, read_offset=0)
+@pytest.mark.parametrize("int_type", [np.uint16, np.uint32, np.uint64])
+@pytest.mark.parametrize("should_byteswap", [True, False])
+def test_int_byteswap(read_offset, number, int_type, should_byteswap):
+ assume(number < 2 ** (8 * int_type(0).itemsize))
+ _test(number, int_type, read_offset, should_byteswap)
+
+
+@pytest.mark.slow
+@pytest.mark.filterwarnings("ignore:overflow encountered:RuntimeWarning")
+@given(read_offset=st.integers(0, 11), number=st.floats())
+@pytest.mark.parametrize("float_type", [np.float32, np.float64])
+@pytest.mark.parametrize("should_byteswap", [True, False])
+def test_float_byteswap(read_offset, number, float_type, should_byteswap):
+ _test(number, float_type, read_offset, should_byteswap)
+
+
+def _test(number, number_type, read_offset, should_byteswap):
+ number = number_type(number)
+ data = np.random.default_rng(2).integers(0, 256, size=20, dtype="uint8")
+ data[read_offset : read_offset + number.itemsize] = number[None].view("uint8")
+ swap_func = {
+ np.float32: read_float_with_byteswap,
+ np.float64: read_double_with_byteswap,
+ np.uint16: read_uint16_with_byteswap,
+ np.uint32: read_uint32_with_byteswap,
+ np.uint64: read_uint64_with_byteswap,
+ }[type(number)]
+ output_number = number_type(swap_func(bytes(data), read_offset, should_byteswap))
+ if should_byteswap:
+ tm.assert_equal(output_number, number.byteswap())
+ else:
+ tm.assert_equal(output_number, number)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/sas/test_sas.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/sas/test_sas.py
new file mode 100644
index 0000000000000000000000000000000000000000..ea072b6e9c4e01822176e883a2cf6c31bb96ca83
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/sas/test_sas.py
@@ -0,0 +1,33 @@
+from io import StringIO
+
+import pytest
+
+from pandas import read_sas
+import pandas._testing as tm
+
+
+class TestSas:
+ def test_sas_buffer_format(self):
+ # see gh-14947
+ b = StringIO("")
+
+ msg = (
+ "If this is a buffer object rather than a string "
+ "name, you must specify a format string"
+ )
+ with pytest.raises(ValueError, match=msg):
+ read_sas(b)
+
+ def test_sas_read_no_format_or_extension(self, temp_file):
+ # see gh-24548
+ msg = "unable to infer format of SAS file.+"
+ with pytest.raises(ValueError, match=msg):
+ read_sas(temp_file)
+
+
+def test_sas_archive(datapath):
+ fname_uncompressed = datapath("io", "sas", "data", "airline.sas7bdat")
+ df_uncompressed = read_sas(fname_uncompressed)
+ fname_compressed = datapath("io", "sas", "data", "airline.sas7bdat.gz")
+ df_compressed = read_sas(fname_compressed, format="sas7bdat")
+ tm.assert_frame_equal(df_uncompressed, df_compressed)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/sas/test_sas7bdat.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/sas/test_sas7bdat.py
new file mode 100644
index 0000000000000000000000000000000000000000..c2ce104e5a3d4387d96bed79641246803eae65d6
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/sas/test_sas7bdat.py
@@ -0,0 +1,416 @@
+import contextlib
+from datetime import datetime
+import io
+import os
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+from pandas.compat._constants import (
+ IS64,
+ WASM,
+)
+from pandas.errors import EmptyDataError
+
+import pandas as pd
+import pandas._testing as tm
+
+from pandas.io.sas.sas7bdat import SAS7BDATReader
+
+
+@pytest.fixture
+def dirpath(datapath):
+ return datapath("io", "sas", "data")
+
+
+@pytest.fixture(params=[(1, range(1, 16)), (2, [16])])
+def data_test_ix(request, dirpath):
+ i, test_ix = request.param
+ fname = os.path.join(dirpath, f"test_sas7bdat_{i}.csv")
+ df = pd.read_csv(fname)
+ epoch = datetime(1960, 1, 1)
+ t1 = pd.to_timedelta(df["Column4"], unit="D")
+ df["Column4"] = (epoch + t1).astype("M8[s]")
+ t2 = pd.to_timedelta(df["Column12"], unit="D")
+ df["Column12"] = (epoch + t2).astype("M8[s]")
+ for k in range(df.shape[1]):
+ col = df.iloc[:, k]
+ if col.dtype == np.int64:
+ df.isetitem(k, df.iloc[:, k].astype(np.float64))
+ return df, test_ix
+
+
+# https://github.com/cython/cython/issues/1720
+class TestSAS7BDAT:
+ @pytest.mark.slow
+ def test_from_file(self, dirpath, data_test_ix):
+ expected, test_ix = data_test_ix
+ for k in test_ix:
+ fname = os.path.join(dirpath, f"test{k}.sas7bdat")
+ df = pd.read_sas(fname, encoding="utf-8")
+ tm.assert_frame_equal(df, expected)
+
+ @pytest.mark.slow
+ def test_from_buffer(self, dirpath, data_test_ix):
+ expected, test_ix = data_test_ix
+ for k in test_ix:
+ fname = os.path.join(dirpath, f"test{k}.sas7bdat")
+ with open(fname, "rb") as f:
+ byts = f.read()
+ buf = io.BytesIO(byts)
+ with pd.read_sas(
+ buf, format="sas7bdat", iterator=True, encoding="utf-8"
+ ) as rdr:
+ df = rdr.read()
+ tm.assert_frame_equal(df, expected)
+
+ @pytest.mark.slow
+ def test_from_iterator(self, dirpath, data_test_ix):
+ expected, test_ix = data_test_ix
+ for k in test_ix:
+ fname = os.path.join(dirpath, f"test{k}.sas7bdat")
+ with pd.read_sas(fname, iterator=True, encoding="utf-8") as rdr:
+ df = rdr.read(2)
+ tm.assert_frame_equal(df, expected.iloc[0:2, :])
+ df = rdr.read(3)
+ tm.assert_frame_equal(df, expected.iloc[2:5, :])
+
+ @pytest.mark.slow
+ def test_path_pathlib(self, dirpath, data_test_ix):
+ expected, test_ix = data_test_ix
+ for k in test_ix:
+ fname = Path(os.path.join(dirpath, f"test{k}.sas7bdat"))
+ df = pd.read_sas(fname, encoding="utf-8")
+ tm.assert_frame_equal(df, expected)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("chunksize", (3, 5, 10, 11))
+ @pytest.mark.parametrize("k", range(1, 17))
+ def test_iterator_loop(self, dirpath, k, chunksize):
+ # github #13654
+ fname = os.path.join(dirpath, f"test{k}.sas7bdat")
+ with pd.read_sas(fname, chunksize=chunksize, encoding="utf-8") as rdr:
+ y = 0
+ for x in rdr:
+ y += x.shape[0]
+ assert y == rdr.row_count
+
+ def test_iterator_read_too_much(self, dirpath):
+ # github #14734
+ fname = os.path.join(dirpath, "test1.sas7bdat")
+ with pd.read_sas(
+ fname, format="sas7bdat", iterator=True, encoding="utf-8"
+ ) as rdr:
+ d1 = rdr.read(rdr.row_count + 20)
+
+ with pd.read_sas(fname, iterator=True, encoding="utf-8") as rdr:
+ d2 = rdr.read(rdr.row_count + 20)
+ tm.assert_frame_equal(d1, d2)
+
+
+def test_encoding_options(datapath):
+ fname = datapath("io", "sas", "data", "test1.sas7bdat")
+ df1 = pd.read_sas(fname)
+ df2 = pd.read_sas(fname, encoding="utf-8")
+ for col in df1.columns:
+ try:
+ df1[col] = df1[col].str.decode("utf-8")
+ except AttributeError:
+ pass
+ tm.assert_frame_equal(df1, df2)
+
+ with contextlib.closing(SAS7BDATReader(fname, convert_header_text=False)) as rdr:
+ df3 = rdr.read()
+ for x, y in zip(df1.columns, df3.columns):
+ assert x == y.decode()
+
+
+def test_encoding_infer(datapath):
+ fname = datapath("io", "sas", "data", "test1.sas7bdat")
+
+ with pd.read_sas(fname, encoding="infer", iterator=True) as df1_reader:
+ # check: is encoding inferred correctly from file
+ assert df1_reader.inferred_encoding == "cp1252"
+ df1 = df1_reader.read()
+
+ with pd.read_sas(fname, encoding="cp1252", iterator=True) as df2_reader:
+ df2 = df2_reader.read()
+
+ # check: reader reads correct information
+ tm.assert_frame_equal(df1, df2)
+
+
+def test_productsales(datapath):
+ fname = datapath("io", "sas", "data", "productsales.sas7bdat")
+ df = pd.read_sas(fname, encoding="utf-8")
+ fname = datapath("io", "sas", "data", "productsales.csv")
+ df0 = pd.read_csv(fname, parse_dates=["MONTH"])
+ vn = ["ACTUAL", "PREDICT", "QUARTER", "YEAR"]
+ df0[vn] = df0[vn].astype(np.float64)
+
+ df0["MONTH"] = df0["MONTH"].astype("M8[s]")
+ tm.assert_frame_equal(df, df0)
+
+
+def test_12659(datapath):
+ fname = datapath("io", "sas", "data", "test_12659.sas7bdat")
+ df = pd.read_sas(fname)
+ fname = datapath("io", "sas", "data", "test_12659.csv")
+ df0 = pd.read_csv(fname)
+ df0 = df0.astype(np.float64)
+ tm.assert_frame_equal(df, df0)
+
+
+def test_airline(datapath):
+ fname = datapath("io", "sas", "data", "airline.sas7bdat")
+ df = pd.read_sas(fname)
+ fname = datapath("io", "sas", "data", "airline.csv")
+ df0 = pd.read_csv(fname)
+ df0 = df0.astype(np.float64)
+ tm.assert_frame_equal(df, df0)
+
+
+@pytest.mark.skipif(WASM, reason="Pyodide/WASM has 32-bitness")
+def test_date_time(datapath):
+ # Support of different SAS date/datetime formats (PR #15871)
+ fname = datapath("io", "sas", "data", "datetime.sas7bdat")
+ df = pd.read_sas(fname)
+ fname = datapath("io", "sas", "data", "datetime.csv")
+ df0 = pd.read_csv(
+ fname, parse_dates=["Date1", "Date2", "DateTime", "DateTimeHi", "Taiw"]
+ )
+ # GH 19732: Timestamps imported from sas will incur floating point errors
+ # See GH#56014 for discussion of the correct "expected" results
+ # We are really just testing that we are "close". This only seems to be
+ # an issue near the implementation bounds.
+
+ df[df.columns[3]] = df.iloc[:, 3].dt.round("us")
+ df0["Date1"] = df0["Date1"].astype("M8[s]")
+ df0["Date2"] = df0["Date2"].astype("M8[s]")
+ df0["DateTime"] = df0["DateTime"].astype("M8[ms]")
+ df0["Taiw"] = df0["Taiw"].astype("M8[s]")
+
+ res = df0["DateTimeHi"].astype("M8[us]").dt.round("ms")
+ df0["DateTimeHi"] = res.astype("M8[ms]")
+
+ if not IS64:
+ # No good reason for this, just what we get on the CI
+ df0.loc[0, "DateTimeHi"] += np.timedelta64(1, "ms")
+ df0.loc[[2, 3], "DateTimeHi"] -= np.timedelta64(1, "ms")
+ tm.assert_frame_equal(df, df0)
+
+
+@pytest.mark.parametrize("column", ["WGT", "CYL"])
+def test_compact_numerical_values(datapath, column):
+ # Regression test for #21616
+ fname = datapath("io", "sas", "data", "cars.sas7bdat")
+ df = pd.read_sas(fname, encoding="latin-1")
+ # The two columns CYL and WGT in cars.sas7bdat have column
+ # width < 8 and only contain integral values.
+ # Test that pandas doesn't corrupt the numbers by adding
+ # decimals.
+ result = df[column]
+ expected = df[column].round()
+ tm.assert_series_equal(result, expected, check_exact=True)
+
+
+def test_many_columns(datapath):
+ # Test for looking for column information in more places (PR #22628)
+ fname = datapath("io", "sas", "data", "many_columns.sas7bdat")
+
+ df = pd.read_sas(fname, encoding="latin-1")
+
+ fname = datapath("io", "sas", "data", "many_columns.csv")
+ df0 = pd.read_csv(fname, encoding="latin-1")
+ tm.assert_frame_equal(df, df0)
+
+
+def test_inconsistent_number_of_rows(datapath):
+ # Regression test for issue #16615. (PR #22628)
+ fname = datapath("io", "sas", "data", "load_log.sas7bdat")
+ df = pd.read_sas(fname, encoding="latin-1")
+ assert len(df) == 2097
+
+
+def test_zero_variables(datapath):
+ # Check if the SAS file has zero variables (PR #18184)
+ fname = datapath("io", "sas", "data", "zero_variables.sas7bdat")
+ with pytest.raises(EmptyDataError, match="No columns to parse from file"):
+ pd.read_sas(fname)
+
+
+@pytest.mark.parametrize("encoding", [None, "utf8"])
+def test_zero_rows(datapath, encoding):
+ # GH 18198
+ fname = datapath("io", "sas", "data", "zero_rows.sas7bdat")
+ result = pd.read_sas(fname, encoding=encoding)
+ str_value = b"a" if encoding is None else "a"
+ expected = pd.DataFrame([{"char_field": str_value, "num_field": 1.0}]).iloc[:0]
+ tm.assert_frame_equal(result, expected)
+
+
+def test_corrupt_read(datapath):
+ # We don't really care about the exact failure, the important thing is
+ # that the resource should be cleaned up afterwards (BUG #35566)
+ fname = datapath("io", "sas", "data", "corrupt.sas7bdat")
+ msg = "'SAS7BDATReader' object has no attribute 'row_count'"
+ with pytest.raises(AttributeError, match=msg):
+ pd.read_sas(fname)
+
+
+@pytest.mark.xfail(WASM, reason="failing with currently set tolerances on WASM")
+def test_max_sas_date(datapath):
+ # GH 20927
+ # NB. max datetime in SAS dataset is 31DEC9999:23:59:59.999
+ # but this is read as 29DEC9999:23:59:59.998993 by a buggy
+ # sas7bdat module
+ # See also GH#56014 for discussion of the correct "expected" results.
+ fname = datapath("io", "sas", "data", "max_sas_date.sas7bdat")
+ df = pd.read_sas(fname, encoding="iso-8859-1")
+
+ expected = pd.DataFrame(
+ {
+ "text": ["max", "normal"],
+ "dt_as_float": [253717747199.999, 1880323199.999],
+ "dt_as_dt": np.array(
+ [
+ datetime(9999, 12, 29, 23, 59, 59, 999000),
+ datetime(2019, 8, 1, 23, 59, 59, 999000),
+ ],
+ dtype="M8[ms]",
+ ),
+ "date_as_float": [2936547.0, 21762.0],
+ "date_as_date": np.array(
+ [
+ datetime(9999, 12, 29),
+ datetime(2019, 8, 1),
+ ],
+ dtype="M8[s]",
+ ),
+ },
+ columns=["text", "dt_as_float", "dt_as_dt", "date_as_float", "date_as_date"],
+ )
+
+ if not IS64:
+ # No good reason for this, just what we get on the CI
+ expected.loc[:, "dt_as_dt"] -= np.timedelta64(1, "ms")
+
+ tm.assert_frame_equal(df, expected)
+
+
+@pytest.mark.xfail(WASM, reason="failing with currently set tolerances on WASM")
+def test_max_sas_date_iterator(datapath):
+ # GH 20927
+ # when called as an iterator, only those chunks with a date > pd.Timestamp.max
+ # are returned as datetime.datetime, if this happens that whole chunk is returned
+ # as datetime.datetime
+ col_order = ["text", "dt_as_float", "dt_as_dt", "date_as_float", "date_as_date"]
+ fname = datapath("io", "sas", "data", "max_sas_date.sas7bdat")
+ results = []
+ for df in pd.read_sas(fname, encoding="iso-8859-1", chunksize=1):
+ # GH 19732: Timestamps imported from sas will incur floating point errors
+ df.reset_index(inplace=True, drop=True)
+ results.append(df)
+ expected = [
+ pd.DataFrame(
+ {
+ "text": ["max"],
+ "dt_as_float": [253717747199.999],
+ "dt_as_dt": np.array(
+ [datetime(9999, 12, 29, 23, 59, 59, 999000)], dtype="M8[ms]"
+ ),
+ "date_as_float": [2936547.0],
+ "date_as_date": np.array([datetime(9999, 12, 29)], dtype="M8[s]"),
+ },
+ columns=col_order,
+ ),
+ pd.DataFrame(
+ {
+ "text": ["normal"],
+ "dt_as_float": [1880323199.999],
+ "dt_as_dt": np.array(["2019-08-01 23:59:59.999"], dtype="M8[ms]"),
+ "date_as_float": [21762.0],
+ "date_as_date": np.array(["2019-08-01"], dtype="M8[s]"),
+ },
+ columns=col_order,
+ ),
+ ]
+ if not IS64:
+ # No good reason for this, just what we get on the CI
+ expected[0].loc[0, "dt_as_dt"] -= np.timedelta64(1, "ms")
+ expected[1].loc[0, "dt_as_dt"] -= np.timedelta64(1, "ms")
+
+ tm.assert_frame_equal(results[0], expected[0])
+ tm.assert_frame_equal(results[1], expected[1])
+
+
+@pytest.mark.skipif(WASM, reason="Pyodide/WASM has 32-bitness")
+def test_null_date(datapath):
+ fname = datapath("io", "sas", "data", "dates_null.sas7bdat")
+ df = pd.read_sas(fname, encoding="utf-8")
+
+ expected = pd.DataFrame(
+ {
+ "datecol": np.array(
+ [
+ datetime(9999, 12, 29),
+ np.datetime64("NaT", "ns"),
+ ],
+ dtype="M8[s]",
+ ),
+ "datetimecol": np.array(
+ [
+ datetime(9999, 12, 29, 23, 59, 59, 999000),
+ np.datetime64("NaT", "ns"),
+ ],
+ dtype="M8[ms]",
+ ),
+ },
+ )
+ if not IS64:
+ # No good reason for this, just what we get on the CI
+ expected.loc[0, "datetimecol"] -= np.timedelta64(1, "ms")
+ tm.assert_frame_equal(df, expected)
+
+
+def test_meta2_page(datapath):
+ # GH 35545
+ fname = datapath("io", "sas", "data", "test_meta2_page.sas7bdat")
+ df = pd.read_sas(fname)
+ assert len(df) == 1000
+
+
+@pytest.mark.parametrize(
+ "test_file, override_offset, override_value, expected_msg",
+ [
+ ("test2.sas7bdat", 0x10000 + 55229, 0x80 | 0x0F, "Out of bounds"),
+ ("test2.sas7bdat", 0x10000 + 55229, 0x10, "unknown control byte"),
+ ("test3.sas7bdat", 118170, 184, "Out of bounds"),
+ ],
+)
+def test_rle_rdc_exceptions(
+ datapath, test_file, override_offset, override_value, expected_msg
+):
+ """Errors in RLE/RDC decompression should propagate."""
+ with open(datapath("io", "sas", "data", test_file), "rb") as fd:
+ data = bytearray(fd.read())
+ data[override_offset] = override_value
+ with pytest.raises(Exception, match=expected_msg):
+ pd.read_sas(io.BytesIO(data), format="sas7bdat")
+
+
+def test_0x40_control_byte(datapath):
+ # GH 31243
+ fname = datapath("io", "sas", "data", "0x40controlbyte.sas7bdat")
+ df = pd.read_sas(fname, encoding="ascii")
+ fname = datapath("io", "sas", "data", "0x40controlbyte.csv")
+ df0 = pd.read_csv(fname, dtype="str")
+ tm.assert_frame_equal(df, df0)
+
+
+def test_0x00_control_byte(datapath):
+ # GH 47099
+ fname = datapath("io", "sas", "data", "0x00controlbyte.sas7bdat.bz2")
+ df = next(pd.read_sas(fname, chunksize=11_000))
+ assert df.shape == (11_000, 20)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/sas/test_xport.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/sas/test_xport.py
new file mode 100644
index 0000000000000000000000000000000000000000..30ca70d7e000963207695da02490c003f61f7d28
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/sas/test_xport.py
@@ -0,0 +1,153 @@
+import numpy as np
+import pytest
+
+import pandas as pd
+import pandas._testing as tm
+
+from pandas.io.sas.sasreader import read_sas
+
+# CSV versions of test xpt files were obtained using the R foreign library
+
+# Numbers in a SAS xport file are always float64, so need to convert
+# before making comparisons.
+
+
+def numeric_as_float(data):
+ for v in data.columns:
+ if data[v].dtype is np.dtype("int64"):
+ data[v] = data[v].astype(np.float64)
+
+
+class TestXport:
+ @pytest.mark.slow
+ def test1_basic(self, datapath):
+ # Tests with DEMO_G.xpt (all numeric file)
+
+ # Compare to this
+ file01 = datapath("io", "sas", "data", "DEMO_G.xpt")
+ data_csv = pd.read_csv(file01.replace(".xpt", ".csv"))
+ numeric_as_float(data_csv)
+
+ # Read full file
+ data = read_sas(file01, format="xport")
+ tm.assert_frame_equal(data, data_csv)
+ num_rows = data.shape[0]
+
+ # Test reading beyond end of file
+ with read_sas(file01, format="xport", iterator=True) as reader:
+ data = reader.read(num_rows + 100)
+ assert data.shape[0] == num_rows
+
+ # Test incremental read with `read` method.
+ with read_sas(file01, format="xport", iterator=True) as reader:
+ data = reader.read(10)
+ tm.assert_frame_equal(data, data_csv.iloc[0:10, :])
+
+ # Test incremental read with `get_chunk` method.
+ with read_sas(file01, format="xport", chunksize=10) as reader:
+ data = reader.get_chunk()
+ tm.assert_frame_equal(data, data_csv.iloc[0:10, :])
+
+ # Test read in loop
+ m = 0
+ with read_sas(file01, format="xport", chunksize=100) as reader:
+ for x in reader:
+ m += x.shape[0]
+ assert m == num_rows
+
+ # Read full file with `read_sas` method
+ data = read_sas(file01)
+ tm.assert_frame_equal(data, data_csv)
+
+ def test1_index(self, datapath):
+ # Tests with DEMO_G.xpt using index (all numeric file)
+
+ # Compare to this
+ file01 = datapath("io", "sas", "data", "DEMO_G.xpt")
+ data_csv = pd.read_csv(file01.replace(".xpt", ".csv"))
+ data_csv = data_csv.set_index("SEQN")
+ numeric_as_float(data_csv)
+
+ # Read full file
+ data = read_sas(file01, index="SEQN", format="xport")
+ tm.assert_frame_equal(data, data_csv, check_index_type=False)
+
+ # Test incremental read with `read` method.
+ with read_sas(file01, index="SEQN", format="xport", iterator=True) as reader:
+ data = reader.read(10)
+ tm.assert_frame_equal(data, data_csv.iloc[0:10, :], check_index_type=False)
+
+ # Test incremental read with `get_chunk` method.
+ with read_sas(file01, index="SEQN", format="xport", chunksize=10) as reader:
+ data = reader.get_chunk()
+ tm.assert_frame_equal(data, data_csv.iloc[0:10, :], check_index_type=False)
+
+ def test1_incremental(self, datapath):
+ # Test with DEMO_G.xpt, reading full file incrementally
+
+ file01 = datapath("io", "sas", "data", "DEMO_G.xpt")
+ data_csv = pd.read_csv(file01.replace(".xpt", ".csv"))
+ data_csv = data_csv.set_index("SEQN")
+ numeric_as_float(data_csv)
+
+ with read_sas(file01, index="SEQN", chunksize=1000) as reader:
+ all_data = list(reader)
+ data = pd.concat(all_data, axis=0)
+
+ tm.assert_frame_equal(data, data_csv, check_index_type=False)
+
+ def test2(self, datapath):
+ # Test with SSHSV1_A.xpt
+
+ file02 = datapath("io", "sas", "data", "SSHSV1_A.xpt")
+ # Compare to this
+ data_csv = pd.read_csv(file02.replace(".xpt", ".csv"))
+ numeric_as_float(data_csv)
+
+ data = read_sas(file02)
+ tm.assert_frame_equal(data, data_csv)
+
+ def test2_binary(self, datapath):
+ # Test with SSHSV1_A.xpt, read as a binary file
+
+ # Compare to this
+ file02 = datapath("io", "sas", "data", "SSHSV1_A.xpt")
+ data_csv = pd.read_csv(file02.replace(".xpt", ".csv"))
+ numeric_as_float(data_csv)
+
+ with open(file02, "rb") as fd:
+ # GH#35693 ensure that if we pass an open file, we
+ # dont incorrectly close it in read_sas
+ data = read_sas(fd, format="xport")
+
+ tm.assert_frame_equal(data, data_csv)
+
+ def test_multiple_types(self, datapath):
+ # Test with DRXFCD_G.xpt (contains text and numeric variables)
+
+ # Compare to this
+ file03 = datapath("io", "sas", "data", "DRXFCD_G.xpt")
+ data_csv = pd.read_csv(file03.replace(".xpt", ".csv"))
+
+ data = read_sas(file03, encoding="utf-8")
+ tm.assert_frame_equal(data, data_csv)
+
+ def test_truncated_float_support(self, datapath):
+ # Test with paxraw_d_short.xpt, a shortened version of:
+ # http://wwwn.cdc.gov/Nchs/Nhanes/2005-2006/PAXRAW_D.ZIP
+ # This file has truncated floats (5 bytes in this case).
+
+ # GH 11713
+ file04 = datapath("io", "sas", "data", "paxraw_d_short.xpt")
+ data_csv = pd.read_csv(file04.replace(".xpt", ".csv"))
+
+ data = read_sas(file04, format="xport")
+ tm.assert_frame_equal(data.astype("int64"), data_csv)
+
+ def test_cport_header_found_raises(self, datapath):
+ # Test with DEMO_PUF.cpt, the beginning of puf2019_1_fall.xpt
+ # from https://www.cms.gov/files/zip/puf2019.zip
+ # (despite the extension, it's a cpt file)
+ msg = "Header record indicates a CPORT file, which is not readable."
+ with pytest.raises(ValueError, match=msg):
+ read_sas(datapath("io", "sas", "data", "DEMO_PUF.cpt"), format="xport")
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_clipboard.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_clipboard.py
new file mode 100644
index 0000000000000000000000000000000000000000..25834c47c09c67c3feadd7817a4902e6bdeff378
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_clipboard.py
@@ -0,0 +1,402 @@
+from textwrap import dedent
+
+import numpy as np
+import pytest
+
+from pandas.errors import (
+ PyperclipException,
+ PyperclipWindowsException,
+)
+
+import pandas as pd
+from pandas import (
+ NA,
+ DataFrame,
+ Series,
+ get_option,
+ read_clipboard,
+)
+import pandas._testing as tm
+
+from pandas.io.clipboard import (
+ CheckedCall,
+ _stringifyText,
+ init_qt_clipboard,
+)
+
+
+def build_kwargs(sep, excel):
+ kwargs = {}
+ if excel != "default":
+ kwargs["excel"] = excel
+ if sep != "default":
+ kwargs["sep"] = sep
+ return kwargs
+
+
+@pytest.fixture(
+ params=[
+ "delims",
+ "utf8",
+ "utf16",
+ "string",
+ "long",
+ "nonascii",
+ "colwidth",
+ "mixed",
+ "float",
+ "int",
+ ]
+)
+def df(request):
+ data_type = request.param
+
+ if data_type == "delims":
+ return DataFrame({"a": ['"a,\t"b|c', "d\tef`"], "b": ["hi'j", "k''lm"]})
+ elif data_type == "utf8":
+ return DataFrame({"a": ["µasd", "Ωœ∑`"], "b": ["øπ∆˚¬", "œ∑`®"]})
+ elif data_type == "utf16":
+ return DataFrame(
+ {"a": ["\U0001f44d\U0001f44d", "\U0001f44d\U0001f44d"], "b": ["abc", "def"]}
+ )
+ elif data_type == "string":
+ return DataFrame(
+ np.array([f"i-{i}" for i in range(15)]).reshape(5, 3), columns=list("abc")
+ )
+ elif data_type == "long":
+ max_rows = get_option("display.max_rows")
+ return DataFrame(
+ np.random.default_rng(2).integers(0, 10, size=(max_rows + 1, 3)),
+ columns=list("abc"),
+ )
+ elif data_type == "nonascii":
+ return DataFrame({"en": "in English".split(), "es": "en español".split()})
+ elif data_type == "colwidth":
+ _cw = get_option("display.max_colwidth") + 1
+ return DataFrame(
+ np.array(["x" * _cw for _ in range(15)]).reshape(5, 3), columns=list("abc")
+ )
+ elif data_type == "mixed":
+ return DataFrame(
+ {
+ "a": np.arange(1.0, 6.0) + 0.01,
+ "b": np.arange(1, 6).astype(np.int64),
+ "c": list("abcde"),
+ }
+ )
+ elif data_type == "float":
+ return DataFrame(np.random.default_rng(2).random((5, 3)), columns=list("abc"))
+ elif data_type == "int":
+ return DataFrame(
+ np.random.default_rng(2).integers(0, 10, (5, 3)), columns=list("abc")
+ )
+ else:
+ raise ValueError
+
+
+@pytest.fixture
+def mock_ctypes(monkeypatch):
+ """
+ Mocks WinError to help with testing the clipboard.
+ """
+
+ def _mock_win_error():
+ return "Window Error"
+
+ # Set raising to False because WinError won't exist on non-windows platforms
+ with monkeypatch.context() as m:
+ m.setattr("ctypes.WinError", _mock_win_error, raising=False)
+ yield
+
+
+@pytest.mark.usefixtures("mock_ctypes")
+def test_checked_call_with_bad_call(monkeypatch):
+ """
+ Give CheckCall a function that returns a falsey value and
+ mock get_errno so it returns false so an exception is raised.
+ """
+
+ def _return_false():
+ return False
+
+ monkeypatch.setattr("pandas.io.clipboard.get_errno", lambda: True)
+ msg = f"Error calling {_return_false.__name__} \\(Window Error\\)"
+
+ with pytest.raises(PyperclipWindowsException, match=msg):
+ CheckedCall(_return_false)()
+
+
+@pytest.mark.usefixtures("mock_ctypes")
+def test_checked_call_with_valid_call(monkeypatch):
+ """
+ Give CheckCall a function that returns a truthy value and
+ mock get_errno so it returns true so an exception is not raised.
+ The function should return the results from _return_true.
+ """
+
+ def _return_true():
+ return True
+
+ monkeypatch.setattr("pandas.io.clipboard.get_errno", lambda: False)
+
+ # Give CheckedCall a callable that returns a truthy value s
+ checked_call = CheckedCall(_return_true)
+ assert checked_call() is True
+
+
+@pytest.mark.parametrize(
+ "text",
+ [
+ "String_test",
+ True,
+ 1,
+ 1.0,
+ 1j,
+ ],
+)
+def test_stringify_text(text):
+ valid_types = (str, int, float, bool)
+
+ if isinstance(text, valid_types):
+ result = _stringifyText(text)
+ assert result == str(text)
+ else:
+ msg = (
+ "only str, int, float, and bool values "
+ f"can be copied to the clipboard, not {type(text).__name__}"
+ )
+ with pytest.raises(PyperclipException, match=msg):
+ _stringifyText(text)
+
+
+@pytest.fixture
+def set_pyqt_clipboard(monkeypatch):
+ qt_cut, qt_paste = init_qt_clipboard()
+ with monkeypatch.context() as m:
+ m.setattr(pd.io.clipboard, "clipboard_set", qt_cut)
+ m.setattr(pd.io.clipboard, "clipboard_get", qt_paste)
+ yield
+
+
+@pytest.fixture
+def clipboard(qapp):
+ clip = qapp.clipboard()
+ yield clip
+ clip.clear()
+
+
+@pytest.mark.single_cpu
+@pytest.mark.clipboard
+@pytest.mark.usefixtures("set_pyqt_clipboard")
+@pytest.mark.usefixtures("clipboard")
+class TestClipboard:
+ # Test that default arguments copy as tab delimited
+ # Test that explicit delimiters are respected
+ @pytest.mark.parametrize("sep", [None, "\t", ",", "|"])
+ @pytest.mark.parametrize("encoding", [None, "UTF-8", "utf-8", "utf8"])
+ def test_round_trip_frame_sep(self, df, sep, encoding):
+ df.to_clipboard(excel=None, sep=sep, encoding=encoding)
+ result = read_clipboard(sep=sep or "\t", index_col=0, encoding=encoding)
+ tm.assert_frame_equal(df, result)
+
+ # Test white space separator
+ def test_round_trip_frame_string(self, df):
+ df.to_clipboard(excel=False, sep=None)
+ result = read_clipboard()
+ assert df.to_string() == result.to_string()
+ assert df.shape == result.shape
+
+ # Two character separator is not supported in to_clipboard
+ # Test that multi-character separators are not silently passed
+ def test_excel_sep_warning(self, df):
+ with tm.assert_produces_warning(
+ UserWarning,
+ match="to_clipboard in excel mode requires a single character separator.",
+ check_stacklevel=False,
+ ):
+ df.to_clipboard(excel=True, sep=r"\t")
+
+ # Separator is ignored when excel=False and should produce a warning
+ def test_copy_delim_warning(self, df):
+ with tm.assert_produces_warning(UserWarning, match="ignores the sep argument"):
+ df.to_clipboard(excel=False, sep="\t")
+
+ # Tests that the default behavior of to_clipboard is tab
+ # delimited and excel="True"
+ @pytest.mark.parametrize("sep", ["\t", None, "default"])
+ @pytest.mark.parametrize("excel", [True, None, "default"])
+ def test_clipboard_copy_tabs_default(self, sep, excel, df, clipboard):
+ kwargs = build_kwargs(sep, excel)
+ df.to_clipboard(**kwargs)
+ assert clipboard.text() == df.to_csv(sep="\t")
+
+ # Tests reading of white space separated tables
+ @pytest.mark.parametrize("sep", [None, "default"])
+ def test_clipboard_copy_strings(self, sep, df):
+ kwargs = build_kwargs(sep, False)
+ df.to_clipboard(**kwargs)
+ result = read_clipboard(sep=r"\s+")
+ assert result.to_string() == df.to_string()
+ assert df.shape == result.shape
+
+ def test_read_clipboard_infer_excel(self, clipboard):
+ # gh-19010: avoid warnings
+ clip_kwargs = {"engine": "python"}
+
+ text = dedent(
+ """
+ John James\tCharlie Mingus
+ 1\t2
+ 4\tHarry Carney
+ """.strip()
+ )
+ clipboard.setText(text)
+ df = read_clipboard(**clip_kwargs)
+
+ # excel data is parsed correctly
+ assert df.iloc[1, 1] == "Harry Carney"
+
+ # having diff tab counts doesn't trigger it
+ text = dedent(
+ """
+ a\t b
+ 1 2
+ 3 4
+ """.strip()
+ )
+ clipboard.setText(text)
+ res = read_clipboard(**clip_kwargs)
+
+ text = dedent(
+ """
+ a b
+ 1 2
+ 3 4
+ """.strip()
+ )
+ clipboard.setText(text)
+ exp = read_clipboard(**clip_kwargs)
+
+ tm.assert_frame_equal(res, exp)
+
+ def test_infer_excel_with_nulls(self, clipboard):
+ # GH41108
+ text = "col1\tcol2\n1\tred\n\tblue\n2\tgreen"
+
+ clipboard.setText(text)
+ df = read_clipboard()
+ df_expected = DataFrame(
+ data={"col1": [1, None, 2], "col2": ["red", "blue", "green"]}
+ )
+
+ # excel data is parsed correctly
+ tm.assert_frame_equal(df, df_expected)
+
+ @pytest.mark.parametrize(
+ "multiindex",
+ [
+ ( # Can't use `dedent` here as it will remove the leading `\t`
+ "\n".join(
+ [
+ "\t\t\tcol1\tcol2",
+ "A\t0\tTrue\t1\tred",
+ "A\t1\tTrue\t\tblue",
+ "B\t0\tFalse\t2\tgreen",
+ ]
+ ),
+ [["A", "A", "B"], [0, 1, 0], [True, True, False]],
+ ),
+ (
+ "\n".join(
+ ["\t\tcol1\tcol2", "A\t0\t1\tred", "A\t1\t\tblue", "B\t0\t2\tgreen"]
+ ),
+ [["A", "A", "B"], [0, 1, 0]],
+ ),
+ ],
+ )
+ def test_infer_excel_with_multiindex(self, clipboard, multiindex):
+ # GH41108
+
+ clipboard.setText(multiindex[0])
+ df = read_clipboard()
+ df_expected = DataFrame(
+ data={"col1": [1, None, 2], "col2": ["red", "blue", "green"]},
+ index=multiindex[1],
+ )
+
+ # excel data is parsed correctly
+ tm.assert_frame_equal(df, df_expected)
+
+ def test_invalid_encoding(self, df):
+ msg = "clipboard only supports utf-8 encoding"
+ # test case for testing invalid encoding
+ with pytest.raises(ValueError, match=msg):
+ df.to_clipboard(encoding="ascii")
+ with pytest.raises(NotImplementedError, match=msg):
+ read_clipboard(encoding="ascii")
+
+ @pytest.mark.parametrize("data", ["\U0001f44d...", "Ωœ∑`...", "abcd..."])
+ def test_raw_roundtrip(self, data):
+ # PR #25040 wide unicode wasn't copied correctly on PY3 on windows
+ df = DataFrame({"data": [data]})
+ df.to_clipboard()
+ result = read_clipboard()
+ tm.assert_frame_equal(df, result)
+
+ @pytest.mark.parametrize("engine", ["c", "python"])
+ def test_read_clipboard_dtype_backend(
+ self, clipboard, string_storage, dtype_backend, engine, using_infer_string
+ ):
+ # GH#50502
+ if dtype_backend == "pyarrow":
+ pa = pytest.importorskip("pyarrow")
+ string_dtype = pd.ArrowDtype(pa.string())
+ else:
+ string_dtype = pd.StringDtype(string_storage)
+
+ text = """a,b,c,d,e,f,g,h,i
+x,1,4.0,x,2,4.0,,True,False
+y,2,5.0,,,,,False,"""
+ clipboard.setText(text)
+
+ with pd.option_context("mode.string_storage", string_storage):
+ result = read_clipboard(sep=",", dtype_backend=dtype_backend, engine=engine)
+
+ expected = DataFrame(
+ {
+ "a": Series(["x", "y"], dtype=string_dtype),
+ "b": Series([1, 2], dtype="Int64"),
+ "c": Series([4.0, 5.0], dtype="Float64"),
+ "d": Series(["x", None], dtype=string_dtype),
+ "e": Series([2, NA], dtype="Int64"),
+ "f": Series([4.0, NA], dtype="Float64"),
+ "g": Series([NA, NA], dtype="Int64"),
+ "h": Series([True, False], dtype="boolean"),
+ "i": Series([False, NA], dtype="boolean"),
+ }
+ )
+ if dtype_backend == "pyarrow":
+ from pandas.arrays import ArrowExtensionArray
+
+ expected = DataFrame(
+ {
+ col: ArrowExtensionArray(pa.array(expected[col], from_pandas=True))
+ for col in expected.columns
+ }
+ )
+ expected["g"] = ArrowExtensionArray(pa.array([None, None]))
+
+ if using_infer_string:
+ expected.columns = expected.columns.astype(
+ pd.StringDtype(string_storage, na_value=np.nan)
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_invalid_dtype_backend(self):
+ msg = (
+ "dtype_backend numpy is invalid, only 'numpy_nullable' and "
+ "'pyarrow' are allowed."
+ )
+ with pytest.raises(ValueError, match=msg):
+ read_clipboard(dtype_backend="numpy")
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_common.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_common.py
new file mode 100644
index 0000000000000000000000000000000000000000..a5081109d2299799f9c982d8117f65157049d3ad
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_common.py
@@ -0,0 +1,688 @@
+"""
+Tests for the pandas.io.common functionalities
+"""
+
+import codecs
+import errno
+from functools import partial
+from io import (
+ BytesIO,
+ StringIO,
+ UnsupportedOperation,
+)
+import mmap
+import os
+from pathlib import Path
+import pickle
+import tempfile
+
+import numpy as np
+import pytest
+
+from pandas.compat import (
+ WASM,
+ is_platform_windows,
+)
+from pandas.compat.pyarrow import pa_version_under19p0
+import pandas.util._test_decorators as td
+
+import pandas as pd
+import pandas._testing as tm
+
+import pandas.io.common as icom
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+class CustomFSPath:
+ """For testing fspath on unknown objects"""
+
+ def __init__(self, path) -> None:
+ self.path = path
+
+ def __fspath__(self):
+ return self.path
+
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+
+
+# https://github.com/cython/cython/issues/1720
+class TestCommonIOCapabilities:
+ data1 = """index,A,B,C,D
+foo,2,3,4,5
+bar,7,8,9,10
+baz,12,13,14,15
+qux,12,13,14,15
+foo2,12,13,14,15
+bar2,12,13,14,15
+"""
+
+ def test_expand_user(self):
+ filename = "~/sometest"
+ expanded_name = icom._expand_user(filename)
+
+ assert expanded_name != filename
+ assert os.path.isabs(expanded_name)
+ assert os.path.expanduser(filename) == expanded_name
+
+ def test_expand_user_normal_path(self):
+ filename = "/somefolder/sometest"
+ expanded_name = icom._expand_user(filename)
+
+ assert expanded_name == filename
+ assert os.path.expanduser(filename) == expanded_name
+
+ def test_stringify_path_pathlib(self):
+ rel_path = icom.stringify_path(Path("."))
+ assert rel_path == "."
+ redundant_path = icom.stringify_path(Path("foo//bar"))
+ assert redundant_path == os.path.join("foo", "bar")
+
+ def test_stringify_path_fspath(self):
+ p = CustomFSPath("foo/bar.csv")
+ result = icom.stringify_path(p)
+ assert result == "foo/bar.csv"
+
+ def test_stringify_file_and_path_like(self, temp_file):
+ # GH 38125: do not stringify file objects that are also path-like
+ fsspec = pytest.importorskip("fsspec")
+ with fsspec.open(f"file://{temp_file}", mode="wb") as fsspec_obj:
+ assert fsspec_obj == icom.stringify_path(fsspec_obj)
+
+ @pytest.mark.parametrize("path_type", [str, CustomFSPath, Path])
+ def test_infer_compression_from_path(self, compression_format, path_type):
+ extension, expected = compression_format
+ path = path_type("foo/bar.csv" + extension)
+ compression = icom.infer_compression(path, compression="infer")
+ assert compression == expected
+
+ @pytest.mark.parametrize("path_type", [str, CustomFSPath, Path])
+ def test_get_handle_with_path(self, path_type):
+ with tempfile.TemporaryDirectory(dir=Path.home()) as tmp:
+ filename = path_type("~/" + Path(tmp).name + "/sometest")
+ with icom.get_handle(filename, "w") as handles:
+ assert Path(handles.handle.name).is_absolute()
+ assert os.path.expanduser(filename) == handles.handle.name
+
+ def test_get_handle_with_buffer(self):
+ with StringIO() as input_buffer:
+ with icom.get_handle(input_buffer, "r") as handles:
+ assert handles.handle == input_buffer
+ assert not input_buffer.closed
+ assert input_buffer.closed
+
+ # Test that BytesIOWrapper(get_handle) returns correct amount of bytes every time
+ def test_bytesiowrapper_returns_correct_bytes(self):
+ # Test latin1, ucs-2, and ucs-4 chars
+ data = """a,b,c
+1,2,3
+©,®,®
+Look,a snake,🐍"""
+ with icom.get_handle(StringIO(data), "rb", is_text=False) as handles:
+ result = b""
+ chunksize = 5
+ while True:
+ chunk = handles.handle.read(chunksize)
+ # Make sure each chunk is correct amount of bytes
+ assert len(chunk) <= chunksize
+ if len(chunk) < chunksize:
+ # Can be less amount of bytes, but only at EOF
+ # which happens when read returns empty
+ assert len(handles.handle.read()) == 0
+ result += chunk
+ break
+ result += chunk
+ assert result == data.encode("utf-8")
+
+ # Test that pyarrow can handle a file opened with get_handle
+ def test_get_handle_pyarrow_compat(sel, using_infer_string):
+ pa_csv = pytest.importorskip("pyarrow.csv")
+
+ # Test latin1, ucs-2, and ucs-4 chars
+ data = """a,b,c
+1,2,3
+©,®,®
+Look,a snake,🐍"""
+ expected = pd.DataFrame(
+ {"a": ["1", "©", "Look"], "b": ["2", "®", "a snake"], "c": ["3", "®", "🐍"]}
+ )
+ s = StringIO(data)
+ with icom.get_handle(s, "rb", is_text=False) as handles:
+ df = pa_csv.read_csv(handles.handle).to_pandas()
+ if pa_version_under19p0:
+ expected = expected.astype("object")
+ elif not using_infer_string:
+ expected = expected.astype(pd.StringDtype(na_value=np.nan))
+ tm.assert_frame_equal(df, expected)
+ assert not s.closed
+
+ def test_iterator(self):
+ with pd.read_csv(StringIO(self.data1), chunksize=1) as reader:
+ result = pd.concat(reader, ignore_index=True)
+ expected = pd.read_csv(StringIO(self.data1))
+ tm.assert_frame_equal(result, expected)
+
+ # GH12153
+ with pd.read_csv(StringIO(self.data1), chunksize=1) as it:
+ first = next(it)
+ tm.assert_frame_equal(first, expected.iloc[[0]])
+ tm.assert_frame_equal(pd.concat(it), expected.iloc[1:])
+
+ @pytest.mark.skipif(WASM, reason="limited file system access on WASM")
+ @pytest.mark.parametrize(
+ "reader, module, error_class, fn_ext",
+ [
+ (pd.read_csv, "os", FileNotFoundError, "csv"),
+ (pd.read_fwf, "os", FileNotFoundError, "txt"),
+ (pd.read_excel, "xlrd", FileNotFoundError, "xlsx"),
+ (pd.read_feather, "pyarrow", OSError, "feather"),
+ (pd.read_hdf, "tables", FileNotFoundError, "h5"),
+ (pd.read_stata, "os", FileNotFoundError, "dta"),
+ (pd.read_sas, "os", FileNotFoundError, "sas7bdat"),
+ (pd.read_json, "os", FileNotFoundError, "json"),
+ (pd.read_pickle, "os", FileNotFoundError, "pickle"),
+ ],
+ )
+ def test_read_non_existent(self, reader, module, error_class, fn_ext):
+ pytest.importorskip(module)
+
+ path = os.path.join(HERE, "data", "does_not_exist." + fn_ext)
+ msg1 = rf"File (b')?.+does_not_exist\.{fn_ext}'? does not exist"
+ msg2 = rf"\[Errno 2\] No such file or directory: '.+does_not_exist\.{fn_ext}'"
+ msg3 = "Expected object or value"
+ msg4 = "path_or_buf needs to be a string file path or file-like"
+ msg5 = (
+ rf"\[Errno 2\] File .+does_not_exist\.{fn_ext} does not exist: "
+ rf"'.+does_not_exist\.{fn_ext}'"
+ )
+ msg6 = rf"\[Errno 2\] 没有那个文件或目录: '.+does_not_exist\.{fn_ext}'"
+ msg7 = (
+ rf"\[Errno 2\] File o directory non esistente: '.+does_not_exist\.{fn_ext}'"
+ )
+ msg8 = rf"Failed to open local file.+does_not_exist\.{fn_ext}"
+
+ with pytest.raises(
+ error_class,
+ match=rf"({msg1}|{msg2}|{msg3}|{msg4}|{msg5}|{msg6}|{msg7}|{msg8})",
+ ):
+ reader(path)
+
+ @pytest.mark.parametrize(
+ "method, module, error_class, fn_ext",
+ [
+ (pd.DataFrame.to_csv, "os", OSError, "csv"),
+ (pd.DataFrame.to_html, "os", OSError, "html"),
+ (pd.DataFrame.to_excel, "xlrd", OSError, "xlsx"),
+ (pd.DataFrame.to_feather, "pyarrow", OSError, "feather"),
+ (pd.DataFrame.to_parquet, "pyarrow", OSError, "parquet"),
+ (pd.DataFrame.to_stata, "os", OSError, "dta"),
+ (pd.DataFrame.to_json, "os", OSError, "json"),
+ (pd.DataFrame.to_pickle, "os", OSError, "pickle"),
+ ],
+ )
+ # NOTE: Missing parent directory for pd.DataFrame.to_hdf is handled by PyTables
+ def test_write_missing_parent_directory(self, method, module, error_class, fn_ext):
+ pytest.importorskip(module)
+
+ dummy_frame = pd.DataFrame({"a": [1, 2, 3], "b": [2, 3, 4], "c": [3, 4, 5]})
+
+ path = os.path.join(HERE, "data", "missing_folder", "does_not_exist." + fn_ext)
+
+ with pytest.raises(
+ error_class,
+ match=r"Cannot save file into a non-existent directory: .*missing_folder",
+ ):
+ method(dummy_frame, path)
+
+ @pytest.mark.skipif(WASM, reason="limited file system access on WASM")
+ @pytest.mark.parametrize(
+ "reader, module, error_class, fn_ext",
+ [
+ (pd.read_csv, "os", FileNotFoundError, "csv"),
+ (pd.read_table, "os", FileNotFoundError, "csv"),
+ (pd.read_fwf, "os", FileNotFoundError, "txt"),
+ (pd.read_excel, "xlrd", FileNotFoundError, "xlsx"),
+ (pd.read_feather, "pyarrow", OSError, "feather"),
+ (pd.read_hdf, "tables", FileNotFoundError, "h5"),
+ (pd.read_stata, "os", FileNotFoundError, "dta"),
+ (pd.read_sas, "os", FileNotFoundError, "sas7bdat"),
+ (pd.read_json, "os", FileNotFoundError, "json"),
+ (pd.read_pickle, "os", FileNotFoundError, "pickle"),
+ ],
+ )
+ def test_read_expands_user_home_dir(
+ self, reader, module, error_class, fn_ext, monkeypatch
+ ):
+ pytest.importorskip(module)
+
+ path = os.path.join("~", "does_not_exist." + fn_ext)
+ monkeypatch.setattr(icom, "_expand_user", lambda x: os.path.join("foo", x))
+
+ msg1 = rf"File (b')?.+does_not_exist\.{fn_ext}'? does not exist"
+ msg2 = rf"\[Errno 2\] No such file or directory: '.+does_not_exist\.{fn_ext}'"
+ msg3 = "Unexpected character found when decoding 'false'"
+ msg4 = "path_or_buf needs to be a string file path or file-like"
+ msg5 = (
+ rf"\[Errno 2\] File .+does_not_exist\.{fn_ext} does not exist: "
+ rf"'.+does_not_exist\.{fn_ext}'"
+ )
+ msg6 = rf"\[Errno 2\] 没有那个文件或目录: '.+does_not_exist\.{fn_ext}'"
+ msg7 = (
+ rf"\[Errno 2\] File o directory non esistente: '.+does_not_exist\.{fn_ext}'"
+ )
+ msg8 = rf"Failed to open local file.+does_not_exist\.{fn_ext}"
+
+ with pytest.raises(
+ error_class,
+ match=rf"({msg1}|{msg2}|{msg3}|{msg4}|{msg5}|{msg6}|{msg7}|{msg8})",
+ ):
+ reader(path)
+
+ @pytest.mark.parametrize(
+ "reader, module, path",
+ [
+ (pd.read_csv, "os", ("io", "data", "csv", "iris.csv")),
+ (pd.read_table, "os", ("io", "data", "csv", "iris.csv")),
+ (
+ pd.read_fwf,
+ "os",
+ ("io", "data", "fixed_width", "fixed_width_format.txt"),
+ ),
+ (pd.read_excel, "xlrd", ("io", "data", "excel", "test1.xlsx")),
+ (
+ pd.read_feather,
+ "pyarrow",
+ ("io", "data", "feather", "feather-0_3_1.feather"),
+ ),
+ (
+ pd.read_hdf,
+ "tables",
+ ("io", "data", "legacy_hdf", "pytables_native2.h5"),
+ ),
+ (pd.read_stata, "os", ("io", "data", "stata", "stata10_115.dta")),
+ (pd.read_sas, "os", ("io", "sas", "data", "test1.sas7bdat")),
+ (pd.read_json, "os", ("io", "json", "data", "tsframe_v012.json")),
+ (
+ pd.read_pickle,
+ "os",
+ ("io", "data", "pickle", "categorical.0.25.0.pickle"),
+ ),
+ ],
+ )
+ def test_read_fspath_all(self, reader, module, path, datapath):
+ pytest.importorskip(module)
+ path = datapath(*path)
+
+ mypath = CustomFSPath(path)
+ result = reader(mypath)
+ expected = reader(path)
+
+ if path.endswith(".pickle"):
+ # categorical
+ tm.assert_categorical_equal(result, expected)
+ else:
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "writer_name, writer_kwargs, module",
+ [
+ ("to_csv", {}, "os"),
+ ("to_excel", {"engine": "openpyxl"}, "openpyxl"),
+ ("to_feather", {}, "pyarrow"),
+ ("to_html", {}, "os"),
+ ("to_json", {}, "os"),
+ ("to_latex", {}, "os"),
+ ("to_pickle", {}, "os"),
+ ("to_stata", {"time_stamp": pd.to_datetime("2019-01-01 00:00")}, "os"),
+ ],
+ )
+ def test_write_fspath_all(self, writer_name, writer_kwargs, module, tmp_path):
+ if writer_name in ["to_latex"]: # uses Styler implementation
+ pytest.importorskip("jinja2")
+ string = str(tmp_path / "string")
+ fspath = str(tmp_path / "fspath")
+ df = pd.DataFrame({"A": [1, 2]})
+
+ pytest.importorskip(module)
+ mypath = CustomFSPath(fspath)
+ writer = getattr(df, writer_name)
+
+ writer(string, **writer_kwargs)
+ writer(mypath, **writer_kwargs)
+ with open(string, "rb") as f_str, open(fspath, "rb") as f_path:
+ if writer_name == "to_excel":
+ # binary representation of excel contains time creation
+ # data that causes flaky CI failures
+ result = pd.read_excel(f_str, **writer_kwargs)
+ expected = pd.read_excel(f_path, **writer_kwargs)
+ tm.assert_frame_equal(result, expected)
+ else:
+ result = f_str.read()
+ expected = f_path.read()
+ assert result == expected
+
+ def test_write_fspath_hdf5(self, tmp_path):
+ # Same test as write_fspath_all, except HDF5 files aren't
+ # necessarily byte-for-byte identical for a given dataframe, so we'll
+ # have to read and compare equality
+ pytest.importorskip("tables")
+
+ df = pd.DataFrame({"A": [1, 2]})
+ string = str(tmp_path / "string")
+ fspath = str(tmp_path / "fspath")
+
+ mypath = CustomFSPath(fspath)
+ df.to_hdf(mypath, key="bar")
+ df.to_hdf(string, key="bar")
+
+ result = pd.read_hdf(fspath, key="bar")
+ expected = pd.read_hdf(string, key="bar")
+
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.fixture
+def mmap_file(datapath):
+ return datapath("io", "data", "csv", "test_mmap.csv")
+
+
+class TestMMapWrapper:
+ @pytest.mark.skipif(WASM, reason="limited file system access on WASM")
+ def test_constructor_bad_file(self, mmap_file):
+ non_file = StringIO("I am not a file")
+ non_file.fileno = lambda: -1
+
+ # the error raised is different on Windows
+ if is_platform_windows():
+ msg = "The parameter is incorrect"
+ err = OSError
+ else:
+ msg = "[Errno 22]"
+ err = mmap.error
+
+ with pytest.raises(err, match=msg):
+ icom._maybe_memory_map(non_file, True)
+
+ with open(mmap_file, encoding="utf-8") as target:
+ pass
+
+ msg = "I/O operation on closed file"
+ with pytest.raises(ValueError, match=msg):
+ icom._maybe_memory_map(target, True)
+
+ @pytest.mark.skipif(WASM, reason="limited file system access on WASM")
+ def test_next(self, mmap_file):
+ with open(mmap_file, encoding="utf-8") as target:
+ lines = target.readlines()
+
+ with icom.get_handle(
+ target, "r", is_text=True, memory_map=True
+ ) as wrappers:
+ wrapper = wrappers.handle
+ assert isinstance(wrapper.buffer.buffer, mmap.mmap)
+
+ for line in lines:
+ next_line = next(wrapper)
+ assert next_line.strip() == line.strip()
+
+ with pytest.raises(StopIteration, match=r"^$"):
+ next(wrapper)
+
+ def test_unknown_engine(self, temp_file):
+ df = pd.DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ )
+ df.to_csv(temp_file)
+ with pytest.raises(ValueError, match="Unknown engine"):
+ pd.read_csv(temp_file, engine="pyt")
+
+ def test_binary_mode(self, temp_file):
+ """
+ 'encoding' shouldn't be passed to 'open' in binary mode.
+
+ GH 35058
+ """
+ df = pd.DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ )
+ df.to_csv(temp_file, mode="w+b")
+ tm.assert_frame_equal(df, pd.read_csv(temp_file, index_col=0))
+
+ @pytest.mark.parametrize("encoding", ["utf-16", "utf-32"])
+ @pytest.mark.parametrize("compression_", ["bz2", "xz"])
+ def test_warning_missing_utf_bom(self, encoding, compression_, temp_file):
+ """
+ bz2 and xz do not write the byte order mark (BOM) for utf-16/32.
+
+ https://stackoverflow.com/questions/55171439
+
+ GH 35681
+ """
+ df = pd.DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ )
+ with tm.assert_produces_warning(UnicodeWarning, match="byte order mark"):
+ df.to_csv(temp_file, compression=compression_, encoding=encoding)
+
+ # reading should fail (otherwise we wouldn't need the warning)
+ msg = (
+ r"UTF-\d+ stream does not start with BOM|"
+ r"'utf-\d+' codec can't decode byte"
+ )
+ with pytest.raises(UnicodeError, match=msg):
+ pd.read_csv(temp_file, compression=compression_, encoding=encoding)
+
+
+def test_is_fsspec_url():
+ assert icom.is_fsspec_url("gcs://pandas/somethingelse.com")
+ assert icom.is_fsspec_url("gs://pandas/somethingelse.com")
+ # the following is the only remote URL that is handled without fsspec
+ assert not icom.is_fsspec_url("http://pandas/somethingelse.com")
+ assert not icom.is_fsspec_url("random:pandas/somethingelse.com")
+ assert not icom.is_fsspec_url("/local/path")
+ assert not icom.is_fsspec_url("relative/local/path")
+ # fsspec URL in string should not be recognized
+ assert not icom.is_fsspec_url("this is not fsspec://url")
+ assert not icom.is_fsspec_url("{'url': 'gs://pandas/somethingelse.com'}")
+ # accept everything that conforms to RFC 3986 schema
+ assert icom.is_fsspec_url("RFC-3986+compliant.spec://something")
+
+
+def test_is_fsspec_url_chained():
+ # GH#48978 Support chained fsspec URLs
+ # See https://filesystem-spec.readthedocs.io/en/latest/features.html#url-chaining.
+ assert icom.is_fsspec_url("filecache::s3://pandas/test.csv")
+ assert icom.is_fsspec_url("zip://test.csv::filecache::gcs://bucket/file.zip")
+ assert icom.is_fsspec_url("filecache::zip://test.csv::gcs://bucket/file.zip")
+ assert icom.is_fsspec_url("filecache::dask::s3://pandas/test.csv")
+ assert not icom.is_fsspec_url("filecache:s3://pandas/test.csv")
+ assert not icom.is_fsspec_url("filecache:::s3://pandas/test.csv")
+ assert not icom.is_fsspec_url("filecache::://pandas/test.csv")
+
+
+@pytest.mark.parametrize("format", ["csv", "json"])
+def test_codecs_encoding(format, temp_file):
+ # GH39247
+ expected = pd.DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ )
+ with open(temp_file, mode="w", encoding="utf-8") as handle:
+ getattr(expected, f"to_{format}")(handle)
+ with open(temp_file, encoding="utf-8") as handle:
+ if format == "csv":
+ df = pd.read_csv(handle, index_col=0)
+ else:
+ df = pd.read_json(handle)
+ tm.assert_frame_equal(expected, df)
+
+
+def test_codecs_get_writer_reader(temp_file):
+ # GH39247
+ expected = pd.DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ )
+ with open(temp_file, "wb") as handle:
+ with codecs.getwriter("utf-8")(handle) as encoded:
+ expected.to_csv(encoded)
+ with open(temp_file, "rb") as handle:
+ with codecs.getreader("utf-8")(handle) as encoded:
+ df = pd.read_csv(encoded, index_col=0)
+ tm.assert_frame_equal(expected, df)
+
+
+@pytest.mark.parametrize(
+ "io_class,mode,msg",
+ [
+ (BytesIO, "t", "a bytes-like object is required, not 'str'"),
+ (StringIO, "b", "string argument expected, got 'bytes'"),
+ ],
+)
+def test_explicit_encoding(io_class, mode, msg):
+ # GH39247; this test makes sure that if a user provides mode="*t" or "*b",
+ # it is used. In the case of this test it leads to an error as intentionally the
+ # wrong mode is requested
+ expected = pd.DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ )
+ with io_class() as buffer:
+ with pytest.raises(TypeError, match=msg):
+ expected.to_csv(buffer, mode=f"w{mode}")
+
+
+@pytest.mark.parametrize("encoding_errors", ["strict", "replace"])
+@pytest.mark.parametrize("format", ["csv", "json"])
+def test_encoding_errors(encoding_errors, format, temp_file):
+ # GH39450
+ msg = "'utf-8' codec can't decode byte"
+ bad_encoding = b"\xe4"
+
+ if format == "csv":
+ content = b"," + bad_encoding + b"\n" + bad_encoding * 2 + b"," + bad_encoding
+ reader = partial(pd.read_csv, index_col=0)
+ else:
+ content = (
+ b'{"'
+ + bad_encoding * 2
+ + b'": {"'
+ + bad_encoding
+ + b'":"'
+ + bad_encoding
+ + b'"}}'
+ )
+ reader = partial(pd.read_json, orient="index")
+ file = temp_file
+ file.write_bytes(content)
+
+ if encoding_errors != "replace":
+ with pytest.raises(UnicodeDecodeError, match=msg):
+ reader(temp_file, encoding_errors=encoding_errors)
+ else:
+ df = reader(temp_file, encoding_errors=encoding_errors)
+ decoded = bad_encoding.decode(errors=encoding_errors)
+ expected = pd.DataFrame({decoded: [decoded]}, index=[decoded * 2])
+ tm.assert_frame_equal(df, expected)
+
+
+@pytest.mark.parametrize("encoding_errors", [0, None])
+def test_encoding_errors_badtype(encoding_errors):
+ # GH 59075
+ content = StringIO("A,B\n1,2\n3,4\n")
+ reader = partial(pd.read_csv, encoding_errors=encoding_errors)
+ expected_error = "encoding_errors must be a string, got "
+ expected_error += f"{type(encoding_errors).__name__}"
+ with pytest.raises(ValueError, match=expected_error):
+ reader(content)
+
+
+def test_bad_encdoing_errors(temp_file):
+ # GH 39777
+ with pytest.raises(LookupError, match="unknown error handler name"):
+ icom.get_handle(temp_file, "w", errors="bad")
+
+
+@pytest.mark.skipif(WASM, reason="limited file system access on WASM")
+def test_errno_attribute():
+ # GH 13872
+ with pytest.raises(FileNotFoundError, match="\\[Errno 2\\]") as err:
+ pd.read_csv("doesnt_exist")
+ assert err.errno == errno.ENOENT
+
+
+def test_fail_mmap():
+ with pytest.raises(UnsupportedOperation, match="fileno"):
+ with BytesIO() as buffer:
+ icom.get_handle(buffer, "rb", memory_map=True)
+
+
+def test_close_on_error():
+ # GH 47136
+ class TestError:
+ def close(self):
+ raise OSError("test")
+
+ with pytest.raises(OSError, match="test"):
+ with BytesIO() as buffer:
+ with icom.get_handle(buffer, "rb") as handles:
+ handles.created_handles.append(TestError())
+
+
+@td.skip_if_no("fsspec")
+@pytest.mark.parametrize("compression", [None, "infer"])
+def test_read_csv_chained_url_no_error(datapath, compression):
+ # GH 60100
+ tar_file_path = datapath("io", "data", "tar", "test-csv.tar")
+ chained_file_url = f"tar://test.csv::file://{tar_file_path}"
+
+ result = pd.read_csv(chained_file_url, compression=compression, sep=";")
+ expected = pd.DataFrame({"1": {0: 3}, "2": {0: 4}})
+
+ tm.assert_frame_equal(expected, result)
+
+
+@pytest.mark.parametrize(
+ "reader",
+ [
+ pd.read_csv,
+ pd.read_fwf,
+ pd.read_excel,
+ pd.read_feather,
+ pd.read_hdf,
+ pd.read_stata,
+ pd.read_sas,
+ pd.read_json,
+ pd.read_pickle,
+ ],
+)
+def test_pickle_reader(reader):
+ # GH 22265
+ with BytesIO() as buffer:
+ pickle.dump(reader, buffer)
+
+
+@td.skip_if_no("pyarrow")
+def test_pyarrow_read_csv_datetime_dtype():
+ # GH 59904
+ data = '"date"\n"20/12/2025"\n""\n"31/12/2020"'
+ result = pd.read_csv(
+ StringIO(data), parse_dates=["date"], dayfirst=True, dtype_backend="pyarrow"
+ )
+
+ expect_data = pd.to_datetime(["20/12/2025", pd.NaT, "31/12/2020"], dayfirst=True)
+ expect = pd.DataFrame({"date": expect_data})
+
+ tm.assert_frame_equal(expect, result)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_compression.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_compression.py
new file mode 100644
index 0000000000000000000000000000000000000000..97b64a29a7f2cff7bfd5c1311fab9b40dcdf1a0a
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_compression.py
@@ -0,0 +1,382 @@
+import gzip
+import io
+import os
+import subprocess
+import sys
+import tarfile
+import textwrap
+import zipfile
+
+import numpy as np
+import pytest
+
+from pandas.compat import is_platform_windows
+
+import pandas as pd
+import pandas._testing as tm
+
+import pandas.io.common as icom
+
+
+@pytest.mark.parametrize(
+ "obj",
+ [
+ pd.DataFrame(
+ 100 * [[0.123456, 0.234567, 0.567567], [12.32112, 123123.2, 321321.2]],
+ columns=["X", "Y", "Z"],
+ ),
+ pd.Series(100 * [0.123456, 0.234567, 0.567567], name="X"),
+ ],
+)
+@pytest.mark.parametrize("method", ["to_pickle", "to_json", "to_csv"])
+def test_compression_size(obj, method, compression_only, temp_file):
+ if compression_only == "tar":
+ compression_only = {"method": "tar", "mode": "w:gz"}
+
+ path = temp_file
+ getattr(obj, method)(path, compression=compression_only)
+ compressed_size = os.path.getsize(path)
+ getattr(obj, method)(path, compression=None)
+ uncompressed_size = os.path.getsize(path)
+ assert uncompressed_size > compressed_size
+
+
+@pytest.mark.parametrize(
+ "obj",
+ [
+ pd.DataFrame(
+ 100 * [[0.123456, 0.234567, 0.567567], [12.32112, 123123.2, 321321.2]],
+ columns=["X", "Y", "Z"],
+ ),
+ pd.Series(100 * [0.123456, 0.234567, 0.567567], name="X"),
+ ],
+)
+@pytest.mark.parametrize("method", ["to_csv", "to_json"])
+def test_compression_size_fh(obj, method, compression_only, temp_file):
+ path = temp_file
+ with icom.get_handle(
+ path,
+ "w:gz" if compression_only == "tar" else "w",
+ compression=compression_only,
+ ) as handles:
+ getattr(obj, method)(handles.handle)
+ assert not handles.handle.closed
+ compressed_size = os.path.getsize(path)
+
+ # Create a new temporary file for uncompressed comparison
+ path2 = temp_file.parent / f"{temp_file.stem}_uncompressed{temp_file.suffix}"
+ path2.touch()
+ with icom.get_handle(path2, "w", compression=None) as handles:
+ getattr(obj, method)(handles.handle)
+ assert not handles.handle.closed
+ uncompressed_size = os.path.getsize(path2)
+ assert uncompressed_size > compressed_size
+
+
+@pytest.mark.parametrize(
+ "write_method, write_kwargs, read_method",
+ [
+ ("to_csv", {"index": False}, pd.read_csv),
+ ("to_json", {}, pd.read_json),
+ ("to_pickle", {}, pd.read_pickle),
+ ],
+)
+def test_dataframe_compression_defaults_to_infer(
+ write_method,
+ write_kwargs,
+ read_method,
+ compression_only,
+ compression_to_extension,
+ temp_file,
+):
+ # GH22004
+ input = pd.DataFrame([[1.0, 0, -4], [3.4, 5, 2]], columns=["X", "Y", "Z"])
+ extension = compression_to_extension[compression_only]
+ path = temp_file.parent / f"compressed{extension}"
+ getattr(input, write_method)(path, **write_kwargs)
+ output = read_method(path, compression=compression_only)
+ tm.assert_frame_equal(output, input)
+
+
+@pytest.mark.parametrize(
+ "write_method,write_kwargs,read_method,read_kwargs",
+ [
+ ("to_csv", {"index": False, "header": True}, pd.read_csv, {"squeeze": True}),
+ ("to_json", {}, pd.read_json, {"typ": "series"}),
+ ("to_pickle", {}, pd.read_pickle, {}),
+ ],
+)
+def test_series_compression_defaults_to_infer(
+ write_method,
+ write_kwargs,
+ read_method,
+ read_kwargs,
+ compression_only,
+ compression_to_extension,
+ temp_file,
+):
+ # GH22004
+ input = pd.Series([0, 5, -2, 10], name="X")
+ extension = compression_to_extension[compression_only]
+ path = temp_file.parent / f"compressed{extension}"
+ getattr(input, write_method)(path, **write_kwargs)
+ if "squeeze" in read_kwargs:
+ kwargs = read_kwargs.copy()
+ del kwargs["squeeze"]
+ output = read_method(path, compression=compression_only, **kwargs).squeeze(
+ "columns"
+ )
+ else:
+ output = read_method(path, compression=compression_only, **read_kwargs)
+ tm.assert_series_equal(output, input, check_names=False)
+
+
+def test_compression_warning(compression_only, temp_file):
+ # Assert that passing a file object to to_csv while explicitly specifying a
+ # compression protocol triggers a RuntimeWarning, as per GH21227.
+ df = pd.DataFrame(
+ 100 * [[0.123456, 0.234567, 0.567567], [12.32112, 123123.2, 321321.2]],
+ columns=["X", "Y", "Z"],
+ )
+ path = temp_file
+ with icom.get_handle(path, "w", compression=compression_only) as handles:
+ with tm.assert_produces_warning(RuntimeWarning, match="has no effect"):
+ df.to_csv(handles.handle, compression=compression_only)
+
+
+def test_compression_binary(compression_only, temp_file):
+ """
+ Binary file handles support compression.
+
+ GH22555
+ """
+ df = pd.DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ )
+
+ # with a file
+ path = temp_file
+ with open(path, mode="wb") as file:
+ df.to_csv(file, mode="wb", compression=compression_only)
+ file.seek(0) # file shouldn't be closed
+ tm.assert_frame_equal(
+ df, pd.read_csv(path, index_col=0, compression=compression_only)
+ )
+
+ # with BytesIO
+ file = io.BytesIO()
+ df.to_csv(file, mode="wb", compression=compression_only)
+ file.seek(0) # file shouldn't be closed
+ tm.assert_frame_equal(
+ df, pd.read_csv(file, index_col=0, compression=compression_only)
+ )
+
+
+def test_gzip_reproducibility_file_name(temp_file):
+ """
+ Gzip should create reproducible archives with mtime.
+
+ Note: Archives created with different filenames will still be different!
+
+ GH 28103
+ """
+ df = pd.DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ )
+ compression_options = {"method": "gzip", "mtime": 1}
+
+ # test for filename
+ path = temp_file
+ df.to_csv(path, compression=compression_options)
+ output = path.read_bytes()
+ df.to_csv(path, compression=compression_options)
+ assert output == path.read_bytes()
+
+
+def test_gzip_reproducibility_file_object():
+ """
+ Gzip should create reproducible archives with mtime.
+
+ GH 28103
+ """
+ df = pd.DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ )
+ compression_options = {"method": "gzip", "mtime": 1}
+
+ # test for file object
+ buffer = io.BytesIO()
+ df.to_csv(buffer, compression=compression_options, mode="wb")
+ output = buffer.getvalue()
+ buffer = io.BytesIO()
+ df.to_csv(buffer, compression=compression_options, mode="wb")
+ assert output == buffer.getvalue()
+
+
+@pytest.mark.single_cpu
+def test_with_missing_lzma():
+ """Tests if import pandas works when lzma is not present."""
+ # https://github.com/pandas-dev/pandas/issues/27575
+ code = textwrap.dedent(
+ """\
+ import sys
+ sys.modules['lzma'] = None
+ import pandas
+ """
+ )
+ subprocess.check_output([sys.executable, "-c", code], stderr=subprocess.PIPE)
+
+
+@pytest.mark.single_cpu
+def test_with_missing_lzma_runtime():
+ """Tests if ModuleNotFoundError is hit when calling lzma without
+ having the module available.
+ """
+ code = textwrap.dedent(
+ """
+ import sys
+ import pytest
+ sys.modules['lzma'] = None
+ import pandas as pd
+ df = pd.DataFrame()
+ with pytest.raises(ModuleNotFoundError, match='import of lzma'):
+ df.to_csv('foo.csv', compression='xz')
+ """
+ )
+ subprocess.check_output([sys.executable, "-c", code], stderr=subprocess.PIPE)
+
+
+@pytest.mark.parametrize(
+ "obj",
+ [
+ pd.DataFrame(
+ 100 * [[0.123456, 0.234567, 0.567567], [12.32112, 123123.2, 321321.2]],
+ columns=["X", "Y", "Z"],
+ ),
+ pd.Series(100 * [0.123456, 0.234567, 0.567567], name="X"),
+ ],
+)
+@pytest.mark.parametrize("method", ["to_pickle", "to_json", "to_csv"])
+def test_gzip_compression_level(obj, method, temp_file):
+ # GH33196
+ path = temp_file
+ getattr(obj, method)(path, compression="gzip")
+ compressed_size_default = os.path.getsize(path)
+ getattr(obj, method)(path, compression={"method": "gzip", "compresslevel": 1})
+ compressed_size_fast = os.path.getsize(path)
+ assert compressed_size_default < compressed_size_fast
+
+
+@pytest.mark.parametrize(
+ "obj",
+ [
+ pd.DataFrame(
+ 100 * [[0.123456, 0.234567, 0.567567], [12.32112, 123123.2, 321321.2]],
+ columns=["X", "Y", "Z"],
+ ),
+ pd.Series(100 * [0.123456, 0.234567, 0.567567], name="X"),
+ ],
+)
+@pytest.mark.parametrize("method", ["to_pickle", "to_json", "to_csv"])
+def test_xz_compression_level_read(obj, method, temp_file):
+ path = temp_file
+ getattr(obj, method)(path, compression="xz")
+ compressed_size_default = os.path.getsize(path)
+ getattr(obj, method)(path, compression={"method": "xz", "preset": 1})
+ compressed_size_fast = os.path.getsize(path)
+ assert compressed_size_default < compressed_size_fast
+ if method == "to_csv":
+ pd.read_csv(path, compression="xz")
+
+
+@pytest.mark.parametrize(
+ "obj",
+ [
+ pd.DataFrame(
+ 100 * [[0.123456, 0.234567, 0.567567], [12.32112, 123123.2, 321321.2]],
+ columns=["X", "Y", "Z"],
+ ),
+ pd.Series(100 * [0.123456, 0.234567, 0.567567], name="X"),
+ ],
+)
+@pytest.mark.parametrize("method", ["to_pickle", "to_json", "to_csv"])
+def test_bzip_compression_level(obj, method, temp_file):
+ """GH33196 bzip needs file size > 100k to show a size difference between
+ compression levels, so here we just check if the call works when
+ compression is passed as a dict.
+ """
+ path = temp_file
+ getattr(obj, method)(path, compression={"method": "bz2", "compresslevel": 1})
+
+
+@pytest.mark.parametrize(
+ "suffix,archive",
+ [
+ (".zip", zipfile.ZipFile),
+ (".tar", tarfile.TarFile),
+ ],
+)
+def test_empty_archive_zip(suffix, archive, temp_file):
+ path = temp_file.parent / f"archive{suffix}"
+ with archive(path, "w"):
+ pass
+ with pytest.raises(ValueError, match="Zero files found"):
+ pd.read_csv(path)
+
+
+def test_ambiguous_archive_zip(temp_file):
+ path = temp_file.parent / "archive.zip"
+ with zipfile.ZipFile(path, "w") as file:
+ file.writestr("a.csv", "foo,bar")
+ file.writestr("b.csv", "foo,bar")
+ with pytest.raises(ValueError, match="Multiple files found in ZIP file"):
+ pd.read_csv(path)
+
+
+def test_ambiguous_archive_tar(tmp_path):
+ csvAPath = tmp_path / "a.csv"
+ with open(csvAPath, "w", encoding="utf-8") as a:
+ a.write("foo,bar\n")
+ csvBPath = tmp_path / "b.csv"
+ with open(csvBPath, "w", encoding="utf-8") as b:
+ b.write("foo,bar\n")
+
+ tarpath = tmp_path / "archive.tar"
+ with tarfile.TarFile(tarpath, "w") as tar:
+ tar.add(csvAPath, "a.csv")
+ tar.add(csvBPath, "b.csv")
+
+ with pytest.raises(ValueError, match="Multiple files found in TAR archive"):
+ pd.read_csv(tarpath)
+
+
+def test_tar_gz_to_different_filename(temp_file):
+ file = temp_file.parent / "archive.foo"
+ pd.DataFrame(
+ [["1", "2"]],
+ columns=["foo", "bar"],
+ ).to_csv(file, compression={"method": "tar", "mode": "w:gz"}, index=False)
+ with gzip.open(file) as uncompressed:
+ with tarfile.TarFile(fileobj=uncompressed) as archive:
+ members = archive.getmembers()
+ assert len(members) == 1
+ content = archive.extractfile(members[0]).read().decode("utf8")
+
+ if is_platform_windows():
+ expected = "foo,bar\r\n1,2\r\n"
+ else:
+ expected = "foo,bar\n1,2\n"
+
+ assert content == expected
+
+
+def test_tar_no_error_on_close():
+ with io.BytesIO() as buffer:
+ with icom._BytesTarFile(fileobj=buffer, mode="w"):
+ pass
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_feather.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_feather.py
new file mode 100644
index 0000000000000000000000000000000000000000..6351a9760b773e2ffdd1547061b7f1918ce325e4
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_feather.py
@@ -0,0 +1,291 @@
+"""test feather-format compat"""
+
+from datetime import datetime
+import zoneinfo
+
+import numpy as np
+import pytest
+
+from pandas.compat.pyarrow import (
+ pa_version_under18p0,
+ pa_version_under19p0,
+)
+
+import pandas as pd
+import pandas._testing as tm
+
+from pandas.io.feather_format import read_feather, to_feather # isort:skip
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+pa = pytest.importorskip("pyarrow")
+
+
+@pytest.mark.single_cpu
+class TestFeather:
+ def check_error_on_write(self, df, exc, err_msg, temp_file):
+ # check that we are raising the exception
+ # on writing
+
+ with pytest.raises(exc, match=err_msg):
+ to_feather(df, temp_file)
+
+ def check_external_error_on_write(self, df, temp_file):
+ # check that we are raising the exception
+ # on writing
+
+ with tm.external_error_raised(Exception):
+ to_feather(df, temp_file)
+
+ def check_round_trip(
+ self, df, temp_file, expected=None, write_kwargs=None, **read_kwargs
+ ):
+ if write_kwargs is None:
+ write_kwargs = {}
+ if expected is None:
+ expected = df.copy()
+
+ to_feather(df, temp_file, **write_kwargs)
+
+ result = read_feather(temp_file, **read_kwargs)
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_error(self, temp_file):
+ msg = "feather only support IO with DataFrames"
+ for obj in [
+ pd.Series([1, 2, 3]),
+ 1,
+ "foo",
+ pd.Timestamp("20130101"),
+ np.array([1, 2, 3]),
+ ]:
+ self.check_error_on_write(obj, ValueError, msg, temp_file)
+
+ def test_basic(self, temp_file):
+ tz = zoneinfo.ZoneInfo("US/Eastern")
+ df = pd.DataFrame(
+ {
+ "string": list("abc"),
+ "int": list(range(1, 4)),
+ "uint": np.arange(3, 6).astype("u1"),
+ "float": np.arange(4.0, 7.0, dtype="float64"),
+ "float_with_null": [1.0, np.nan, 3],
+ "bool": [True, False, True],
+ "bool_with_null": [True, np.nan, False],
+ "cat": pd.Categorical(list("abc")),
+ "dt": pd.DatetimeIndex(
+ list(pd.date_range("20130101", periods=3)), freq=None
+ ),
+ "dttz": pd.DatetimeIndex(
+ list(pd.date_range("20130101", periods=3, tz=tz)),
+ freq=None,
+ ),
+ "dt_with_null": [
+ pd.Timestamp("20130101"),
+ pd.NaT,
+ pd.Timestamp("20130103"),
+ ],
+ "dtns": pd.DatetimeIndex(
+ list(pd.date_range("20130101", periods=3, freq="ns")), freq=None
+ ),
+ }
+ )
+ df["periods"] = pd.period_range("2013", freq="M", periods=3)
+ df["timedeltas"] = pd.timedelta_range("1 day", periods=3)
+ df["intervals"] = pd.interval_range(0, 3, 3)
+
+ assert df.dttz.dtype.tz.key == "US/Eastern"
+
+ expected = df.copy()
+ expected.loc[1, "bool_with_null"] = None
+ self.check_round_trip(df, temp_file, expected=expected)
+
+ def test_duplicate_columns(self, temp_file):
+ # https://github.com/wesm/feather/issues/53
+ # not currently able to handle duplicate columns
+ df = pd.DataFrame(np.arange(12).reshape(4, 3), columns=list("aaa")).copy()
+ self.check_external_error_on_write(df, temp_file)
+
+ def test_read_columns(self, temp_file):
+ # GH 24025
+ df = pd.DataFrame(
+ {
+ "col1": list("abc"),
+ "col2": list(range(1, 4)),
+ "col3": list("xyz"),
+ "col4": list(range(4, 7)),
+ }
+ )
+ columns = ["col1", "col3"]
+ self.check_round_trip(df, temp_file, expected=df[columns], columns=columns)
+
+ def test_read_columns_different_order(self, temp_file):
+ # GH 33878
+ df = pd.DataFrame({"A": [1, 2], "B": ["x", "y"], "C": [True, False]})
+ expected = df[["B", "A"]]
+ self.check_round_trip(df, temp_file, expected, columns=["B", "A"])
+
+ def test_unsupported_other(self, temp_file):
+ # mixed python objects
+ df = pd.DataFrame({"a": ["a", 1, 2.0]})
+ self.check_external_error_on_write(df, temp_file)
+
+ def test_rw_use_threads(self, temp_file):
+ df = pd.DataFrame({"A": np.arange(100000)})
+ self.check_round_trip(df, temp_file, use_threads=True)
+ self.check_round_trip(df, temp_file, use_threads=False)
+
+ def test_path_pathlib(self, temp_file):
+ df = pd.DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ ).reset_index()
+ result = tm.round_trip_pathlib(df.to_feather, read_feather, temp_file)
+ tm.assert_frame_equal(df, result)
+
+ def test_passthrough_keywords(self, temp_file):
+ df = pd.DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ ).reset_index()
+ self.check_round_trip(df, temp_file, write_kwargs={"version": 1})
+
+ @pytest.mark.network
+ @pytest.mark.single_cpu
+ def test_http_path(self, feather_file, httpserver):
+ # GH 29055
+ expected = read_feather(feather_file)
+ with open(feather_file, "rb") as f:
+ httpserver.serve_content(content=f.read())
+ res = read_feather(httpserver.url)
+ tm.assert_frame_equal(expected, res)
+
+ def test_read_feather_dtype_backend(
+ self, string_storage, dtype_backend, using_infer_string, temp_file
+ ):
+ # GH#50765
+ df = pd.DataFrame(
+ {
+ "a": pd.Series([1, pd.NA, 3], dtype="Int64"),
+ "b": pd.Series([1, 2, 3], dtype="Int64"),
+ "c": pd.Series([1.5, pd.NA, 2.5], dtype="Float64"),
+ "d": pd.Series([1.5, 2.0, 2.5], dtype="Float64"),
+ "e": [True, False, None],
+ "f": [True, False, True],
+ "g": ["a", "b", "c"],
+ "h": ["a", "b", None],
+ }
+ )
+
+ to_feather(df, temp_file)
+ with pd.option_context("mode.string_storage", string_storage):
+ result = read_feather(temp_file, dtype_backend=dtype_backend)
+
+ if dtype_backend == "pyarrow":
+ pa = pytest.importorskip("pyarrow")
+ if using_infer_string:
+ string_dtype = pd.ArrowDtype(pa.large_string())
+ else:
+ string_dtype = pd.ArrowDtype(pa.string())
+ else:
+ string_dtype = pd.StringDtype(string_storage)
+
+ expected = pd.DataFrame(
+ {
+ "a": pd.Series([1, pd.NA, 3], dtype="Int64"),
+ "b": pd.Series([1, 2, 3], dtype="Int64"),
+ "c": pd.Series([1.5, pd.NA, 2.5], dtype="Float64"),
+ "d": pd.Series([1.5, 2.0, 2.5], dtype="Float64"),
+ "e": pd.Series([True, False, pd.NA], dtype="boolean"),
+ "f": pd.Series([True, False, True], dtype="boolean"),
+ "g": pd.Series(["a", "b", "c"], dtype=string_dtype),
+ "h": pd.Series(["a", "b", None], dtype=string_dtype),
+ }
+ )
+
+ if dtype_backend == "pyarrow":
+ from pandas.arrays import ArrowExtensionArray
+
+ expected = pd.DataFrame(
+ {
+ col: ArrowExtensionArray(pa.array(expected[col], from_pandas=True))
+ for col in expected.columns
+ }
+ )
+
+ if using_infer_string:
+ expected.columns = expected.columns.astype(
+ pd.StringDtype(string_storage, na_value=np.nan)
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_int_columns_and_index(self, temp_file):
+ df = pd.DataFrame({"a": [1, 2, 3]}, index=pd.Index([3, 4, 5], name="test"))
+ self.check_round_trip(df, temp_file)
+
+ def test_invalid_dtype_backend(self, temp_file):
+ msg = (
+ "dtype_backend numpy is invalid, only 'numpy_nullable' and "
+ "'pyarrow' are allowed."
+ )
+ df = pd.DataFrame({"int": list(range(1, 4))})
+ df.to_feather(temp_file)
+ with pytest.raises(ValueError, match=msg):
+ read_feather(temp_file, dtype_backend="numpy")
+
+ def test_string_inference(self, temp_file, using_infer_string):
+ # GH#54431
+ df = pd.DataFrame(data={"a": ["x", "y"]})
+ df.to_feather(temp_file)
+ with pd.option_context("future.infer_string", True):
+ result = read_feather(temp_file)
+ dtype = pd.StringDtype(na_value=np.nan)
+ expected = pd.DataFrame(
+ data={"a": ["x", "y"]}, dtype=pd.StringDtype(na_value=np.nan)
+ )
+ expected = pd.DataFrame(
+ data={"a": ["x", "y"]},
+ dtype=dtype,
+ columns=pd.Index(
+ ["a"],
+ dtype=object
+ if pa_version_under19p0 and not using_infer_string
+ else dtype,
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.skipif(pa_version_under18p0, reason="not supported before 18.0")
+ def test_string_inference_string_view_type(self, temp_file):
+ # GH#54798
+ import pyarrow as pa
+ from pyarrow import feather
+
+ table = pa.table({"a": pa.array([None, "b", "c"], pa.string_view())})
+ feather.write_feather(table, temp_file)
+
+ with pd.option_context("future.infer_string", True):
+ result = read_feather(temp_file)
+
+ expected = pd.DataFrame(
+ data={"a": [None, "b", "c"]}, dtype=pd.StringDtype(na_value=np.nan)
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_out_of_bounds_datetime_to_feather(self, temp_file):
+ # GH#47832
+ df = pd.DataFrame(
+ {
+ "date": [
+ datetime.fromisoformat("1654-01-01"),
+ datetime.fromisoformat("1920-01-01"),
+ ],
+ }
+ )
+ self.check_round_trip(df, temp_file)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_fsspec.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_fsspec.py
new file mode 100644
index 0000000000000000000000000000000000000000..be374d042fd487d00e081f332779650ecacb366f
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_fsspec.py
@@ -0,0 +1,355 @@
+import io
+
+import numpy as np
+import pytest
+
+from pandas._config import using_string_dtype
+
+from pandas.compat import HAS_PYARROW
+from pandas.compat.pyarrow import pa_version_under14p0
+
+from pandas import (
+ DataFrame,
+ date_range,
+ read_csv,
+ read_excel,
+ read_feather,
+ read_json,
+ read_parquet,
+ read_pickle,
+ read_stata,
+ read_table,
+)
+import pandas._testing as tm
+from pandas.util import _test_decorators as td
+from pandas.util.version import Version
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+@pytest.fixture
+def fsspectest():
+ pytest.importorskip("fsspec")
+ from fsspec import register_implementation
+ from fsspec.implementations.memory import MemoryFileSystem
+ from fsspec.registry import _registry as registry
+
+ class TestMemoryFS(MemoryFileSystem):
+ protocol = "testmem"
+ test = [None]
+
+ def __init__(self, **kwargs) -> None:
+ self.test[0] = kwargs.pop("test", None)
+ super().__init__(**kwargs)
+
+ register_implementation("testmem", TestMemoryFS, clobber=True)
+ yield TestMemoryFS()
+ registry.pop("testmem", None)
+ TestMemoryFS.test[0] = None
+ TestMemoryFS.store.clear()
+
+
+@pytest.fixture
+def df1():
+ return DataFrame(
+ {
+ "int": [1, 3],
+ "float": [2.0, np.nan],
+ "str": ["t", "s"],
+ "dt": date_range("2018-06-18", periods=2),
+ }
+ )
+
+
+@pytest.fixture
+def cleared_fs():
+ fsspec = pytest.importorskip("fsspec")
+
+ memfs = fsspec.filesystem("memory")
+ yield memfs
+ memfs.store.clear()
+
+
+def test_read_csv(cleared_fs, df1):
+ text = str(df1.to_csv(index=False)).encode()
+ with cleared_fs.open("test/test.csv", "wb") as w:
+ w.write(text)
+ df2 = read_csv("memory://test/test.csv", parse_dates=["dt"])
+
+ expected = df1.copy()
+ expected["dt"] = expected["dt"].astype("M8[us]")
+ tm.assert_frame_equal(df2, expected)
+
+
+def test_reasonable_error(monkeypatch, cleared_fs):
+ from fsspec.registry import known_implementations
+
+ with pytest.raises(ValueError, match="nosuchprotocol"):
+ read_csv("nosuchprotocol://test/test.csv")
+ err_msg = "test error message"
+ monkeypatch.setitem(
+ known_implementations,
+ "couldexist",
+ {"class": "unimportable.CouldExist", "err": err_msg},
+ )
+ with pytest.raises(ImportError, match=err_msg):
+ read_csv("couldexist://test/test.csv")
+
+
+def test_to_csv(cleared_fs, df1):
+ df1.to_csv("memory://test/test.csv", index=True)
+
+ df2 = read_csv("memory://test/test.csv", parse_dates=["dt"], index_col=0)
+
+ expected = df1.copy()
+ expected["dt"] = expected["dt"].astype("M8[us]")
+ tm.assert_frame_equal(df2, expected)
+
+
+def test_to_excel(cleared_fs, df1):
+ pytest.importorskip("openpyxl")
+ ext = "xlsx"
+ path = f"memory://test/test.{ext}"
+ df1.to_excel(path, index=True)
+
+ df2 = read_excel(path, parse_dates=["dt"], index_col=0)
+
+ expected = df1.copy()
+ expected["dt"] = expected["dt"].astype("M8[us]")
+ tm.assert_frame_equal(df2, expected)
+
+
+@pytest.mark.parametrize("binary_mode", [False, True])
+def test_to_csv_fsspec_object(cleared_fs, binary_mode, df1):
+ fsspec = pytest.importorskip("fsspec")
+
+ path = "memory://test/test.csv"
+ mode = "wb" if binary_mode else "w"
+ with fsspec.open(path, mode=mode).open() as fsspec_object:
+ df1.to_csv(fsspec_object, index=True)
+ assert not fsspec_object.closed
+
+ mode = mode.replace("w", "r")
+ with fsspec.open(path, mode=mode) as fsspec_object:
+ df2 = read_csv(
+ fsspec_object,
+ parse_dates=["dt"],
+ index_col=0,
+ )
+ assert not fsspec_object.closed
+
+ expected = df1.copy()
+ expected["dt"] = expected["dt"].astype("M8[us]")
+ tm.assert_frame_equal(df2, expected)
+
+
+def test_csv_options(fsspectest):
+ df = DataFrame({"a": [0]})
+ df.to_csv(
+ "testmem://test/test.csv", storage_options={"test": "csv_write"}, index=False
+ )
+ assert fsspectest.test[0] == "csv_write"
+ read_csv("testmem://test/test.csv", storage_options={"test": "csv_read"})
+ assert fsspectest.test[0] == "csv_read"
+
+
+def test_read_table_options(fsspectest):
+ # GH #39167
+ df = DataFrame({"a": [0]})
+ df.to_csv(
+ "testmem://test/test.csv", storage_options={"test": "csv_write"}, index=False
+ )
+ assert fsspectest.test[0] == "csv_write"
+ read_table("testmem://test/test.csv", storage_options={"test": "csv_read"})
+ assert fsspectest.test[0] == "csv_read"
+
+
+def test_excel_options(fsspectest):
+ pytest.importorskip("openpyxl")
+ extension = "xlsx"
+
+ df = DataFrame({"a": [0]})
+
+ path = f"testmem://test/test.{extension}"
+
+ df.to_excel(path, storage_options={"test": "write"}, index=False)
+ assert fsspectest.test[0] == "write"
+ read_excel(path, storage_options={"test": "read"})
+ assert fsspectest.test[0] == "read"
+
+
+def test_to_parquet_new_file(cleared_fs, df1, request):
+ """Regression test for writing to a not-yet-existent GCS Parquet file."""
+ fp = pytest.importorskip("fastparquet")
+
+ request.applymarker(
+ pytest.mark.xfail(
+ using_string_dtype()
+ and HAS_PYARROW
+ and not pa_version_under14p0
+ and Version(fp.__version__) <= Version("2025.12.0"),
+ reason="TODO(infer_string) fastparquet",
+ )
+ )
+
+ df1.to_parquet(
+ "memory://test/test.csv", index=True, engine="fastparquet", compression=None
+ )
+
+
+def test_arrowparquet_options(fsspectest):
+ """Regression test for writing to a not-yet-existent GCS Parquet file."""
+ pytest.importorskip("pyarrow")
+ df = DataFrame({"a": [0]})
+ df.to_parquet(
+ "testmem://test/test.csv",
+ engine="pyarrow",
+ compression=None,
+ storage_options={"test": "parquet_write"},
+ )
+ assert fsspectest.test[0] == "parquet_write"
+ read_parquet(
+ "testmem://test/test.csv",
+ engine="pyarrow",
+ storage_options={"test": "parquet_read"},
+ )
+ assert fsspectest.test[0] == "parquet_read"
+
+
+def test_fastparquet_options(fsspectest):
+ """Regression test for writing to a not-yet-existent GCS Parquet file."""
+ pytest.importorskip("fastparquet")
+
+ df = DataFrame({"a": [0]})
+ df.to_parquet(
+ "testmem://test/test.csv",
+ engine="fastparquet",
+ compression=None,
+ storage_options={"test": "parquet_write"},
+ )
+ assert fsspectest.test[0] == "parquet_write"
+ read_parquet(
+ "testmem://test/test.csv",
+ engine="fastparquet",
+ storage_options={"test": "parquet_read"},
+ )
+ assert fsspectest.test[0] == "parquet_read"
+
+
+@pytest.mark.single_cpu
+@pytest.mark.parametrize("compression_suffix", ["", ".gz", ".bz2"])
+def test_from_s3_csv(s3_bucket_public_with_data, s3so, tips_file, compression_suffix):
+ pytest.importorskip("s3fs")
+ df_from_s3 = read_csv(
+ f"s3://{s3_bucket_public_with_data.name}/tips.csv{compression_suffix}",
+ storage_options=s3so,
+ )
+ df_from_local = read_csv(tips_file)
+ tm.assert_equal(df_from_s3, df_from_local)
+
+
+@pytest.mark.single_cpu
+@pytest.mark.parametrize("protocol", ["s3", "s3a", "s3n"])
+def test_s3_protocols(s3_bucket_public_with_data, s3so, tips_file, protocol):
+ pytest.importorskip("s3fs")
+ df_from_s3 = read_csv(
+ f"{protocol}://{s3_bucket_public_with_data.name}/tips.csv",
+ storage_options=s3so,
+ )
+ df_from_local = read_csv(tips_file)
+ tm.assert_equal(df_from_s3, df_from_local)
+
+
+@pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string) fastparquet")
+@pytest.mark.single_cpu
+def test_s3_parquet(s3_bucket_public, s3so, df1):
+ pytest.importorskip("fastparquet")
+ pytest.importorskip("s3fs")
+
+ fn = f"s3://{s3_bucket_public.name}/test.parquet"
+ df1.to_parquet(
+ fn, index=False, engine="fastparquet", compression=None, storage_options=s3so
+ )
+ df2 = read_parquet(fn, engine="fastparquet", storage_options=s3so)
+ tm.assert_equal(df1, df2)
+
+
+@td.skip_if_installed("fsspec")
+def test_not_present_exception():
+ msg = "`Import fsspec` failed. Use pip or conda to install the fsspec package."
+ with pytest.raises(ImportError, match=msg):
+ read_csv("memory://test/test.csv")
+
+
+def test_feather_options(fsspectest):
+ pytest.importorskip("pyarrow")
+ df = DataFrame({"a": [0]})
+ df.to_feather("testmem://mockfile", storage_options={"test": "feather_write"})
+ assert fsspectest.test[0] == "feather_write"
+ out = read_feather("testmem://mockfile", storage_options={"test": "feather_read"})
+ assert fsspectest.test[0] == "feather_read"
+ tm.assert_frame_equal(df, out)
+
+
+def test_pickle_options(fsspectest):
+ df = DataFrame({"a": [0]})
+ df.to_pickle("testmem://mockfile", storage_options={"test": "pickle_write"})
+ assert fsspectest.test[0] == "pickle_write"
+ out = read_pickle("testmem://mockfile", storage_options={"test": "pickle_read"})
+ assert fsspectest.test[0] == "pickle_read"
+ tm.assert_frame_equal(df, out)
+
+
+def test_json_options(fsspectest, compression):
+ df = DataFrame({"a": [0]})
+ df.to_json(
+ "testmem://mockfile",
+ compression=compression,
+ storage_options={"test": "json_write"},
+ )
+ assert fsspectest.test[0] == "json_write"
+ out = read_json(
+ "testmem://mockfile",
+ compression=compression,
+ storage_options={"test": "json_read"},
+ )
+ assert fsspectest.test[0] == "json_read"
+ tm.assert_frame_equal(df, out)
+
+
+def test_stata_options(fsspectest):
+ df = DataFrame({"a": [0]})
+ df.to_stata(
+ "testmem://mockfile", storage_options={"test": "stata_write"}, write_index=False
+ )
+ assert fsspectest.test[0] == "stata_write"
+ out = read_stata("testmem://mockfile", storage_options={"test": "stata_read"})
+ assert fsspectest.test[0] == "stata_read"
+ tm.assert_frame_equal(df, out.astype("int64"))
+
+
+def test_markdown_options(fsspectest):
+ pytest.importorskip("tabulate")
+ df = DataFrame({"a": [0]})
+ df.to_markdown("testmem://mockfile", storage_options={"test": "md_write"})
+ assert fsspectest.test[0] == "md_write"
+ assert fsspectest.cat("testmem://mockfile")
+
+
+def test_non_fsspec_options():
+ pytest.importorskip("pyarrow")
+ with pytest.raises(ValueError, match="storage_options"):
+ read_csv("localfile", storage_options={"a": True})
+ with pytest.raises(ValueError, match="storage_options"):
+ # separate test for parquet, which has a different code path
+ read_parquet("localfile", storage_options={"a": True})
+ by = io.BytesIO()
+
+ with pytest.raises(ValueError, match="storage_options"):
+ read_csv(by, storage_options={"a": True})
+
+ df = DataFrame({"a": [0]})
+ with pytest.raises(ValueError, match="storage_options"):
+ df.to_parquet("nonfsspecpath", storage_options={"a": True})
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_gcs.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_gcs.py
new file mode 100644
index 0000000000000000000000000000000000000000..022fd89c1f555551a47d6361f0368dcbb3d28ab8
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_gcs.py
@@ -0,0 +1,233 @@
+from io import BytesIO
+import os
+import pathlib
+import tarfile
+import zipfile
+
+import numpy as np
+import pytest
+
+from pandas.compat.pyarrow import pa_version_under17p0
+
+from pandas import (
+ DataFrame,
+ Index,
+ date_range,
+ read_csv,
+ read_excel,
+ read_json,
+ read_parquet,
+)
+import pandas._testing as tm
+from pandas.util import _test_decorators as td
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+@pytest.fixture
+def gcs_buffer():
+ """Emulate GCS using a binary buffer."""
+ pytest.importorskip("gcsfs")
+ fsspec = pytest.importorskip("fsspec")
+
+ gcs_buffer = BytesIO()
+ gcs_buffer.close = lambda: True
+
+ class MockGCSFileSystem(fsspec.AbstractFileSystem):
+ @staticmethod
+ def open(*args, **kwargs):
+ gcs_buffer.seek(0)
+ return gcs_buffer
+
+ def ls(self, path, **kwargs):
+ # needed for pyarrow
+ return [{"name": path, "type": "file"}]
+
+ # Overwrites the default implementation from gcsfs to our mock class
+ fsspec.register_implementation("gs", MockGCSFileSystem, clobber=True)
+
+ return gcs_buffer
+
+
+# Patches pyarrow; other processes should not pick up change
+@pytest.mark.single_cpu
+@pytest.mark.parametrize("format", ["csv", "json", "parquet", "excel", "markdown"])
+def test_to_read_gcs(gcs_buffer, format, monkeypatch, capsys, request):
+ """
+ Test that many to/read functions support GCS.
+
+ GH 33987
+ """
+
+ df1 = DataFrame(
+ {
+ "int": [1, 3],
+ "float": [2.0, np.nan],
+ "str": ["t", "s"],
+ "dt": date_range("2018-06-18", periods=2, unit="ns"),
+ }
+ )
+
+ path = f"gs://test/test.{format}"
+
+ if format == "csv":
+ df1.to_csv(path, index=True)
+ df2 = read_csv(path, parse_dates=["dt"], index_col=0)
+ elif format == "excel":
+ path = "gs://test/test.xlsx"
+ df1.to_excel(path)
+ df2 = read_excel(path, parse_dates=["dt"], index_col=0)
+ elif format == "json":
+ df1.to_json(path, date_format="iso")
+ df2 = read_json(path, convert_dates=["dt"])
+ elif format == "parquet":
+ pytest.importorskip("pyarrow")
+ pa_fs = pytest.importorskip("pyarrow.fs")
+
+ class MockFileSystem(pa_fs.FileSystem):
+ @staticmethod
+ def from_uri(path):
+ print("Using pyarrow filesystem")
+ to_local = pathlib.Path(path.replace("gs://", "")).absolute().as_uri()
+ return pa_fs.LocalFileSystem(to_local)
+
+ request.applymarker(
+ pytest.mark.xfail(
+ not pa_version_under17p0,
+ raises=TypeError,
+ reason="pyarrow 17 broke the mocked filesystem",
+ )
+ )
+ with monkeypatch.context() as m:
+ m.setattr(pa_fs, "FileSystem", MockFileSystem)
+ df1.to_parquet(path)
+ df2 = read_parquet(path)
+ captured = capsys.readouterr()
+ assert captured.out == "Using pyarrow filesystem\nUsing pyarrow filesystem\n"
+ elif format == "markdown":
+ pytest.importorskip("tabulate")
+ df1.to_markdown(path)
+ df2 = df1
+
+ expected = df1[:]
+ if format in ["csv", "excel", "json"]:
+ expected["dt"] = expected["dt"].dt.as_unit("us")
+
+ tm.assert_frame_equal(df2, expected)
+
+
+def assert_equal_zip_safe(result: bytes, expected: bytes, compression: str):
+ """
+ For zip compression, only compare the CRC-32 checksum of the file contents
+ to avoid checking the time-dependent last-modified timestamp which
+ in some CI builds is off-by-one
+
+ See https://en.wikipedia.org/wiki/ZIP_(file_format)#File_headers
+ """
+ if compression == "zip":
+ # Only compare the CRC checksum of the file contents
+ with (
+ zipfile.ZipFile(BytesIO(result)) as exp,
+ zipfile.ZipFile(BytesIO(expected)) as res,
+ ):
+ for res_info, exp_info in zip(res.infolist(), exp.infolist()):
+ assert res_info.CRC == exp_info.CRC
+ elif compression == "tar":
+ with (
+ tarfile.open(fileobj=BytesIO(result)) as tar_exp,
+ tarfile.open(fileobj=BytesIO(expected)) as tar_res,
+ ):
+ for tar_res_info, tar_exp_info in zip(
+ tar_res.getmembers(), tar_exp.getmembers()
+ ):
+ actual_file = tar_res.extractfile(tar_res_info)
+ expected_file = tar_exp.extractfile(tar_exp_info)
+ assert (actual_file is None) == (expected_file is None)
+ if actual_file is not None and expected_file is not None:
+ assert actual_file.read() == expected_file.read()
+ else:
+ assert result == expected
+
+
+@pytest.mark.parametrize("encoding", ["utf-8", "cp1251"])
+def test_to_csv_compression_encoding_gcs(
+ gcs_buffer, compression_only, encoding, compression_to_extension
+):
+ """
+ Compression and encoding should with GCS.
+
+ GH 35677 (to_csv, compression), GH 26124 (to_csv, encoding), and
+ GH 32392 (read_csv, encoding)
+ """
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD")),
+ index=Index([f"i-{i}" for i in range(30)]),
+ )
+
+ # reference of compressed and encoded file
+ compression = {"method": compression_only}
+ if compression_only == "gzip":
+ compression["mtime"] = 1 # be reproducible
+ buffer = BytesIO()
+ df.to_csv(buffer, compression=compression, encoding=encoding, mode="wb")
+
+ # write compressed file with explicit compression
+ path_gcs = "gs://test/test.csv"
+ df.to_csv(path_gcs, compression=compression, encoding=encoding)
+ res = gcs_buffer.getvalue()
+ expected = buffer.getvalue()
+ assert_equal_zip_safe(res, expected, compression_only)
+
+ read_df = read_csv(
+ path_gcs, index_col=0, compression=compression_only, encoding=encoding
+ )
+ tm.assert_frame_equal(df, read_df)
+
+ # write compressed file with implicit compression
+ file_ext = compression_to_extension[compression_only]
+ compression["method"] = "infer"
+ path_gcs += f".{file_ext}"
+ df.to_csv(path_gcs, compression=compression, encoding=encoding)
+
+ res = gcs_buffer.getvalue()
+ expected = buffer.getvalue()
+ assert_equal_zip_safe(res, expected, compression_only)
+
+ read_df = read_csv(path_gcs, index_col=0, compression="infer", encoding=encoding)
+ tm.assert_frame_equal(df, read_df)
+
+
+def test_to_parquet_gcs_new_file(monkeypatch, tmpdir):
+ """Regression test for writing to a not-yet-existent GCS Parquet file."""
+ pytest.importorskip("fastparquet")
+ pytest.importorskip("gcsfs")
+
+ from fsspec import AbstractFileSystem
+
+ df1 = DataFrame(
+ {
+ "int": [1, 3],
+ "float": [2.0, np.nan],
+ "dt": date_range("2018-06-18", periods=2),
+ }
+ )
+
+ class MockGCSFileSystem(AbstractFileSystem):
+ def open(self, path, mode="r", *args):
+ if "w" not in mode:
+ raise FileNotFoundError
+ return open(os.path.join(tmpdir, "test.parquet"), mode, encoding="utf-8")
+
+ monkeypatch.setattr("gcsfs.GCSFileSystem", MockGCSFileSystem)
+ df1.to_parquet(
+ "gs://test/test.csv", index=True, engine="fastparquet", compression=None
+ )
+
+
+@td.skip_if_installed("gcsfs")
+def test_gcs_not_present_exception():
+ with tm.external_error_raised(ImportError):
+ read_csv("gs://test/test.csv")
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_html.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_html.py
new file mode 100644
index 0000000000000000000000000000000000000000..abc0cfbb36332daeb3ea606450a1df75d455ed50
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_html.py
@@ -0,0 +1,1669 @@
+from collections.abc import Iterator
+from functools import partial
+from io import (
+ BytesIO,
+ StringIO,
+)
+import os
+from pathlib import Path
+import re
+import threading
+from urllib.error import URLError
+
+import numpy as np
+import pytest
+
+from pandas.compat import is_platform_windows
+import pandas.util._test_decorators as td
+
+import pandas as pd
+from pandas import (
+ NA,
+ DataFrame,
+ MultiIndex,
+ Series,
+ Timestamp,
+ date_range,
+ read_csv,
+ read_html,
+ to_datetime,
+)
+import pandas._testing as tm
+
+from pandas.io.common import file_path_to_url
+
+
+@pytest.fixture(
+ params=[
+ "chinese_utf-16.html",
+ "chinese_utf-32.html",
+ "chinese_utf-8.html",
+ "letz_latin1.html",
+ ]
+)
+def html_encoding_file(request, datapath):
+ """Parametrized fixture for HTML encoding test filenames."""
+ return datapath("io", "data", "html_encoding", request.param)
+
+
+def assert_framelist_equal(list1, list2, *args, **kwargs):
+ assert len(list1) == len(list2), (
+ "lists are not of equal size "
+ f"len(list1) == {len(list1)}, "
+ f"len(list2) == {len(list2)}"
+ )
+ msg = "not all list elements are DataFrames"
+ both_frames = all(
+ map(
+ lambda x, y: isinstance(x, DataFrame) and isinstance(y, DataFrame),
+ list1,
+ list2,
+ )
+ )
+ assert both_frames, msg
+ for frame_i, frame_j in zip(list1, list2):
+ tm.assert_frame_equal(frame_i, frame_j, *args, **kwargs)
+ assert not frame_i.empty, "frames are both empty"
+
+
+def test_bs4_version_fails(monkeypatch, datapath):
+ bs4 = pytest.importorskip("bs4")
+ pytest.importorskip("html5lib")
+
+ monkeypatch.setattr(bs4, "__version__", "4.2")
+ with pytest.raises(ImportError, match="Pandas requires version"):
+ read_html(datapath("io", "data", "html", "spam.html"), flavor="bs4")
+
+
+def test_invalid_flavor():
+ url = "google.com"
+ flavor = "invalid flavor"
+ msg = r"\{" + flavor + r"\} is not a valid set of flavors"
+
+ with pytest.raises(ValueError, match=msg):
+ read_html(StringIO(url), match="google", flavor=flavor)
+
+
+def test_same_ordering(datapath):
+ pytest.importorskip("bs4")
+ pytest.importorskip("lxml")
+ pytest.importorskip("html5lib")
+
+ filename = datapath("io", "data", "html", "valid_markup.html")
+ dfs_lxml = read_html(filename, index_col=0, flavor=["lxml"])
+ dfs_bs4 = read_html(filename, index_col=0, flavor=["bs4"])
+ assert_framelist_equal(dfs_lxml, dfs_bs4)
+
+
+@pytest.fixture(
+ params=[
+ pytest.param("bs4", marks=[td.skip_if_no("bs4"), td.skip_if_no("html5lib")]),
+ pytest.param("lxml", marks=td.skip_if_no("lxml")),
+ ],
+)
+def flavor_read_html(request):
+ return partial(read_html, flavor=request.param)
+
+
+class TestReadHtml:
+ def test_literal_html_deprecation(self, flavor_read_html):
+ # GH 53785
+ msg = r"\[Errno 2\] No such file or director"
+
+ with pytest.raises(FileNotFoundError, match=msg):
+ flavor_read_html(
+ """
+
+
+ A
+ B
+
+
+
+
+ 1
+ 2
+
+
+
+
+ 3
+ 4
+
+
+
"""
+ )
+
+ @pytest.fixture
+ def spam_data(self, datapath):
+ return datapath("io", "data", "html", "spam.html")
+
+ @pytest.fixture
+ def banklist_data(self, datapath):
+ return datapath("io", "data", "html", "banklist.html")
+
+ def test_to_html_compat(self, flavor_read_html):
+ df = (
+ DataFrame(
+ np.random.default_rng(2).random((4, 3)),
+ columns=pd.Index(list("abc")),
+ )
+ .map("{:.3f}".format)
+ .astype(float)
+ )
+ out = df.to_html()
+ res = flavor_read_html(
+ StringIO(out), attrs={"class": "dataframe"}, index_col=0
+ )[0]
+ tm.assert_frame_equal(res, df)
+
+ def test_dtype_backend(self, string_storage, dtype_backend, flavor_read_html):
+ # GH#50286
+ df = DataFrame(
+ {
+ "a": Series([1, NA, 3], dtype="Int64"),
+ "b": Series([1, 2, 3], dtype="Int64"),
+ "c": Series([1.5, NA, 2.5], dtype="Float64"),
+ "d": Series([1.5, 2.0, 2.5], dtype="Float64"),
+ "e": [True, False, None],
+ "f": [True, False, True],
+ "g": ["a", "b", "c"],
+ "h": ["a", "b", None],
+ }
+ )
+
+ out = df.to_html(index=False)
+ with pd.option_context("mode.string_storage", string_storage):
+ result = flavor_read_html(StringIO(out), dtype_backend=dtype_backend)[0]
+
+ if dtype_backend == "pyarrow":
+ pa = pytest.importorskip("pyarrow")
+ string_dtype = pd.ArrowDtype(pa.string())
+ else:
+ string_dtype = pd.StringDtype(string_storage)
+
+ expected = DataFrame(
+ {
+ "a": Series([1, NA, 3], dtype="Int64"),
+ "b": Series([1, 2, 3], dtype="Int64"),
+ "c": Series([1.5, NA, 2.5], dtype="Float64"),
+ "d": Series([1.5, 2.0, 2.5], dtype="Float64"),
+ "e": Series([True, False, NA], dtype="boolean"),
+ "f": Series([True, False, True], dtype="boolean"),
+ "g": Series(["a", "b", "c"], dtype=string_dtype),
+ "h": Series(["a", "b", None], dtype=string_dtype),
+ }
+ )
+
+ if dtype_backend == "pyarrow":
+ import pyarrow as pa
+
+ from pandas.arrays import ArrowExtensionArray
+
+ expected = DataFrame(
+ {
+ col: ArrowExtensionArray(pa.array(expected[col], from_pandas=True))
+ for col in expected.columns
+ }
+ )
+
+ # the storage of the str columns' Index is also affected by the
+ # string_storage setting -> ignore that for checking the result
+ tm.assert_frame_equal(result, expected, check_column_type=False)
+
+ @pytest.mark.network
+ @pytest.mark.single_cpu
+ def test_banklist_url(self, httpserver, banklist_data, flavor_read_html):
+ with open(banklist_data, encoding="utf-8") as f:
+ httpserver.serve_content(content=f.read())
+ df1 = flavor_read_html(
+ # lxml cannot find attrs leave out for now
+ httpserver.url,
+ match="First Federal Bank of Florida", # attrs={"class": "dataTable"}
+ )
+ # lxml cannot find attrs leave out for now
+ df2 = flavor_read_html(
+ httpserver.url,
+ match="Metcalf Bank",
+ ) # attrs={"class": "dataTable"})
+
+ assert_framelist_equal(df1, df2)
+
+ @pytest.mark.network
+ @pytest.mark.single_cpu
+ def test_spam_url(self, httpserver, spam_data, flavor_read_html):
+ with open(spam_data, encoding="utf-8") as f:
+ httpserver.serve_content(content=f.read())
+ df1 = flavor_read_html(httpserver.url, match=".*Water.*")
+ df2 = flavor_read_html(httpserver.url, match="Unit")
+
+ assert_framelist_equal(df1, df2)
+
+ @pytest.mark.slow
+ def test_banklist(self, banklist_data, flavor_read_html):
+ df1 = flavor_read_html(
+ banklist_data, match=".*Florida.*", attrs={"id": "table"}
+ )
+ df2 = flavor_read_html(
+ banklist_data, match="Metcalf Bank", attrs={"id": "table"}
+ )
+
+ assert_framelist_equal(df1, df2)
+
+ def test_spam(self, spam_data, flavor_read_html):
+ df1 = flavor_read_html(spam_data, match=".*Water.*")
+ df2 = flavor_read_html(spam_data, match="Unit")
+ assert_framelist_equal(df1, df2)
+
+ assert df1[0].iloc[0, 0] == "Proximates"
+ assert df1[0].columns[0] == "Nutrient"
+
+ def test_spam_no_match(self, spam_data, flavor_read_html):
+ dfs = flavor_read_html(spam_data)
+ for df in dfs:
+ assert isinstance(df, DataFrame)
+
+ def test_banklist_no_match(self, banklist_data, flavor_read_html):
+ dfs = flavor_read_html(banklist_data, attrs={"id": "table"})
+ for df in dfs:
+ assert isinstance(df, DataFrame)
+
+ def test_spam_header(self, spam_data, flavor_read_html):
+ df = flavor_read_html(spam_data, match=".*Water.*", header=2)[0]
+ assert df.columns[0] == "Proximates"
+ assert not df.empty
+
+ def test_skiprows_int(self, spam_data, flavor_read_html):
+ df1 = flavor_read_html(spam_data, match=".*Water.*", skiprows=1)
+ df2 = flavor_read_html(spam_data, match="Unit", skiprows=1)
+
+ assert_framelist_equal(df1, df2)
+
+ def test_skiprows_range(self, spam_data, flavor_read_html):
+ df1 = flavor_read_html(spam_data, match=".*Water.*", skiprows=range(2))
+ df2 = flavor_read_html(spam_data, match="Unit", skiprows=range(2))
+
+ assert_framelist_equal(df1, df2)
+
+ def test_skiprows_list(self, spam_data, flavor_read_html):
+ df1 = flavor_read_html(spam_data, match=".*Water.*", skiprows=[1, 2])
+ df2 = flavor_read_html(spam_data, match="Unit", skiprows=[2, 1])
+
+ assert_framelist_equal(df1, df2)
+
+ def test_skiprows_set(self, spam_data, flavor_read_html):
+ df1 = flavor_read_html(spam_data, match=".*Water.*", skiprows={1, 2})
+ df2 = flavor_read_html(spam_data, match="Unit", skiprows={2, 1})
+
+ assert_framelist_equal(df1, df2)
+
+ def test_skiprows_slice(self, spam_data, flavor_read_html):
+ df1 = flavor_read_html(spam_data, match=".*Water.*", skiprows=1)
+ df2 = flavor_read_html(spam_data, match="Unit", skiprows=1)
+
+ assert_framelist_equal(df1, df2)
+
+ def test_skiprows_slice_short(self, spam_data, flavor_read_html):
+ df1 = flavor_read_html(spam_data, match=".*Water.*", skiprows=slice(2))
+ df2 = flavor_read_html(spam_data, match="Unit", skiprows=slice(2))
+
+ assert_framelist_equal(df1, df2)
+
+ def test_skiprows_slice_long(self, spam_data, flavor_read_html):
+ df1 = flavor_read_html(spam_data, match=".*Water.*", skiprows=slice(2, 5))
+ df2 = flavor_read_html(spam_data, match="Unit", skiprows=slice(4, 1, -1))
+
+ assert_framelist_equal(df1, df2)
+
+ def test_skiprows_ndarray(self, spam_data, flavor_read_html):
+ df1 = flavor_read_html(spam_data, match=".*Water.*", skiprows=np.arange(2))
+ df2 = flavor_read_html(spam_data, match="Unit", skiprows=np.arange(2))
+
+ assert_framelist_equal(df1, df2)
+
+ def test_skiprows_invalid(self, spam_data, flavor_read_html):
+ with pytest.raises(TypeError, match=("is not a valid type for skipping rows")):
+ flavor_read_html(spam_data, match=".*Water.*", skiprows="asdf")
+
+ def test_index(self, spam_data, flavor_read_html):
+ df1 = flavor_read_html(spam_data, match=".*Water.*", index_col=0)
+ df2 = flavor_read_html(spam_data, match="Unit", index_col=0)
+ assert_framelist_equal(df1, df2)
+
+ def test_header_and_index_no_types(self, spam_data, flavor_read_html):
+ df1 = flavor_read_html(spam_data, match=".*Water.*", header=1, index_col=0)
+ df2 = flavor_read_html(spam_data, match="Unit", header=1, index_col=0)
+ assert_framelist_equal(df1, df2)
+
+ def test_header_and_index_with_types(self, spam_data, flavor_read_html):
+ df1 = flavor_read_html(spam_data, match=".*Water.*", header=1, index_col=0)
+ df2 = flavor_read_html(spam_data, match="Unit", header=1, index_col=0)
+ assert_framelist_equal(df1, df2)
+
+ def test_infer_types(self, spam_data, flavor_read_html):
+ # 10892 infer_types removed
+ df1 = flavor_read_html(spam_data, match=".*Water.*", index_col=0)
+ df2 = flavor_read_html(spam_data, match="Unit", index_col=0)
+ assert_framelist_equal(df1, df2)
+
+ def test_string_io(self, spam_data, flavor_read_html):
+ with open(spam_data, encoding="UTF-8") as f:
+ data1 = StringIO(f.read())
+
+ with open(spam_data, encoding="UTF-8") as f:
+ data2 = StringIO(f.read())
+
+ df1 = flavor_read_html(data1, match=".*Water.*")
+ df2 = flavor_read_html(data2, match="Unit")
+ assert_framelist_equal(df1, df2)
+
+ def test_string(self, spam_data, flavor_read_html):
+ with open(spam_data, encoding="UTF-8") as f:
+ data = f.read()
+
+ df1 = flavor_read_html(StringIO(data), match=".*Water.*")
+ df2 = flavor_read_html(StringIO(data), match="Unit")
+
+ assert_framelist_equal(df1, df2)
+
+ def test_file_like(self, spam_data, flavor_read_html):
+ with open(spam_data, encoding="UTF-8") as f:
+ df1 = flavor_read_html(f, match=".*Water.*")
+
+ with open(spam_data, encoding="UTF-8") as f:
+ df2 = flavor_read_html(f, match="Unit")
+
+ assert_framelist_equal(df1, df2)
+
+ @pytest.mark.network
+ @pytest.mark.single_cpu
+ def test_bad_url_protocol(self, httpserver, flavor_read_html):
+ httpserver.serve_content("urlopen error unknown url type: git", code=404)
+ with pytest.raises(URLError, match="urlopen error unknown url type: git"):
+ flavor_read_html("git://github.com", match=".*Water.*")
+
+ @pytest.mark.slow
+ @pytest.mark.network
+ @pytest.mark.single_cpu
+ def test_invalid_url(self, httpserver, flavor_read_html):
+ httpserver.serve_content("Name or service not known", code=404)
+ try:
+ with pytest.raises(
+ (URLError, ValueError), match="HTTP Error 404: NOT FOUND"
+ ) as err:
+ flavor_read_html(httpserver.url, match=".*Water.*")
+ finally:
+ if isinstance(err.value, URLError):
+ # Has a file-like handle that we can close
+ # https://docs.python.org/3/library/urllib.error.html#urllib.error.HTTPError
+ err.value.close()
+
+ @pytest.mark.slow
+ def test_file_url(self, banklist_data, flavor_read_html):
+ url = banklist_data
+ dfs = flavor_read_html(
+ file_path_to_url(os.path.abspath(url)), match="First", attrs={"id": "table"}
+ )
+ assert isinstance(dfs, list)
+ for df in dfs:
+ assert isinstance(df, DataFrame)
+
+ @pytest.mark.slow
+ def test_invalid_table_attrs(self, banklist_data, flavor_read_html):
+ url = banklist_data
+ with pytest.raises(ValueError, match="No tables found"):
+ flavor_read_html(
+ url, match="First Federal Bank of Florida", attrs={"id": "tasdfable"}
+ )
+
+ @pytest.mark.slow
+ def test_multiindex_header(self, banklist_data, flavor_read_html):
+ df = flavor_read_html(
+ banklist_data, match="Metcalf", attrs={"id": "table"}, header=[0, 1]
+ )[0]
+ assert isinstance(df.columns, MultiIndex)
+
+ @pytest.mark.slow
+ def test_multiindex_index(self, banklist_data, flavor_read_html):
+ df = flavor_read_html(
+ banklist_data, match="Metcalf", attrs={"id": "table"}, index_col=[0, 1]
+ )[0]
+ assert isinstance(df.index, MultiIndex)
+
+ @pytest.mark.slow
+ def test_multiindex_header_index(self, banklist_data, flavor_read_html):
+ df = flavor_read_html(
+ banklist_data,
+ match="Metcalf",
+ attrs={"id": "table"},
+ header=[0, 1],
+ index_col=[0, 1],
+ )[0]
+ assert isinstance(df.columns, MultiIndex)
+ assert isinstance(df.index, MultiIndex)
+
+ @pytest.mark.slow
+ def test_multiindex_header_skiprows_tuples(self, banklist_data, flavor_read_html):
+ df = flavor_read_html(
+ banklist_data,
+ match="Metcalf",
+ attrs={"id": "table"},
+ header=[0, 1],
+ skiprows=1,
+ )[0]
+ assert isinstance(df.columns, MultiIndex)
+
+ @pytest.mark.slow
+ def test_multiindex_header_skiprows(self, banklist_data, flavor_read_html):
+ df = flavor_read_html(
+ banklist_data,
+ match="Metcalf",
+ attrs={"id": "table"},
+ header=[0, 1],
+ skiprows=1,
+ )[0]
+ assert isinstance(df.columns, MultiIndex)
+
+ @pytest.mark.slow
+ def test_multiindex_header_index_skiprows(self, banklist_data, flavor_read_html):
+ df = flavor_read_html(
+ banklist_data,
+ match="Metcalf",
+ attrs={"id": "table"},
+ header=[0, 1],
+ index_col=[0, 1],
+ skiprows=1,
+ )[0]
+ assert isinstance(df.index, MultiIndex)
+ assert isinstance(df.columns, MultiIndex)
+
+ @pytest.mark.slow
+ def test_regex_idempotency(self, banklist_data, flavor_read_html):
+ url = banklist_data
+ dfs = flavor_read_html(
+ file_path_to_url(os.path.abspath(url)),
+ match=re.compile(re.compile("Florida")),
+ attrs={"id": "table"},
+ )
+ assert isinstance(dfs, list)
+ for df in dfs:
+ assert isinstance(df, DataFrame)
+
+ def test_negative_skiprows(self, spam_data, flavor_read_html):
+ msg = r"\(you passed a negative value\)"
+ with pytest.raises(ValueError, match=msg):
+ flavor_read_html(spam_data, match="Water", skiprows=-1)
+
+ @pytest.fixture
+ def python_docs(self):
+ return """
+
+
+ What's new in Python 2.7?
+ or all "What's new" documents since 2.0
+ Tutorial
+ start here
+ Library Reference
+ keep this under your pillow
+ Language Reference
+ describes syntax and language elements
+ Python Setup and Usage
+ how to use Python on different platforms
+ Python HOWTOs
+ in-depth documents on specific topics
+
+ Installing Python Modules
+ installing from the Python Package Index & other sources
+ Distributing Python Modules
+ publishing modules for installation by others
+ Extending and Embedding
+ tutorial for C/C++ programmers
+ Python/C API
+ reference for C/C++ programmers
+ FAQs
+ frequently asked questions (with answers!)
+
+
+
+ Indices and tables:
+
+
+ Python Global Module Index
+ quick access to all modules
+ General Index
+ all functions, classes, terms
+ Glossary
+ the most important terms explained
+
+ Search page
+ search this documentation
+ Complete Table of Contents
+ lists all sections and subsections
+
+
+ """ # noqa: E501
+
+ @pytest.mark.network
+ @pytest.mark.single_cpu
+ def test_multiple_matches(self, python_docs, httpserver, flavor_read_html):
+ httpserver.serve_content(content=python_docs)
+ dfs = flavor_read_html(httpserver.url, match="Python")
+ assert len(dfs) > 1
+
+ @pytest.mark.network
+ @pytest.mark.single_cpu
+ def test_python_docs_table(self, python_docs, httpserver, flavor_read_html):
+ httpserver.serve_content(content=python_docs)
+ dfs = flavor_read_html(httpserver.url, match="Python")
+ zz = [df.iloc[0, 0][0:4] for df in dfs]
+ assert sorted(zz) == ["Pyth", "What"]
+
+ def test_empty_tables(self, flavor_read_html):
+ """
+ Make sure that read_html ignores empty tables.
+ """
+ html = """
+
+
+
+ A
+ B
+
+
+
+
+ 1
+ 2
+
+
+
+
+ """
+ result = flavor_read_html(StringIO(html))
+ assert len(result) == 1
+
+ def test_multiple_tbody(self, flavor_read_html):
+ # GH-20690
+ # Read all tbody tags within a single table.
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+ A
+ B
+
+
+
+
+ 1
+ 2
+
+
+
+
+ 3
+ 4
+
+
+
"""
+ )
+ )[0]
+
+ expected = DataFrame(data=[[1, 2], [3, 4]], columns=["A", "B"])
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_header_and_one_column(self, flavor_read_html):
+ """
+ Don't fail with bs4 when there is a header and only one column
+ as described in issue #9178
+ """
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+ Header
+
+
+
+
+ first
+
+
+
"""
+ )
+ )[0]
+
+ expected = DataFrame(data={"Header": "first"}, index=[0])
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_thead_without_tr(self, flavor_read_html):
+ """
+ Ensure parser adds within on malformed HTML.
+ """
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+ Country
+ Municipality
+ Year
+
+
+
+
+ Ukraine
+ Odessa
+ 1944
+
+
+
"""
+ )
+ )[0]
+
+ expected = DataFrame(
+ data=[["Ukraine", "Odessa", 1944]],
+ columns=["Country", "Municipality", "Year"],
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_tfoot_read(self, flavor_read_html):
+ """
+ Make sure that read_html reads tfoot, containing td or th.
+ Ignores empty tfoot
+ """
+ data_template = """
+
+
+ A
+ B
+
+
+
+
+ bodyA
+ bodyB
+
+
+
+ {footer}
+
+
"""
+
+ expected1 = DataFrame(data=[["bodyA", "bodyB"]], columns=["A", "B"])
+
+ expected2 = DataFrame(
+ data=[["bodyA", "bodyB"], ["footA", "footB"]], columns=["A", "B"]
+ )
+
+ data1 = data_template.format(footer="")
+ data2 = data_template.format(footer="footA footB ")
+
+ result1 = flavor_read_html(StringIO(data1))[0]
+ result2 = flavor_read_html(StringIO(data2))[0]
+
+ tm.assert_frame_equal(result1, expected1)
+ tm.assert_frame_equal(result2, expected2)
+
+ def test_parse_header_of_non_string_column(self, flavor_read_html):
+ # GH5048: if header is specified explicitly, an int column should be
+ # parsed as int while its header is parsed as str
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+ S
+ I
+
+
+ text
+ 1944
+
+
+ """
+ ),
+ header=0,
+ )[0]
+
+ expected = DataFrame([["text", 1944]], columns=("S", "I"))
+
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.slow
+ def test_banklist_header(self, banklist_data, datapath, flavor_read_html):
+ from pandas.io.html import _remove_whitespace
+
+ def try_remove_ws(x):
+ try:
+ return _remove_whitespace(x)
+ except AttributeError:
+ return x
+
+ df = flavor_read_html(banklist_data, match="Metcalf", attrs={"id": "table"})[0]
+ ground_truth = read_csv(
+ datapath("io", "data", "csv", "banklist.csv"),
+ converters={"Updated Date": Timestamp, "Closing Date": Timestamp},
+ )
+ # html is a truncated version of banklist since bs4 is slow to parse it
+ assert df.shape == (len(df), ground_truth.shape[1])
+ old = [
+ "First Vietnamese American Bank In Vietnamese",
+ "Westernbank Puerto Rico En Espanol",
+ "R-G Premier Bank of Puerto Rico En Espanol",
+ "Eurobank En Espanol",
+ "Sanderson State Bank En Espanol",
+ "Washington Mutual Bank (Including its subsidiary Washington "
+ "Mutual Bank FSB)",
+ "Silver State Bank En Espanol",
+ "AmTrade International Bank En Espanol",
+ "Hamilton Bank, NA En Espanol",
+ "The Citizens Savings Bank Pioneer Community Bank, Inc.",
+ ]
+ new = [
+ "First Vietnamese American Bank",
+ "Westernbank Puerto Rico",
+ "R-G Premier Bank of Puerto Rico",
+ "Eurobank",
+ "Sanderson State Bank",
+ "Washington Mutual Bank",
+ "Silver State Bank",
+ "AmTrade International Bank",
+ "Hamilton Bank, NA",
+ "The Citizens Savings Bank",
+ ]
+ dfnew = df.map(try_remove_ws).replace(old, new)
+ gtnew = ground_truth.map(try_remove_ws)
+ converted = dfnew
+ date_cols = ["Closing Date", "Updated Date"]
+ converted[date_cols] = converted[date_cols].apply(to_datetime)
+ gtnew = gtnew[gtnew["Bank Name"].isin(converted["Bank Name"])].reset_index(
+ drop=True
+ )
+ tm.assert_frame_equal(converted, gtnew)
+
+ @pytest.mark.slow
+ def test_heartland_bank(self, banklist_data, flavor_read_html):
+ gc = "Heartland Bank"
+ with open(banklist_data, encoding="utf-8") as f:
+ raw_text = f.read()
+
+ assert gc in raw_text
+ df = flavor_read_html(banklist_data, match=gc, attrs={"id": "table"})[0]
+ assert gc in df.to_string()
+
+ def test_different_number_of_cols(self, flavor_read_html):
+ expected = flavor_read_html(
+ StringIO(
+ """
+
+
+
+ C_l0_g0
+ C_l0_g1
+ C_l0_g2
+ C_l0_g3
+ C_l0_g4
+
+
+
+
+ R_l0_g0
+ 0.763
+ 0.233
+ nan
+ nan
+ nan
+
+
+ R_l0_g1
+ 0.244
+ 0.285
+ 0.392
+ 0.137
+ 0.222
+
+
+
"""
+ ),
+ index_col=0,
+ )[0]
+
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+
+ C_l0_g0
+ C_l0_g1
+ C_l0_g2
+ C_l0_g3
+ C_l0_g4
+
+
+
+
+ R_l0_g0
+ 0.763
+ 0.233
+
+
+ R_l0_g1
+ 0.244
+ 0.285
+ 0.392
+ 0.137
+ 0.222
+
+
+
"""
+ ),
+ index_col=0,
+ )[0]
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_colspan_rowspan_1(self, flavor_read_html):
+ # GH17054
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+ A
+ B
+ C
+
+
+ a
+ b
+ c
+
+
+ """
+ )
+ )[0]
+
+ expected = DataFrame([["a", "b", "c"]], columns=["A", "B", "C"])
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_colspan_rowspan_copy_values(self, flavor_read_html):
+ # GH17054
+
+ # In ASCII, with lowercase letters being copies:
+ #
+ # X x Y Z W
+ # A B b z C
+
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+ X
+ Y
+ Z
+ W
+
+
+ A
+ B
+ C
+
+
+ """
+ ),
+ header=0,
+ )[0]
+
+ expected = DataFrame(
+ data=[["A", "B", "B", "Z", "C"]], columns=["X", "X.1", "Y", "Z", "W"]
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_colspan_rowspan_both_not_1(self, flavor_read_html):
+ # GH17054
+
+ # In ASCII, with lowercase letters being copies:
+ #
+ # A B b b C
+ # a b b b D
+
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+ A
+ B
+ C
+
+
+ D
+
+
+ """
+ ),
+ header=0,
+ )[0]
+
+ expected = DataFrame(
+ data=[["A", "B", "B", "B", "D"]], columns=["A", "B", "B.1", "B.2", "C"]
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_rowspan_at_end_of_row(self, flavor_read_html):
+ # GH17054
+
+ # In ASCII, with lowercase letters being copies:
+ #
+ # A B
+ # C b
+
+ result = flavor_read_html(
+ StringIO(
+ """
+
+ """
+ ),
+ header=0,
+ )[0]
+
+ expected = DataFrame(data=[["C", "B"]], columns=["A", "B"])
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_rowspan_only_rows(self, flavor_read_html):
+ # GH17054
+
+ result = flavor_read_html(
+ StringIO(
+ """
+
+ """
+ ),
+ header=0,
+ )[0]
+
+ expected = DataFrame(data=[["A", "B"], ["A", "B"]], columns=["A", "B"])
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_rowspan_in_header_overflowing_to_body(self, flavor_read_html):
+ # GH60210
+
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+ A
+ B
+
+
+ 1
+
+
+ C
+ 2
+
+
+ """
+ )
+ )[0]
+
+ expected = DataFrame(data=[["A", 1], ["C", 2]], columns=["A", "B"])
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_header_inferred_from_rows_with_only_th(self, flavor_read_html):
+ # GH17054
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+ A
+ B
+
+
+ a
+ b
+
+
+ 1
+ 2
+
+
+ """
+ )
+ )[0]
+
+ columns = MultiIndex(levels=[["A", "B"], ["a", "b"]], codes=[[0, 1], [0, 1]])
+ expected = DataFrame(data=[[1, 2]], columns=columns)
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_parse_dates_list(self, flavor_read_html):
+ df = DataFrame({"date": date_range("1/1/2001", periods=10)})
+
+ expected = df[:]
+ expected["date"] = expected["date"].dt.as_unit("us")
+
+ str_df = df.to_html()
+ res = flavor_read_html(StringIO(str_df), parse_dates=[1], index_col=0)
+ tm.assert_frame_equal(expected, res[0])
+ res = flavor_read_html(StringIO(str_df), parse_dates=["date"], index_col=0)
+ tm.assert_frame_equal(expected, res[0])
+
+ def test_wikipedia_states_table(self, datapath, flavor_read_html):
+ data = datapath("io", "data", "html", "wikipedia_states.html")
+ assert os.path.isfile(data), f"{data!r} is not a file"
+ assert os.path.getsize(data), f"{data!r} is an empty file"
+ result = flavor_read_html(data, match="Arizona", header=1)[0]
+ assert result.shape == (60, 12)
+ assert "Unnamed" in result.columns[-1]
+ assert result["sq mi"].dtype == np.dtype("float64")
+ assert np.allclose(result.loc[0, "sq mi"], 665384.04)
+
+ def test_wikipedia_states_multiindex(self, datapath, flavor_read_html):
+ data = datapath("io", "data", "html", "wikipedia_states.html")
+ result = flavor_read_html(data, match="Arizona", index_col=0)[0]
+ assert result.shape == (60, 11)
+ assert "Unnamed" in result.columns[-1][1]
+ assert result.columns.nlevels == 2
+ assert np.allclose(result.loc["Alaska", ("Total area[2]", "sq mi")], 665384.04)
+
+ def test_parser_error_on_empty_header_row(self, flavor_read_html):
+ result = flavor_read_html(
+ StringIO(
+ """
+
+ """
+ ),
+ header=[0, 1],
+ )
+ expected = DataFrame(
+ [["a", "b"]],
+ columns=MultiIndex.from_tuples(
+ [("Unnamed: 0_level_0", "A"), ("Unnamed: 1_level_0", "B")]
+ ),
+ )
+ tm.assert_frame_equal(result[0], expected)
+
+ def test_decimal_rows(self, flavor_read_html):
+ # GH 12907
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+
+
+ Header
+
+
+
+
+ 1100#101
+
+
+
+
+ """
+ ),
+ decimal="#",
+ )[0]
+
+ expected = DataFrame(data={"Header": 1100.101}, index=[0])
+
+ assert result["Header"].dtype == np.dtype("float64")
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("arg", [True, False])
+ def test_bool_header_arg(self, spam_data, arg, flavor_read_html):
+ # GH 6114
+ msg = re.escape(
+ "Passing a bool to header is invalid. Use header=None for no header or "
+ "header=int or list-like of ints to specify the row(s) making up the "
+ "column names"
+ )
+ with pytest.raises(TypeError, match=msg):
+ flavor_read_html(spam_data, header=arg)
+
+ def test_converters(self, flavor_read_html):
+ # GH 13461
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+ a
+
+
+
+
+ 0.763
+
+
+ 0.244
+
+
+
"""
+ ),
+ converters={"a": str},
+ )[0]
+
+ expected = DataFrame({"a": ["0.763", "0.244"]})
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_na_values(self, flavor_read_html):
+ # GH 13461
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+ a
+
+
+
+
+ 0.763
+
+
+ 0.244
+
+
+
"""
+ ),
+ na_values=[0.244],
+ )[0]
+
+ expected = DataFrame({"a": [0.763, np.nan]})
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_keep_default_na(self, flavor_read_html):
+ html_data = """
+
+
+ a
+
+
+
+
+ N/A
+
+
+ NA
+
+
+
"""
+
+ expected_df = DataFrame({"a": ["N/A", "NA"]})
+ html_df = flavor_read_html(StringIO(html_data), keep_default_na=False)[0]
+ tm.assert_frame_equal(expected_df, html_df)
+
+ expected_df = DataFrame({"a": [np.nan, np.nan]})
+ html_df = flavor_read_html(StringIO(html_data), keep_default_na=True)[0]
+ tm.assert_frame_equal(expected_df, html_df)
+
+ def test_preserve_empty_rows(self, flavor_read_html):
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+ A
+ B
+
+
+ a
+ b
+
+
+
+
+
+
+ """
+ )
+ )[0]
+
+ expected = DataFrame(data=[["a", "b"], [np.nan, np.nan]], columns=["A", "B"])
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_ignore_empty_rows_when_inferring_header(self, flavor_read_html):
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+
+ A B
+ a b
+
+
+ 1 2
+
+
+ """
+ )
+ )[0]
+
+ columns = MultiIndex(levels=[["A", "B"], ["a", "b"]], codes=[[0, 1], [0, 1]])
+ expected = DataFrame(data=[[1, 2]], columns=columns)
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_multiple_header_rows(self, flavor_read_html):
+ # Issue #13434
+ expected_df = DataFrame(
+ data=[("Hillary", 68, "D"), ("Bernie", 74, "D"), ("Donald", 69, "R")]
+ )
+ expected_df.columns = [
+ ["Unnamed: 0_level_0", "Age", "Party"],
+ ["Name", "Unnamed: 1_level_1", "Unnamed: 2_level_1"],
+ ]
+ html = expected_df.to_html(index=False)
+ html_df = flavor_read_html(StringIO(html))[0]
+ tm.assert_frame_equal(expected_df, html_df)
+
+ def test_works_on_valid_markup(self, datapath, flavor_read_html):
+ filename = datapath("io", "data", "html", "valid_markup.html")
+ dfs = flavor_read_html(filename, index_col=0)
+ assert isinstance(dfs, list)
+ assert isinstance(dfs[0], DataFrame)
+
+ @pytest.mark.slow
+ def test_fallback_success(self, datapath, flavor_read_html):
+ banklist_data = datapath("io", "data", "html", "banklist.html")
+
+ flavor_read_html(banklist_data, match=".*Water.*", flavor=["lxml", "html5lib"])
+
+ def test_to_html_timestamp(self):
+ rng = date_range("2000-01-01", periods=10)
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 4)), index=rng)
+
+ result = df.to_html()
+ assert "2000-01-01" in result
+
+ def test_to_html_borderless(self):
+ df = DataFrame([{"A": 1, "B": 2}])
+ out_border_default = df.to_html()
+ out_border_true = df.to_html(border=True)
+ out_border_explicit_default = df.to_html(border=1)
+ out_border_nondefault = df.to_html(border=2)
+ out_border_zero = df.to_html(border=0)
+
+ out_border_false = df.to_html(border=False)
+
+ assert ' border="1"' in out_border_default
+ assert out_border_true == out_border_default
+ assert out_border_default == out_border_explicit_default
+ assert out_border_default != out_border_nondefault
+ assert ' border="2"' in out_border_nondefault
+ assert ' border="0"' not in out_border_zero
+ assert " border" not in out_border_false
+ assert out_border_zero == out_border_false
+
+ @pytest.mark.parametrize(
+ "displayed_only,exp0,exp1",
+ [
+ (True, ["foo"], None),
+ (False, ["foo bar baz qux"], DataFrame(["foo"])),
+ ],
+ )
+ def test_displayed_only(self, displayed_only, exp0, exp1, flavor_read_html):
+ # GH 20027
+ data = """
+
+
+
+
+ foo
+ bar
+ baz
+ qux
+
+
+
+
+
+ """
+
+ exp0 = DataFrame(exp0)
+ dfs = flavor_read_html(StringIO(data), displayed_only=displayed_only)
+ tm.assert_frame_equal(dfs[0], exp0)
+
+ if exp1 is not None:
+ tm.assert_frame_equal(dfs[1], exp1)
+ else:
+ assert len(dfs) == 1 # Should not parse hidden table
+
+ @pytest.mark.parametrize("displayed_only", [True, False])
+ def test_displayed_only_with_many_elements(self, displayed_only, flavor_read_html):
+ html_table = """
+
+
+ A
+ B
+
+
+ 1
+ 2
+
+
+ 4
+ 5
+
+
+ """
+ result = flavor_read_html(StringIO(html_table), displayed_only=displayed_only)[
+ 0
+ ]
+ expected = DataFrame({"A": [1, 4], "B": [2, 5]})
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.filterwarnings(
+ "ignore:You provided Unicode markup but also provided a value for "
+ "from_encoding.*:UserWarning"
+ )
+ def test_encode(self, html_encoding_file, flavor_read_html):
+ base_path = os.path.basename(html_encoding_file)
+ root = os.path.splitext(base_path)[0]
+ _, encoding = root.split("_")
+
+ try:
+ with open(html_encoding_file, "rb") as fobj:
+ from_string = flavor_read_html(
+ BytesIO(fobj.read()), encoding=encoding, index_col=0
+ ).pop()
+
+ with open(html_encoding_file, "rb") as fobj:
+ from_file_like = flavor_read_html(
+ BytesIO(fobj.read()), encoding=encoding, index_col=0
+ ).pop()
+
+ from_filename = flavor_read_html(
+ html_encoding_file, encoding=encoding, index_col=0
+ ).pop()
+ tm.assert_frame_equal(from_string, from_file_like)
+ tm.assert_frame_equal(from_string, from_filename)
+ except Exception:
+ # seems utf-16/32 fail on windows
+ if is_platform_windows():
+ if "16" in encoding or "32" in encoding:
+ pytest.skip()
+ raise
+
+ def test_parse_failure_unseekable(self, flavor_read_html):
+ # Issue #17975
+
+ if flavor_read_html.keywords.get("flavor") == "lxml":
+ pytest.skip("Not applicable for lxml")
+
+ class UnseekableStringIO(StringIO):
+ def seekable(self):
+ return False
+
+ bad = UnseekableStringIO(
+ """
+ """
+ )
+
+ assert flavor_read_html(bad)
+
+ with pytest.raises(ValueError, match="passed a non-rewindable file object"):
+ flavor_read_html(bad)
+
+ def test_parse_failure_rewinds(self, flavor_read_html):
+ # Issue #17975
+
+ class MockFile:
+ def __init__(self, data) -> None:
+ self.data = data
+ self.at_end = False
+
+ def read(self, size=None):
+ data = "" if self.at_end else self.data
+ self.at_end = True
+ return data
+
+ def seek(self, offset):
+ self.at_end = False
+
+ def seekable(self):
+ return True
+
+ def __next__(self): ...
+
+ def __iter__(self) -> Iterator:
+ # `is_file_like` depends on the presence of
+ # the __iter__ attribute.
+ return self
+
+ good = MockFile("")
+ bad = MockFile("")
+
+ assert flavor_read_html(good)
+ assert flavor_read_html(bad)
+
+ @pytest.mark.slow
+ @pytest.mark.single_cpu
+ def test_importcheck_thread_safety(self, datapath, flavor_read_html):
+ # see gh-16928
+
+ class ErrorThread(threading.Thread):
+ def run(self):
+ try:
+ super().run()
+ except Exception as err:
+ self.err = err
+ else:
+ self.err = None
+
+ filename = datapath("io", "data", "html", "valid_markup.html")
+ helper_thread1 = ErrorThread(target=flavor_read_html, args=(filename,))
+ helper_thread2 = ErrorThread(target=flavor_read_html, args=(filename,))
+
+ helper_thread1.start()
+ helper_thread2.start()
+
+ while helper_thread1.is_alive() or helper_thread2.is_alive():
+ pass
+ assert None is helper_thread1.err is helper_thread2.err
+
+ def test_parse_path_object(self, datapath, flavor_read_html):
+ # GH 37705
+ file_path_string = datapath("io", "data", "html", "spam.html")
+ file_path = Path(file_path_string)
+ df1 = flavor_read_html(file_path_string)[0]
+ df2 = flavor_read_html(file_path)[0]
+ tm.assert_frame_equal(df1, df2)
+
+ def test_parse_br_as_space(self, flavor_read_html):
+ # GH 29528: pd.read_html() convert to space
+ result = flavor_read_html(
+ StringIO(
+ """
+
+
+ A
+
+
+ word1 word2
+
+
+ """
+ )
+ )[0]
+
+ expected = DataFrame(data=[["word1 word2"]], columns=["A"])
+
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("arg", ["all", "body", "header", "footer"])
+ def test_extract_links(self, arg, flavor_read_html):
+ gh_13141_data = """
+
+ """
+
+ gh_13141_expected = {
+ "head_ignore": ["HTTP", "FTP", "Linkless"],
+ "head_extract": [
+ ("HTTP", None),
+ ("FTP", None),
+ ("Linkless", "https://en.wiktionary.org/wiki/linkless"),
+ ],
+ "body_ignore": ["Wikipedia", "SURROUNDING Debian TEXT", "Linkless"],
+ "body_extract": [
+ ("Wikipedia", "https://en.wikipedia.org/"),
+ ("SURROUNDING Debian TEXT", "ftp://ftp.us.debian.org/"),
+ ("Linkless", None),
+ ],
+ "footer_ignore": [
+ "Footer",
+ "Multiple links: Only first captured.",
+ None,
+ ],
+ "footer_extract": [
+ ("Footer", "https://en.wikipedia.org/wiki/Page_footer"),
+ ("Multiple links: Only first captured.", "1"),
+ None,
+ ],
+ }
+
+ data_exp = gh_13141_expected["body_ignore"]
+ foot_exp = gh_13141_expected["footer_ignore"]
+ head_exp = gh_13141_expected["head_ignore"]
+ if arg == "all":
+ data_exp = gh_13141_expected["body_extract"]
+ foot_exp = gh_13141_expected["footer_extract"]
+ head_exp = gh_13141_expected["head_extract"]
+ elif arg == "body":
+ data_exp = gh_13141_expected["body_extract"]
+ elif arg == "footer":
+ foot_exp = gh_13141_expected["footer_extract"]
+ elif arg == "header":
+ head_exp = gh_13141_expected["head_extract"]
+
+ result = flavor_read_html(StringIO(gh_13141_data), extract_links=arg)[0]
+ expected = DataFrame([data_exp, foot_exp], columns=head_exp)
+ expected = expected.fillna(np.nan)
+ tm.assert_frame_equal(result, expected)
+
+ def test_extract_links_bad(self, spam_data):
+ msg = (
+ "`extract_links` must be one of "
+ '{None, "header", "footer", "body", "all"}, got "incorrect"'
+ )
+ with pytest.raises(ValueError, match=msg):
+ read_html(spam_data, extract_links="incorrect")
+
+ def test_extract_links_all_no_header(self, flavor_read_html):
+ # GH 48316
+ data = """
+
+ """
+ result = flavor_read_html(StringIO(data), extract_links="all")[0]
+ expected = DataFrame([[("Google.com", "https://google.com")]])
+ tm.assert_frame_equal(result, expected)
+
+ def test_invalid_dtype_backend(self):
+ msg = (
+ "dtype_backend numpy is invalid, only 'numpy_nullable' and "
+ "'pyarrow' are allowed."
+ )
+ with pytest.raises(ValueError, match=msg):
+ read_html("test", dtype_backend="numpy")
+
+ def test_style_tag(self, flavor_read_html):
+ # GH 48316
+ data = """
+
+
+
+
+ A
+
+ B
+
+
+ A1
+ B1
+
+
+ A2
+ B2
+
+
+ """
+ result = flavor_read_html(StringIO(data))[0]
+ expected = DataFrame(data=[["A1", "B1"], ["A2", "B2"]], columns=["A", "B"])
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_http_headers.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_http_headers.py
new file mode 100644
index 0000000000000000000000000000000000000000..3b9c8769ad9dc909f99d346395d5cd2113984992
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_http_headers.py
@@ -0,0 +1,174 @@
+"""
+Tests for the pandas custom headers in http(s) requests
+"""
+
+from functools import partial
+import gzip
+from io import BytesIO
+
+import pytest
+
+from pandas._config import using_string_dtype
+
+import pandas.util._test_decorators as td
+
+import pandas as pd
+import pandas._testing as tm
+
+pytestmark = [
+ pytest.mark.single_cpu,
+ pytest.mark.network,
+ pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+ ),
+]
+
+
+def gzip_bytes(response_bytes):
+ with BytesIO() as bio:
+ with gzip.GzipFile(fileobj=bio, mode="w") as zipper:
+ zipper.write(response_bytes)
+ return bio.getvalue()
+
+
+def csv_responder(df):
+ return df.to_csv(index=False).encode("utf-8")
+
+
+def gz_csv_responder(df):
+ return gzip_bytes(csv_responder(df))
+
+
+def json_responder(df):
+ return df.to_json().encode("utf-8")
+
+
+def gz_json_responder(df):
+ return gzip_bytes(json_responder(df))
+
+
+def html_responder(df):
+ return df.to_html(index=False).encode("utf-8")
+
+
+def parquetpyarrow_reponder(df):
+ return df.to_parquet(index=False, engine="pyarrow")
+
+
+def parquetfastparquet_responder(df):
+ # the fastparquet engine doesn't like to write to a buffer
+ # it can do it via the open_with function being set appropriately
+ # however it automatically calls the close method and wipes the buffer
+ # so just overwrite that attribute on this instance to not do that
+
+ # protected by an importorskip in the respective test
+ import fsspec
+
+ df.to_parquet(
+ "memory://fastparquet_user_agent.parquet",
+ index=False,
+ engine="fastparquet",
+ compression=None,
+ )
+ with fsspec.open("memory://fastparquet_user_agent.parquet", "rb") as f:
+ return f.read()
+
+
+def pickle_respnder(df):
+ with BytesIO() as bio:
+ df.to_pickle(bio)
+ return bio.getvalue()
+
+
+def stata_responder(df):
+ with BytesIO() as bio:
+ df.to_stata(bio, write_index=False)
+ return bio.getvalue()
+
+
+@pytest.mark.parametrize(
+ "responder, read_method",
+ [
+ (csv_responder, pd.read_csv),
+ (json_responder, pd.read_json),
+ (
+ html_responder,
+ lambda *args, **kwargs: pd.read_html(*args, **kwargs)[0],
+ ),
+ pytest.param(
+ parquetpyarrow_reponder,
+ partial(pd.read_parquet, engine="pyarrow"),
+ marks=td.skip_if_no("pyarrow"),
+ ),
+ pytest.param(
+ parquetfastparquet_responder,
+ partial(pd.read_parquet, engine="fastparquet"),
+ marks=[
+ td.skip_if_no("fastparquet"),
+ td.skip_if_no("fsspec"),
+ pytest.mark.xfail(using_string_dtype(), reason="TODO(infer_string"),
+ ],
+ ),
+ (pickle_respnder, pd.read_pickle),
+ (stata_responder, pd.read_stata),
+ (gz_csv_responder, pd.read_csv),
+ (gz_json_responder, pd.read_json),
+ ],
+)
+@pytest.mark.parametrize(
+ "storage_options",
+ [
+ None,
+ {"User-Agent": "foo"},
+ {"User-Agent": "foo", "Auth": "bar"},
+ ],
+)
+def test_request_headers(responder, read_method, httpserver, storage_options):
+ expected = pd.DataFrame({"a": ["b"]})
+ default_headers = ["Accept-Encoding", "Host", "Connection", "User-Agent"]
+ if "gz" in responder.__name__:
+ extra = {"Content-Encoding": "gzip"}
+ if storage_options is None:
+ storage_options = extra
+ else:
+ storage_options |= extra
+ else:
+ extra = None
+ expected_headers = set(default_headers).union(
+ storage_options.keys() if storage_options else []
+ )
+ httpserver.serve_content(content=responder(expected), headers=extra)
+ result = read_method(httpserver.url, storage_options=storage_options)
+ tm.assert_frame_equal(result, expected)
+
+ request_headers = dict(httpserver.requests[0].headers)
+ for header in expected_headers:
+ exp = request_headers.pop(header)
+ if storage_options and header in storage_options:
+ assert exp == storage_options[header]
+ # No extra headers added
+ assert not request_headers
+
+
+@pytest.mark.parametrize(
+ "engine",
+ [
+ "pyarrow",
+ "fastparquet",
+ ],
+)
+def test_to_parquet_to_disk_with_storage_options(engine):
+ headers = {
+ "User-Agent": "custom",
+ "Auth": "other_custom",
+ }
+
+ pytest.importorskip(engine)
+
+ true_df = pd.DataFrame({"column_name": ["column_value"]})
+ msg = (
+ "storage_options passed with file object or non-fsspec file path|"
+ "storage_options passed with buffer, or non-supported URL"
+ )
+ with pytest.raises(ValueError, match=msg):
+ true_df.to_parquet("/tmp/junk.parquet", storage_options=headers, engine=engine)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_iceberg.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_iceberg.py
new file mode 100644
index 0000000000000000000000000000000000000000..689eddb1985e6344d72c26fbb30a5237489cdadd
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_iceberg.py
@@ -0,0 +1,222 @@
+"""
+Tests for the Apache Iceberg format.
+
+Tests in this file use a simple Iceberg catalog based on SQLite, with the same
+data used for Parquet tests (``pandas/tests/io/data/parquet/simple.parquet``).
+"""
+
+import collections
+import importlib
+import pathlib
+
+import pytest
+
+import pandas as pd
+import pandas._testing as tm
+
+from pandas.io.iceberg import read_iceberg
+
+pytestmark = pytest.mark.single_cpu
+
+pyiceberg = pytest.importorskip("pyiceberg")
+pyiceberg_catalog = pytest.importorskip("pyiceberg.catalog")
+pq = pytest.importorskip("pyarrow.parquet")
+
+Catalog = collections.namedtuple("Catalog", ["name", "uri", "warehouse"])
+
+
+@pytest.fixture
+def catalog(request, tmp_path):
+ # the catalog stores the full path of data files, so the catalog needs to be
+ # created dynamically, and not saved in pandas/tests/io/data as other formats
+ uri = f"sqlite:///{tmp_path}/catalog.sqlite"
+ warehouse = f"file://{tmp_path}"
+ catalog_name = request.param if hasattr(request, "param") else None
+ catalog = pyiceberg_catalog.load_catalog(
+ catalog_name or "default",
+ type="sql",
+ uri=uri,
+ warehouse=warehouse,
+ )
+ catalog.create_namespace("ns")
+
+ df = pq.read_table(
+ pathlib.Path(__file__).parent / "data" / "parquet" / "simple.parquet"
+ )
+ table = catalog.create_table("ns.my_table", schema=df.schema)
+ table.append(df)
+
+ if catalog_name is not None:
+ config_path = pathlib.Path.home() / ".pyiceberg.yaml"
+ with open(config_path, "w", encoding="utf-8") as f:
+ f.write(f"""\
+catalog:
+ {catalog_name}:
+ type: sql
+ uri: {uri}
+ warehouse: {warehouse}""")
+
+ importlib.reload(pyiceberg_catalog) # needed to reload the config file
+
+ yield Catalog(name=catalog_name or "default", uri=uri, warehouse=warehouse)
+
+ if catalog_name is not None:
+ config_path.unlink()
+
+
+class TestIceberg:
+ def test_read(self, catalog):
+ expected = pd.DataFrame(
+ {
+ "A": [1, 2, 3],
+ "B": ["foo", "foo", "foo"],
+ }
+ )
+ result = read_iceberg(
+ "ns.my_table",
+ catalog_properties={"uri": catalog.uri},
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("catalog", ["default", "pandas_tests"], indirect=True)
+ def test_read_by_catalog_name(self, catalog):
+ expected = pd.DataFrame(
+ {
+ "A": [1, 2, 3],
+ "B": ["foo", "foo", "foo"],
+ }
+ )
+ result = read_iceberg(
+ "ns.my_table",
+ catalog_name=catalog.name,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_read_with_row_filter(self, catalog):
+ expected = pd.DataFrame(
+ {
+ "A": [2, 3],
+ "B": ["foo", "foo"],
+ }
+ )
+ result = read_iceberg(
+ "ns.my_table",
+ catalog_properties={"uri": catalog.uri},
+ row_filter="A > 1",
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_read_with_case_sensitive(self, catalog):
+ expected = pd.DataFrame(
+ {
+ "A": [1, 2, 3],
+ }
+ )
+ result = read_iceberg(
+ "ns.my_table",
+ catalog_properties={"uri": catalog.uri},
+ columns=["a"],
+ case_sensitive=False,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ with pytest.raises(ValueError, match="^Could not find column"):
+ read_iceberg(
+ "ns.my_table",
+ catalog_properties={"uri": catalog.uri},
+ columns=["a"],
+ case_sensitive=True,
+ )
+
+ def test_read_with_limit(self, catalog):
+ expected = pd.DataFrame(
+ {
+ "A": [1, 2],
+ "B": ["foo", "foo"],
+ }
+ )
+ result = read_iceberg(
+ "ns.my_table",
+ catalog_properties={"uri": catalog.uri},
+ limit=2,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_write(self, catalog):
+ df = pd.DataFrame(
+ {
+ "A": [1, 2, 3],
+ "B": ["foo", "foo", "foo"],
+ }
+ )
+ df.to_iceberg(
+ "ns.new_table",
+ catalog_properties={"uri": catalog.uri},
+ location=catalog.warehouse,
+ )
+ result = read_iceberg(
+ "ns.new_table",
+ catalog_properties={"uri": catalog.uri},
+ )
+ tm.assert_frame_equal(result, df)
+
+ @pytest.mark.parametrize("catalog", ["default", "pandas_tests"], indirect=True)
+ def test_write_by_catalog_name(self, catalog):
+ df = pd.DataFrame(
+ {
+ "A": [1, 2, 3],
+ "B": ["foo", "foo", "foo"],
+ }
+ )
+ df.to_iceberg(
+ "ns.new_table",
+ catalog_name=catalog.name,
+ )
+ result = read_iceberg(
+ "ns.new_table",
+ catalog_name=catalog.name,
+ )
+ tm.assert_frame_equal(result, df)
+
+ def test_write_existing_table_with_append_true(self, catalog):
+ original = read_iceberg(
+ "ns.my_table",
+ catalog_properties={"uri": catalog.uri},
+ )
+ new = pd.DataFrame(
+ {
+ "A": [1, 2, 3],
+ "B": ["foo", "foo", "foo"],
+ }
+ )
+ expected = pd.concat([original, new], ignore_index=True)
+ new.to_iceberg(
+ "ns.my_table",
+ catalog_properties={"uri": catalog.uri},
+ location=catalog.warehouse,
+ append=True,
+ )
+ result = read_iceberg(
+ "ns.my_table",
+ catalog_properties={"uri": catalog.uri},
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_write_existing_table_with_append_false(self, catalog):
+ df = pd.DataFrame(
+ {
+ "A": [1, 2, 3],
+ "B": ["foo", "foo", "foo"],
+ }
+ )
+ df.to_iceberg(
+ "ns.my_table",
+ catalog_properties={"uri": catalog.uri},
+ location=catalog.warehouse,
+ append=False,
+ )
+ result = read_iceberg(
+ "ns.my_table",
+ catalog_properties={"uri": catalog.uri},
+ )
+ tm.assert_frame_equal(result, df)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_orc.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_orc.py
new file mode 100644
index 0000000000000000000000000000000000000000..2e61494103355c5836f3ad0e9187d712093610dc
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_orc.py
@@ -0,0 +1,432 @@
+"""test orc compat"""
+
+import datetime
+from decimal import Decimal
+from io import BytesIO
+import os
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import read_orc
+import pandas._testing as tm
+
+pytest.importorskip("pyarrow.orc")
+
+import pyarrow as pa
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+@pytest.fixture
+def dirpath(datapath):
+ return datapath("io", "data", "orc")
+
+
+def test_orc_reader_empty(dirpath, using_infer_string):
+ columns = [
+ "boolean1",
+ "byte1",
+ "short1",
+ "int1",
+ "long1",
+ "float1",
+ "double1",
+ "bytes1",
+ "string1",
+ ]
+ dtypes = [
+ "bool",
+ "int8",
+ "int16",
+ "int32",
+ "int64",
+ "float32",
+ "float64",
+ "object",
+ "str" if using_infer_string else "object",
+ ]
+ expected = pd.DataFrame(index=pd.RangeIndex(0))
+ for colname, dtype in zip(columns, dtypes, strict=True):
+ expected[colname] = pd.Series(dtype=dtype)
+ expected.columns = expected.columns.astype("str")
+
+ inputfile = os.path.join(dirpath, "TestOrcFile.emptyFile.orc")
+ got = read_orc(inputfile, columns=columns)
+
+ tm.assert_equal(expected, got)
+
+
+def test_orc_reader_basic(dirpath):
+ data = {
+ "boolean1": np.array([False, True], dtype="bool"),
+ "byte1": np.array([1, 100], dtype="int8"),
+ "short1": np.array([1024, 2048], dtype="int16"),
+ "int1": np.array([65536, 65536], dtype="int32"),
+ "long1": np.array([9223372036854775807, 9223372036854775807], dtype="int64"),
+ "float1": np.array([1.0, 2.0], dtype="float32"),
+ "double1": np.array([-15.0, -5.0], dtype="float64"),
+ "bytes1": np.array([b"\x00\x01\x02\x03\x04", b""], dtype="object"),
+ "string1": np.array(["hi", "bye"], dtype="object"),
+ }
+ expected = pd.DataFrame.from_dict(data)
+
+ inputfile = os.path.join(dirpath, "TestOrcFile.test1.orc")
+ got = read_orc(inputfile, columns=data.keys())
+
+ tm.assert_equal(expected, got)
+
+
+def test_orc_reader_decimal(dirpath):
+ # Only testing the first 10 rows of data
+ data = {
+ "_col0": np.array(
+ [
+ Decimal("-1000.50000"),
+ Decimal("-999.60000"),
+ Decimal("-998.70000"),
+ Decimal("-997.80000"),
+ Decimal("-996.90000"),
+ Decimal("-995.10000"),
+ Decimal("-994.11000"),
+ Decimal("-993.12000"),
+ Decimal("-992.13000"),
+ Decimal("-991.14000"),
+ ],
+ dtype="object",
+ )
+ }
+ expected = pd.DataFrame.from_dict(data)
+
+ inputfile = os.path.join(dirpath, "TestOrcFile.decimal.orc")
+ got = read_orc(inputfile).iloc[:10]
+
+ tm.assert_equal(expected, got)
+
+
+def test_orc_reader_date_low(dirpath):
+ data = {
+ "time": np.array(
+ [
+ "1900-05-05 12:34:56.100000",
+ "1900-05-05 12:34:56.100100",
+ "1900-05-05 12:34:56.100200",
+ "1900-05-05 12:34:56.100300",
+ "1900-05-05 12:34:56.100400",
+ "1900-05-05 12:34:56.100500",
+ "1900-05-05 12:34:56.100600",
+ "1900-05-05 12:34:56.100700",
+ "1900-05-05 12:34:56.100800",
+ "1900-05-05 12:34:56.100900",
+ ],
+ dtype="datetime64[ns]",
+ ),
+ "date": np.array(
+ [
+ datetime.date(1900, 12, 25),
+ datetime.date(1900, 12, 25),
+ datetime.date(1900, 12, 25),
+ datetime.date(1900, 12, 25),
+ datetime.date(1900, 12, 25),
+ datetime.date(1900, 12, 25),
+ datetime.date(1900, 12, 25),
+ datetime.date(1900, 12, 25),
+ datetime.date(1900, 12, 25),
+ datetime.date(1900, 12, 25),
+ ],
+ dtype="object",
+ ),
+ }
+ expected = pd.DataFrame.from_dict(data)
+
+ inputfile = os.path.join(dirpath, "TestOrcFile.testDate1900.orc")
+ got = read_orc(inputfile).iloc[:10]
+
+ tm.assert_equal(expected, got)
+
+
+def test_orc_reader_date_high(dirpath):
+ data = {
+ "time": np.array(
+ [
+ "2038-05-05 12:34:56.100000",
+ "2038-05-05 12:34:56.100100",
+ "2038-05-05 12:34:56.100200",
+ "2038-05-05 12:34:56.100300",
+ "2038-05-05 12:34:56.100400",
+ "2038-05-05 12:34:56.100500",
+ "2038-05-05 12:34:56.100600",
+ "2038-05-05 12:34:56.100700",
+ "2038-05-05 12:34:56.100800",
+ "2038-05-05 12:34:56.100900",
+ ],
+ dtype="datetime64[ns]",
+ ),
+ "date": np.array(
+ [
+ datetime.date(2038, 12, 25),
+ datetime.date(2038, 12, 25),
+ datetime.date(2038, 12, 25),
+ datetime.date(2038, 12, 25),
+ datetime.date(2038, 12, 25),
+ datetime.date(2038, 12, 25),
+ datetime.date(2038, 12, 25),
+ datetime.date(2038, 12, 25),
+ datetime.date(2038, 12, 25),
+ datetime.date(2038, 12, 25),
+ ],
+ dtype="object",
+ ),
+ }
+ expected = pd.DataFrame.from_dict(data)
+
+ inputfile = os.path.join(dirpath, "TestOrcFile.testDate2038.orc")
+ got = read_orc(inputfile).iloc[:10]
+
+ tm.assert_equal(expected, got)
+
+
+def test_orc_reader_snappy_compressed(dirpath):
+ data = {
+ "int1": np.array(
+ [
+ -1160101563,
+ 1181413113,
+ 2065821249,
+ -267157795,
+ 172111193,
+ 1752363137,
+ 1406072123,
+ 1911809390,
+ -1308542224,
+ -467100286,
+ ],
+ dtype="int32",
+ ),
+ "string1": np.array(
+ [
+ "f50dcb8",
+ "382fdaaa",
+ "90758c6",
+ "9e8caf3f",
+ "ee97332b",
+ "d634da1",
+ "2bea4396",
+ "d67d89e8",
+ "ad71007e",
+ "e8c82066",
+ ],
+ dtype="object",
+ ),
+ }
+ expected = pd.DataFrame.from_dict(data)
+
+ inputfile = os.path.join(dirpath, "TestOrcFile.testSnappy.orc")
+ got = read_orc(inputfile).iloc[:10]
+
+ tm.assert_equal(expected, got)
+
+
+def test_orc_roundtrip_file(dirpath, temp_file):
+ # GH44554
+ # PyArrow gained ORC write support with the current argument order
+ pytest.importorskip("pyarrow")
+
+ data = {
+ "boolean1": np.array([False, True], dtype="bool"),
+ "byte1": np.array([1, 100], dtype="int8"),
+ "short1": np.array([1024, 2048], dtype="int16"),
+ "int1": np.array([65536, 65536], dtype="int32"),
+ "long1": np.array([9223372036854775807, 9223372036854775807], dtype="int64"),
+ "float1": np.array([1.0, 2.0], dtype="float32"),
+ "double1": np.array([-15.0, -5.0], dtype="float64"),
+ "bytes1": np.array([b"\x00\x01\x02\x03\x04", b""], dtype="object"),
+ "string1": np.array(["hi", "bye"], dtype="object"),
+ }
+ expected = pd.DataFrame.from_dict(data)
+
+ expected.to_orc(temp_file)
+ got = read_orc(temp_file)
+
+ tm.assert_equal(expected, got)
+
+
+def test_orc_roundtrip_bytesio():
+ # GH44554
+ # PyArrow gained ORC write support with the current argument order
+ pytest.importorskip("pyarrow")
+
+ data = {
+ "boolean1": np.array([False, True], dtype="bool"),
+ "byte1": np.array([1, 100], dtype="int8"),
+ "short1": np.array([1024, 2048], dtype="int16"),
+ "int1": np.array([65536, 65536], dtype="int32"),
+ "long1": np.array([9223372036854775807, 9223372036854775807], dtype="int64"),
+ "float1": np.array([1.0, 2.0], dtype="float32"),
+ "double1": np.array([-15.0, -5.0], dtype="float64"),
+ "bytes1": np.array([b"\x00\x01\x02\x03\x04", b""], dtype="object"),
+ "string1": np.array(["hi", "bye"], dtype="object"),
+ }
+ expected = pd.DataFrame.from_dict(data)
+
+ bytes = expected.to_orc()
+ got = read_orc(BytesIO(bytes))
+
+ tm.assert_equal(expected, got)
+
+
+@pytest.mark.parametrize(
+ "orc_writer_dtypes_not_supported",
+ [
+ np.array([1, 20], dtype="uint64"),
+ pd.Series(["a", "b", "a"], dtype="category"),
+ [pd.Interval(left=0, right=2), pd.Interval(left=0, right=5)],
+ [pd.Period("2022-01-03", freq="D"), pd.Period("2022-01-04", freq="D")],
+ ],
+)
+def test_orc_writer_dtypes_not_supported(orc_writer_dtypes_not_supported):
+ # GH44554
+ # PyArrow gained ORC write support with the current argument order
+ pytest.importorskip("pyarrow")
+
+ df = pd.DataFrame({"unimpl": orc_writer_dtypes_not_supported})
+ msg = "The dtype of one or more columns is not supported yet."
+ with pytest.raises(NotImplementedError, match=msg):
+ df.to_orc()
+
+
+def test_orc_dtype_backend_pyarrow(using_infer_string):
+ pytest.importorskip("pyarrow")
+ df = pd.DataFrame(
+ {
+ "string": list("abc"),
+ "string_with_nan": ["a", np.nan, "c"],
+ "string_with_none": ["a", None, "c"],
+ "bytes": [b"foo", b"bar", None],
+ "int": list(range(1, 4)),
+ "float": np.arange(4.0, 7.0, dtype="float64"),
+ "float_with_nan": [2.0, np.nan, 3.0],
+ "bool": [True, False, True],
+ "bool_with_na": [True, False, None],
+ "datetime": pd.date_range("20130101", periods=3, unit="ns"),
+ "datetime_with_nat": [
+ pd.Timestamp("20130101"),
+ pd.NaT,
+ pd.Timestamp("20130103"),
+ ],
+ }
+ )
+ # FIXME: without casting to ns we do not round-trip correctly
+ df["datetime_with_nat"] = df["datetime_with_nat"].astype("M8[ns]")
+
+ bytes_data = df.copy().to_orc()
+ result = read_orc(BytesIO(bytes_data), dtype_backend="pyarrow")
+
+ expected = pd.DataFrame(
+ {
+ col: pd.arrays.ArrowExtensionArray(pa.array(df[col], from_pandas=True))
+ for col in df.columns
+ }
+ )
+ if using_infer_string:
+ # ORC does not preserve distinction between string and large string
+ # -> the default large string comes back as string
+ string_dtype = pd.ArrowDtype(pa.string())
+ expected["string"] = expected["string"].astype(string_dtype)
+ expected["string_with_nan"] = expected["string_with_nan"].astype(string_dtype)
+ expected["string_with_none"] = expected["string_with_none"].astype(string_dtype)
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_orc_dtype_backend_numpy_nullable():
+ # GH#50503
+ pytest.importorskip("pyarrow")
+ df = pd.DataFrame(
+ {
+ "string": list("abc"),
+ "string_with_nan": ["a", np.nan, "c"],
+ "string_with_none": ["a", None, "c"],
+ "int": list(range(1, 4)),
+ "int_with_nan": pd.Series([1, pd.NA, 3], dtype="Int64"),
+ "na_only": pd.Series([pd.NA, pd.NA, pd.NA], dtype="Int64"),
+ "float": np.arange(4.0, 7.0, dtype="float64"),
+ "float_with_nan": [2.0, np.nan, 3.0],
+ "bool": [True, False, True],
+ "bool_with_na": [True, False, None],
+ }
+ )
+
+ bytes_data = df.copy().to_orc()
+ result = read_orc(BytesIO(bytes_data), dtype_backend="numpy_nullable")
+
+ expected = pd.DataFrame(
+ {
+ "string": pd.array(["a", "b", "c"], dtype=pd.StringDtype()),
+ "string_with_nan": pd.array(["a", pd.NA, "c"], dtype=pd.StringDtype()),
+ "string_with_none": pd.array(["a", pd.NA, "c"], dtype=pd.StringDtype()),
+ "int": pd.Series([1, 2, 3], dtype="Int64"),
+ "int_with_nan": pd.Series([1, pd.NA, 3], dtype="Int64"),
+ "na_only": pd.Series([pd.NA, pd.NA, pd.NA], dtype="Int64"),
+ "float": pd.Series([4.0, 5.0, 6.0], dtype="Float64"),
+ "float_with_nan": pd.Series([2.0, pd.NA, 3.0], dtype="Float64"),
+ "bool": pd.Series([True, False, True], dtype="boolean"),
+ "bool_with_na": pd.Series([True, False, pd.NA], dtype="boolean"),
+ }
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_orc_uri_path(temp_file):
+ expected = pd.DataFrame({"int": list(range(1, 4))})
+ expected.to_orc(temp_file)
+ uri = temp_file.as_uri()
+ result = read_orc(uri)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "index",
+ [
+ pd.RangeIndex(start=2, stop=5, step=1),
+ pd.RangeIndex(start=0, stop=3, step=1, name="non-default"),
+ pd.Index([1, 2, 3]),
+ ],
+)
+def test_to_orc_non_default_index(index):
+ df = pd.DataFrame({"a": [1, 2, 3]}, index=index)
+ msg = (
+ "orc does not support serializing a non-default index|"
+ "orc does not serialize index meta-data"
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.to_orc()
+
+
+def test_invalid_dtype_backend(temp_file):
+ msg = (
+ "dtype_backend numpy is invalid, only 'numpy_nullable' and "
+ "'pyarrow' are allowed."
+ )
+ df = pd.DataFrame({"int": list(range(1, 4))})
+ df.to_orc(temp_file)
+ with pytest.raises(ValueError, match=msg):
+ read_orc(temp_file, dtype_backend="numpy")
+
+
+def test_string_inference(temp_file):
+ # GH#54431
+ df = pd.DataFrame(data={"a": ["x", "y"]})
+ df.to_orc(temp_file)
+ with pd.option_context("future.infer_string", True):
+ result = read_orc(temp_file)
+ expected = pd.DataFrame(
+ data={"a": ["x", "y"]},
+ dtype=pd.StringDtype(na_value=np.nan),
+ columns=pd.Index(["a"], dtype=pd.StringDtype(na_value=np.nan)),
+ )
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_parquet.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_parquet.py
new file mode 100644
index 0000000000000000000000000000000000000000..e6f37e452d0745efe36554d6b3759367617140ee
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_parquet.py
@@ -0,0 +1,1480 @@
+"""test parquet compat"""
+
+import datetime
+from decimal import Decimal
+from io import BytesIO
+import os
+import pathlib
+
+import numpy as np
+import pytest
+
+from pandas._config import using_string_dtype
+
+from pandas.compat import is_platform_windows
+from pandas.compat.pyarrow import (
+ pa_version_under15p0,
+ pa_version_under17p0,
+ pa_version_under19p0,
+ pa_version_under20p0,
+)
+
+import pandas as pd
+import pandas._testing as tm
+from pandas.util.version import Version
+
+from pandas.io.parquet import (
+ FastParquetImpl,
+ PyArrowImpl,
+ get_engine,
+ read_parquet,
+ to_parquet,
+)
+
+try:
+ import pyarrow
+
+ _HAVE_PYARROW = True
+except ImportError:
+ _HAVE_PYARROW = False
+
+try:
+ import fastparquet
+
+ _HAVE_FASTPARQUET = True
+ _fp_version_lt_2025 = Version(fastparquet.__version__) < Version("2025.12.0")
+ _fp_version_lt_2026_5 = Version(fastparquet.__version__) < Version("2026.5.0")
+except ImportError:
+ _HAVE_FASTPARQUET = False
+ _fp_version_lt_2025 = False
+ _fp_version_lt_2026_5 = False
+
+
+pytestmark = [
+ pytest.mark.filterwarnings("ignore:DataFrame._data is deprecated:FutureWarning"),
+ pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+ ),
+]
+
+
+# setup engines & skips
+@pytest.fixture(
+ params=[
+ pytest.param(
+ "fastparquet",
+ marks=[
+ pytest.mark.skipif(
+ not _HAVE_FASTPARQUET,
+ reason="fastparquet is not installed",
+ ),
+ pytest.mark.xfail(
+ using_string_dtype(),
+ reason="TODO(infer_string) fastparquet",
+ strict=False,
+ ),
+ ],
+ ),
+ pytest.param(
+ "pyarrow",
+ marks=pytest.mark.skipif(
+ not _HAVE_PYARROW, reason="pyarrow is not installed"
+ ),
+ ),
+ ]
+)
+def engine(request):
+ return request.param
+
+
+@pytest.fixture
+def pa():
+ if not _HAVE_PYARROW:
+ pytest.skip("pyarrow is not installed")
+ return "pyarrow"
+
+
+@pytest.fixture
+def fp(request):
+ if not _HAVE_FASTPARQUET:
+ pytest.skip("fastparquet is not installed")
+ if using_string_dtype():
+ request.applymarker(
+ pytest.mark.xfail(reason="TODO(infer_string) fastparquet", strict=False)
+ )
+ return "fastparquet"
+
+
+@pytest.fixture
+def df_compat():
+ return pd.DataFrame({"A": [1, 2, 3], "B": "foo"}, columns=pd.Index(["A", "B"]))
+
+
+@pytest.fixture
+def df_cross_compat():
+ df = pd.DataFrame(
+ {
+ "a": list("abc"),
+ "b": list(range(1, 4)),
+ # 'c': np.arange(3, 6).astype('u1'),
+ "d": np.arange(4.0, 7.0, dtype="float64"),
+ "e": [True, False, True],
+ "f": pd.date_range("20130101", periods=3),
+ # 'g': pd.date_range('20130101', periods=3,
+ # tz='US/Eastern'),
+ # 'h': pd.date_range('20130101', periods=3, freq='ns')
+ }
+ )
+ return df
+
+
+@pytest.fixture
+def df_full():
+ return pd.DataFrame(
+ {
+ "string": list("abc"),
+ "string_with_nan": ["a", np.nan, "c"],
+ "string_with_none": ["a", None, "c"],
+ "bytes": [b"foo", b"bar", b"baz"],
+ "unicode": ["foo", "bar", "baz"],
+ "int": list(range(1, 4)),
+ "uint": np.arange(3, 6).astype("u1"),
+ "float": np.arange(4.0, 7.0, dtype="float64"),
+ "float_with_nan": [2.0, np.nan, 3.0],
+ "bool": [True, False, True],
+ "datetime": pd.date_range("20130101", periods=3, unit="ns"),
+ "datetime_with_nat": [
+ pd.Timestamp("20130101"),
+ pd.NaT,
+ pd.Timestamp("20130103"),
+ ],
+ }
+ )
+
+
+@pytest.fixture(
+ params=[
+ datetime.datetime.now(datetime.UTC),
+ datetime.datetime.now(datetime.timezone.min),
+ datetime.datetime.now(datetime.timezone.max),
+ datetime.datetime.strptime("2019-01-04T16:41:24+0200", "%Y-%m-%dT%H:%M:%S%z"),
+ datetime.datetime.strptime("2019-01-04T16:41:24+0215", "%Y-%m-%dT%H:%M:%S%z"),
+ datetime.datetime.strptime("2019-01-04T16:41:24-0200", "%Y-%m-%dT%H:%M:%S%z"),
+ datetime.datetime.strptime("2019-01-04T16:41:24-0215", "%Y-%m-%dT%H:%M:%S%z"),
+ ]
+)
+def timezone_aware_date_list(request):
+ return request.param
+
+
+def check_round_trip(
+ df,
+ temp_file,
+ engine=None,
+ path=None,
+ write_kwargs=None,
+ read_kwargs=None,
+ expected=None,
+ check_names=True,
+ check_like=False,
+ check_dtype=True,
+ repeat=2,
+):
+ """Verify parquet serializer and deserializer produce the same results.
+
+ Performs a pandas to disk and disk to pandas round trip,
+ then compares the 2 resulting DataFrames to verify equality.
+
+ Parameters
+ ----------
+ df: Dataframe
+ engine: str, optional
+ 'pyarrow' or 'fastparquet'
+ path: str, optional
+ write_kwargs: dict of str:str, optional
+ read_kwargs: dict of str:str, optional
+ expected: DataFrame, optional
+ Expected deserialization result, otherwise will be equal to `df`
+ check_names: list of str, optional
+ Closed set of column names to be compared
+ check_like: bool, optional
+ If True, ignore the order of index & columns.
+ repeat: int, optional
+ How many times to repeat the test
+ """
+ if not isinstance(temp_file, pathlib.Path):
+ raise ValueError("temp_file must be a pathlib.Path")
+ write_kwargs = write_kwargs or {"compression": None}
+ read_kwargs = read_kwargs or {}
+
+ if expected is None:
+ expected = df
+
+ if engine:
+ write_kwargs["engine"] = engine
+ read_kwargs["engine"] = engine
+
+ def compare(repeat):
+ for _ in range(repeat):
+ df.to_parquet(path, **write_kwargs)
+ actual = read_parquet(path, **read_kwargs)
+
+ if "string_with_nan" in expected:
+ expected.loc[1, "string_with_nan"] = None
+ tm.assert_frame_equal(
+ actual,
+ expected,
+ check_names=check_names,
+ check_like=check_like,
+ check_dtype=check_dtype,
+ )
+
+ if path is None:
+ path = temp_file
+ compare(repeat)
+ else:
+ compare(repeat)
+
+
+def check_partition_names(path, expected):
+ """Check partitions of a parquet file are as expected.
+
+ Parameters
+ ----------
+ path: str
+ Path of the dataset.
+ expected: iterable of str
+ Expected partition names.
+ """
+ import pyarrow.dataset as ds
+
+ dataset = ds.dataset(path, partitioning="hive")
+ assert dataset.partitioning.schema.names == expected
+
+
+def test_invalid_engine(df_compat, temp_file):
+ msg = "engine must be one of 'pyarrow', 'fastparquet'"
+ with pytest.raises(ValueError, match=msg):
+ check_round_trip(df_compat, temp_file, "foo", "bar")
+
+
+def test_options_py(df_compat, pa, using_infer_string, temp_file):
+ # use the set option
+ if using_infer_string and not pa_version_under19p0:
+ df_compat.columns = df_compat.columns.astype("str")
+
+ with pd.option_context("io.parquet.engine", "pyarrow"):
+ check_round_trip(df_compat, temp_file)
+
+
+def test_options_fp(df_compat, fp, temp_file):
+ # use the set option
+
+ with pd.option_context("io.parquet.engine", "fastparquet"):
+ check_round_trip(df_compat, temp_file)
+
+
+def test_options_auto(df_compat, fp, pa, temp_file):
+ # use the set option
+
+ with pd.option_context("io.parquet.engine", "auto"):
+ check_round_trip(df_compat, temp_file)
+
+
+def test_options_get_engine(fp, pa):
+ assert isinstance(get_engine("pyarrow"), PyArrowImpl)
+ assert isinstance(get_engine("fastparquet"), FastParquetImpl)
+
+ with pd.option_context("io.parquet.engine", "pyarrow"):
+ assert isinstance(get_engine("auto"), PyArrowImpl)
+ assert isinstance(get_engine("pyarrow"), PyArrowImpl)
+ assert isinstance(get_engine("fastparquet"), FastParquetImpl)
+
+ with pd.option_context("io.parquet.engine", "fastparquet"):
+ assert isinstance(get_engine("auto"), FastParquetImpl)
+ assert isinstance(get_engine("pyarrow"), PyArrowImpl)
+ assert isinstance(get_engine("fastparquet"), FastParquetImpl)
+
+ with pd.option_context("io.parquet.engine", "auto"):
+ assert isinstance(get_engine("auto"), PyArrowImpl)
+ assert isinstance(get_engine("pyarrow"), PyArrowImpl)
+ assert isinstance(get_engine("fastparquet"), FastParquetImpl)
+
+
+def test_get_engine_auto_error_message():
+ # Expect different error messages from get_engine(engine="auto")
+ # if engines aren't installed vs. are installed but bad version
+ from pandas.compat._optional import VERSIONS
+
+ # Do we have engines installed, but a bad version of them?
+ pa_min_ver = VERSIONS.get("pyarrow")
+ fp_min_ver = VERSIONS.get("fastparquet")
+ have_pa_bad_version = (
+ False
+ if not _HAVE_PYARROW
+ else Version(pyarrow.__version__) < Version(pa_min_ver)
+ )
+ have_fp_bad_version = (
+ False
+ if not _HAVE_FASTPARQUET
+ else Version(fastparquet.__version__) < Version(fp_min_ver)
+ )
+ # Do we have usable engines installed?
+ have_usable_pa = _HAVE_PYARROW and not have_pa_bad_version
+ have_usable_fp = _HAVE_FASTPARQUET and not have_fp_bad_version
+
+ if not have_usable_pa and not have_usable_fp:
+ # No usable engines found.
+ if have_pa_bad_version:
+ match = f"Pandas requires version .{pa_min_ver}. or newer of .pyarrow."
+ with pytest.raises(ImportError, match=match):
+ get_engine("auto")
+ else:
+ match = "Unable to find a usable engine; tried using: 'pyarrow'"
+ with pytest.raises(ImportError, match=match):
+ get_engine("auto")
+
+ if have_fp_bad_version:
+ match = f"Pandas requires version .{fp_min_ver}. or newer of .fastparquet."
+ with pytest.raises(ImportError, match=match):
+ get_engine("auto")
+ else:
+ match = "Use pip or conda to install the fastparquet package"
+ with pytest.raises(ImportError, match=match):
+ get_engine("auto")
+
+
+def test_cross_engine_pa_fp(df_cross_compat, pa, fp, temp_file):
+ # cross-compat with differing reading/writing engines
+
+ df = df_cross_compat
+ df.to_parquet(temp_file, engine=pa, compression=None)
+
+ result = read_parquet(temp_file, engine=fp)
+ tm.assert_frame_equal(result, df)
+
+ result = read_parquet(temp_file, engine=fp, columns=["a", "d"])
+ tm.assert_frame_equal(result, df[["a", "d"]])
+
+
+@pytest.mark.xfail(
+ using_string_dtype()
+ and _HAVE_PYARROW
+ and not _fp_version_lt_2025
+ and _fp_version_lt_2026_5,
+ reason="fastparquet >= 2025.12.0, < 2026.5.0 can't write ArrowStringArray",
+)
+def test_cross_engine_fp_pa(df_cross_compat, pa, fp, temp_file):
+ # cross-compat with differing reading/writing engines
+ df = df_cross_compat
+
+ df.to_parquet(temp_file, engine=fp, compression=None)
+
+ result = read_parquet(temp_file, engine=pa)
+ tm.assert_frame_equal(result, df)
+
+ result = read_parquet(temp_file, engine=pa, columns=["a", "d"])
+ tm.assert_frame_equal(result, df[["a", "d"]])
+
+
+class Base:
+ def check_error_on_write(self, df, engine, exc, err_msg, temp_file_path):
+ # check that we are raising the exception on writing
+ with pytest.raises(exc, match=err_msg):
+ to_parquet(df, temp_file_path, engine, compression=None)
+
+ def check_external_error_on_write(self, df, engine, exc, temp_file_path):
+ # check that an external library is raising the exception on writing
+ with tm.external_error_raised(exc):
+ to_parquet(df, temp_file_path, engine, compression=None)
+
+
+class TestBasic(Base):
+ def test_error(self, engine, temp_file):
+ for obj in [
+ pd.Series([1, 2, 3]),
+ 1,
+ "foo",
+ pd.Timestamp("20130101"),
+ np.array([1, 2, 3]),
+ ]:
+ msg = "to_parquet only supports IO with DataFrames"
+ self.check_error_on_write(obj, engine, ValueError, msg, temp_file)
+
+ def test_columns_dtypes(self, engine, temp_file):
+ df = pd.DataFrame({"string": list("abc"), "int": list(range(1, 4))})
+
+ # unicode
+ df.columns = ["foo", "bar"]
+ check_round_trip(df, temp_file, engine)
+
+ @pytest.mark.parametrize("compression", [None, "gzip", "snappy", "brotli"])
+ def test_compression(self, engine, compression, temp_file):
+ df = pd.DataFrame({"A": [1, 2, 3]})
+ check_round_trip(
+ df, temp_file, engine, write_kwargs={"compression": compression}
+ )
+
+ def test_read_columns(self, engine, temp_file):
+ # GH18154
+ df = pd.DataFrame({"string": list("abc"), "int": list(range(1, 4))})
+
+ expected = pd.DataFrame({"string": list("abc")})
+ check_round_trip(
+ df,
+ temp_file,
+ engine,
+ expected=expected,
+ read_kwargs={"columns": ["string"]},
+ )
+
+ def test_read_filters(self, engine, tmp_path):
+ df = pd.DataFrame(
+ {
+ "int": list(range(4)),
+ "part": list("aabb"),
+ }
+ )
+
+ expected = pd.DataFrame({"int": [0, 1]})
+ check_round_trip(
+ df,
+ tmp_path,
+ engine,
+ expected=expected,
+ write_kwargs={"partition_cols": ["part"]},
+ read_kwargs={"filters": [("part", "==", "a")], "columns": ["int"]},
+ repeat=1,
+ )
+
+ def test_write_index(self, temp_file):
+ pytest.importorskip("pyarrow")
+ df = pd.DataFrame({"A": [1, 2, 3]})
+ check_round_trip(df, temp_file, "pyarrow")
+
+ indexes = [
+ [2, 3, 4],
+ pd.date_range("20130101", periods=3, unit="ns"),
+ list("abc"),
+ [1, 3, 4],
+ ]
+ # non-default index
+ for index in indexes:
+ df.index = index
+ if isinstance(index, pd.DatetimeIndex):
+ df.index = df.index._with_freq(None) # freq doesn't round-trip
+ check_round_trip(df, temp_file, "pyarrow")
+
+ # index with meta-data
+ df.index = [0, 1, 2]
+ df.index.name = "foo"
+ check_round_trip(df, temp_file, "pyarrow")
+
+ def test_write_multiindex(self, pa, temp_file):
+ # Not supported in fastparquet as of 0.1.3 or older pyarrow version
+ engine = pa
+
+ df = pd.DataFrame({"A": [1, 2, 3]})
+ index = pd.MultiIndex.from_tuples([("a", 1), ("a", 2), ("b", 1)])
+ df.index = index
+ check_round_trip(df, temp_file, engine)
+
+ def test_multiindex_with_columns(self, pa, temp_file):
+ engine = pa
+ dates = pd.date_range("01-Jan-2018", "01-Dec-2018", freq="MS", unit="ns")
+ df = pd.DataFrame(
+ np.random.default_rng(2).standard_normal((2 * len(dates), 3)),
+ columns=list("ABC"),
+ )
+ index1 = pd.MultiIndex.from_product(
+ [["Level1", "Level2"], dates], names=["level", "date"]
+ )
+ index2 = index1.copy(names=None)
+ for index in [index1, index2]:
+ df.index = index
+
+ check_round_trip(df, temp_file, engine)
+ check_round_trip(
+ df,
+ temp_file,
+ engine,
+ read_kwargs={"columns": ["A", "B"]},
+ expected=df[["A", "B"]],
+ )
+
+ def test_write_ignoring_index(self, engine, temp_file):
+ # ENH 20768
+ # Ensure index=False omits the index from the written Parquet file.
+ df = pd.DataFrame({"a": [1, 2, 3], "b": ["q", "r", "s"]})
+
+ write_kwargs = {"compression": None, "index": False}
+
+ # Because we're dropping the index, we expect the loaded dataframe to
+ # have the default integer index.
+ expected = df.reset_index(drop=True)
+
+ check_round_trip(
+ df, temp_file, engine, write_kwargs=write_kwargs, expected=expected
+ )
+
+ # Ignore custom index
+ df = pd.DataFrame(
+ {"a": [1, 2, 3], "b": ["q", "r", "s"]}, index=["zyx", "wvu", "tsr"]
+ )
+
+ check_round_trip(
+ df, temp_file, engine, write_kwargs=write_kwargs, expected=expected
+ )
+
+ # Ignore multi-indexes as well.
+ arrays = [
+ ["bar", "bar", "baz", "baz", "foo", "foo", "qux", "qux"],
+ ["one", "two", "one", "two", "one", "two", "one", "two"],
+ ]
+ df = pd.DataFrame(
+ {"one": list(range(8)), "two": [-i for i in range(8)]}, index=arrays
+ )
+
+ expected = df.reset_index(drop=True)
+ check_round_trip(
+ df, temp_file, engine, write_kwargs=write_kwargs, expected=expected
+ )
+
+ def test_write_column_multiindex(self, engine, temp_file):
+ # Not able to write column multi-indexes with non-string column names.
+ mi_columns = pd.MultiIndex.from_tuples([("a", 1), ("a", 2), ("b", 1)])
+ df = pd.DataFrame(
+ np.random.default_rng(2).standard_normal((4, 3)), columns=mi_columns
+ )
+
+ if engine == "fastparquet":
+ self.check_error_on_write(
+ df, engine, TypeError, "Column name must be a string", temp_file
+ )
+ elif engine == "pyarrow":
+ check_round_trip(df, temp_file, engine)
+
+ def test_write_column_multiindex_nonstring(self, engine, temp_file):
+ # GH #34777
+
+ # Not able to write column multi-indexes with non-string column names
+ arrays = [
+ ["bar", "bar", "baz", "baz", "foo", "foo", "qux", "qux"],
+ [1, 2, 1, 2, 1, 2, 1, 2],
+ ]
+ df = pd.DataFrame(
+ np.random.default_rng(2).standard_normal((8, 8)), columns=arrays
+ )
+ df.columns.names = ["Level1", "Level2"]
+ if engine == "fastparquet":
+ self.check_error_on_write(df, engine, ValueError, "Column name", temp_file)
+ elif engine == "pyarrow":
+ check_round_trip(df, temp_file, engine)
+
+ def test_write_column_multiindex_string(self, pa, temp_file):
+ # GH #34777
+ # Not supported in fastparquet as of 0.1.3
+ engine = pa
+
+ # Write column multi-indexes with string column names
+ arrays = [
+ ["bar", "bar", "baz", "baz", "foo", "foo", "qux", "qux"],
+ ["one", "two", "one", "two", "one", "two", "one", "two"],
+ ]
+ df = pd.DataFrame(
+ np.random.default_rng(2).standard_normal((8, 8)), columns=arrays
+ )
+ df.columns.names = ["ColLevel1", "ColLevel2"]
+
+ check_round_trip(df, temp_file, engine)
+
+ def test_write_column_index_string(self, pa, temp_file):
+ # GH #34777
+ # Not supported in fastparquet as of 0.1.3
+ engine = pa
+
+ # Write column indexes with string column names
+ arrays = ["bar", "baz", "foo", "qux"]
+ df = pd.DataFrame(
+ np.random.default_rng(2).standard_normal((8, 4)), columns=arrays
+ )
+ df.columns.name = "StringCol"
+
+ check_round_trip(df, temp_file, engine)
+
+ def test_write_column_index_nonstring(self, engine, temp_file):
+ # GH #34777
+
+ # Write column indexes with string column names
+ arrays = [1, 2, 3, 4]
+ df = pd.DataFrame(
+ np.random.default_rng(2).standard_normal((8, 4)), columns=arrays
+ )
+ df.columns.name = "NonStringCol"
+ if engine == "fastparquet":
+ self.check_error_on_write(
+ df, engine, TypeError, "Column name must be a string", temp_file
+ )
+ else:
+ check_round_trip(df, temp_file, engine)
+
+ def test_dtype_backend(self, engine, request, temp_file):
+ pq = pytest.importorskip("pyarrow.parquet")
+
+ if engine == "fastparquet":
+ # We are manually disabling fastparquet's
+ # nullable dtype support pending discussion
+ mark = pytest.mark.xfail(
+ reason="Fastparquet nullable dtype support is disabled"
+ )
+ request.applymarker(mark)
+
+ table = pyarrow.table(
+ {
+ "a": pyarrow.array([1, 2, 3, None], "int64"),
+ "b": pyarrow.array([1, 2, 3, None], "uint8"),
+ "c": pyarrow.array(["a", "b", "c", None]),
+ "d": pyarrow.array([True, False, True, None]),
+ # Test that nullable dtypes used even in absence of nulls
+ "e": pyarrow.array([1, 2, 3, 4], "int64"),
+ # GH 45694
+ "f": pyarrow.array([1.0, 2.0, 3.0, None], "float32"),
+ "g": pyarrow.array([1.0, 2.0, 3.0, None], "float64"),
+ }
+ )
+ # write manually with pyarrow to write integers
+ pq.write_table(table, temp_file)
+ result1 = read_parquet(temp_file, engine=engine)
+ result2 = read_parquet(temp_file, engine=engine, dtype_backend="numpy_nullable")
+
+ assert result1["a"].dtype == np.dtype("float64")
+ expected = pd.DataFrame(
+ {
+ "a": pd.array([1, 2, 3, None], dtype="Int64"),
+ "b": pd.array([1, 2, 3, None], dtype="UInt8"),
+ "c": pd.array(["a", "b", "c", None], dtype="string"),
+ "d": pd.array([True, False, True, None], dtype="boolean"),
+ "e": pd.array([1, 2, 3, 4], dtype="Int64"),
+ "f": pd.array([1.0, 2.0, 3.0, None], dtype="Float32"),
+ "g": pd.array([1.0, 2.0, 3.0, None], dtype="Float64"),
+ }
+ )
+ if engine == "fastparquet":
+ # Fastparquet doesn't support string columns yet
+ # Only int and boolean
+ result2 = result2.drop("c", axis=1)
+ expected = expected.drop("c", axis=1)
+ tm.assert_frame_equal(result2, expected)
+
+ @pytest.mark.parametrize(
+ "dtype",
+ [
+ "Int64",
+ "UInt8",
+ "boolean",
+ "object",
+ "datetime64[ns, UTC]",
+ "float",
+ "period[D]",
+ "Float64",
+ "string",
+ ],
+ )
+ def test_read_empty_array(self, pa, dtype, temp_file):
+ # GH #41241
+ df = pd.DataFrame(
+ {
+ "value": pd.array([], dtype=dtype),
+ }
+ )
+ pytest.importorskip("pyarrow", "11.0.0")
+ # GH 45694
+ expected = None
+ if dtype == "float":
+ expected = pd.DataFrame(
+ {
+ "value": pd.array([], dtype="Float64"),
+ }
+ )
+ check_round_trip(
+ df,
+ temp_file,
+ pa,
+ read_kwargs={"dtype_backend": "numpy_nullable"},
+ expected=expected,
+ )
+
+ @pytest.mark.network
+ @pytest.mark.single_cpu
+ def test_parquet_read_from_url(self, httpserver, datapath, df_compat, engine):
+ if engine != "auto":
+ pytest.importorskip(engine)
+ with open(datapath("io", "data", "parquet", "simple.parquet"), mode="rb") as f:
+ httpserver.serve_content(content=f.read())
+ df = read_parquet(httpserver.url, engine=engine)
+
+ expected = df_compat
+ if pa_version_under19p0:
+ expected.columns = expected.columns.astype(object)
+ tm.assert_frame_equal(df, expected)
+
+
+class TestParquetPyArrow(Base):
+ def test_basic(self, pa, df_full, temp_file):
+ df = df_full
+ pytest.importorskip("pyarrow", "11.0.0")
+
+ # additional supported types for pyarrow
+ dti = pd.date_range("20130101", periods=3, tz="Europe/Brussels")
+ dti = dti._with_freq(None) # freq doesn't round-trip
+ df["datetime_tz"] = dti
+ df["bool_with_none"] = [True, None, True]
+
+ check_round_trip(df, temp_file, pa)
+
+ def test_basic_subset_columns(self, pa, df_full, temp_file):
+ # GH18628
+
+ df = df_full
+ # additional supported types for pyarrow
+ df["datetime_tz"] = pd.date_range("20130101", periods=3, tz="Europe/Brussels")
+
+ check_round_trip(
+ df,
+ temp_file,
+ pa,
+ expected=df[["string", "int"]],
+ read_kwargs={"columns": ["string", "int"]},
+ )
+
+ def test_to_bytes_without_path_or_buf_provided(self, pa, df_full):
+ # GH 37105
+ buf_bytes = df_full.to_parquet(engine=pa)
+ assert isinstance(buf_bytes, bytes)
+
+ buf_stream = BytesIO(buf_bytes)
+ res = read_parquet(buf_stream)
+
+ expected = df_full.copy()
+ expected.loc[1, "string_with_nan"] = None
+ expected["datetime_with_nat"] = expected["datetime_with_nat"].astype("M8[us]")
+ tm.assert_frame_equal(res, expected)
+
+ def test_duplicate_columns(self, pa, temp_file):
+ # not currently able to handle duplicate columns
+ df = pd.DataFrame(np.arange(12).reshape(4, 3), columns=list("aaa")).copy()
+ self.check_error_on_write(
+ df, pa, ValueError, "Duplicate column names found", temp_file
+ )
+
+ def test_timedelta(self, pa, temp_file):
+ df = pd.DataFrame({"a": pd.timedelta_range("1 day", periods=3)})
+ check_round_trip(df, temp_file, pa)
+
+ def test_unsupported(self, pa, temp_file):
+ # mixed python objects
+ df = pd.DataFrame({"a": ["a", 1, 2.0]})
+ # pyarrow 0.11 raises ArrowTypeError
+ # older pyarrows raise ArrowInvalid
+ self.check_external_error_on_write(df, pa, pyarrow.ArrowException, temp_file)
+
+ def test_unsupported_float16(self, pa, temp_file):
+ # #44847, #44914
+ # Not able to write float 16 column using pyarrow.
+ data = np.arange(2, 10, dtype=np.float16)
+ df = pd.DataFrame(data=data, columns=["fp16"])
+ if pa_version_under15p0:
+ self.check_external_error_on_write(
+ df, pa, pyarrow.ArrowException, temp_file
+ )
+ else:
+ check_round_trip(df, temp_file, pa)
+
+ @pytest.mark.xfail(
+ is_platform_windows(),
+ reason=(
+ "PyArrow does not cleanup of partial files dumps when unsupported "
+ "dtypes are passed to_parquet function in windows"
+ ),
+ )
+ @pytest.mark.skipif(not pa_version_under15p0, reason="float16 works on 15")
+ @pytest.mark.parametrize("path_type", [str, pathlib.Path])
+ def test_unsupported_float16_cleanup(self, pa, path_type, temp_file):
+ # #44847, #44914
+ # Not able to write float 16 column using pyarrow.
+ # Tests cleanup by pyarrow in case of an error
+ data = np.arange(2, 10, dtype=np.float16)
+ df = pd.DataFrame(data=data, columns=["fp16"])
+
+ path = path_type(temp_file)
+ with tm.external_error_raised(pyarrow.ArrowException):
+ df.to_parquet(path=path, engine=pa)
+ assert not os.path.isfile(path)
+
+ def test_categorical(self, pa, temp_file):
+ # supported in >= 0.7.0
+ df = pd.DataFrame(
+ {
+ "a": pd.Categorical(list("abcdef")),
+ # test for null, out-of-order values, and unobserved category
+ "b": pd.Categorical(
+ ["bar", "foo", "foo", "bar", None, "bar"],
+ dtype=pd.CategoricalDtype(["foo", "bar", "baz"]),
+ ),
+ # test for ordered flag
+ "c": pd.Categorical(
+ [None, "b", "c", None, "c", "b"],
+ categories=["b", "c", "d"],
+ ordered=True,
+ ),
+ }
+ )
+
+ check_round_trip(df, temp_file, pa)
+
+ @pytest.mark.single_cpu
+ def test_s3_roundtrip_explicit_fs(
+ self, df_compat, s3_bucket_public, s3so, pa, temp_file
+ ):
+ s3fs = pytest.importorskip("s3fs")
+ s3 = s3fs.S3FileSystem(**s3so)
+ kw = {"filesystem": s3}
+ check_round_trip(
+ df_compat,
+ temp_file,
+ pa,
+ path=f"{s3_bucket_public.name}/pyarrow.parquet",
+ read_kwargs=kw,
+ write_kwargs=kw,
+ )
+
+ @pytest.mark.single_cpu
+ def test_s3_roundtrip(self, df_compat, s3_bucket_public, s3so, pa, temp_file):
+ # GH #19134
+ s3so = {"storage_options": s3so}
+ check_round_trip(
+ df_compat,
+ temp_file,
+ pa,
+ path=f"s3://{s3_bucket_public.name}/pyarrow.parquet",
+ read_kwargs=s3so,
+ write_kwargs=s3so,
+ )
+
+ @pytest.mark.single_cpu
+ @pytest.mark.parametrize("partition_col", [["A"], []])
+ def test_s3_roundtrip_for_dir(
+ self, df_compat, s3_bucket_public, pa, partition_col, s3so, temp_file
+ ):
+ pytest.importorskip("s3fs")
+ # GH #26388
+ expected_df = df_compat.copy()
+
+ # GH #35791
+ if partition_col:
+ expected_df = expected_df.astype(dict.fromkeys(partition_col, np.int32))
+ partition_col_type = "category"
+
+ expected_df[partition_col] = expected_df[partition_col].astype(
+ partition_col_type
+ )
+
+ check_round_trip(
+ df_compat,
+ temp_file,
+ pa,
+ expected=expected_df,
+ path=f"s3://{s3_bucket_public.name}/parquet_dir",
+ read_kwargs={"storage_options": s3so},
+ write_kwargs={
+ "partition_cols": partition_col,
+ "compression": None,
+ "storage_options": s3so,
+ },
+ check_like=True,
+ repeat=1,
+ )
+
+ def test_read_file_like_obj_support(self, df_compat, using_infer_string):
+ pytest.importorskip("pyarrow")
+ buffer = BytesIO()
+ df_compat.to_parquet(buffer)
+ df_from_buf = read_parquet(buffer)
+ if using_infer_string and not pa_version_under19p0:
+ df_compat.columns = df_compat.columns.astype("str")
+ tm.assert_frame_equal(df_compat, df_from_buf)
+
+ def test_expand_user(self, df_compat, monkeypatch):
+ pytest.importorskip("pyarrow")
+ monkeypatch.setenv("HOME", "TestingUser")
+ monkeypatch.setenv("USERPROFILE", "TestingUser")
+ with pytest.raises(OSError, match=r".*TestingUser.*"):
+ read_parquet("~/file.parquet")
+ with pytest.raises(OSError, match=r".*TestingUser.*"):
+ df_compat.to_parquet("~/file.parquet")
+
+ def test_partition_cols_supported(self, tmp_path, pa, df_full):
+ # GH #23283
+ partition_cols = ["bool", "int"]
+ df = df_full
+ df.to_parquet(tmp_path, partition_cols=partition_cols, compression=None)
+ check_partition_names(tmp_path, partition_cols)
+ assert read_parquet(tmp_path).shape == df.shape
+
+ def test_partition_cols_string(self, tmp_path, pa, df_full):
+ # GH #27117
+ partition_cols = "bool"
+ partition_cols_list = [partition_cols]
+ df = df_full
+ df.to_parquet(tmp_path, partition_cols=partition_cols, compression=None)
+ check_partition_names(tmp_path, partition_cols_list)
+ assert read_parquet(tmp_path).shape == df.shape
+
+ @pytest.mark.parametrize(
+ "path_type", [str, lambda x: x], ids=["string", "pathlib.Path"]
+ )
+ def test_partition_cols_pathlib(self, tmp_path, pa, df_compat, path_type):
+ # GH 35902
+
+ partition_cols = "B"
+ partition_cols_list = [partition_cols]
+ df = df_compat
+
+ path = path_type(tmp_path)
+ df.to_parquet(path, partition_cols=partition_cols_list)
+ assert read_parquet(path).shape == df.shape
+
+ def test_empty_dataframe(self, pa, temp_file):
+ # GH #27339
+ df = pd.DataFrame(index=[], columns=[])
+ check_round_trip(df, temp_file, pa)
+
+ def test_write_with_schema(self, pa, temp_file):
+ import pyarrow
+
+ df = pd.DataFrame({"x": [0, 1]})
+ schema = pyarrow.schema([pyarrow.field("x", type=pyarrow.bool_())])
+ out_df = df.astype(bool)
+ check_round_trip(
+ df, temp_file, pa, write_kwargs={"schema": schema}, expected=out_df
+ )
+
+ def test_additional_extension_arrays(self, pa, using_infer_string, temp_file):
+ # test additional ExtensionArrays that are supported through the
+ # __arrow_array__ protocol
+ pytest.importorskip("pyarrow")
+ df = pd.DataFrame(
+ {
+ "a": pd.Series([1, 2, 3], dtype="Int64"),
+ "b": pd.Series([1, 2, 3], dtype="UInt32"),
+ "c": pd.Series(["a", None, "c"], dtype="string"),
+ }
+ )
+ if using_infer_string and pa_version_under19p0:
+ check_round_trip(df, temp_file, pa, expected=df.astype({"c": "str"}))
+ else:
+ check_round_trip(df, temp_file, pa)
+
+ df = pd.DataFrame({"a": pd.Series([1, 2, 3, None], dtype="Int64")})
+ check_round_trip(df, temp_file, pa)
+
+ def test_pyarrow_backed_string_array(
+ self, pa, string_storage, using_infer_string, temp_file
+ ):
+ # test ArrowStringArray supported through the __arrow_array__ protocol
+ pytest.importorskip("pyarrow")
+ df = pd.DataFrame({"a": pd.Series(["a", None, "c"], dtype="string[pyarrow]")})
+ with pd.option_context("string_storage", string_storage):
+ if using_infer_string:
+ if pa_version_under19p0:
+ expected = df.astype("str")
+ else:
+ expected = df.astype(f"string[{string_storage}]")
+ expected.columns = expected.columns.astype("str")
+ else:
+ expected = df.astype(f"string[{string_storage}]")
+ check_round_trip(df, temp_file, pa, expected=expected)
+
+ def test_additional_extension_types(self, pa, temp_file):
+ # test additional ExtensionArrays that are supported through the
+ # __arrow_array__ protocol + by defining a custom ExtensionType
+ pytest.importorskip("pyarrow")
+ df = pd.DataFrame(
+ {
+ "c": pd.IntervalIndex.from_tuples([(0, 1), (1, 2), (3, 4)]),
+ "d": pd.period_range("2012-01-01", periods=3, freq="D"),
+ # GH-45881 issue with interval with datetime64[ns] subtype
+ "e": pd.IntervalIndex.from_breaks(
+ pd.date_range("2012-01-01", periods=4, freq="D")
+ ),
+ }
+ )
+ check_round_trip(df, temp_file, pa)
+
+ def test_timestamp_nanoseconds(self, pa, temp_file):
+ # with version 2.6, pyarrow defaults to writing the nanoseconds, so
+ # this should work without error, even for pyarrow < 13
+ ver = "2.6"
+ df = pd.DataFrame({"a": pd.date_range("2017-01-01", freq="1ns", periods=10)})
+ check_round_trip(df, temp_file, pa, write_kwargs={"version": ver})
+
+ def test_timezone_aware_index(self, pa, timezone_aware_date_list, temp_file):
+ idx = 5 * [timezone_aware_date_list]
+ df = pd.DataFrame(index=idx, data={"index_as_col": idx})
+ check_round_trip(df, temp_file, pa, check_dtype=False)
+
+ def test_filter_row_groups(self, pa, temp_file):
+ # https://github.com/pandas-dev/pandas/issues/26551
+ pytest.importorskip("pyarrow")
+ df = pd.DataFrame({"a": list(range(3))})
+ df.to_parquet(temp_file, engine=pa)
+ result = read_parquet(temp_file, pa, filters=[("a", "==", 0)])
+ assert len(result) == 1
+
+ @pytest.mark.filterwarnings("ignore:make_block is deprecated:DeprecationWarning")
+ def test_read_dtype_backend_pyarrow_config(self, pa, df_full, temp_file):
+ import pyarrow
+
+ df = df_full
+
+ # additional supported types for pyarrow
+ dti = pd.date_range("20130101", periods=3, tz="Europe/Brussels", unit="ns")
+ dti = dti._with_freq(None) # freq doesn't round-trip
+ df["datetime_tz"] = dti
+ df["bool_with_none"] = [True, None, True]
+
+ pa_table = pyarrow.Table.from_pandas(df)
+ expected = pa_table.to_pandas(types_mapper=pd.ArrowDtype)
+ expected["datetime_with_nat"] = expected["datetime_with_nat"].astype(
+ "timestamp[us][pyarrow]"
+ )
+
+ check_round_trip(
+ df,
+ temp_file,
+ engine=pa,
+ read_kwargs={"dtype_backend": "pyarrow"},
+ expected=expected,
+ )
+
+ def test_read_dtype_backend_pyarrow_config_index(self, pa, temp_file):
+ df = pd.DataFrame(
+ {"a": [1, 2]}, index=pd.Index([3, 4], name="test"), dtype="int64[pyarrow]"
+ )
+ expected = df.copy()
+
+ expected.index = expected.index.astype("int64[pyarrow]")
+ check_round_trip(
+ df,
+ temp_file,
+ engine=pa,
+ read_kwargs={"dtype_backend": "pyarrow"},
+ expected=expected,
+ )
+
+ @pytest.mark.parametrize(
+ "columns",
+ [
+ [0, 1],
+ pytest.param(
+ [b"foo", b"bar"],
+ marks=pytest.mark.xfail(
+ pa_version_under20p0,
+ raises=NotImplementedError,
+ reason="https://github.com/apache/arrow/pull/44171",
+ ),
+ ),
+ pytest.param(
+ [
+ datetime.datetime(2011, 1, 1, 0, 0),
+ datetime.datetime(2011, 1, 1, 1, 1),
+ ],
+ marks=pytest.mark.xfail(
+ pa_version_under17p0,
+ reason="pa.pandas_compat passes 'datetime64' to .astype",
+ ),
+ ),
+ ],
+ )
+ def test_columns_dtypes_not_invalid(self, pa, columns, temp_file):
+ df = pd.DataFrame({"string": list("abc"), "int": list(range(1, 4))})
+
+ df.columns = columns
+ check_round_trip(df, temp_file, pa)
+
+ def test_empty_columns(self, pa, temp_file):
+ # GH 52034
+ df = pd.DataFrame(index=pd.Index(["a", "b", "c"], name="custom name"))
+ check_round_trip(df, temp_file, pa)
+
+ def test_df_attrs_persistence(self, temp_file, pa):
+ df = pd.DataFrame(data={1: [1]})
+ df.attrs = {"test_attribute": 1}
+ df.to_parquet(temp_file, engine=pa)
+ new_df = read_parquet(temp_file, engine=pa)
+ assert new_df.attrs == df.attrs
+
+ def test_string_inference(self, temp_file, pa, using_infer_string):
+ # GH#54431
+ df = pd.DataFrame(data={"a": ["x", "y"]}, index=["a", "b"])
+ df.to_parquet(temp_file, engine=pa)
+ with pd.option_context("future.infer_string", True):
+ result = read_parquet(temp_file, engine=pa)
+ dtype = pd.StringDtype(na_value=np.nan)
+ expected = pd.DataFrame(
+ data={"a": ["x", "y"]},
+ dtype=dtype,
+ index=pd.Index(["a", "b"], dtype=dtype),
+ columns=pd.Index(
+ ["a"],
+ dtype=(
+ object if pa_version_under19p0 and not using_infer_string else dtype
+ ),
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_roundtrip_decimal(self, temp_file, pa):
+ # GH#54768
+ import pyarrow as pa
+
+ df = pd.DataFrame({"a": [Decimal("123.00")]}, dtype="string[pyarrow]")
+ df.to_parquet(temp_file, schema=pa.schema([("a", pa.decimal128(5))]))
+ result = read_parquet(temp_file)
+ if pa_version_under19p0:
+ expected = pd.DataFrame({"a": ["123"]}, dtype="string")
+ else:
+ expected = pd.DataFrame({"a": [Decimal("123.00")]}, dtype="object")
+ tm.assert_frame_equal(result, expected)
+
+ def test_infer_string_large_string_type(self, temp_file, pa):
+ # GH#54798
+ import pyarrow as pa
+ import pyarrow.parquet as pq
+
+ table = pa.table({"a": pa.array([None, "b", "c"], pa.large_string())})
+ pq.write_table(table, temp_file)
+
+ with pd.option_context("future.infer_string", True):
+ result = read_parquet(temp_file)
+ expected = pd.DataFrame(
+ data={"a": [None, "b", "c"]},
+ dtype=pd.StringDtype(na_value=np.nan),
+ columns=pd.Index(["a"], dtype=pd.StringDtype(na_value=np.nan)),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ # NOTE: this test is not run by default, because it requires a lot of memory (>5GB)
+ # @pytest.mark.slow
+ # def test_string_column_above_2GB(self, tmp_path, pa):
+ # # https://github.com/pandas-dev/pandas/issues/55606
+ # # above 2GB of string data
+ # v1 = b"x" * 100000000
+ # v2 = b"x" * 147483646
+ # df = pd.DataFrame({"strings": [v1] * 20 + [v2] + ["x"] * 20}, dtype="string")
+ # df.to_parquet(tmp_path / "test.parquet")
+ # result = read_parquet(tmp_path / "test.parquet")
+ # assert result["strings"].dtype == "string"
+ # FIXME: don't leave commented-out
+
+ def test_non_nanosecond_timestamps(self, temp_file):
+ # GH#49236
+ pa = pytest.importorskip("pyarrow", "13.0.0")
+ pq = pytest.importorskip("pyarrow.parquet")
+
+ arr = pa.array([datetime.datetime(1600, 1, 1)], type=pa.timestamp("us"))
+ table = pa.table([arr], names=["timestamp"])
+ pq.write_table(table, temp_file)
+ result = read_parquet(temp_file)
+ expected = pd.DataFrame(
+ data={"timestamp": [datetime.datetime(1600, 1, 1)]},
+ dtype="datetime64[us]",
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_maps_as_pydicts(self, pa, temp_file):
+ pyarrow = pytest.importorskip("pyarrow", "13.0.0")
+
+ schema = pyarrow.schema(
+ [("foo", pyarrow.map_(pyarrow.string(), pyarrow.int64()))]
+ )
+ df = pd.DataFrame([{"foo": {"A": 1}}, {"foo": {"B": 2}}])
+ check_round_trip(
+ df,
+ temp_file,
+ pa,
+ write_kwargs={"schema": schema},
+ read_kwargs={"to_pandas_kwargs": {"maps_as_pydicts": "strict"}},
+ )
+
+
+class TestParquetFastParquet(Base):
+ def test_basic(self, fp, df_full, request, temp_file):
+ pytz = pytest.importorskip("pytz")
+
+ tz = pytz.timezone("US/Eastern")
+ df = df_full
+
+ dti = pd.date_range("20130101", periods=3, tz=tz)
+ dti = dti._with_freq(None) # freq doesn't round-trip
+ df["datetime_tz"] = dti
+ df["timedelta"] = pd.timedelta_range("1 day", periods=3)
+ check_round_trip(df, temp_file, fp)
+
+ def test_columns_dtypes_invalid(self, fp, temp_file):
+ df = pd.DataFrame({"string": list("abc"), "int": list(range(1, 4))})
+
+ err = TypeError
+ msg = "Column name must be a string"
+
+ # numeric
+ df.columns = [0, 1]
+ self.check_error_on_write(df, fp, err, msg, temp_file)
+
+ # bytes
+ df.columns = [b"foo", b"bar"]
+ self.check_error_on_write(df, fp, err, msg, temp_file)
+
+ # python object
+ df.columns = [
+ datetime.datetime(2011, 1, 1, 0, 0),
+ datetime.datetime(2011, 1, 1, 1, 1),
+ ]
+ self.check_error_on_write(df, fp, err, msg, temp_file)
+
+ def test_duplicate_columns(self, fp, temp_file):
+ # not currently able to handle duplicate columns
+ df = pd.DataFrame(np.arange(12).reshape(4, 3), columns=list("aaa")).copy()
+ msg = "Cannot create parquet dataset with duplicate column names"
+ self.check_error_on_write(df, fp, ValueError, msg, temp_file)
+
+ def test_bool_with_none(self, fp, request, temp_file):
+ df = pd.DataFrame({"a": [True, None, False]})
+ expected = pd.DataFrame({"a": [1.0, np.nan, 0.0]}, dtype="float16")
+ # Fastparquet bug in 0.7.1 makes it so that this dtype becomes
+ # float64
+ check_round_trip(df, temp_file, fp, expected=expected, check_dtype=False)
+
+ def test_unsupported(self, fp, temp_file):
+ # period
+ df = pd.DataFrame({"a": pd.period_range("2013", freq="M", periods=3)})
+ # error from fastparquet -> don't check exact error message
+ self.check_error_on_write(df, fp, ValueError, None, temp_file)
+
+ # mixed
+ df = pd.DataFrame({"a": ["a", 1, 2.0]})
+ msg = "Can't infer object conversion type"
+ self.check_error_on_write(df, fp, ValueError, msg, temp_file)
+
+ @pytest.mark.xfail(
+ using_string_dtype()
+ and _HAVE_PYARROW
+ and not _fp_version_lt_2025
+ and _fp_version_lt_2026_5,
+ reason="fastparquet >= 2025.12.0, < 2026.5.0 can't write ArrowStringArray",
+ )
+ def test_categorical(self, fp, temp_file):
+ df = pd.DataFrame({"a": pd.Categorical(list("abc"))})
+ check_round_trip(df, temp_file, fp)
+
+ def test_filter_row_groups(self, fp, temp_file):
+ d = {"a": list(range(3))}
+ df = pd.DataFrame(d)
+ df.to_parquet(temp_file, engine=fp, compression=None, row_group_offsets=1)
+ result = read_parquet(temp_file, fp, filters=[("a", "==", 0)])
+ assert len(result) == 1
+
+ @pytest.mark.single_cpu
+ def test_s3_roundtrip(self, df_compat, s3_bucket_public, s3so, fp, temp_file):
+ # GH #19134
+ check_round_trip(
+ df_compat,
+ temp_file,
+ fp,
+ path=f"s3://{s3_bucket_public.name}/fastparquet.parquet",
+ read_kwargs={"storage_options": s3so},
+ write_kwargs={"compression": None, "storage_options": s3so},
+ )
+
+ @pytest.mark.xfail(
+ using_string_dtype()
+ and _HAVE_PYARROW
+ and not _fp_version_lt_2025
+ and _fp_version_lt_2026_5,
+ reason="fastparquet >= 2025.12.0, < 2026.5.0 can't write ArrowStringArray",
+ )
+ def test_partition_cols_supported(self, tmp_path, fp, df_full):
+ # GH #23283
+ partition_cols = ["bool", "int"]
+ df = df_full
+ df.to_parquet(
+ tmp_path,
+ engine="fastparquet",
+ partition_cols=partition_cols,
+ compression=None,
+ )
+ assert os.path.exists(tmp_path)
+ import fastparquet
+
+ actual_partition_cols = fastparquet.ParquetFile(str(tmp_path), False).cats
+ assert len(actual_partition_cols) == 2
+
+ @pytest.mark.xfail(
+ using_string_dtype()
+ and _HAVE_PYARROW
+ and not _fp_version_lt_2025
+ and _fp_version_lt_2026_5,
+ reason="fastparquet >= 2025.12.0, < 2026.5.0 can't write ArrowStringArray",
+ )
+ def test_partition_cols_string(self, tmp_path, fp, df_full):
+ # GH #27117
+ partition_cols = "bool"
+ df = df_full
+ df.to_parquet(
+ tmp_path,
+ engine="fastparquet",
+ partition_cols=partition_cols,
+ compression=None,
+ )
+ assert os.path.exists(tmp_path)
+ import fastparquet
+
+ actual_partition_cols = fastparquet.ParquetFile(str(tmp_path), False).cats
+ assert len(actual_partition_cols) == 1
+
+ @pytest.mark.xfail(
+ using_string_dtype()
+ and _HAVE_PYARROW
+ and not _fp_version_lt_2025
+ and _fp_version_lt_2026_5,
+ reason="fastparquet >= 2025.12.0, < 2026.5.0 can't write ArrowStringArray",
+ )
+ def test_partition_on_supported(self, tmp_path, fp, df_full):
+ # GH #23283
+ partition_cols = ["bool", "int"]
+ df = df_full
+ df.to_parquet(
+ tmp_path,
+ engine="fastparquet",
+ compression=None,
+ partition_on=partition_cols,
+ )
+ assert os.path.exists(tmp_path)
+ import fastparquet
+
+ actual_partition_cols = fastparquet.ParquetFile(str(tmp_path), False).cats
+ assert len(actual_partition_cols) == 2
+
+ def test_error_on_using_partition_cols_and_partition_on(
+ self, tmp_path, fp, df_full
+ ):
+ # GH #23283
+ partition_cols = ["bool", "int"]
+ df = df_full
+ msg = (
+ "Cannot use both partition_on and partition_cols. Use partition_cols for "
+ "partitioning data"
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.to_parquet(
+ tmp_path,
+ engine="fastparquet",
+ compression=None,
+ partition_on=partition_cols,
+ partition_cols=partition_cols,
+ )
+
+ def test_empty_dataframe(self, fp, temp_file):
+ # GH #27339
+ df = pd.DataFrame()
+ expected = df.copy()
+ check_round_trip(df, temp_file, fp, expected=expected)
+
+ def test_timezone_aware_index(
+ self, fp, timezone_aware_date_list, request, temp_file
+ ):
+ idx = 5 * [timezone_aware_date_list]
+
+ df = pd.DataFrame(index=idx, data={"index_as_col": idx})
+
+ expected = df.copy()
+ expected.index.name = "index"
+ check_round_trip(df, temp_file, fp, expected=expected)
+
+ def test_close_file_handle_on_read_error(self, temp_file):
+ pathlib.Path(temp_file).write_bytes(b"breakit")
+ with tm.external_error_raised(Exception): # Not important which exception
+ read_parquet(temp_file, engine="fastparquet")
+ # The next line raises an error on Windows if the file is still open
+ pathlib.Path(temp_file).unlink(missing_ok=False)
+
+ def test_bytes_file_name(self, engine, temp_file):
+ # GH#48944
+ df = pd.DataFrame(data={"A": [0, 1], "B": [1, 0]})
+ with open(temp_file, "wb") as f:
+ df.to_parquet(f)
+
+ result = read_parquet(temp_file, engine=engine)
+ tm.assert_frame_equal(result, df)
+
+ def test_filesystem_notimplemented(self, temp_file):
+ pytest.importorskip("fastparquet")
+ df = pd.DataFrame(data={"A": [0, 1], "B": [1, 0]})
+ with pytest.raises(NotImplementedError, match="filesystem is not implemented"):
+ df.to_parquet(temp_file, engine="fastparquet", filesystem="foo")
+
+ pathlib.Path(temp_file).write_bytes(b"foo")
+ with pytest.raises(NotImplementedError, match="filesystem is not implemented"):
+ read_parquet(temp_file, engine="fastparquet", filesystem="foo")
+
+ def test_invalid_filesystem(self, temp_file):
+ pytest.importorskip("pyarrow")
+ df = pd.DataFrame(data={"A": [0, 1], "B": [1, 0]})
+
+ with pytest.raises(
+ ValueError, match="filesystem must be a pyarrow or fsspec FileSystem"
+ ):
+ df.to_parquet(temp_file, engine="pyarrow", filesystem="foo")
+
+ pathlib.Path(temp_file).write_bytes(b"foo")
+ with pytest.raises(
+ ValueError, match="filesystem must be a pyarrow or fsspec FileSystem"
+ ):
+ read_parquet(temp_file, engine="pyarrow", filesystem="foo")
+
+ def test_unsupported_pa_filesystem_storage_options(self, temp_file):
+ pa_fs = pytest.importorskip("pyarrow.fs")
+ df = pd.DataFrame(data={"A": [0, 1], "B": [1, 0]})
+
+ with pytest.raises(
+ NotImplementedError,
+ match="storage_options not supported with a pyarrow FileSystem.",
+ ):
+ df.to_parquet(
+ temp_file,
+ engine="pyarrow",
+ filesystem=pa_fs.LocalFileSystem(),
+ storage_options={"foo": "bar"},
+ )
+
+ pathlib.Path(temp_file).write_bytes(b"foo")
+ with pytest.raises(
+ NotImplementedError,
+ match="storage_options not supported with a pyarrow FileSystem.",
+ ):
+ read_parquet(
+ temp_file,
+ engine="pyarrow",
+ filesystem=pa_fs.LocalFileSystem(),
+ storage_options={"foo": "bar"},
+ )
+
+ def test_invalid_dtype_backend(self, engine, temp_file):
+ msg = (
+ "dtype_backend numpy is invalid, only 'numpy_nullable' and "
+ "'pyarrow' are allowed."
+ )
+ df = pd.DataFrame({"int": list(range(1, 4))})
+ df.to_parquet(temp_file)
+ with pytest.raises(ValueError, match=msg):
+ read_parquet(temp_file, dtype_backend="numpy")
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_pickle.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_pickle.py
new file mode 100644
index 0000000000000000000000000000000000000000..7754c58a88ef137825906a543a3783f2e21c71b0
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_pickle.py
@@ -0,0 +1,590 @@
+"""
+manage legacy pickle tests
+
+How to add pickle tests:
+
+1. Install pandas version intended to output the pickle.
+
+2. Execute "generate_legacy_storage_files.py" to create the pickle.
+$ python generate_legacy_storage_files.py pickle
+
+3. Move the created pickle to "data/legacy_pickle/" directory.
+"""
+
+from __future__ import annotations
+
+import bz2
+import datetime
+import functools
+from functools import partial
+import gzip
+import io
+import os
+from pathlib import Path
+import pickle
+import shutil
+import tarfile
+from typing import Any
+import uuid
+import zipfile
+
+import numpy as np
+import pytest
+
+from pandas.compat import is_platform_little_endian
+from pandas.compat._optional import import_optional_dependency
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ Series,
+ period_range,
+)
+import pandas._testing as tm
+from pandas.tests.io.generate_legacy_storage_files import create_pickle_data
+from pandas.util.version import Version
+
+import pandas.io.common as icom
+from pandas.tseries.offsets import (
+ Day,
+ MonthEnd,
+)
+
+
+# ---------------------
+# comparison functions
+# ---------------------
+def compare_element(result, expected, typ):
+ if isinstance(expected, Index):
+ tm.assert_index_equal(result, expected)
+ return
+
+ if typ.startswith("sp_"):
+ tm.assert_equal(result, expected)
+ elif typ == "timestamp":
+ if expected is pd.NaT:
+ assert result is pd.NaT
+ else:
+ assert result == expected
+ else:
+ comparator = getattr(tm, f"assert_{typ}_equal", tm.assert_almost_equal)
+ comparator(result, expected)
+
+
+# ---------------------
+# tests
+# ---------------------
+
+
+def test_pickles(datapath):
+ pytest.importorskip("pytz")
+ if not is_platform_little_endian():
+ pytest.skip("known failure on non-little endian")
+
+ current_data = create_pickle_data()
+
+ # For loop for compat with --strict-data-files
+ for legacy_pickle in Path(__file__).parent.glob("data/legacy_pickle/*/*.p*kl*"):
+ legacy_version = Version(legacy_pickle.parent.name)
+ legacy_pickle = datapath(legacy_pickle)
+
+ data = pd.read_pickle(legacy_pickle)
+
+ for typ, dv in data.items():
+ for dt, result in dv.items():
+ expected = current_data[typ][dt]
+
+ if (
+ typ == "timestamp"
+ and dt in ("tz", "both")
+ and legacy_version < Version("1.3.0")
+ ):
+ # convert to wall time
+ # (bug since pandas 2.0 that tz gets dropped for older pickle files)
+ expected = expected.tz_convert(None)
+
+ if legacy_version < Version("3.0.0.dev0"):
+ # before 3.0, we had:
+ # - object dtype instead of string
+ # - ns instead of us as the default unit
+ if typ in ("frame", "sp_frame"):
+ expected.columns = expected.columns.astype("object")
+ if dt in ("mixed", "mixed_dup"):
+ expected["C"] = expected["C"].astype(object)
+ expected["D"] = expected["D"].dt.as_unit("ns")
+ elif dt in ("cat_onecol", "cat_and_float"):
+ expected["A"] = expected["A"].astype(
+ pd.CategoricalDtype(
+ expected["A"].cat.categories.astype(object)
+ )
+ )
+ elif typ == "sp_frame" and dt == "float":
+ expected.index = expected.index.as_unit("ns")
+ elif dt == "mi":
+ expected.index = expected.index.set_levels(
+ [
+ level.astype("object")
+ for level in expected.index.levels
+ ],
+ )
+ elif typ in ("series", "sp_series"):
+ if dt == "ts":
+ expected.index = expected.index.as_unit("ns")
+ elif dt in ("dt", "dt_tz"):
+ expected = expected.dt.as_unit("ns")
+ elif dt == "cat":
+ expected = expected.astype(
+ pd.CategoricalDtype(
+ expected.cat.categories.astype(object)
+ )
+ )
+ elif dt == "dup":
+ expected.index = expected.index.astype(object)
+ elif typ == "index" and dt in ("date", "timedelta"):
+ expected = expected.as_unit("ns")
+ elif typ == "mi":
+ expected = expected.set_levels(
+ [level.astype("object") for level in expected.levels],
+ )
+ if dt == "string":
+ # we switched from python to pyarrow as default storage in 3.0
+ expected = expected.astype(pd.StringDtype("python"))
+
+ if dt in ("dt_mixed_tzs", "dt_mixed2_tzs"):
+ if legacy_version < Version("2.1"):
+ # in pandas < 2.0, Timestamp() unit defaulted to 'ns'
+ expected_unit = "ns"
+ elif Version("2.1") <= legacy_version < Version("3.0.0.dev0"):
+ # in pandas 2.x, Timestamp() unit depended on input
+ expected_unit = "s"
+ else:
+ expected_unit = "us"
+ for col in expected.columns:
+ expected[col] = expected[col].dt.as_unit(expected_unit)
+ if typ == "index" and dt == "int" and "windows" in legacy_pickle:
+ expected = expected.astype(np.int32)
+
+ if typ == "series" and dt == "ts":
+ # GH 7748
+ tm.assert_series_equal(result, expected)
+ assert result.index.freq == expected.index.freq
+ assert not result.index.freq.normalize
+ tm.assert_series_equal(result > 0, expected > 0)
+
+ # GH 9291
+ freq = result.index.freq
+ assert freq + Day(1) == Day(2)
+
+ res = freq + pd.Timedelta(hours=1)
+ assert isinstance(res, pd.Timedelta)
+ assert res == pd.Timedelta(days=1, hours=1)
+
+ res = freq + pd.Timedelta(nanoseconds=1)
+ assert isinstance(res, pd.Timedelta)
+ assert res == pd.Timedelta(days=1, nanoseconds=1)
+ elif typ == "index" and dt == "period":
+ tm.assert_index_equal(result, expected)
+ assert isinstance(result.freq, MonthEnd)
+ assert result.freq == MonthEnd()
+ assert result.freqstr == "M"
+ tm.assert_index_equal(result.shift(2), expected.shift(2))
+ elif typ == "series" and dt in ("dt_tz", "cat"):
+ tm.assert_series_equal(result, expected)
+ elif typ == "frame" and dt in (
+ "dt_mixed_tzs",
+ "cat_onecol",
+ "cat_and_float",
+ ):
+ tm.assert_frame_equal(result, expected)
+ else:
+ compare_element(result, expected, typ)
+
+
+def python_pickler(obj, path):
+ with open(path, "wb") as fh:
+ pickle.dump(obj, fh, protocol=-1)
+
+
+def python_unpickler(path):
+ with open(path, "rb") as fh:
+ fh.seek(0)
+ return pickle.load(fh)
+
+
+def flatten(data: dict) -> list[tuple[str, Any]]:
+ """Flatten create_pickle_data"""
+ return [
+ (typ, example)
+ for typ, examples in data.items()
+ for example in examples.values()
+ ]
+
+
+@pytest.mark.parametrize(
+ "pickle_writer",
+ [
+ pytest.param(python_pickler, id="python"),
+ pytest.param(pd.to_pickle, id="pandas_proto_default"),
+ pytest.param(
+ functools.partial(pd.to_pickle, protocol=pickle.HIGHEST_PROTOCOL),
+ id="pandas_proto_highest",
+ ),
+ pytest.param(functools.partial(pd.to_pickle, protocol=4), id="pandas_proto_4"),
+ pytest.param(
+ functools.partial(pd.to_pickle, protocol=5),
+ id="pandas_proto_5",
+ ),
+ ],
+)
+@pytest.mark.parametrize("writer", [pd.to_pickle, python_pickler])
+@pytest.mark.parametrize("typ, expected", flatten(create_pickle_data()))
+def test_round_trip_current(typ, expected, pickle_writer, writer, temp_file):
+ path = temp_file
+ # test writing with each pickler
+ pickle_writer(expected, path)
+
+ # test reading with each unpickler
+ result = pd.read_pickle(path)
+ compare_element(result, expected, typ)
+
+ result = python_unpickler(path)
+ compare_element(result, expected, typ)
+
+ # and the same for file objects (GH 35679)
+ with open(path, mode="wb") as handle:
+ writer(expected, path)
+ handle.seek(0) # shouldn't close file handle
+ with open(path, mode="rb") as handle:
+ result = pd.read_pickle(handle)
+ handle.seek(0) # shouldn't close file handle
+ compare_element(result, expected, typ)
+
+
+def test_pickle_path_pathlib(temp_file):
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ result = tm.round_trip_pathlib(df.to_pickle, pd.read_pickle, temp_file)
+ tm.assert_frame_equal(df, result)
+
+
+# ---------------------
+# test pickle compression
+# ---------------------
+
+
+@pytest.fixture
+def get_random_path():
+ return f"__{uuid.uuid4()}__.pickle"
+
+
+class TestCompression:
+ _extension_to_compression = icom.extension_to_compression
+
+ def compress_file(self, src_path, dest_path, compression):
+ if compression is None:
+ shutil.copyfile(src_path, dest_path)
+ return
+
+ if compression == "gzip":
+ f = gzip.open(dest_path, "w")
+ elif compression == "bz2":
+ f = bz2.BZ2File(dest_path, "w")
+ elif compression == "zip":
+ with zipfile.ZipFile(dest_path, "w", compression=zipfile.ZIP_DEFLATED) as f:
+ f.write(src_path, os.path.basename(src_path))
+ elif compression == "tar":
+ with open(src_path, "rb") as fh:
+ with tarfile.open(dest_path, mode="w") as tar:
+ tarinfo = tar.gettarinfo(src_path, os.path.basename(src_path))
+ tar.addfile(tarinfo, fh)
+ elif compression == "xz":
+ import lzma
+
+ f = lzma.LZMAFile(dest_path, "w")
+ elif compression == "zstd":
+ f = import_optional_dependency("zstandard").open(dest_path, "wb")
+ else:
+ msg = f"Unrecognized compression type: {compression}"
+ raise ValueError(msg)
+
+ if compression not in ["zip", "tar"]:
+ with open(src_path, "rb") as fh:
+ with f:
+ f.write(fh.read())
+
+ def test_write_explicit(self, compression, get_random_path, temp_file):
+ p1 = temp_file.parent / f"{temp_file.stem}.compressed"
+ p2 = temp_file.parent / f"{temp_file.stem}.raw"
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+
+ # write to compressed file
+ df.to_pickle(p1, compression=compression)
+
+ # decompress
+ with tm.decompress_file(p1, compression=compression) as f:
+ with open(p2, "wb") as fh:
+ fh.write(f.read())
+
+ # read decompressed file
+ df2 = pd.read_pickle(p2, compression=None)
+
+ tm.assert_frame_equal(df, df2)
+
+ @pytest.mark.parametrize("compression", ["", "None", "bad", "7z"])
+ def test_write_explicit_bad(self, compression, get_random_path, temp_file):
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ path = temp_file
+ with pytest.raises(ValueError, match="Unrecognized compression type"):
+ df.to_pickle(path, compression=compression)
+
+ def test_write_infer(self, compression_ext, get_random_path, temp_file):
+ p1 = temp_file.parent / f"{temp_file.stem}{compression_ext}"
+ p2 = temp_file.parent / f"{temp_file.stem}.raw"
+ compression = self._extension_to_compression.get(compression_ext.lower())
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+
+ # write to compressed file by inferred compression method
+ df.to_pickle(p1)
+
+ # decompress
+ with tm.decompress_file(p1, compression=compression) as f:
+ with open(p2, "wb") as fh:
+ fh.write(f.read())
+
+ # read decompressed file
+ df2 = pd.read_pickle(p2, compression=None)
+
+ tm.assert_frame_equal(df, df2)
+
+ def test_read_explicit(self, compression, get_random_path, temp_file):
+ p1 = temp_file.parent / f"{temp_file.stem}.raw"
+ p2 = temp_file.parent / f"{temp_file.stem}.compressed"
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+
+ # write to uncompressed file
+ df.to_pickle(p1, compression=None)
+
+ # compress
+ self.compress_file(p1, p2, compression=compression)
+
+ # read compressed file
+ df2 = pd.read_pickle(p2, compression=compression)
+ tm.assert_frame_equal(df, df2)
+
+ def test_read_infer(self, compression_ext, get_random_path, temp_file):
+ p1 = temp_file.parent / f"{temp_file.stem}.raw"
+ p2 = temp_file.parent / f"{temp_file.stem}{compression_ext}"
+ compression = self._extension_to_compression.get(compression_ext.lower())
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+
+ # write to uncompressed file
+ df.to_pickle(p1, compression=None)
+
+ # compress
+ self.compress_file(p1, p2, compression=compression)
+
+ # read compressed file by inferred compression method
+ df2 = pd.read_pickle(p2)
+ tm.assert_frame_equal(df, df2)
+
+
+# ---------------------
+# test pickle compression
+# ---------------------
+
+
+class TestProtocol:
+ @pytest.mark.parametrize("protocol", [-1, 0, 1, 2])
+ def test_read(self, protocol, get_random_path, temp_file):
+ path = temp_file
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ df.to_pickle(path, protocol=protocol)
+ df2 = pd.read_pickle(path)
+ tm.assert_frame_equal(df, df2)
+
+
+def test_pickle_buffer_roundtrip(temp_file):
+ path = temp_file
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ with open(path, "wb") as fh:
+ df.to_pickle(fh)
+ with open(path, "rb") as fh:
+ result = pd.read_pickle(fh)
+ tm.assert_frame_equal(df, result)
+
+
+def test_pickle_fsspec_roundtrip(temp_file):
+ pytest.importorskip("fsspec")
+ # Using temp_file for context, but fsspec uses memory URL
+ mockurl = "memory://mockfile"
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+ df.to_pickle(mockurl)
+ result = pd.read_pickle(mockurl)
+ tm.assert_frame_equal(df, result)
+
+
+class MyTz(datetime.tzinfo):
+ def __init__(self) -> None:
+ pass
+
+
+def test_read_pickle_with_subclass(temp_file):
+ # GH 12163
+ expected = Series(dtype=object), MyTz()
+ result = tm.round_trip_pickle(expected, temp_file)
+
+ tm.assert_series_equal(result[0], expected[0])
+ assert isinstance(result[1], MyTz)
+
+
+def test_pickle_binary_object_compression(compression, temp_file):
+ """
+ Read/write from binary file-objects w/wo compression.
+
+ GH 26237, GH 29054, and GH 29570
+ """
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(30)], dtype=object),
+ )
+
+ # reference for compression
+ path = temp_file
+ df.to_pickle(path, compression=compression)
+ reference = path.read_bytes()
+
+ # write
+ buffer = io.BytesIO()
+ df.to_pickle(buffer, compression=compression)
+ buffer.seek(0)
+
+ # gzip and zip safe the filename: cannot compare the compressed content
+ assert buffer.getvalue() == reference or compression in ("gzip", "zip", "tar")
+
+ # read
+ read_df = pd.read_pickle(buffer, compression=compression)
+ buffer.seek(0)
+ tm.assert_frame_equal(df, read_df)
+
+
+def test_pickle_dataframe_with_multilevel_index(
+ multiindex_year_month_day_dataframe_random_data,
+ multiindex_dataframe_random_data,
+ temp_file,
+):
+ ymd = multiindex_year_month_day_dataframe_random_data
+ frame = multiindex_dataframe_random_data
+
+ def _test_roundtrip(frame, temp_file):
+ unpickled = tm.round_trip_pickle(frame, temp_file)
+ tm.assert_frame_equal(frame, unpickled)
+
+ _test_roundtrip(frame, temp_file)
+ _test_roundtrip(frame.T, temp_file)
+ _test_roundtrip(ymd, temp_file)
+ _test_roundtrip(ymd.T, temp_file)
+
+
+def test_pickle_timeseries_periodindex(temp_file):
+ # GH#2891
+ prng = period_range("1/1/2011", "1/1/2012", freq="M")
+ ts = Series(np.random.default_rng(2).standard_normal(len(prng)), prng)
+ new_ts = tm.round_trip_pickle(ts, temp_file)
+ assert new_ts.index.freqstr == "M"
+
+
+@pytest.mark.parametrize(
+ "name", [777, 777.0, "name", datetime.datetime(2001, 11, 11), (1, 2)]
+)
+def test_pickle_preserve_name(name, temp_file):
+ unpickled = tm.round_trip_pickle(
+ Series(np.arange(10, dtype=np.float64), name=name), temp_file
+ )
+ assert unpickled.name == name
+
+
+def test_pickle_datetimes(datetime_series, temp_file):
+ unp_ts = tm.round_trip_pickle(datetime_series, temp_file)
+ tm.assert_series_equal(unp_ts, datetime_series)
+
+
+def test_pickle_strings(string_series, temp_file):
+ unp_series = tm.round_trip_pickle(string_series, temp_file)
+ tm.assert_series_equal(unp_series, string_series)
+
+
+def test_pickle_preserves_block_ndim(temp_file):
+ # GH#37631
+ ser = Series(list("abc")).astype("category").iloc[[0]]
+ res = tm.round_trip_pickle(ser, temp_file)
+
+ assert res._mgr.blocks[0].ndim == 1
+ assert res._mgr.blocks[0].shape == (1,)
+
+ # GH#37631 OP issue was about indexing, underlying problem was pickle
+ tm.assert_series_equal(res[[True]], ser)
+
+
+@pytest.mark.parametrize("protocol", [pickle.DEFAULT_PROTOCOL, pickle.HIGHEST_PROTOCOL])
+def test_pickle_big_dataframe_compression(protocol, compression, temp_file):
+ # GH#39002
+ df = DataFrame(range(100000))
+ result = tm.round_trip_pathlib(
+ partial(df.to_pickle, protocol=protocol, compression=compression),
+ partial(pd.read_pickle, compression=compression),
+ temp_file,
+ )
+ tm.assert_frame_equal(df, result)
+
+
+def test_pickle_frame_v124_unpickle_130(datapath):
+ # GH#42345 DataFrame created in 1.2.x, unpickle in 1.3.x
+ path = datapath(
+ Path(__file__).parent,
+ "data",
+ "legacy_pickle",
+ "1.2.4",
+ "empty_frame_v1_2_4-GH#42345.pkl",
+ )
+ with open(path, "rb") as fd:
+ df = pickle.load(fd)
+
+ expected = DataFrame(index=[], columns=[])
+ tm.assert_frame_equal(df, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_s3.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_s3.py
new file mode 100644
index 0000000000000000000000000000000000000000..31d22223b0a33218db58120cea1db393827ff699
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_s3.py
@@ -0,0 +1,33 @@
+from io import BytesIO
+
+import pytest
+
+from pandas import read_csv
+
+
+@pytest.mark.parametrize("data", [b"foo,bar,baz\n1,2,3\n4,5,6\n", b"just,the,header\n"])
+def test_streaming_s3_objects(data):
+ # GH 17135
+ # botocore gained iteration support in 1.10.47, can now be used in read_*
+ pytest.importorskip("botocore", minversion="1.10.47")
+ from botocore.response import StreamingBody
+
+ body = StreamingBody(BytesIO(data), content_length=len(data))
+ read_csv(body)
+
+
+@pytest.mark.single_cpu
+@pytest.mark.parametrize("header", ["infer", None])
+def test_read_with_and_without_creds_from_pub_bucket(
+ s3_bucket_public_with_data, s3so, header
+):
+ # GH 34626
+ pytest.importorskip("s3fs")
+ nrows = 5
+ df = read_csv(
+ f"s3://{s3_bucket_public_with_data.name}/tips.csv",
+ nrows=nrows,
+ header=header,
+ storage_options=s3so,
+ )
+ assert len(df) == nrows
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_spss.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_spss.py
new file mode 100644
index 0000000000000000000000000000000000000000..6210c0289a160e0ec853cb810dd80e404e11c37e
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_spss.py
@@ -0,0 +1,169 @@
+import datetime
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+import pandas as pd
+import pandas._testing as tm
+
+pyreadstat = pytest.importorskip("pyreadstat")
+
+
+# TODO(CoW) - detection of chained assignment in cython
+# https://github.com/pandas-dev/pandas/issues/51315
+@pytest.mark.filterwarnings("ignore::pandas.errors.ChainedAssignmentError")
+@pytest.mark.parametrize("path_klass", [lambda p: p, Path])
+def test_spss_labelled_num(path_klass, datapath):
+ # test file from the Haven project (https://haven.tidyverse.org/)
+ # Licence at LICENSES/HAVEN_LICENSE, LICENSES/HAVEN_MIT
+ fname = path_klass(datapath("io", "data", "spss", "labelled-num.sav"))
+
+ df = pd.read_spss(fname, convert_categoricals=True)
+ expected = pd.DataFrame({"VAR00002": "This is one"}, index=[0])
+ expected["VAR00002"] = pd.Categorical(expected["VAR00002"])
+ tm.assert_frame_equal(df, expected)
+
+ df = pd.read_spss(fname, convert_categoricals=False)
+ expected = pd.DataFrame({"VAR00002": 1.0}, index=[0])
+ tm.assert_frame_equal(df, expected)
+
+
+@pytest.mark.filterwarnings("ignore::pandas.errors.ChainedAssignmentError")
+def test_spss_labelled_num_na(datapath):
+ # test file from the Haven project (https://haven.tidyverse.org/)
+ # Licence at LICENSES/HAVEN_LICENSE, LICENSES/HAVEN_MIT
+ fname = datapath("io", "data", "spss", "labelled-num-na.sav")
+
+ df = pd.read_spss(fname, convert_categoricals=True)
+ expected = pd.DataFrame({"VAR00002": ["This is one", None]})
+ expected["VAR00002"] = pd.Categorical(expected["VAR00002"])
+ tm.assert_frame_equal(df, expected)
+
+ df = pd.read_spss(fname, convert_categoricals=False)
+ expected = pd.DataFrame({"VAR00002": [1.0, np.nan]})
+ tm.assert_frame_equal(df, expected)
+
+
+@pytest.mark.filterwarnings("ignore::pandas.errors.ChainedAssignmentError")
+def test_spss_labelled_str(datapath):
+ # test file from the Haven project (https://haven.tidyverse.org/)
+ # Licence at LICENSES/HAVEN_LICENSE, LICENSES/HAVEN_MIT
+ fname = datapath("io", "data", "spss", "labelled-str.sav")
+
+ df = pd.read_spss(fname, convert_categoricals=True)
+ expected = pd.DataFrame({"gender": ["Male", "Female"]})
+ expected["gender"] = pd.Categorical(expected["gender"])
+ tm.assert_frame_equal(df, expected)
+
+ df = pd.read_spss(fname, convert_categoricals=False)
+ expected = pd.DataFrame({"gender": ["M", "F"]})
+ tm.assert_frame_equal(df, expected)
+
+
+@pytest.mark.filterwarnings("ignore::pandas.errors.ChainedAssignmentError")
+def test_spss_kwargs(datapath):
+ # test file from the Haven project (https://haven.tidyverse.org/)
+ # Licence at LICENSES/HAVEN_LICENSE, LICENSES/HAVEN_MIT
+ fname = datapath("io", "data", "spss", "labelled-str.sav")
+
+ df = pd.read_spss(fname, convert_categoricals=True, row_limit=1)
+ expected = pd.DataFrame({"gender": ["Male"]}, dtype="category")
+ tm.assert_frame_equal(df, expected)
+
+ df = pd.read_spss(fname, convert_categoricals=False, row_offset=1)
+ expected = pd.DataFrame({"gender": ["F"]})
+ tm.assert_frame_equal(df, expected)
+
+
+@pytest.mark.filterwarnings("ignore::pandas.errors.ChainedAssignmentError")
+def test_spss_umlauts(datapath):
+ # test file from the Haven project (https://haven.tidyverse.org/)
+ # Licence at LICENSES/HAVEN_LICENSE, LICENSES/HAVEN_MIT
+ fname = datapath("io", "data", "spss", "umlauts.sav")
+
+ df = pd.read_spss(fname, convert_categoricals=True)
+ expected = pd.DataFrame(
+ {"var1": ["the ä umlaut", "the ü umlaut", "the ä umlaut", "the ö umlaut"]}
+ )
+ expected["var1"] = pd.Categorical(expected["var1"])
+ tm.assert_frame_equal(df, expected)
+
+ df = pd.read_spss(fname, convert_categoricals=False)
+ expected = pd.DataFrame({"var1": [1.0, 2.0, 1.0, 3.0]})
+ tm.assert_frame_equal(df, expected)
+
+
+def test_spss_usecols(datapath):
+ # usecols must be list-like
+ fname = datapath("io", "data", "spss", "labelled-num.sav")
+
+ with pytest.raises(TypeError, match="usecols must be list-like."):
+ pd.read_spss(fname, usecols="VAR00002")
+
+
+def test_spss_umlauts_dtype_backend(datapath, dtype_backend):
+ # test file from the Haven project (https://haven.tidyverse.org/)
+ # Licence at LICENSES/HAVEN_LICENSE, LICENSES/HAVEN_MIT
+ fname = datapath("io", "data", "spss", "umlauts.sav")
+
+ df = pd.read_spss(fname, convert_categoricals=False, dtype_backend=dtype_backend)
+ expected = pd.DataFrame({"var1": [1.0, 2.0, 1.0, 3.0]}, dtype="Int64")
+
+ if dtype_backend == "pyarrow":
+ pa = pytest.importorskip("pyarrow")
+
+ from pandas.arrays import ArrowExtensionArray
+
+ expected = pd.DataFrame(
+ {
+ col: ArrowExtensionArray(pa.array(expected[col], from_pandas=True))
+ for col in expected.columns
+ }
+ )
+
+ tm.assert_frame_equal(df, expected)
+
+
+def test_invalid_dtype_backend():
+ msg = (
+ "dtype_backend numpy is invalid, only 'numpy_nullable' and "
+ "'pyarrow' are allowed."
+ )
+ with pytest.raises(ValueError, match=msg):
+ pd.read_spss("test", dtype_backend="numpy")
+
+
+@pytest.mark.filterwarnings("ignore::pandas.errors.ChainedAssignmentError")
+def test_spss_metadata(datapath):
+ # GH 54264
+ fname = datapath("io", "data", "spss", "labelled-num.sav")
+
+ df = pd.read_spss(fname)
+ metadata = {
+ "column_names": ["VAR00002"],
+ "column_labels": [None],
+ "column_names_to_labels": {"VAR00002": None},
+ "file_encoding": "UTF-8",
+ "number_columns": 1,
+ "number_rows": 1,
+ "variable_value_labels": {"VAR00002": {1.0: "This is one"}},
+ "value_labels": {"labels0": {1.0: "This is one"}},
+ "variable_to_label": {"VAR00002": "labels0"},
+ "notes": [],
+ "original_variable_types": {"VAR00002": "F8.0"},
+ "readstat_variable_types": {"VAR00002": "double"},
+ "table_name": None,
+ "missing_ranges": {},
+ "missing_user_values": {},
+ "variable_storage_width": {"VAR00002": 8},
+ "variable_display_width": {"VAR00002": 8},
+ "variable_alignment": {"VAR00002": "unknown"},
+ "variable_measure": {"VAR00002": "unknown"},
+ "file_label": None,
+ "file_format": "sav/zsav",
+ "creation_time": datetime.datetime(2015, 2, 6, 14, 33, 36),
+ "modification_time": datetime.datetime(2015, 2, 6, 14, 33, 36),
+ "mr_sets": {},
+ }
+ tm.assert_dict_equal(df.attrs, metadata)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_sql.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_sql.py
new file mode 100644
index 0000000000000000000000000000000000000000..c879c4b7796632ee8dc2042a8469d74780617758
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_sql.py
@@ -0,0 +1,4426 @@
+from __future__ import annotations
+
+import contextlib
+import csv
+from datetime import (
+ date,
+ datetime,
+ time,
+ timedelta,
+)
+from decimal import Decimal
+from io import StringIO
+from pathlib import Path
+import sqlite3
+from typing import TYPE_CHECKING
+import uuid
+
+import numpy as np
+import pytest
+
+from pandas._config import using_string_dtype
+
+from pandas._libs import lib
+from pandas.compat import pa_version_under14p1
+from pandas.compat._optional import import_optional_dependency
+import pandas.util._test_decorators as td
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ MultiIndex,
+ Series,
+ Timestamp,
+ concat,
+ date_range,
+ isna,
+ to_datetime,
+ to_timedelta,
+)
+import pandas._testing as tm
+from pandas.util.version import Version
+
+from pandas.io import sql
+from pandas.io.sql import (
+ SQLAlchemyEngine,
+ SQLDatabase,
+ SQLiteDatabase,
+ get_engine,
+ pandasSQL_builder,
+ read_sql_query,
+ read_sql_table,
+)
+
+if TYPE_CHECKING:
+ import sqlalchemy
+
+
+pytestmark = [
+ pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+ ),
+ pytest.mark.single_cpu,
+]
+
+
+@pytest.fixture
+def sql_strings():
+ return {
+ "read_parameters": {
+ "sqlite": "SELECT * FROM iris WHERE Name=? AND SepalLength=?",
+ "mysql": "SELECT * FROM iris WHERE `Name`=%s AND `SepalLength`=%s",
+ "postgresql": 'SELECT * FROM iris WHERE "Name"=%s AND "SepalLength"=%s',
+ },
+ "read_named_parameters": {
+ "sqlite": """
+ SELECT * FROM iris WHERE Name=:name AND SepalLength=:length
+ """,
+ "mysql": """
+ SELECT * FROM iris WHERE
+ `Name`=%(name)s AND `SepalLength`=%(length)s
+ """,
+ "postgresql": """
+ SELECT * FROM iris WHERE
+ "Name"=%(name)s AND "SepalLength"=%(length)s
+ """,
+ },
+ "read_no_parameters_with_percent": {
+ "sqlite": "SELECT * FROM iris WHERE Name LIKE '%'",
+ "mysql": "SELECT * FROM iris WHERE `Name` LIKE '%'",
+ "postgresql": "SELECT * FROM iris WHERE \"Name\" LIKE '%'",
+ },
+ }
+
+
+def iris_table_metadata():
+ import sqlalchemy
+ from sqlalchemy import (
+ Column,
+ Double,
+ Float,
+ MetaData,
+ String,
+ Table,
+ )
+
+ dtype = Double if Version(sqlalchemy.__version__) >= Version("2.0.0") else Float
+ metadata = MetaData()
+ iris = Table(
+ "iris",
+ metadata,
+ Column("SepalLength", dtype),
+ Column("SepalWidth", dtype),
+ Column("PetalLength", dtype),
+ Column("PetalWidth", dtype),
+ Column("Name", String(200)),
+ )
+ return iris
+
+
+def create_and_load_iris_sqlite3(conn, iris_file: Path):
+ stmt = """CREATE TABLE iris (
+ "SepalLength" REAL,
+ "SepalWidth" REAL,
+ "PetalLength" REAL,
+ "PetalWidth" REAL,
+ "Name" TEXT
+ )"""
+
+ cur = conn.cursor()
+ cur.execute(stmt)
+ with iris_file.open(newline=None, encoding="utf-8") as csvfile:
+ reader = csv.reader(csvfile)
+ next(reader)
+ stmt = "INSERT INTO iris VALUES(?, ?, ?, ?, ?)"
+ # ADBC requires explicit types - no implicit str -> float conversion
+ records = []
+ records = [
+ (
+ float(row[0]),
+ float(row[1]),
+ float(row[2]),
+ float(row[3]),
+ row[4],
+ )
+ for row in reader
+ ]
+
+ cur.executemany(stmt, records)
+ cur.close()
+
+ conn.commit()
+
+
+def create_and_load_iris_postgresql(conn, iris_file: Path):
+ stmt = """CREATE TABLE iris (
+ "SepalLength" DOUBLE PRECISION,
+ "SepalWidth" DOUBLE PRECISION,
+ "PetalLength" DOUBLE PRECISION,
+ "PetalWidth" DOUBLE PRECISION,
+ "Name" TEXT
+ )"""
+ with conn.cursor() as cur:
+ cur.execute(stmt)
+ with iris_file.open(newline=None, encoding="utf-8") as csvfile:
+ reader = csv.reader(csvfile)
+ next(reader)
+ stmt = "INSERT INTO iris VALUES($1, $2, $3, $4, $5)"
+ # ADBC requires explicit types - no implicit str -> float conversion
+ records = [
+ (
+ float(row[0]),
+ float(row[1]),
+ float(row[2]),
+ float(row[3]),
+ row[4],
+ )
+ for row in reader
+ ]
+
+ cur.executemany(stmt, records)
+
+ conn.commit()
+
+
+def create_and_load_iris(conn, iris_file: Path):
+ from sqlalchemy import insert
+
+ iris = iris_table_metadata()
+
+ with iris_file.open(newline=None, encoding="utf-8") as csvfile:
+ reader = csv.reader(csvfile)
+ header = next(reader)
+ params = [dict(zip(header, row)) for row in reader]
+ stmt = insert(iris).values(params)
+ with conn.begin() as con:
+ iris.drop(con, checkfirst=True)
+ iris.create(bind=con)
+ con.execute(stmt)
+
+
+def create_and_load_iris_view(conn):
+ stmt = "CREATE VIEW iris_view AS SELECT * FROM iris"
+ if isinstance(conn, sqlite3.Connection):
+ cur = conn.cursor()
+ cur.execute(stmt)
+ else:
+ adbc = import_optional_dependency("adbc_driver_manager.dbapi", errors="ignore")
+ if adbc and isinstance(conn, adbc.Connection):
+ with conn.cursor() as cur:
+ cur.execute(stmt)
+ conn.commit()
+ else:
+ from sqlalchemy import text
+
+ stmt = text(stmt)
+ with conn.begin() as con:
+ con.execute(stmt)
+
+
+def types_table_metadata(dialect: str):
+ from sqlalchemy import (
+ TEXT,
+ Boolean,
+ Column,
+ DateTime,
+ Float,
+ Integer,
+ MetaData,
+ Table,
+ )
+
+ date_type = TEXT if dialect == "sqlite" else DateTime
+ bool_type = Integer if dialect == "sqlite" else Boolean
+ metadata = MetaData()
+ types = Table(
+ "types",
+ metadata,
+ Column("TextCol", TEXT),
+ # error: Cannot infer type argument 1 of "Column"
+ Column("DateCol", date_type), # type: ignore[misc]
+ Column("IntDateCol", Integer),
+ Column("IntDateOnlyCol", Integer),
+ Column("FloatCol", Float),
+ Column("IntCol", Integer),
+ # error: Cannot infer type argument 1 of "Column"
+ Column("BoolCol", bool_type), # type: ignore[misc]
+ Column("IntColWithNull", Integer),
+ # error: Cannot infer type argument 1 of "Column"
+ Column("BoolColWithNull", bool_type), # type: ignore[misc]
+ )
+ return types
+
+
+def create_and_load_types_sqlite3(conn, types_data: list[dict]):
+ stmt = """CREATE TABLE types (
+ "TextCol" TEXT,
+ "DateCol" TEXT,
+ "IntDateCol" INTEGER,
+ "IntDateOnlyCol" INTEGER,
+ "FloatCol" REAL,
+ "IntCol" INTEGER,
+ "BoolCol" INTEGER,
+ "IntColWithNull" INTEGER,
+ "BoolColWithNull" INTEGER
+ )"""
+
+ ins_stmt = """
+ INSERT INTO types
+ VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)
+ """
+
+ if isinstance(conn, sqlite3.Connection):
+ cur = conn.cursor()
+ cur.execute(stmt)
+ cur.executemany(ins_stmt, types_data)
+ else:
+ with conn.cursor() as cur:
+ cur.execute(stmt)
+ cur.executemany(ins_stmt, types_data)
+
+ conn.commit()
+
+
+def create_and_load_types_postgresql(conn, types_data: list[dict]):
+ with conn.cursor() as cur:
+ stmt = """CREATE TABLE types (
+ "TextCol" TEXT,
+ "DateCol" TIMESTAMP,
+ "IntDateCol" INTEGER,
+ "IntDateOnlyCol" INTEGER,
+ "FloatCol" DOUBLE PRECISION,
+ "IntCol" INTEGER,
+ "BoolCol" BOOLEAN,
+ "IntColWithNull" INTEGER,
+ "BoolColWithNull" BOOLEAN
+ )"""
+ cur.execute(stmt)
+
+ stmt = """
+ INSERT INTO types
+ VALUES($1, $2::timestamp, $3, $4, $5, $6, $7, $8, $9)
+ """
+
+ cur.executemany(stmt, types_data)
+
+ conn.commit()
+
+
+def create_and_load_types(conn, types_data: list[dict], dialect: str):
+ from sqlalchemy import insert
+ from sqlalchemy.engine import Engine
+
+ types = types_table_metadata(dialect)
+
+ stmt = insert(types).values(types_data)
+ if isinstance(conn, Engine):
+ with conn.connect() as conn:
+ with conn.begin():
+ types.drop(conn, checkfirst=True)
+ types.create(bind=conn)
+ conn.execute(stmt)
+ else:
+ with conn.begin():
+ types.drop(conn, checkfirst=True)
+ types.create(bind=conn)
+ conn.execute(stmt)
+
+
+def create_and_load_postgres_datetz(conn):
+ from sqlalchemy import (
+ Column,
+ DateTime,
+ MetaData,
+ Table,
+ insert,
+ )
+ from sqlalchemy.engine import Engine
+
+ metadata = MetaData()
+ datetz = Table("datetz", metadata, Column("DateColWithTz", DateTime(timezone=True)))
+ datetz_data = [
+ {
+ "DateColWithTz": "2000-01-01 00:00:00-08:00",
+ },
+ {
+ "DateColWithTz": "2000-06-01 00:00:00-07:00",
+ },
+ ]
+ stmt = insert(datetz).values(datetz_data)
+ if isinstance(conn, Engine):
+ with conn.connect() as conn:
+ with conn.begin():
+ datetz.drop(conn, checkfirst=True)
+ datetz.create(bind=conn)
+ conn.execute(stmt)
+ else:
+ with conn.begin():
+ datetz.drop(conn, checkfirst=True)
+ datetz.create(bind=conn)
+ conn.execute(stmt)
+
+ # "2000-01-01 00:00:00-08:00" should convert to
+ # "2000-01-01 08:00:00"
+ # "2000-06-01 00:00:00-07:00" should convert to
+ # "2000-06-01 07:00:00"
+ # GH 6415
+ expected_data = [
+ Timestamp("2000-01-01 08:00:00", tz="UTC"),
+ Timestamp("2000-06-01 07:00:00", tz="UTC"),
+ ]
+ return Series(expected_data, name="DateColWithTz").astype("M8[us, UTC]")
+
+
+def check_iris_frame(frame: DataFrame):
+ pytype = frame.dtypes.iloc[0].type
+ row = frame.iloc[0]
+ assert issubclass(pytype, np.floating)
+ tm.assert_series_equal(
+ row, Series([5.1, 3.5, 1.4, 0.2, "Iris-setosa"], index=frame.columns, name=0)
+ )
+ assert frame.shape in ((150, 5), (8, 5))
+
+
+def count_rows(conn, table_name: str):
+ stmt = f"SELECT count(*) AS count_1 FROM {table_name}"
+ adbc = import_optional_dependency("adbc_driver_manager.dbapi", errors="ignore")
+ if isinstance(conn, sqlite3.Connection):
+ cur = conn.cursor()
+ return cur.execute(stmt).fetchone()[0]
+ elif adbc and isinstance(conn, adbc.Connection):
+ with conn.cursor() as cur:
+ cur.execute(stmt)
+ return cur.fetchone()[0]
+ else:
+ from sqlalchemy import create_engine
+ from sqlalchemy.engine import Engine
+
+ if isinstance(conn, str):
+ try:
+ engine = create_engine(conn)
+ with engine.connect() as conn:
+ return conn.exec_driver_sql(stmt).scalar_one()
+ finally:
+ engine.dispose()
+ elif isinstance(conn, Engine):
+ with conn.connect() as conn:
+ return conn.exec_driver_sql(stmt).scalar_one()
+ else:
+ return conn.exec_driver_sql(stmt).scalar_one()
+
+
+@pytest.fixture
+def iris_path(datapath):
+ iris_path = datapath("io", "data", "csv", "iris.csv")
+ return Path(iris_path)
+
+
+@pytest.fixture
+def types_data():
+ return [
+ {
+ "TextCol": "first",
+ "DateCol": "2000-01-03 00:00:00",
+ "IntDateCol": 535852800,
+ "IntDateOnlyCol": 20101010,
+ "FloatCol": 10.10,
+ "IntCol": 1,
+ "BoolCol": False,
+ "IntColWithNull": 1,
+ "BoolColWithNull": False,
+ },
+ {
+ "TextCol": "first",
+ "DateCol": "2000-01-04 00:00:00",
+ "IntDateCol": 1356998400,
+ "IntDateOnlyCol": 20101212,
+ "FloatCol": 10.10,
+ "IntCol": 1,
+ "BoolCol": False,
+ "IntColWithNull": None,
+ "BoolColWithNull": None,
+ },
+ ]
+
+
+@pytest.fixture
+def types_data_frame(types_data):
+ dtypes = {
+ "TextCol": "str",
+ "DateCol": "str",
+ "IntDateCol": "int64",
+ "IntDateOnlyCol": "int64",
+ "FloatCol": "float",
+ "IntCol": "int64",
+ "BoolCol": "int64",
+ "IntColWithNull": "float",
+ "BoolColWithNull": "float",
+ }
+ df = DataFrame(types_data)
+ return df[dtypes.keys()].astype(dtypes)
+
+
+@pytest.fixture
+def test_frame1():
+ columns = ["index", "A", "B", "C", "D"]
+ data = [
+ (
+ "2000-01-03 00:00:00",
+ 0.980268513777,
+ 3.68573087906,
+ -0.364216805298,
+ -1.15973806169,
+ ),
+ (
+ "2000-01-04 00:00:00",
+ 1.04791624281,
+ -0.0412318367011,
+ -0.16181208307,
+ 0.212549316967,
+ ),
+ (
+ "2000-01-05 00:00:00",
+ 0.498580885705,
+ 0.731167677815,
+ -0.537677223318,
+ 1.34627041952,
+ ),
+ (
+ "2000-01-06 00:00:00",
+ 1.12020151869,
+ 1.56762092543,
+ 0.00364077397681,
+ 0.67525259227,
+ ),
+ ]
+ return DataFrame(data, columns=columns)
+
+
+@pytest.fixture
+def test_frame3():
+ columns = ["index", "A", "B"]
+ data = [
+ ("2000-01-03 00:00:00", 2**31 - 1, -1.987670),
+ ("2000-01-04 00:00:00", -29, -0.0412318367011),
+ ("2000-01-05 00:00:00", 20000, 0.731167677815),
+ ("2000-01-06 00:00:00", -290867, 1.56762092543),
+ ]
+ return DataFrame(data, columns=columns)
+
+
+def get_all_views(conn):
+ if isinstance(conn, sqlite3.Connection):
+ c = conn.execute("SELECT name FROM sqlite_master WHERE type='view'")
+ return [view[0] for view in c.fetchall()]
+ else:
+ adbc = import_optional_dependency("adbc_driver_manager.dbapi", errors="ignore")
+ if adbc and isinstance(conn, adbc.Connection):
+ results = []
+ info = conn.adbc_get_objects().read_all().to_pylist()
+ for catalog in info:
+ catalog["catalog_name"]
+ for schema in catalog["catalog_db_schemas"]:
+ schema["db_schema_name"]
+ for table in schema["db_schema_tables"]:
+ if table["table_type"] == "view":
+ view_name = table["table_name"]
+ results.append(view_name)
+
+ return results
+ else:
+ from sqlalchemy import inspect
+
+ return inspect(conn).get_view_names()
+
+
+def get_all_tables(conn):
+ if isinstance(conn, sqlite3.Connection):
+ c = conn.execute("SELECT name FROM sqlite_master WHERE type='table'")
+ return [table[0] for table in c.fetchall()]
+ else:
+ adbc = import_optional_dependency("adbc_driver_manager.dbapi", errors="ignore")
+
+ if adbc and isinstance(conn, adbc.Connection):
+ results = []
+ info = conn.adbc_get_objects().read_all().to_pylist()
+ for catalog in info:
+ for schema in catalog["catalog_db_schemas"]:
+ for table in schema["db_schema_tables"]:
+ if table["table_type"] == "table":
+ table_name = table["table_name"]
+ results.append(table_name)
+
+ return results
+ else:
+ from sqlalchemy import inspect
+
+ return inspect(conn).get_table_names()
+
+
+def drop_table(
+ table_name: str,
+ conn: sqlite3.Connection | sqlalchemy.engine.Engine | sqlalchemy.engine.Connection,
+):
+ if isinstance(conn, sqlite3.Connection):
+ conn.execute(f"DROP TABLE IF EXISTS {sql._get_valid_sqlite_name(table_name)}")
+ conn.commit()
+
+ else:
+ adbc = import_optional_dependency("adbc_driver_manager.dbapi", errors="ignore")
+ if adbc and isinstance(conn, adbc.Connection):
+ with conn.cursor() as cur:
+ cur.execute(f'DROP TABLE IF EXISTS "{table_name}"')
+ else:
+ with conn.begin() as con:
+ with sql.SQLDatabase(con) as db:
+ db.drop_table(table_name)
+
+
+def drop_view(
+ view_name: str,
+ conn: sqlite3.Connection | sqlalchemy.engine.Engine | sqlalchemy.engine.Connection,
+):
+ import sqlalchemy
+
+ if isinstance(conn, sqlite3.Connection):
+ conn.execute(f"DROP VIEW IF EXISTS {sql._get_valid_sqlite_name(view_name)}")
+ conn.commit()
+ else:
+ adbc = import_optional_dependency("adbc_driver_manager.dbapi", errors="ignore")
+ if adbc and isinstance(conn, adbc.Connection):
+ with conn.cursor() as cur:
+ cur.execute(f'DROP VIEW IF EXISTS "{view_name}"')
+ else:
+ quoted_view = conn.engine.dialect.identifier_preparer.quote_identifier(
+ view_name
+ )
+ stmt = sqlalchemy.text(f"DROP VIEW IF EXISTS {quoted_view}")
+ with conn.begin() as con:
+ con.execute(stmt) # type: ignore[union-attr]
+
+
+@pytest.fixture
+def mysql_pymysql_engine():
+ sqlalchemy = pytest.importorskip("sqlalchemy")
+ pymysql = pytest.importorskip("pymysql")
+ engine = sqlalchemy.create_engine(
+ "mysql+pymysql://root@localhost:3306/pandas",
+ connect_args={"client_flag": pymysql.constants.CLIENT.MULTI_STATEMENTS},
+ poolclass=sqlalchemy.pool.NullPool,
+ )
+ yield engine
+ for view in get_all_views(engine):
+ drop_view(view, engine)
+ for tbl in get_all_tables(engine):
+ drop_table(tbl, engine)
+ engine.dispose()
+
+
+@pytest.fixture
+def mysql_pymysql_engine_iris(mysql_pymysql_engine, iris_path):
+ create_and_load_iris(mysql_pymysql_engine, iris_path)
+ create_and_load_iris_view(mysql_pymysql_engine)
+ return mysql_pymysql_engine
+
+
+@pytest.fixture
+def mysql_pymysql_engine_types(mysql_pymysql_engine, types_data):
+ create_and_load_types(mysql_pymysql_engine, types_data, "mysql")
+ return mysql_pymysql_engine
+
+
+@pytest.fixture
+def mysql_pymysql_conn(mysql_pymysql_engine):
+ with mysql_pymysql_engine.connect() as conn:
+ yield conn
+
+
+@pytest.fixture
+def mysql_pymysql_conn_iris(mysql_pymysql_engine_iris):
+ with mysql_pymysql_engine_iris.connect() as conn:
+ yield conn
+
+
+@pytest.fixture
+def mysql_pymysql_conn_types(mysql_pymysql_engine_types):
+ with mysql_pymysql_engine_types.connect() as conn:
+ yield conn
+
+
+@pytest.fixture
+def postgresql_psycopg2_engine():
+ sqlalchemy = pytest.importorskip("sqlalchemy")
+ pytest.importorskip("psycopg2")
+ engine = sqlalchemy.create_engine(
+ "postgresql+psycopg2://postgres:postgres@localhost:5432/pandas",
+ poolclass=sqlalchemy.pool.NullPool,
+ )
+ yield engine
+ for view in get_all_views(engine):
+ drop_view(view, engine)
+ for tbl in get_all_tables(engine):
+ drop_table(tbl, engine)
+ engine.dispose()
+
+
+@pytest.fixture
+def postgresql_psycopg2_engine_iris(postgresql_psycopg2_engine, iris_path):
+ create_and_load_iris(postgresql_psycopg2_engine, iris_path)
+ create_and_load_iris_view(postgresql_psycopg2_engine)
+ return postgresql_psycopg2_engine
+
+
+@pytest.fixture
+def postgresql_psycopg2_engine_types(postgresql_psycopg2_engine, types_data):
+ create_and_load_types(postgresql_psycopg2_engine, types_data, "postgres")
+ return postgresql_psycopg2_engine
+
+
+@pytest.fixture
+def postgresql_psycopg2_conn(postgresql_psycopg2_engine):
+ with postgresql_psycopg2_engine.connect() as conn:
+ yield conn
+
+
+@pytest.fixture
+def postgresql_adbc_conn():
+ pytest.importorskip("pyarrow")
+ pytest.importorskip("adbc_driver_postgresql")
+ from adbc_driver_postgresql import dbapi
+
+ uri = "postgresql://postgres:postgres@localhost:5432/pandas"
+ with dbapi.connect(uri) as conn:
+ yield conn
+ for view in get_all_views(conn):
+ drop_view(view, conn)
+ for tbl in get_all_tables(conn):
+ drop_table(tbl, conn)
+ conn.commit()
+
+
+@pytest.fixture
+def postgresql_adbc_iris(postgresql_adbc_conn, iris_path):
+ import adbc_driver_manager as mgr
+
+ conn = postgresql_adbc_conn
+
+ try:
+ conn.adbc_get_table_schema("iris")
+ except mgr.ProgrammingError:
+ conn.rollback()
+ create_and_load_iris_postgresql(conn, iris_path)
+ try:
+ conn.adbc_get_table_schema("iris_view")
+ except mgr.ProgrammingError: # note arrow-adbc issue 1022
+ conn.rollback()
+ create_and_load_iris_view(conn)
+ return conn
+
+
+@pytest.fixture
+def postgresql_adbc_types(postgresql_adbc_conn, types_data):
+ import adbc_driver_manager as mgr
+
+ conn = postgresql_adbc_conn
+
+ try:
+ conn.adbc_get_table_schema("types")
+ except mgr.ProgrammingError:
+ conn.rollback()
+ new_data = [tuple(entry.values()) for entry in types_data]
+
+ create_and_load_types_postgresql(conn, new_data)
+
+ return conn
+
+
+@pytest.fixture
+def postgresql_psycopg2_conn_iris(postgresql_psycopg2_engine_iris):
+ with postgresql_psycopg2_engine_iris.connect() as conn:
+ yield conn
+
+
+@pytest.fixture
+def postgresql_psycopg2_conn_types(postgresql_psycopg2_engine_types):
+ with postgresql_psycopg2_engine_types.connect() as conn:
+ yield conn
+
+
+@pytest.fixture
+def sqlite_str(temp_file):
+ pytest.importorskip("sqlalchemy")
+ return f"sqlite:///{temp_file}"
+
+
+@pytest.fixture
+def sqlite_engine(sqlite_str):
+ sqlalchemy = pytest.importorskip("sqlalchemy")
+ engine = sqlalchemy.create_engine(sqlite_str, poolclass=sqlalchemy.pool.NullPool)
+ yield engine
+ for view in get_all_views(engine):
+ drop_view(view, engine)
+ for tbl in get_all_tables(engine):
+ drop_table(tbl, engine)
+ engine.dispose()
+
+
+@pytest.fixture
+def sqlite_conn(sqlite_engine):
+ with sqlite_engine.connect() as conn:
+ yield conn
+
+
+@pytest.fixture
+def sqlite_str_iris(sqlite_str, iris_path):
+ sqlalchemy = pytest.importorskip("sqlalchemy")
+ engine = sqlalchemy.create_engine(sqlite_str)
+ create_and_load_iris(engine, iris_path)
+ create_and_load_iris_view(engine)
+ engine.dispose()
+ return sqlite_str
+
+
+@pytest.fixture
+def sqlite_engine_iris(sqlite_engine, iris_path):
+ create_and_load_iris(sqlite_engine, iris_path)
+ create_and_load_iris_view(sqlite_engine)
+ return sqlite_engine
+
+
+@pytest.fixture
+def sqlite_conn_iris(sqlite_engine_iris):
+ with sqlite_engine_iris.connect() as conn:
+ yield conn
+
+
+@pytest.fixture
+def sqlite_str_types(sqlite_str, types_data):
+ sqlalchemy = pytest.importorskip("sqlalchemy")
+ engine = sqlalchemy.create_engine(sqlite_str)
+ create_and_load_types(engine, types_data, "sqlite")
+ engine.dispose()
+ return sqlite_str
+
+
+@pytest.fixture
+def sqlite_engine_types(sqlite_engine, types_data):
+ create_and_load_types(sqlite_engine, types_data, "sqlite")
+ return sqlite_engine
+
+
+@pytest.fixture
+def sqlite_conn_types(sqlite_engine_types):
+ with sqlite_engine_types.connect() as conn:
+ yield conn
+
+
+@pytest.fixture
+def sqlite_adbc_conn(temp_file):
+ pytest.importorskip("pyarrow")
+ pytest.importorskip("adbc_driver_sqlite")
+ from adbc_driver_sqlite import dbapi
+
+ uri = f"file:{temp_file}"
+ with dbapi.connect(uri) as conn:
+ yield conn
+ for view in get_all_views(conn):
+ drop_view(view, conn)
+ for tbl in get_all_tables(conn):
+ drop_table(tbl, conn)
+ conn.commit()
+
+
+@pytest.fixture
+def sqlite_adbc_iris(sqlite_adbc_conn, iris_path):
+ import adbc_driver_manager as mgr
+
+ conn = sqlite_adbc_conn
+ try:
+ conn.adbc_get_table_schema("iris")
+ except mgr.ProgrammingError:
+ conn.rollback()
+ create_and_load_iris_sqlite3(conn, iris_path)
+ try:
+ conn.adbc_get_table_schema("iris_view")
+ except mgr.ProgrammingError:
+ conn.rollback()
+ create_and_load_iris_view(conn)
+ return conn
+
+
+@pytest.fixture
+def sqlite_adbc_types(sqlite_adbc_conn, types_data):
+ import adbc_driver_manager as mgr
+
+ conn = sqlite_adbc_conn
+ try:
+ conn.adbc_get_table_schema("types")
+ except mgr.ProgrammingError:
+ conn.rollback()
+ new_data = []
+ for entry in types_data:
+ entry["BoolCol"] = int(entry["BoolCol"])
+ if entry["BoolColWithNull"] is not None:
+ entry["BoolColWithNull"] = int(entry["BoolColWithNull"])
+ new_data.append(tuple(entry.values()))
+
+ create_and_load_types_sqlite3(conn, new_data)
+ conn.commit()
+
+ return conn
+
+
+@pytest.fixture
+def sqlite_buildin():
+ with contextlib.closing(sqlite3.connect(":memory:")) as closing_conn:
+ with closing_conn as conn:
+ yield conn
+
+
+@pytest.fixture
+def sqlite_buildin_iris(sqlite_buildin, iris_path):
+ create_and_load_iris_sqlite3(sqlite_buildin, iris_path)
+ create_and_load_iris_view(sqlite_buildin)
+ return sqlite_buildin
+
+
+@pytest.fixture
+def sqlite_buildin_types(sqlite_buildin, types_data):
+ types_data = [tuple(entry.values()) for entry in types_data]
+ create_and_load_types_sqlite3(sqlite_buildin, types_data)
+ return sqlite_buildin
+
+
+mysql_connectable = [
+ pytest.param("mysql_pymysql_engine", marks=pytest.mark.db),
+ pytest.param("mysql_pymysql_conn", marks=pytest.mark.db),
+]
+
+mysql_connectable_iris = [
+ pytest.param("mysql_pymysql_engine_iris", marks=pytest.mark.db),
+ pytest.param("mysql_pymysql_conn_iris", marks=pytest.mark.db),
+]
+
+mysql_connectable_types = [
+ pytest.param("mysql_pymysql_engine_types", marks=pytest.mark.db),
+ pytest.param("mysql_pymysql_conn_types", marks=pytest.mark.db),
+]
+
+postgresql_connectable = [
+ pytest.param("postgresql_psycopg2_engine", marks=pytest.mark.db),
+ pytest.param("postgresql_psycopg2_conn", marks=pytest.mark.db),
+]
+
+postgresql_connectable_iris = [
+ pytest.param("postgresql_psycopg2_engine_iris", marks=pytest.mark.db),
+ pytest.param("postgresql_psycopg2_conn_iris", marks=pytest.mark.db),
+]
+
+postgresql_connectable_types = [
+ pytest.param("postgresql_psycopg2_engine_types", marks=pytest.mark.db),
+ pytest.param("postgresql_psycopg2_conn_types", marks=pytest.mark.db),
+]
+
+sqlite_connectable = [
+ "sqlite_engine",
+ "sqlite_conn",
+ "sqlite_str",
+]
+
+sqlite_connectable_iris = [
+ "sqlite_engine_iris",
+ "sqlite_conn_iris",
+ "sqlite_str_iris",
+]
+
+sqlite_connectable_types = [
+ "sqlite_engine_types",
+ "sqlite_conn_types",
+ "sqlite_str_types",
+]
+
+sqlalchemy_connectable = mysql_connectable + postgresql_connectable + sqlite_connectable
+
+sqlalchemy_connectable_iris = (
+ mysql_connectable_iris + postgresql_connectable_iris + sqlite_connectable_iris
+)
+
+sqlalchemy_connectable_types = (
+ mysql_connectable_types + postgresql_connectable_types + sqlite_connectable_types
+)
+
+adbc_connectable = [
+ "sqlite_adbc_conn",
+ pytest.param("postgresql_adbc_conn", marks=pytest.mark.db),
+]
+
+adbc_connectable_iris = [
+ pytest.param("postgresql_adbc_iris", marks=pytest.mark.db),
+ "sqlite_adbc_iris",
+]
+
+adbc_connectable_types = [
+ pytest.param("postgresql_adbc_types", marks=pytest.mark.db),
+ "sqlite_adbc_types",
+]
+
+
+all_connectable = [*sqlalchemy_connectable, "sqlite_buildin", *adbc_connectable]
+
+all_connectable_iris = [
+ *sqlalchemy_connectable_iris,
+ "sqlite_buildin_iris",
+ *adbc_connectable_iris,
+]
+
+all_connectable_types = [
+ *sqlalchemy_connectable_types,
+ "sqlite_buildin_types",
+ *adbc_connectable_types,
+]
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_dataframe_to_sql(conn, test_frame1, request):
+ # GH 51086 if conn is sqlite_engine
+ conn = request.getfixturevalue(conn)
+ test_frame1.to_sql(name="test", con=conn, if_exists="append", index=False)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_dataframe_to_sql_empty(conn, test_frame1, request):
+ if conn == "postgresql_adbc_conn" and not using_string_dtype():
+ adbc_pg = pytest.importorskip("adbc_driver_postgresql")
+ if Version(adbc_pg.__version__) < Version("1.11"):
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason=(
+ "postgres ADBC driver < 1.11 cannot insert index with null type"
+ ),
+ )
+ )
+
+ # GH 51086 if conn is sqlite_engine
+ conn = request.getfixturevalue(conn)
+ empty_df = test_frame1.iloc[:0]
+ empty_df.to_sql(name="test", con=conn, if_exists="append", index=False)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_dataframe_to_sql_arrow_dtypes(conn, request):
+ # GH 52046
+ pytest.importorskip("pyarrow")
+ df = DataFrame(
+ {
+ "int": pd.array([1], dtype="int8[pyarrow]"),
+ "datetime": pd.array(
+ [datetime(2023, 1, 1)], dtype="timestamp[ns][pyarrow]"
+ ),
+ "date": pd.array([date(2023, 1, 1)], dtype="date32[day][pyarrow]"),
+ "timedelta": pd.array([timedelta(1)], dtype="duration[ns][pyarrow]"),
+ "string": pd.array(["a"], dtype="string[pyarrow]"),
+ }
+ )
+
+ if "adbc" in conn:
+ if conn == "sqlite_adbc_conn":
+ df = df.drop(columns=["timedelta"])
+ if pa_version_under14p1:
+ exp_warning = DeprecationWarning
+ msg = "is_sparse is deprecated"
+ else:
+ exp_warning = None
+ msg = ""
+ else:
+ exp_warning = UserWarning
+ msg = "the 'timedelta'"
+
+ conn = request.getfixturevalue(conn)
+ with tm.assert_produces_warning(exp_warning, match=msg, check_stacklevel=False):
+ df.to_sql(name="test_arrow", con=conn, if_exists="replace", index=False)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_dataframe_to_sql_arrow_dtypes_missing(conn, request, nulls_fixture):
+ # GH 52046
+ pytest.importorskip("pyarrow")
+ if isinstance(nulls_fixture, Decimal):
+ pytest.skip(
+ # GH#61773
+ reason="Decimal('NaN') not supported in constructor for timestamp dtype"
+ )
+
+ df = DataFrame(
+ {
+ "datetime": pd.array(
+ [datetime(2023, 1, 1), nulls_fixture], dtype="timestamp[ns][pyarrow]"
+ ),
+ }
+ )
+ conn = request.getfixturevalue(conn)
+ df.to_sql(name="test_arrow", con=conn, if_exists="replace", index=False)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+@pytest.mark.parametrize("method", [None, "multi"])
+def test_to_sql(conn, method, test_frame1, request):
+ if method == "multi" and "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason="'method' not implemented for ADBC drivers", strict=True
+ )
+ )
+
+ conn = request.getfixturevalue(conn)
+ with pandasSQL_builder(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.to_sql(test_frame1, "test_frame", method=method)
+ assert pandasSQL.has_table("test_frame")
+ assert count_rows(conn, "test_frame") == len(test_frame1)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+@pytest.mark.parametrize(
+ "mode, num_row_coef", [("replace", 1), ("append", 2), ("delete_rows", 1)]
+)
+def test_to_sql_exist(conn, mode, num_row_coef, test_frame1, request):
+ conn = request.getfixturevalue(conn)
+ with pandasSQL_builder(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.to_sql(test_frame1, "test_frame", if_exists="fail")
+ pandasSQL.to_sql(test_frame1, "test_frame", if_exists=mode)
+ assert pandasSQL.has_table("test_frame")
+ assert count_rows(conn, "test_frame") == num_row_coef * len(test_frame1)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_to_sql_exist_fail(conn, test_frame1, request):
+ conn = request.getfixturevalue(conn)
+ with pandasSQL_builder(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.to_sql(test_frame1, "test_frame", if_exists="fail")
+ assert pandasSQL.has_table("test_frame")
+
+ msg = "Table 'test_frame' already exists"
+ with pytest.raises(ValueError, match=msg):
+ pandasSQL.to_sql(test_frame1, "test_frame", if_exists="fail")
+
+
+@pytest.mark.parametrize("conn", all_connectable_iris)
+def test_read_iris_query(conn, request):
+ conn = request.getfixturevalue(conn)
+ iris_frame = read_sql_query("SELECT * FROM iris", conn)
+ check_iris_frame(iris_frame)
+ iris_frame = pd.read_sql("SELECT * FROM iris", conn)
+ check_iris_frame(iris_frame)
+ iris_frame = pd.read_sql("SELECT * FROM iris where 0=1", conn)
+ assert iris_frame.shape == (0, 5)
+ assert "SepalWidth" in iris_frame.columns
+
+
+@pytest.mark.parametrize("conn", all_connectable_iris)
+def test_read_iris_query_chunksize(conn, request):
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason="'chunksize' not implemented for ADBC drivers",
+ strict=True,
+ )
+ )
+ conn = request.getfixturevalue(conn)
+ iris_frame = concat(read_sql_query("SELECT * FROM iris", conn, chunksize=7))
+ check_iris_frame(iris_frame)
+ iris_frame = concat(pd.read_sql("SELECT * FROM iris", conn, chunksize=7))
+ check_iris_frame(iris_frame)
+ iris_frame = concat(pd.read_sql("SELECT * FROM iris where 0=1", conn, chunksize=7))
+ assert iris_frame.shape == (0, 5)
+ assert "SepalWidth" in iris_frame.columns
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris)
+def test_read_iris_query_expression_with_parameter(conn, request):
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason="'chunksize' not implemented for ADBC drivers",
+ strict=True,
+ )
+ )
+ conn = request.getfixturevalue(conn)
+ from sqlalchemy import (
+ MetaData,
+ Table,
+ create_engine,
+ select,
+ )
+
+ metadata = MetaData()
+ autoload_con = create_engine(conn) if isinstance(conn, str) else conn
+ iris = Table("iris", metadata, autoload_with=autoload_con)
+ iris_frame = read_sql_query(
+ select(iris), conn, params={"name": "Iris-setosa", "length": 5.1}
+ )
+ check_iris_frame(iris_frame)
+ if isinstance(conn, str):
+ autoload_con.dispose()
+
+
+@pytest.mark.parametrize("conn", all_connectable_iris)
+def test_read_iris_query_string_with_parameter(conn, request, sql_strings):
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason="'chunksize' not implemented for ADBC drivers",
+ strict=True,
+ )
+ )
+
+ for db, query in sql_strings["read_parameters"].items():
+ if db in conn:
+ break
+ else:
+ raise KeyError(f"No part of {conn} found in sql_strings['read_parameters']")
+ conn = request.getfixturevalue(conn)
+ iris_frame = read_sql_query(query, conn, params=("Iris-setosa", 5.1))
+ check_iris_frame(iris_frame)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris)
+def test_read_iris_table(conn, request):
+ # GH 51015 if conn = sqlite_iris_str
+ conn = request.getfixturevalue(conn)
+ iris_frame = read_sql_table("iris", conn)
+ check_iris_frame(iris_frame)
+ iris_frame = pd.read_sql("iris", conn)
+ check_iris_frame(iris_frame)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris)
+def test_read_iris_table_chunksize(conn, request):
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(reason="chunksize argument NotImplemented with ADBC")
+ )
+ conn = request.getfixturevalue(conn)
+ iris_frame = concat(read_sql_table("iris", conn, chunksize=7))
+ check_iris_frame(iris_frame)
+ iris_frame = concat(pd.read_sql("iris", conn, chunksize=7))
+ check_iris_frame(iris_frame)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_to_sql_callable(conn, test_frame1, request):
+ conn = request.getfixturevalue(conn)
+
+ check = [] # used to double check function below is really being used
+
+ def sample(pd_table, conn, keys, data_iter):
+ check.append(1)
+ data = [dict(zip(keys, row)) for row in data_iter]
+ conn.execute(pd_table.table.insert(), data)
+
+ with pandasSQL_builder(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.to_sql(test_frame1, "test_frame", method=sample)
+ assert pandasSQL.has_table("test_frame")
+ assert check == [1]
+ assert count_rows(conn, "test_frame") == len(test_frame1)
+
+
+@pytest.mark.parametrize("conn", all_connectable_types)
+def test_default_type_conversion(conn, request):
+ conn_name = conn
+ if conn_name == "sqlite_buildin_types":
+ request.applymarker(
+ pytest.mark.xfail(
+ reason="sqlite_buildin connection does not implement read_sql_table"
+ )
+ )
+
+ conn = request.getfixturevalue(conn)
+ df = sql.read_sql_table("types", conn)
+
+ assert issubclass(df.FloatCol.dtype.type, np.floating)
+ assert issubclass(df.IntCol.dtype.type, np.integer)
+
+ # MySQL/sqlite has no real BOOL type
+ if "postgresql" in conn_name:
+ assert issubclass(df.BoolCol.dtype.type, np.bool_)
+ else:
+ assert issubclass(df.BoolCol.dtype.type, np.integer)
+
+ # Int column with NA values stays as float
+ assert issubclass(df.IntColWithNull.dtype.type, np.floating)
+
+ # Bool column with NA = int column with NA values => becomes float
+ if "postgresql" in conn_name:
+ assert issubclass(df.BoolColWithNull.dtype.type, object)
+ else:
+ assert issubclass(df.BoolColWithNull.dtype.type, np.floating)
+
+
+@pytest.mark.parametrize("conn", mysql_connectable)
+def test_read_procedure(conn, request):
+ conn = request.getfixturevalue(conn)
+
+ # GH 7324
+ # Although it is more an api test, it is added to the
+ # mysql tests as sqlite does not have stored procedures
+ from sqlalchemy import text
+ from sqlalchemy.engine import Engine
+
+ df = DataFrame({"a": [1, 2, 3], "b": [0.1, 0.2, 0.3]})
+ df.to_sql(name="test_frame", con=conn, index=False)
+
+ proc = """DROP PROCEDURE IF EXISTS get_testdb;
+
+ CREATE PROCEDURE get_testdb ()
+
+ BEGIN
+ SELECT * FROM test_frame;
+ END"""
+ proc = text(proc)
+ if isinstance(conn, Engine):
+ with conn.connect() as engine_conn:
+ with engine_conn.begin():
+ engine_conn.execute(proc)
+ else:
+ with conn.begin():
+ conn.execute(proc)
+
+ res1 = sql.read_sql_query("CALL get_testdb();", conn)
+ tm.assert_frame_equal(df, res1)
+
+ # test delegation to read_sql_query
+ res2 = sql.read_sql("CALL get_testdb();", conn)
+ tm.assert_frame_equal(df, res2)
+
+
+@pytest.mark.parametrize("conn", postgresql_connectable)
+@pytest.mark.parametrize("expected_count", [2, "Success!"])
+def test_copy_from_callable_insertion_method(conn, expected_count, request):
+ # GH 8953
+ # Example in io.rst found under _io.sql.method
+ # not available in sqlite, mysql
+ def psql_insert_copy(table, conn, keys, data_iter):
+ # gets a DBAPI connection that can provide a cursor
+ dbapi_conn = conn.connection
+ with dbapi_conn.cursor() as cur:
+ s_buf = StringIO()
+ writer = csv.writer(s_buf)
+ writer.writerows(data_iter)
+ s_buf.seek(0)
+
+ columns = ", ".join([f'"{k}"' for k in keys])
+ if table.schema:
+ table_name = f"{table.schema}.{table.name}"
+ else:
+ table_name = table.name
+
+ sql_query = f"COPY {table_name} ({columns}) FROM STDIN WITH CSV"
+ cur.copy_expert(sql=sql_query, file=s_buf)
+ return expected_count
+
+ conn = request.getfixturevalue(conn)
+ expected = DataFrame({"col1": [1, 2], "col2": [0.1, 0.2], "col3": ["a", "n"]})
+ result_count = expected.to_sql(
+ name="test_frame", con=conn, index=False, method=psql_insert_copy
+ )
+ # GH 46891
+ if expected_count is None:
+ assert result_count is None
+ else:
+ assert result_count == expected_count
+ result = sql.read_sql_table("test_frame", conn)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("conn", postgresql_connectable)
+def test_insertion_method_on_conflict_do_nothing(conn, request):
+ # GH 15988: Example in to_sql docstring
+ conn = request.getfixturevalue(conn)
+
+ from sqlalchemy.dialects.postgresql import insert
+ from sqlalchemy.engine import Engine
+ from sqlalchemy.sql import text
+
+ def insert_on_conflict(table, conn, keys, data_iter):
+ data = [dict(zip(keys, row)) for row in data_iter]
+ stmt = (
+ insert(table.table)
+ .values(data)
+ .on_conflict_do_nothing(index_elements=["a"])
+ )
+ result = conn.execute(stmt)
+ return result.rowcount
+
+ create_sql = text(
+ """
+ CREATE TABLE test_insert_conflict (
+ a integer PRIMARY KEY,
+ b numeric,
+ c text
+ );
+ """
+ )
+ if isinstance(conn, Engine):
+ with conn.connect() as con:
+ with con.begin():
+ con.execute(create_sql)
+ else:
+ with conn.begin():
+ conn.execute(create_sql)
+
+ expected = DataFrame([[1, 2.1, "a"]], columns=list("abc"))
+ expected.to_sql(
+ name="test_insert_conflict", con=conn, if_exists="append", index=False
+ )
+
+ df_insert = DataFrame([[1, 3.2, "b"]], columns=list("abc"))
+ inserted = df_insert.to_sql(
+ name="test_insert_conflict",
+ con=conn,
+ index=False,
+ if_exists="append",
+ method=insert_on_conflict,
+ )
+ result = sql.read_sql_table("test_insert_conflict", conn)
+ tm.assert_frame_equal(result, expected)
+ assert inserted == 0
+
+ # Cleanup
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_insert_conflict")
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_to_sql_on_public_schema(conn, request):
+ if "sqlite" in conn or "mysql" in conn:
+ request.applymarker(
+ pytest.mark.xfail(
+ reason="test for public schema only specific to postgresql"
+ )
+ )
+
+ conn = request.getfixturevalue(conn)
+
+ test_data = DataFrame([[1, 2.1, "a"], [2, 3.1, "b"]], columns=list("abc"))
+ test_data.to_sql(
+ name="test_public_schema",
+ con=conn,
+ if_exists="append",
+ index=False,
+ schema="public",
+ )
+
+ df_out = sql.read_sql_table("test_public_schema", conn, schema="public")
+ tm.assert_frame_equal(test_data, df_out)
+
+
+@pytest.mark.parametrize("conn", mysql_connectable)
+def test_insertion_method_on_conflict_update(conn, request):
+ # GH 14553: Example in to_sql docstring
+ conn = request.getfixturevalue(conn)
+
+ from sqlalchemy.dialects.mysql import insert
+ from sqlalchemy.engine import Engine
+ from sqlalchemy.sql import text
+
+ def insert_on_conflict(table, conn, keys, data_iter):
+ data = [dict(zip(keys, row)) for row in data_iter]
+ stmt = insert(table.table).values(data)
+ stmt = stmt.on_duplicate_key_update(b=stmt.inserted.b, c=stmt.inserted.c)
+ result = conn.execute(stmt)
+ return result.rowcount
+
+ create_sql = text(
+ """
+ CREATE TABLE test_insert_conflict (
+ a INT PRIMARY KEY,
+ b FLOAT,
+ c VARCHAR(10)
+ );
+ """
+ )
+ if isinstance(conn, Engine):
+ with conn.connect() as con:
+ with con.begin():
+ con.execute(create_sql)
+ else:
+ with conn.begin():
+ conn.execute(create_sql)
+
+ df = DataFrame([[1, 2.1, "a"]], columns=list("abc"))
+ df.to_sql(name="test_insert_conflict", con=conn, if_exists="append", index=False)
+
+ expected = DataFrame([[1, 3.2, "b"]], columns=list("abc"))
+ inserted = expected.to_sql(
+ name="test_insert_conflict",
+ con=conn,
+ index=False,
+ if_exists="append",
+ method=insert_on_conflict,
+ )
+ result = sql.read_sql_table("test_insert_conflict", conn)
+ tm.assert_frame_equal(result, expected)
+ assert inserted == 2
+
+ # Cleanup
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_insert_conflict")
+
+
+@pytest.mark.parametrize("conn", postgresql_connectable)
+def test_read_view_postgres(conn, request):
+ # GH 52969
+ conn = request.getfixturevalue(conn)
+
+ from sqlalchemy.engine import Engine
+ from sqlalchemy.sql import text
+
+ table_name = f"group_{uuid.uuid4().hex}"
+ view_name = f"group_view_{uuid.uuid4().hex}"
+
+ sql_stmt = text(
+ f"""
+ CREATE TABLE {table_name} (
+ group_id INTEGER,
+ name TEXT
+ );
+ INSERT INTO {table_name} VALUES
+ (1, 'name');
+ CREATE VIEW {view_name}
+ AS
+ SELECT * FROM {table_name};
+ """
+ )
+ if isinstance(conn, Engine):
+ with conn.connect() as con:
+ with con.begin():
+ con.execute(sql_stmt)
+ else:
+ with conn.begin():
+ conn.execute(sql_stmt)
+ result = read_sql_table(view_name, conn)
+ expected = DataFrame({"group_id": [1], "name": "name"})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_read_view_sqlite(sqlite_buildin):
+ # GH 52969
+ create_table = """
+CREATE TABLE groups (
+ group_id INTEGER,
+ name TEXT
+);
+"""
+ insert_into = """
+INSERT INTO groups VALUES
+ (1, 'name');
+"""
+ create_view = """
+CREATE VIEW group_view
+AS
+SELECT * FROM groups;
+"""
+ sqlite_buildin.execute(create_table)
+ sqlite_buildin.execute(insert_into)
+ sqlite_buildin.execute(create_view)
+ result = pd.read_sql("SELECT * FROM group_view", sqlite_buildin)
+ expected = DataFrame({"group_id": [1], "name": "name"})
+ tm.assert_frame_equal(result, expected)
+
+
+def flavor(conn_name):
+ if "postgresql" in conn_name:
+ return "postgresql"
+ elif "sqlite" in conn_name:
+ return "sqlite"
+ elif "mysql" in conn_name:
+ return "mysql"
+
+ raise ValueError(f"unsupported connection: {conn_name}")
+
+
+@pytest.mark.parametrize("conn", all_connectable_iris)
+def test_read_sql_iris_parameter(conn, request, sql_strings):
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason="'params' not implemented for ADBC drivers",
+ strict=True,
+ )
+ )
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ query = sql_strings["read_parameters"][flavor(conn_name)]
+ params = ("Iris-setosa", 5.1)
+ with pandasSQL_builder(conn) as pandasSQL:
+ with pandasSQL.run_transaction():
+ iris_frame = pandasSQL.read_query(query, params=params)
+ check_iris_frame(iris_frame)
+
+
+@pytest.mark.parametrize("conn", all_connectable_iris)
+def test_read_sql_iris_named_parameter(conn, request, sql_strings):
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason="'params' not implemented for ADBC drivers",
+ strict=True,
+ )
+ )
+
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ query = sql_strings["read_named_parameters"][flavor(conn_name)]
+ params = {"name": "Iris-setosa", "length": 5.1}
+ with pandasSQL_builder(conn) as pandasSQL:
+ with pandasSQL.run_transaction():
+ iris_frame = pandasSQL.read_query(query, params=params)
+ check_iris_frame(iris_frame)
+
+
+@pytest.mark.parametrize("conn", all_connectable_iris)
+def test_read_sql_iris_no_parameter_with_percent(conn, request, sql_strings):
+ if "mysql" in conn or ("postgresql" in conn and "adbc" not in conn):
+ request.applymarker(pytest.mark.xfail(reason="broken test"))
+
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+
+ query = sql_strings["read_no_parameters_with_percent"][flavor(conn_name)]
+ with pandasSQL_builder(conn) as pandasSQL:
+ with pandasSQL.run_transaction():
+ iris_frame = pandasSQL.read_query(query, params=None)
+ check_iris_frame(iris_frame)
+
+
+# -----------------------------------------------------------------------------
+# -- Testing the public API
+
+
+@pytest.mark.parametrize("conn", all_connectable_iris)
+def test_api_read_sql_view(conn, request):
+ conn = request.getfixturevalue(conn)
+ iris_frame = sql.read_sql_query("SELECT * FROM iris_view", conn)
+ check_iris_frame(iris_frame)
+
+
+@pytest.mark.parametrize("conn", all_connectable_iris)
+def test_api_read_sql_with_chunksize_no_result(conn, request):
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(reason="chunksize argument NotImplemented with ADBC")
+ )
+ conn = request.getfixturevalue(conn)
+ query = 'SELECT * FROM iris_view WHERE "SepalLength" < 0.0'
+ with_batch = sql.read_sql_query(query, conn, chunksize=5)
+ without_batch = sql.read_sql_query(query, conn)
+ tm.assert_frame_equal(concat(with_batch), without_batch)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_to_sql(conn, request, test_frame1):
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_frame1", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_frame1")
+
+ sql.to_sql(test_frame1, "test_frame1", conn)
+ assert sql.has_table("test_frame1", conn)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_to_sql_fail(conn, request, test_frame1):
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_frame2", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_frame2")
+
+ sql.to_sql(test_frame1, "test_frame2", conn, if_exists="fail")
+ assert sql.has_table("test_frame2", conn)
+
+ msg = "Table 'test_frame2' already exists"
+ with pytest.raises(ValueError, match=msg):
+ sql.to_sql(test_frame1, "test_frame2", conn, if_exists="fail")
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_to_sql_replace(conn, request, test_frame1):
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_frame3", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_frame3")
+
+ sql.to_sql(test_frame1, "test_frame3", conn, if_exists="fail")
+ # Add to table again
+ sql.to_sql(test_frame1, "test_frame3", conn, if_exists="replace")
+ assert sql.has_table("test_frame3", conn)
+
+ num_entries = len(test_frame1)
+ num_rows = count_rows(conn, "test_frame3")
+
+ assert num_rows == num_entries
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_to_sql_append(conn, request, test_frame1):
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_frame4", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_frame4")
+
+ assert sql.to_sql(test_frame1, "test_frame4", conn, if_exists="fail") == 4
+
+ # Add to table again
+ assert sql.to_sql(test_frame1, "test_frame4", conn, if_exists="append") == 4
+ assert sql.has_table("test_frame4", conn)
+
+ num_entries = 2 * len(test_frame1)
+ num_rows = count_rows(conn, "test_frame4")
+
+ assert num_rows == num_entries
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_to_sql_type_mapping(conn, request, test_frame3):
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_frame5", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_frame5")
+
+ sql.to_sql(test_frame3, "test_frame5", conn, index=False)
+ result = sql.read_sql("SELECT * FROM test_frame5", conn)
+
+ tm.assert_frame_equal(test_frame3, result)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_to_sql_series(conn, request):
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_series", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_series")
+
+ s = Series(np.arange(5, dtype="int64"), name="series")
+ sql.to_sql(s, "test_series", conn, index=False)
+ s2 = sql.read_sql_query("SELECT * FROM test_series", conn)
+ tm.assert_frame_equal(s.to_frame(), s2)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_roundtrip(conn, request, test_frame1):
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_frame_roundtrip", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_frame_roundtrip")
+
+ sql.to_sql(test_frame1, "test_frame_roundtrip", con=conn)
+ result = sql.read_sql_query("SELECT * FROM test_frame_roundtrip", con=conn)
+
+ # HACK!
+ if "adbc" in conn_name:
+ result = result.drop(columns="__index_level_0__")
+ else:
+ result = result.drop(columns="level_0")
+ tm.assert_frame_equal(result, test_frame1)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_roundtrip_chunksize(conn, request, test_frame1):
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(reason="chunksize argument NotImplemented with ADBC")
+ )
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_frame_roundtrip", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_frame_roundtrip")
+
+ sql.to_sql(
+ test_frame1,
+ "test_frame_roundtrip",
+ con=conn,
+ index=False,
+ chunksize=2,
+ )
+ result = sql.read_sql_query("SELECT * FROM test_frame_roundtrip", con=conn)
+ tm.assert_frame_equal(result, test_frame1)
+
+
+@pytest.mark.parametrize("conn", all_connectable_iris)
+def test_api_execute_sql(conn, request):
+ # drop_sql = "DROP TABLE IF EXISTS test" # should already be done
+ conn = request.getfixturevalue(conn)
+ with sql.pandasSQL_builder(conn) as pandas_sql:
+ iris_results = pandas_sql.execute("SELECT * FROM iris")
+ row = iris_results.fetchone()
+ iris_results.close()
+ assert list(row) == [5.1, 3.5, 1.4, 0.2, "Iris-setosa"]
+
+
+@pytest.mark.parametrize("conn", all_connectable_types)
+def test_api_date_parsing(conn, request):
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ # Test date parsing in read_sql
+ # No Parsing
+ df = sql.read_sql_query("SELECT * FROM types", conn)
+ if not ("mysql" in conn_name or "postgres" in conn_name):
+ assert not issubclass(df.DateCol.dtype.type, np.datetime64)
+
+ df = sql.read_sql_query("SELECT * FROM types", conn, parse_dates=["DateCol"])
+ assert issubclass(df.DateCol.dtype.type, np.datetime64)
+ assert df.DateCol.tolist() == [
+ Timestamp(2000, 1, 3, 0, 0, 0),
+ Timestamp(2000, 1, 4, 0, 0, 0),
+ ]
+
+ df = sql.read_sql_query(
+ "SELECT * FROM types",
+ conn,
+ parse_dates={"DateCol": "%Y-%m-%d %H:%M:%S"},
+ )
+ assert issubclass(df.DateCol.dtype.type, np.datetime64)
+ assert df.DateCol.tolist() == [
+ Timestamp(2000, 1, 3, 0, 0, 0),
+ Timestamp(2000, 1, 4, 0, 0, 0),
+ ]
+
+ df = sql.read_sql_query("SELECT * FROM types", conn, parse_dates=["IntDateCol"])
+ assert issubclass(df.IntDateCol.dtype.type, np.datetime64)
+ assert df.IntDateCol.tolist() == [
+ Timestamp(1986, 12, 25, 0, 0, 0),
+ Timestamp(2013, 1, 1, 0, 0, 0),
+ ]
+
+ df = sql.read_sql_query(
+ "SELECT * FROM types", conn, parse_dates={"IntDateCol": "s"}
+ )
+ assert issubclass(df.IntDateCol.dtype.type, np.datetime64)
+ assert df.IntDateCol.tolist() == [
+ Timestamp(1986, 12, 25, 0, 0, 0),
+ Timestamp(2013, 1, 1, 0, 0, 0),
+ ]
+
+ df = sql.read_sql_query(
+ "SELECT * FROM types",
+ conn,
+ parse_dates={"IntDateOnlyCol": "%Y%m%d"},
+ )
+ assert issubclass(df.IntDateOnlyCol.dtype.type, np.datetime64)
+ assert df.IntDateOnlyCol.tolist() == [
+ Timestamp("2010-10-10"),
+ Timestamp("2010-12-12"),
+ ]
+
+
+@pytest.mark.parametrize("conn", all_connectable_types)
+@pytest.mark.parametrize("error", ["raise", "coerce"])
+@pytest.mark.parametrize(
+ "read_sql, text, mode",
+ [
+ (sql.read_sql, "SELECT * FROM types", ("sqlalchemy", "fallback")),
+ (sql.read_sql, "types", ("sqlalchemy")),
+ (
+ sql.read_sql_query,
+ "SELECT * FROM types",
+ ("sqlalchemy", "fallback"),
+ ),
+ (sql.read_sql_table, "types", ("sqlalchemy")),
+ ],
+)
+def test_api_custom_dateparsing_error(
+ conn, request, read_sql, text, mode, error, types_data_frame
+):
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ if text == "types" and conn_name == "sqlite_buildin_types":
+ request.applymarker(
+ pytest.mark.xfail(reason="failing combination of arguments")
+ )
+
+ expected = types_data_frame.astype({"DateCol": "datetime64[us]"})
+
+ result = read_sql(
+ text,
+ con=conn,
+ parse_dates={
+ "DateCol": {"errors": error},
+ },
+ )
+ if "postgres" in conn_name:
+ # TODO: clean up types_data_frame fixture
+ result["BoolCol"] = result["BoolCol"].astype(int)
+ result["BoolColWithNull"] = result["BoolColWithNull"].astype(float)
+
+ if conn_name == "postgresql_adbc_types":
+ expected = expected.astype(
+ {
+ "IntDateCol": "int32",
+ "IntDateOnlyCol": "int32",
+ "IntCol": "int32",
+ }
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("conn", all_connectable_types)
+def test_api_date_and_index(conn, request):
+ # Test case where same column appears in parse_date and index_col
+ conn = request.getfixturevalue(conn)
+ df = sql.read_sql_query(
+ "SELECT * FROM types",
+ conn,
+ index_col="DateCol",
+ parse_dates=["DateCol", "IntDateCol"],
+ )
+
+ assert issubclass(df.index.dtype.type, np.datetime64)
+ assert issubclass(df.IntDateCol.dtype.type, np.datetime64)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_timedelta(conn, request):
+ # see #6921
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_timedelta", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_timedelta")
+
+ df = to_timedelta(Series(["00:00:01", "00:00:03"], name="foo")).to_frame()
+
+ if conn_name == "sqlite_adbc_conn":
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason="sqlite ADBC driver doesn't implement timedelta",
+ )
+ )
+
+ if "adbc" in conn_name:
+ if pa_version_under14p1:
+ exp_warning = DeprecationWarning
+ else:
+ exp_warning = None
+ else:
+ exp_warning = UserWarning
+
+ with tm.assert_produces_warning(exp_warning, check_stacklevel=False):
+ result_count = df.to_sql(name="test_timedelta", con=conn)
+ assert result_count == 2
+ result = sql.read_sql_query("SELECT * FROM test_timedelta", conn)
+
+ if conn_name == "postgresql_adbc_conn":
+ # TODO: Postgres stores an INTERVAL, which ADBC reads as a Month-Day-Nano
+ # Interval; the default pandas type mapper maps this to a DateOffset
+ # but maybe we should try and restore the timedelta here?
+ expected = Series(
+ [
+ pd.DateOffset(months=0, days=0, microseconds=1000000, nanoseconds=0),
+ pd.DateOffset(months=0, days=0, microseconds=3000000, nanoseconds=0),
+ ],
+ name="foo",
+ )
+ else:
+ expected = df["foo"].astype("int64")
+ tm.assert_series_equal(result["foo"], expected)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_complex_raises(conn, request):
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ df = DataFrame({"a": [1 + 1j, 2j]})
+
+ if "adbc" in conn_name:
+ msg = "datatypes not supported"
+ else:
+ msg = "Complex datatypes not supported"
+ with pytest.raises(ValueError, match=msg):
+ assert df.to_sql("test_complex", con=conn) is None
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+@pytest.mark.parametrize(
+ "index_name,index_label,expected",
+ [
+ # no index name, defaults to 'index'
+ (None, None, "index"),
+ # specifying index_label
+ (None, "other_label", "other_label"),
+ # using the index name
+ ("index_name", None, "index_name"),
+ # has index name, but specifying index_label
+ ("index_name", "other_label", "other_label"),
+ # index name is integer
+ (0, None, "0"),
+ # index name is None but index label is integer
+ (None, 0, "0"),
+ ],
+)
+def test_api_to_sql_index_label(conn, request, index_name, index_label, expected):
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(reason="index_label argument NotImplemented with ADBC")
+ )
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_index_label", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_index_label")
+
+ temp_frame = DataFrame({"col1": range(4)})
+ temp_frame.index.name = index_name
+ query = "SELECT * FROM test_index_label"
+ sql.to_sql(temp_frame, "test_index_label", conn, index_label=index_label)
+ frame = sql.read_sql_query(query, conn)
+ assert frame.columns[0] == expected
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_to_sql_index_label_multiindex(conn, request):
+ conn_name = conn
+ if "mysql" in conn_name:
+ request.applymarker(
+ pytest.mark.xfail(
+ reason="MySQL can fail using TEXT without length as key", strict=False
+ )
+ )
+ elif "adbc" in conn_name:
+ request.node.add_marker(
+ pytest.mark.xfail(reason="index_label argument NotImplemented with ADBC")
+ )
+
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_index_label", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_index_label")
+
+ expected_row_count = 4
+ temp_frame = DataFrame(
+ {"col1": range(4)},
+ index=MultiIndex.from_product([("A0", "A1"), ("B0", "B1")]),
+ )
+
+ # no index name, defaults to 'level_0' and 'level_1'
+ result = sql.to_sql(temp_frame, "test_index_label", conn)
+ assert result == expected_row_count
+ frame = sql.read_sql_query("SELECT * FROM test_index_label", conn)
+ assert frame.columns[0] == "level_0"
+ assert frame.columns[1] == "level_1"
+
+ # specifying index_label
+ result = sql.to_sql(
+ temp_frame,
+ "test_index_label",
+ conn,
+ if_exists="replace",
+ index_label=["A", "B"],
+ )
+ assert result == expected_row_count
+ frame = sql.read_sql_query("SELECT * FROM test_index_label", conn)
+ assert frame.columns[:2].tolist() == ["A", "B"]
+
+ # using the index name
+ temp_frame.index.names = ["A", "B"]
+ result = sql.to_sql(temp_frame, "test_index_label", conn, if_exists="replace")
+ assert result == expected_row_count
+ frame = sql.read_sql_query("SELECT * FROM test_index_label", conn)
+ assert frame.columns[:2].tolist() == ["A", "B"]
+
+ # has index name, but specifying index_label
+ result = sql.to_sql(
+ temp_frame,
+ "test_index_label",
+ conn,
+ if_exists="replace",
+ index_label=["C", "D"],
+ )
+ assert result == expected_row_count
+ frame = sql.read_sql_query("SELECT * FROM test_index_label", conn)
+ assert frame.columns[:2].tolist() == ["C", "D"]
+
+ msg = "Length of 'index_label' should match number of levels, which is 2"
+ with pytest.raises(ValueError, match=msg):
+ sql.to_sql(
+ temp_frame,
+ "test_index_label",
+ conn,
+ if_exists="replace",
+ index_label="C",
+ )
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_multiindex_roundtrip(conn, request):
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_multiindex_roundtrip", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_multiindex_roundtrip")
+
+ df = DataFrame.from_records(
+ [(1, 2.1, "line1"), (2, 1.5, "line2")],
+ columns=["A", "B", "C"],
+ index=["A", "B"],
+ )
+
+ df.to_sql(name="test_multiindex_roundtrip", con=conn)
+ result = sql.read_sql_query(
+ "SELECT * FROM test_multiindex_roundtrip", conn, index_col=["A", "B"]
+ )
+ tm.assert_frame_equal(df, result, check_index_type=True)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+@pytest.mark.parametrize(
+ "dtype",
+ [
+ None,
+ int,
+ float,
+ {"A": int, "B": float},
+ ],
+)
+def test_api_dtype_argument(conn, request, dtype):
+ # GH10285 Add dtype argument to read_sql_query
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_dtype_argument", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_dtype_argument")
+
+ df = DataFrame([[1.2, 3.4], [5.6, 7.8]], columns=["A", "B"])
+ assert df.to_sql(name="test_dtype_argument", con=conn) == 2
+
+ expected = df.astype(dtype)
+
+ if "postgres" in conn_name:
+ query = 'SELECT "A", "B" FROM test_dtype_argument'
+ else:
+ query = "SELECT A, B FROM test_dtype_argument"
+ result = sql.read_sql_query(query, con=conn, dtype=dtype)
+
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_integer_col_names(conn, request):
+ conn = request.getfixturevalue(conn)
+ df = DataFrame([[1, 2], [3, 4]], columns=[0, 1])
+ sql.to_sql(df, "test_frame_integer_col_names", conn, if_exists="replace")
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_get_schema(conn, request, test_frame1):
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason="'get_schema' not implemented for ADBC drivers",
+ strict=True,
+ )
+ )
+ conn = request.getfixturevalue(conn)
+ create_sql = sql.get_schema(test_frame1, "test", con=conn)
+ assert "CREATE" in create_sql
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_get_schema_with_schema(conn, request, test_frame1):
+ # GH28486
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason="'get_schema' not implemented for ADBC drivers",
+ strict=True,
+ )
+ )
+ conn = request.getfixturevalue(conn)
+ create_sql = sql.get_schema(test_frame1, "test", con=conn, schema="pypi")
+ assert "CREATE TABLE pypi." in create_sql
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_get_schema_dtypes(conn, request):
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason="'get_schema' not implemented for ADBC drivers",
+ strict=True,
+ )
+ )
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ float_frame = DataFrame({"a": [1.1, 1.2], "b": [2.1, 2.2]})
+
+ if conn_name == "sqlite_buildin":
+ dtype = "INTEGER"
+ else:
+ from sqlalchemy import Integer
+
+ dtype = Integer
+ create_sql = sql.get_schema(float_frame, "test", con=conn, dtype={"b": dtype})
+ assert "CREATE" in create_sql
+ assert "INTEGER" in create_sql
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_get_schema_keys(conn, request, test_frame1):
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason="'get_schema' not implemented for ADBC drivers",
+ strict=True,
+ )
+ )
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ frame = DataFrame({"Col1": [1.1, 1.2], "Col2": [2.1, 2.2]})
+ create_sql = sql.get_schema(frame, "test", con=conn, keys="Col1")
+
+ if "mysql" in conn_name:
+ constraint_sentence = "CONSTRAINT test_pk PRIMARY KEY (`Col1`)"
+ else:
+ constraint_sentence = 'CONSTRAINT test_pk PRIMARY KEY ("Col1")'
+ assert constraint_sentence in create_sql
+
+ # multiple columns as key (GH10385)
+ create_sql = sql.get_schema(test_frame1, "test", con=conn, keys=["A", "B"])
+ if "mysql" in conn_name:
+ constraint_sentence = "CONSTRAINT test_pk PRIMARY KEY (`A`, `B`)"
+ else:
+ constraint_sentence = 'CONSTRAINT test_pk PRIMARY KEY ("A", "B")'
+ assert constraint_sentence in create_sql
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_chunksize_read(conn, request):
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(reason="chunksize argument NotImplemented with ADBC")
+ )
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_chunksize", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_chunksize")
+
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((22, 5)), columns=list("abcde")
+ )
+ df.to_sql(name="test_chunksize", con=conn, index=False)
+
+ # reading the query in one time
+ res1 = sql.read_sql_query("select * from test_chunksize", conn)
+
+ # reading the query in chunks with read_sql_query
+ res2 = DataFrame()
+ i = 0
+ sizes = [5, 5, 5, 5, 2]
+
+ for chunk in sql.read_sql_query("select * from test_chunksize", conn, chunksize=5):
+ res2 = concat([res2, chunk], ignore_index=True)
+ assert len(chunk) == sizes[i]
+ i += 1
+
+ tm.assert_frame_equal(res1, res2)
+
+ # reading the query in chunks with read_sql_query
+ if conn_name == "sqlite_buildin":
+ with pytest.raises(NotImplementedError, match="^$"):
+ sql.read_sql_table("test_chunksize", conn, chunksize=5)
+ else:
+ res3 = DataFrame()
+ i = 0
+ sizes = [5, 5, 5, 5, 2]
+
+ for chunk in sql.read_sql_table("test_chunksize", conn, chunksize=5):
+ res3 = concat([res3, chunk], ignore_index=True)
+ assert len(chunk) == sizes[i]
+ i += 1
+
+ tm.assert_frame_equal(res1, res3)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_categorical(conn, request):
+ if conn == "postgresql_adbc_conn":
+ adbc = import_optional_dependency("adbc_driver_postgresql", errors="ignore")
+ if adbc is not None and Version(adbc.__version__) < Version("0.9.0"):
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason="categorical dtype not implemented for ADBC postgres driver",
+ strict=True,
+ )
+ )
+ # GH8624
+ # test that categorical gets written correctly as dense column
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_categorical", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_categorical")
+
+ df = DataFrame(
+ {
+ "person_id": [1, 2, 3],
+ "person_name": ["John P. Doe", "Jane Dove", "John P. Doe"],
+ }
+ )
+ df2 = df.copy()
+ df2["person_name"] = df2["person_name"].astype("category")
+
+ df2.to_sql(name="test_categorical", con=conn, index=False)
+ res = sql.read_sql_query("SELECT * FROM test_categorical", conn)
+
+ tm.assert_frame_equal(res, df)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_unicode_column_name(conn, request):
+ # GH 11431
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_unicode", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_unicode")
+
+ df = DataFrame([[1, 2], [3, 4]], columns=["\xe9", "b"])
+ df.to_sql(name="test_unicode", con=conn, index=False)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_escaped_table_name(conn, request):
+ # GH 13206
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("d1187b08-4943-4c8d-a7f6", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("d1187b08-4943-4c8d-a7f6")
+
+ df = DataFrame({"A": [0, 1, 2], "B": [0.2, np.nan, 5.6]})
+ df.to_sql(name="d1187b08-4943-4c8d-a7f6", con=conn, index=False)
+
+ if "postgres" in conn_name:
+ query = 'SELECT * FROM "d1187b08-4943-4c8d-a7f6"'
+ else:
+ query = "SELECT * FROM `d1187b08-4943-4c8d-a7f6`"
+ res = sql.read_sql_query(query, conn)
+
+ tm.assert_frame_equal(res, df)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_table_name_quoted(conn, request):
+ # GH#65065 ADBCDatabase did not escape SQL identifiers in
+ # delete_rows, read_table, and to_sql (DROP TABLE path)
+ conn_name = conn
+ if conn_name == "sqlite_buildin":
+ pytest.skip("sqlite_buildin connection does not implement read_sql_table")
+
+ conn = request.getfixturevalue(conn)
+
+ tricky_name = "my table"
+ df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+
+ # to_sql: adbc_ingest handles quoting internally — should pass
+ df.to_sql(tricky_name, conn, index=False, if_exists="replace")
+ # read_table: was interpolating name raw into f-string
+ result = read_sql_table(tricky_name, conn)
+ tm.assert_frame_equal(result, df)
+ # delete_rows: was interpolating name raw into DELETE FROM f-string
+ df.to_sql(tricky_name, conn, index=False, if_exists="delete_rows")
+ result = read_sql_table(tricky_name, conn)
+ tm.assert_frame_equal(result, df)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_api_read_sql_duplicate_columns(conn, request):
+ # GH#53117
+ if "adbc" in conn:
+ pa = pytest.importorskip("pyarrow")
+ if not (
+ Version(pa.__version__) >= Version("16.0")
+ and conn in ["sqlite_adbc_conn", "postgresql_adbc_conn"]
+ ):
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason="pyarrow->pandas throws ValueError", strict=True
+ )
+ )
+ conn = request.getfixturevalue(conn)
+ if sql.has_table("test_table", conn):
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("test_table")
+
+ df = DataFrame({"a": [1, 2, 3], "b": [0.1, 0.2, 0.3], "c": 1})
+ df.to_sql(name="test_table", con=conn, index=False)
+
+ result = pd.read_sql("SELECT a, b, a +1 as a, c FROM test_table", conn)
+ expected = DataFrame(
+ [[1, 0.1, 2, 1], [2, 0.2, 3, 1], [3, 0.3, 4, 1]],
+ columns=["a", "b", "a", "c"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_read_table_columns(conn, request, test_frame1):
+ # test columns argument in read_table
+ conn_name = conn
+ if conn_name == "sqlite_buildin":
+ request.applymarker(pytest.mark.xfail(reason="Not Implemented"))
+
+ conn = request.getfixturevalue(conn)
+ sql.to_sql(test_frame1, "test_frame", conn)
+
+ cols = ["A", "B"]
+
+ result = sql.read_sql_table("test_frame", conn, columns=cols)
+ assert result.columns.tolist() == cols
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_read_table_index_col(conn, request, test_frame1):
+ # test columns argument in read_table
+ conn_name = conn
+ if conn_name == "sqlite_buildin":
+ request.applymarker(pytest.mark.xfail(reason="Not Implemented"))
+
+ conn = request.getfixturevalue(conn)
+ sql.to_sql(test_frame1, "test_frame", conn)
+
+ result = sql.read_sql_table("test_frame", conn, index_col="index")
+ assert result.index.names == ["index"]
+
+ result = sql.read_sql_table("test_frame", conn, index_col=["A", "B"])
+ assert result.index.names == ["A", "B"]
+
+ result = sql.read_sql_table(
+ "test_frame", conn, index_col=["A", "B"], columns=["C", "D"]
+ )
+ assert result.index.names == ["A", "B"]
+ assert result.columns.tolist() == ["C", "D"]
+
+
+@pytest.mark.parametrize("conn", all_connectable_iris)
+def test_read_sql_delegate(conn, request):
+ if conn == "sqlite_buildin_iris":
+ request.applymarker(
+ pytest.mark.xfail(
+ reason="sqlite_buildin connection does not implement read_sql_table"
+ )
+ )
+
+ conn = request.getfixturevalue(conn)
+ iris_frame1 = sql.read_sql_query("SELECT * FROM iris", conn)
+ iris_frame2 = sql.read_sql("SELECT * FROM iris", conn)
+ tm.assert_frame_equal(iris_frame1, iris_frame2)
+
+ iris_frame1 = sql.read_sql_table("iris", conn)
+ iris_frame2 = sql.read_sql("iris", conn)
+ tm.assert_frame_equal(iris_frame1, iris_frame2)
+
+
+def test_not_reflect_all_tables(sqlite_conn):
+ conn = sqlite_conn
+ from sqlalchemy import text
+ from sqlalchemy.engine import Engine
+
+ # create invalid table
+ query_list = [
+ text("CREATE TABLE invalid (x INTEGER, y UNKNOWN);"),
+ text("CREATE TABLE other_table (x INTEGER, y INTEGER);"),
+ ]
+
+ for query in query_list:
+ if isinstance(conn, Engine):
+ with conn.connect() as conn:
+ with conn.begin():
+ conn.execute(query)
+ else:
+ with conn.begin():
+ conn.execute(query)
+
+ with tm.assert_produces_warning(None):
+ sql.read_sql_table("other_table", conn)
+ sql.read_sql_query("SELECT * FROM other_table", conn)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_warning_case_insensitive_table_name(conn, request, test_frame1):
+ conn_name = conn
+ if conn_name == "sqlite_buildin" or "adbc" in conn_name:
+ request.applymarker(pytest.mark.xfail(reason="Does not raise warning"))
+
+ conn = request.getfixturevalue(conn)
+ # see gh-7815
+ with tm.assert_produces_warning(
+ UserWarning,
+ match=(
+ r"The provided table name 'TABLE1' is not found exactly as such in "
+ r"the database after writing the table, possibly due to case "
+ r"sensitivity issues. Consider using lower case table names."
+ ),
+ ):
+ with sql.SQLDatabase(conn) as db:
+ db.check_case_sensitive("TABLE1", "")
+
+ # Test that the warning is certainly NOT triggered in a normal case.
+ with tm.assert_produces_warning(None):
+ test_frame1.to_sql(name="CaseSensitive", con=conn)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_sqlalchemy_type_mapping(conn, request):
+ conn = request.getfixturevalue(conn)
+ from sqlalchemy import TIMESTAMP
+
+ # Test Timestamp objects (no datetime64 because of timezone) (GH9085)
+ df = DataFrame(
+ {"time": to_datetime(["2014-12-12 01:54", "2014-12-11 02:54"], utc=True)}
+ )
+ with sql.SQLDatabase(conn) as db:
+ table = sql.SQLTable("test_type", db, frame=df)
+ # GH 9086: TIMESTAMP is the suggested type for datetimes with timezones
+ assert isinstance(table.table.c["time"].type, TIMESTAMP)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+@pytest.mark.parametrize(
+ "integer, expected",
+ [
+ ("int8", "SMALLINT"),
+ ("Int8", "SMALLINT"),
+ ("uint8", "SMALLINT"),
+ ("UInt8", "SMALLINT"),
+ ("int16", "SMALLINT"),
+ ("Int16", "SMALLINT"),
+ ("uint16", "INTEGER"),
+ ("UInt16", "INTEGER"),
+ ("int32", "INTEGER"),
+ ("Int32", "INTEGER"),
+ ("uint32", "BIGINT"),
+ ("UInt32", "BIGINT"),
+ ("int64", "BIGINT"),
+ ("Int64", "BIGINT"),
+ (int, "BIGINT" if np.dtype(int).name == "int64" else "INTEGER"),
+ ],
+)
+def test_sqlalchemy_integer_mapping(conn, request, integer, expected):
+ # GH35076 Map pandas integer to optimal SQLAlchemy integer type
+ conn = request.getfixturevalue(conn)
+ df = DataFrame([0, 1], columns=["a"], dtype=integer)
+ with sql.SQLDatabase(conn) as db:
+ table = sql.SQLTable("test_type", db, frame=df)
+
+ result = str(table.table.c.a.type)
+ assert result == expected
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+@pytest.mark.parametrize("integer", ["uint64", "UInt64"])
+def test_sqlalchemy_integer_overload_mapping(conn, request, integer):
+ conn = request.getfixturevalue(conn)
+ # GH35076 Map pandas integer to optimal SQLAlchemy integer type
+ df = DataFrame([0, 1], columns=["a"], dtype=integer)
+ with sql.SQLDatabase(conn) as db:
+ with pytest.raises(
+ ValueError, match="Unsigned 64 bit integer datatype is not supported"
+ ):
+ sql.SQLTable("test_type", db, frame=df)
+
+
+def test_database_uri_string(temp_file, request, test_frame1):
+ pytest.importorskip("sqlalchemy")
+ # Test read_sql and .to_sql method with a database URI (GH10654)
+ # db_uri = 'sqlite:///:memory:' # raises
+ # sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) near
+ # "iris": syntax error [SQL: 'iris']
+ name = str(temp_file)
+ db_uri = "sqlite:///" + name
+ table = "iris"
+ test_frame1.to_sql(name=table, con=db_uri, if_exists="replace", index=False)
+ test_frame2 = sql.read_sql(table, db_uri)
+ test_frame3 = sql.read_sql_table(table, db_uri)
+ query = "SELECT * FROM iris"
+ test_frame4 = sql.read_sql_query(query, db_uri)
+ tm.assert_frame_equal(test_frame1, test_frame2)
+ tm.assert_frame_equal(test_frame1, test_frame3)
+ tm.assert_frame_equal(test_frame1, test_frame4)
+
+
+@td.skip_if_installed("pg8000")
+def test_pg8000_sqlalchemy_passthrough_error(request):
+ pytest.importorskip("sqlalchemy")
+ # using driver that will not be installed on CI to trigger error
+ # in sqlalchemy.create_engine -> test passing of this error to user
+ db_uri = "postgresql+pg8000://user:pass@host/dbname"
+ with pytest.raises(ImportError, match="pg8000"):
+ sql.read_sql("select * from table", db_uri)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris)
+def test_query_by_text_obj(conn, request):
+ # WIP : GH10846
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ from sqlalchemy import text
+
+ if "postgres" in conn_name:
+ name_text = text('select * from iris where "Name"=:name')
+ else:
+ name_text = text("select * from iris where name=:name")
+ iris_df = sql.read_sql(name_text, conn, params={"name": "Iris-versicolor"})
+ all_names = set(iris_df["Name"])
+ assert all_names == {"Iris-versicolor"}
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris)
+def test_query_by_select_obj(conn, request):
+ conn = request.getfixturevalue(conn)
+ # WIP : GH10846
+ from sqlalchemy import (
+ bindparam,
+ select,
+ )
+
+ iris = iris_table_metadata()
+ name_select = select(iris).where(iris.c.Name == bindparam("name"))
+ iris_df = sql.read_sql(name_select, conn, params={"name": "Iris-setosa"})
+ all_names = set(iris_df["Name"])
+ assert all_names == {"Iris-setosa"}
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_column_with_percentage(conn, request):
+ # GH 37157
+ conn_name = conn
+ if conn_name == "sqlite_buildin":
+ request.applymarker(pytest.mark.xfail(reason="Not Implemented"))
+
+ conn = request.getfixturevalue(conn)
+ df = DataFrame({"A": [0, 1, 2], "%_variation": [3, 4, 5]})
+ df.to_sql(name="test_column_percentage", con=conn, index=False)
+
+ res = sql.read_sql_table("test_column_percentage", conn)
+
+ tm.assert_frame_equal(res, df)
+
+
+def test_sql_open_close(temp_file, test_frame3):
+ # Test if the IO in the database still work if the connection closed
+ # between the writing and reading (as in many real situations).
+
+ with contextlib.closing(sqlite3.connect(temp_file)) as conn:
+ assert sql.to_sql(test_frame3, "test_frame3_legacy", conn, index=False) == 4
+
+ with contextlib.closing(sqlite3.connect(temp_file)) as conn:
+ result = sql.read_sql_query("SELECT * FROM test_frame3_legacy;", conn)
+
+ tm.assert_frame_equal(test_frame3, result)
+
+
+@td.skip_if_installed("sqlalchemy")
+def test_con_string_import_error():
+ conn = "mysql://root@localhost/pandas"
+ msg = "Using URI string without sqlalchemy installed"
+ with pytest.raises(ImportError, match=msg):
+ sql.read_sql("SELECT * FROM iris", conn)
+
+
+@td.skip_if_installed("sqlalchemy")
+def test_con_unknown_dbapi2_class_does_not_error_without_sql_alchemy_installed():
+ class MockSqliteConnection:
+ def __init__(self, *args, **kwargs) -> None:
+ self.conn = sqlite3.Connection(*args, **kwargs)
+
+ def __getattr__(self, name):
+ return getattr(self.conn, name)
+
+ def close(self):
+ self.conn.close()
+
+ with contextlib.closing(MockSqliteConnection(":memory:")) as conn:
+ with tm.assert_produces_warning(UserWarning, match="only supports SQLAlchemy"):
+ sql.read_sql("SELECT 1", conn)
+
+
+def test_sqlite_read_sql_delegate(sqlite_buildin_iris):
+ conn = sqlite_buildin_iris
+ iris_frame1 = sql.read_sql_query("SELECT * FROM iris", conn)
+ iris_frame2 = sql.read_sql("SELECT * FROM iris", conn)
+ tm.assert_frame_equal(iris_frame1, iris_frame2)
+
+ msg = "Execution failed on sql 'iris': near \"iris\": syntax error"
+ with pytest.raises(sql.DatabaseError, match=msg):
+ sql.read_sql("iris", conn)
+
+
+def test_get_schema2(test_frame1):
+ # without providing a connection object (available for backwards comp)
+ create_sql = sql.get_schema(test_frame1, "test")
+ assert "CREATE" in create_sql
+
+
+def test_sqlite_type_mapping(sqlite_buildin):
+ # Test Timestamp objects (no datetime64 because of timezone) (GH9085)
+ conn = sqlite_buildin
+ df = DataFrame(
+ {"time": to_datetime(["2014-12-12 01:54", "2014-12-11 02:54"], utc=True)}
+ )
+ db = sql.SQLiteDatabase(conn)
+ table = sql.SQLiteTable("test_type", db, frame=df)
+ schema = table.sql_schema()
+ for col in schema.split("\n"):
+ if col.split()[0].strip('"') == "time":
+ assert col.split()[1] == "TIMESTAMP"
+
+
+# -----------------------------------------------------------------------------
+# -- Database flavor specific tests
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_create_table(conn, request):
+ if conn == "sqlite_str":
+ pytest.skip("sqlite_str has no inspection system")
+
+ conn = request.getfixturevalue(conn)
+
+ from sqlalchemy import inspect
+
+ temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]})
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4
+
+ insp = inspect(conn)
+ assert insp.has_table("temp_frame")
+
+ # Cleanup
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("temp_frame")
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_drop_table(conn, request):
+ if conn == "sqlite_str":
+ pytest.skip("sqlite_str has no inspection system")
+
+ conn = request.getfixturevalue(conn)
+
+ from sqlalchemy import inspect
+
+ temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]})
+ with sql.SQLDatabase(conn) as pandasSQL:
+ with pandasSQL.run_transaction():
+ assert pandasSQL.to_sql(temp_frame, "temp_frame") == 4
+
+ insp = inspect(conn)
+ assert insp.has_table("temp_frame")
+
+ with pandasSQL.run_transaction():
+ pandasSQL.drop_table("temp_frame")
+ try:
+ insp.clear_cache() # needed with SQLAlchemy 2.0, unavailable prior
+ except AttributeError:
+ pass
+ assert not insp.has_table("temp_frame")
+
+
+@pytest.mark.parametrize("conn_name", all_connectable)
+def test_delete_rows_success(conn_name, test_frame1, request):
+ table_name = "temp_delete_rows_frame"
+ conn = request.getfixturevalue(conn_name)
+
+ with pandasSQL_builder(conn) as pandasSQL:
+ with pandasSQL.run_transaction():
+ assert pandasSQL.to_sql(test_frame1, table_name) == test_frame1.shape[0]
+
+ with pandasSQL.run_transaction():
+ assert pandasSQL.delete_rows(table_name) is None
+
+ assert count_rows(conn, table_name) == 0
+ assert pandasSQL.has_table(table_name)
+
+
+@pytest.mark.parametrize("conn_name", all_connectable)
+def test_delete_rows_is_atomic(conn_name, request):
+ sqlalchemy = pytest.importorskip("sqlalchemy")
+
+ table_name = "temp_delete_rows_atomic_frame"
+ table_stmt = f"CREATE TABLE {table_name} (a INTEGER, b INTEGER UNIQUE NOT NULL)"
+
+ if conn_name != "sqlite_buildin" and "adbc" not in conn_name:
+ table_stmt = sqlalchemy.text(table_stmt)
+
+ # setting dtype is mandatory for adbc related tests
+ original_df = DataFrame({"a": [1, 2], "b": [3, 4]}, dtype="int32")
+ replacing_df = DataFrame({"a": [5, 6, 7], "b": [8, 8, 8]}, dtype="int32")
+
+ conn = request.getfixturevalue(conn_name)
+ with pandasSQL_builder(conn) as pandasSQL:
+ with pandasSQL.run_transaction() as cur:
+ cur.execute(table_stmt)
+
+ with pandasSQL.run_transaction():
+ pandasSQL.to_sql(original_df, table_name, if_exists="append", index=False)
+
+ # inserting duplicated values in a UNIQUE constraint column
+ with pytest.raises(pd.errors.DatabaseError):
+ with pandasSQL.run_transaction():
+ pandasSQL.to_sql(
+ replacing_df, table_name, if_exists="delete_rows", index=False
+ )
+
+ # failed "delete_rows" is rolled back preserving original data
+ with pandasSQL.run_transaction():
+ result_df = pandasSQL.read_query(
+ f"SELECT * FROM {table_name}", dtype="int32"
+ )
+ tm.assert_frame_equal(result_df, original_df)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_roundtrip(conn, request, test_frame1):
+ if conn == "sqlite_str":
+ pytest.skip("sqlite_str has no inspection system")
+
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ with pandasSQL_builder(conn) as pandasSQL:
+ with pandasSQL.run_transaction():
+ assert pandasSQL.to_sql(test_frame1, "test_frame_roundtrip") == 4
+ result = pandasSQL.read_query("SELECT * FROM test_frame_roundtrip")
+
+ if "adbc" in conn_name:
+ result = result.rename(columns={"__index_level_0__": "level_0"})
+ result.set_index("level_0", inplace=True)
+ # result.index.astype(int)
+
+ result.index.name = None
+
+ tm.assert_frame_equal(result, test_frame1)
+
+
+@pytest.mark.parametrize("conn", all_connectable_iris)
+def test_execute_sql(conn, request):
+ conn = request.getfixturevalue(conn)
+ with pandasSQL_builder(conn) as pandasSQL:
+ with pandasSQL.run_transaction():
+ iris_results = pandasSQL.execute("SELECT * FROM iris")
+ row = iris_results.fetchone()
+ iris_results.close()
+ assert list(row) == [5.1, 3.5, 1.4, 0.2, "Iris-setosa"]
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris)
+def test_sqlalchemy_read_table(conn, request):
+ conn = request.getfixturevalue(conn)
+ iris_frame = sql.read_sql_table("iris", con=conn)
+ check_iris_frame(iris_frame)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris)
+def test_sqlalchemy_read_table_columns(conn, request):
+ conn = request.getfixturevalue(conn)
+ iris_frame = sql.read_sql_table(
+ "iris", con=conn, columns=["SepalLength", "SepalLength"]
+ )
+ tm.assert_index_equal(iris_frame.columns, Index(["SepalLength", "SepalLength__1"]))
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable_iris)
+def test_read_table_absent_raises(conn, request):
+ conn = request.getfixturevalue(conn)
+ msg = "Table this_doesnt_exist not found"
+ with pytest.raises(ValueError, match=msg):
+ sql.read_sql_table("this_doesnt_exist", con=conn)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable_types)
+def test_sqlalchemy_default_type_conversion(conn, request):
+ conn_name = conn
+ if conn_name == "sqlite_str":
+ pytest.skip("types tables not created in sqlite_str fixture")
+ elif "mysql" in conn_name or "sqlite" in conn_name:
+ request.applymarker(
+ pytest.mark.xfail(reason="boolean dtype not inferred properly")
+ )
+
+ conn = request.getfixturevalue(conn)
+ df = sql.read_sql_table("types", conn)
+
+ assert issubclass(df.FloatCol.dtype.type, np.floating)
+ assert issubclass(df.IntCol.dtype.type, np.integer)
+ assert issubclass(df.BoolCol.dtype.type, np.bool_)
+
+ # Int column with NA values stays as float
+ assert issubclass(df.IntColWithNull.dtype.type, np.floating)
+ # Bool column with NA values becomes object
+ assert issubclass(df.BoolColWithNull.dtype.type, object)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_bigint(conn, request):
+ # int64 should be converted to BigInteger, GH7433
+ conn = request.getfixturevalue(conn)
+ df = DataFrame(data={"i64": [2**62]})
+ assert df.to_sql(name="test_bigint", con=conn, index=False) == 1
+ result = sql.read_sql_table("test_bigint", conn)
+
+ tm.assert_frame_equal(df, result)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable_types)
+def test_default_date_load(conn, request):
+ conn_name = conn
+ if conn_name == "sqlite_str":
+ pytest.skip("types tables not created in sqlite_str fixture")
+ elif "sqlite" in conn_name:
+ request.applymarker(
+ pytest.mark.xfail(reason="sqlite does not read date properly")
+ )
+
+ conn = request.getfixturevalue(conn)
+ df = sql.read_sql_table("types", conn)
+
+ assert issubclass(df.DateCol.dtype.type, np.datetime64)
+
+
+@pytest.mark.parametrize("conn", postgresql_connectable)
+@pytest.mark.parametrize("parse_dates", [None, ["DateColWithTz"]])
+def test_datetime_with_timezone_query(conn, request, parse_dates):
+ # edge case that converts postgresql datetime with time zone types
+ # to datetime64[ns,psycopg2.tz.FixedOffsetTimezone..], which is ok
+ # but should be more natural, so coerce to datetime64[ns] for now
+ conn = request.getfixturevalue(conn)
+ expected = create_and_load_postgres_datetz(conn)
+
+ # GH11216
+ df = read_sql_query("select * from datetz", conn, parse_dates=parse_dates)
+ col = df.DateColWithTz
+ tm.assert_series_equal(col, expected)
+
+
+@pytest.mark.parametrize("conn", postgresql_connectable)
+def test_datetime_with_timezone_query_chunksize(conn, request):
+ conn = request.getfixturevalue(conn)
+ expected = create_and_load_postgres_datetz(conn)
+
+ df = concat(
+ list(read_sql_query("select * from datetz", conn, chunksize=1)),
+ ignore_index=True,
+ )
+ col = df.DateColWithTz
+ tm.assert_series_equal(col, expected)
+
+
+@pytest.mark.parametrize("conn", postgresql_connectable)
+def test_datetime_with_timezone_table(conn, request):
+ conn = request.getfixturevalue(conn)
+ expected = create_and_load_postgres_datetz(conn)
+ result = sql.read_sql_table("datetz", conn)
+
+ exp_frame = expected.to_frame()
+ tm.assert_frame_equal(result, exp_frame)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_datetime_with_timezone_roundtrip(conn, request):
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ # GH 9086
+ # Write datetimetz data to a db and read it back
+ # For dbs that support timestamps with timezones, should get back UTC
+ # otherwise naive data should be returned
+ expected = DataFrame(
+ {"A": date_range("2013-01-01 09:00:00", periods=3, tz="US/Pacific", unit="us")}
+ )
+ assert expected.to_sql(name="test_datetime_tz", con=conn, index=False) == 3
+
+ if "postgresql" in conn_name:
+ # SQLAlchemy "timezones" (i.e. offsets) are coerced to UTC
+ expected["A"] = expected["A"].dt.tz_convert("UTC")
+ else:
+ # Otherwise, timestamps are returned as local, naive
+ expected["A"] = expected["A"].dt.tz_localize(None)
+
+ result = sql.read_sql_table("test_datetime_tz", conn)
+ tm.assert_frame_equal(result, expected)
+
+ result = sql.read_sql_query("SELECT * FROM test_datetime_tz", conn)
+ if "sqlite" in conn_name:
+ # read_sql_query does not return datetime type like read_sql_table
+ assert isinstance(result.loc[0, "A"], str)
+ result["A"] = to_datetime(result["A"]).dt.as_unit("us")
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_out_of_bounds_datetime(conn, request):
+ # GH 26761
+ conn = request.getfixturevalue(conn)
+ data = DataFrame({"date": datetime(9999, 1, 1)}, index=[0])
+ assert data.to_sql(name="test_datetime_obb", con=conn, index=False) == 1
+ result = sql.read_sql_table("test_datetime_obb", conn)
+ expected = DataFrame(
+ np.array([datetime(9999, 1, 1)], dtype="M8[us]"), columns=["date"]
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_naive_datetimeindex_roundtrip(conn, request):
+ # GH 23510
+ # Ensure that a naive DatetimeIndex isn't converted to UTC
+ conn = request.getfixturevalue(conn)
+ dates = date_range("2018-01-01", periods=5, freq="6h", unit="us")._with_freq(None)
+ expected = DataFrame({"nums": range(5)}, index=dates)
+ assert expected.to_sql(name="foo_table", con=conn, index_label="info_date") == 5
+ result = sql.read_sql_table("foo_table", conn, index_col="info_date")
+ # result index with gain a name from a set_index operation; expected
+ tm.assert_frame_equal(result, expected, check_names=False)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable_types)
+def test_date_parsing(conn, request):
+ # No Parsing
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ df = sql.read_sql_table("types", conn)
+ expected_type = object if "sqlite" in conn_name else np.datetime64
+ assert issubclass(df.DateCol.dtype.type, expected_type)
+
+ df = sql.read_sql_table("types", conn, parse_dates=["DateCol"])
+ assert issubclass(df.DateCol.dtype.type, np.datetime64)
+
+ df = sql.read_sql_table("types", conn, parse_dates={"DateCol": "%Y-%m-%d %H:%M:%S"})
+ assert issubclass(df.DateCol.dtype.type, np.datetime64)
+
+ df = sql.read_sql_table(
+ "types",
+ conn,
+ parse_dates={"DateCol": {"format": "%Y-%m-%d %H:%M:%S"}},
+ )
+ assert issubclass(df.DateCol.dtype.type, np.datetime64)
+
+ df = sql.read_sql_table("types", conn, parse_dates=["IntDateCol"])
+ assert issubclass(df.IntDateCol.dtype.type, np.datetime64)
+
+ df = sql.read_sql_table("types", conn, parse_dates={"IntDateCol": "s"})
+ assert issubclass(df.IntDateCol.dtype.type, np.datetime64)
+
+ df = sql.read_sql_table("types", conn, parse_dates={"IntDateCol": {"unit": "s"}})
+ assert issubclass(df.IntDateCol.dtype.type, np.datetime64)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_datetime(conn, request):
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ df = DataFrame(
+ {"A": date_range("2013-01-01 09:00:00", periods=3), "B": np.arange(3.0)}
+ )
+ assert df.to_sql(name="test_datetime", con=conn) == 3
+
+ # with read_table -> type information from schema used
+ result = sql.read_sql_table("test_datetime", conn)
+ result = result.drop("index", axis=1)
+
+ expected = df[:]
+ expected["A"] = expected["A"].astype("M8[us]")
+ tm.assert_frame_equal(result, expected)
+
+ # with read_sql -> no type information -> sqlite has no native
+ result = sql.read_sql_query("SELECT * FROM test_datetime", conn)
+ result = result.drop("index", axis=1)
+ if "sqlite" in conn_name:
+ assert isinstance(result.loc[0, "A"], str)
+ result["A"] = to_datetime(result["A"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_datetime_NaT(conn, request):
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ df = DataFrame(
+ {"A": date_range("2013-01-01 09:00:00", periods=3), "B": np.arange(3.0)}
+ )
+ df.loc[1, "A"] = np.nan
+ assert df.to_sql(name="test_datetime", con=conn, index=False) == 3
+
+ # with read_table -> type information from schema used
+ result = sql.read_sql_table("test_datetime", conn)
+ expected = df[:]
+ expected["A"] = expected["A"].astype("M8[us]")
+ tm.assert_frame_equal(result, expected)
+
+ # with read_sql -> no type information -> sqlite has no native
+ result = sql.read_sql_query("SELECT * FROM test_datetime", conn)
+ if "sqlite" in conn_name:
+ assert isinstance(result.loc[0, "A"], str)
+ result["A"] = to_datetime(result["A"], errors="coerce")
+
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_datetime_date(conn, request):
+ # test support for datetime.date
+ conn = request.getfixturevalue(conn)
+ df = DataFrame([date(2014, 1, 1), date(2014, 1, 2)], columns=["a"])
+ assert df.to_sql(name="test_date", con=conn, index=False) == 2
+ res = read_sql_table("test_date", conn)
+ result = res["a"]
+ expected = to_datetime(df["a"])
+ # comes back as datetime64
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_datetime_time(conn, request, sqlite_buildin):
+ # test support for datetime.time
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ df = DataFrame([time(9, 0, 0), time(9, 1, 30)], columns=["a"])
+ assert df.to_sql(name="test_time", con=conn, index=False) == 2
+ res = read_sql_table("test_time", conn)
+ tm.assert_frame_equal(res, df)
+
+ # GH8341
+ # first, use the fallback to have the sqlite adapter put in place
+ sqlite_conn = sqlite_buildin
+ assert sql.to_sql(df, "test_time2", sqlite_conn, index=False) == 2
+ res = sql.read_sql_query("SELECT * FROM test_time2", sqlite_conn)
+ ref = df.map(lambda _: _.strftime("%H:%M:%S.%f"))
+ tm.assert_frame_equal(ref, res) # check if adapter is in place
+ # then test if sqlalchemy is unaffected by the sqlite adapter
+ assert sql.to_sql(df, "test_time3", conn, index=False) == 2
+ if "sqlite" in conn_name:
+ res = sql.read_sql_query("SELECT * FROM test_time3", conn)
+ ref = df.map(lambda _: _.strftime("%H:%M:%S.%f"))
+ tm.assert_frame_equal(ref, res)
+ res = sql.read_sql_table("test_time3", conn)
+ tm.assert_frame_equal(df, res)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_mixed_dtype_insert(conn, request):
+ # see GH6509
+ conn = request.getfixturevalue(conn)
+ s1 = Series(2**25 + 1, dtype=np.int32)
+ s2 = Series(0.0, dtype=np.float32)
+ df = DataFrame({"s1": s1, "s2": s2})
+
+ # write and read again
+ assert df.to_sql(name="test_read_write", con=conn, index=False) == 1
+ df2 = sql.read_sql_table("test_read_write", conn)
+
+ tm.assert_frame_equal(df, df2, check_dtype=False, check_exact=True)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_nan_numeric(conn, request):
+ # NaNs in numeric float column
+ conn = request.getfixturevalue(conn)
+ df = DataFrame({"A": [0, 1, 2], "B": [0.2, np.nan, 5.6]})
+ assert df.to_sql(name="test_nan", con=conn, index=False) == 3
+
+ # with read_table
+ result = sql.read_sql_table("test_nan", conn)
+ tm.assert_frame_equal(result, df)
+
+ # with read_sql
+ result = sql.read_sql_query("SELECT * FROM test_nan", conn)
+ tm.assert_frame_equal(result, df)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_nan_fullcolumn(conn, request):
+ # full NaN column (numeric float column)
+ conn = request.getfixturevalue(conn)
+ df = DataFrame({"A": [0, 1, 2], "B": [np.nan, np.nan, np.nan]})
+ assert df.to_sql(name="test_nan", con=conn, index=False) == 3
+
+ # with read_table
+ result = sql.read_sql_table("test_nan", conn)
+ tm.assert_frame_equal(result, df)
+
+ # with read_sql -> not type info from table -> stays None
+ df["B"] = df["B"].astype("object")
+ df["B"] = None
+ result = sql.read_sql_query("SELECT * FROM test_nan", conn)
+ tm.assert_frame_equal(result, df)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_nan_string(conn, request):
+ # NaNs in string column
+ conn = request.getfixturevalue(conn)
+ df = DataFrame({"A": [0, 1, 2], "B": ["a", "b", np.nan]})
+ assert df.to_sql(name="test_nan", con=conn, index=False) == 3
+
+ # NaNs are coming back as None
+ df.loc[2, "B"] = None
+
+ # with read_table
+ result = sql.read_sql_table("test_nan", conn)
+ tm.assert_frame_equal(result, df)
+
+ # with read_sql
+ result = sql.read_sql_query("SELECT * FROM test_nan", conn)
+ tm.assert_frame_equal(result, df)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_to_sql_save_index(conn, request):
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(
+ reason="ADBC implementation does not create index", strict=True
+ )
+ )
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ df = DataFrame.from_records(
+ [(1, 2.1, "line1"), (2, 1.5, "line2")], columns=["A", "B", "C"], index=["A"]
+ )
+
+ tbl_name = "test_to_sql_saves_index"
+ with pandasSQL_builder(conn) as pandasSQL:
+ with pandasSQL.run_transaction():
+ assert pandasSQL.to_sql(df, tbl_name) == 2
+
+ if conn_name in {"sqlite_buildin", "sqlite_str"}:
+ ixs = sql.read_sql_query(
+ "SELECT * FROM sqlite_master WHERE type = 'index' "
+ f"AND tbl_name = '{tbl_name}'",
+ conn,
+ )
+ ix_cols = []
+ for ix_name in ixs.name:
+ ix_info = sql.read_sql_query(f"PRAGMA index_info({ix_name})", conn)
+ ix_cols.append(ix_info.name.tolist())
+ else:
+ from sqlalchemy import inspect
+
+ insp = inspect(conn)
+
+ ixs = insp.get_indexes(tbl_name)
+ ix_cols = [i["column_names"] for i in ixs]
+
+ assert ix_cols == [["A"]]
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_transactions(conn, request):
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+
+ stmt = "CREATE TABLE test_trans (A INT, B TEXT)"
+ if conn_name != "sqlite_buildin" and "adbc" not in conn_name:
+ from sqlalchemy import text
+
+ stmt = text(stmt)
+
+ with pandasSQL_builder(conn) as pandasSQL:
+ with pandasSQL.run_transaction() as trans:
+ trans.execute(stmt)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_transaction_rollback(conn, request):
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ with pandasSQL_builder(conn) as pandasSQL:
+ with pandasSQL.run_transaction() as trans:
+ stmt = "CREATE TABLE test_trans (A INT, B TEXT)"
+ if "adbc" in conn_name or isinstance(pandasSQL, SQLiteDatabase):
+ trans.execute(stmt)
+ else:
+ from sqlalchemy import text
+
+ stmt = text(stmt)
+ trans.execute(stmt)
+
+ class DummyException(Exception):
+ pass
+
+ # Make sure when transaction is rolled back, no rows get inserted
+ ins_sql = "INSERT INTO test_trans (A,B) VALUES (1, 'blah')"
+ if isinstance(pandasSQL, SQLDatabase):
+ from sqlalchemy import text
+
+ ins_sql = text(ins_sql)
+ try:
+ with pandasSQL.run_transaction() as trans:
+ trans.execute(ins_sql)
+ raise DummyException("error")
+ except DummyException:
+ # ignore raised exception
+ pass
+ with pandasSQL.run_transaction():
+ res = pandasSQL.read_query("SELECT * FROM test_trans")
+ assert len(res) == 0
+
+ # Make sure when transaction is committed, rows do get inserted
+ with pandasSQL.run_transaction() as trans:
+ trans.execute(ins_sql)
+ res2 = pandasSQL.read_query("SELECT * FROM test_trans")
+ assert len(res2) == 1
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_get_schema_create_table(conn, request, test_frame3):
+ # Use a dataframe without a bool column, since MySQL converts bool to
+ # TINYINT (which read_sql_table returns as an int and causes a dtype
+ # mismatch)
+ if conn == "sqlite_str":
+ request.applymarker(
+ pytest.mark.xfail(reason="test does not support sqlite_str fixture")
+ )
+
+ conn = request.getfixturevalue(conn)
+
+ from sqlalchemy import text
+ from sqlalchemy.engine import Engine
+
+ tbl = "test_get_schema_create_table"
+ create_sql = sql.get_schema(test_frame3, tbl, con=conn)
+ blank_test_df = test_frame3.iloc[:0]
+
+ create_sql = text(create_sql)
+ if isinstance(conn, Engine):
+ with conn.connect() as newcon:
+ with newcon.begin():
+ newcon.execute(create_sql)
+ else:
+ conn.execute(create_sql)
+ returned_df = sql.read_sql_table(tbl, conn)
+ tm.assert_frame_equal(returned_df, blank_test_df, check_index_type=False)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_dtype(conn, request):
+ if conn == "sqlite_str":
+ pytest.skip("sqlite_str has no inspection system")
+
+ conn = request.getfixturevalue(conn)
+
+ from sqlalchemy import (
+ TEXT,
+ String,
+ )
+ from sqlalchemy.schema import MetaData
+
+ cols = ["A", "B"]
+ data = [(0.8, True), (0.9, None)]
+ df = DataFrame(data, columns=cols)
+ assert df.to_sql(name="dtype_test", con=conn) == 2
+ assert df.to_sql(name="dtype_test2", con=conn, dtype={"B": TEXT}) == 2
+ meta = MetaData()
+ meta.reflect(bind=conn)
+ sqltype = meta.tables["dtype_test2"].columns["B"].type
+ assert isinstance(sqltype, TEXT)
+ msg = "The type of B is not a SQLAlchemy type"
+ with pytest.raises(ValueError, match=msg):
+ df.to_sql(name="error", con=conn, dtype={"B": str})
+
+ # GH9083
+ assert df.to_sql(name="dtype_test3", con=conn, dtype={"B": String(10)}) == 2
+ meta.reflect(bind=conn)
+ sqltype = meta.tables["dtype_test3"].columns["B"].type
+ assert isinstance(sqltype, String)
+ assert sqltype.length == 10
+
+ # single dtype
+ assert df.to_sql(name="single_dtype_test", con=conn, dtype=TEXT) == 2
+ meta.reflect(bind=conn)
+ sqltypea = meta.tables["single_dtype_test"].columns["A"].type
+ sqltypeb = meta.tables["single_dtype_test"].columns["B"].type
+ assert isinstance(sqltypea, TEXT)
+ assert isinstance(sqltypeb, TEXT)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_notna_dtype(conn, request):
+ if conn == "sqlite_str":
+ pytest.skip("sqlite_str has no inspection system")
+
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+
+ from sqlalchemy import (
+ Boolean,
+ DateTime,
+ Float,
+ Integer,
+ )
+ from sqlalchemy.schema import MetaData
+
+ cols = {
+ "Bool": Series([True, None]),
+ "Date": Series([datetime(2012, 5, 1), None]),
+ "Int": Series([1, None], dtype="object"),
+ "Float": Series([1.1, None]),
+ }
+ df = DataFrame(cols)
+
+ tbl = "notna_dtype_test"
+ assert df.to_sql(name=tbl, con=conn) == 2
+ _ = sql.read_sql_table(tbl, conn)
+ meta = MetaData()
+ meta.reflect(bind=conn)
+ my_type = Integer if "mysql" in conn_name else Boolean
+ col_dict = meta.tables[tbl].columns
+ assert isinstance(col_dict["Bool"].type, my_type)
+ assert isinstance(col_dict["Date"].type, DateTime)
+ assert isinstance(col_dict["Int"].type, Integer)
+ assert isinstance(col_dict["Float"].type, Float)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_double_precision(conn, request):
+ if conn == "sqlite_str":
+ pytest.skip("sqlite_str has no inspection system")
+
+ conn = request.getfixturevalue(conn)
+
+ from sqlalchemy import (
+ BigInteger,
+ Float,
+ Integer,
+ )
+ from sqlalchemy.schema import MetaData
+
+ V = 1.23456789101112131415
+
+ df = DataFrame(
+ {
+ "f32": Series([V], dtype="float32"),
+ "f64": Series([V], dtype="float64"),
+ "f64_as_f32": Series([V], dtype="float64"),
+ "i32": Series([5], dtype="int32"),
+ "i64": Series([5], dtype="int64"),
+ }
+ )
+
+ assert (
+ df.to_sql(
+ name="test_dtypes",
+ con=conn,
+ index=False,
+ if_exists="replace",
+ dtype={"f64_as_f32": Float(precision=23)},
+ )
+ == 1
+ )
+ res = sql.read_sql_table("test_dtypes", conn)
+
+ # check precision of float64
+ assert np.round(df["f64"].iloc[0], 14) == np.round(res["f64"].iloc[0], 14)
+
+ # check sql types
+ meta = MetaData()
+ meta.reflect(bind=conn)
+ col_dict = meta.tables["test_dtypes"].columns
+ assert str(col_dict["f32"].type) == str(col_dict["f64_as_f32"].type)
+ assert isinstance(col_dict["f32"].type, Float)
+ assert isinstance(col_dict["f64"].type, Float)
+ assert isinstance(col_dict["i32"].type, Integer)
+ assert isinstance(col_dict["i64"].type, BigInteger)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_connectable_issue_example(conn, request):
+ conn = request.getfixturevalue(conn)
+
+ # This tests the example raised in issue
+ # https://github.com/pandas-dev/pandas/issues/10104
+ from sqlalchemy.engine import Engine
+
+ def test_select(connection):
+ query = "SELECT test_foo_data FROM test_foo_data"
+ return sql.read_sql_query(query, con=connection)
+
+ def test_append(connection, data):
+ data.to_sql(name="test_foo_data", con=connection, if_exists="append")
+
+ def test_connectable(conn):
+ # https://github.com/sqlalchemy/sqlalchemy/commit/
+ # 00b5c10846e800304caa86549ab9da373b42fa5d#r48323973
+ foo_data = test_select(conn)
+ test_append(conn, foo_data)
+
+ def main(connectable):
+ if isinstance(connectable, Engine):
+ with connectable.connect() as conn:
+ with conn.begin():
+ test_connectable(conn)
+ else:
+ test_connectable(connectable)
+
+ assert (
+ DataFrame({"test_foo_data": [0, 1, 2]}).to_sql(name="test_foo_data", con=conn)
+ == 3
+ )
+ main(conn)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+@pytest.mark.parametrize(
+ "input",
+ [{"foo": [np.inf]}, {"foo": [-np.inf]}, {"foo": [-np.inf], "infe0": ["bar"]}],
+)
+def test_to_sql_with_negative_npinf(conn, request, input):
+ # GH 34431
+
+ df = DataFrame(input)
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+
+ if "mysql" in conn_name:
+ # GH 36465
+ # The input {"foo": [-np.inf], "infe0": ["bar"]} does not raise any error
+ # for pymysql version >= 0.10
+ msg = "Execution failed on sql"
+ with pytest.raises(pd.errors.DatabaseError, match=msg):
+ df.to_sql(name="foobar", con=conn, index=False)
+ else:
+ assert df.to_sql(name="foobar", con=conn, index=False) == 1
+ res = sql.read_sql_table("foobar", conn)
+ tm.assert_equal(df, res)
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_temporary_table(conn, request):
+ if conn == "sqlite_str":
+ pytest.skip("test does not work with str connection")
+
+ conn = request.getfixturevalue(conn)
+
+ from sqlalchemy import (
+ Column,
+ Integer,
+ Unicode,
+ select,
+ )
+ from sqlalchemy.orm import (
+ Session,
+ declarative_base,
+ )
+
+ test_data = "Hello, World!"
+ expected = DataFrame({"spam": [test_data]})
+ Base = declarative_base()
+
+ class Temporary(Base):
+ __tablename__ = "temp_test"
+ __table_args__ = {"prefixes": ["TEMPORARY"]}
+ id = Column(Integer, primary_key=True)
+ spam = Column(Unicode(30), nullable=False)
+
+ with Session(conn) as session:
+ with session.begin():
+ conn = session.connection()
+ Temporary.__table__.create(conn)
+ session.add(Temporary(spam=test_data))
+ session.flush()
+ df = sql.read_sql_query(sql=select(Temporary.spam), con=conn)
+ tm.assert_frame_equal(df, expected)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_invalid_engine(conn, request, test_frame1):
+ if conn == "sqlite_buildin" or "adbc" in conn:
+ request.applymarker(
+ pytest.mark.xfail(
+ reason="SQLiteDatabase/ADBCDatabase does not raise for bad engine"
+ )
+ )
+
+ conn = request.getfixturevalue(conn)
+ msg = "engine must be one of 'auto', 'sqlalchemy'"
+ with pandasSQL_builder(conn) as pandasSQL:
+ with pytest.raises(ValueError, match=msg):
+ pandasSQL.to_sql(test_frame1, "test_frame1", engine="bad_engine")
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_to_sql_with_sql_engine(conn, request, test_frame1):
+ """`to_sql` with the `engine` param"""
+ # mostly copied from this class's `_to_sql()` method
+ conn = request.getfixturevalue(conn)
+ with pandasSQL_builder(conn) as pandasSQL:
+ with pandasSQL.run_transaction():
+ assert pandasSQL.to_sql(test_frame1, "test_frame1", engine="auto") == 4
+ assert pandasSQL.has_table("test_frame1")
+
+ num_entries = len(test_frame1)
+ num_rows = count_rows(conn, "test_frame1")
+ assert num_rows == num_entries
+
+
+@pytest.mark.parametrize("conn", sqlalchemy_connectable)
+def test_options_sqlalchemy(conn, request, test_frame1):
+ # use the set option
+ conn = request.getfixturevalue(conn)
+ with pd.option_context("io.sql.engine", "sqlalchemy"):
+ with pandasSQL_builder(conn) as pandasSQL:
+ with pandasSQL.run_transaction():
+ assert pandasSQL.to_sql(test_frame1, "test_frame1") == 4
+ assert pandasSQL.has_table("test_frame1")
+
+ num_entries = len(test_frame1)
+ num_rows = count_rows(conn, "test_frame1")
+ assert num_rows == num_entries
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_options_auto(conn, request, test_frame1):
+ # use the set option
+ conn = request.getfixturevalue(conn)
+ with pd.option_context("io.sql.engine", "auto"):
+ with pandasSQL_builder(conn) as pandasSQL:
+ with pandasSQL.run_transaction():
+ assert pandasSQL.to_sql(test_frame1, "test_frame1") == 4
+ assert pandasSQL.has_table("test_frame1")
+
+ num_entries = len(test_frame1)
+ num_rows = count_rows(conn, "test_frame1")
+ assert num_rows == num_entries
+
+
+def test_options_get_engine():
+ pytest.importorskip("sqlalchemy")
+ assert isinstance(get_engine("sqlalchemy"), SQLAlchemyEngine)
+
+ with pd.option_context("io.sql.engine", "sqlalchemy"):
+ assert isinstance(get_engine("auto"), SQLAlchemyEngine)
+ assert isinstance(get_engine("sqlalchemy"), SQLAlchemyEngine)
+
+ with pd.option_context("io.sql.engine", "auto"):
+ assert isinstance(get_engine("auto"), SQLAlchemyEngine)
+ assert isinstance(get_engine("sqlalchemy"), SQLAlchemyEngine)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+@pytest.mark.parametrize("func", ["read_sql", "read_sql_query"])
+def test_read_sql_dtype_backend(
+ conn,
+ request,
+ string_storage,
+ func,
+ dtype_backend,
+ dtype_backend_data,
+ dtype_backend_expected,
+):
+ # GH#50048
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ table = "test"
+ df = dtype_backend_data
+ df.to_sql(name=table, con=conn, index=False, if_exists="replace")
+
+ with pd.option_context("mode.string_storage", string_storage):
+ result = getattr(pd, func)(
+ f"Select * from {table}", conn, dtype_backend=dtype_backend
+ )
+ expected = dtype_backend_expected(string_storage, dtype_backend, conn_name)
+
+ tm.assert_frame_equal(result, expected)
+
+ if "adbc" in conn_name:
+ # adbc does not support chunksize argument
+ request.applymarker(
+ pytest.mark.xfail(reason="adbc does not support chunksize argument")
+ )
+
+ with pd.option_context("mode.string_storage", string_storage):
+ iterator = getattr(pd, func)(
+ f"Select * from {table}",
+ con=conn,
+ dtype_backend=dtype_backend,
+ chunksize=3,
+ )
+ expected = dtype_backend_expected(string_storage, dtype_backend, conn_name)
+ for result in iterator:
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+@pytest.mark.parametrize("func", ["read_sql", "read_sql_table"])
+def test_read_sql_dtype_backend_table(
+ conn,
+ request,
+ string_storage,
+ func,
+ dtype_backend,
+ dtype_backend_data,
+ dtype_backend_expected,
+):
+ if "sqlite" in conn and "adbc" not in conn:
+ request.applymarker(
+ pytest.mark.xfail(
+ reason=(
+ "SQLite actually returns proper boolean values via "
+ "read_sql_table, but before pytest refactor was skipped"
+ )
+ )
+ )
+ # GH#50048
+ conn_name = conn
+ conn = request.getfixturevalue(conn)
+ table = "test"
+ df = dtype_backend_data
+ df.to_sql(name=table, con=conn, index=False, if_exists="replace")
+
+ with pd.option_context("mode.string_storage", string_storage):
+ result = getattr(pd, func)(table, conn, dtype_backend=dtype_backend)
+ expected = dtype_backend_expected(string_storage, dtype_backend, conn_name)
+ tm.assert_frame_equal(result, expected)
+
+ if "adbc" in conn_name:
+ # adbc does not support chunksize argument
+ return
+
+ with pd.option_context("mode.string_storage", string_storage):
+ iterator = getattr(pd, func)(
+ table,
+ conn,
+ dtype_backend=dtype_backend,
+ chunksize=3,
+ )
+ expected = dtype_backend_expected(string_storage, dtype_backend, conn_name)
+ for result in iterator:
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+@pytest.mark.parametrize("func", ["read_sql", "read_sql_table", "read_sql_query"])
+def test_read_sql_invalid_dtype_backend_table(conn, request, func, dtype_backend_data):
+ conn = request.getfixturevalue(conn)
+ table = "test"
+ df = dtype_backend_data
+ df.to_sql(name=table, con=conn, index=False, if_exists="replace")
+
+ msg = (
+ "dtype_backend numpy is invalid, only 'numpy_nullable' and "
+ "'pyarrow' are allowed."
+ )
+ with pytest.raises(ValueError, match=msg):
+ getattr(pd, func)(table, conn, dtype_backend="numpy")
+
+
+@pytest.fixture
+def dtype_backend_data() -> DataFrame:
+ return DataFrame(
+ {
+ "a": Series([1, pd.NA, 3], dtype="Int64"),
+ "b": Series([1, 2, 3], dtype="Int64"),
+ "c": Series([1.5, pd.NA, 2.5], dtype="Float64"),
+ "d": Series([1.5, 2.0, 2.5], dtype="Float64"),
+ "e": [True, False, None],
+ "f": [True, False, True],
+ "g": ["a", "b", "c"],
+ "h": ["a", "b", None],
+ }
+ )
+
+
+@pytest.fixture
+def dtype_backend_expected():
+ def func(string_storage, dtype_backend, conn_name) -> DataFrame:
+ string_dtype: pd.StringDtype | pd.ArrowDtype
+ if dtype_backend == "pyarrow":
+ pa = pytest.importorskip("pyarrow")
+ string_dtype = pd.ArrowDtype(pa.string())
+ else:
+ string_dtype = pd.StringDtype(string_storage)
+
+ df = DataFrame(
+ {
+ "a": Series([1, pd.NA, 3], dtype="Int64"),
+ "b": Series([1, 2, 3], dtype="Int64"),
+ "c": Series([1.5, pd.NA, 2.5], dtype="Float64"),
+ "d": Series([1.5, 2.0, 2.5], dtype="Float64"),
+ "e": Series([True, False, pd.NA], dtype="boolean"),
+ "f": Series([True, False, True], dtype="boolean"),
+ "g": Series(["a", "b", "c"], dtype=string_dtype),
+ "h": Series(["a", "b", None], dtype=string_dtype),
+ }
+ )
+ if dtype_backend == "pyarrow":
+ pa = pytest.importorskip("pyarrow")
+
+ from pandas.arrays import ArrowExtensionArray
+
+ df = DataFrame(
+ {
+ col: ArrowExtensionArray(pa.array(df[col], from_pandas=True))
+ for col in df.columns
+ }
+ )
+
+ if "mysql" in conn_name or "sqlite" in conn_name:
+ if dtype_backend == "numpy_nullable":
+ df = df.astype({"e": "Int64", "f": "Int64"})
+ else:
+ df = df.astype({"e": "int64[pyarrow]", "f": "int64[pyarrow]"})
+
+ return df
+
+ return func
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+def test_chunksize_empty_dtypes(conn, request):
+ # GH#50245
+ if "adbc" in conn:
+ request.node.add_marker(
+ pytest.mark.xfail(reason="chunksize argument NotImplemented with ADBC")
+ )
+ conn = request.getfixturevalue(conn)
+ dtypes = {"a": "int64", "b": "object"}
+ df = DataFrame(columns=["a", "b"]).astype(dtypes)
+ expected = df.copy()
+ df.to_sql(name="test", con=conn, index=False, if_exists="replace")
+
+ for result in read_sql_query(
+ "SELECT * FROM test",
+ conn,
+ dtype=dtypes,
+ chunksize=1,
+ ):
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("conn", all_connectable)
+@pytest.mark.parametrize("dtype_backend", [lib.no_default, "numpy_nullable"])
+@pytest.mark.parametrize("func", ["read_sql", "read_sql_query"])
+def test_read_sql_dtype(conn, request, func, dtype_backend):
+ # GH#50797
+ conn = request.getfixturevalue(conn)
+ table = "test"
+ df = DataFrame({"a": [1, 2, 3], "b": 5})
+ df.to_sql(name=table, con=conn, index=False, if_exists="replace")
+
+ result = getattr(pd, func)(
+ f"Select * from {table}",
+ conn,
+ dtype={"a": np.float64},
+ dtype_backend=dtype_backend,
+ )
+ expected = DataFrame(
+ {
+ "a": Series([1, 2, 3], dtype=np.float64),
+ "b": Series(
+ [5, 5, 5],
+ dtype="int64" if not dtype_backend == "numpy_nullable" else "Int64",
+ ),
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_bigint_warning(sqlite_engine):
+ conn = sqlite_engine
+ # test no warning for BIGINT (to support int64) is raised (GH7433)
+ df = DataFrame({"a": [1, 2]}, dtype="int64")
+ assert df.to_sql(name="test_bigintwarning", con=conn, index=False) == 2
+
+ with tm.assert_produces_warning(None):
+ sql.read_sql_table("test_bigintwarning", conn)
+
+
+def test_valueerror_exception(sqlite_engine):
+ conn = sqlite_engine
+ df = DataFrame({"col1": [1, 2], "col2": [3, 4]})
+ with pytest.raises(ValueError, match="Empty table name specified"):
+ df.to_sql(name="", con=conn, if_exists="replace", index=False)
+
+
+def test_row_object_is_named_tuple(sqlite_engine):
+ conn = sqlite_engine
+ # GH 40682
+ # Test for the is_named_tuple() function
+ # Placed here due to its usage of sqlalchemy
+
+ from sqlalchemy import (
+ Column,
+ Integer,
+ String,
+ )
+ from sqlalchemy.orm import (
+ declarative_base,
+ sessionmaker,
+ )
+
+ BaseModel = declarative_base()
+
+ class Test(BaseModel):
+ __tablename__ = "test_frame"
+ id = Column(Integer, primary_key=True)
+ string_column = Column(String(50))
+
+ with conn.begin():
+ BaseModel.metadata.create_all(conn)
+ Session = sessionmaker(bind=conn)
+ with Session() as session:
+ df = DataFrame({"id": [0, 1], "string_column": ["hello", "world"]})
+ assert (
+ df.to_sql(name="test_frame", con=conn, index=False, if_exists="replace")
+ == 2
+ )
+ session.commit()
+ test_query = session.query(Test.id, Test.string_column)
+ df = DataFrame(test_query)
+
+ assert list(df.columns) == ["id", "string_column"]
+
+
+def test_read_sql_string_inference(sqlite_engine):
+ conn = sqlite_engine
+ # GH#54430
+ table = "test"
+ df = DataFrame({"a": ["x", "y"]})
+ df.to_sql(table, con=conn, index=False, if_exists="replace")
+
+ with pd.option_context("future.infer_string", True):
+ result = read_sql_table(table, conn)
+
+ dtype = pd.StringDtype(na_value=np.nan)
+ expected = DataFrame(
+ {"a": ["x", "y"]}, dtype=dtype, columns=Index(["a"], dtype=dtype)
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_roundtripping_datetimes(sqlite_engine):
+ conn = sqlite_engine
+ # GH#54877
+ df = DataFrame({"t": [datetime(2020, 12, 31, 12)]}, dtype="datetime64[ns]")
+ df.to_sql("test", conn, if_exists="replace", index=False)
+ result = pd.read_sql("select * from test", conn).iloc[0, 0]
+ assert result == "2020-12-31 12:00:00.000000"
+
+
+@pytest.fixture
+def sqlite_builtin_detect_types():
+ with contextlib.closing(
+ sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_DECLTYPES)
+ ) as closing_conn:
+ with closing_conn as conn:
+ yield conn
+
+
+def test_roundtripping_datetimes_detect_types(sqlite_builtin_detect_types):
+ # https://github.com/pandas-dev/pandas/issues/55554
+ conn = sqlite_builtin_detect_types
+ df = DataFrame({"t": [datetime(2020, 12, 31, 12)]}, dtype="datetime64[ns]")
+ df.to_sql("test", conn, if_exists="replace", index=False)
+ result = pd.read_sql("select * from test", conn).iloc[0, 0]
+ assert result == Timestamp("2020-12-31 12:00:00.000000")
+
+
+@pytest.mark.db
+def test_psycopg2_schema_support(postgresql_psycopg2_engine):
+ conn = postgresql_psycopg2_engine
+
+ # only test this for postgresql (schema's not supported in
+ # mysql/sqlite)
+ df = DataFrame({"col1": [1, 2], "col2": [0.1, 0.2], "col3": ["a", "n"]})
+
+ # create a schema
+ with conn.connect() as con:
+ with con.begin():
+ con.exec_driver_sql("DROP SCHEMA IF EXISTS other CASCADE;")
+ con.exec_driver_sql("CREATE SCHEMA other;")
+
+ # write dataframe to different schema's
+ assert df.to_sql(name="test_schema_public", con=conn, index=False) == 2
+ assert (
+ df.to_sql(
+ name="test_schema_public_explicit",
+ con=conn,
+ index=False,
+ schema="public",
+ )
+ == 2
+ )
+ assert (
+ df.to_sql(name="test_schema_other", con=conn, index=False, schema="other") == 2
+ )
+
+ # read dataframes back in
+ res1 = sql.read_sql_table("test_schema_public", conn)
+ tm.assert_frame_equal(df, res1)
+ res2 = sql.read_sql_table("test_schema_public_explicit", conn)
+ tm.assert_frame_equal(df, res2)
+ res3 = sql.read_sql_table("test_schema_public_explicit", conn, schema="public")
+ tm.assert_frame_equal(df, res3)
+ res4 = sql.read_sql_table("test_schema_other", conn, schema="other")
+ tm.assert_frame_equal(df, res4)
+ msg = "Table test_schema_other not found"
+ with pytest.raises(ValueError, match=msg):
+ sql.read_sql_table("test_schema_other", conn, schema="public")
+
+ # different if_exists options
+
+ # create a schema
+ with conn.connect() as con:
+ with con.begin():
+ con.exec_driver_sql("DROP SCHEMA IF EXISTS other CASCADE;")
+ con.exec_driver_sql("CREATE SCHEMA other;")
+
+ # write dataframe with different if_exists options
+ assert (
+ df.to_sql(name="test_schema_other", con=conn, schema="other", index=False) == 2
+ )
+ df.to_sql(
+ name="test_schema_other",
+ con=conn,
+ schema="other",
+ index=False,
+ if_exists="replace",
+ )
+ assert (
+ df.to_sql(
+ name="test_schema_other",
+ con=conn,
+ schema="other",
+ index=False,
+ if_exists="append",
+ )
+ == 2
+ )
+ res = sql.read_sql_table("test_schema_other", conn, schema="other")
+ tm.assert_frame_equal(concat([df, df], ignore_index=True), res)
+
+
+@pytest.mark.db
+def test_self_join_date_columns(postgresql_psycopg2_engine):
+ # GH 44421
+ conn = postgresql_psycopg2_engine
+ from sqlalchemy.sql import text
+
+ create_table = text(
+ """
+ CREATE TABLE person
+ (
+ id serial constraint person_pkey primary key,
+ created_dt timestamp with time zone
+ );
+
+ INSERT INTO person
+ VALUES (1, '2021-01-01T00:00:00Z');
+ """
+ )
+ with conn.connect() as con:
+ with con.begin():
+ con.execute(create_table)
+
+ sql_query = (
+ 'SELECT * FROM "person" AS p1 INNER JOIN "person" AS p2 ON p1.id = p2.id;'
+ )
+ result = pd.read_sql(sql_query, conn)
+ expected = DataFrame(
+ [[1, Timestamp("2021", tz="UTC")] * 2], columns=["id", "created_dt"] * 2
+ )
+ expected["created_dt"] = expected["created_dt"].astype("M8[us, UTC]")
+ tm.assert_frame_equal(result, expected)
+
+ # Cleanup
+ with sql.SQLDatabase(conn, need_transaction=True) as pandasSQL:
+ pandasSQL.drop_table("person")
+
+
+def test_create_and_drop_table(sqlite_engine):
+ conn = sqlite_engine
+ temp_frame = DataFrame({"one": [1.0, 2.0, 3.0, 4.0], "two": [4.0, 3.0, 2.0, 1.0]})
+ with sql.SQLDatabase(conn) as pandasSQL:
+ with pandasSQL.run_transaction():
+ assert pandasSQL.to_sql(temp_frame, "drop_test_frame") == 4
+
+ assert pandasSQL.has_table("drop_test_frame")
+
+ with pandasSQL.run_transaction():
+ pandasSQL.drop_table("drop_test_frame")
+
+ assert not pandasSQL.has_table("drop_test_frame")
+
+
+def test_sqlite_datetime_date(sqlite_buildin):
+ conn = sqlite_buildin
+ df = DataFrame([date(2014, 1, 1), date(2014, 1, 2)], columns=["a"])
+ assert df.to_sql(name="test_date", con=conn, index=False) == 2
+ res = read_sql_query("SELECT * FROM test_date", conn)
+ # comes back as strings
+ tm.assert_frame_equal(res, df.astype(str))
+
+
+@pytest.mark.parametrize("tz_aware", [False, True])
+def test_sqlite_datetime_time(tz_aware, sqlite_buildin):
+ conn = sqlite_buildin
+ # test support for datetime.time, GH #8341
+ if not tz_aware:
+ tz_times = [time(9, 0, 0), time(9, 1, 30)]
+ else:
+ tz_dt = date_range("2013-01-01 09:00:00", periods=2, tz="US/Pacific")
+ tz_times = Series(tz_dt.to_pydatetime()).map(lambda dt: dt.timetz())
+
+ df = DataFrame(tz_times, columns=["a"])
+
+ assert df.to_sql(name="test_time", con=conn, index=False) == 2
+ res = read_sql_query("SELECT * FROM test_time", conn)
+ # comes back as strings
+ expected = df.map(lambda _: _.strftime("%H:%M:%S.%f"))
+ tm.assert_frame_equal(res, expected)
+
+
+def get_sqlite_column_type(conn, table, column):
+ recs = conn.execute(f"PRAGMA table_info({table})")
+ for cid, name, ctype, not_null, default, pk in recs:
+ if name == column:
+ return ctype
+ raise ValueError(f"Table {table}, column {column} not found")
+
+
+def test_sqlite_test_dtype(sqlite_buildin):
+ conn = sqlite_buildin
+ cols = ["A", "B"]
+ data = [(0.8, True), (0.9, None)]
+ df = DataFrame(data, columns=cols)
+ assert df.to_sql(name="dtype_test", con=conn) == 2
+ assert df.to_sql(name="dtype_test2", con=conn, dtype={"B": "STRING"}) == 2
+
+ # sqlite stores Boolean values as INTEGER
+ assert get_sqlite_column_type(conn, "dtype_test", "B") == "INTEGER"
+
+ assert get_sqlite_column_type(conn, "dtype_test2", "B") == "STRING"
+ msg = r"B \(\) not a string"
+ with pytest.raises(ValueError, match=msg):
+ df.to_sql(name="error", con=conn, dtype={"B": bool})
+
+ # single dtype
+ assert df.to_sql(name="single_dtype_test", con=conn, dtype="STRING") == 2
+ assert get_sqlite_column_type(conn, "single_dtype_test", "A") == "STRING"
+ assert get_sqlite_column_type(conn, "single_dtype_test", "B") == "STRING"
+
+
+def test_sqlite_notna_dtype(sqlite_buildin):
+ conn = sqlite_buildin
+ cols = {
+ "Bool": Series([True, None]),
+ "Date": Series([datetime(2012, 5, 1), None]),
+ "Int": Series([1, None], dtype="object"),
+ "Float": Series([1.1, None]),
+ }
+ df = DataFrame(cols)
+
+ tbl = "notna_dtype_test"
+ assert df.to_sql(name=tbl, con=conn) == 2
+
+ assert get_sqlite_column_type(conn, tbl, "Bool") == "INTEGER"
+ assert get_sqlite_column_type(conn, tbl, "Date") == "TIMESTAMP"
+ assert get_sqlite_column_type(conn, tbl, "Int") == "INTEGER"
+ assert get_sqlite_column_type(conn, tbl, "Float") == "REAL"
+
+
+def test_sqlite_illegal_names(sqlite_buildin):
+ # For sqlite, these should work fine
+ conn = sqlite_buildin
+ df = DataFrame([[1, 2], [3, 4]], columns=["a", "b"])
+
+ msg = "Empty table or column name specified"
+ with pytest.raises(ValueError, match=msg):
+ df.to_sql(name="", con=conn)
+
+ for ndx, weird_name in enumerate(
+ [
+ "test_weird_name]",
+ "test_weird_name[",
+ "test_weird_name`",
+ 'test_weird_name"',
+ "test_weird_name'",
+ "_b.test_weird_name_01-30",
+ '"_b.test_weird_name_01-30"',
+ "99beginswithnumber",
+ "12345",
+ "\xe9",
+ ]
+ ):
+ assert df.to_sql(name=weird_name, con=conn) == 2
+ sql.table_exists(weird_name, conn)
+
+ df2 = DataFrame([[1, 2], [3, 4]], columns=["a", weird_name])
+ c_tbl = f"test_weird_col_name{ndx:d}"
+ assert df2.to_sql(name=c_tbl, con=conn) == 2
+ sql.table_exists(c_tbl, conn)
+
+
+def format_query(sql, *args):
+ _formatters = {
+ datetime: "'{}'".format,
+ str: "'{}'".format,
+ np.str_: "'{}'".format,
+ bytes: "'{}'".format,
+ float: "{:.8f}".format,
+ int: "{:d}".format,
+ type(None): lambda x: "NULL",
+ np.float64: "{:.10f}".format,
+ bool: "'{!s}'".format,
+ }
+ processed_args = []
+ for arg in args:
+ if isinstance(arg, float) and isna(arg):
+ arg = None
+
+ formatter = _formatters[type(arg)]
+ processed_args.append(formatter(arg))
+
+ return sql % tuple(processed_args)
+
+
+def tquery(query, con=None):
+ """Replace removed sql.tquery function"""
+ with sql.pandasSQL_builder(con) as pandas_sql:
+ res = pandas_sql.execute(query).fetchall()
+ return None if res is None else list(res)
+
+
+def test_xsqlite_basic(sqlite_buildin):
+ frame = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ assert sql.to_sql(frame, name="test_table", con=sqlite_buildin, index=False) == 10
+ result = sql.read_sql("select * from test_table", sqlite_buildin)
+
+ # HACK! Change this once indexes are handled properly.
+ result.index = frame.index
+
+ expected = frame
+ tm.assert_frame_equal(result, frame)
+
+ frame["txt"] = ["a"] * len(frame)
+ frame2 = frame.copy()
+ new_idx = Index(np.arange(len(frame2)), dtype=np.int64) + 10
+ frame2["Idx"] = new_idx.copy()
+ assert sql.to_sql(frame2, name="test_table2", con=sqlite_buildin, index=False) == 10
+ result = sql.read_sql("select * from test_table2", sqlite_buildin, index_col="Idx")
+ expected = frame.copy()
+ expected.index = new_idx
+ expected.index.name = "Idx"
+ tm.assert_frame_equal(expected, result)
+
+
+def test_xsqlite_write_row_by_row(sqlite_buildin):
+ frame = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ frame.iloc[0, 0] = np.nan
+ create_sql = sql.get_schema(frame, "test")
+ cur = sqlite_buildin.cursor()
+ cur.execute(create_sql)
+
+ ins = "INSERT INTO test VALUES (%s, %s, %s, %s)"
+ for _, row in frame.iterrows():
+ fmt_sql = format_query(ins, *row)
+ tquery(fmt_sql, con=sqlite_buildin)
+
+ sqlite_buildin.commit()
+
+ result = sql.read_sql("select * from test", con=sqlite_buildin)
+ result.index = frame.index
+ tm.assert_frame_equal(result, frame, rtol=1e-3)
+
+
+def test_xsqlite_execute(sqlite_buildin):
+ frame = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ create_sql = sql.get_schema(frame, "test")
+ cur = sqlite_buildin.cursor()
+ cur.execute(create_sql)
+ ins = "INSERT INTO test VALUES (?, ?, ?, ?)"
+
+ row = frame.iloc[0]
+ with sql.pandasSQL_builder(sqlite_buildin) as pandas_sql:
+ pandas_sql.execute(ins, tuple(row))
+ sqlite_buildin.commit()
+
+ result = sql.read_sql("select * from test", sqlite_buildin)
+ result.index = frame.index[:1]
+ tm.assert_frame_equal(result, frame[:1])
+
+
+def test_xsqlite_schema(sqlite_buildin):
+ frame = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ create_sql = sql.get_schema(frame, "test")
+ lines = create_sql.splitlines()
+ for line in lines:
+ tokens = line.split(" ")
+ if len(tokens) == 2 and tokens[0] == "A":
+ assert tokens[1] == "DATETIME"
+
+ create_sql = sql.get_schema(frame, "test", keys=["A", "B"])
+ lines = create_sql.splitlines()
+ assert 'PRIMARY KEY ("A", "B")' in create_sql
+ cur = sqlite_buildin.cursor()
+ cur.execute(create_sql)
+
+
+def test_xsqlite_execute_fail(sqlite_buildin):
+ create_sql = """
+ CREATE TABLE test
+ (
+ a TEXT,
+ b TEXT,
+ c REAL,
+ PRIMARY KEY (a, b)
+ );
+ """
+ cur = sqlite_buildin.cursor()
+ cur.execute(create_sql)
+
+ with sql.pandasSQL_builder(sqlite_buildin) as pandas_sql:
+ pandas_sql.execute("INSERT INTO test VALUES('foo', 'bar', 1.234)")
+ pandas_sql.execute("INSERT INTO test VALUES('foo', 'baz', 2.567)")
+
+ with pytest.raises(sql.DatabaseError, match="Execution failed on sql"):
+ pandas_sql.execute("INSERT INTO test VALUES('foo', 'bar', 7)")
+
+
+def test_xsqlite_execute_closed_connection():
+ create_sql = """
+ CREATE TABLE test
+ (
+ a TEXT,
+ b TEXT,
+ c REAL,
+ PRIMARY KEY (a, b)
+ );
+ """
+ with contextlib.closing(sqlite3.connect(":memory:")) as conn:
+ cur = conn.cursor()
+ cur.execute(create_sql)
+
+ with sql.pandasSQL_builder(conn) as pandas_sql:
+ pandas_sql.execute("INSERT INTO test VALUES('foo', 'bar', 1.234)")
+
+ msg = "Cannot operate on a closed database."
+ with pytest.raises(sqlite3.ProgrammingError, match=msg):
+ tquery("select * from test", con=conn)
+
+
+def test_xsqlite_keyword_as_column_names(sqlite_buildin):
+ df = DataFrame({"From": np.ones(5)})
+ assert sql.to_sql(df, con=sqlite_buildin, name="testkeywords", index=False) == 5
+
+
+def test_xsqlite_onecolumn_of_integer(sqlite_buildin):
+ # GH 3628
+ # a column_of_integers dataframe should transfer well to sql
+
+ mono_df = DataFrame([1, 2], columns=["c0"])
+ assert sql.to_sql(mono_df, con=sqlite_buildin, name="mono_df", index=False) == 2
+ # computing the sum via sql
+ con_x = sqlite_buildin
+ the_sum = sum(my_c0[0] for my_c0 in con_x.execute("select * from mono_df"))
+ # it should not fail, and gives 3 ( Issue #3628 )
+ assert the_sum == 3
+
+ result = sql.read_sql("select * from mono_df", con_x)
+ tm.assert_frame_equal(result, mono_df)
+
+
+def test_xsqlite_if_exists(sqlite_buildin):
+ df_if_exists_1 = DataFrame({"col1": [1, 2], "col2": ["A", "B"]})
+ df_if_exists_2 = DataFrame({"col1": [3, 4, 5], "col2": ["C", "D", "E"]})
+ table_name = "table_if_exists"
+ sql_select = f"SELECT * FROM {table_name}"
+
+ msg = "'notvalidvalue' is not valid for if_exists"
+ with pytest.raises(ValueError, match=msg):
+ sql.to_sql(
+ frame=df_if_exists_1,
+ con=sqlite_buildin,
+ name=table_name,
+ if_exists="notvalidvalue",
+ )
+ drop_table(table_name, sqlite_buildin)
+
+ # test if_exists='fail'
+ sql.to_sql(
+ frame=df_if_exists_1, con=sqlite_buildin, name=table_name, if_exists="fail"
+ )
+ msg = "Table 'table_if_exists' already exists"
+ with pytest.raises(ValueError, match=msg):
+ sql.to_sql(
+ frame=df_if_exists_1,
+ con=sqlite_buildin,
+ name=table_name,
+ if_exists="fail",
+ )
+ # test if_exists='replace'
+ sql.to_sql(
+ frame=df_if_exists_1,
+ con=sqlite_buildin,
+ name=table_name,
+ if_exists="replace",
+ index=False,
+ )
+ assert tquery(sql_select, con=sqlite_buildin) == [(1, "A"), (2, "B")]
+ assert (
+ sql.to_sql(
+ frame=df_if_exists_2,
+ con=sqlite_buildin,
+ name=table_name,
+ if_exists="replace",
+ index=False,
+ )
+ == 3
+ )
+ assert tquery(sql_select, con=sqlite_buildin) == [(3, "C"), (4, "D"), (5, "E")]
+ drop_table(table_name, sqlite_buildin)
+
+ # test if_exists='append'
+ assert (
+ sql.to_sql(
+ frame=df_if_exists_1,
+ con=sqlite_buildin,
+ name=table_name,
+ if_exists="fail",
+ index=False,
+ )
+ == 2
+ )
+ assert tquery(sql_select, con=sqlite_buildin) == [(1, "A"), (2, "B")]
+ assert (
+ sql.to_sql(
+ frame=df_if_exists_2,
+ con=sqlite_buildin,
+ name=table_name,
+ if_exists="append",
+ index=False,
+ )
+ == 3
+ )
+ assert tquery(sql_select, con=sqlite_buildin) == [
+ (1, "A"),
+ (2, "B"),
+ (3, "C"),
+ (4, "D"),
+ (5, "E"),
+ ]
+ drop_table(table_name, sqlite_buildin)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_stata.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_stata.py
new file mode 100644
index 0000000000000000000000000000000000000000..f69ec1f6105605f80eadccd0072c7c0683158b49
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_stata.py
@@ -0,0 +1,2624 @@
+import bz2
+import datetime as dt
+from datetime import datetime
+import gzip
+import io
+import itertools
+import os
+import string
+import struct
+import tarfile
+import zipfile
+
+import numpy as np
+import pytest
+
+from pandas.errors import Pandas4Warning
+import pandas.util._test_decorators as td
+
+import pandas as pd
+from pandas import CategoricalDtype
+import pandas._testing as tm
+from pandas.core.frame import (
+ DataFrame,
+ Series,
+)
+
+from pandas.io.parsers import read_csv
+from pandas.io.stata import (
+ CategoricalConversionWarning,
+ InvalidColumnName,
+ PossiblePrecisionLoss,
+ StataMissingValue,
+ StataReader,
+ StataWriter,
+ StataWriterUTF8,
+ ValueLabelTypeMismatch,
+ read_stata,
+)
+
+
+@pytest.fixture
+def mixed_frame():
+ return DataFrame(
+ {
+ "a": [1, 2, 3, 4],
+ "b": [1.0, 3.0, 27.0, 81.0],
+ "c": ["Atlanta", "Birmingham", "Cincinnati", "Detroit"],
+ }
+ )
+
+
+@pytest.fixture
+def parsed_114(datapath):
+ dta14_114 = datapath("io", "data", "stata", "stata5_114.dta")
+ parsed_114 = read_stata(dta14_114, convert_dates=True)
+ parsed_114.index.name = "index"
+ return parsed_114
+
+
+class TestStata:
+ def read_dta(self, file):
+ # Legacy default reader configuration
+ return read_stata(file, convert_dates=True)
+
+ def read_csv(self, file):
+ return read_csv(file, parse_dates=True)
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ def test_read_empty_dta(self, version, temp_file):
+ empty_ds = DataFrame(columns=["unit"])
+ # GH 7369, make sure can read a 0-obs dta file
+ path = temp_file
+ empty_ds.to_stata(path, write_index=False, version=version)
+ empty_ds2 = read_stata(path)
+ tm.assert_frame_equal(empty_ds, empty_ds2)
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ def test_read_empty_dta_with_dtypes(self, version, temp_file):
+ # GH 46240
+ # Fixing above bug revealed that types are not correctly preserved when
+ # writing empty DataFrames
+ empty_df_typed = DataFrame(
+ {
+ "i8": np.array([0], dtype=np.int8),
+ "i16": np.array([0], dtype=np.int16),
+ "i32": np.array([0], dtype=np.int32),
+ "i64": np.array([0], dtype=np.int64),
+ "u8": np.array([0], dtype=np.uint8),
+ "u16": np.array([0], dtype=np.uint16),
+ "u32": np.array([0], dtype=np.uint32),
+ "u64": np.array([0], dtype=np.uint64),
+ "f32": np.array([0], dtype=np.float32),
+ "f64": np.array([0], dtype=np.float64),
+ }
+ )
+ # GH 7369, make sure can read a 0-obs dta file
+ path = temp_file
+ empty_df_typed.to_stata(path, write_index=False, version=version)
+ empty_reread = read_stata(path)
+
+ expected = empty_df_typed
+ # No uint# support. Downcast since values in range for int#
+ expected["u8"] = expected["u8"].astype(np.int8)
+ expected["u16"] = expected["u16"].astype(np.int16)
+ expected["u32"] = expected["u32"].astype(np.int32)
+ # No int64 supported at all. Downcast since values in range for int32
+ expected["u64"] = expected["u64"].astype(np.int32)
+ expected["i64"] = expected["i64"].astype(np.int32)
+
+ tm.assert_frame_equal(expected, empty_reread)
+ tm.assert_series_equal(expected.dtypes, empty_reread.dtypes)
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ def test_read_index_col_none(self, version, temp_file):
+ df = DataFrame({"a": range(5), "b": ["b1", "b2", "b3", "b4", "b5"]})
+ # GH 7369, make sure can read a 0-obs dta file
+ path = temp_file
+ df.to_stata(path, write_index=False, version=version)
+ read_df = read_stata(path)
+
+ assert isinstance(read_df.index, pd.RangeIndex)
+ expected = df
+ expected["a"] = expected["a"].astype(np.int32)
+ tm.assert_frame_equal(read_df, expected, check_index_type=True)
+
+ @pytest.mark.parametrize(
+ "version", [102, 103, 104, 105, 108, 110, 111, 113, 114, 115, 117, 118, 119]
+ )
+ def test_read_dta1(self, version, datapath):
+ file = datapath("io", "data", "stata", f"stata1_{version}.dta")
+ parsed = self.read_dta(file)
+
+ # Pandas uses np.nan as missing value.
+ # Thus, all columns will be of type float, regardless of their name.
+ expected = DataFrame(
+ [(np.nan, np.nan, np.nan, np.nan, np.nan)],
+ columns=["float_miss", "double_miss", "byte_miss", "int_miss", "long_miss"],
+ )
+
+ # this is an oddity as really the nan should be float64, but
+ # the casting doesn't fail so need to match stata here
+ expected["float_miss"] = expected["float_miss"].astype(np.float32)
+
+ # Column names too long for older Stata formats
+ if version <= 108:
+ expected = expected.rename(
+ columns={
+ "float_miss": "f_miss",
+ "double_miss": "d_miss",
+ "byte_miss": "b_miss",
+ "int_miss": "i_miss",
+ "long_miss": "l_miss",
+ }
+ )
+
+ tm.assert_frame_equal(parsed, expected)
+
+ def test_read_dta2(self, datapath):
+ expected = DataFrame.from_records(
+ [
+ (
+ datetime(2006, 11, 19, 23, 13, 20),
+ 1479596223000,
+ datetime(2010, 1, 20),
+ datetime(2010, 1, 8),
+ datetime(2010, 1, 1),
+ datetime(1974, 7, 1),
+ datetime(2010, 1, 1),
+ datetime(2010, 1, 1),
+ ),
+ (
+ datetime(1959, 12, 31, 20, 3, 20),
+ -1479590,
+ datetime(1953, 10, 2),
+ datetime(1948, 6, 10),
+ datetime(1955, 1, 1),
+ datetime(1955, 7, 1),
+ datetime(1955, 1, 1),
+ datetime(2, 1, 1),
+ ),
+ (pd.NaT, pd.NaT, pd.NaT, pd.NaT, pd.NaT, pd.NaT, pd.NaT, pd.NaT),
+ ],
+ columns=[
+ "datetime_c",
+ "datetime_big_c",
+ "date",
+ "weekly_date",
+ "monthly_date",
+ "quarterly_date",
+ "half_yearly_date",
+ "yearly_date",
+ ],
+ )
+ # TODO(GH#55564): just pass M8[s] to the constructor
+ expected["datetime_c"] = expected["datetime_c"].astype("M8[ms]")
+ expected["date"] = expected["date"].astype("M8[s]")
+ expected["weekly_date"] = expected["weekly_date"].astype("M8[s]")
+ expected["monthly_date"] = expected["monthly_date"].astype("M8[s]")
+ expected["quarterly_date"] = expected["quarterly_date"].astype("M8[s]")
+ expected["half_yearly_date"] = expected["half_yearly_date"].astype("M8[s]")
+ expected["yearly_date"] = expected["yearly_date"].astype("M8[s]")
+
+ path1 = datapath("io", "data", "stata", "stata2_114.dta")
+ path2 = datapath("io", "data", "stata", "stata2_115.dta")
+ path3 = datapath("io", "data", "stata", "stata2_117.dta")
+
+ msg = "Leaving in Stata Internal Format"
+ with tm.assert_produces_warning(UserWarning, match=msg):
+ parsed_114 = self.read_dta(path1)
+ with tm.assert_produces_warning(UserWarning, match=msg):
+ parsed_115 = self.read_dta(path2)
+ with tm.assert_produces_warning(UserWarning, match=msg):
+ parsed_117 = self.read_dta(path3)
+ # FIXME: don't leave commented-out
+ # 113 is buggy due to limits of date format support in Stata
+ # parsed_113 = self.read_dta(
+ # datapath("io", "data", "stata", "stata2_113.dta")
+ # )
+
+ # FIXME: don't leave commented-out
+ # buggy test because of the NaT comparison on certain platforms
+ # Format 113 test fails since it does not support tc and tC formats
+ # tm.assert_frame_equal(parsed_113, expected)
+ tm.assert_frame_equal(parsed_114, expected)
+ tm.assert_frame_equal(parsed_115, expected)
+ tm.assert_frame_equal(parsed_117, expected)
+
+ @pytest.mark.parametrize(
+ "file", ["stata3_113", "stata3_114", "stata3_115", "stata3_117"]
+ )
+ def test_read_dta3(self, file, datapath):
+ file = datapath("io", "data", "stata", f"{file}.dta")
+ parsed = self.read_dta(file)
+
+ # match stata here
+ expected = self.read_csv(datapath("io", "data", "stata", "stata3.csv"))
+ expected = expected.astype(np.float32)
+ expected["year"] = expected["year"].astype(np.int16)
+ expected["quarter"] = expected["quarter"].astype(np.int8)
+
+ tm.assert_frame_equal(parsed, expected)
+
+ @pytest.mark.parametrize("version", [110, 111, 113, 114, 115, 117])
+ def test_read_dta4(self, version, datapath):
+ file = datapath("io", "data", "stata", f"stata4_{version}.dta")
+ parsed = self.read_dta(file)
+
+ expected = DataFrame.from_records(
+ [
+ ["one", "ten", "one", "one", "one"],
+ ["two", "nine", "two", "two", "two"],
+ ["three", "eight", "three", "three", "three"],
+ ["four", "seven", 4, "four", "four"],
+ ["five", "six", 5, np.nan, "five"],
+ ["six", "five", 6, np.nan, "six"],
+ ["seven", "four", 7, np.nan, "seven"],
+ ["eight", "three", 8, np.nan, "eight"],
+ ["nine", "two", 9, np.nan, "nine"],
+ ["ten", "one", "ten", np.nan, "ten"],
+ ],
+ columns=[
+ "fully_labeled",
+ "fully_labeled2",
+ "incompletely_labeled",
+ "labeled_with_missings",
+ "float_labelled",
+ ],
+ )
+
+ # these are all categoricals
+ for col in expected:
+ orig = expected[col].copy()
+
+ categories = np.asarray(expected["fully_labeled"][orig.notna()])
+ if col == "incompletely_labeled":
+ categories = orig
+
+ cat = orig.astype("category")._values
+ cat = cat.set_categories(categories, ordered=True)
+ cat.categories.rename(None, inplace=True)
+
+ expected[col] = cat
+
+ # stata doesn't save .category metadata
+ tm.assert_frame_equal(parsed, expected)
+
+ @pytest.mark.parametrize("version", [102, 103, 104, 105, 108])
+ def test_readold_dta4(self, version, datapath):
+ # This test is the same as test_read_dta4 above except that the columns
+ # had to be renamed to match the restrictions in older file format
+ file = datapath("io", "data", "stata", f"stata4_{version}.dta")
+ parsed = self.read_dta(file)
+
+ expected = DataFrame.from_records(
+ [
+ ["one", "ten", "one", "one", "one"],
+ ["two", "nine", "two", "two", "two"],
+ ["three", "eight", "three", "three", "three"],
+ ["four", "seven", 4, "four", "four"],
+ ["five", "six", 5, np.nan, "five"],
+ ["six", "five", 6, np.nan, "six"],
+ ["seven", "four", 7, np.nan, "seven"],
+ ["eight", "three", 8, np.nan, "eight"],
+ ["nine", "two", 9, np.nan, "nine"],
+ ["ten", "one", "ten", np.nan, "ten"],
+ ],
+ columns=[
+ "fulllab",
+ "fulllab2",
+ "incmplab",
+ "misslab",
+ "floatlab",
+ ],
+ )
+
+ # these are all categoricals
+ for col in expected:
+ orig = expected[col].copy()
+
+ categories = np.asarray(expected["fulllab"][orig.notna()])
+ if col == "incmplab":
+ categories = orig
+
+ cat = orig.astype("category")._values
+ cat = cat.set_categories(categories, ordered=True)
+ cat.categories.rename(None, inplace=True)
+
+ expected[col] = cat
+
+ # stata doesn't save .category metadata
+ tm.assert_frame_equal(parsed, expected)
+
+ # File containing strls
+ @pytest.mark.parametrize(
+ "file",
+ [
+ "stata12_117",
+ "stata12_be_117",
+ "stata12_118",
+ "stata12_be_118",
+ "stata12_119",
+ "stata12_be_119",
+ ],
+ )
+ def test_read_dta_strl(self, file, datapath):
+ parsed = self.read_dta(datapath("io", "data", "stata", f"{file}.dta"))
+ expected = DataFrame.from_records(
+ [
+ [1, "abc", "abcdefghi"],
+ [3, "cba", "qwertywertyqwerty"],
+ [93, "", "strl"],
+ ],
+ columns=["x", "y", "z"],
+ )
+
+ tm.assert_frame_equal(parsed, expected, check_dtype=False)
+
+ # 117 is not included in this list as it uses ASCII strings
+ @pytest.mark.parametrize(
+ "file",
+ [
+ "stata14_118",
+ "stata14_be_118",
+ "stata14_119",
+ "stata14_be_119",
+ ],
+ )
+ def test_read_dta118_119(self, file, datapath):
+ parsed_118 = self.read_dta(datapath("io", "data", "stata", f"{file}.dta"))
+ parsed_118["Bytes"] = parsed_118["Bytes"].astype("O")
+ expected = DataFrame.from_records(
+ [
+ ["Cat", "Bogota", "Bogotá", 1, 1.0, "option b Ünicode", 1.0],
+ ["Dog", "Boston", "Uzunköprü", np.nan, np.nan, np.nan, np.nan],
+ ["Plane", "Rome", "Tromsø", 0, 0.0, "option a", 0.0],
+ ["Potato", "Tokyo", "Elâzığ", -4, 4.0, 4, 4], # noqa: RUF001
+ ["", "", "", 0, 0.3332999, "option a", 1 / 3.0],
+ ],
+ columns=[
+ "Things",
+ "Cities",
+ "Unicode_Cities_Strl",
+ "Ints",
+ "Floats",
+ "Bytes",
+ "Longs",
+ ],
+ )
+ expected["Floats"] = expected["Floats"].astype(np.float32)
+ for col in parsed_118.columns:
+ tm.assert_almost_equal(parsed_118[col], expected[col])
+
+ with StataReader(datapath("io", "data", "stata", f"{file}.dta")) as rdr:
+ vl = rdr.variable_labels()
+ vl_expected = {
+ "Unicode_Cities_Strl": "Here are some strls with Ünicode chars",
+ "Longs": "long data",
+ "Things": "Here are some things",
+ "Bytes": "byte data",
+ "Ints": "int data",
+ "Cities": "Here are some cities",
+ "Floats": "float data",
+ }
+ tm.assert_dict_equal(vl, vl_expected)
+
+ assert rdr.data_label == "This is a Ünicode data label"
+
+ def test_read_write_dta5(self, temp_file):
+ original = DataFrame(
+ [(np.nan, np.nan, np.nan, np.nan, np.nan)],
+ columns=["float_miss", "double_miss", "byte_miss", "int_miss", "long_miss"],
+ )
+ original.index.name = "index"
+
+ path = temp_file
+ original.to_stata(path, convert_dates=None)
+ written_and_read_again = self.read_dta(path)
+
+ expected = original
+ expected.index = expected.index.astype(np.int32)
+ tm.assert_frame_equal(written_and_read_again.set_index("index"), expected)
+
+ def test_write_dta6(self, datapath, temp_file):
+ original = self.read_csv(datapath("io", "data", "stata", "stata3.csv"))
+ original.index.name = "index"
+ original.index = original.index.astype(np.int32)
+ original["year"] = original["year"].astype(np.int32)
+ original["quarter"] = original["quarter"].astype(np.int32)
+
+ path = temp_file
+ original.to_stata(path, convert_dates=None)
+ written_and_read_again = self.read_dta(path)
+ tm.assert_frame_equal(
+ written_and_read_again.set_index("index"),
+ original,
+ check_index_type=False,
+ )
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ def test_read_write_dta10(self, version, temp_file, using_infer_string):
+ original = DataFrame(
+ data=[["string", "object", 1, 1.1, np.datetime64("2003-12-25")]],
+ columns=["string", "object", "integer", "floating", "datetime"],
+ )
+ original["object"] = Series(original["object"], dtype=object)
+ original.index.name = "index"
+ original.index = original.index.astype(np.int32)
+ original["integer"] = original["integer"].astype(np.int32)
+
+ path = temp_file
+ original.to_stata(path, convert_dates={"datetime": "tc"}, version=version)
+ written_and_read_again = self.read_dta(path)
+
+ expected = original.copy()
+ # "tc" convert_dates means we store in ms
+ expected["datetime"] = expected["datetime"].astype("M8[ms]")
+ if using_infer_string:
+ expected["object"] = expected["object"].astype("str")
+
+ tm.assert_frame_equal(
+ written_and_read_again.set_index("index"),
+ expected,
+ )
+
+ def test_stata_doc_examples(self, temp_file):
+ path = temp_file
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)), columns=list("AB")
+ )
+ df.to_stata(path)
+
+ def test_write_preserves_original(self, temp_file):
+ # 9795
+
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((5, 4)), columns=list("abcd")
+ )
+ df.loc[2, "a":"c"] = np.nan
+ df_copy = df.copy()
+ path = temp_file
+ df.to_stata(path, write_index=False)
+ tm.assert_frame_equal(df, df_copy)
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ def test_encoding(self, version, datapath, temp_file):
+ # GH 4626, proper encoding handling
+ raw = read_stata(datapath("io", "data", "stata", "stata1_encoding.dta"))
+ encoded = read_stata(datapath("io", "data", "stata", "stata1_encoding.dta"))
+ result = encoded.kreis1849[0]
+
+ expected = raw.kreis1849[0]
+ assert result == expected
+ assert isinstance(result, str)
+
+ path = temp_file
+ encoded.to_stata(path, write_index=False, version=version)
+ reread_encoded = read_stata(path)
+ tm.assert_frame_equal(encoded, reread_encoded)
+
+ def test_read_write_dta11(self, temp_file):
+ original = DataFrame(
+ [(1, 2, 3, 4)],
+ columns=[
+ "good",
+ "b\u00e4d",
+ "8number",
+ "astringwithmorethan32characters______",
+ ],
+ )
+ formatted = DataFrame(
+ [(1, 2, 3, 4)],
+ columns=["good", "b_d", "_8number", "astringwithmorethan32characters_"],
+ )
+ formatted.index.name = "index"
+ formatted = formatted.astype(np.int32)
+
+ path = temp_file
+ msg = "Not all pandas column names were valid Stata variable names"
+ with tm.assert_produces_warning(InvalidColumnName, match=msg):
+ original.to_stata(path, convert_dates=None)
+
+ written_and_read_again = self.read_dta(path)
+
+ expected = formatted
+ expected.index = expected.index.astype(np.int32)
+ tm.assert_frame_equal(written_and_read_again.set_index("index"), expected)
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ def test_read_write_dta12(self, version, temp_file):
+ original = DataFrame(
+ [(1, 2, 3, 4, 5, 6)],
+ columns=[
+ "astringwithmorethan32characters_1",
+ "astringwithmorethan32characters_2",
+ "+",
+ "-",
+ "short",
+ "delete",
+ ],
+ )
+ formatted = DataFrame(
+ [(1, 2, 3, 4, 5, 6)],
+ columns=[
+ "astringwithmorethan32characters_",
+ "_0astringwithmorethan32character",
+ "_",
+ "_1_",
+ "_short",
+ "_delete",
+ ],
+ )
+ formatted.index.name = "index"
+ formatted = formatted.astype(np.int32)
+
+ path = temp_file
+ msg = "Not all pandas column names were valid Stata variable names"
+ with tm.assert_produces_warning(InvalidColumnName, match=msg):
+ original.to_stata(path, convert_dates=None, version=version)
+ # should get a warning for that format.
+
+ written_and_read_again = self.read_dta(path)
+
+ expected = formatted
+ expected.index = expected.index.astype(np.int32)
+ tm.assert_frame_equal(written_and_read_again.set_index("index"), expected)
+
+ def test_read_write_dta13(self, temp_file):
+ s1 = Series(2**9, dtype=np.int16)
+ s2 = Series(2**17, dtype=np.int32)
+ s3 = Series(2**33, dtype=np.int64)
+ original = DataFrame({"int16": s1, "int32": s2, "int64": s3})
+ original.index.name = "index"
+
+ formatted = original
+ formatted["int64"] = formatted["int64"].astype(np.float64)
+
+ path = temp_file
+ original.to_stata(path)
+ written_and_read_again = self.read_dta(path)
+
+ expected = formatted
+ expected.index = expected.index.astype(np.int32)
+ tm.assert_frame_equal(written_and_read_again.set_index("index"), expected)
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ @pytest.mark.parametrize(
+ "file", ["stata5_113", "stata5_114", "stata5_115", "stata5_117"]
+ )
+ def test_read_write_reread_dta14(
+ self, file, parsed_114, version, datapath, temp_file
+ ):
+ file = datapath("io", "data", "stata", f"{file}.dta")
+ parsed = self.read_dta(file)
+ parsed.index.name = "index"
+
+ tm.assert_frame_equal(parsed_114, parsed)
+
+ path = temp_file
+ parsed_114.to_stata(path, convert_dates={"date_td": "td"}, version=version)
+ written_and_read_again = self.read_dta(path)
+
+ expected = parsed_114.copy()
+ tm.assert_frame_equal(written_and_read_again.set_index("index"), expected)
+
+ @pytest.mark.parametrize(
+ "file", ["stata6_113", "stata6_114", "stata6_115", "stata6_117"]
+ )
+ def test_read_write_reread_dta15(self, file, datapath):
+ expected = self.read_csv(datapath("io", "data", "stata", "stata6.csv"))
+ expected["byte_"] = expected["byte_"].astype(np.int8)
+ expected["int_"] = expected["int_"].astype(np.int16)
+ expected["long_"] = expected["long_"].astype(np.int32)
+ expected["float_"] = expected["float_"].astype(np.float32)
+ expected["double_"] = expected["double_"].astype(np.float64)
+
+ # TODO(GH#55564): directly cast to M8[s]
+ arr = expected["date_td"].astype("Period[D]")._values.asfreq("s", how="S")
+ expected["date_td"] = arr.view("M8[s]")
+
+ file = datapath("io", "data", "stata", f"{file}.dta")
+ parsed = self.read_dta(file)
+
+ tm.assert_frame_equal(expected, parsed)
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ def test_timestamp_and_label(self, version, temp_file):
+ original = DataFrame([(1,)], columns=["variable"])
+ time_stamp = datetime(2000, 2, 29, 14, 21)
+ data_label = "This is a data file."
+ path = temp_file
+ original.to_stata(
+ path, time_stamp=time_stamp, data_label=data_label, version=version
+ )
+
+ with StataReader(path) as reader:
+ assert reader.time_stamp == "29 Feb 2000 14:21"
+ assert reader.data_label == data_label
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ def test_invalid_timestamp(self, version, temp_file):
+ original = DataFrame([(1,)], columns=["variable"])
+ time_stamp = "01 Jan 2000, 00:00:00"
+ path = temp_file
+ msg = "time_stamp should be datetime type"
+ with pytest.raises(ValueError, match=msg):
+ original.to_stata(path, time_stamp=time_stamp, version=version)
+ assert not os.path.isfile(path)
+
+ def test_numeric_column_names(self, temp_file):
+ original = DataFrame(np.reshape(np.arange(25.0), (5, 5)))
+ original.index.name = "index"
+ path = temp_file
+ # should get a warning for that format.
+ msg = "Not all pandas column names were valid Stata variable names"
+ with tm.assert_produces_warning(InvalidColumnName, match=msg):
+ original.to_stata(path)
+
+ written_and_read_again = self.read_dta(path)
+
+ written_and_read_again = written_and_read_again.set_index("index")
+ columns = list(written_and_read_again.columns)
+ convert_col_name = lambda x: int(x[1])
+ written_and_read_again.columns = map(convert_col_name, columns)
+
+ expected = original
+ tm.assert_frame_equal(expected, written_and_read_again)
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ def test_nan_to_missing_value(self, version, temp_file):
+ s1 = Series(np.arange(4.0), dtype=np.float32)
+ s2 = Series(np.arange(4.0), dtype=np.float64)
+ s1[::2] = np.nan
+ s2[1::2] = np.nan
+ original = DataFrame({"s1": s1, "s2": s2})
+ original.index.name = "index"
+
+ path = temp_file
+ original.to_stata(path, version=version)
+ written_and_read_again = self.read_dta(path)
+
+ written_and_read_again = written_and_read_again.set_index("index")
+ expected = original
+ tm.assert_frame_equal(written_and_read_again, expected)
+
+ def test_no_index(self, temp_file):
+ columns = ["x", "y"]
+ original = DataFrame(np.reshape(np.arange(10.0), (5, 2)), columns=columns)
+ original.index.name = "index_not_written"
+ path = temp_file
+ original.to_stata(path, write_index=False)
+ written_and_read_again = self.read_dta(path)
+ with pytest.raises(KeyError, match=original.index.name):
+ written_and_read_again["index_not_written"]
+
+ def test_string_no_dates(self, temp_file):
+ s1 = Series(["a", "A longer string"])
+ s2 = Series([1.0, 2.0], dtype=np.float64)
+ original = DataFrame({"s1": s1, "s2": s2})
+ original.index.name = "index"
+ path = temp_file
+ original.to_stata(path)
+ written_and_read_again = self.read_dta(path)
+
+ expected = original
+ tm.assert_frame_equal(written_and_read_again.set_index("index"), expected)
+
+ def test_large_value_conversion(self, temp_file):
+ s0 = Series([1, 99], dtype=np.int8)
+ s1 = Series([1, 127], dtype=np.int8)
+ s2 = Series([1, 2**15 - 1], dtype=np.int16)
+ s3 = Series([1, 2**63 - 1], dtype=np.int64)
+ original = DataFrame({"s0": s0, "s1": s1, "s2": s2, "s3": s3})
+ original.index.name = "index"
+ path = temp_file
+ with tm.assert_produces_warning(PossiblePrecisionLoss, match="from int64 to"):
+ original.to_stata(path)
+
+ written_and_read_again = self.read_dta(path)
+
+ modified = original
+ modified["s1"] = Series(modified["s1"], dtype=np.int16)
+ modified["s2"] = Series(modified["s2"], dtype=np.int32)
+ modified["s3"] = Series(modified["s3"], dtype=np.float64)
+ tm.assert_frame_equal(written_and_read_again.set_index("index"), modified)
+
+ def test_dates_invalid_column(self, temp_file):
+ original = DataFrame([datetime(2006, 11, 19, 23, 13, 20)])
+ original.index.name = "index"
+ path = temp_file
+ msg = "Not all pandas column names were valid Stata variable names"
+ with tm.assert_produces_warning(InvalidColumnName, match=msg):
+ original.to_stata(path, convert_dates={0: "tc"})
+
+ written_and_read_again = self.read_dta(path)
+
+ expected = original.copy()
+ expected.columns = ["_0"]
+ expected.index = original.index.astype(np.int32)
+ expected["_0"] = expected["_0"].astype("M8[ms]")
+ tm.assert_frame_equal(written_and_read_again.set_index("index"), expected)
+
+ def test_105(self, datapath):
+ # Data obtained from:
+ # http://go.worldbank.org/ZXY29PVJ21
+ dpath = datapath("io", "data", "stata", "S4_EDUC1.dta")
+ df = read_stata(dpath)
+ df0 = [[1, 1, 3, -2], [2, 1, 2, -2], [4, 1, 1, -2]]
+ df0 = DataFrame(df0)
+ df0.columns = ["clustnum", "pri_schl", "psch_num", "psch_dis"]
+ df0["clustnum"] = df0["clustnum"].astype(np.int16)
+ df0["pri_schl"] = df0["pri_schl"].astype(np.int8)
+ df0["psch_num"] = df0["psch_num"].astype(np.int8)
+ df0["psch_dis"] = df0["psch_dis"].astype(np.float32)
+ tm.assert_frame_equal(df.head(3), df0)
+
+ def test_value_labels_old_format(self, datapath):
+ # GH 19417
+ #
+ # Test that value_labels() returns an empty dict if the file format
+ # predates supporting value labels.
+ dpath = datapath("io", "data", "stata", "S4_EDUC1.dta")
+ with StataReader(dpath) as reader:
+ assert reader.value_labels() == {}
+
+ def test_date_export_formats(self, temp_file):
+ columns = ["tc", "td", "tw", "tm", "tq", "th", "ty"]
+ conversions = {c: c for c in columns}
+ data = [datetime(2006, 11, 20, 23, 13, 20)] * len(columns)
+ original = DataFrame([data], columns=columns)
+ original.index.name = "index"
+ expected_values = [
+ datetime(2006, 11, 20, 23, 13, 20), # Time
+ datetime(2006, 11, 20), # Day
+ datetime(2006, 11, 19), # Week
+ datetime(2006, 11, 1), # Month
+ datetime(2006, 10, 1), # Quarter year
+ datetime(2006, 7, 1), # Half year
+ datetime(2006, 1, 1),
+ ] # Year
+
+ expected = DataFrame(
+ [expected_values],
+ index=pd.Index([0], dtype=np.int32, name="index"),
+ columns=columns,
+ dtype="M8[s]",
+ )
+ expected["tc"] = expected["tc"].astype("M8[ms]")
+
+ path = temp_file
+ original.to_stata(path, convert_dates=conversions)
+ written_and_read_again = self.read_dta(path)
+
+ tm.assert_frame_equal(written_and_read_again.set_index("index"), expected)
+
+ def test_write_missing_strings(self, temp_file):
+ original = DataFrame([["1"], [None]], columns=["foo"])
+
+ expected = DataFrame(
+ [["1"], [""]],
+ index=pd.RangeIndex(2, name="index"),
+ columns=["foo"],
+ )
+
+ path = temp_file
+ original.to_stata(path)
+ written_and_read_again = self.read_dta(path)
+
+ tm.assert_frame_equal(written_and_read_again.set_index("index"), expected)
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ @pytest.mark.parametrize("byteorder", [">", "<"])
+ def test_bool_uint(self, byteorder, version, temp_file):
+ s0 = Series([0, 1, True], dtype=np.bool_)
+ s1 = Series([0, 1, 100], dtype=np.uint8)
+ s2 = Series([0, 1, 255], dtype=np.uint8)
+ s3 = Series([0, 1, 2**15 - 100], dtype=np.uint16)
+ s4 = Series([0, 1, 2**16 - 1], dtype=np.uint16)
+ s5 = Series([0, 1, 2**31 - 100], dtype=np.uint32)
+ s6 = Series([0, 1, 2**32 - 1], dtype=np.uint32)
+
+ original = DataFrame(
+ {"s0": s0, "s1": s1, "s2": s2, "s3": s3, "s4": s4, "s5": s5, "s6": s6}
+ )
+ original.index.name = "index"
+
+ path = temp_file
+ original.to_stata(path, byteorder=byteorder, version=version)
+ written_and_read_again = self.read_dta(path)
+
+ written_and_read_again = written_and_read_again.set_index("index")
+
+ expected = original
+ expected_types = (
+ np.int8,
+ np.int8,
+ np.int16,
+ np.int16,
+ np.int32,
+ np.int32,
+ np.float64,
+ )
+ for c, t in zip(expected.columns, expected_types):
+ expected[c] = expected[c].astype(t)
+
+ tm.assert_frame_equal(written_and_read_again, expected)
+
+ def test_variable_labels(self, datapath):
+ with StataReader(datapath("io", "data", "stata", "stata7_115.dta")) as rdr:
+ sr_115 = rdr.variable_labels()
+ with StataReader(datapath("io", "data", "stata", "stata7_117.dta")) as rdr:
+ sr_117 = rdr.variable_labels()
+ keys = ("var1", "var2", "var3")
+ labels = ("label1", "label2", "label3")
+ for k, v in sr_115.items():
+ assert k in sr_117
+ assert v == sr_117[k]
+ assert k in keys
+ assert v in labels
+
+ def test_minimal_size_col(self, temp_file):
+ str_lens = (1, 100, 244)
+ s = {}
+ for str_len in str_lens:
+ s["s" + str(str_len)] = Series(
+ ["a" * str_len, "b" * str_len, "c" * str_len]
+ )
+ original = DataFrame(s)
+ path = temp_file
+ original.to_stata(path, write_index=False)
+
+ with StataReader(path) as sr:
+ sr._ensure_open() # The `_*list` variables are initialized here
+ for variable, fmt, typ in zip(sr._varlist, sr._fmtlist, sr._typlist):
+ assert int(variable[1:]) == int(fmt[1:-1])
+ assert int(variable[1:]) == typ
+
+ def test_excessively_long_string(self, temp_file):
+ str_lens = (1, 244, 500)
+ s = {}
+ for str_len in str_lens:
+ s["s" + str(str_len)] = Series(
+ ["a" * str_len, "b" * str_len, "c" * str_len]
+ )
+ original = DataFrame(s)
+ msg = (
+ r"Fixed width strings in Stata \.dta files are limited to 244 "
+ r"\(or fewer\)\ncharacters\. Column 's500' does not satisfy "
+ r"this restriction\. Use the\n'version=117' parameter to write "
+ r"the newer \(Stata 13 and later\) format\."
+ )
+ with pytest.raises(ValueError, match=msg):
+ path = temp_file
+ original.to_stata(path)
+
+ def test_missing_value_generator(self, temp_file):
+ types = ("b", "h", "l")
+ df = DataFrame([[0.0]], columns=["float_"])
+ path = temp_file
+ df.to_stata(path)
+ with StataReader(path) as rdr:
+ valid_range = rdr.VALID_RANGE
+ expected_values = ["." + chr(97 + i) for i in range(26)]
+ expected_values.insert(0, ".")
+ for t in types:
+ offset = valid_range[t][1]
+ for i in range(27):
+ val = StataMissingValue(offset + 1 + i)
+ assert val.string == expected_values[i]
+
+ # Test extremes for floats
+ val = StataMissingValue(struct.unpack(" DataFrame:
+ """
+ Emulate the categorical casting behavior we expect from roundtripping.
+ """
+ for col in from_frame:
+ ser = from_frame[col]
+ if isinstance(ser.dtype, CategoricalDtype):
+ cat = ser._values.remove_unused_categories()
+ if cat.categories.dtype == object:
+ categories = pd.Index._with_infer(
+ cat.categories._values, copy=False
+ )
+ cat = cat.set_categories(categories)
+ elif cat.categories.dtype == "string" and len(cat.categories) == 0:
+ # if the read categories are empty, it comes back as object dtype
+ categories = cat.categories.astype(object)
+ cat = cat.set_categories(categories)
+ from_frame[col] = cat
+ return from_frame
+
+ def test_iterator(self, datapath):
+ fname = datapath("io", "data", "stata", "stata12_117.dta")
+
+ parsed = read_stata(fname)
+ expected = parsed.iloc[0:5, :]
+
+ with read_stata(fname, iterator=True) as itr:
+ chunk = itr.read(5)
+ tm.assert_frame_equal(expected, chunk)
+
+ with read_stata(fname, chunksize=5) as itr:
+ chunk = next(itr)
+ tm.assert_frame_equal(expected, chunk)
+
+ with read_stata(fname, iterator=True) as itr:
+ chunk = itr.get_chunk(5)
+ tm.assert_frame_equal(expected, chunk)
+
+ with read_stata(fname, chunksize=5) as itr:
+ chunk = itr.get_chunk()
+ tm.assert_frame_equal(expected, chunk)
+
+ # GH12153
+ with read_stata(fname, chunksize=4) as itr:
+ from_chunks = pd.concat(itr)
+ tm.assert_frame_equal(parsed, from_chunks)
+
+ @pytest.mark.filterwarnings("ignore::UserWarning")
+ @pytest.mark.parametrize(
+ "file",
+ [
+ "stata2_115",
+ "stata3_115",
+ "stata4_115",
+ "stata5_115",
+ "stata6_115",
+ "stata7_115",
+ "stata8_115",
+ "stata9_115",
+ "stata10_115",
+ "stata11_115",
+ ],
+ )
+ @pytest.mark.parametrize("chunksize", [1, 2])
+ @pytest.mark.parametrize("convert_categoricals", [False, True])
+ @pytest.mark.parametrize("convert_dates", [False, True])
+ def test_read_chunks_115(
+ self, file, chunksize, convert_categoricals, convert_dates, datapath
+ ):
+ fname = datapath("io", "data", "stata", f"{file}.dta")
+
+ # Read the whole file
+ parsed = read_stata(
+ fname,
+ convert_categoricals=convert_categoricals,
+ convert_dates=convert_dates,
+ )
+
+ # Compare to what we get when reading by chunk
+ with read_stata(
+ fname,
+ iterator=True,
+ convert_dates=convert_dates,
+ convert_categoricals=convert_categoricals,
+ ) as itr:
+ pos = 0
+ for j in range(5):
+ try:
+ chunk = itr.read(chunksize)
+ except StopIteration:
+ break
+ from_frame = parsed.iloc[pos : pos + chunksize, :].copy()
+ from_frame = self._convert_categorical(from_frame)
+ tm.assert_frame_equal(
+ from_frame,
+ chunk,
+ check_dtype=False,
+ )
+ pos += chunksize
+
+ def test_read_chunks_columns(self, datapath):
+ fname = datapath("io", "data", "stata", "stata3_117.dta")
+ columns = ["quarter", "cpi", "m1"]
+ chunksize = 2
+
+ parsed = read_stata(fname, columns=columns)
+ with read_stata(fname, iterator=True) as itr:
+ pos = 0
+ for j in range(5):
+ chunk = itr.read(chunksize, columns=columns)
+ if chunk is None:
+ break
+ from_frame = parsed.iloc[pos : pos + chunksize, :]
+ tm.assert_frame_equal(from_frame, chunk, check_dtype=False)
+ pos += chunksize
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ def test_write_variable_labels(self, version, mixed_frame, temp_file):
+ # GH 13631, add support for writing variable labels
+ mixed_frame.index.name = "index"
+ variable_labels = {"a": "City Rank", "b": "City Exponent", "c": "City"}
+ path = temp_file
+ mixed_frame.to_stata(path, variable_labels=variable_labels, version=version)
+ with StataReader(path) as sr:
+ read_labels = sr.variable_labels()
+ expected_labels = {
+ "index": "",
+ "a": "City Rank",
+ "b": "City Exponent",
+ "c": "City",
+ }
+ assert read_labels == expected_labels
+
+ variable_labels["index"] = "The Index"
+ path = temp_file
+ mixed_frame.to_stata(path, variable_labels=variable_labels, version=version)
+ with StataReader(path) as sr:
+ read_labels = sr.variable_labels()
+ assert read_labels == variable_labels
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ def test_invalid_variable_labels(self, version, mixed_frame, temp_file):
+ mixed_frame.index.name = "index"
+ variable_labels = {"a": "very long" * 10, "b": "City Exponent", "c": "City"}
+ path = temp_file
+ msg = "Variable labels must be 80 characters or fewer"
+ with pytest.raises(ValueError, match=msg):
+ mixed_frame.to_stata(path, variable_labels=variable_labels, version=version)
+
+ @pytest.mark.parametrize("version", [114, 117])
+ def test_invalid_variable_label_encoding(self, version, mixed_frame, temp_file):
+ mixed_frame.index.name = "index"
+ variable_labels = {"a": "very long" * 10, "b": "City Exponent", "c": "City"}
+ variable_labels["a"] = "invalid character Œ"
+ path = temp_file
+ with pytest.raises(
+ ValueError, match="Variable labels must contain only characters"
+ ):
+ mixed_frame.to_stata(path, variable_labels=variable_labels, version=version)
+
+ def test_write_variable_label_errors(self, mixed_frame, temp_file):
+ values = ["\u03a1", "\u0391", "\u039d", "\u0394", "\u0391", "\u03a3"]
+
+ variable_labels_utf8 = {
+ "a": "City Rank",
+ "b": "City Exponent",
+ "c": "".join(values),
+ }
+
+ msg = (
+ "Variable labels must contain only characters that can be "
+ "encoded in Latin-1"
+ )
+ with pytest.raises(ValueError, match=msg):
+ path = temp_file
+ mixed_frame.to_stata(path, variable_labels=variable_labels_utf8)
+
+ variable_labels_long = {
+ "a": "City Rank",
+ "b": "City Exponent",
+ "c": "A very, very, very long variable label "
+ "that is too long for Stata which means "
+ "that it has more than 80 characters",
+ }
+
+ msg = "Variable labels must be 80 characters or fewer"
+ with pytest.raises(ValueError, match=msg):
+ path = temp_file
+ mixed_frame.to_stata(path, variable_labels=variable_labels_long)
+
+ def test_default_date_conversion(self, temp_file):
+ # GH 12259
+ dates = [
+ dt.datetime(1999, 12, 31, 12, 12, 12, 12000),
+ dt.datetime(2012, 12, 21, 12, 21, 12, 21000),
+ dt.datetime(1776, 7, 4, 7, 4, 7, 4000),
+ ]
+ original = DataFrame(
+ {
+ "nums": [1.0, 2.0, 3.0],
+ "strs": ["apple", "banana", "cherry"],
+ "dates": dates,
+ }
+ )
+
+ expected = original[:]
+ # "tc" for convert_dates below stores with "ms" resolution
+ expected["dates"] = expected["dates"].astype("M8[ms]")
+
+ path = temp_file
+ original.to_stata(path, write_index=False)
+ reread = read_stata(path, convert_dates=True)
+ tm.assert_frame_equal(expected, reread)
+
+ original.to_stata(path, write_index=False, convert_dates={"dates": "tc"})
+ direct = read_stata(path, convert_dates=True)
+ tm.assert_frame_equal(reread, direct)
+
+ dates_idx = original.columns.tolist().index("dates")
+ original.to_stata(path, write_index=False, convert_dates={dates_idx: "tc"})
+ direct = read_stata(path, convert_dates=True)
+ tm.assert_frame_equal(reread, direct)
+
+ def test_unsupported_type(self, temp_file):
+ original = DataFrame({"a": [1 + 2j, 2 + 4j]})
+
+ msg = "Data type complex128 not supported"
+ with pytest.raises(NotImplementedError, match=msg):
+ path = temp_file
+ original.to_stata(path)
+
+ def test_unsupported_datetype(self, temp_file):
+ dates = [
+ dt.datetime(1999, 12, 31, 12, 12, 12, 12000),
+ dt.datetime(2012, 12, 21, 12, 21, 12, 21000),
+ dt.datetime(1776, 7, 4, 7, 4, 7, 4000),
+ ]
+ original = DataFrame(
+ {
+ "nums": [1.0, 2.0, 3.0],
+ "strs": ["apple", "banana", "cherry"],
+ "dates": dates,
+ }
+ )
+
+ msg = "Format %tC not implemented"
+ with pytest.raises(NotImplementedError, match=msg):
+ path = temp_file
+ original.to_stata(path, convert_dates={"dates": "tC"})
+
+ dates = pd.date_range("1-1-1990", periods=3, tz="Asia/Hong_Kong")
+ original = DataFrame(
+ {
+ "nums": [1.0, 2.0, 3.0],
+ "strs": ["apple", "banana", "cherry"],
+ "dates": dates,
+ }
+ )
+ with pytest.raises(NotImplementedError, match="Data type datetime64"):
+ path = temp_file
+ original.to_stata(path)
+
+ def test_repeated_column_labels(self, datapath):
+ # GH 13923, 25772
+ msg = """
+Value labels for column ethnicsn are not unique. These cannot be converted to
+pandas categoricals.
+
+Either read the file with `convert_categoricals` set to False or use the
+low level interface in `StataReader` to separately read the values and the
+value_labels.
+
+The repeated labels are:\n-+\nwolof
+"""
+ with pytest.raises(ValueError, match=msg):
+ read_stata(
+ datapath("io", "data", "stata", "stata15.dta"),
+ convert_categoricals=True,
+ )
+
+ def test_stata_111(self, datapath):
+ # 111 is an old version but still used by current versions of
+ # SAS when exporting to Stata format. We do not know of any
+ # on-line documentation for this version.
+ df = read_stata(datapath("io", "data", "stata", "stata7_111.dta"))
+ original = DataFrame(
+ {
+ "y": [1, 1, 1, 1, 1, 0, 0, np.nan, 0, 0],
+ "x": [1, 2, 1, 3, np.nan, 4, 3, 5, 1, 6],
+ "w": [2, np.nan, 5, 2, 4, 4, 3, 1, 2, 3],
+ "z": ["a", "b", "c", "d", "e", "", "g", "h", "i", "j"],
+ }
+ )
+ original = original[["y", "x", "w", "z"]]
+ tm.assert_frame_equal(original, df)
+
+ def test_out_of_range_double(self, temp_file):
+ # GH 14618
+ df = DataFrame(
+ {
+ "ColumnOk": [0.0, np.finfo(np.double).eps, 4.49423283715579e307],
+ "ColumnTooBig": [0.0, np.finfo(np.double).eps, np.finfo(np.double).max],
+ }
+ )
+ msg = (
+ r"Column ColumnTooBig has a maximum value \(.+\) outside the range "
+ r"supported by Stata \(.+\)"
+ )
+ with pytest.raises(ValueError, match=msg):
+ path = temp_file
+ df.to_stata(path)
+
+ def test_out_of_range_float(self, temp_file):
+ original = DataFrame(
+ {
+ "ColumnOk": [
+ 0.0,
+ np.finfo(np.float32).eps,
+ np.finfo(np.float32).max / 10.0,
+ ],
+ "ColumnTooBig": [
+ 0.0,
+ np.finfo(np.float32).eps,
+ np.finfo(np.float32).max,
+ ],
+ }
+ )
+ original.index.name = "index"
+ for col in original:
+ original[col] = original[col].astype(np.float32)
+
+ path = temp_file
+ original.to_stata(path)
+ reread = read_stata(path)
+
+ original["ColumnTooBig"] = original["ColumnTooBig"].astype(np.float64)
+ expected = original
+ tm.assert_frame_equal(reread.set_index("index"), expected)
+
+ @pytest.mark.parametrize("infval", [np.inf, -np.inf])
+ def test_inf(self, infval, temp_file):
+ # GH 45350
+ df = DataFrame({"WithoutInf": [0.0, 1.0], "WithInf": [2.0, infval]})
+ msg = (
+ "Column WithInf contains infinity or -infinity"
+ "which is outside the range supported by Stata."
+ )
+ with pytest.raises(ValueError, match=msg):
+ path = temp_file
+ df.to_stata(path)
+
+ def test_path_pathlib(self, temp_file):
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ )
+ df.index.name = "index"
+ reader = lambda x: read_stata(x).set_index("index")
+ result = tm.round_trip_pathlib(df.to_stata, reader, temp_file)
+ tm.assert_frame_equal(df, result)
+
+ @pytest.mark.parametrize("write_index", [True, False])
+ def test_value_labels_iterator(self, write_index, temp_file):
+ # GH 16923
+ d = {"A": ["B", "E", "C", "A", "E"]}
+ df = DataFrame(data=d)
+ df["A"] = df["A"].astype("category")
+ path = temp_file
+ df.to_stata(path, write_index=write_index)
+
+ with read_stata(path, iterator=True) as dta_iter:
+ value_labels = dta_iter.value_labels()
+ assert value_labels == {"A": {0: "A", 1: "B", 2: "C", 3: "E"}}
+
+ def test_set_index(self, temp_file):
+ # GH 17328
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ )
+ df.index.name = "index"
+ path = temp_file
+ df.to_stata(path)
+ reread = read_stata(path, index_col="index")
+ tm.assert_frame_equal(df, reread)
+
+ @pytest.mark.parametrize(
+ "column", ["ms", "day", "week", "month", "qtr", "half", "yr"]
+ )
+ def test_date_parsing_ignores_format_details(self, column, datapath):
+ # GH 17797
+ #
+ # Test that display formats are ignored when determining if a numeric
+ # column is a date value.
+ #
+ # All date types are stored as numbers and format associated with the
+ # column denotes both the type of the date and the display format.
+ #
+ # STATA supports 9 date types which each have distinct units. We test 7
+ # of the 9 types, ignoring %tC and %tb. %tC is a variant of %tc that
+ # accounts for leap seconds and %tb relies on STATAs business calendar.
+ df = read_stata(datapath("io", "data", "stata", "stata13_dates.dta"))
+ unformatted = df.loc[0, column]
+ formatted = df.loc[0, column + "_fmt"]
+ assert unformatted == formatted
+
+ @pytest.mark.parametrize("byteorder", ["little", "big"])
+ def test_writer_117(self, byteorder, temp_file, using_infer_string):
+ original = DataFrame(
+ data=[
+ [
+ "string",
+ "object",
+ 1,
+ 1,
+ 1,
+ 1.1,
+ 1.1,
+ np.datetime64("2003-12-25"),
+ "a",
+ "a" * 2045,
+ "a" * 5000,
+ "a",
+ ],
+ [
+ "string-1",
+ "object-1",
+ 1,
+ 1,
+ 1,
+ 1.1,
+ 1.1,
+ np.datetime64("2003-12-26"),
+ "b",
+ "b" * 2045,
+ "",
+ "",
+ ],
+ ],
+ columns=[
+ "string",
+ "object",
+ "int8",
+ "int16",
+ "int32",
+ "float32",
+ "float64",
+ "datetime",
+ "s1",
+ "s2045",
+ "srtl",
+ "forced_strl",
+ ],
+ )
+ original["object"] = Series(original["object"], dtype=object)
+ original["int8"] = Series(original["int8"], dtype=np.int8)
+ original["int16"] = Series(original["int16"], dtype=np.int16)
+ original["int32"] = original["int32"].astype(np.int32)
+ original["float32"] = Series(original["float32"], dtype=np.float32)
+ original.index.name = "index"
+ copy = original.copy()
+ path = temp_file
+ original.to_stata(
+ path,
+ convert_dates={"datetime": "tc"},
+ byteorder=byteorder,
+ convert_strl=["forced_strl"],
+ version=117,
+ )
+ written_and_read_again = self.read_dta(path)
+
+ expected = original[:]
+ # "tc" for convert_dates means we store with "ms" resolution
+ expected["datetime"] = expected["datetime"].astype("M8[ms]")
+ if using_infer_string:
+ # object dtype (with only strings/None) comes back as string dtype
+ expected["object"] = expected["object"].astype("str")
+
+ tm.assert_frame_equal(
+ written_and_read_again.set_index("index"),
+ expected,
+ )
+ tm.assert_frame_equal(original, copy)
+
+ def test_convert_strl_name_swap(self, temp_file):
+ original = DataFrame(
+ [["a" * 3000, "A", "apple"], ["b" * 1000, "B", "banana"]],
+ columns=["long1" * 10, "long", 1],
+ )
+ original.index.name = "index"
+
+ msg = "Not all pandas column names were valid Stata variable names"
+ with tm.assert_produces_warning(InvalidColumnName, match=msg):
+ path = temp_file
+ original.to_stata(path, convert_strl=["long", 1], version=117)
+ reread = self.read_dta(path)
+ reread = reread.set_index("index")
+ reread.columns = original.columns
+ tm.assert_frame_equal(reread, original, check_index_type=False)
+
+ def test_invalid_date_conversion(self, temp_file):
+ # GH 12259
+ dates = [
+ dt.datetime(1999, 12, 31, 12, 12, 12, 12000),
+ dt.datetime(2012, 12, 21, 12, 21, 12, 21000),
+ dt.datetime(1776, 7, 4, 7, 4, 7, 4000),
+ ]
+ original = DataFrame(
+ {
+ "nums": [1.0, 2.0, 3.0],
+ "strs": ["apple", "banana", "cherry"],
+ "dates": dates,
+ }
+ )
+
+ path = temp_file
+ msg = "convert_dates key must be a column or an integer"
+ with pytest.raises(ValueError, match=msg):
+ original.to_stata(path, convert_dates={"wrong_name": "tc"})
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ def test_nonfile_writing(self, version, temp_file):
+ # GH 21041
+ bio = io.BytesIO()
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ )
+ df.index.name = "index"
+ path = temp_file
+ df.to_stata(bio, version=version)
+ bio.seek(0)
+ with open(path, "wb") as dta:
+ dta.write(bio.read())
+ reread = read_stata(path, index_col="index")
+ tm.assert_frame_equal(df, reread)
+
+ def test_gzip_writing(self, temp_file):
+ # writing version 117 requires seek and cannot be used with gzip
+ df = DataFrame(
+ 1.1 * np.arange(120).reshape((30, 4)),
+ columns=pd.Index(list("ABCD")),
+ index=pd.Index([f"i-{i}" for i in range(30)]),
+ )
+ df.index.name = "index"
+ path = temp_file
+ with gzip.GzipFile(path, "wb") as gz:
+ df.to_stata(gz, version=114)
+ with gzip.GzipFile(path, "rb") as gz:
+ reread = read_stata(gz, index_col="index")
+ tm.assert_frame_equal(df, reread)
+
+ # 117 is not included in this list as it uses ASCII strings
+ @pytest.mark.parametrize(
+ "file",
+ [
+ "stata16_118",
+ "stata16_be_118",
+ "stata16_119",
+ "stata16_be_119",
+ ],
+ )
+ def test_unicode_dta_118_119(self, file, datapath):
+ unicode_df = self.read_dta(datapath("io", "data", "stata", f"{file}.dta"))
+
+ columns = ["utf8", "latin1", "ascii", "utf8_strl", "ascii_strl"]
+ values = [
+ ["ραηδας", "PÄNDÄS", "p", "ραηδας", "p"],
+ ["ƤĀńĐąŜ", "Ö", "a", "ƤĀńĐąŜ", "a"],
+ ["ᴘᴀᴎᴅᴀS", "Ü", "n", "ᴘᴀᴎᴅᴀS", "n"],
+ [" ", " ", "d", " ", "d"],
+ [" ", "", "a", " ", "a"],
+ ["", "", "s", "", "s"],
+ ["", "", " ", "", " "],
+ ]
+ expected = DataFrame(values, columns=columns)
+
+ tm.assert_frame_equal(unicode_df, expected)
+
+ def test_mixed_string_strl(self, temp_file, using_infer_string):
+ # GH 23633
+ output = [{"mixed": "string" * 500, "number": 0}, {"mixed": None, "number": 1}]
+ output = DataFrame(output)
+ output.number = output.number.astype("int32")
+
+ path = temp_file
+ output.to_stata(path, write_index=False, version=117)
+ reread = read_stata(path)
+ expected = output.fillna("")
+ tm.assert_frame_equal(reread, expected)
+
+ # Check strl supports all None (null)
+ output["mixed"] = None
+ output.to_stata(path, write_index=False, convert_strl=["mixed"], version=117)
+ reread = read_stata(path)
+ expected = output.fillna("")
+ if using_infer_string:
+ expected["mixed"] = expected["mixed"].astype("str")
+ tm.assert_frame_equal(reread, expected)
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ def test_all_none_exception(self, version, temp_file):
+ output = [{"none": "none", "number": 0}, {"none": None, "number": 1}]
+ output = DataFrame(output)
+ output["none"] = None
+ with pytest.raises(ValueError, match="Column `none` cannot be exported"):
+ output.to_stata(temp_file, version=version)
+
+ @pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+ def test_invalid_file_not_written(self, version, temp_file):
+ content = "Here is one __�__ Another one __·__ Another one __½__"
+ df = DataFrame([content], columns=["invalid"])
+ msg1 = (
+ r"'latin-1' codec can't encode character '\\ufffd' "
+ r"in position 14: ordinal not in range\(256\)"
+ )
+ msg2 = (
+ "'ascii' codec can't decode byte 0xef in position 14: "
+ r"ordinal not in range\(128\)"
+ )
+ with pytest.raises(UnicodeEncodeError, match=f"{msg1}|{msg2}"):
+ df.to_stata(temp_file)
+
+ def test_strl_latin1(self, temp_file):
+ # GH 23573, correct GSO data to reflect correct size
+ output = DataFrame(
+ [["pandas"] * 2, ["þâÑÐŧ"] * 2], columns=["var_str", "var_strl"]
+ )
+
+ output.to_stata(temp_file, version=117, convert_strl=["var_strl"])
+ with open(temp_file, "rb") as reread:
+ content = reread.read()
+ expected = "þâÑÐŧ"
+ assert expected.encode("latin-1") in content
+ assert expected.encode("utf-8") in content
+ gsos = content.split(b"strls")[1][1:-2]
+ for gso in gsos.split(b"GSO")[1:]:
+ val = gso.split(b"\x00")[-2]
+ size = gso[gso.find(b"\x82") + 1]
+ assert len(val) == size - 1
+
+ def test_encoding_latin1_118(self, datapath):
+ # GH 25960
+ msg = """
+One or more strings in the dta file could not be decoded using utf-8, and
+so the fallback encoding of latin-1 is being used. This can happen when a file
+has been incorrectly encoded by Stata or some other software. You should verify
+the string values returned are correct."""
+ # Move path outside of read_stata, or else assert_produces_warning
+ # will block pytests skip mechanism from triggering (failing the test)
+ # if the path is not present
+ path = datapath("io", "data", "stata", "stata1_encoding_118.dta")
+ with tm.assert_produces_warning(UnicodeWarning, filter_level="once") as w:
+ encoded = read_stata(path)
+ # with filter_level="always", produces 151 warnings which can be slow
+ assert len(w) == 1
+ assert w[0].message.args[0] == msg
+
+ expected = DataFrame([["Düsseldorf"]] * 151, columns=["kreis1849"])
+ tm.assert_frame_equal(encoded, expected)
+
+ @pytest.mark.slow
+ def test_stata_119(self, datapath):
+ # Gzipped since contains 32,999 variables and uncompressed is 20MiB
+ # Just validate that the reader reports correct number of variables
+ # to avoid high peak memory
+ with gzip.open(
+ datapath("io", "data", "stata", "stata1_119.dta.gz"), "rb"
+ ) as gz:
+ with StataReader(gz) as reader:
+ reader._ensure_open()
+ assert reader._nvar == 32999
+
+ @pytest.mark.parametrize("version", [118, 119, None])
+ @pytest.mark.parametrize("byteorder", ["little", "big"])
+ def test_utf8_writer(self, version, byteorder, temp_file):
+ cat = pd.Categorical(["a", "β", "ĉ"], ordered=True)
+ data = DataFrame(
+ [
+ [1.0, 1, "ᴬ", "ᴀ relatively long ŝtring"],
+ [2.0, 2, "ᴮ", ""],
+ [3.0, 3, "ᴰ", None],
+ ],
+ columns=["Å", "β", "ĉ", "strls"],
+ )
+ data["ᴐᴬᵀ"] = cat
+ variable_labels = {
+ "Å": "apple",
+ "β": "ᵈᵉᵊ",
+ "ĉ": "ᴎტჄႲႳႴႶႺ",
+ "strls": "Long Strings",
+ "ᴐᴬᵀ": "",
+ }
+ data_label = "ᴅaᵀa-label"
+ value_labels = {"β": {1: "label", 2: "æøå", 3: "ŋot valid latin-1"}}
+ data["β"] = data["β"].astype(np.int32)
+ writer = StataWriterUTF8(
+ temp_file,
+ data,
+ data_label=data_label,
+ convert_strl=["strls"],
+ variable_labels=variable_labels,
+ write_index=False,
+ byteorder=byteorder,
+ version=version,
+ value_labels=value_labels,
+ )
+ writer.write_file()
+ reread_encoded = read_stata(temp_file)
+ # Missing is intentionally converted to empty strl
+ data["strls"] = data["strls"].fillna("")
+ # Variable with value labels is reread as categorical
+ data["β"] = (
+ data["β"].replace(value_labels["β"]).astype("category").cat.as_ordered()
+ )
+ tm.assert_frame_equal(data, reread_encoded)
+ with StataReader(temp_file) as reader:
+ assert reader.data_label == data_label
+ assert reader.variable_labels() == variable_labels
+
+ data.to_stata(temp_file, version=version, write_index=False)
+ reread_to_stata = read_stata(temp_file)
+ tm.assert_frame_equal(data, reread_to_stata)
+
+ def test_writer_118_exceptions(self, temp_file):
+ df = DataFrame(np.zeros((1, 33000), dtype=np.int8))
+ with pytest.raises(ValueError, match="version must be either 118 or 119."):
+ StataWriterUTF8(temp_file, df, version=117)
+ with pytest.raises(ValueError, match="You must use version 119"):
+ StataWriterUTF8(temp_file, df, version=118)
+
+ @pytest.mark.parametrize(
+ "dtype_backend",
+ ["numpy_nullable", pytest.param("pyarrow", marks=td.skip_if_no("pyarrow"))],
+ )
+ def test_read_write_ea_dtypes(self, dtype_backend, temp_file, tmp_path):
+ dtype = "Int64" if dtype_backend == "numpy_nullable" else "int64[pyarrow]"
+ df = DataFrame(
+ {
+ "a": pd.array([1, 2, None], dtype=dtype),
+ "b": ["a", "b", "c"],
+ "c": [True, False, None],
+ "d": [1.5, 2.5, 3.5],
+ "e": pd.date_range("2020-12-31", periods=3, freq="D"),
+ },
+ index=pd.Index([0, 1, 2], name="index"),
+ )
+ df = df.convert_dtypes(dtype_backend=dtype_backend)
+ stata_path = tmp_path / "test_stata.dta"
+ df.to_stata(stata_path, version=118)
+
+ df.to_stata(temp_file)
+ written_and_read_again = self.read_dta(temp_file)
+
+ expected = DataFrame(
+ {
+ "a": [1, 2, np.nan],
+ "b": ["a", "b", "c"],
+ "c": [1.0, 0, np.nan],
+ "d": [1.5, 2.5, 3.5],
+ # stata stores with ms unit, so unit does not round-trip exactly
+ "e": pd.date_range("2020-12-31", periods=3, freq="D", unit="ms"),
+ },
+ index=pd.RangeIndex(range(3), name="index"),
+ )
+
+ tm.assert_frame_equal(written_and_read_again.set_index("index"), expected)
+
+ @pytest.mark.parametrize("version", [113, 114, 115, 117, 118, 119])
+ def test_read_data_int_validranges(self, version, datapath):
+ expected = DataFrame(
+ {
+ "byte": np.array([-127, 100], dtype=np.int8),
+ "int": np.array([-32767, 32740], dtype=np.int16),
+ "long": np.array([-2147483647, 2147483620], dtype=np.int32),
+ }
+ )
+
+ parsed = read_stata(
+ datapath("io", "data", "stata", f"stata_int_validranges_{version}.dta")
+ )
+ tm.assert_frame_equal(parsed, expected)
+
+ @pytest.mark.parametrize("version", [104, 105, 108, 110, 111])
+ def test_read_data_int_validranges_compat(self, version, datapath):
+ expected = DataFrame(
+ {
+ "byte": np.array([-128, 126], dtype=np.int8),
+ "int": np.array([-32768, 32766], dtype=np.int16),
+ "long": np.array([-2147483648, 2147483646], dtype=np.int32),
+ }
+ )
+
+ parsed = read_stata(
+ datapath("io", "data", "stata", f"stata_int_validranges_{version}.dta")
+ )
+ tm.assert_frame_equal(parsed, expected)
+
+ # The byte type was not supported prior to the 104 format
+ @pytest.mark.parametrize("version", [102, 103])
+ def test_read_data_int_validranges_compat_nobyte(self, version, datapath):
+ expected = DataFrame(
+ {
+ "byte": np.array([-128, 126], dtype=np.int16),
+ "int": np.array([-32768, 32766], dtype=np.int16),
+ "long": np.array([-2147483648, 2147483646], dtype=np.int32),
+ }
+ )
+
+ parsed = read_stata(
+ datapath("io", "data", "stata", f"stata_int_validranges_{version}.dta")
+ )
+ tm.assert_frame_equal(parsed, expected)
+
+
+@pytest.mark.parametrize("version", [105, 108, 110, 111, 113, 114])
+def test_backward_compat(version, datapath):
+ data_base = datapath("io", "data", "stata")
+ ref = os.path.join(data_base, "stata-compat-118.dta")
+ old = os.path.join(data_base, f"stata-compat-{version}.dta")
+ expected = read_stata(ref)
+ old_dta = read_stata(old)
+ tm.assert_frame_equal(old_dta, expected, check_dtype=False)
+
+
+@pytest.mark.parametrize("version", [103, 104])
+def test_backward_compat_nodateconversion(version, datapath):
+ # The Stata data format prior to 105 did not support a date format
+ # so read the raw values for comparison
+ data_base = datapath("io", "data", "stata")
+ ref = os.path.join(data_base, "stata-compat-118.dta")
+ old = os.path.join(data_base, f"stata-compat-{version}.dta")
+ expected = read_stata(ref, convert_dates=False)
+ old_dta = read_stata(old, convert_dates=False)
+ tm.assert_frame_equal(old_dta, expected, check_dtype=False)
+
+
+@pytest.mark.parametrize("version", [102])
+def test_backward_compat_nostring(version, datapath):
+ # The Stata data format prior to 105 did not support a date format
+ # so read the raw values for comparison
+ ref = datapath("io", "data", "stata", "stata-compat-118.dta")
+ old = datapath("io", "data", "stata", f"stata-compat-{version}.dta")
+ expected = read_stata(ref, convert_dates=False)
+ # The Stata data format prior to 103 did not support string data
+ expected = expected.drop(columns=["s10"])
+ old_dta = read_stata(old, convert_dates=False)
+ tm.assert_frame_equal(old_dta, expected, check_dtype=False)
+
+
+@pytest.mark.parametrize("version", [105, 108, 110, 111, 113, 114, 118])
+def test_bigendian(version, datapath):
+ ref = datapath("io", "data", "stata", f"stata-compat-{version}.dta")
+ big = datapath("io", "data", "stata", f"stata-compat-be-{version}.dta")
+ expected = read_stata(ref)
+ big_dta = read_stata(big)
+ tm.assert_frame_equal(big_dta, expected)
+
+
+# Note: 102 format does not support big-endian byte order
+@pytest.mark.parametrize("version", [103, 104])
+def test_bigendian_nodateconversion(version, datapath):
+ # The Stata data format prior to 105 did not support a date format
+ # so read the raw values for comparison
+ ref = datapath("io", "data", "stata", f"stata-compat-{version}.dta")
+ big = datapath("io", "data", "stata", f"stata-compat-be-{version}.dta")
+ expected = read_stata(ref, convert_dates=False)
+ big_dta = read_stata(big, convert_dates=False)
+ tm.assert_frame_equal(big_dta, expected)
+
+
+def test_direct_read(datapath, monkeypatch):
+ file_path = datapath("io", "data", "stata", "stata-compat-118.dta")
+
+ # Test that opening a file path doesn't buffer the file.
+ with StataReader(file_path) as reader:
+ # Must not have been buffered to memory
+ assert not reader.read().empty
+ assert not isinstance(reader._path_or_buf, io.BytesIO)
+
+ # Test that we use a given fp exactly, if possible.
+ with open(file_path, "rb") as fp:
+ with StataReader(fp) as reader:
+ assert not reader.read().empty
+ assert reader._path_or_buf is fp
+
+ # Test that we use a given BytesIO exactly, if possible.
+ with open(file_path, "rb") as fp:
+ with io.BytesIO(fp.read()) as bio:
+ with StataReader(bio) as reader:
+ assert not reader.read().empty
+ assert reader._path_or_buf is bio
+
+
+@pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+@pytest.mark.parametrize("use_dict", [True, False])
+@pytest.mark.parametrize("infer", [True, False])
+def test_compression(
+ compression, version, use_dict, infer, compression_to_extension, tmp_path
+):
+ file_name = "dta_inferred_compression.dta"
+ if compression:
+ if use_dict:
+ file_ext = compression
+ else:
+ file_ext = compression_to_extension[compression]
+ file_name += f".{file_ext}"
+ compression_arg = compression
+ if infer:
+ compression_arg = "infer"
+ if use_dict:
+ compression_arg = {"method": compression}
+
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)), columns=list("AB")
+ )
+ df.index.name = "index"
+ path = tmp_path / file_name
+ path.touch()
+ df.to_stata(path, version=version, compression=compression_arg)
+ if compression == "gzip":
+ with gzip.open(path, "rb") as comp:
+ fp = io.BytesIO(comp.read())
+ elif compression == "zip":
+ with zipfile.ZipFile(path, "r") as comp:
+ fp = io.BytesIO(comp.read(comp.filelist[0]))
+ elif compression == "tar":
+ with tarfile.open(path) as tar:
+ fp = io.BytesIO(tar.extractfile(tar.getnames()[0]).read())
+ elif compression == "bz2":
+ with bz2.open(path, "rb") as comp:
+ fp = io.BytesIO(comp.read())
+ elif compression == "zstd":
+ zstd = pytest.importorskip("zstandard")
+ with zstd.open(path, "rb") as comp:
+ fp = io.BytesIO(comp.read())
+ elif compression == "xz":
+ lzma = pytest.importorskip("lzma")
+ with lzma.open(path, "rb") as comp:
+ fp = io.BytesIO(comp.read())
+ elif compression is None:
+ fp = path
+ reread = read_stata(fp, index_col="index")
+
+ expected = df
+ tm.assert_frame_equal(reread, expected)
+
+
+@pytest.mark.parametrize("method", ["zip", "infer"])
+@pytest.mark.parametrize("file_ext", [None, "dta", "zip"])
+def test_compression_dict(method, file_ext, tmp_path):
+ file_name = f"test.{file_ext}"
+ archive_name = "test.dta"
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)), columns=list("AB")
+ )
+ df.index.name = "index"
+ compression = {"method": method, "archive_name": archive_name}
+ path = tmp_path / file_name
+ path.touch()
+ df.to_stata(path, compression=compression)
+ if method == "zip" or file_ext == "zip":
+ with zipfile.ZipFile(path, "r") as zp:
+ assert len(zp.filelist) == 1
+ assert zp.filelist[0].filename == archive_name
+ fp = io.BytesIO(zp.read(zp.filelist[0]))
+ else:
+ fp = path
+ reread = read_stata(fp, index_col="index")
+
+ expected = df
+ tm.assert_frame_equal(reread, expected)
+
+
+@pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+def test_chunked_categorical(version, temp_file):
+ df = DataFrame({"cats": Series(["a", "b", "a", "b", "c"], dtype="category")})
+ df.index.name = "index"
+
+ expected = df.copy()
+
+ df.to_stata(temp_file, version=version)
+ with StataReader(temp_file, chunksize=2, order_categoricals=False) as reader:
+ for i, block in enumerate(reader):
+ block = block.set_index("index")
+ assert "cats" in block
+ tm.assert_series_equal(
+ block.cats,
+ expected.cats.iloc[2 * i : 2 * (i + 1)],
+ check_index_type=len(block) > 1,
+ )
+
+
+def test_chunked_categorical_partial(datapath):
+ dta_file = datapath("io", "data", "stata", "stata-dta-partially-labeled.dta")
+ values = ["a", "b", "a", "b", 3.0]
+ msg = "series with value labels are not fully labeled"
+ with StataReader(dta_file, chunksize=2) as reader:
+ with tm.assert_produces_warning(CategoricalConversionWarning, match=msg):
+ for i, block in enumerate(reader):
+ assert list(block.cats) == values[2 * i : 2 * (i + 1)]
+ if i < 2:
+ idx = pd.Index(["a", "b"])
+ else:
+ idx = pd.Index([3.0], dtype="float64")
+ tm.assert_index_equal(block.cats.cat.categories, idx)
+ with tm.assert_produces_warning(CategoricalConversionWarning, match=msg):
+ with StataReader(dta_file, chunksize=5) as reader:
+ large_chunk = reader.__next__()
+ direct = read_stata(dta_file)
+ tm.assert_frame_equal(direct, large_chunk)
+
+
+@pytest.mark.parametrize("chunksize", (-1, 0, "apple"))
+def test_iterator_errors(datapath, chunksize):
+ dta_file = datapath("io", "data", "stata", "stata-dta-partially-labeled.dta")
+ with pytest.raises(ValueError, match="chunksize must be a positive"):
+ with StataReader(dta_file, chunksize=chunksize):
+ pass
+
+
+def test_iterator_value_labels(temp_file):
+ # GH 31544
+ values = ["c_label", "b_label"] + ["a_label"] * 500
+ df = DataFrame({f"col{k}": pd.Categorical(values, ordered=True) for k in range(2)})
+ df.to_stata(temp_file, write_index=False)
+ expected = pd.Index(["a_label", "b_label", "c_label"])
+ with read_stata(temp_file, chunksize=100) as reader:
+ for j, chunk in enumerate(reader):
+ for i in range(2):
+ tm.assert_index_equal(chunk.dtypes.iloc[i].categories, expected)
+ tm.assert_frame_equal(chunk, df.iloc[j * 100 : (j + 1) * 100])
+
+
+def test_precision_loss(temp_file):
+ df = DataFrame(
+ [[sum(2**i for i in range(60)), sum(2**i for i in range(52))]],
+ columns=["big", "little"],
+ )
+ with tm.assert_produces_warning(
+ PossiblePrecisionLoss, match="Column converted from int64 to float64"
+ ):
+ df.to_stata(temp_file, write_index=False)
+ reread = read_stata(temp_file)
+ expected_dt = Series([np.float64, np.float64], index=["big", "little"])
+ tm.assert_series_equal(reread.dtypes, expected_dt)
+ assert reread.loc[0, "little"] == df.loc[0, "little"]
+ assert reread.loc[0, "big"] == float(df.loc[0, "big"])
+
+
+def test_compression_roundtrip(compression, temp_file):
+ df = DataFrame(
+ [[0.123456, 0.234567, 0.567567], [12.32112, 123123.2, 321321.2]],
+ index=["A", "B"],
+ columns=["X", "Y", "Z"],
+ )
+ df.index.name = "index"
+
+ df.to_stata(temp_file, compression=compression)
+ reread = read_stata(temp_file, compression=compression, index_col="index")
+ tm.assert_frame_equal(df, reread)
+
+ # explicitly ensure file was compressed.
+ with tm.decompress_file(temp_file, compression) as fh:
+ contents = io.BytesIO(fh.read())
+ reread = read_stata(contents, index_col="index")
+ tm.assert_frame_equal(df, reread)
+
+
+@pytest.mark.parametrize("to_infer", [True, False])
+@pytest.mark.parametrize("read_infer", [True, False])
+def test_stata_compression(
+ compression_only, read_infer, to_infer, compression_to_extension, tmp_path
+):
+ compression = compression_only
+
+ ext = compression_to_extension[compression]
+ filename = f"test.{ext}"
+
+ df = DataFrame(
+ [[0.123456, 0.234567, 0.567567], [12.32112, 123123.2, 321321.2]],
+ index=["A", "B"],
+ columns=["X", "Y", "Z"],
+ )
+ df.index.name = "index"
+
+ to_compression = "infer" if to_infer else compression
+ read_compression = "infer" if read_infer else compression
+
+ path = tmp_path / filename
+ path.touch()
+ df.to_stata(path, compression=to_compression)
+ result = read_stata(path, compression=read_compression, index_col="index")
+ tm.assert_frame_equal(result, df)
+
+
+def test_non_categorical_value_labels(temp_file):
+ data = DataFrame(
+ {
+ "fully_labelled": [1, 2, 3, 3, 1],
+ "partially_labelled": [1.0, 2.0, np.nan, 9.0, np.nan],
+ "Y": [7, 7, 9, 8, 10],
+ "Z": pd.Categorical(["j", "k", "l", "k", "j"]),
+ }
+ )
+
+ path = temp_file
+ value_labels = {
+ "fully_labelled": {1: "one", 2: "two", 3: "three"},
+ "partially_labelled": {1.0: "one", 2.0: "two"},
+ }
+ expected = {**value_labels, "Z": {0: "j", 1: "k", 2: "l"}}
+
+ writer = StataWriter(path, data, value_labels=value_labels)
+ writer.write_file()
+
+ with StataReader(path) as reader:
+ reader_value_labels = reader.value_labels()
+ assert reader_value_labels == expected
+
+ msg = "Can't create value labels for notY, it wasn't found in the dataset."
+ value_labels = {"notY": {7: "label1", 8: "label2"}}
+ with pytest.raises(KeyError, match=msg):
+ StataWriter(path, data, value_labels=value_labels)
+
+ msg = (
+ "Can't create value labels for Z, value labels "
+ "can only be applied to numeric columns."
+ )
+ value_labels = {"Z": {1: "a", 2: "k", 3: "j", 4: "i"}}
+ with pytest.raises(ValueError, match=msg):
+ StataWriter(path, data, value_labels=value_labels)
+
+
+def test_non_categorical_value_label_name_conversion(temp_file):
+ # Check conversion of invalid variable names
+ data = DataFrame(
+ {
+ "invalid~!": [1, 1, 2, 3, 5, 8], # Only alphanumeric and _
+ "6_invalid": [1, 1, 2, 3, 5, 8], # Must start with letter or _
+ "invalid_name_longer_than_32_characters": [8, 8, 9, 9, 8, 8], # Too long
+ "aggregate": [2, 5, 5, 6, 6, 9], # Reserved words
+ (1, 2): [1, 2, 3, 4, 5, 6], # Hashable non-string
+ }
+ )
+
+ value_labels = {
+ "invalid~!": {1: "label1", 2: "label2"},
+ "6_invalid": {1: "label1", 2: "label2"},
+ "invalid_name_longer_than_32_characters": {8: "eight", 9: "nine"},
+ "aggregate": {5: "five"},
+ (1, 2): {3: "three"},
+ }
+
+ expected = {
+ "invalid__": {1: "label1", 2: "label2"},
+ "_6_invalid": {1: "label1", 2: "label2"},
+ "invalid_name_longer_than_32_char": {8: "eight", 9: "nine"},
+ "_aggregate": {5: "five"},
+ "_1__2_": {3: "three"},
+ }
+
+ msg = "Not all pandas column names were valid Stata variable names"
+ with tm.assert_produces_warning(InvalidColumnName, match=msg):
+ data.to_stata(temp_file, value_labels=value_labels)
+
+ with StataReader(temp_file) as reader:
+ reader_value_labels = reader.value_labels()
+ assert reader_value_labels == expected
+
+
+def test_non_categorical_value_label_convert_categoricals_error(temp_file):
+ # Mapping more than one value to the same label is valid for Stata
+ # labels, but can't be read with convert_categoricals=True
+ value_labels = {
+ "repeated_labels": {10: "Ten", 20: "More than ten", 40: "More than ten"}
+ }
+
+ data = DataFrame(
+ {
+ "repeated_labels": [10, 10, 20, 20, 40, 40],
+ }
+ )
+
+ data.to_stata(temp_file, value_labels=value_labels)
+
+ with StataReader(temp_file, convert_categoricals=False) as reader:
+ reader_value_labels = reader.value_labels()
+ assert reader_value_labels == value_labels
+
+ col = "repeated_labels"
+ repeats = "-" * 80 + "\n" + "\n".join(["More than ten"])
+
+ msg = f"""
+Value labels for column {col} are not unique. These cannot be converted to
+pandas categoricals.
+
+Either read the file with `convert_categoricals` set to False or use the
+low level interface in `StataReader` to separately read the values and the
+value_labels.
+
+The repeated labels are:
+{repeats}
+"""
+ with pytest.raises(ValueError, match=msg):
+ read_stata(temp_file, convert_categoricals=True)
+
+
+@pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+@pytest.mark.parametrize(
+ "dtype",
+ [
+ pd.BooleanDtype,
+ pd.Int8Dtype,
+ pd.Int16Dtype,
+ pd.Int32Dtype,
+ pd.Int64Dtype,
+ pd.UInt8Dtype,
+ pd.UInt16Dtype,
+ pd.UInt32Dtype,
+ pd.UInt64Dtype,
+ ],
+)
+def test_nullable_support(dtype, version, temp_file):
+ df = DataFrame(
+ {
+ "a": Series([1.0, 2.0, 3.0]),
+ "b": Series([1, pd.NA, pd.NA], dtype=dtype.name),
+ "c": Series(["a", "b", None]),
+ }
+ )
+ dtype_name = df.b.dtype.numpy_dtype.name
+ # Only use supported names: no uint, bool or int64
+ dtype_name = dtype_name.replace("u", "")
+ if dtype_name == "int64":
+ dtype_name = "int32"
+ elif dtype_name == "bool":
+ dtype_name = "int8"
+ value = StataMissingValue.BASE_MISSING_VALUES[dtype_name]
+ smv = StataMissingValue(value)
+ expected_b = Series([1, smv, smv], dtype=object, name="b")
+ expected_c = Series(["a", "b", ""], name="c")
+ df.to_stata(temp_file, write_index=False, version=version)
+ reread = read_stata(temp_file, convert_missing=True)
+ tm.assert_series_equal(df.a, reread.a)
+ tm.assert_series_equal(reread.b, expected_b)
+ tm.assert_series_equal(reread.c, expected_c)
+
+
+def test_empty_frame(temp_file):
+ # GH 46240
+ # create an empty DataFrame with int64 and float64 dtypes
+ df = DataFrame(data={"a": range(3), "b": [1.0, 2.0, 3.0]}).head(0)
+ path = temp_file
+ df.to_stata(path, write_index=False, version=117)
+ # Read entire dataframe
+ df2 = read_stata(path)
+ assert "b" in df2
+ # Dtypes don't match since no support for int32
+ dtypes = Series({"a": np.dtype("int32"), "b": np.dtype("float64")})
+ tm.assert_series_equal(df2.dtypes, dtypes)
+ # read one column of empty .dta file
+ df3 = read_stata(path, columns=["a"])
+ assert "b" not in df3
+ tm.assert_series_equal(df3.dtypes, dtypes.loc[["a"]])
+
+
+@pytest.mark.parametrize("version", [114, 117, 118, 119, None])
+def test_many_strl(temp_file, version):
+ n = 65534
+ df = DataFrame(np.arange(n), columns=["col"])
+ lbls = ["".join(v) for v in itertools.product(*([string.ascii_letters] * 3))]
+ value_labels = {"col": {i: lbls[i] for i in range(n)}}
+ df.to_stata(temp_file, value_labels=value_labels, version=version)
+
+
+@pytest.mark.parametrize("version", [117, 118, 119, None])
+def test_strl_missings(temp_file, version):
+ # GH 23633
+ # Check that strl supports None and pd.NA
+ df = DataFrame(
+ [
+ {"str1": "string" * 500, "number": 0},
+ {"str1": None, "number": 1},
+ {"str1": pd.NA, "number": 1},
+ ]
+ )
+ df.to_stata(temp_file, version=version)
+
+
+@pytest.mark.parametrize("version", [117, 118, 119, None])
+def test_ascii_error(temp_file, version):
+ # GH #61583
+ # Check that 2 byte long unicode characters doesn't cause export error
+ df = DataFrame({"doubleByteCol": ["§" * 1500]})
+ df.to_stata(temp_file, write_index=0, version=version)
+ df_input = read_stata(temp_file)
+ tm.assert_frame_equal(df, df_input)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/test_util.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_util.py
new file mode 100644
index 0000000000000000000000000000000000000000..555b66d129b886dffa9557fa9e11737cb9303308
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/test_util.py
@@ -0,0 +1,69 @@
+import zoneinfo
+
+import pytest
+
+from pandas.compat import (
+ pa_version_under18p0,
+ pa_version_under23p0,
+)
+
+import pandas as pd
+import pandas._testing as tm
+
+from pandas.io._util import arrow_table_to_pandas
+
+pa = pytest.importorskip("pyarrow")
+pytz = pytest.importorskip("pytz")
+
+
+pytestmark = pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager to DataFrame:DeprecationWarning"
+)
+
+
+def test_arrow_table_to_pandas_normalize_timezones():
+ df = pd.DataFrame(
+ {"ts": pd.date_range("2024-03-01", periods=2, tz="America/New_York")},
+ index=pd.date_range("2024-01-01", periods=2, tz="US/Eastern"),
+ )
+ expected = df.copy()
+ expected.index = expected.index._with_freq(None)
+
+ table = pa.Table.from_pandas(df)
+ result = arrow_table_to_pandas(table)
+
+ tm.assert_frame_equal(result, expected)
+ assert isinstance(result["ts"].dtype.tz, zoneinfo.ZoneInfo)
+ assert isinstance(result.index.tz, zoneinfo.ZoneInfo)
+
+
+def test_arrow_table_to_pandas_normalize_timezones_columns():
+ df = pd.DataFrame(
+ [[1, 2], [3, 4]],
+ columns=pd.date_range("2024-02-01", periods=2, tz="Europe/Berlin"),
+ )
+ expected = df.copy()
+
+ table = pa.Table.from_pandas(df)
+ result = arrow_table_to_pandas(table)
+
+ if pa_version_under23p0 and not pa_version_under18p0:
+ expected.columns = expected.columns.as_unit("ns")
+
+ tm.assert_frame_equal(result, expected)
+ assert isinstance(result.columns.tz, zoneinfo.ZoneInfo)
+
+
+def test_arrow_table_to_pandas_normalize_timezones_multiindex():
+ df = pd.DataFrame(
+ {"ts": pd.date_range("2024-03-01", periods=2, tz="America/New_York")},
+ index=pd.Index([1, 2], name="index"),
+ ).set_index("ts", append=True, drop=False)
+ expected = df.copy()
+
+ table = pa.Table.from_pandas(df)
+ result = arrow_table_to_pandas(table)
+
+ tm.assert_frame_equal(result, expected)
+ assert isinstance(result["ts"].dtype.tz, zoneinfo.ZoneInfo)
+ assert isinstance(result.index.get_level_values("ts").tz, zoneinfo.ZoneInfo)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/xml/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/xml/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/xml/conftest.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/xml/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..8900b0dc46754a2b7b5191c3619b75afe57eb232
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/xml/conftest.py
@@ -0,0 +1,105 @@
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def xml_data_path():
+ """
+ Returns a Path object to the XML example directory.
+
+ Examples
+ --------
+ >>> def test_read_xml(xml_data_path):
+ ... pd.read_xml(xml_data_path / "file.xsl")
+ """
+ return Path(__file__).parent.parent / "data" / "xml"
+
+
+@pytest.fixture
+def xml_books(xml_data_path, datapath):
+ """
+ Returns the path (as `str`) to the `books.xml` example file.
+
+ Examples
+ --------
+ >>> def test_read_xml(xml_books):
+ ... pd.read_xml(xml_books)
+ """
+ return datapath(xml_data_path / "books.xml")
+
+
+@pytest.fixture
+def xml_doc_ch_utf(xml_data_path, datapath):
+ """
+ Returns the path (as `str`) to the `doc_ch_utf.xml` example file.
+
+ Examples
+ --------
+ >>> def test_read_xml(xml_doc_ch_utf):
+ ... pd.read_xml(xml_doc_ch_utf)
+ """
+ return datapath(xml_data_path / "doc_ch_utf.xml")
+
+
+@pytest.fixture
+def xml_baby_names(xml_data_path, datapath):
+ """
+ Returns the path (as `str`) to the `baby_names.xml` example file.
+
+ Examples
+ --------
+ >>> def test_read_xml(xml_baby_names):
+ ... pd.read_xml(xml_baby_names)
+ """
+ return datapath(xml_data_path / "baby_names.xml")
+
+
+@pytest.fixture
+def kml_cta_rail_lines(xml_data_path, datapath):
+ """
+ Returns the path (as `str`) to the `cta_rail_lines.kml` example file.
+
+ Examples
+ --------
+ >>> def test_read_xml(kml_cta_rail_lines):
+ ... pd.read_xml(
+ ... kml_cta_rail_lines,
+ ... xpath=".//k:Placemark",
+ ... namespaces={"k": "http://www.opengis.net/kml/2.2"},
+ ... stylesheet=xsl_flatten_doc,
+ ... )
+ """
+ return datapath(xml_data_path / "cta_rail_lines.kml")
+
+
+@pytest.fixture
+def xsl_flatten_doc(xml_data_path, datapath):
+ """
+ Returns the path (as `str`) to the `flatten_doc.xsl` example file.
+
+ Examples
+ --------
+ >>> def test_read_xsl(xsl_flatten_doc, mode):
+ ... with open(
+ ... xsl_flatten_doc, mode, encoding="utf-8" if mode == "r" else None
+ ... ) as f:
+ ... xsl_obj = f.read()
+ """
+ return datapath(xml_data_path / "flatten_doc.xsl")
+
+
+@pytest.fixture
+def xsl_row_field_output(xml_data_path, datapath):
+ """
+ Returns the path (as `str`) to the `row_field_output.xsl` example file.
+
+ Examples
+ --------
+ >>> def test_read_xsl(xsl_row_field_output, mode):
+ ... with open(
+ ... xsl_row_field_output, mode, encoding="utf-8" if mode == "r" else None
+ ... ) as f:
+ ... xsl_obj = f.read()
+ """
+ return datapath(xml_data_path / "row_field_output.xsl")
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/xml/test_to_xml.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/xml/test_to_xml.py
new file mode 100644
index 0000000000000000000000000000000000000000..2178d904d8461ad5e691f8158217afca17851998
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/xml/test_to_xml.py
@@ -0,0 +1,1357 @@
+from __future__ import annotations
+
+from io import (
+ BytesIO,
+ StringIO,
+)
+import os
+
+import numpy as np
+import pytest
+
+import pandas.util._test_decorators as td
+
+from pandas import (
+ NA,
+ DataFrame,
+ Index,
+)
+import pandas._testing as tm
+
+from pandas.io.common import get_handle
+from pandas.io.xml import read_xml
+
+# CHECKLIST
+
+# [x] - ValueError: "Values for parser can only be lxml or etree."
+
+# etree
+# [x] - ImportError: "lxml not found, please install or use the etree parser."
+# [X] - TypeError: "...is not a valid type for attr_cols"
+# [X] - TypeError: "...is not a valid type for elem_cols"
+# [X] - LookupError: "unknown encoding"
+# [X] - KeyError: "...is not included in namespaces"
+# [X] - KeyError: "no valid column"
+# [X] - ValueError: "To use stylesheet, you need lxml installed..."
+# [] - OSError: (NEED PERMISSION ISSUE, DISK FULL, ETC.)
+# [X] - FileNotFoundError: "No such file or directory"
+# [X] - PermissionError: "Forbidden"
+
+# lxml
+# [X] - TypeError: "...is not a valid type for attr_cols"
+# [X] - TypeError: "...is not a valid type for elem_cols"
+# [X] - LookupError: "unknown encoding"
+# [] - OSError: (NEED PERMISSION ISSUE, DISK FULL, ETC.)
+# [X] - FileNotFoundError: "No such file or directory"
+# [X] - KeyError: "...is not included in namespaces"
+# [X] - KeyError: "no valid column"
+# [X] - ValueError: "stylesheet is not a url, file, or xml string."
+# [] - LookupError: (NEED WRONG ENCODING FOR FILE OUTPUT)
+# [] - URLError: (USUALLY DUE TO NETWORKING)
+# [] - HTTPError: (NEED AN ONLINE STYLESHEET)
+# [X] - OSError: "failed to load external entity"
+# [X] - XMLSyntaxError: "Opening and ending tag mismatch"
+# [X] - XSLTApplyError: "Cannot resolve URI"
+# [X] - XSLTParseError: "failed to compile"
+# [X] - PermissionError: "Forbidden"
+
+
+@pytest.fixture
+def geom_df():
+ return DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": [360, 360, 180],
+ "sides": [4, np.nan, 3],
+ }
+ )
+
+
+@pytest.fixture
+def planet_df():
+ return DataFrame(
+ {
+ "planet": [
+ "Mercury",
+ "Venus",
+ "Earth",
+ "Mars",
+ "Jupiter",
+ "Saturn",
+ "Uranus",
+ "Neptune",
+ ],
+ "type": [
+ "terrestrial",
+ "terrestrial",
+ "terrestrial",
+ "terrestrial",
+ "gas giant",
+ "gas giant",
+ "ice giant",
+ "ice giant",
+ ],
+ "location": [
+ "inner",
+ "inner",
+ "inner",
+ "inner",
+ "outer",
+ "outer",
+ "outer",
+ "outer",
+ ],
+ "mass": [
+ 0.330114,
+ 4.86747,
+ 5.97237,
+ 0.641712,
+ 1898.187,
+ 568.3174,
+ 86.8127,
+ 102.4126,
+ ],
+ }
+ )
+
+
+@pytest.fixture
+def from_file_expected():
+ return """\
+
+
+
+ 0
+ cooking
+ Everyday Italian
+ Giada De Laurentiis
+ 2005
+ 30.0
+
+
+ 1
+ children
+ Harry Potter
+ J K. Rowling
+ 2005
+ 29.99
+
+
+ 2
+ web
+ Learning XML
+ Erik T. Ray
+ 2003
+ 39.95
+
+ """
+
+
+def equalize_decl(doc):
+ # etree and lxml differ on quotes and case in xml declaration
+ if doc is not None:
+ doc = doc.replace(
+ '
+
+
+ cooking
+ Everyday Italian
+ Giada De Laurentiis
+ 2005
+ 30.0
+
+
+ children
+ Harry Potter
+ J K. Rowling
+ 2005
+ 29.99
+
+
+ web
+ Learning XML
+ Erik T. Ray
+ 2003
+ 39.95
+
+ """
+
+ df_file = read_xml(xml_books, parser=parser)
+
+ df_file.to_xml(temp_file, index=False, parser=parser)
+ output = temp_file.read_text(encoding="utf-8").strip()
+
+ output = equalize_decl(output)
+
+ assert output == expected
+
+
+def test_index_false_rename_row_root(xml_books, parser, temp_file):
+ expected = """\
+
+
+
+ cooking
+ Everyday Italian
+ Giada De Laurentiis
+ 2005
+ 30.0
+
+
+ children
+ Harry Potter
+ J K. Rowling
+ 2005
+ 29.99
+
+
+ web
+ Learning XML
+ Erik T. Ray
+ 2003
+ 39.95
+
+ """
+
+ df_file = read_xml(xml_books, parser=parser)
+
+ df_file.to_xml(
+ temp_file, index=False, root_name="books", row_name="book", parser=parser
+ )
+ output = temp_file.read_text(encoding="utf-8").strip()
+
+ output = equalize_decl(output)
+
+ assert output == expected
+
+
+@pytest.mark.parametrize("typ", [int, str])
+def test_index_false_with_offset_input_index(parser, typ, geom_df):
+ """
+ Tests that the output does not contain the `` field when the index of the
+ input Dataframe has an offset.
+
+ This is a regression test for issue #42458.
+ """
+
+ expected = """\
+
+
+
+ square
+ 360
+ 4.0
+
+
+ circle
+ 360
+
+
+
+ triangle
+ 180
+ 3.0
+
+ """
+ offset_index = [typ(i) for i in range(10, 13)]
+ offset_geom_df = geom_df.copy()
+ offset_geom_df.index = Index(offset_index)
+ output = offset_geom_df.to_xml(index=False, parser=parser)
+ output = equalize_decl(output)
+
+ assert output == expected
+
+
+# NA_REP
+
+na_expected = """\
+
+
+
+ 0
+ square
+ 360
+ 4.0
+
+
+ 1
+ circle
+ 360
+
+
+
+ 2
+ triangle
+ 180
+ 3.0
+
+ """
+
+
+def test_na_elem_output(parser, geom_df):
+ output = geom_df.to_xml(parser=parser)
+ output = equalize_decl(output)
+
+ assert output == na_expected
+
+
+def test_na_empty_str_elem_option(parser, geom_df):
+ output = geom_df.to_xml(na_rep="", parser=parser)
+ output = equalize_decl(output)
+
+ assert output == na_expected
+
+
+def test_na_empty_elem_option(parser, geom_df):
+ expected = """\
+
+
+
+ 0
+ square
+ 360
+ 4.0
+
+
+ 1
+ circle
+ 360
+ 0.0
+
+
+ 2
+ triangle
+ 180
+ 3.0
+
+ """
+
+ output = geom_df.to_xml(na_rep="0.0", parser=parser)
+ output = equalize_decl(output)
+
+ assert output == expected
+
+
+# ATTR_COLS
+
+
+def test_attrs_cols_nan_output(parser, geom_df):
+ expected = """\
+
+
+
+
+
+ """
+
+ output = geom_df.to_xml(attr_cols=["shape", "degrees", "sides"], parser=parser)
+ output = equalize_decl(output)
+
+ assert output == expected
+
+
+def test_attrs_cols_prefix(parser, geom_df):
+ expected = """\
+
+
+
+
+
+ """
+
+ output = geom_df.to_xml(
+ attr_cols=["index", "shape", "degrees", "sides"],
+ namespaces={"doc": "http://example.xom"},
+ prefix="doc",
+ parser=parser,
+ )
+ output = equalize_decl(output)
+
+ assert output == expected
+
+
+def test_attrs_unknown_column(parser, geom_df):
+ with pytest.raises(KeyError, match=("no valid column")):
+ geom_df.to_xml(attr_cols=["shape", "degree", "sides"], parser=parser)
+
+
+def test_attrs_wrong_type(parser, geom_df):
+ with pytest.raises(TypeError, match=("is not a valid type for attr_cols")):
+ geom_df.to_xml(attr_cols='"shape", "degree", "sides"', parser=parser)
+
+
+# ELEM_COLS
+
+
+def test_elems_cols_nan_output(parser, geom_df):
+ elems_cols_expected = """\
+
+
+
+ 360
+ 4.0
+ square
+
+
+ 360
+
+ circle
+
+
+ 180
+ 3.0
+ triangle
+
+ """
+
+ output = geom_df.to_xml(
+ index=False, elem_cols=["degrees", "sides", "shape"], parser=parser
+ )
+ output = equalize_decl(output)
+
+ assert output == elems_cols_expected
+
+
+def test_elems_unknown_column(parser, geom_df):
+ with pytest.raises(KeyError, match=("no valid column")):
+ geom_df.to_xml(elem_cols=["shape", "degree", "sides"], parser=parser)
+
+
+def test_elems_wrong_type(parser, geom_df):
+ with pytest.raises(TypeError, match=("is not a valid type for elem_cols")):
+ geom_df.to_xml(elem_cols='"shape", "degree", "sides"', parser=parser)
+
+
+def test_elems_and_attrs_cols(parser, geom_df):
+ elems_cols_expected = """\
+
+
+
+ 360
+ 4.0
+
+
+ 360
+
+
+
+ 180
+ 3.0
+
+ """
+
+ output = geom_df.to_xml(
+ index=False,
+ elem_cols=["degrees", "sides"],
+ attr_cols=["shape"],
+ parser=parser,
+ )
+ output = equalize_decl(output)
+
+ assert output == elems_cols_expected
+
+
+# HIERARCHICAL COLUMNS
+
+
+def test_hierarchical_columns(parser, planet_df):
+ expected = """\
+
+
+
+ inner
+ terrestrial
+ 4
+ 11.81
+ 2.95
+
+
+ outer
+ gas giant
+ 2
+ 2466.5
+ 1233.25
+
+
+ outer
+ ice giant
+ 2
+ 189.23
+ 94.61
+
+
+ All
+
+ 8
+ 2667.54
+ 333.44
+
+ """
+
+ pvt = planet_df.pivot_table(
+ index=["location", "type"],
+ values="mass",
+ aggfunc=["count", "sum", "mean"],
+ margins=True,
+ ).round(2)
+
+ output = pvt.to_xml(parser=parser)
+ output = equalize_decl(output)
+
+ assert output == expected
+
+
+def test_hierarchical_attrs_columns(parser, planet_df):
+ expected = """\
+
+
+
+
+
+
+ """
+
+ pvt = planet_df.pivot_table(
+ index=["location", "type"],
+ values="mass",
+ aggfunc=["count", "sum", "mean"],
+ margins=True,
+ ).round(2)
+
+ output = pvt.to_xml(attr_cols=list(pvt.reset_index().columns.values), parser=parser)
+ output = equalize_decl(output)
+
+ assert output == expected
+
+
+# MULTIINDEX
+
+
+def test_multi_index(parser, planet_df):
+ expected = """\
+
+
+
+ inner
+ terrestrial
+ 4
+ 11.81
+ 2.95
+
+
+ outer
+ gas giant
+ 2
+ 2466.5
+ 1233.25
+
+
+ outer
+ ice giant
+ 2
+ 189.23
+ 94.61
+
+ """
+
+ agg = (
+ planet_df.groupby(["location", "type"])["mass"]
+ .agg(["count", "sum", "mean"])
+ .round(2)
+ )
+
+ output = agg.to_xml(parser=parser)
+ output = equalize_decl(output)
+
+ assert output == expected
+
+
+def test_multi_index_attrs_cols(parser, planet_df):
+ expected = """\
+
+
+
+
+
+ """
+
+ agg = (
+ planet_df.groupby(["location", "type"])["mass"]
+ .agg(["count", "sum", "mean"])
+ .round(2)
+ )
+ output = agg.to_xml(attr_cols=list(agg.reset_index().columns.values), parser=parser)
+ output = equalize_decl(output)
+
+ assert output == expected
+
+
+# NAMESPACE
+
+
+def test_default_namespace(parser, geom_df):
+ expected = """\
+
+
+
+ 0
+ square
+ 360
+ 4.0
+
+
+ 1
+ circle
+ 360
+
+
+
+ 2
+ triangle
+ 180
+ 3.0
+
+ """
+
+ output = geom_df.to_xml(namespaces={"": "http://example.com"}, parser=parser)
+ output = equalize_decl(output)
+
+ assert output == expected
+
+
+def test_unused_namespaces(parser, geom_df):
+ expected = """\
+
+
+
+ 0
+ square
+ 360
+ 4.0
+
+
+ 1
+ circle
+ 360
+
+
+
+ 2
+ triangle
+ 180
+ 3.0
+
+ """
+
+ output = geom_df.to_xml(
+ namespaces={"oth": "http://other.org", "ex": "http://example.com"},
+ parser=parser,
+ )
+ output = equalize_decl(output)
+
+ assert output == expected
+
+
+# PREFIX
+
+
+def test_namespace_prefix(parser, geom_df):
+ expected = """\
+
+
+
+ 0
+ square
+ 360
+ 4.0
+
+
+ 1
+ circle
+ 360
+
+
+
+ 2
+ triangle
+ 180
+ 3.0
+
+ """
+
+ output = geom_df.to_xml(
+ namespaces={"doc": "http://example.com"}, prefix="doc", parser=parser
+ )
+ output = equalize_decl(output)
+
+ assert output == expected
+
+
+def test_missing_prefix_in_nmsp(parser, geom_df):
+ with pytest.raises(KeyError, match=("doc is not included in namespaces")):
+ geom_df.to_xml(
+ namespaces={"": "http://example.com"}, prefix="doc", parser=parser
+ )
+
+
+def test_namespace_prefix_and_default(parser, geom_df):
+ expected = """\
+
+
+
+ 0
+ square
+ 360
+ 4.0
+
+
+ 1
+ circle
+ 360
+
+
+
+ 2
+ triangle
+ 180
+ 3.0
+
+ """
+
+ output = geom_df.to_xml(
+ namespaces={"": "http://example.com", "doc": "http://other.org"},
+ prefix="doc",
+ parser=parser,
+ )
+ output = equalize_decl(output)
+
+ assert output == expected
+
+
+# ENCODING
+
+encoding_expected = """\
+
+
+
+ 0
+ 1
+ José
+ Sofía
+
+
+ 1
+ 2
+ Luis
+ Valentina
+
+
+ 2
+ 3
+ Carlos
+ Isabella
+
+
+ 3
+ 4
+ Juan
+ Camila
+
+
+ 4
+ 5
+ Jorge
+ Valeria
+
+ """
+
+
+def test_encoding_option_str(xml_baby_names, parser):
+ df_file = read_xml(xml_baby_names, parser=parser, encoding="ISO-8859-1").head(5)
+
+ output = df_file.to_xml(encoding="ISO-8859-1", parser=parser)
+
+ if output is not None:
+ # etree and lxml differ on quotes and case in xml declaration
+ output = output.replace(
+ '
+
+ 0
+ square
+ 360
+ 4.0
+
+
+ 1
+ circle
+ 360
+
+
+
+ 2
+ triangle
+ 180
+ 3.0
+
+"""
+
+ output = geom_df.to_xml(xml_declaration=False)
+
+ assert output == expected
+
+
+def test_no_pretty_print_with_decl(parser, geom_df):
+ expected = (
+ "\n"
+ "0 square "
+ "360 4.0 |
"
+ "1 circle 360"
+ "
2 "
+ "triangle 180 3.0 "
+ "|
"
+ )
+
+ output = geom_df.to_xml(pretty_print=False, parser=parser)
+ output = equalize_decl(output)
+
+ # etree adds space for closed tags
+ if output is not None:
+ output = output.replace(" />", "/>")
+
+ assert output == expected
+
+
+def test_no_pretty_print_no_decl(parser, geom_df):
+ expected = (
+ "0 square "
+ "360 4.0 |
"
+ "1 circle 360"
+ "
2 "
+ "triangle 180 3.0 "
+ "|
"
+ )
+
+ output = geom_df.to_xml(xml_declaration=False, pretty_print=False, parser=parser)
+
+ # etree adds space for closed tags
+ if output is not None:
+ output = output.replace(" />", "/>")
+
+ assert output == expected
+
+
+# PARSER
+
+
+@td.skip_if_installed("lxml")
+def test_default_parser_no_lxml(geom_df):
+ with pytest.raises(
+ ImportError, match=("lxml not found, please install or use the etree parser.")
+ ):
+ geom_df.to_xml()
+
+
+def test_unknown_parser(geom_df):
+ with pytest.raises(
+ ValueError, match=("Values for parser can only be lxml or etree.")
+ ):
+ geom_df.to_xml(parser="bs4")
+
+
+# STYLESHEET
+
+xsl_expected = """\
+
+
+
+ 0
+ square
+ 360
+ 4.0
+
+
+ 1
+ circle
+ 360
+
+
+
+ 2
+ triangle
+ 180
+ 3.0
+
+ """
+
+
+def test_stylesheet_file_like(xsl_row_field_output, mode, geom_df):
+ pytest.importorskip("lxml")
+ with open(
+ xsl_row_field_output, mode, encoding="utf-8" if mode == "r" else None
+ ) as f:
+ assert geom_df.to_xml(stylesheet=f) == xsl_expected
+
+
+def test_stylesheet_io(xsl_row_field_output, mode, geom_df):
+ # note: By default the bodies of untyped functions are not checked,
+ # consider using --check-untyped-defs
+ pytest.importorskip("lxml")
+ xsl_obj: BytesIO | StringIO # type: ignore[annotation-unchecked]
+
+ with open(
+ xsl_row_field_output, mode, encoding="utf-8" if mode == "r" else None
+ ) as f:
+ if mode == "rb":
+ xsl_obj = BytesIO(f.read())
+ else:
+ xsl_obj = StringIO(f.read())
+
+ output = geom_df.to_xml(stylesheet=xsl_obj)
+
+ assert output == xsl_expected
+
+
+def test_stylesheet_buffered_reader(xsl_row_field_output, mode, geom_df):
+ pytest.importorskip("lxml")
+ with open(
+ xsl_row_field_output, mode, encoding="utf-8" if mode == "r" else None
+ ) as f:
+ output = geom_df.to_xml(stylesheet=f)
+
+ assert output == xsl_expected
+
+
+def test_stylesheet_wrong_path(geom_df):
+ pytest.importorskip("lxml.etree")
+
+ xsl = os.path.join("does", "not", "exist", "row_field_output.xslt")
+
+ with pytest.raises(
+ FileNotFoundError, match=r"\[Errno 2\] No such file or director"
+ ):
+ geom_df.to_xml(stylesheet=xsl)
+
+
+@pytest.mark.parametrize("val", [StringIO(""), BytesIO(b"")])
+def test_empty_string_stylesheet(val, geom_df):
+ lxml_etree = pytest.importorskip("lxml.etree")
+
+ msg = "|".join(
+ [
+ "Document is empty",
+ "Start tag expected, '<' not found",
+ # Seen on Mac with lxml 4.9.1
+ r"None \(line 0\)",
+ ]
+ )
+
+ with pytest.raises(lxml_etree.XMLSyntaxError, match=msg):
+ geom_df.to_xml(stylesheet=val)
+
+
+def test_incorrect_xsl_syntax(geom_df):
+ lxml_etree = pytest.importorskip("lxml.etree")
+
+ xsl = """\
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ """
+
+ with pytest.raises(
+ lxml_etree.XMLSyntaxError, match="Opening and ending tag mismatch"
+ ):
+ geom_df.to_xml(stylesheet=StringIO(xsl))
+
+
+def test_incorrect_xsl_eval(geom_df):
+ lxml_etree = pytest.importorskip("lxml.etree")
+
+ xsl = """\
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ """
+
+ with pytest.raises(lxml_etree.XSLTParseError, match="failed to compile"):
+ geom_df.to_xml(stylesheet=StringIO(xsl))
+
+
+def test_incorrect_xsl_apply(geom_df, temp_file):
+ lxml_etree = pytest.importorskip("lxml.etree")
+
+ xsl = """\
+
+
+
+
+
+
+
+
+
+ """
+
+ with pytest.raises(lxml_etree.XSLTApplyError, match="Cannot resolve URI"):
+ geom_df.to_xml(temp_file, stylesheet=StringIO(xsl))
+
+
+def test_stylesheet_with_etree(geom_df):
+ xsl = """\
+
+
+
+
+
+
+
+
+ """
+
+ with pytest.raises(ValueError, match="To use stylesheet, you need lxml installed"):
+ geom_df.to_xml(parser="etree", stylesheet=StringIO(xsl))
+
+
+def test_style_to_csv(geom_df):
+ pytest.importorskip("lxml")
+ xsl = """\
+
+
+
+
+ ,
+
+ ,shape,degrees,sides
+
+
+
+
+
+
+
+ """
+
+ out_csv = geom_df.to_csv(lineterminator="\n")
+
+ if out_csv is not None:
+ out_csv = out_csv.strip()
+ out_xml = geom_df.to_xml(stylesheet=StringIO(xsl))
+
+ assert out_csv == out_xml
+
+
+def test_style_to_string(geom_df):
+ pytest.importorskip("lxml")
+ xsl = """\
+
+
+
+
+
+
+ shape degrees sides
+
+
+
+
+
+
+
+ """
+
+ out_str = geom_df.to_string()
+ out_xml = geom_df.to_xml(na_rep="NaN", stylesheet=StringIO(xsl))
+
+ assert out_xml == out_str
+
+
+def test_style_to_json(geom_df):
+ pytest.importorskip("lxml")
+ xsl = """\
+
+
+
+
+ "
+
+
+ {"shape":{
+
+ },"degrees":{
+
+ },"sides":{
+
+ }}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ,
+
+
+ """
+
+ out_json = geom_df.to_json()
+ out_xml = geom_df.to_xml(stylesheet=StringIO(xsl))
+
+ assert out_json == out_xml
+
+
+# COMPRESSION
+
+
+geom_xml = """\
+
+
+
+ 0
+ square
+ 360
+ 4.0
+
+
+ 1
+ circle
+ 360
+
+
+
+ 2
+ triangle
+ 180
+ 3.0
+
+ """
+
+
+def test_compression_output(parser, compression_only, geom_df, temp_file):
+ path = temp_file
+ geom_df.to_xml(path, parser=parser, compression=compression_only)
+
+ with get_handle(
+ path,
+ "r",
+ compression=compression_only,
+ ) as handle_obj:
+ output = handle_obj.handle.read()
+
+ output = equalize_decl(output)
+
+ assert geom_xml == output.strip()
+
+
+def test_filename_and_suffix_comp(
+ parser, compression_only, geom_df, compression_to_extension, tmp_path
+):
+ compfile = "xml." + compression_to_extension[compression_only]
+ path = tmp_path / compfile
+ geom_df.to_xml(path, parser=parser, compression=compression_only)
+
+ with get_handle(
+ path,
+ "r",
+ compression=compression_only,
+ ) as handle_obj:
+ output = handle_obj.handle.read()
+
+ output = equalize_decl(output)
+
+ assert geom_xml == output.strip()
+
+
+def test_ea_dtypes(any_numeric_ea_dtype, parser):
+ # GH#43903
+ expected = """
+
+
+ 0
+
+
+ """
+ df = DataFrame({"a": [NA]}).astype(any_numeric_ea_dtype)
+ result = df.to_xml(parser=parser)
+ assert equalize_decl(result).strip() == expected
+
+
+def test_unsupported_compression(parser, geom_df, temp_file):
+ with pytest.raises(ValueError, match="Unrecognized compression type"):
+ path = temp_file
+ geom_df.to_xml(path, parser=parser, compression="7z")
+
+
+# STORAGE OPTIONS
+
+
+@pytest.mark.single_cpu
+def test_s3_permission_output(parser, s3_bucket_public, geom_df):
+ s3fs = pytest.importorskip("s3fs")
+ pytest.importorskip("lxml")
+
+ with tm.external_error_raised((PermissionError, FileNotFoundError)):
+ fs = s3fs.S3FileSystem(anon=True)
+ fs.ls(s3_bucket_public.name)
+
+ geom_df.to_xml(
+ f"s3://{s3_bucket_public.name}/geom.xml", compression="zip", parser=parser
+ )
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/xml/test_xml.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/xml/test_xml.py
new file mode 100644
index 0000000000000000000000000000000000000000..8317a3337e0e19bcf1df35ec45f310eae907b48a
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/xml/test_xml.py
@@ -0,0 +1,2067 @@
+from __future__ import annotations
+
+from io import (
+ BytesIO,
+ StringIO,
+)
+from lzma import LZMAError
+import os
+from tarfile import ReadError
+from urllib.error import HTTPError
+from xml.etree.ElementTree import ParseError
+from zipfile import BadZipFile
+
+import numpy as np
+import pytest
+
+from pandas.compat import WASM
+from pandas.compat._optional import import_optional_dependency
+from pandas.errors import (
+ EmptyDataError,
+ ParserError,
+)
+import pandas.util._test_decorators as td
+
+import pandas as pd
+from pandas import (
+ NA,
+ DataFrame,
+ Series,
+)
+import pandas._testing as tm
+
+from pandas.io.common import get_handle
+from pandas.io.xml import read_xml
+
+# CHECK LIST
+
+# [x] - ValueError: "Values for parser can only be lxml or etree."
+
+# etree
+# [X] - ImportError: "lxml not found, please install or use the etree parser."
+# [X] - TypeError: "expected str, bytes or os.PathLike object, not NoneType"
+# [X] - ValueError: "Either element or attributes can be parsed not both."
+# [X] - ValueError: "xpath does not return any nodes..."
+# [X] - SyntaxError: "You have used an incorrect or unsupported XPath"
+# [X] - ValueError: "names does not match length of child elements in xpath."
+# [X] - TypeError: "...is not a valid type for names"
+# [X] - ValueError: "To use stylesheet, you need lxml installed..."
+# [] - URLError: (GENERAL ERROR WITH HTTPError AS SUBCLASS)
+# [X] - HTTPError: "HTTP Error 404: Not Found"
+# [] - OSError: (GENERAL ERROR WITH FileNotFoundError AS SUBCLASS)
+# [X] - FileNotFoundError: "No such file or directory"
+# [] - ParseError (FAILSAFE CATCH ALL FOR VERY COMPLEX XML)
+# [X] - UnicodeDecodeError: "'utf-8' codec can't decode byte 0xe9..."
+# [X] - UnicodeError: "UTF-16 stream does not start with BOM"
+# [X] - BadZipFile: "File is not a zip file"
+# [X] - OSError: "Invalid data stream"
+# [X] - LZMAError: "Input format not supported by decoder"
+# [X] - ValueError: "Unrecognized compression type"
+# [X] - PermissionError: "Forbidden"
+
+# lxml
+# [X] - ValueError: "Either element or attributes can be parsed not both."
+# [X] - AttributeError: "__enter__"
+# [X] - XSLTApplyError: "Cannot resolve URI"
+# [X] - XSLTParseError: "document is not a stylesheet"
+# [X] - ValueError: "xpath does not return any nodes."
+# [X] - XPathEvalError: "Invalid expression"
+# [] - XPathSyntaxError: (OLD VERSION IN lxml FOR XPATH ERRORS)
+# [X] - TypeError: "empty namespace prefix is not supported in XPath"
+# [X] - ValueError: "names does not match length of child elements in xpath."
+# [X] - TypeError: "...is not a valid type for names"
+# [X] - LookupError: "unknown encoding"
+# [] - URLError: (USUALLY DUE TO NETWORKING)
+# [X - HTTPError: "HTTP Error 404: Not Found"
+# [X] - OSError: "failed to load external entity"
+# [X] - XMLSyntaxError: "Start tag expected, '<' not found"
+# [] - ParserError: (FAILSAFE CATCH ALL FOR VERY COMPLEX XML
+# [X] - ValueError: "Values for parser can only be lxml or etree."
+# [X] - UnicodeDecodeError: "'utf-8' codec can't decode byte 0xe9..."
+# [X] - UnicodeError: "UTF-16 stream does not start with BOM"
+# [X] - BadZipFile: "File is not a zip file"
+# [X] - OSError: "Invalid data stream"
+# [X] - LZMAError: "Input format not supported by decoder"
+# [X] - ValueError: "Unrecognized compression type"
+# [X] - PermissionError: "Forbidden"
+
+geom_df = DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": [360, 360, 180],
+ "sides": [4, np.nan, 3],
+ }
+)
+
+xml_default_nmsp = """\
+
+
+
+ square
+ 360
+ 4
+
+
+ circle
+ 360
+
+
+
+ triangle
+ 180
+ 3
+
+ """
+
+xml_prefix_nmsp = """\
+
+
+
+ square
+ 360
+ 4.0
+
+
+ circle
+ 360
+
+
+
+ triangle
+ 180
+ 3.0
+
+ """
+
+
+df_kml = DataFrame(
+ {
+ "id": {
+ 0: "ID_00001",
+ 1: "ID_00002",
+ 2: "ID_00003",
+ 3: "ID_00004",
+ 4: "ID_00005",
+ },
+ "name": {
+ 0: "Blue Line (Forest Park)",
+ 1: "Red, Purple Line",
+ 2: "Red, Purple Line",
+ 3: "Red, Purple Line",
+ 4: "Red, Purple Line",
+ },
+ "styleUrl": {
+ 0: "#LineStyle01",
+ 1: "#LineStyle01",
+ 2: "#LineStyle01",
+ 3: "#LineStyle01",
+ 4: "#LineStyle01",
+ },
+ "extrude": {0: 0, 1: 0, 2: 0, 3: 0, 4: 0},
+ "altitudeMode": {
+ 0: "clampedToGround",
+ 1: "clampedToGround",
+ 2: "clampedToGround",
+ 3: "clampedToGround",
+ 4: "clampedToGround",
+ },
+ "coordinates": {
+ 0: (
+ "-87.77678526964958,41.8708863930319,0 "
+ "-87.77826234150609,41.87097820122218,0 "
+ "-87.78251583439344,41.87130129991005,0 "
+ "-87.78418294588424,41.87145055520308,0 "
+ "-87.7872369165933,41.8717239119163,0 "
+ "-87.79160214925886,41.87210797280065,0"
+ ),
+ 1: (
+ "-87.65758750947528,41.96427269188822,0 "
+ "-87.65802133507393,41.96581929055245,0 "
+ "-87.65819033925305,41.96621846093642,0 "
+ "-87.6583189819129,41.96650362897086,0 "
+ "-87.65835858701473,41.96669002089185,0 "
+ "-87.65838428411853,41.96688150295095,0 "
+ "-87.65842208882658,41.96745896091846,0 "
+ "-87.65846556843937,41.9683761425439,0 "
+ "-87.65849296214573,41.96913893870342,0"
+ ),
+ 2: (
+ "-87.65492939166126,41.95377494531437,0 "
+ "-87.65557043199591,41.95376544118533,0 "
+ "-87.65606302030132,41.95376391658746,0 "
+ "-87.65623502146268,41.95377379126367,0 "
+ "-87.65634748981634,41.95380103566435,0 "
+ "-87.65646537904269,41.95387703994676,0 "
+ "-87.65656532461145,41.95396622645799,0 "
+ "-87.65664760856414,41.95404201996044,0 "
+ "-87.65671750555913,41.95416647054043,0 "
+ "-87.65673983607117,41.95429949810849,0 "
+ "-87.65673866475777,41.95441024240925,0 "
+ "-87.6567690255541,41.95490657227902,0 "
+ "-87.65683672482363,41.95692259283837,0 "
+ "-87.6568900886376,41.95861070983142,0 "
+ "-87.65699865558875,41.96181418669004,0 "
+ "-87.65756347177603,41.96397045777844,0 "
+ "-87.65758750947528,41.96427269188822,0"
+ ),
+ 3: (
+ "-87.65362593118043,41.94742799535678,0 "
+ "-87.65363554415794,41.94819886386848,0 "
+ "-87.6536456393239,41.95059994675451,0 "
+ "-87.65365831235026,41.95108288489359,0 "
+ "-87.6536604873874,41.9519954657554,0 "
+ "-87.65362592053201,41.95245597302328,0 "
+ "-87.65367158496069,41.95311153649393,0 "
+ "-87.65368468595476,41.9533202828916,0 "
+ "-87.65369271253692,41.95343095587119,0 "
+ "-87.65373335834569,41.95351536301472,0 "
+ "-87.65378605844126,41.95358212680591,0 "
+ "-87.65385067928185,41.95364452823767,0 "
+ "-87.6539390793817,41.95370263886964,0 "
+ "-87.6540786298351,41.95373403675265,0 "
+ "-87.65430648647626,41.9537535411832,0 "
+ "-87.65492939166126,41.95377494531437,0"
+ ),
+ 4: (
+ "-87.65345391792157,41.94217681262115,0 "
+ "-87.65342448305786,41.94237224420864,0 "
+ "-87.65339745703922,41.94268217746244,0 "
+ "-87.65337753982941,41.94288140770284,0 "
+ "-87.65336256753105,41.94317369618263,0 "
+ "-87.65338799707138,41.94357253961736,0 "
+ "-87.65340240886648,41.94389158188269,0 "
+ "-87.65341837392448,41.94406444407721,0 "
+ "-87.65342275247338,41.94421065714904,0 "
+ "-87.65347469646018,41.94434829382345,0 "
+ "-87.65351486483024,41.94447699917548,0 "
+ "-87.65353483605053,41.9453896864472,0 "
+ "-87.65361975532807,41.94689193720703,0 "
+ "-87.65362593118043,41.94742799535678,0"
+ ),
+ },
+ },
+ index=range(5),
+)
+
+
+def test_literal_xml_raises():
+ # GH 53809
+ pytest.importorskip("lxml")
+ msg = "|".join([r".*No such file or directory", r".*Invalid argument"])
+
+ with pytest.raises((FileNotFoundError, OSError), match=msg):
+ read_xml(xml_default_nmsp)
+
+
+@pytest.fixture(params=["rb", "r"])
+def mode(request):
+ return request.param
+
+
+@pytest.fixture(params=[pytest.param("lxml", marks=td.skip_if_no("lxml")), "etree"])
+def parser(request):
+ return request.param
+
+
+def read_xml_iterparse(data, temp_path, **kwargs):
+ temp_path.write_text(data, encoding="utf-8")
+ return read_xml(temp_path, **kwargs)
+
+
+def read_xml_iterparse_comp(comp_path, compression_only, temp_path, **kwargs):
+ with get_handle(comp_path, "r", compression=compression_only) as handles:
+ temp_path.write_text(handles.handle.read(), encoding="utf-8")
+ return read_xml(temp_path, **kwargs)
+
+
+# FILE / URL
+
+
+def test_parser_consistency_file(xml_books):
+ pytest.importorskip("lxml")
+ df_file_lxml = read_xml(xml_books, parser="lxml")
+ df_file_etree = read_xml(xml_books, parser="etree")
+
+ df_iter_lxml = read_xml(
+ xml_books,
+ parser="lxml",
+ iterparse={"book": ["category", "title", "year", "author", "price"]},
+ )
+ df_iter_etree = read_xml(
+ xml_books,
+ parser="etree",
+ iterparse={"book": ["category", "title", "year", "author", "price"]},
+ )
+
+ tm.assert_frame_equal(df_file_lxml, df_file_etree)
+ tm.assert_frame_equal(df_file_lxml, df_iter_lxml)
+ tm.assert_frame_equal(df_iter_lxml, df_iter_etree)
+
+
+@pytest.mark.network
+@pytest.mark.single_cpu
+def test_parser_consistency_url(parser, httpserver):
+ httpserver.serve_content(content=xml_default_nmsp)
+
+ df_xpath = read_xml(StringIO(xml_default_nmsp), parser=parser)
+ df_iter = read_xml(
+ BytesIO(xml_default_nmsp.encode()),
+ parser=parser,
+ iterparse={"row": ["shape", "degrees", "sides"]},
+ )
+
+ tm.assert_frame_equal(df_xpath, df_iter)
+
+
+def test_file_like(xml_books, parser, mode):
+ with open(xml_books, mode, encoding="utf-8" if mode == "r" else None) as f:
+ df_file = read_xml(f, parser=parser)
+
+ df_expected = DataFrame(
+ {
+ "category": ["cooking", "children", "web"],
+ "title": ["Everyday Italian", "Harry Potter", "Learning XML"],
+ "author": ["Giada De Laurentiis", "J K. Rowling", "Erik T. Ray"],
+ "year": [2005, 2005, 2003],
+ "price": [30.00, 29.99, 39.95],
+ }
+ )
+
+ tm.assert_frame_equal(df_file, df_expected)
+
+
+def test_file_io(xml_books, parser, mode):
+ with open(xml_books, mode, encoding="utf-8" if mode == "r" else None) as f:
+ xml_obj = f.read()
+
+ df_io = read_xml(
+ (BytesIO(xml_obj) if isinstance(xml_obj, bytes) else StringIO(xml_obj)),
+ parser=parser,
+ )
+
+ df_expected = DataFrame(
+ {
+ "category": ["cooking", "children", "web"],
+ "title": ["Everyday Italian", "Harry Potter", "Learning XML"],
+ "author": ["Giada De Laurentiis", "J K. Rowling", "Erik T. Ray"],
+ "year": [2005, 2005, 2003],
+ "price": [30.00, 29.99, 39.95],
+ }
+ )
+
+ tm.assert_frame_equal(df_io, df_expected)
+
+
+def test_file_buffered_reader_string(xml_books, parser, mode):
+ with open(xml_books, mode, encoding="utf-8" if mode == "r" else None) as f:
+ xml_obj = f.read()
+
+ if mode == "rb":
+ xml_obj = StringIO(xml_obj.decode())
+ elif mode == "r":
+ xml_obj = StringIO(xml_obj)
+
+ df_str = read_xml(xml_obj, parser=parser)
+
+ df_expected = DataFrame(
+ {
+ "category": ["cooking", "children", "web"],
+ "title": ["Everyday Italian", "Harry Potter", "Learning XML"],
+ "author": ["Giada De Laurentiis", "J K. Rowling", "Erik T. Ray"],
+ "year": [2005, 2005, 2003],
+ "price": [30.00, 29.99, 39.95],
+ }
+ )
+
+ tm.assert_frame_equal(df_str, df_expected)
+
+
+def test_file_buffered_reader_no_xml_declaration(xml_books, parser, mode):
+ with open(xml_books, mode, encoding="utf-8" if mode == "r" else None) as f:
+ next(f)
+ xml_obj = f.read()
+
+ if mode == "rb":
+ xml_obj = StringIO(xml_obj.decode())
+ elif mode == "r":
+ xml_obj = StringIO(xml_obj)
+
+ df_str = read_xml(xml_obj, parser=parser)
+
+ df_expected = DataFrame(
+ {
+ "category": ["cooking", "children", "web"],
+ "title": ["Everyday Italian", "Harry Potter", "Learning XML"],
+ "author": ["Giada De Laurentiis", "J K. Rowling", "Erik T. Ray"],
+ "year": [2005, 2005, 2003],
+ "price": [30.00, 29.99, 39.95],
+ }
+ )
+
+ tm.assert_frame_equal(df_str, df_expected)
+
+
+def test_string_charset(parser):
+ txt = "<中文標籤>1 2
中文標籤>"
+
+ df_str = read_xml(StringIO(txt), parser=parser)
+
+ df_expected = DataFrame({"c1": 1, "c2": 2}, index=range(1))
+
+ tm.assert_frame_equal(df_str, df_expected)
+
+
+def test_file_charset(xml_doc_ch_utf, parser):
+ df_file = read_xml(xml_doc_ch_utf, parser=parser)
+
+ df_expected = DataFrame(
+ {
+ "問": [
+ "問 若箇是邪而言破邪 何者是正而道(Sorry, this is Big5 only)申正",
+ "問 既破有得申無得 亦應但破性執申假名以不",
+ "問 既破性申假 亦應但破有申無 若有無兩洗 亦應性假雙破耶",
+ ],
+ "答": [
+ "".join(
+ [
+ "答 邪既無量 正亦多途 大略為言不出二種 謂",
+ "有得與無得 有得是邪須破 無得是正須申\n\t\t故",
+ ]
+ ),
+ None,
+ "答 不例 有無皆是性 所以須雙破 既分性假異 故有破不破",
+ ],
+ "a": [
+ None,
+ "答 性執是有得 假名是無得 今破有得申無得 即是破性執申假名也",
+ None,
+ ],
+ }
+ )
+
+ tm.assert_frame_equal(df_file, df_expected)
+
+
+def test_file_handle_close(xml_books, parser):
+ with open(xml_books, "rb") as f:
+ read_xml(BytesIO(f.read()), parser=parser)
+
+ assert not f.closed
+
+
+@pytest.mark.parametrize("val", ["", b""])
+def test_empty_string_lxml(val):
+ lxml_etree = pytest.importorskip("lxml.etree")
+
+ msg = "|".join(
+ [
+ "Document is empty",
+ # Seen on Mac with lxml 4.91
+ r"None \(line 0\)",
+ ]
+ )
+ if isinstance(val, str):
+ data = StringIO(val)
+ else:
+ data = BytesIO(val)
+ with pytest.raises(lxml_etree.XMLSyntaxError, match=msg):
+ read_xml(data, parser="lxml")
+
+
+@pytest.mark.parametrize("val", ["", b""])
+def test_empty_string_etree(val):
+ if isinstance(val, str):
+ data = StringIO(val)
+ else:
+ data = BytesIO(val)
+ with pytest.raises(ParseError, match="no element found"):
+ read_xml(data, parser="etree")
+
+
+@pytest.mark.skipif(WASM, reason="limited file system access on WASM")
+def test_wrong_file_path(parser):
+ filename = os.path.join("does", "not", "exist", "books.xml")
+
+ with pytest.raises(
+ FileNotFoundError, match=r"\[Errno 2\] No such file or directory"
+ ):
+ read_xml(filename, parser=parser)
+
+
+@pytest.mark.network
+@pytest.mark.single_cpu
+def test_url(httpserver, xml_file):
+ pytest.importorskip("lxml")
+ with open(xml_file, encoding="utf-8") as f:
+ httpserver.serve_content(content=f.read())
+ df_url = read_xml(httpserver.url, xpath=".//book[count(*)=4]")
+
+ df_expected = DataFrame(
+ {
+ "category": ["cooking", "children", "web"],
+ "title": ["Everyday Italian", "Harry Potter", "Learning XML"],
+ "author": ["Giada De Laurentiis", "J K. Rowling", "Erik T. Ray"],
+ "year": [2005, 2005, 2003],
+ "price": [30.00, 29.99, 39.95],
+ }
+ )
+
+ tm.assert_frame_equal(df_url, df_expected)
+
+
+@pytest.mark.network
+@pytest.mark.single_cpu
+def test_wrong_url(parser, httpserver):
+ httpserver.serve_content("NOT FOUND", code=404)
+ try:
+ with pytest.raises(HTTPError, match=("HTTP Error 404: NOT FOUND")) as err:
+ read_xml(httpserver.url, xpath=".//book[count(*)=4]", parser=parser)
+ finally:
+ if isinstance(err.value, HTTPError):
+ # Has a file-like handle that we can close
+ # https://docs.python.org/3/library/urllib.error.html#urllib.error.HTTPError
+ err.value.close()
+
+
+# CONTENT
+
+
+def test_whitespace(parser, temp_file):
+ xml = """
+
+
+
+ square
+
+ 360
+
+
+
+ circle
+
+ 360
+
+
+
+ triangle
+
+ 180
+
+ """
+
+ df_xpath = read_xml(StringIO(xml), parser=parser, dtype="string")
+
+ df_iter = read_xml_iterparse(
+ xml,
+ temp_file,
+ parser=parser,
+ iterparse={"row": ["sides", "shape", "degrees"]},
+ dtype="string",
+ )
+
+ df_expected = DataFrame(
+ {
+ "sides": [" 4 ", " 0 ", " 3 "],
+ "shape": [
+ "\n square\n ",
+ "\n circle\n ",
+ "\n triangle\n ",
+ ],
+ "degrees": ["\t360\t", "\t360\t", "\t180\t"],
+ },
+ dtype="string",
+ )
+
+ tm.assert_frame_equal(df_xpath, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+# XPATH
+
+
+def test_empty_xpath_lxml(xml_books):
+ pytest.importorskip("lxml")
+ with pytest.raises(ValueError, match=("xpath does not return any nodes")):
+ read_xml(xml_books, xpath=".//python", parser="lxml")
+
+
+def test_bad_xpath_etree(xml_books):
+ with pytest.raises(
+ SyntaxError, match=("You have used an incorrect or unsupported XPath")
+ ):
+ read_xml(xml_books, xpath=".//[book]", parser="etree")
+
+
+def test_bad_xpath_lxml(xml_books):
+ lxml_etree = pytest.importorskip("lxml.etree")
+
+ with pytest.raises(lxml_etree.XPathEvalError, match=("Invalid expression")):
+ read_xml(xml_books, xpath=".//[book]", parser="lxml")
+
+
+# NAMESPACE
+
+
+def test_default_namespace(parser, temp_file):
+ df_nmsp = read_xml(
+ StringIO(xml_default_nmsp),
+ xpath=".//ns:row",
+ namespaces={"ns": "http://example.com"},
+ parser=parser,
+ )
+
+ df_iter = read_xml_iterparse(
+ xml_default_nmsp,
+ temp_file,
+ parser=parser,
+ iterparse={"row": ["shape", "degrees", "sides"]},
+ )
+
+ df_expected = DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": [360, 360, 180],
+ "sides": [4.0, float("nan"), 3.0],
+ }
+ )
+
+ tm.assert_frame_equal(df_nmsp, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_prefix_namespace(parser, temp_file):
+ df_nmsp = read_xml(
+ StringIO(xml_prefix_nmsp),
+ xpath=".//doc:row",
+ namespaces={"doc": "http://example.com"},
+ parser=parser,
+ )
+ df_iter = read_xml_iterparse(
+ xml_prefix_nmsp,
+ temp_file,
+ parser=parser,
+ iterparse={"row": ["shape", "degrees", "sides"]},
+ )
+
+ df_expected = DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": [360, 360, 180],
+ "sides": [4.0, float("nan"), 3.0],
+ }
+ )
+
+ tm.assert_frame_equal(df_nmsp, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_consistency_default_namespace():
+ pytest.importorskip("lxml")
+ df_lxml = read_xml(
+ StringIO(xml_default_nmsp),
+ xpath=".//ns:row",
+ namespaces={"ns": "http://example.com"},
+ parser="lxml",
+ )
+
+ df_etree = read_xml(
+ StringIO(xml_default_nmsp),
+ xpath=".//doc:row",
+ namespaces={"doc": "http://example.com"},
+ parser="etree",
+ )
+
+ tm.assert_frame_equal(df_lxml, df_etree)
+
+
+def test_consistency_prefix_namespace():
+ pytest.importorskip("lxml")
+ df_lxml = read_xml(
+ StringIO(xml_prefix_nmsp),
+ xpath=".//doc:row",
+ namespaces={"doc": "http://example.com"},
+ parser="lxml",
+ )
+
+ df_etree = read_xml(
+ StringIO(xml_prefix_nmsp),
+ xpath=".//doc:row",
+ namespaces={"doc": "http://example.com"},
+ parser="etree",
+ )
+
+ tm.assert_frame_equal(df_lxml, df_etree)
+
+
+# PREFIX
+
+
+def test_missing_prefix_with_default_namespace(xml_books, parser):
+ with pytest.raises(ValueError, match=("xpath does not return any nodes")):
+ read_xml(xml_books, xpath=".//Placemark", parser=parser)
+
+
+def test_missing_prefix_definition_etree(kml_cta_rail_lines):
+ with pytest.raises(SyntaxError, match=("you used an undeclared namespace prefix")):
+ read_xml(kml_cta_rail_lines, xpath=".//kml:Placemark", parser="etree")
+
+
+def test_missing_prefix_definition_lxml(kml_cta_rail_lines):
+ lxml_etree = pytest.importorskip("lxml.etree")
+
+ with pytest.raises(lxml_etree.XPathEvalError, match=("Undefined namespace prefix")):
+ read_xml(kml_cta_rail_lines, xpath=".//kml:Placemark", parser="lxml")
+
+
+@pytest.mark.parametrize("key", ["", None])
+def test_none_namespace_prefix(key):
+ pytest.importorskip("lxml")
+ with pytest.raises(
+ TypeError, match=("empty namespace prefix is not supported in XPath")
+ ):
+ read_xml(
+ StringIO(xml_default_nmsp),
+ xpath=".//kml:Placemark",
+ namespaces={key: "http://www.opengis.net/kml/2.2"},
+ parser="lxml",
+ )
+
+
+# ELEMS AND ATTRS
+
+
+def test_file_elems_and_attrs(xml_books, parser):
+ df_file = read_xml(xml_books, parser=parser)
+ df_iter = read_xml(
+ xml_books,
+ parser=parser,
+ iterparse={"book": ["category", "title", "author", "year", "price"]},
+ )
+ df_expected = DataFrame(
+ {
+ "category": ["cooking", "children", "web"],
+ "title": ["Everyday Italian", "Harry Potter", "Learning XML"],
+ "author": ["Giada De Laurentiis", "J K. Rowling", "Erik T. Ray"],
+ "year": [2005, 2005, 2003],
+ "price": [30.00, 29.99, 39.95],
+ }
+ )
+
+ tm.assert_frame_equal(df_file, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_file_only_attrs(xml_books, parser):
+ df_file = read_xml(xml_books, attrs_only=True, parser=parser)
+ df_iter = read_xml(xml_books, parser=parser, iterparse={"book": ["category"]})
+ df_expected = DataFrame({"category": ["cooking", "children", "web"]})
+
+ tm.assert_frame_equal(df_file, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_file_only_elems(xml_books, parser):
+ df_file = read_xml(xml_books, elems_only=True, parser=parser)
+ df_iter = read_xml(
+ xml_books,
+ parser=parser,
+ iterparse={"book": ["title", "author", "year", "price"]},
+ )
+ df_expected = DataFrame(
+ {
+ "title": ["Everyday Italian", "Harry Potter", "Learning XML"],
+ "author": ["Giada De Laurentiis", "J K. Rowling", "Erik T. Ray"],
+ "year": [2005, 2005, 2003],
+ "price": [30.00, 29.99, 39.95],
+ }
+ )
+
+ tm.assert_frame_equal(df_file, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_elem_and_attrs_only(kml_cta_rail_lines, parser):
+ with pytest.raises(
+ ValueError,
+ match=("Either element or attributes can be parsed not both"),
+ ):
+ read_xml(kml_cta_rail_lines, elems_only=True, attrs_only=True, parser=parser)
+
+
+def test_empty_attrs_only(parser):
+ xml = """
+
+
+ square
+ 360
+
+
+ circle
+ 360
+
+
+ triangle
+ 180
+
+ """
+
+ with pytest.raises(
+ ValueError,
+ match=("xpath does not return any nodes or attributes"),
+ ):
+ read_xml(StringIO(xml), xpath="./row", attrs_only=True, parser=parser)
+
+
+def test_empty_elems_only(parser):
+ xml = """
+
+
+
+
+ """
+
+ with pytest.raises(
+ ValueError,
+ match=("xpath does not return any nodes or attributes"),
+ ):
+ read_xml(StringIO(xml), xpath="./row", elems_only=True, parser=parser)
+
+
+def test_attribute_centric_xml(temp_file):
+ pytest.importorskip("lxml")
+ xml = """\
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ """
+
+ df_lxml = read_xml(StringIO(xml), xpath=".//station")
+ df_etree = read_xml(StringIO(xml), xpath=".//station", parser="etree")
+
+ df_iter_lx = read_xml_iterparse(
+ xml, temp_file, iterparse={"station": ["Name", "coords"]}
+ )
+ df_iter_et = read_xml_iterparse(
+ xml, temp_file, parser="etree", iterparse={"station": ["Name", "coords"]}
+ )
+
+ tm.assert_frame_equal(df_lxml, df_etree)
+ tm.assert_frame_equal(df_iter_lx, df_iter_et)
+
+
+# NAMES
+
+
+def test_names_option_output(xml_books, parser):
+ df_file = read_xml(
+ xml_books, names=["Col1", "Col2", "Col3", "Col4", "Col5"], parser=parser
+ )
+ df_iter = read_xml(
+ xml_books,
+ parser=parser,
+ names=["Col1", "Col2", "Col3", "Col4", "Col5"],
+ iterparse={"book": ["category", "title", "author", "year", "price"]},
+ )
+
+ df_expected = DataFrame(
+ {
+ "Col1": ["cooking", "children", "web"],
+ "Col2": ["Everyday Italian", "Harry Potter", "Learning XML"],
+ "Col3": ["Giada De Laurentiis", "J K. Rowling", "Erik T. Ray"],
+ "Col4": [2005, 2005, 2003],
+ "Col5": [30.00, 29.99, 39.95],
+ }
+ )
+
+ tm.assert_frame_equal(df_file, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_repeat_names(parser, temp_file):
+ xml = """\
+
+
+ circle
+ curved
+
+
+ sphere
+ curved
+
+ """
+ df_xpath = read_xml(
+ StringIO(xml),
+ xpath=".//shape",
+ parser=parser,
+ names=["type_dim", "shape", "type_edge"],
+ )
+
+ df_iter = read_xml_iterparse(
+ xml,
+ temp_file,
+ parser=parser,
+ iterparse={"shape": ["type", "name", "type"]},
+ names=["type_dim", "shape", "type_edge"],
+ )
+
+ df_expected = DataFrame(
+ {
+ "type_dim": ["2D", "3D"],
+ "shape": ["circle", "sphere"],
+ "type_edge": ["curved", "curved"],
+ }
+ )
+
+ tm.assert_frame_equal(df_xpath, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_repeat_values_new_names(parser, temp_file):
+ xml = """\
+
+
+ rectangle
+ rectangle
+
+
+ square
+ rectangle
+
+
+ ellipse
+ ellipse
+
+
+ circle
+ ellipse
+
+ """
+ df_xpath = read_xml(
+ StringIO(xml), xpath=".//shape", parser=parser, names=["name", "group"]
+ )
+
+ df_iter = read_xml_iterparse(
+ xml,
+ temp_file,
+ parser=parser,
+ iterparse={"shape": ["name", "family"]},
+ names=["name", "group"],
+ )
+
+ df_expected = DataFrame(
+ {
+ "name": ["rectangle", "square", "ellipse", "circle"],
+ "group": ["rectangle", "rectangle", "ellipse", "ellipse"],
+ }
+ )
+
+ tm.assert_frame_equal(df_xpath, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_repeat_elements(parser, temp_file):
+ xml = """\
+
+
+ circle
+ ellipse
+ 360
+ 0
+
+
+ triangle
+ polygon
+ 180
+ 3
+
+
+ square
+ polygon
+ 360
+ 4
+
+ """
+ df_xpath = read_xml(
+ StringIO(xml),
+ xpath=".//shape",
+ parser=parser,
+ names=["name", "family", "degrees", "sides"],
+ )
+
+ df_iter = read_xml_iterparse(
+ xml,
+ temp_file,
+ parser=parser,
+ iterparse={"shape": ["value", "value", "value", "value"]},
+ names=["name", "family", "degrees", "sides"],
+ )
+
+ df_expected = DataFrame(
+ {
+ "name": ["circle", "triangle", "square"],
+ "family": ["ellipse", "polygon", "polygon"],
+ "degrees": [360, 180, 360],
+ "sides": [0, 3, 4],
+ }
+ )
+
+ tm.assert_frame_equal(df_xpath, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_names_option_wrong_length(xml_books, parser):
+ with pytest.raises(ValueError, match=("names does not match length")):
+ read_xml(xml_books, names=["Col1", "Col2", "Col3"], parser=parser)
+
+
+def test_names_option_wrong_type(xml_books, parser):
+ with pytest.raises(TypeError, match=("is not a valid type for names")):
+ read_xml(xml_books, names="Col1, Col2, Col3", parser=parser)
+
+
+# ENCODING
+
+
+def test_wrong_encoding(xml_baby_names, parser):
+ with pytest.raises(UnicodeDecodeError, match=("'utf-8' codec can't decode")):
+ read_xml(xml_baby_names, parser=parser)
+
+
+def test_utf16_encoding(xml_baby_names, parser):
+ with pytest.raises(
+ UnicodeError,
+ match=(
+ "UTF-16 stream does not start with BOM|"
+ "'utf-16(-le)?' codec can't decode byte"
+ ),
+ ):
+ read_xml(xml_baby_names, encoding="UTF-16", parser=parser)
+
+
+def test_unknown_encoding(xml_baby_names, parser):
+ with pytest.raises(LookupError, match=("unknown encoding: UFT-8")):
+ read_xml(xml_baby_names, encoding="UFT-8", parser=parser)
+
+
+def test_ascii_encoding(xml_baby_names, parser):
+ with pytest.raises(UnicodeDecodeError, match=("'ascii' codec can't decode byte")):
+ read_xml(xml_baby_names, encoding="ascii", parser=parser)
+
+
+def test_parser_consistency_with_encoding(xml_baby_names):
+ pytest.importorskip("lxml")
+ df_xpath_lxml = read_xml(xml_baby_names, parser="lxml", encoding="ISO-8859-1")
+ df_xpath_etree = read_xml(xml_baby_names, parser="etree", encoding="iso-8859-1")
+
+ df_iter_lxml = read_xml(
+ xml_baby_names,
+ parser="lxml",
+ encoding="ISO-8859-1",
+ iterparse={"row": ["rank", "malename", "femalename"]},
+ )
+ df_iter_etree = read_xml(
+ xml_baby_names,
+ parser="etree",
+ encoding="ISO-8859-1",
+ iterparse={"row": ["rank", "malename", "femalename"]},
+ )
+
+ tm.assert_frame_equal(df_xpath_lxml, df_xpath_etree)
+ tm.assert_frame_equal(df_xpath_etree, df_iter_etree)
+ tm.assert_frame_equal(df_iter_lxml, df_iter_etree)
+
+
+def test_wrong_encoding_for_lxml():
+ pytest.importorskip("lxml")
+ # GH#45133
+ data = """
+
+ c
+
+
+"""
+ with pytest.raises(TypeError, match="encoding None"):
+ read_xml(StringIO(data), parser="lxml", encoding=None)
+
+
+def test_none_encoding_etree():
+ # GH#45133
+ data = """
+
+ c
+
+
+"""
+ result = read_xml(StringIO(data), parser="etree", encoding=None)
+ expected = DataFrame({"a": ["c"]})
+ tm.assert_frame_equal(result, expected)
+
+
+# PARSER
+
+
+@td.skip_if_installed("lxml")
+def test_default_parser_no_lxml(xml_books):
+ with pytest.raises(
+ ImportError, match=("lxml not found, please install or use the etree parser.")
+ ):
+ read_xml(xml_books)
+
+
+def test_wrong_parser(xml_books):
+ with pytest.raises(
+ ValueError, match=("Values for parser can only be lxml or etree.")
+ ):
+ read_xml(xml_books, parser="bs4")
+
+
+# STYLESHEET
+
+
+def test_stylesheet_file(kml_cta_rail_lines, xsl_flatten_doc):
+ pytest.importorskip("lxml")
+ df_style = read_xml(
+ kml_cta_rail_lines,
+ xpath=".//k:Placemark",
+ namespaces={"k": "http://www.opengis.net/kml/2.2"},
+ stylesheet=xsl_flatten_doc,
+ )
+
+ df_iter = read_xml(
+ kml_cta_rail_lines,
+ iterparse={
+ "Placemark": [
+ "id",
+ "name",
+ "styleUrl",
+ "extrude",
+ "altitudeMode",
+ "coordinates",
+ ]
+ },
+ )
+
+ tm.assert_frame_equal(df_kml, df_style)
+ tm.assert_frame_equal(df_kml, df_iter)
+
+
+def test_stylesheet_file_like(kml_cta_rail_lines, xsl_flatten_doc, mode):
+ pytest.importorskip("lxml")
+ with open(xsl_flatten_doc, mode, encoding="utf-8" if mode == "r" else None) as f:
+ df_style = read_xml(
+ kml_cta_rail_lines,
+ xpath=".//k:Placemark",
+ namespaces={"k": "http://www.opengis.net/kml/2.2"},
+ stylesheet=f,
+ )
+
+ tm.assert_frame_equal(df_kml, df_style)
+
+
+def test_stylesheet_io(kml_cta_rail_lines, xsl_flatten_doc, mode):
+ # note: By default the bodies of untyped functions are not checked,
+ # consider using --check-untyped-defs
+ pytest.importorskip("lxml")
+ xsl_obj: BytesIO | StringIO # type: ignore[annotation-unchecked]
+
+ with open(xsl_flatten_doc, mode, encoding="utf-8" if mode == "r" else None) as f:
+ if mode == "rb":
+ xsl_obj = BytesIO(f.read())
+ else:
+ xsl_obj = StringIO(f.read())
+
+ df_style = read_xml(
+ kml_cta_rail_lines,
+ xpath=".//k:Placemark",
+ namespaces={"k": "http://www.opengis.net/kml/2.2"},
+ stylesheet=xsl_obj,
+ )
+
+ tm.assert_frame_equal(df_kml, df_style)
+
+
+def test_stylesheet_buffered_reader(kml_cta_rail_lines, xsl_flatten_doc, mode):
+ pytest.importorskip("lxml")
+ with open(xsl_flatten_doc, mode, encoding="utf-8" if mode == "r" else None) as f:
+ df_style = read_xml(
+ kml_cta_rail_lines,
+ xpath=".//k:Placemark",
+ namespaces={"k": "http://www.opengis.net/kml/2.2"},
+ stylesheet=f,
+ )
+
+ tm.assert_frame_equal(df_kml, df_style)
+
+
+def test_style_charset():
+ pytest.importorskip("lxml")
+ xml = "<中文標籤>1 2
中文標籤>"
+
+ xsl = """\
+
+
+
+
+
+
+
+
+
+
+
+ <根>
+
+ 根>
+
+
+ """
+
+ df_orig = read_xml(StringIO(xml))
+ df_style = read_xml(StringIO(xml), stylesheet=StringIO(xsl))
+
+ tm.assert_frame_equal(df_orig, df_style)
+
+
+def test_not_stylesheet(kml_cta_rail_lines, xml_books):
+ lxml_etree = pytest.importorskip("lxml.etree")
+
+ with pytest.raises(
+ lxml_etree.XSLTParseError, match=("document is not a stylesheet")
+ ):
+ read_xml(kml_cta_rail_lines, stylesheet=xml_books)
+
+
+def test_incorrect_xsl_syntax(kml_cta_rail_lines):
+ lxml_etree = pytest.importorskip("lxml.etree")
+
+ xsl = """\
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ """
+
+ with pytest.raises(
+ lxml_etree.XMLSyntaxError, match="Extra content at the end of the document"
+ ):
+ read_xml(kml_cta_rail_lines, stylesheet=StringIO(xsl))
+
+
+def test_incorrect_xsl_eval(kml_cta_rail_lines):
+ lxml_etree = pytest.importorskip("lxml.etree")
+
+ xsl = """\
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ """
+
+ with pytest.raises(lxml_etree.XSLTParseError, match="failed to compile"):
+ read_xml(kml_cta_rail_lines, stylesheet=StringIO(xsl))
+
+
+def test_incorrect_xsl_apply(kml_cta_rail_lines):
+ lxml_etree = pytest.importorskip("lxml.etree")
+
+ xsl = """\
+
+
+
+
+
+
+
+
+
+ """
+
+ with pytest.raises(lxml_etree.XSLTApplyError, match="Cannot resolve URI"):
+ read_xml(kml_cta_rail_lines, stylesheet=StringIO(xsl))
+
+
+def test_wrong_stylesheet(kml_cta_rail_lines, xml_data_path):
+ pytest.importorskip("lxml.etree")
+
+ xsl = xml_data_path / "flatten_doesnt_exist.xsl"
+
+ with pytest.raises(
+ FileNotFoundError, match=r"\[Errno 2\] No such file or directory"
+ ):
+ read_xml(kml_cta_rail_lines, stylesheet=xsl)
+
+
+def test_stylesheet_file_close(kml_cta_rail_lines, xsl_flatten_doc, mode):
+ # note: By default the bodies of untyped functions are not checked,
+ # consider using --check-untyped-defs
+ pytest.importorskip("lxml")
+ xsl_obj: BytesIO | StringIO # type: ignore[annotation-unchecked]
+
+ with open(xsl_flatten_doc, mode, encoding="utf-8" if mode == "r" else None) as f:
+ if mode == "rb":
+ xsl_obj = BytesIO(f.read())
+ else:
+ xsl_obj = StringIO(f.read())
+
+ read_xml(kml_cta_rail_lines, stylesheet=xsl_obj)
+
+ assert not f.closed
+
+
+def test_stylesheet_with_etree(kml_cta_rail_lines, xsl_flatten_doc):
+ pytest.importorskip("lxml")
+ with pytest.raises(
+ ValueError, match=("To use stylesheet, you need lxml installed")
+ ):
+ read_xml(kml_cta_rail_lines, parser="etree", stylesheet=xsl_flatten_doc)
+
+
+@pytest.mark.parametrize("val", [StringIO(""), BytesIO(b"")])
+def test_empty_stylesheet(val, kml_cta_rail_lines):
+ lxml_etree = pytest.importorskip("lxml.etree")
+ with pytest.raises(lxml_etree.XMLSyntaxError):
+ read_xml(kml_cta_rail_lines, stylesheet=val)
+
+
+# ITERPARSE
+def test_file_like_iterparse(xml_books, parser, mode):
+ with open(xml_books, mode, encoding="utf-8" if mode == "r" else None) as f:
+ if mode == "r" and parser == "lxml":
+ with pytest.raises(
+ TypeError, match=("reading file objects must return bytes objects")
+ ):
+ read_xml(
+ f,
+ parser=parser,
+ iterparse={
+ "book": ["category", "title", "year", "author", "price"]
+ },
+ )
+ return None
+ else:
+ df_filelike = read_xml(
+ f,
+ parser=parser,
+ iterparse={"book": ["category", "title", "year", "author", "price"]},
+ )
+
+ df_expected = DataFrame(
+ {
+ "category": ["cooking", "children", "web"],
+ "title": ["Everyday Italian", "Harry Potter", "Learning XML"],
+ "author": ["Giada De Laurentiis", "J K. Rowling", "Erik T. Ray"],
+ "year": [2005, 2005, 2003],
+ "price": [30.00, 29.99, 39.95],
+ }
+ )
+
+ tm.assert_frame_equal(df_filelike, df_expected)
+
+
+def test_file_io_iterparse(xml_books, parser, mode):
+ funcIO = StringIO if mode == "r" else BytesIO
+ with open(
+ xml_books,
+ mode,
+ encoding="utf-8" if mode == "r" else None,
+ ) as f:
+ with funcIO(f.read()) as b:
+ if mode == "r" and parser == "lxml":
+ with pytest.raises(
+ TypeError, match=("reading file objects must return bytes objects")
+ ):
+ read_xml(
+ b,
+ parser=parser,
+ iterparse={
+ "book": ["category", "title", "year", "author", "price"]
+ },
+ )
+ return None
+ else:
+ df_fileio = read_xml(
+ b,
+ parser=parser,
+ iterparse={
+ "book": ["category", "title", "year", "author", "price"]
+ },
+ )
+
+ df_expected = DataFrame(
+ {
+ "category": ["cooking", "children", "web"],
+ "title": ["Everyday Italian", "Harry Potter", "Learning XML"],
+ "author": ["Giada De Laurentiis", "J K. Rowling", "Erik T. Ray"],
+ "year": [2005, 2005, 2003],
+ "price": [30.00, 29.99, 39.95],
+ }
+ )
+
+ tm.assert_frame_equal(df_fileio, df_expected)
+
+
+@pytest.mark.network
+@pytest.mark.single_cpu
+def test_url_path_error(parser, httpserver, xml_file):
+ with open(xml_file, encoding="utf-8") as f:
+ httpserver.serve_content(content=f.read())
+ with pytest.raises(
+ ParserError, match=("iterparse is designed for large XML files")
+ ):
+ read_xml(
+ httpserver.url,
+ parser=parser,
+ iterparse={"row": ["shape", "degrees", "sides", "date"]},
+ )
+
+
+def test_compression_error(parser, compression_only, tmp_path):
+ path = tmp_path / "geom_xml.zip"
+ geom_df.to_xml(path, parser=parser, compression=compression_only)
+
+ with pytest.raises(
+ ParserError, match=("iterparse is designed for large XML files")
+ ):
+ read_xml(
+ path,
+ parser=parser,
+ iterparse={"row": ["shape", "degrees", "sides", "date"]},
+ compression=compression_only,
+ )
+
+
+def test_wrong_dict_type(xml_books, parser):
+ with pytest.raises(TypeError, match="list is not a valid type for iterparse"):
+ read_xml(
+ xml_books,
+ parser=parser,
+ iterparse=["category", "title", "year", "author", "price"],
+ )
+
+
+def test_wrong_dict_value(xml_books, parser):
+ with pytest.raises(
+ TypeError, match=" is not a valid type for value in iterparse"
+ ):
+ read_xml(xml_books, parser=parser, iterparse={"book": "category"})
+
+
+def test_bad_xml(parser, temp_file):
+ bad_xml = """\
+
+
+ square
+ 00360
+ 4.0
+ 2020-01-01
+
+
+ circle
+ 00360
+
+ 2021-01-01
+
+
+ triangle
+ 00180
+ 3.0
+ 2022-01-01
+
+"""
+ temp_file.write_text(bad_xml, encoding="utf-8")
+
+ with pytest.raises(
+ SyntaxError,
+ match="Extra content at the end of the document|junk after document element",
+ ):
+ read_xml(
+ temp_file,
+ parser=parser,
+ parse_dates=["date"],
+ iterparse={"row": ["shape", "degrees", "sides", "date"]},
+ )
+
+
+def test_comment(parser, temp_file):
+ xml = """\
+
+
+
+
+ circle
+ 2D
+
+
+ sphere
+ 3D
+
+
+
+
+"""
+
+ df_xpath = read_xml(StringIO(xml), xpath=".//shape", parser=parser)
+
+ df_iter = read_xml_iterparse(
+ xml, temp_file, parser=parser, iterparse={"shape": ["name", "type"]}
+ )
+
+ df_expected = DataFrame(
+ {
+ "name": ["circle", "sphere"],
+ "type": ["2D", "3D"],
+ }
+ )
+
+ tm.assert_frame_equal(df_xpath, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_dtd(parser, temp_file):
+ xml = """\
+
+
+
+
+]>
+
+
+ circle
+ 2D
+
+
+ sphere
+ 3D
+
+ """
+
+ df_xpath = read_xml(StringIO(xml), xpath=".//shape", parser=parser)
+
+ df_iter = read_xml_iterparse(
+ xml, temp_file, parser=parser, iterparse={"shape": ["name", "type"]}
+ )
+
+ df_expected = DataFrame(
+ {
+ "name": ["circle", "sphere"],
+ "type": ["2D", "3D"],
+ }
+ )
+
+ tm.assert_frame_equal(df_xpath, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_processing_instruction(parser, temp_file):
+ xml = """\
+
+
+
+
+
+, , ?>
+
+
+ circle
+ 2D
+
+
+ sphere
+ 3D
+
+ """
+
+ df_xpath = read_xml(StringIO(xml), xpath=".//shape", parser=parser)
+
+ df_iter = read_xml_iterparse(
+ xml, temp_file, parser=parser, iterparse={"shape": ["name", "type"]}
+ )
+
+ df_expected = DataFrame(
+ {
+ "name": ["circle", "sphere"],
+ "type": ["2D", "3D"],
+ }
+ )
+
+ tm.assert_frame_equal(df_xpath, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_no_result(xml_books, parser):
+ with pytest.raises(
+ ParserError, match="No result from selected items in iterparse."
+ ):
+ read_xml(
+ xml_books,
+ parser=parser,
+ iterparse={"node": ["attr1", "elem1", "elem2", "elem3"]},
+ )
+
+
+def test_empty_data(xml_books, parser):
+ with pytest.raises(EmptyDataError, match="No columns to parse from file"):
+ read_xml(
+ xml_books,
+ parser=parser,
+ iterparse={"book": ["attr1", "elem1", "elem2", "elem3"]},
+ )
+
+
+def test_online_stylesheet():
+ pytest.importorskip("lxml")
+ xml = """\
+
+
+
+ Empire Burlesque
+ Bob Dylan
+ USA
+ Columbia
+ 10.90
+ 1985
+
+
+ Hide your heart
+ Bonnie Tyler
+ UK
+ CBS Records
+ 9.90
+ 1988
+
+
+ Greatest Hits
+ Dolly Parton
+ USA
+ RCA
+ 9.90
+ 1982
+
+
+ Still got the blues
+ Gary Moore
+ UK
+ Virgin records
+ 10.20
+ 1990
+
+
+ Eros
+ Eros Ramazzotti
+ EU
+ BMG
+ 9.90
+ 1997
+
+
+ One night only
+ Bee Gees
+ UK
+ Polydor
+ 10.90
+ 1998
+
+
+ Sylvias Mother
+ Dr.Hook
+ UK
+ CBS
+ 8.10
+ 1973
+
+
+ Maggie May
+ Rod Stewart
+ UK
+ Pickwick
+ 8.50
+ 1990
+
+
+ Romanza
+ Andrea Bocelli
+ EU
+ Polydor
+ 10.80
+ 1996
+
+
+ When a man loves a woman
+ Percy Sledge
+ USA
+ Atlantic
+ 8.70
+ 1987
+
+
+ Black angel
+ Savage Rose
+ EU
+ Mega
+ 10.90
+ 1995
+
+
+ 1999 Grammy Nominees
+ Many
+ USA
+ Grammy
+ 10.20
+ 1999
+
+
+ For the good times
+ Kenny Rogers
+ UK
+ Mucik Master
+ 8.70
+ 1995
+
+
+ Big Willie style
+ Will Smith
+ USA
+ Columbia
+ 9.90
+ 1997
+
+
+ Tupelo Honey
+ Van Morrison
+ UK
+ Polydor
+ 8.20
+ 1971
+
+
+ Soulsville
+ Jorn Hoel
+ Norway
+ WEA
+ 7.90
+ 1996
+
+
+ The very best of
+ Cat Stevens
+ UK
+ Island
+ 8.90
+ 1990
+
+
+ Stop
+ Sam Brown
+ UK
+ A and M
+ 8.90
+ 1988
+
+
+ Bridge of Spies
+ T`Pau
+ UK
+ Siren
+ 7.90
+ 1987
+
+
+ Private Dancer
+ Tina Turner
+ UK
+ Capitol
+ 8.90
+ 1983
+
+
+ Midt om natten
+ Kim Larsen
+ EU
+ Medley
+ 7.80
+ 1983
+
+
+ Pavarotti Gala Concert
+ Luciano Pavarotti
+ UK
+ DECCA
+ 9.90
+ 1991
+
+
+ The dock of the bay
+ Otis Redding
+ USA
+ Stax Records
+ 7.90
+ 1968
+
+
+ Picture book
+ Simply Red
+ EU
+ Elektra
+ 7.20
+ 1985
+
+
+ Red
+ The Communards
+ UK
+ London
+ 7.80
+ 1987
+
+
+ Unchain my heart
+ Joe Cocker
+ USA
+ EMI
+ 8.20
+ 1987
+
+
+"""
+ xsl = """\
+
+
+
+
+
+ My CD Collection
+
+
+ Title
+ Artist
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+ df_xsl = read_xml(
+ StringIO(xml),
+ xpath=".//tr[td and position() <= 6]",
+ names=["title", "artist"],
+ stylesheet=StringIO(xsl),
+ )
+
+ df_expected = DataFrame(
+ {
+ "title": {
+ 0: "Empire Burlesque",
+ 1: "Hide your heart",
+ 2: "Greatest Hits",
+ 3: "Still got the blues",
+ 4: "Eros",
+ },
+ "artist": {
+ 0: "Bob Dylan",
+ 1: "Bonnie Tyler",
+ 2: "Dolly Parton",
+ 3: "Gary Moore",
+ 4: "Eros Ramazzotti",
+ },
+ }
+ )
+
+ tm.assert_frame_equal(df_expected, df_xsl)
+
+
+# COMPRESSION
+
+
+def test_compression_read(parser, compression_only, tmp_path, temp_file):
+ comp_path = tmp_path / "test.xml"
+ geom_df.to_xml(comp_path, index=False, parser=parser, compression=compression_only)
+
+ df_xpath = read_xml(comp_path, parser=parser, compression=compression_only)
+
+ df_iter = read_xml_iterparse_comp(
+ comp_path,
+ compression_only,
+ temp_file,
+ parser=parser,
+ iterparse={"row": ["shape", "degrees", "sides"]},
+ compression=compression_only,
+ )
+
+ tm.assert_frame_equal(df_xpath, geom_df)
+ tm.assert_frame_equal(df_iter, geom_df)
+
+
+def test_wrong_compression(parser, compression, compression_only, temp_file):
+ actual_compression = compression
+ attempted_compression = compression_only
+
+ if actual_compression == attempted_compression:
+ pytest.skip(f"{actual_compression} == {attempted_compression}")
+
+ errors = {
+ "bz2": (OSError, "Invalid data stream"),
+ "gzip": (OSError, "Not a gzipped file"),
+ "zip": (BadZipFile, "File is not a zip file"),
+ "tar": (ReadError, "file could not be opened successfully"),
+ }
+ zstd = import_optional_dependency("zstandard", errors="ignore")
+ if zstd is not None:
+ errors["zstd"] = (zstd.ZstdError, "Unknown frame descriptor")
+ lzma = import_optional_dependency("lzma", errors="ignore")
+ if lzma is not None:
+ errors["xz"] = (LZMAError, "Input format not supported by decoder")
+ error_cls, error_str = errors[attempted_compression]
+
+ geom_df.to_xml(temp_file, parser=parser, compression=actual_compression)
+
+ with pytest.raises(error_cls, match=error_str):
+ read_xml(temp_file, parser=parser, compression=attempted_compression)
+
+
+def test_unsupported_compression(parser, temp_file):
+ with pytest.raises(ValueError, match="Unrecognized compression type"):
+ read_xml(temp_file, parser=parser, compression="7z")
+
+
+# STORAGE OPTIONS
+
+
+@pytest.mark.network
+@pytest.mark.single_cpu
+def test_s3_parser_consistency(s3_bucket_public_with_data, s3so):
+ pytest.importorskip("s3fs")
+ pytest.importorskip("lxml")
+ s3 = f"s3://{s3_bucket_public_with_data.name}/books.xml"
+ df_lxml = read_xml(s3, parser="lxml", storage_options=s3so)
+
+ df_etree = read_xml(s3, parser="etree", storage_options=s3so)
+
+ tm.assert_frame_equal(df_lxml, df_etree)
+
+
+def test_read_xml_nullable_dtypes(
+ parser, string_storage, dtype_backend, using_infer_string
+):
+ # GH#50500
+ data = """
+
+
+ x
+ 1
+ 4.0
+ x
+ 2
+ 4.0
+
+ True
+ False
+
+
+ y
+ 2
+ 5.0
+
+
+
+
+ False
+
+
+ """
+
+ with pd.option_context("mode.string_storage", string_storage):
+ result = read_xml(StringIO(data), parser=parser, dtype_backend=dtype_backend)
+
+ if dtype_backend == "pyarrow":
+ pa = pytest.importorskip("pyarrow")
+ string_dtype = pd.ArrowDtype(pa.string())
+ else:
+ string_dtype = pd.StringDtype(string_storage)
+
+ expected = DataFrame(
+ {
+ "a": Series(["x", "y"], dtype=string_dtype),
+ "b": Series([1, 2], dtype="Int64"),
+ "c": Series([4.0, 5.0], dtype="Float64"),
+ "d": Series(["x", None], dtype=string_dtype),
+ "e": Series([2, NA], dtype="Int64"),
+ "f": Series([4.0, NA], dtype="Float64"),
+ "g": Series([NA, NA], dtype="Int64"),
+ "h": Series([True, False], dtype="boolean"),
+ "i": Series([False, NA], dtype="boolean"),
+ }
+ )
+
+ if dtype_backend == "pyarrow":
+ pa = pytest.importorskip("pyarrow")
+ from pandas.arrays import ArrowExtensionArray
+
+ expected = DataFrame(
+ {
+ col: ArrowExtensionArray(pa.array(expected[col], from_pandas=True))
+ for col in expected.columns
+ }
+ )
+ expected["g"] = ArrowExtensionArray(pa.array([None, None]))
+
+ # the storage of the str columns' Index is also affected by the
+ # string_storage setting -> ignore that for checking the result
+ tm.assert_frame_equal(result, expected, check_column_type=False)
+
+
+def test_invalid_dtype_backend():
+ msg = (
+ "dtype_backend numpy is invalid, only 'numpy_nullable' and "
+ "'pyarrow' are allowed."
+ )
+ with pytest.raises(ValueError, match=msg):
+ read_xml("test", dtype_backend="numpy")
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/io/xml/test_xml_dtypes.py b/.venv/lib/python3.13/site-packages/pandas/tests/io/xml/test_xml_dtypes.py
new file mode 100644
index 0000000000000000000000000000000000000000..825bc3a7f9f96ee2d9c112f9609e04911000759f
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/io/xml/test_xml_dtypes.py
@@ -0,0 +1,443 @@
+from __future__ import annotations
+
+from io import StringIO
+
+import pytest
+
+from pandas.errors import ParserWarning
+import pandas.util._test_decorators as td
+
+from pandas import (
+ NA,
+ DataFrame,
+ DatetimeIndex,
+ Series,
+ to_datetime,
+)
+import pandas._testing as tm
+
+from pandas.io.xml import read_xml
+
+
+@pytest.fixture(params=[pytest.param("lxml", marks=td.skip_if_no("lxml")), "etree"])
+def parser(request):
+ return request.param
+
+
+@pytest.fixture(
+ params=[None, {"book": ["category", "title", "author", "year", "price"]}]
+)
+def iterparse(request):
+ return request.param
+
+
+def read_xml_iterparse(data, temp_file, **kwargs):
+ with open(temp_file, "w", encoding="utf-8") as f:
+ f.write(data)
+ return read_xml(temp_file, **kwargs)
+
+
+xml_types = """\
+
+
+
+ square
+ 00360
+ 4.0
+
+
+ circle
+ 00360
+
+
+
+ triangle
+ 00180
+ 3.0
+
+ """
+
+xml_dates = """
+
+
+ square
+ 00360
+ 4.0
+ 2020-01-01
+
+
+ circle
+ 00360
+
+ 2021-01-01
+
+
+ triangle
+ 00180
+ 3.0
+ 2022-01-01
+
+ """
+
+
+# DTYPE
+
+
+def test_dtype_single_str(parser, temp_file):
+ df_result = read_xml(StringIO(xml_types), dtype={"degrees": "str"}, parser=parser)
+ df_iter = read_xml_iterparse(
+ xml_types,
+ parser=parser,
+ dtype={"degrees": "str"},
+ iterparse={"row": ["shape", "degrees", "sides"]},
+ temp_file=temp_file,
+ )
+
+ df_expected = DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": ["00360", "00360", "00180"],
+ "sides": [4.0, float("nan"), 3.0],
+ }
+ )
+
+ tm.assert_frame_equal(df_result, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_dtypes_all_str(parser, temp_file):
+ df_result = read_xml(StringIO(xml_dates), dtype="string", parser=parser)
+ df_iter = read_xml_iterparse(
+ xml_dates,
+ parser=parser,
+ dtype="string",
+ iterparse={"row": ["shape", "degrees", "sides", "date"]},
+ temp_file=temp_file,
+ )
+
+ df_expected = DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": ["00360", "00360", "00180"],
+ "sides": ["4.0", None, "3.0"],
+ "date": ["2020-01-01", "2021-01-01", "2022-01-01"],
+ },
+ dtype="string",
+ )
+
+ tm.assert_frame_equal(df_result, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_dtypes_with_names(parser, temp_file):
+ df_result = read_xml(
+ StringIO(xml_dates),
+ names=["Col1", "Col2", "Col3", "Col4"],
+ dtype={"Col2": "string", "Col3": "Int64", "Col4": "datetime64[ns]"},
+ parser=parser,
+ )
+ df_iter = read_xml_iterparse(
+ xml_dates,
+ parser=parser,
+ names=["Col1", "Col2", "Col3", "Col4"],
+ dtype={"Col2": "string", "Col3": "Int64", "Col4": "datetime64[ns]"},
+ iterparse={"row": ["shape", "degrees", "sides", "date"]},
+ temp_file=temp_file,
+ )
+
+ df_expected = DataFrame(
+ {
+ "Col1": ["square", "circle", "triangle"],
+ "Col2": Series(["00360", "00360", "00180"], dtype="string"),
+ "Col3": Series([4.0, NA, 3.0], dtype="Int64"),
+ "Col4": DatetimeIndex(
+ ["2020-01-01", "2021-01-01", "2022-01-01"], dtype="M8[ns]"
+ ),
+ }
+ )
+
+ tm.assert_frame_equal(df_result, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_dtype_nullable_int(parser, temp_file):
+ df_result = read_xml(StringIO(xml_types), dtype={"sides": "Int64"}, parser=parser)
+ df_iter = read_xml_iterparse(
+ xml_types,
+ parser=parser,
+ dtype={"sides": "Int64"},
+ iterparse={"row": ["shape", "degrees", "sides"]},
+ temp_file=temp_file,
+ )
+
+ df_expected = DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": [360, 360, 180],
+ "sides": Series([4.0, NA, 3.0], dtype="Int64"),
+ }
+ )
+
+ tm.assert_frame_equal(df_result, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_dtype_float(parser, temp_file):
+ df_result = read_xml(StringIO(xml_types), dtype={"degrees": "float"}, parser=parser)
+ df_iter = read_xml_iterparse(
+ xml_types,
+ parser=parser,
+ dtype={"degrees": "float"},
+ iterparse={"row": ["shape", "degrees", "sides"]},
+ temp_file=temp_file,
+ )
+
+ df_expected = DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": Series([360, 360, 180]).astype("float"),
+ "sides": [4.0, float("nan"), 3.0],
+ }
+ )
+
+ tm.assert_frame_equal(df_result, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_wrong_dtype(xml_books, parser, iterparse):
+ with pytest.raises(
+ ValueError, match=('Unable to parse string "Everyday Italian" at position 0')
+ ):
+ read_xml(
+ xml_books, dtype={"title": "Int64"}, parser=parser, iterparse=iterparse
+ )
+
+
+def test_both_dtype_converters(parser, temp_file):
+ df_expected = DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": ["00360", "00360", "00180"],
+ "sides": [4.0, float("nan"), 3.0],
+ }
+ )
+
+ with tm.assert_produces_warning(ParserWarning, match="Both a converter and dtype"):
+ df_result = read_xml(
+ StringIO(xml_types),
+ dtype={"degrees": "str"},
+ converters={"degrees": str},
+ parser=parser,
+ )
+ df_iter = read_xml_iterparse(
+ xml_types,
+ dtype={"degrees": "str"},
+ converters={"degrees": str},
+ parser=parser,
+ iterparse={"row": ["shape", "degrees", "sides"]},
+ temp_file=temp_file,
+ )
+
+ tm.assert_frame_equal(df_result, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+# CONVERTERS
+
+
+def test_converters_str(parser, temp_file):
+ df_result = read_xml(
+ StringIO(xml_types), converters={"degrees": str}, parser=parser
+ )
+ df_iter = read_xml_iterparse(
+ xml_types,
+ parser=parser,
+ converters={"degrees": str},
+ iterparse={"row": ["shape", "degrees", "sides"]},
+ temp_file=temp_file,
+ )
+
+ df_expected = DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": ["00360", "00360", "00180"],
+ "sides": [4.0, float("nan"), 3.0],
+ }
+ )
+
+ tm.assert_frame_equal(df_result, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_converters_date(parser, temp_file):
+ convert_to_datetime = lambda x: to_datetime(x)
+ df_result = read_xml(
+ StringIO(xml_dates), converters={"date": convert_to_datetime}, parser=parser
+ )
+ df_iter = read_xml_iterparse(
+ xml_dates,
+ parser=parser,
+ converters={"date": convert_to_datetime},
+ iterparse={"row": ["shape", "degrees", "sides", "date"]},
+ temp_file=temp_file,
+ )
+
+ df_expected = DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": [360, 360, 180],
+ "sides": [4.0, float("nan"), 3.0],
+ "date": to_datetime(["2020-01-01", "2021-01-01", "2022-01-01"]),
+ }
+ )
+
+ tm.assert_frame_equal(df_result, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_wrong_converters_type(xml_books, parser, iterparse):
+ with pytest.raises(TypeError, match=("Type converters must be a dict or subclass")):
+ read_xml(
+ xml_books, converters={"year", str}, parser=parser, iterparse=iterparse
+ )
+
+
+def test_callable_func_converters(xml_books, parser, iterparse):
+ with pytest.raises(TypeError, match=("'float' object is not callable")):
+ read_xml(
+ xml_books, converters={"year": float()}, parser=parser, iterparse=iterparse
+ )
+
+
+def test_callable_str_converters(xml_books, parser, iterparse):
+ with pytest.raises(TypeError, match=("'str' object is not callable")):
+ read_xml(
+ xml_books, converters={"year": "float"}, parser=parser, iterparse=iterparse
+ )
+
+
+# PARSE DATES
+
+
+def test_parse_dates_column_name(parser, temp_file):
+ df_result = read_xml(StringIO(xml_dates), parse_dates=["date"], parser=parser)
+ df_iter = read_xml_iterparse(
+ xml_dates,
+ parser=parser,
+ parse_dates=["date"],
+ iterparse={"row": ["shape", "degrees", "sides", "date"]},
+ temp_file=temp_file,
+ )
+
+ df_expected = DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": [360, 360, 180],
+ "sides": [4.0, float("nan"), 3.0],
+ "date": to_datetime(["2020-01-01", "2021-01-01", "2022-01-01"]),
+ }
+ )
+
+ tm.assert_frame_equal(df_result, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_parse_dates_column_index(parser, temp_file):
+ df_result = read_xml(StringIO(xml_dates), parse_dates=[3], parser=parser)
+ df_iter = read_xml_iterparse(
+ xml_dates,
+ parser=parser,
+ parse_dates=[3],
+ iterparse={"row": ["shape", "degrees", "sides", "date"]},
+ temp_file=temp_file,
+ )
+
+ df_expected = DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": [360, 360, 180],
+ "sides": [4.0, float("nan"), 3.0],
+ "date": to_datetime(["2020-01-01", "2021-01-01", "2022-01-01"]),
+ }
+ )
+
+ tm.assert_frame_equal(df_result, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_parse_dates_true(parser, temp_file):
+ df_result = read_xml(StringIO(xml_dates), parse_dates=True, parser=parser)
+
+ df_iter = read_xml_iterparse(
+ xml_dates,
+ parser=parser,
+ parse_dates=True,
+ iterparse={"row": ["shape", "degrees", "sides", "date"]},
+ temp_file=temp_file,
+ )
+
+ df_expected = DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": [360, 360, 180],
+ "sides": [4.0, float("nan"), 3.0],
+ "date": ["2020-01-01", "2021-01-01", "2022-01-01"],
+ }
+ )
+
+ tm.assert_frame_equal(df_result, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_day_first_parse_dates(parser, temp_file):
+ xml = """\
+
+
+
+ square
+ 00360
+ 4.0
+ 31/12/2020
+
+
+ circle
+ 00360
+
+ 31/12/2021
+
+
+ triangle
+ 00180
+ 3.0
+ 31/12/2022
+
+ """
+
+ df_expected = DataFrame(
+ {
+ "shape": ["square", "circle", "triangle"],
+ "degrees": [360, 360, 180],
+ "sides": [4.0, float("nan"), 3.0],
+ "date": to_datetime(["2020-12-31", "2021-12-31", "2022-12-31"]),
+ }
+ )
+
+ with tm.assert_produces_warning(
+ UserWarning, match="Parsing dates in %d/%m/%Y format"
+ ):
+ df_result = read_xml(StringIO(xml), parse_dates=["date"], parser=parser)
+ df_iter = read_xml_iterparse(
+ xml,
+ parse_dates=["date"],
+ parser=parser,
+ iterparse={"row": ["shape", "degrees", "sides", "date"]},
+ temp_file=temp_file,
+ )
+
+ tm.assert_frame_equal(df_result, df_expected)
+ tm.assert_frame_equal(df_iter, df_expected)
+
+
+def test_wrong_parse_dates_type(xml_books, parser, iterparse):
+ with pytest.raises(TypeError, match="Only booleans and lists are accepted"):
+ read_xml(xml_books, parse_dates={"date"}, parser=parser, iterparse=iterparse)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/libs/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/libs/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/libs/test_hashtable.py b/.venv/lib/python3.13/site-packages/pandas/tests/libs/test_hashtable.py
new file mode 100644
index 0000000000000000000000000000000000000000..1f24f87348595b7416e6717f929b92f0c9df22c4
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/libs/test_hashtable.py
@@ -0,0 +1,782 @@
+from collections import namedtuple
+from collections.abc import Generator
+from contextlib import contextmanager
+import re
+import struct
+import tracemalloc
+
+import numpy as np
+import pytest
+
+from pandas._libs import hashtable as ht
+
+import pandas as pd
+import pandas._testing as tm
+from pandas.core.algorithms import isin
+
+
+@contextmanager
+def activated_tracemalloc() -> Generator[None, None, None]:
+ tracemalloc.start()
+ try:
+ yield
+ finally:
+ tracemalloc.stop()
+
+
+def get_allocated_khash_memory():
+ snapshot = tracemalloc.take_snapshot()
+ snapshot = snapshot.filter_traces(
+ (tracemalloc.DomainFilter(True, ht.get_hashtable_trace_domain()),)
+ )
+ return sum(x.size for x in snapshot.traces)
+
+
+@pytest.mark.parametrize(
+ "table_type, dtype",
+ [
+ (ht.PyObjectHashTable, np.object_),
+ (ht.Complex128HashTable, np.complex128),
+ (ht.Int64HashTable, np.int64),
+ (ht.UInt64HashTable, np.uint64),
+ (ht.Float64HashTable, np.float64),
+ (ht.Complex64HashTable, np.complex64),
+ (ht.Int32HashTable, np.int32),
+ (ht.UInt32HashTable, np.uint32),
+ (ht.Float32HashTable, np.float32),
+ (ht.Int16HashTable, np.int16),
+ (ht.UInt16HashTable, np.uint16),
+ (ht.Int8HashTable, np.int8),
+ (ht.UInt8HashTable, np.uint8),
+ (ht.IntpHashTable, np.intp),
+ ],
+)
+class TestHashTable:
+ def test_get_set_contains_len(self, table_type, dtype):
+ index = 5
+ table = table_type(55)
+ assert len(table) == 0
+ assert index not in table
+
+ table.set_item(index, 42)
+ assert len(table) == 1
+ assert index in table
+ assert table.get_item(index) == 42
+
+ table.set_item(index + 1, 41)
+ assert index in table
+ assert index + 1 in table
+ assert len(table) == 2
+ assert table.get_item(index) == 42
+ assert table.get_item(index + 1) == 41
+
+ table.set_item(index, 21)
+ assert index in table
+ assert index + 1 in table
+ assert len(table) == 2
+ assert table.get_item(index) == 21
+ assert table.get_item(index + 1) == 41
+ assert index + 2 not in table
+
+ table.set_item(index + 1, 21)
+ assert index in table
+ assert index + 1 in table
+ assert len(table) == 2
+ assert table.get_item(index) == 21
+ assert table.get_item(index + 1) == 21
+
+ with pytest.raises(KeyError, match=str(index + 2)):
+ table.get_item(index + 2)
+
+ def test_get_set_contains_len_mask(self, table_type, dtype):
+ if table_type == ht.PyObjectHashTable:
+ pytest.skip("Mask not supported for object")
+ index = 5
+ table = table_type(55, uses_mask=True)
+ assert len(table) == 0
+ assert index not in table
+
+ table.set_item(index, 42)
+ assert len(table) == 1
+ assert index in table
+ assert table.get_item(index) == 42
+ with pytest.raises(KeyError, match="NA"):
+ table.get_na()
+
+ table.set_item(index + 1, 41)
+ table.set_na(41)
+ assert pd.NA in table
+ assert index in table
+ assert index + 1 in table
+ assert len(table) == 3
+ assert table.get_item(index) == 42
+ assert table.get_item(index + 1) == 41
+ assert table.get_na() == 41
+
+ table.set_na(21)
+ assert index in table
+ assert index + 1 in table
+ assert len(table) == 3
+ assert table.get_item(index + 1) == 41
+ assert table.get_na() == 21
+ assert index + 2 not in table
+
+ with pytest.raises(KeyError, match=str(index + 2)):
+ table.get_item(index + 2)
+
+ def test_map_keys_to_values(self, table_type, dtype, writable):
+ # only Int64HashTable has this method
+ if table_type == ht.Int64HashTable:
+ N = 77
+ table = table_type()
+ keys = np.arange(N).astype(dtype)
+ vals = np.arange(N).astype(np.int64) + N
+ keys.flags.writeable = writable
+ vals.flags.writeable = writable
+ table.map_keys_to_values(keys, vals)
+ for i in range(N):
+ assert table.get_item(keys[i]) == i + N
+
+ def test_map_locations(self, table_type, dtype, writable):
+ N = 8
+ table = table_type()
+ keys = (np.arange(N) + N).astype(dtype)
+ keys.flags.writeable = writable
+ table.map_locations(keys)
+ for i in range(N):
+ assert table.get_item(keys[i]) == i
+
+ def test_map_locations_mask(self, table_type, dtype, writable):
+ if table_type == ht.PyObjectHashTable:
+ pytest.skip("Mask not supported for object")
+ N = 129 # must be > 128 to test GH#58924
+ table = table_type(uses_mask=True)
+ keys = (np.arange(N) + N).astype(dtype)
+ keys.flags.writeable = writable
+ mask = np.concatenate([np.repeat(False, N - 1), [True]], axis=0)
+ table.map_locations(keys, mask)
+ for i in range(N - 1):
+ assert table.get_item(keys[i]) == i
+
+ with pytest.raises(KeyError, match=re.escape(str(keys[N - 1]))):
+ table.get_item(keys[N - 1])
+
+ assert table.get_na() == N - 1
+
+ def test_lookup(self, table_type, dtype, writable):
+ N = 3
+ table = table_type()
+ keys = (np.arange(N) + N).astype(dtype)
+ keys.flags.writeable = writable
+ table.map_locations(keys)
+ result = table.lookup(keys)
+ expected = np.arange(N)
+ tm.assert_numpy_array_equal(result.astype(np.int64), expected.astype(np.int64))
+
+ def test_lookup_wrong(self, table_type, dtype):
+ if dtype in (np.int8, np.uint8):
+ N = 100
+ else:
+ N = 512
+ table = table_type()
+ keys = (np.arange(N) + N).astype(dtype)
+ table.map_locations(keys)
+ wrong_keys = np.arange(N).astype(dtype)
+ result = table.lookup(wrong_keys)
+ assert np.all(result == -1)
+
+ def test_lookup_mask(self, table_type, dtype, writable):
+ if table_type == ht.PyObjectHashTable:
+ pytest.skip("Mask not supported for object")
+ N = 3
+ table = table_type(uses_mask=True)
+ keys = (np.arange(N) + N).astype(dtype)
+ mask = np.array([False, True, False])
+ keys.flags.writeable = writable
+ table.map_locations(keys, mask)
+ result = table.lookup(keys, mask)
+ expected = np.arange(N)
+ tm.assert_numpy_array_equal(result.astype(np.int64), expected.astype(np.int64))
+
+ result = table.lookup(np.array([1 + N]).astype(dtype), np.array([False]))
+ tm.assert_numpy_array_equal(
+ result.astype(np.int64), np.array([-1], dtype=np.int64)
+ )
+
+ def test_unique(self, table_type, dtype, writable):
+ if dtype in (np.int8, np.uint8):
+ N = 88
+ else:
+ N = 1000
+ table = table_type()
+ expected = (np.arange(N) + N).astype(dtype)
+ keys = np.repeat(expected, 5)
+ keys.flags.writeable = writable
+ unique = table.unique(keys)
+ tm.assert_numpy_array_equal(unique, expected)
+
+ def test_tracemalloc_works(self, table_type, dtype):
+ if dtype in (np.int8, np.uint8):
+ N = 256
+ else:
+ N = 30000
+ keys = np.arange(N).astype(dtype)
+ with activated_tracemalloc():
+ table = table_type()
+ table.map_locations(keys)
+ used = get_allocated_khash_memory()
+ my_size = table.sizeof()
+ assert used == my_size
+ del table
+ assert get_allocated_khash_memory() == 0
+
+ def test_tracemalloc_for_empty(self, table_type, dtype):
+ with activated_tracemalloc():
+ table = table_type()
+ used = get_allocated_khash_memory()
+ my_size = table.sizeof()
+ assert used == my_size
+ del table
+ assert get_allocated_khash_memory() == 0
+
+ def test_get_state(self, table_type, dtype):
+ table = table_type(1000)
+ state = table.get_state()
+ assert state["size"] == 0
+ assert state["n_occupied"] == 0
+ assert "n_buckets" in state
+ assert "upper_bound" in state
+
+ @pytest.mark.parametrize("N", range(1, 110, 4))
+ def test_no_reallocation(self, table_type, dtype, N):
+ keys = np.arange(N).astype(dtype)
+ preallocated_table = table_type(N)
+ n_buckets_start = preallocated_table.get_state()["n_buckets"]
+ preallocated_table.map_locations(keys)
+ n_buckets_end = preallocated_table.get_state()["n_buckets"]
+ # original number of buckets was enough:
+ assert n_buckets_start == n_buckets_end
+ # check with clean table (not too much preallocated)
+ clean_table = table_type()
+ clean_table.map_locations(keys)
+ assert n_buckets_start == clean_table.get_state()["n_buckets"]
+
+
+class TestHashTableUnsorted:
+ # TODO: moved from test_algos; may be redundancies with other tests
+ def test_string_hashtable_set_item_signature(self):
+ # GH#30419 fix typing in StringHashTable.set_item to prevent segfault
+ tbl = ht.StringHashTable()
+
+ tbl.set_item("key", 1)
+ assert tbl.get_item("key") == 1
+
+ with pytest.raises(TypeError, match="'key' has incorrect type"):
+ # key arg typed as string, not object
+ tbl.set_item(4, 6)
+ with pytest.raises(TypeError, match="'val' has incorrect type"):
+ tbl.get_item(4)
+
+ def test_lookup_nan(self, writable):
+ # GH#21688 ensure we can deal with readonly memory views
+ xs = np.array([2.718, 3.14, np.nan, -7, 5, 2, 3])
+ xs.setflags(write=writable)
+ m = ht.Float64HashTable()
+ m.map_locations(xs)
+ tm.assert_numpy_array_equal(m.lookup(xs), np.arange(len(xs), dtype=np.intp))
+
+ def test_add_signed_zeros(self):
+ # GH#21866 inconsistent hash-function for float64
+ # default hash-function would lead to different hash-buckets
+ # for 0.0 and -0.0 if there are more than 2^30 hash-buckets
+ # but this would mean 16GB
+ N = 4 # 12 * 10**8 would trigger the error, if you have enough memory
+ m = ht.Float64HashTable(N)
+ m.set_item(0.0, 0)
+ m.set_item(-0.0, 0)
+ assert len(m) == 1 # 0.0 and -0.0 are equivalent
+
+ def test_add_different_nans(self):
+ # GH#21866 inconsistent hash-function for float64
+ # create different nans from bit-patterns:
+ NAN1 = struct.unpack("d", struct.pack("=Q", 0x7FF8000000000000))[0]
+ NAN2 = struct.unpack("d", struct.pack("=Q", 0x7FF8000000000001))[0]
+ assert NAN1 != NAN1
+ assert NAN2 != NAN2
+ # default hash function would lead to different hash-buckets
+ # for NAN1 and NAN2 even if there are only 4 buckets:
+ m = ht.Float64HashTable()
+ m.set_item(NAN1, 0)
+ m.set_item(NAN2, 0)
+ assert len(m) == 1 # NAN1 and NAN2 are equivalent
+
+ def test_lookup_overflow(self, writable):
+ xs = np.array([1, 2, 2**63], dtype=np.uint64)
+ # GH 21688 ensure we can deal with readonly memory views
+ xs.setflags(write=writable)
+ m = ht.UInt64HashTable()
+ m.map_locations(xs)
+ tm.assert_numpy_array_equal(m.lookup(xs), np.arange(len(xs), dtype=np.intp))
+
+ @pytest.mark.parametrize("nvals", [0, 10]) # resizing to 0 is special case
+ @pytest.mark.parametrize(
+ "htable, uniques, dtype, safely_resizes",
+ [
+ (ht.PyObjectHashTable, ht.ObjectVector, "object", False),
+ (ht.StringHashTable, ht.ObjectVector, "object", True),
+ (ht.Float64HashTable, ht.Float64Vector, "float64", False),
+ (ht.Int64HashTable, ht.Int64Vector, "int64", False),
+ (ht.Int32HashTable, ht.Int32Vector, "int32", False),
+ (ht.UInt64HashTable, ht.UInt64Vector, "uint64", False),
+ ],
+ )
+ def test_vector_resize(
+ self, writable, htable, uniques, dtype, safely_resizes, nvals
+ ):
+ # Test for memory errors after internal vector
+ # reallocations (GH 7157)
+ # Changed from using np.random.default_rng(2).rand to range
+ # which could cause flaky CI failures when safely_resizes=False
+ vals = np.array(range(1000), dtype=dtype)
+
+ # GH 21688 ensures we can deal with read-only memory views
+ vals.setflags(write=writable)
+
+ # initialise instances; cannot initialise in parametrization,
+ # as otherwise external views would be held on the array (which is
+ # one of the things this test is checking)
+ htable = htable()
+ uniques = uniques()
+
+ # get_labels may append to uniques
+ htable.get_labels(vals[:nvals], uniques, 0, -1)
+ # to_array() sets an external_view_exists flag on uniques.
+ tmp = uniques.to_array()
+ oldshape = tmp.shape
+
+ # subsequent get_labels() calls can no longer append to it
+ # (except for StringHashTables + ObjectVector)
+ if safely_resizes:
+ htable.get_labels(vals, uniques, 0, -1)
+ else:
+ with pytest.raises(ValueError, match="external reference.*"):
+ htable.get_labels(vals, uniques, 0, -1)
+
+ uniques.to_array() # should not raise here
+ assert tmp.shape == oldshape
+
+ @pytest.mark.parametrize(
+ "hashtable",
+ [
+ ht.PyObjectHashTable,
+ ht.StringHashTable,
+ ht.Float64HashTable,
+ ht.Int64HashTable,
+ ht.Int32HashTable,
+ ht.UInt64HashTable,
+ ],
+ )
+ def test_hashtable_large_sizehint(self, hashtable):
+ # GH#22729 smoketest for not raising when passing a large size_hint
+ size_hint = np.iinfo(np.uint32).max + 1
+ hashtable(size_hint=size_hint)
+
+
+class TestPyObjectHashTableWithNans:
+ def test_nan_float(self):
+ nan1 = float("nan")
+ nan2 = float("nan")
+ assert nan1 is not nan2
+ table = ht.PyObjectHashTable()
+ table.set_item(nan1, 42)
+ assert table.get_item(nan2) == 42
+
+ def test_nan_complex_both(self):
+ nan1 = complex(float("nan"), float("nan"))
+ nan2 = complex(float("nan"), float("nan"))
+ assert nan1 is not nan2
+ table = ht.PyObjectHashTable()
+ table.set_item(nan1, 42)
+ assert table.get_item(nan2) == 42
+
+ def test_nan_complex_real(self):
+ nan1 = complex(float("nan"), 1)
+ nan2 = complex(float("nan"), 1)
+ other = complex(float("nan"), 2)
+ assert nan1 is not nan2
+ table = ht.PyObjectHashTable()
+ table.set_item(nan1, 42)
+ assert table.get_item(nan2) == 42
+ with pytest.raises(KeyError, match=re.escape(repr(other))):
+ table.get_item(other)
+
+ def test_nan_complex_imag(self):
+ nan1 = complex(1, float("nan"))
+ nan2 = complex(1, float("nan"))
+ other = complex(2, float("nan"))
+ assert nan1 is not nan2
+ table = ht.PyObjectHashTable()
+ table.set_item(nan1, 42)
+ assert table.get_item(nan2) == 42
+ with pytest.raises(KeyError, match=re.escape(repr(other))):
+ table.get_item(other)
+
+ def test_nan_in_tuple(self):
+ nan1 = (float("nan"),)
+ nan2 = (float("nan"),)
+ assert nan1[0] is not nan2[0]
+ table = ht.PyObjectHashTable()
+ table.set_item(nan1, 42)
+ assert table.get_item(nan2) == 42
+
+ def test_nan_in_nested_tuple(self):
+ nan1 = (1, (2, (float("nan"),)))
+ nan2 = (1, (2, (float("nan"),)))
+ other = (1, 2)
+ table = ht.PyObjectHashTable()
+ table.set_item(nan1, 42)
+ assert table.get_item(nan2) == 42
+ with pytest.raises(KeyError, match=re.escape(repr(other))):
+ table.get_item(other)
+
+ def test_nan_in_namedtuple(self):
+ T = namedtuple("T", ["x"])
+ nan1 = T(float("nan"))
+ nan2 = T(float("nan"))
+ assert nan1.x is not nan2.x
+ table = ht.PyObjectHashTable()
+ table.set_item(nan1, 42)
+ assert table.get_item(nan2) == 42
+
+ def test_nan_in_nested_namedtuple(self):
+ T = namedtuple("T", ["x", "y"])
+ nan1 = T(1, (2, (float("nan"),)))
+ nan2 = T(1, (2, (float("nan"),)))
+ other = T(1, 2)
+ table = ht.PyObjectHashTable()
+ table.set_item(nan1, 42)
+ assert table.get_item(nan2) == 42
+ with pytest.raises(KeyError, match=re.escape(repr(other))):
+ table.get_item(other)
+
+
+def test_hash_equal_tuple_with_nans():
+ a = (float("nan"), (float("nan"), float("nan")))
+ b = (float("nan"), (float("nan"), float("nan")))
+ assert ht.object_hash(a) == ht.object_hash(b)
+ assert ht.objects_are_equal(a, b)
+
+
+def test_hash_equal_namedtuple_with_nans():
+ T = namedtuple("T", ["x", "y"])
+ a = T(float("nan"), (float("nan"), float("nan")))
+ b = T(float("nan"), (float("nan"), float("nan")))
+ assert ht.object_hash(a) == ht.object_hash(b)
+ assert ht.objects_are_equal(a, b)
+
+
+def test_hash_equal_namedtuple_and_tuple():
+ T = namedtuple("T", ["x", "y"])
+ a = T(1, (2, 3))
+ b = (1, (2, 3))
+ assert ht.object_hash(a) == ht.object_hash(b)
+ assert ht.objects_are_equal(a, b)
+
+
+def test_get_labels_groupby_for_Int64(writable):
+ table = ht.Int64HashTable()
+ vals = np.array([1, 2, -1, 2, 1, -1], dtype=np.int64)
+ vals.flags.writeable = writable
+ arr, unique = table.get_labels_groupby(vals)
+ expected_arr = np.array([0, 1, -1, 1, 0, -1], dtype=np.intp)
+ expected_unique = np.array([1, 2], dtype=np.int64)
+ tm.assert_numpy_array_equal(arr, expected_arr)
+ tm.assert_numpy_array_equal(unique, expected_unique)
+
+
+def test_tracemalloc_works_for_StringHashTable():
+ N = 1000
+ keys = np.arange(N).astype(np.str_).astype(np.object_)
+ with activated_tracemalloc():
+ table = ht.StringHashTable()
+ table.map_locations(keys)
+ used = get_allocated_khash_memory()
+ my_size = table.sizeof()
+ assert used == my_size
+ del table
+ assert get_allocated_khash_memory() == 0
+
+
+def test_tracemalloc_for_empty_StringHashTable():
+ with activated_tracemalloc():
+ table = ht.StringHashTable()
+ used = get_allocated_khash_memory()
+ my_size = table.sizeof()
+ assert used == my_size
+ del table
+ assert get_allocated_khash_memory() == 0
+
+
+@pytest.mark.parametrize("N", range(1, 110, 4))
+def test_no_reallocation_StringHashTable(N):
+ keys = np.arange(N).astype(np.str_).astype(np.object_)
+ preallocated_table = ht.StringHashTable(N)
+ n_buckets_start = preallocated_table.get_state()["n_buckets"]
+ preallocated_table.map_locations(keys)
+ n_buckets_end = preallocated_table.get_state()["n_buckets"]
+ # original number of buckets was enough:
+ assert n_buckets_start == n_buckets_end
+ # check with clean table (not too much preallocated)
+ clean_table = ht.StringHashTable()
+ clean_table.map_locations(keys)
+ assert n_buckets_start == clean_table.get_state()["n_buckets"]
+
+
+@pytest.mark.parametrize(
+ "table_type, dtype",
+ [
+ (ht.Float64HashTable, np.float64),
+ (ht.Float32HashTable, np.float32),
+ (ht.Complex128HashTable, np.complex128),
+ (ht.Complex64HashTable, np.complex64),
+ ],
+)
+class TestHashTableWithNans:
+ def test_get_set_contains_len(self, table_type, dtype):
+ index = float("nan")
+ table = table_type()
+ assert index not in table
+
+ table.set_item(index, 42)
+ assert len(table) == 1
+ assert index in table
+ assert table.get_item(index) == 42
+
+ table.set_item(index, 41)
+ assert len(table) == 1
+ assert index in table
+ assert table.get_item(index) == 41
+
+ def test_map_locations(self, table_type, dtype):
+ N = 10
+ table = table_type()
+ keys = np.full(N, np.nan, dtype=dtype)
+ table.map_locations(keys)
+ assert len(table) == 1
+ assert table.get_item(np.nan) == N - 1
+
+ def test_unique(self, table_type, dtype):
+ N = 1020
+ table = table_type()
+ keys = np.full(N, np.nan, dtype=dtype)
+ unique = table.unique(keys)
+ assert np.all(np.isnan(unique)) and len(unique) == 1
+
+
+def test_unique_for_nan_objects_floats():
+ table = ht.PyObjectHashTable()
+ keys = np.array([float("nan") for i in range(50)], dtype=np.object_)
+ unique = table.unique(keys)
+ assert len(unique) == 1
+
+
+def test_unique_for_nan_objects_complex():
+ table = ht.PyObjectHashTable()
+ keys = np.array([complex(float("nan"), 1.0) for i in range(50)], dtype=np.object_)
+ unique = table.unique(keys)
+ assert len(unique) == 1
+
+
+def test_unique_for_nan_objects_tuple():
+ table = ht.PyObjectHashTable()
+ keys = np.array(
+ [1] + [(1.0, (float("nan"), 1.0)) for i in range(50)], dtype=np.object_
+ )
+ unique = table.unique(keys)
+ assert len(unique) == 2
+
+
+@pytest.mark.parametrize(
+ "dtype",
+ [
+ np.object_,
+ np.complex128,
+ np.int64,
+ np.uint64,
+ np.float64,
+ np.complex64,
+ np.int32,
+ np.uint32,
+ np.float32,
+ np.int16,
+ np.uint16,
+ np.int8,
+ np.uint8,
+ np.intp,
+ ],
+)
+class TestHelpFunctions:
+ def test_value_count(self, dtype, writable):
+ N = 43
+ expected = (np.arange(N) + N).astype(dtype)
+ values = np.repeat(expected, 5)
+ values.flags.writeable = writable
+ keys, counts, _ = ht.value_count(values, False)
+ tm.assert_numpy_array_equal(np.sort(keys), expected)
+ assert np.all(counts == 5)
+
+ def test_value_count_mask(self, dtype):
+ if dtype == np.object_:
+ pytest.skip("mask not implemented for object dtype")
+ values = np.array([1] * 5, dtype=dtype)
+ mask = np.zeros((5,), dtype=np.bool_)
+ mask[1] = True
+ mask[4] = True
+ keys, counts, na_counter = ht.value_count(values, False, mask=mask)
+ assert len(keys) == 2
+ assert na_counter == 2
+
+ def test_value_count_stable(self, dtype, writable):
+ # GH12679
+ values = np.array([2, 1, 5, 22, 3, -1, 8]).astype(dtype)
+ values.flags.writeable = writable
+ keys, counts, _ = ht.value_count(values, False)
+ tm.assert_numpy_array_equal(keys, values)
+ assert np.all(counts == 1)
+
+ def test_duplicated_first(self, dtype, writable):
+ N = 100
+ values = np.repeat(np.arange(N).astype(dtype), 5)
+ values.flags.writeable = writable
+ result = ht.duplicated(values)
+ expected = np.ones_like(values, dtype=np.bool_)
+ expected[::5] = False
+ tm.assert_numpy_array_equal(result, expected)
+
+ def test_ismember_yes(self, dtype, writable):
+ N = 127
+ arr = np.arange(N).astype(dtype)
+ values = np.arange(N).astype(dtype)
+ arr.flags.writeable = writable
+ values.flags.writeable = writable
+ result = ht.ismember(arr, values)
+ expected = np.ones_like(values, dtype=np.bool_)
+ tm.assert_numpy_array_equal(result, expected)
+
+ def test_ismember_no(self, dtype):
+ N = 17
+ arr = np.arange(N).astype(dtype)
+ values = (np.arange(N) + N).astype(dtype)
+ result = ht.ismember(arr, values)
+ expected = np.zeros_like(values, dtype=np.bool_)
+ tm.assert_numpy_array_equal(result, expected)
+
+ def test_mode(self, dtype, writable):
+ if dtype in (np.int8, np.uint8):
+ N = 53
+ else:
+ N = 11111
+ values = np.repeat(np.arange(N).astype(dtype), 5)
+ values[0] = 42
+ values.flags.writeable = writable
+ result = ht.mode(values, False)[0]
+ assert result == 42
+
+ def test_mode_stable(self, dtype, writable):
+ values = np.array([2, 1, 5, 22, 3, -1, 8]).astype(dtype)
+ values.flags.writeable = writable
+ keys = ht.mode(values, False)[0]
+ tm.assert_numpy_array_equal(keys, values)
+
+
+def test_modes_with_nans():
+ # GH42688, nans aren't mangled
+ nulls = [pd.NA, np.nan, pd.NaT, None]
+ values = np.array([True] + nulls * 2, dtype=np.object_)
+ modes = ht.mode(values, False)[0]
+ assert modes.size == len(nulls)
+
+
+def test_unique_label_indices_intp(writable):
+ keys = np.array([1, 2, 2, 2, 1, 3], dtype=np.intp)
+ keys.flags.writeable = writable
+ result = ht.unique_label_indices(keys)
+ expected = np.array([0, 1, 5], dtype=np.intp)
+ tm.assert_numpy_array_equal(result, expected)
+
+
+def test_unique_label_indices():
+ a = np.random.default_rng(2).integers(1, 1 << 10, 1 << 15).astype(np.intp)
+
+ left = ht.unique_label_indices(a)
+ right = np.unique(a, return_index=True)[1]
+
+ tm.assert_numpy_array_equal(left, right, check_dtype=False)
+
+ a[np.random.default_rng(2).choice(len(a), 10)] = -1
+ left = ht.unique_label_indices(a)
+ right = np.unique(a, return_index=True)[1][1:]
+ tm.assert_numpy_array_equal(left, right, check_dtype=False)
+
+
+@pytest.mark.parametrize(
+ "dtype",
+ [
+ np.float64,
+ np.float32,
+ np.complex128,
+ np.complex64,
+ ],
+)
+class TestHelpFunctionsWithNans:
+ def test_value_count(self, dtype):
+ values = np.array([np.nan, np.nan, np.nan], dtype=dtype)
+ keys, counts, _ = ht.value_count(values, True)
+ assert len(keys) == 0
+ keys, counts, _ = ht.value_count(values, False)
+ assert len(keys) == 1 and np.all(np.isnan(keys))
+ assert counts[0] == 3
+
+ def test_duplicated_first(self, dtype):
+ values = np.array([np.nan, np.nan, np.nan], dtype=dtype)
+ result = ht.duplicated(values)
+ expected = np.array([False, True, True])
+ tm.assert_numpy_array_equal(result, expected)
+
+ def test_ismember_yes(self, dtype):
+ arr = np.array([np.nan, np.nan, np.nan], dtype=dtype)
+ values = np.array([np.nan, np.nan], dtype=dtype)
+ result = ht.ismember(arr, values)
+ expected = np.array([True, True, True], dtype=np.bool_)
+ tm.assert_numpy_array_equal(result, expected)
+
+ def test_ismember_no(self, dtype):
+ arr = np.array([np.nan, np.nan, np.nan], dtype=dtype)
+ values = np.array([1], dtype=dtype)
+ result = ht.ismember(arr, values)
+ expected = np.array([False, False, False], dtype=np.bool_)
+ tm.assert_numpy_array_equal(result, expected)
+
+ def test_mode(self, dtype):
+ values = np.array([42, np.nan, np.nan, np.nan], dtype=dtype)
+ assert ht.mode(values, True)[0] == 42
+ assert np.isnan(ht.mode(values, False)[0])
+
+
+def test_ismember_tuple_with_nans():
+ # GH-41836
+ values = np.empty(2, dtype=object)
+ values[:] = [("a", float("nan")), ("b", 1)]
+ comps = [("a", float("nan"))]
+
+ result = isin(values, comps)
+ expected = np.array([True, False], dtype=np.bool_)
+ tm.assert_numpy_array_equal(result, expected)
+
+
+def test_float_complex_int_are_equal_as_objects():
+ values = ["a", 5, 5.0, 5.0 + 0j]
+ comps = list(range(129))
+ result = isin(np.array(values, dtype=object), np.asarray(comps))
+ expected = np.array([False, True, True, True], dtype=np.bool_)
+ tm.assert_numpy_array_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/libs/test_join.py b/.venv/lib/python3.13/site-packages/pandas/tests/libs/test_join.py
new file mode 100644
index 0000000000000000000000000000000000000000..bf8b4fabc54cbab338e9b4a3a061eb09b19714da
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/libs/test_join.py
@@ -0,0 +1,388 @@
+import numpy as np
+import pytest
+
+from pandas._libs import join as libjoin
+from pandas._libs.join import (
+ inner_join,
+ left_outer_join,
+)
+
+import pandas._testing as tm
+
+
+class TestIndexer:
+ @pytest.mark.parametrize(
+ "dtype", ["int32", "int64", "float32", "float64", "object"]
+ )
+ def test_outer_join_indexer(self, dtype):
+ indexer = libjoin.outer_join_indexer
+
+ left = np.arange(3, dtype=dtype)
+ right = np.arange(2, 5, dtype=dtype)
+ empty = np.array([], dtype=dtype)
+
+ result, lindexer, rindexer = indexer(left, right)
+ assert isinstance(result, np.ndarray)
+ assert isinstance(lindexer, np.ndarray)
+ assert isinstance(rindexer, np.ndarray)
+ tm.assert_numpy_array_equal(result, np.arange(5, dtype=dtype))
+ exp = np.array([0, 1, 2, -1, -1], dtype=np.intp)
+ tm.assert_numpy_array_equal(lindexer, exp)
+ exp = np.array([-1, -1, 0, 1, 2], dtype=np.intp)
+ tm.assert_numpy_array_equal(rindexer, exp)
+
+ result, lindexer, rindexer = indexer(empty, right)
+ tm.assert_numpy_array_equal(result, right)
+ exp = np.array([-1, -1, -1], dtype=np.intp)
+ tm.assert_numpy_array_equal(lindexer, exp)
+ exp = np.array([0, 1, 2], dtype=np.intp)
+ tm.assert_numpy_array_equal(rindexer, exp)
+
+ result, lindexer, rindexer = indexer(left, empty)
+ tm.assert_numpy_array_equal(result, left)
+ exp = np.array([0, 1, 2], dtype=np.intp)
+ tm.assert_numpy_array_equal(lindexer, exp)
+ exp = np.array([-1, -1, -1], dtype=np.intp)
+ tm.assert_numpy_array_equal(rindexer, exp)
+
+ def test_cython_left_outer_join(self):
+ left = np.array([0, 1, 2, 1, 2, 0, 0, 1, 2, 3, 3], dtype=np.intp)
+ right = np.array([1, 1, 0, 4, 2, 2, 1], dtype=np.intp)
+ max_group = 5
+
+ ls, rs = left_outer_join(left, right, max_group)
+
+ exp_ls = left.argsort(kind="mergesort")
+ exp_rs = right.argsort(kind="mergesort")
+
+ exp_li = np.array([0, 1, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 7, 7, 8, 8, 9, 10])
+ exp_ri = np.array(
+ [0, 0, 0, 1, 2, 3, 1, 2, 3, 1, 2, 3, 4, 5, 4, 5, 4, 5, -1, -1]
+ )
+
+ exp_ls = exp_ls.take(exp_li)
+ exp_ls[exp_li == -1] = -1
+
+ exp_rs = exp_rs.take(exp_ri)
+ exp_rs[exp_ri == -1] = -1
+
+ tm.assert_numpy_array_equal(ls, exp_ls, check_dtype=False)
+ tm.assert_numpy_array_equal(rs, exp_rs, check_dtype=False)
+
+ def test_cython_right_outer_join(self):
+ left = np.array([0, 1, 2, 1, 2, 0, 0, 1, 2, 3, 3], dtype=np.intp)
+ right = np.array([1, 1, 0, 4, 2, 2, 1], dtype=np.intp)
+ max_group = 5
+
+ rs, ls = left_outer_join(right, left, max_group)
+
+ exp_ls = left.argsort(kind="mergesort")
+ exp_rs = right.argsort(kind="mergesort")
+
+ # 0 1 1 1
+ exp_li = np.array(
+ [
+ 0,
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 3,
+ 4,
+ 5,
+ 3,
+ 4,
+ 5,
+ # 2 2 4
+ 6,
+ 7,
+ 8,
+ 6,
+ 7,
+ 8,
+ -1,
+ ]
+ )
+ exp_ri = np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6])
+
+ exp_ls = exp_ls.take(exp_li)
+ exp_ls[exp_li == -1] = -1
+
+ exp_rs = exp_rs.take(exp_ri)
+ exp_rs[exp_ri == -1] = -1
+
+ tm.assert_numpy_array_equal(ls, exp_ls)
+ tm.assert_numpy_array_equal(rs, exp_rs)
+
+ def test_cython_inner_join(self):
+ left = np.array([0, 1, 2, 1, 2, 0, 0, 1, 2, 3, 3], dtype=np.intp)
+ right = np.array([1, 1, 0, 4, 2, 2, 1, 4], dtype=np.intp)
+ max_group = 5
+
+ ls, rs = inner_join(left, right, max_group)
+
+ exp_ls = left.argsort(kind="mergesort")
+ exp_rs = right.argsort(kind="mergesort")
+
+ exp_li = np.array([0, 1, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 7, 7, 8, 8])
+ exp_ri = np.array([0, 0, 0, 1, 2, 3, 1, 2, 3, 1, 2, 3, 4, 5, 4, 5, 4, 5])
+
+ exp_ls = exp_ls.take(exp_li)
+ exp_ls[exp_li == -1] = -1
+
+ exp_rs = exp_rs.take(exp_ri)
+ exp_rs[exp_ri == -1] = -1
+
+ tm.assert_numpy_array_equal(ls, exp_ls)
+ tm.assert_numpy_array_equal(rs, exp_rs)
+
+
+def test_left_join_indexer_unique(writable):
+ a = np.array([1, 2, 3, 4, 5], dtype=np.int64)
+ b = np.array([2, 2, 3, 4, 4], dtype=np.int64)
+ # GH#37312, GH#37264
+ a.setflags(write=writable)
+ b.setflags(write=writable)
+
+ result = libjoin.left_join_indexer_unique(b, a)
+ expected = np.array([1, 1, 2, 3, 3], dtype=np.intp)
+ tm.assert_numpy_array_equal(result, expected)
+
+
+def test_left_outer_join_bug():
+ left = np.array(
+ [
+ 0,
+ 1,
+ 0,
+ 1,
+ 1,
+ 2,
+ 3,
+ 1,
+ 0,
+ 2,
+ 1,
+ 2,
+ 0,
+ 1,
+ 1,
+ 2,
+ 3,
+ 2,
+ 3,
+ 2,
+ 1,
+ 1,
+ 3,
+ 0,
+ 3,
+ 2,
+ 3,
+ 0,
+ 0,
+ 2,
+ 3,
+ 2,
+ 0,
+ 3,
+ 1,
+ 3,
+ 0,
+ 1,
+ 3,
+ 0,
+ 0,
+ 1,
+ 0,
+ 3,
+ 1,
+ 0,
+ 1,
+ 0,
+ 1,
+ 1,
+ 0,
+ 2,
+ 2,
+ 2,
+ 2,
+ 2,
+ 0,
+ 3,
+ 1,
+ 2,
+ 0,
+ 0,
+ 3,
+ 1,
+ 3,
+ 2,
+ 2,
+ 0,
+ 1,
+ 3,
+ 0,
+ 2,
+ 3,
+ 2,
+ 3,
+ 3,
+ 2,
+ 3,
+ 3,
+ 1,
+ 3,
+ 2,
+ 0,
+ 0,
+ 3,
+ 1,
+ 1,
+ 1,
+ 0,
+ 2,
+ 3,
+ 3,
+ 1,
+ 2,
+ 0,
+ 3,
+ 1,
+ 2,
+ 0,
+ 2,
+ ],
+ dtype=np.intp,
+ )
+
+ right = np.array([3, 1], dtype=np.intp)
+ max_groups = 4
+
+ lidx, ridx = libjoin.left_outer_join(left, right, max_groups, sort=False)
+
+ exp_lidx = np.arange(len(left), dtype=np.intp)
+ exp_ridx = -np.ones(len(left), dtype=np.intp)
+
+ exp_ridx[left == 1] = 1
+ exp_ridx[left == 3] = 0
+
+ tm.assert_numpy_array_equal(lidx, exp_lidx)
+ tm.assert_numpy_array_equal(ridx, exp_ridx)
+
+
+def test_inner_join_indexer():
+ a = np.array([1, 2, 3, 4, 5], dtype=np.int64)
+ b = np.array([0, 3, 5, 7, 9], dtype=np.int64)
+
+ index, ares, bres = libjoin.inner_join_indexer(a, b)
+
+ index_exp = np.array([3, 5], dtype=np.int64)
+ tm.assert_almost_equal(index, index_exp)
+
+ aexp = np.array([2, 4], dtype=np.intp)
+ bexp = np.array([1, 2], dtype=np.intp)
+ tm.assert_almost_equal(ares, aexp)
+ tm.assert_almost_equal(bres, bexp)
+
+ a = np.array([5], dtype=np.int64)
+ b = np.array([5], dtype=np.int64)
+
+ index, ares, bres = libjoin.inner_join_indexer(a, b)
+ tm.assert_numpy_array_equal(index, np.array([5], dtype=np.int64))
+ tm.assert_numpy_array_equal(ares, np.array([0], dtype=np.intp))
+ tm.assert_numpy_array_equal(bres, np.array([0], dtype=np.intp))
+
+
+def test_outer_join_indexer():
+ a = np.array([1, 2, 3, 4, 5], dtype=np.int64)
+ b = np.array([0, 3, 5, 7, 9], dtype=np.int64)
+
+ index, ares, bres = libjoin.outer_join_indexer(a, b)
+
+ index_exp = np.array([0, 1, 2, 3, 4, 5, 7, 9], dtype=np.int64)
+ tm.assert_almost_equal(index, index_exp)
+
+ aexp = np.array([-1, 0, 1, 2, 3, 4, -1, -1], dtype=np.intp)
+ bexp = np.array([0, -1, -1, 1, -1, 2, 3, 4], dtype=np.intp)
+ tm.assert_almost_equal(ares, aexp)
+ tm.assert_almost_equal(bres, bexp)
+
+ a = np.array([5], dtype=np.int64)
+ b = np.array([5], dtype=np.int64)
+
+ index, ares, bres = libjoin.outer_join_indexer(a, b)
+ tm.assert_numpy_array_equal(index, np.array([5], dtype=np.int64))
+ tm.assert_numpy_array_equal(ares, np.array([0], dtype=np.intp))
+ tm.assert_numpy_array_equal(bres, np.array([0], dtype=np.intp))
+
+
+def test_left_join_indexer():
+ a = np.array([1, 2, 3, 4, 5], dtype=np.int64)
+ b = np.array([0, 3, 5, 7, 9], dtype=np.int64)
+
+ index, ares, bres = libjoin.left_join_indexer(a, b)
+
+ tm.assert_almost_equal(index, a)
+
+ aexp = np.array([0, 1, 2, 3, 4], dtype=np.intp)
+ bexp = np.array([-1, -1, 1, -1, 2], dtype=np.intp)
+ tm.assert_almost_equal(ares, aexp)
+ tm.assert_almost_equal(bres, bexp)
+
+ a = np.array([5], dtype=np.int64)
+ b = np.array([5], dtype=np.int64)
+
+ index, ares, bres = libjoin.left_join_indexer(a, b)
+ tm.assert_numpy_array_equal(index, np.array([5], dtype=np.int64))
+ tm.assert_numpy_array_equal(ares, np.array([0], dtype=np.intp))
+ tm.assert_numpy_array_equal(bres, np.array([0], dtype=np.intp))
+
+
+def test_left_join_indexer2():
+ idx = np.array([1, 1, 2, 5], dtype=np.int64)
+ idx2 = np.array([1, 2, 5, 7, 9], dtype=np.int64)
+
+ res, lidx, ridx = libjoin.left_join_indexer(idx2, idx)
+
+ exp_res = np.array([1, 1, 2, 5, 7, 9], dtype=np.int64)
+ tm.assert_almost_equal(res, exp_res)
+
+ exp_lidx = np.array([0, 0, 1, 2, 3, 4], dtype=np.intp)
+ tm.assert_almost_equal(lidx, exp_lidx)
+
+ exp_ridx = np.array([0, 1, 2, 3, -1, -1], dtype=np.intp)
+ tm.assert_almost_equal(ridx, exp_ridx)
+
+
+def test_outer_join_indexer2():
+ idx = np.array([1, 1, 2, 5], dtype=np.int64)
+ idx2 = np.array([1, 2, 5, 7, 9], dtype=np.int64)
+
+ res, lidx, ridx = libjoin.outer_join_indexer(idx2, idx)
+
+ exp_res = np.array([1, 1, 2, 5, 7, 9], dtype=np.int64)
+ tm.assert_almost_equal(res, exp_res)
+
+ exp_lidx = np.array([0, 0, 1, 2, 3, 4], dtype=np.intp)
+ tm.assert_almost_equal(lidx, exp_lidx)
+
+ exp_ridx = np.array([0, 1, 2, 3, -1, -1], dtype=np.intp)
+ tm.assert_almost_equal(ridx, exp_ridx)
+
+
+def test_inner_join_indexer2():
+ idx = np.array([1, 1, 2, 5], dtype=np.int64)
+ idx2 = np.array([1, 2, 5, 7, 9], dtype=np.int64)
+
+ res, lidx, ridx = libjoin.inner_join_indexer(idx2, idx)
+
+ exp_res = np.array([1, 1, 2, 5], dtype=np.int64)
+ tm.assert_almost_equal(res, exp_res)
+
+ exp_lidx = np.array([0, 0, 1, 2], dtype=np.intp)
+ tm.assert_almost_equal(lidx, exp_lidx)
+
+ exp_ridx = np.array([0, 1, 2, 3], dtype=np.intp)
+ tm.assert_almost_equal(ridx, exp_ridx)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/libs/test_lib.py b/.venv/lib/python3.13/site-packages/pandas/tests/libs/test_lib.py
new file mode 100644
index 0000000000000000000000000000000000000000..792b2ef121cf2f0bd8266050217031b1c04f06c2
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/libs/test_lib.py
@@ -0,0 +1,358 @@
+import pickle
+
+import numpy as np
+import pytest
+
+from pandas._libs import (
+ Timedelta,
+ lib,
+ writers as libwriters,
+)
+from pandas.compat import IS64
+
+from pandas import Index
+import pandas._testing as tm
+
+
+class TestMisc:
+ def test_max_len_string_array(self):
+ arr = a = np.array(["foo", "b", np.nan], dtype="object")
+ assert libwriters.max_len_string_array(arr) == 3
+
+ # unicode
+ arr = a.astype("U").astype(object)
+ assert libwriters.max_len_string_array(arr) == 3
+
+ # bytes for python3
+ arr = a.astype("S").astype(object)
+ assert libwriters.max_len_string_array(arr) == 3
+
+ # raises
+ msg = "No matching signature found"
+ with pytest.raises(TypeError, match=msg):
+ libwriters.max_len_string_array(arr.astype("U"))
+
+ def test_fast_unique_multiple_list_gen_sort(self):
+ keys = [["p", "a"], ["n", "d"], ["a", "s"]]
+
+ gen = (key for key in keys)
+ expected = np.array(["a", "d", "n", "p", "s"])
+ out = lib.fast_unique_multiple_list_gen(gen, sort=True)
+ tm.assert_numpy_array_equal(np.array(out), expected)
+
+ gen = (key for key in keys)
+ expected = np.array(["p", "a", "n", "d", "s"])
+ out = lib.fast_unique_multiple_list_gen(gen, sort=False)
+ tm.assert_numpy_array_equal(np.array(out), expected)
+
+ def test_fast_multiget_timedelta_resos(self):
+ # This will become relevant for test_constructor_dict_timedelta64_index
+ # once Timedelta constructor preserves reso when passed a
+ # np.timedelta64 object
+ td = Timedelta(days=1)
+
+ mapping1 = {td: 1}
+ mapping2 = {td.as_unit("s"): 1}
+
+ oindex = Index([td * n for n in range(3)])._values.astype(object)
+
+ expected = lib.fast_multiget(mapping1, oindex)
+ result = lib.fast_multiget(mapping2, oindex)
+ tm.assert_numpy_array_equal(result, expected)
+
+ # case that can't be cast to td64ns
+ td = Timedelta(np.timedelta64(146000, "D"))
+ assert hash(td) == hash(td.as_unit("ms"))
+ assert hash(td) == hash(td.as_unit("us"))
+ mapping1 = {td: 1}
+ mapping2 = {td.as_unit("ms"): 1}
+
+ oindex = Index([td * n for n in range(3)])._values.astype(object)
+
+ expected = lib.fast_multiget(mapping1, oindex)
+ result = lib.fast_multiget(mapping2, oindex)
+ tm.assert_numpy_array_equal(result, expected)
+
+
+class TestIndexing:
+ def test_maybe_indices_to_slice_left_edge(self):
+ target = np.arange(100)
+
+ # slice
+ indices = np.array([], dtype=np.intp)
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+
+ assert isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(target[indices], target[maybe_slice])
+
+ @pytest.mark.parametrize("end", [1, 2, 5, 20, 99])
+ @pytest.mark.parametrize("step", [1, 2, 4])
+ def test_maybe_indices_to_slice_left_edge_not_slice_end_steps(self, end, step):
+ target = np.arange(100)
+ indices = np.arange(0, end, step, dtype=np.intp)
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+
+ assert isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(target[indices], target[maybe_slice])
+
+ # reverse
+ indices = indices[::-1]
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+
+ assert isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(target[indices], target[maybe_slice])
+
+ @pytest.mark.parametrize(
+ "case", [[2, 1, 2, 0], [2, 2, 1, 0], [0, 1, 2, 1], [-2, 0, 2], [2, 0, -2]]
+ )
+ def test_maybe_indices_to_slice_left_edge_not_slice(self, case):
+ # not slice
+ target = np.arange(100)
+ indices = np.array(case, dtype=np.intp)
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+
+ assert not isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(maybe_slice, indices)
+ tm.assert_numpy_array_equal(target[indices], target[maybe_slice])
+
+ @pytest.mark.parametrize("start", [0, 2, 5, 20, 97, 98])
+ @pytest.mark.parametrize("step", [1, 2, 4])
+ def test_maybe_indices_to_slice_right_edge(self, start, step):
+ target = np.arange(100)
+
+ # slice
+ indices = np.arange(start, 99, step, dtype=np.intp)
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+
+ assert isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(target[indices], target[maybe_slice])
+
+ # reverse
+ indices = indices[::-1]
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+
+ assert isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(target[indices], target[maybe_slice])
+
+ def test_maybe_indices_to_slice_right_edge_not_slice(self):
+ # not slice
+ target = np.arange(100)
+ indices = np.array([97, 98, 99, 100], dtype=np.intp)
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+
+ assert not isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(maybe_slice, indices)
+
+ msg = "index 100 is out of bounds for axis (0|1) with size 100"
+
+ with pytest.raises(IndexError, match=msg):
+ target[indices]
+ with pytest.raises(IndexError, match=msg):
+ target[maybe_slice]
+
+ indices = np.array([100, 99, 98, 97], dtype=np.intp)
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+
+ assert not isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(maybe_slice, indices)
+
+ with pytest.raises(IndexError, match=msg):
+ target[indices]
+ with pytest.raises(IndexError, match=msg):
+ target[maybe_slice]
+
+ @pytest.mark.parametrize(
+ "case", [[99, 97, 99, 96], [99, 99, 98, 97], [98, 98, 97, 96]]
+ )
+ def test_maybe_indices_to_slice_right_edge_cases(self, case):
+ target = np.arange(100)
+ indices = np.array(case, dtype=np.intp)
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+
+ assert not isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(maybe_slice, indices)
+ tm.assert_numpy_array_equal(target[indices], target[maybe_slice])
+
+ @pytest.mark.parametrize("step", [1, 2, 4, 5, 8, 9])
+ def test_maybe_indices_to_slice_both_edges(self, step):
+ target = np.arange(10)
+
+ # slice
+ indices = np.arange(0, 9, step, dtype=np.intp)
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+ assert isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(target[indices], target[maybe_slice])
+
+ # reverse
+ indices = indices[::-1]
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+ assert isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(target[indices], target[maybe_slice])
+
+ @pytest.mark.parametrize("case", [[4, 2, 0, -2], [2, 2, 1, 0], [0, 1, 2, 1]])
+ def test_maybe_indices_to_slice_both_edges_not_slice(self, case):
+ # not slice
+ target = np.arange(10)
+ indices = np.array(case, dtype=np.intp)
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+ assert not isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(maybe_slice, indices)
+ tm.assert_numpy_array_equal(target[indices], target[maybe_slice])
+
+ @pytest.mark.parametrize("start, end", [(2, 10), (5, 25), (65, 97)])
+ @pytest.mark.parametrize("step", [1, 2, 4, 20])
+ def test_maybe_indices_to_slice_middle(self, start, end, step):
+ target = np.arange(100)
+
+ # slice
+ indices = np.arange(start, end, step, dtype=np.intp)
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+
+ assert isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(target[indices], target[maybe_slice])
+
+ # reverse
+ indices = indices[::-1]
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+
+ assert isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(target[indices], target[maybe_slice])
+
+ @pytest.mark.parametrize(
+ "case", [[14, 12, 10, 12], [12, 12, 11, 10], [10, 11, 12, 11]]
+ )
+ def test_maybe_indices_to_slice_middle_not_slice(self, case):
+ # not slice
+ target = np.arange(100)
+ indices = np.array(case, dtype=np.intp)
+ maybe_slice = lib.maybe_indices_to_slice(indices, len(target))
+
+ assert not isinstance(maybe_slice, slice)
+ tm.assert_numpy_array_equal(maybe_slice, indices)
+ tm.assert_numpy_array_equal(target[indices], target[maybe_slice])
+
+ def test_maybe_booleans_to_slice(self):
+ arr = np.array([0, 0, 1, 1, 1, 0, 1], dtype=np.uint8)
+ result = lib.maybe_booleans_to_slice(arr)
+ assert result.dtype == np.bool_
+
+ result = lib.maybe_booleans_to_slice(arr[:0])
+ assert result == slice(0, 0)
+
+ def test_get_reverse_indexer(self):
+ indexer = np.array([-1, -1, 1, 2, 0, -1, 3, 4], dtype=np.intp)
+ result = lib.get_reverse_indexer(indexer, 5)
+ expected = np.array([4, 2, 3, 6, 7], dtype=np.intp)
+ tm.assert_numpy_array_equal(result, expected)
+
+ @pytest.mark.parametrize("dtype", ["int64", "int32"])
+ def test_is_range_indexer(self, dtype):
+ # GH#50592
+ left = np.arange(0, 100, dtype=dtype)
+ assert lib.is_range_indexer(left, 100)
+
+ @pytest.mark.skipif(
+ not IS64,
+ reason="2**31 is too big for Py_ssize_t on 32-bit. "
+ "It doesn't matter though since you cannot create an array that long on 32-bit",
+ )
+ @pytest.mark.parametrize("dtype", ["int64", "int32"])
+ def test_is_range_indexer_big_n(self, dtype):
+ # GH53616
+ left = np.arange(0, 100, dtype=dtype)
+
+ assert not lib.is_range_indexer(left, 2**31)
+
+ @pytest.mark.parametrize("dtype", ["int64", "int32"])
+ def test_is_range_indexer_not_equal(self, dtype):
+ # GH#50592
+ left = np.array([1, 2], dtype=dtype)
+ assert not lib.is_range_indexer(left, 2)
+
+ @pytest.mark.parametrize("dtype", ["int64", "int32"])
+ def test_is_range_indexer_not_equal_shape(self, dtype):
+ # GH#50592
+ left = np.array([0, 1, 2], dtype=dtype)
+ assert not lib.is_range_indexer(left, 2)
+
+
+def test_cache_readonly_preserve_docstrings():
+ # GH18197
+ assert Index.hasnans.__doc__ is not None
+
+
+def test_no_default_pickle(temp_file):
+ # GH#40397
+ obj = tm.round_trip_pickle(lib.no_default, temp_file)
+ assert obj is lib.no_default
+
+
+def test_ensure_string_array_copy():
+ # ensure the original array is not modified in case of copy=False with
+ # pickle-roundtripped object dtype array
+ # https://github.com/pandas-dev/pandas/issues/54654
+ arr = np.array(["a", None], dtype=object)
+ arr = pickle.loads(pickle.dumps(arr))
+ result = lib.ensure_string_array(arr, copy=False)
+ assert not np.shares_memory(arr, result)
+ assert arr[1] is None
+ assert result[1] is np.nan
+
+
+def test_ensure_string_array_list_of_lists():
+ # GH#61155: ensure list of lists doesn't get converted to string
+ arr = [list("test"), list("word")]
+ result = lib.ensure_string_array(arr)
+
+ # Each item in result should still be a list, not a stringified version
+ expected = np.array(["['t', 'e', 's', 't']", "['w', 'o', 'r', 'd']"], dtype=object)
+ tm.assert_numpy_array_equal(result, expected)
+
+
+def test_item_from_zerodim_for_subclasses():
+ # GH#62981 Ensure item_from_zerodim preserves subclasses of ndarray
+ # Define a custom ndarray subclass
+ class TestArray(np.ndarray):
+ def __new__(cls, input_array):
+ return np.asarray(input_array).view(cls)
+
+ def __array_finalize__(self, obj) -> None:
+ self._is_test_array = True
+
+ # Define test data
+ val_0_dim = 1
+ val_1_dim = [1, 2, 3]
+
+ # 0-dim and 1-dim numpy arrays
+ arr_0_dim = np.array(val_0_dim)
+ arr_1_dim = np.array(val_1_dim)
+
+ # 0-dim and 1-dim TestArray arrays
+ test_arr_0_dim = TestArray(val_0_dim)
+ test_arr_1_dim = TestArray(val_1_dim)
+
+ # Check that behavior did not change for regular numpy arrays
+ # Test with regular numpy 0-dim array
+ result = lib.item_from_zerodim(arr_0_dim)
+ expected = val_0_dim
+ assert result == expected
+ assert np.isscalar(result)
+
+ # Test with regular numpy 1-dim array
+ result = lib.item_from_zerodim(arr_1_dim)
+ expected = arr_1_dim
+ tm.assert_numpy_array_equal(result, expected)
+ assert isinstance(result, np.ndarray)
+
+ # Check that behaviour for subclasses now is as expected
+ # Test with TestArray 0-dim array
+ result = lib.item_from_zerodim(test_arr_0_dim)
+ expected = test_arr_0_dim
+ assert result == expected
+ assert isinstance(result, TestArray)
+
+ # Test with TestArray 1-dim array
+ result = lib.item_from_zerodim(test_arr_1_dim)
+ expected = test_arr_1_dim
+ assert np.all(result == expected)
+ assert isinstance(result, TestArray)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/libs/test_libalgos.py b/.venv/lib/python3.13/site-packages/pandas/tests/libs/test_libalgos.py
new file mode 100644
index 0000000000000000000000000000000000000000..42d09c72aab2baa9636093d172d864cbe0e41b12
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/libs/test_libalgos.py
@@ -0,0 +1,162 @@
+from datetime import datetime
+from itertools import permutations
+
+import numpy as np
+
+from pandas._libs import algos as libalgos
+
+import pandas._testing as tm
+
+
+def test_ensure_platform_int():
+ arr = np.arange(100, dtype=np.intp)
+
+ result = libalgos.ensure_platform_int(arr)
+ assert result is arr
+
+
+def test_is_lexsorted():
+ failure = [
+ np.array(
+ ([3] * 32) + ([2] * 32) + ([1] * 32) + ([0] * 32),
+ dtype="int64",
+ ),
+ np.array(
+ list(range(31))[::-1] * 4,
+ dtype="int64",
+ ),
+ ]
+
+ assert not libalgos.is_lexsorted(failure)
+
+
+def test_groupsort_indexer():
+ a = np.random.default_rng(2).integers(0, 1000, 100).astype(np.intp)
+ b = np.random.default_rng(2).integers(0, 1000, 100).astype(np.intp)
+
+ result = libalgos.groupsort_indexer(a, 1000)[0]
+
+ # need to use a stable sort
+ # np.argsort returns int, groupsort_indexer
+ # always returns intp
+ expected = np.argsort(a, kind="mergesort")
+ expected = expected.astype(np.intp)
+
+ tm.assert_numpy_array_equal(result, expected)
+
+ # compare with lexsort
+ # np.lexsort returns int, groupsort_indexer
+ # always returns intp
+ key = a * 1000 + b
+ result = libalgos.groupsort_indexer(key, 1000000)[0]
+ expected = np.lexsort((b, a))
+ expected = expected.astype(np.intp)
+
+ tm.assert_numpy_array_equal(result, expected)
+
+
+class TestPadBackfill:
+ def test_backfill(self):
+ old = np.array([1, 5, 10], dtype=np.int64)
+ new = np.array(list(range(12)), dtype=np.int64)
+
+ filler = libalgos.backfill["int64_t"](old, new)
+
+ expect_filler = np.array([0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, -1], dtype=np.intp)
+ tm.assert_numpy_array_equal(filler, expect_filler)
+
+ # corner case
+ old = np.array([1, 4], dtype=np.int64)
+ new = np.array(list(range(5, 10)), dtype=np.int64)
+ filler = libalgos.backfill["int64_t"](old, new)
+
+ expect_filler = np.array([-1, -1, -1, -1, -1], dtype=np.intp)
+ tm.assert_numpy_array_equal(filler, expect_filler)
+
+ def test_pad(self):
+ old = np.array([1, 5, 10], dtype=np.int64)
+ new = np.array(list(range(12)), dtype=np.int64)
+
+ filler = libalgos.pad["int64_t"](old, new)
+
+ expect_filler = np.array([-1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 2], dtype=np.intp)
+ tm.assert_numpy_array_equal(filler, expect_filler)
+
+ # corner case
+ old = np.array([5, 10], dtype=np.int64)
+ new = np.arange(5, dtype=np.int64)
+ filler = libalgos.pad["int64_t"](old, new)
+ expect_filler = np.array([-1, -1, -1, -1, -1], dtype=np.intp)
+ tm.assert_numpy_array_equal(filler, expect_filler)
+
+ def test_pad_backfill_object_segfault(self):
+ old = np.array([], dtype="O")
+ new = np.array([datetime(2010, 12, 31)], dtype="O")
+
+ result = libalgos.pad["object"](old, new)
+ expected = np.array([-1], dtype=np.intp)
+ tm.assert_numpy_array_equal(result, expected)
+
+ result = libalgos.pad["object"](new, old)
+ expected = np.array([], dtype=np.intp)
+ tm.assert_numpy_array_equal(result, expected)
+
+ result = libalgos.backfill["object"](old, new)
+ expected = np.array([-1], dtype=np.intp)
+ tm.assert_numpy_array_equal(result, expected)
+
+ result = libalgos.backfill["object"](new, old)
+ expected = np.array([], dtype=np.intp)
+ tm.assert_numpy_array_equal(result, expected)
+
+
+class TestInfinity:
+ def test_infinity_sort(self):
+ # GH#13445
+ # numpy's argsort can be unhappy if something is less than
+ # itself. Instead, let's give our infinities a self-consistent
+ # ordering, but outside the float extended real line.
+
+ Inf = libalgos.Infinity()
+ NegInf = libalgos.NegInfinity()
+
+ ref_nums = [NegInf, float("-inf"), -1e100, 0, 1e100, float("inf"), Inf]
+
+ assert all(Inf >= x for x in ref_nums)
+ assert all(Inf > x or x is Inf for x in ref_nums)
+ assert Inf >= Inf and Inf == Inf
+ assert not Inf < Inf and not Inf > Inf
+ assert libalgos.Infinity() == libalgos.Infinity()
+ assert not libalgos.Infinity() != libalgos.Infinity()
+
+ assert all(NegInf <= x for x in ref_nums)
+ assert all(NegInf < x or x is NegInf for x in ref_nums)
+ assert NegInf <= NegInf and NegInf == NegInf
+ assert not NegInf < NegInf and not NegInf > NegInf
+ assert libalgos.NegInfinity() == libalgos.NegInfinity()
+ assert not libalgos.NegInfinity() != libalgos.NegInfinity()
+
+ for perm in permutations(ref_nums):
+ assert sorted(perm) == ref_nums
+
+ # smoke tests
+ np.array([libalgos.Infinity()] * 32).argsort()
+ np.array([libalgos.NegInfinity()] * 32).argsort()
+
+ def test_infinity_against_nan(self):
+ Inf = libalgos.Infinity()
+ NegInf = libalgos.NegInfinity()
+
+ assert not Inf > np.nan
+ assert not Inf >= np.nan
+ assert not Inf < np.nan
+ assert not Inf <= np.nan
+ assert not Inf == np.nan
+ assert Inf != np.nan
+
+ assert not NegInf > np.nan
+ assert not NegInf >= np.nan
+ assert not NegInf < np.nan
+ assert not NegInf <= np.nan
+ assert not NegInf == np.nan
+ assert NegInf != np.nan
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/common.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/common.py
new file mode 100644
index 0000000000000000000000000000000000000000..588bbf88e856243ce539a5378b22468228bcf280
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/common.py
@@ -0,0 +1,579 @@
+"""
+Module consolidating common testing functions for checking plotting.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from pandas.core.dtypes.api import is_list_like
+
+import pandas as pd
+from pandas import Series
+import pandas._testing as tm
+
+if TYPE_CHECKING:
+ from collections.abc import Sequence
+
+ from matplotlib.axes import Axes
+
+
+def _check_legend_labels(axes, labels=None, visible=True):
+ """
+ Check each axes has expected legend labels
+
+ Parameters
+ ----------
+ axes : matplotlib Axes object, or its list-like
+ labels : list-like
+ expected legend labels
+ visible : bool
+ expected legend visibility. labels are checked only when visible is
+ True
+ """
+ if visible and (labels is None):
+ raise ValueError("labels must be specified when visible is True")
+ axes = _flatten_visible(axes)
+ for ax in axes:
+ if visible:
+ assert ax.get_legend() is not None
+ _check_text_labels(ax.get_legend().get_texts(), labels)
+ else:
+ assert ax.get_legend() is None
+
+
+def _check_legend_marker(ax, expected_markers=None, visible=True):
+ """
+ Check ax has expected legend markers
+
+ Parameters
+ ----------
+ ax : matplotlib Axes object
+ expected_markers : list-like
+ expected legend markers
+ visible : bool
+ expected legend visibility. labels are checked only when visible is
+ True
+ """
+ if visible and (expected_markers is None):
+ raise ValueError("Markers must be specified when visible is True")
+ if visible:
+ handles, _ = ax.get_legend_handles_labels()
+ markers = [handle.get_marker() for handle in handles]
+ assert markers == expected_markers
+ else:
+ assert ax.get_legend() is None
+
+
+def _check_data(xp, rs):
+ """
+ Check each axes has identical lines
+
+ Parameters
+ ----------
+ xp : matplotlib Axes object
+ rs : matplotlib Axes object
+ """
+ xp_lines = xp.get_lines()
+ rs_lines = rs.get_lines()
+
+ assert len(xp_lines) == len(rs_lines)
+ for xpl, rsl in zip(xp_lines, rs_lines, strict=True):
+ xpdata = xpl.get_xydata()
+ rsdata = rsl.get_xydata()
+ tm.assert_almost_equal(xpdata, rsdata)
+
+
+def _check_visible(collections, visible=True):
+ """
+ Check each artist is visible or not
+
+ Parameters
+ ----------
+ collections : matplotlib Artist or its list-like
+ target Artist or its list or collection
+ visible : bool
+ expected visibility
+ """
+ from matplotlib.collections import Collection
+
+ if not isinstance(collections, Collection) and not is_list_like(collections):
+ collections = [collections]
+
+ for patch in collections:
+ assert patch.get_visible() == visible
+
+
+def _check_patches_all_filled(axes: Axes | Sequence[Axes], filled: bool = True) -> None:
+ """
+ Check for each artist whether it is filled or not
+
+ Parameters
+ ----------
+ axes : matplotlib Axes object, or its list-like
+ filled : bool
+ expected filling
+ """
+
+ axes = _flatten_visible(axes)
+ for ax in axes:
+ for patch in ax.patches:
+ assert patch.fill == filled
+
+
+def _get_colors_mapped(series, colors):
+ unique = series.unique()
+ # unique and colors length can be differed
+ # depending on slice value
+ mapped = dict(zip(unique, colors))
+ return [mapped[v] for v in series.values]
+
+
+def _check_colors(collections, linecolors=None, facecolors=None, mapping=None):
+ """
+ Check each artist has expected line colors and face colors
+
+ Parameters
+ ----------
+ collections : list-like
+ list or collection of target artist
+ linecolors : list-like which has the same length as collections
+ list of expected line colors
+ facecolors : list-like which has the same length as collections
+ list of expected face colors
+ mapping : Series
+ Series used for color grouping key
+ used for andrew_curves, parallel_coordinates, radviz test
+ """
+ from matplotlib import colors
+ from matplotlib.collections import (
+ Collection,
+ LineCollection,
+ PolyCollection,
+ )
+ from matplotlib.lines import Line2D
+
+ conv = colors.ColorConverter
+ if linecolors is not None:
+ if mapping is not None:
+ linecolors = _get_colors_mapped(mapping, linecolors)
+ linecolors = linecolors[: len(collections)]
+
+ assert len(collections) == len(linecolors)
+ for patch, color in zip(collections, linecolors, strict=True):
+ if isinstance(patch, Line2D):
+ result = patch.get_color()
+ # Line2D may contains string color expression
+ result = conv.to_rgba(result)
+ elif isinstance(patch, (PolyCollection, LineCollection)):
+ result = tuple(patch.get_edgecolor()[0])
+ else:
+ result = patch.get_edgecolor()
+
+ expected = conv.to_rgba(color)
+ assert result == expected
+
+ if facecolors is not None:
+ if mapping is not None:
+ facecolors = _get_colors_mapped(mapping, facecolors)
+ facecolors = facecolors[: len(collections)]
+
+ assert len(collections) == len(facecolors)
+ for patch, color in zip(collections, facecolors, strict=True):
+ if isinstance(patch, Collection):
+ # returned as list of np.array
+ result = patch.get_facecolor()[0]
+ else:
+ result = patch.get_facecolor()
+
+ if isinstance(result, np.ndarray):
+ result = tuple(result)
+
+ expected = conv.to_rgba(color)
+ assert result == expected
+
+
+def _check_text_labels(texts, expected):
+ """
+ Check each text has expected labels
+
+ Parameters
+ ----------
+ texts : matplotlib Text object, or its list-like
+ target text, or its list
+ expected : str or list-like which has the same length as texts
+ expected text label, or its list
+ """
+ if not is_list_like(texts):
+ assert texts.get_text() == expected
+ else:
+ labels = [t.get_text() for t in texts]
+ assert len(labels) == len(expected)
+ for label, e in zip(labels, expected, strict=True):
+ assert label == e
+
+
+def _check_ticks_props(axes, xlabelsize=None, xrot=None, ylabelsize=None, yrot=None):
+ """
+ Check each axes has expected tick properties
+
+ Parameters
+ ----------
+ axes : matplotlib Axes object, or its list-like
+ xlabelsize : number
+ expected xticks font size
+ xrot : number
+ expected xticks rotation
+ ylabelsize : number
+ expected yticks font size
+ yrot : number
+ expected yticks rotation
+ """
+ from matplotlib.ticker import NullFormatter
+
+ axes = _flatten_visible(axes)
+ for ax in axes:
+ if xlabelsize is not None or xrot is not None:
+ if isinstance(ax.xaxis.get_minor_formatter(), NullFormatter):
+ # If minor ticks has NullFormatter, rot / fontsize are not
+ # retained
+ labels = ax.get_xticklabels()
+ else:
+ labels = ax.get_xticklabels() + ax.get_xticklabels(minor=True)
+
+ for label in labels:
+ if xlabelsize is not None:
+ tm.assert_almost_equal(label.get_fontsize(), xlabelsize)
+ if xrot is not None:
+ tm.assert_almost_equal(label.get_rotation(), xrot)
+
+ if ylabelsize is not None or yrot is not None:
+ if isinstance(ax.yaxis.get_minor_formatter(), NullFormatter):
+ labels = ax.get_yticklabels()
+ else:
+ labels = ax.get_yticklabels() + ax.get_yticklabels(minor=True)
+
+ for label in labels:
+ if ylabelsize is not None:
+ tm.assert_almost_equal(label.get_fontsize(), ylabelsize)
+ if yrot is not None:
+ tm.assert_almost_equal(label.get_rotation(), yrot)
+
+
+def _check_ax_scales(axes, xaxis="linear", yaxis="linear"):
+ """
+ Check each axes has expected scales
+
+ Parameters
+ ----------
+ axes : matplotlib Axes object, or its list-like
+ xaxis : {'linear', 'log'}
+ expected xaxis scale
+ yaxis : {'linear', 'log'}
+ expected yaxis scale
+ """
+ axes = _flatten_visible(axes)
+ for ax in axes:
+ assert ax.xaxis.get_scale() == xaxis
+ assert ax.yaxis.get_scale() == yaxis
+
+
+def _check_axes_shape(axes, axes_num=None, layout=None, figsize=None):
+ """
+ Check expected number of axes is drawn in expected layout
+
+ Parameters
+ ----------
+ axes : matplotlib Axes object, or its list-like
+ axes_num : number
+ expected number of axes. Unnecessary axes should be set to
+ invisible.
+ layout : tuple
+ expected layout, (expected number of rows , columns)
+ figsize : tuple
+ expected figsize. default is matplotlib default
+ """
+ from pandas.plotting._matplotlib.tools import flatten_axes
+
+ if figsize is None:
+ figsize = (6.4, 4.8)
+ visible_axes = _flatten_visible(axes)
+
+ if axes_num is not None:
+ assert len(visible_axes) == axes_num
+ for ax in visible_axes:
+ # check something drawn on visible axes
+ assert len(ax.get_children()) > 0
+
+ if layout is not None:
+ x_set = set()
+ y_set = set()
+ for ax in flatten_axes(axes):
+ # check axes coordinates to estimate layout
+ points = ax.get_position().get_points()
+ x_set.add(points[0][0])
+ y_set.add(points[0][1])
+ result = (len(y_set), len(x_set))
+ assert result == layout
+
+ tm.assert_numpy_array_equal(
+ visible_axes[0].figure.get_size_inches(),
+ np.array(figsize, dtype=np.float64),
+ )
+
+
+def _flatten_visible(axes: Axes | Sequence[Axes]) -> Sequence[Axes]:
+ """
+ Flatten axes, and filter only visible
+
+ Parameters
+ ----------
+ axes : matplotlib Axes object, or its list-like
+
+ """
+ from pandas.plotting._matplotlib.tools import flatten_axes
+
+ axes_ndarray = flatten_axes(axes)
+ axes = [ax for ax in axes_ndarray if ax.get_visible()]
+ return axes
+
+
+def _check_has_errorbars(axes, xerr=0, yerr=0):
+ """
+ Check axes has expected number of errorbars
+
+ Parameters
+ ----------
+ axes : matplotlib Axes object, or its list-like
+ xerr : number
+ expected number of x errorbar
+ yerr : number
+ expected number of y errorbar
+ """
+ axes = _flatten_visible(axes)
+ for ax in axes:
+ containers = ax.containers
+ xerr_count = 0
+ yerr_count = 0
+ for c in containers:
+ has_xerr = getattr(c, "has_xerr", False)
+ has_yerr = getattr(c, "has_yerr", False)
+ if has_xerr:
+ xerr_count += 1
+ if has_yerr:
+ yerr_count += 1
+ assert xerr == xerr_count
+ assert yerr == yerr_count
+
+
+def _check_box_return_type(
+ returned, return_type, expected_keys=None, check_ax_title=True
+):
+ """
+ Check box returned type is correct
+
+ Parameters
+ ----------
+ returned : object to be tested, returned from boxplot
+ return_type : str
+ return_type passed to boxplot
+ expected_keys : list-like, optional
+ group labels in subplot case. If not passed,
+ the function checks assuming boxplot uses single ax
+ check_ax_title : bool
+ Whether to check the ax.title is the same as expected_key
+ Intended to be checked by calling from ``boxplot``.
+ Normal ``plot`` doesn't attach ``ax.title``, it must be disabled.
+ """
+ from matplotlib.axes import Axes
+
+ types = {"dict": dict, "axes": Axes, "both": tuple}
+ if expected_keys is None:
+ # should be fixed when the returning default is changed
+ if return_type is None:
+ return_type = "dict"
+
+ assert isinstance(returned, types[return_type])
+ if return_type == "both":
+ assert isinstance(returned.ax, Axes)
+ assert isinstance(returned.lines, dict)
+ else:
+ # should be fixed when the returning default is changed
+ if return_type is None:
+ for r in _flatten_visible(returned):
+ assert isinstance(r, Axes)
+ return
+
+ assert isinstance(returned, Series)
+
+ assert sorted(returned.keys()) == sorted(expected_keys)
+ for key, value in returned.items():
+ assert isinstance(value, types[return_type])
+ # check returned dict has correct mapping
+ if return_type == "axes":
+ if check_ax_title:
+ assert value.get_title() == key
+ elif return_type == "both":
+ if check_ax_title:
+ assert value.ax.get_title() == key
+ assert isinstance(value.ax, Axes)
+ assert isinstance(value.lines, dict)
+ elif return_type == "dict":
+ line = value["medians"][0]
+ axes = line.axes
+ if check_ax_title:
+ assert axes.get_title() == key
+ else:
+ raise AssertionError
+
+
+def _check_grid_settings(obj, kinds, kws=None):
+ # Make sure plot defaults to rcParams['axes.grid'] setting, GH 9792
+
+ import matplotlib as mpl
+
+ def is_grid_on():
+ xticks = mpl.pyplot.gca().xaxis.get_major_ticks()
+ yticks = mpl.pyplot.gca().yaxis.get_major_ticks()
+ xoff = all(not g.gridline.get_visible() for g in xticks)
+ yoff = all(not g.gridline.get_visible() for g in yticks)
+
+ return not (xoff and yoff)
+
+ if kws is None:
+ kws = {}
+ spndx = 1
+ for kind in kinds:
+ mpl.pyplot.subplot(1, 4 * len(kinds), spndx)
+ spndx += 1
+ mpl.rc("axes", grid=False)
+ obj.plot(kind=kind, **kws)
+ assert not is_grid_on()
+ mpl.pyplot.clf()
+
+ mpl.pyplot.subplot(1, 4 * len(kinds), spndx)
+ spndx += 1
+ mpl.rc("axes", grid=True)
+ obj.plot(kind=kind, grid=False, **kws)
+ assert not is_grid_on()
+ mpl.pyplot.clf()
+
+ if kind not in ["pie", "hexbin", "scatter"]:
+ mpl.pyplot.subplot(1, 4 * len(kinds), spndx)
+ spndx += 1
+ mpl.rc("axes", grid=True)
+ obj.plot(kind=kind, **kws)
+ assert is_grid_on()
+ mpl.pyplot.clf()
+
+ mpl.pyplot.subplot(1, 4 * len(kinds), spndx)
+ spndx += 1
+ mpl.rc("axes", grid=False)
+ obj.plot(kind=kind, grid=True, **kws)
+ assert is_grid_on()
+ mpl.pyplot.clf()
+
+
+def _unpack_cycler(rcParams, field="color"):
+ """
+ Auxiliary function for correctly unpacking cycler after MPL >= 1.5
+ """
+ return [v[field] for v in rcParams["axes.prop_cycle"]]
+
+
+def get_x_axis(ax):
+ return ax._shared_axes["x"]
+
+
+def get_y_axis(ax):
+ return ax._shared_axes["y"]
+
+
+def assert_is_valid_plot_return_object(objs) -> None:
+ from matplotlib.artist import Artist
+ from matplotlib.axes import Axes
+
+ if isinstance(objs, (Series, np.ndarray)):
+ if isinstance(objs, Series):
+ objs = objs._values
+ for el in objs.reshape(-1):
+ msg = (
+ "one of 'objs' is not a matplotlib Axes instance, "
+ f"type encountered {type(el).__name__!r}"
+ )
+ assert isinstance(el, (Axes, dict)), msg
+ else:
+ msg = (
+ "objs is neither an ndarray of Artist instances nor a single "
+ "ArtistArtist instance, tuple, or dict, 'objs' is a "
+ f"{type(objs).__name__!r}"
+ )
+ assert isinstance(objs, (Artist, tuple, dict)), msg
+
+
+def _check_plot_works(f, default_axes=False, **kwargs):
+ """
+ Create plot and ensure that plot return object is valid.
+
+ Parameters
+ ----------
+ f : func
+ Plotting function.
+ default_axes : bool, optional
+ If False (default):
+ - If `ax` not in `kwargs`, then create subplot(211) and plot there
+ - Create new subplot(212) and plot there as well
+ - Mind special corner case for bootstrap_plot (see `_gen_two_subplots`)
+ If True:
+ - Simply run plotting function with kwargs provided
+ - All required axes instances will be created automatically
+ - It is recommended to use it when the plotting function
+ creates multiple axes itself. It helps avoid warnings like
+ 'UserWarning: To output multiple subplots,
+ the figure containing the passed axes is being cleared'
+ **kwargs
+ Keyword arguments passed to the plotting function.
+
+ Returns
+ -------
+ Plot object returned by the last plotting.
+ """
+ import matplotlib.pyplot as plt
+
+ if default_axes:
+ gen_plots = _gen_default_plot
+ else:
+ gen_plots = _gen_two_subplots
+
+ ret = None
+ fig = kwargs.get("figure", plt.gcf())
+ fig.clf()
+
+ for ret in gen_plots(f, fig, **kwargs):
+ assert_is_valid_plot_return_object(ret)
+
+ return ret
+
+
+def _gen_default_plot(f, fig, **kwargs):
+ """
+ Create plot in a default way.
+ """
+ yield f(**kwargs)
+
+
+def _gen_two_subplots(f, fig, **kwargs):
+ """
+ Create plot on two subplots forcefully created.
+ """
+ if "ax" not in kwargs:
+ fig.add_subplot(211)
+ yield f(**kwargs)
+
+ if f is pd.plotting.bootstrap_plot:
+ assert "ax" not in kwargs
+ else:
+ kwargs["ax"] = fig.add_subplot(212)
+ yield f(**kwargs)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/conftest.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..eb5a1f1f6382e3fe48fd0fa4050a673160807a9a
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/conftest.py
@@ -0,0 +1,39 @@
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ to_datetime,
+)
+
+
+@pytest.fixture(autouse=True)
+def autouse_mpl_cleanup(mpl_cleanup):
+ pass
+
+
+@pytest.fixture
+def hist_df():
+ n = 50
+ rng = np.random.default_rng(10)
+ gender = rng.choice(["Male", "Female"], size=n)
+ classroom = rng.choice(["A", "B", "C"], size=n)
+
+ hist_df = DataFrame(
+ {
+ "gender": gender,
+ "classroom": classroom,
+ "height": rng.normal(66, 4, size=n),
+ "weight": rng.normal(161, 32, size=n),
+ "category": rng.integers(4, size=n),
+ "datetime": to_datetime(
+ rng.integers(
+ 812419200000000000,
+ 819331200000000000,
+ size=n,
+ dtype=np.int64,
+ )
+ ),
+ }
+ )
+ return hist_df
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame.py
new file mode 100644
index 0000000000000000000000000000000000000000..2696005b0091d6f192bdcedd1de486df76e168aa
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame.py
@@ -0,0 +1,2661 @@
+"""Test cases for DataFrame.plot"""
+
+from datetime import (
+ date,
+ datetime,
+)
+import itertools
+import re
+import string
+
+import numpy as np
+import pytest
+
+import pandas.util._test_decorators as td
+
+from pandas.core.dtypes.api import is_list_like
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ MultiIndex,
+ PeriodIndex,
+ Series,
+ bdate_range,
+ date_range,
+ option_context,
+ plotting,
+)
+import pandas._testing as tm
+from pandas.tests.plotting.common import (
+ _check_ax_scales,
+ _check_axes_shape,
+ _check_box_return_type,
+ _check_colors,
+ _check_data,
+ _check_grid_settings,
+ _check_has_errorbars,
+ _check_legend_labels,
+ _check_plot_works,
+ _check_text_labels,
+ _check_ticks_props,
+ _check_visible,
+ get_y_axis,
+)
+from pandas.util.version import Version
+
+from pandas.io.formats.printing import pprint_thing
+
+mpl = pytest.importorskip("matplotlib")
+plt = pytest.importorskip("matplotlib.pyplot")
+
+pytestmark = [
+ pytest.mark.filterwarnings(
+ "ignore:divide by zero encountered in scalar divide:RuntimeWarning"
+ ),
+ pytest.mark.filterwarnings(
+ "ignore:invalid value encountered in scalar multiply:RuntimeWarning"
+ ),
+]
+
+
+class TestDataFramePlots:
+ @pytest.mark.slow
+ def test_plot(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ _check_plot_works(df.plot, grid=False)
+
+ @pytest.mark.slow
+ def test_plot_subplots(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ # _check_plot_works adds an ax so use default_axes=True to avoid warning
+ axes = _check_plot_works(df.plot, default_axes=True, subplots=True)
+ _check_axes_shape(axes, axes_num=4, layout=(4, 1))
+
+ @pytest.mark.slow
+ def test_plot_subplots_negative_layout(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ axes = _check_plot_works(
+ df.plot,
+ default_axes=True,
+ subplots=True,
+ layout=(-1, 2),
+ )
+ _check_axes_shape(axes, axes_num=4, layout=(2, 2))
+
+ @pytest.mark.slow
+ def test_plot_subplots_use_index(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ axes = _check_plot_works(
+ df.plot,
+ default_axes=True,
+ subplots=True,
+ use_index=False,
+ )
+ _check_ticks_props(axes, xrot=0)
+ _check_axes_shape(axes, axes_num=4, layout=(4, 1))
+
+ @pytest.mark.xfail(reason="Api changed in 3.6.0")
+ @pytest.mark.slow
+ def test_plot_invalid_arg(self):
+ df = DataFrame({"x": [1, 2], "y": [3, 4]})
+ msg = "'Line2D' object has no property 'blarg'"
+ with pytest.raises(AttributeError, match=msg):
+ df.plot.line(blarg=True)
+
+ @pytest.mark.slow
+ def test_plot_tick_props(self):
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=list(string.ascii_letters[:10]),
+ )
+
+ ax = _check_plot_works(df.plot, use_index=True)
+ _check_ticks_props(ax, xrot=0)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"yticks": [1, 5, 10]},
+ {"xticks": [1, 5, 10]},
+ {"ylim": (-100, 100), "xlim": (-100, 100)},
+ {"default_axes": True, "subplots": True, "title": "blah"},
+ ],
+ )
+ def test_plot_other_args(self, kwargs):
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=list(string.ascii_letters[:10]),
+ )
+ _check_plot_works(df.plot, **kwargs)
+
+ @pytest.mark.slow
+ def test_plot_visible_ax(self):
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=list(string.ascii_letters[:10]),
+ )
+ # We have to redo it here because _check_plot_works does two plots,
+ # once without an ax kwarg and once with an ax kwarg and the new sharex
+ # behaviour does not remove the visibility of the latter axis (as ax is
+ # present). see: https://github.com/pandas-dev/pandas/issues/9737
+
+ axes = df.plot(subplots=True, title="blah")
+ _check_axes_shape(axes, axes_num=3, layout=(3, 1))
+ for ax in axes[:2]:
+ _check_visible(ax.xaxis) # xaxis must be visible for grid
+ _check_visible(ax.get_xticklabels(), visible=False)
+ _check_visible(ax.get_xticklabels(minor=True), visible=False)
+ _check_visible([ax.xaxis.get_label()], visible=False)
+ for ax in [axes[2]]:
+ _check_visible(ax.xaxis)
+ _check_visible(ax.get_xticklabels())
+ _check_visible([ax.xaxis.get_label()])
+ _check_ticks_props(ax, xrot=0)
+
+ @pytest.mark.slow
+ def test_plot_title(self):
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=list(string.ascii_letters[:10]),
+ )
+ _check_plot_works(df.plot, title="blah")
+
+ @pytest.mark.slow
+ def test_plot_multiindex(self):
+ tuples = zip(string.ascii_letters[:10], range(10), strict=True)
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=MultiIndex.from_tuples(tuples),
+ )
+ ax = _check_plot_works(df.plot, use_index=True)
+ _check_ticks_props(ax, xrot=0)
+
+ @pytest.mark.slow
+ def test_plot_multiindex_unicode(self):
+ # unicode
+ index = MultiIndex.from_tuples(
+ [
+ ("\u03b1", 0),
+ ("\u03b1", 1),
+ ("\u03b2", 2),
+ ("\u03b2", 3),
+ ("\u03b3", 4),
+ ("\u03b3", 5),
+ ("\u03b4", 6),
+ ("\u03b4", 7),
+ ],
+ names=["i0", "i1"],
+ )
+ columns = MultiIndex.from_tuples(
+ [("bar", "\u0394"), ("bar", "\u0395")], names=["c0", "c1"]
+ )
+ df = DataFrame(
+ np.random.default_rng(2).integers(0, 10, (8, 2)),
+ columns=columns,
+ index=index,
+ )
+ _check_plot_works(df.plot, title="\u03a3")
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("layout", [None, (-1, 1)])
+ def test_plot_single_column_bar(self, layout):
+ # GH 6951
+ # Test with single column
+ df = DataFrame({"x": np.random.default_rng(2).random(10)})
+ axes = _check_plot_works(df.plot.bar, subplots=True, layout=layout)
+ _check_axes_shape(axes, axes_num=1, layout=(1, 1))
+
+ @pytest.mark.slow
+ def test_plot_passed_ax(self):
+ # When ax is supplied and required number of axes is 1,
+ # passed ax should be used:
+ df = DataFrame({"x": np.random.default_rng(2).random(10)})
+ _, ax = mpl.pyplot.subplots()
+ axes = df.plot.bar(subplots=True, ax=ax)
+ assert len(axes) == 1
+ result = ax.axes
+ assert result is axes[0]
+
+ @pytest.mark.parametrize(
+ "cols, x, y",
+ [
+ [list("ABCDE"), "A", "B"],
+ [["A", "B"], "A", "B"],
+ [["C", "A"], "C", "A"],
+ [["A", "C"], "A", "C"],
+ [["B", "C"], "B", "C"],
+ [["A", "D"], "A", "D"],
+ [["A", "E"], "A", "E"],
+ ],
+ )
+ def test_nullable_int_plot(self, cols, x, y):
+ # GH 32073
+ dates = ["2008", "2009", None, "2011", "2012"]
+ df = DataFrame(
+ {
+ "A": [1, 2, 3, 4, 5],
+ "B": [1, 2, 3, 4, 5],
+ "C": np.array([7, 5, np.nan, 3, 2], dtype=object),
+ "D": pd.to_datetime(dates, format="%Y").view("i8"),
+ "E": pd.to_datetime(dates, format="%Y", utc=True).view("i8"),
+ }
+ )
+
+ _check_plot_works(df[cols].plot, x=x, y=y)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("plot", ["line", "bar", "hist", "pie"])
+ def test_integer_array_plot_series(self, plot):
+ # GH 25587
+ arr = pd.array([1, 2, 3, 4], dtype="UInt32")
+
+ s = Series(arr)
+ _check_plot_works(getattr(s.plot, plot))
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "plot, kwargs",
+ [
+ ["line", {}],
+ ["bar", {}],
+ ["hist", {}],
+ ["pie", {"y": "y"}],
+ ["scatter", {"x": "x", "y": "y"}],
+ ["hexbin", {"x": "x", "y": "y"}],
+ ],
+ )
+ def test_integer_array_plot_df(self, plot, kwargs):
+ # GH 25587
+ arr = pd.array([1, 2, 3, 4], dtype="UInt32")
+ df = DataFrame({"x": arr, "y": arr})
+ _check_plot_works(getattr(df.plot, plot), **kwargs)
+
+ def test_nonnumeric_exclude(self):
+ df = DataFrame({"A": ["x", "y", "z"], "B": [1, 2, 3]})
+ ax = df.plot()
+ assert len(ax.get_lines()) == 1 # B was plotted
+
+ def test_implicit_label(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 3)), columns=["a", "b", "c"]
+ )
+ ax = df.plot(x="a", y="b")
+ _check_text_labels(ax.xaxis.get_label(), "a")
+
+ def test_donot_overwrite_index_name(self):
+ # GH 8494
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((2, 2)), columns=["a", "b"]
+ )
+ df.index.name = "NAME"
+ df.plot(y="b", label="LABEL")
+ assert df.index.name == "NAME"
+
+ def test_plot_xy(self):
+ # columns.inferred_type == 'string'
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((5, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=5, freq="B"),
+ )
+ _check_data(df.plot(x=0, y=1), df.set_index("A")["B"].plot())
+ _check_data(df.plot(x=0), df.set_index("A").plot())
+ _check_data(df.plot(y=0), df.B.plot())
+ _check_data(df.plot(x="A", y="B"), df.set_index("A").B.plot())
+ _check_data(df.plot(x="A"), df.set_index("A").plot())
+ _check_data(df.plot(y="B"), df.B.plot())
+
+ def test_plot_xy_int_cols(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((5, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=5, freq="B"),
+ )
+ # columns.inferred_type == 'integer'
+ df.columns = np.arange(1, len(df.columns) + 1)
+ _check_data(df.plot(x=1, y=2), df.set_index(1)[2].plot())
+ _check_data(df.plot(x=1), df.set_index(1).plot())
+ _check_data(df.plot(y=1), df[1].plot())
+
+ def test_plot_xy_figsize_and_title(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((5, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=5, freq="B"),
+ )
+ # figsize and title
+ ax = df.plot(x=1, y=2, title="Test", figsize=(16, 8))
+ _check_text_labels(ax.title, "Test")
+ _check_axes_shape(ax, axes_num=1, layout=(1, 1), figsize=(16.0, 8.0))
+
+ # columns.inferred_type == 'mixed'
+ # TODO add MultiIndex test
+
+ @pytest.mark.parametrize(
+ "input_log, expected_log", [(True, "log"), ("sym", "symlog")]
+ )
+ def test_logscales(self, input_log, expected_log):
+ df = DataFrame({"a": np.arange(100)}, index=np.arange(100))
+
+ ax = df.plot(logy=input_log)
+ _check_ax_scales(ax, yaxis=expected_log)
+ assert ax.get_yscale() == expected_log
+
+ ax = df.plot(logx=input_log)
+ _check_ax_scales(ax, xaxis=expected_log)
+ assert ax.get_xscale() == expected_log
+
+ ax = df.plot(loglog=input_log)
+ _check_ax_scales(ax, xaxis=expected_log, yaxis=expected_log)
+ assert ax.get_xscale() == expected_log
+ assert ax.get_yscale() == expected_log
+
+ @pytest.mark.parametrize("input_param", ["logx", "logy", "loglog"])
+ def test_invalid_logscale(self, input_param):
+ # GH: 24867
+ df = DataFrame({"a": np.arange(100)}, index=np.arange(100))
+
+ msg = f"keyword '{input_param}' should be bool, None, or 'sym', not 'sm'"
+ with pytest.raises(ValueError, match=msg):
+ df.plot(**{input_param: "sm"})
+
+ msg = f"PiePlot ignores the '{input_param}' keyword"
+ with tm.assert_produces_warning(UserWarning, match=msg):
+ df.plot.pie(subplots=True, **{input_param: True})
+
+ def test_xcompat(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ ax = df.plot(x_compat=True)
+ lines = ax.get_lines()
+ assert not isinstance(lines[0].get_xdata(), PeriodIndex)
+ _check_ticks_props(ax, xrot=30)
+
+ def test_xcompat_plot_params(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ plotting.plot_params["xaxis.compat"] = True
+ ax = df.plot()
+ lines = ax.get_lines()
+ assert not isinstance(lines[0].get_xdata(), PeriodIndex)
+ _check_ticks_props(ax, xrot=30)
+
+ def test_xcompat_plot_params_x_compat(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ plotting.plot_params["x_compat"] = False
+
+ ax = df.plot()
+ lines = ax.get_lines()
+ assert not isinstance(lines[0].get_xdata(), PeriodIndex)
+ msg = r"PeriodDtype\[B\] is deprecated"
+ with tm.assert_produces_warning(FutureWarning, match=msg):
+ assert isinstance(PeriodIndex(lines[0].get_xdata()), PeriodIndex)
+
+ def test_xcompat_plot_params_context_manager(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ # useful if you're plotting a bunch together
+ with plotting.plot_params.use("x_compat", True):
+ ax = df.plot()
+ lines = ax.get_lines()
+ assert not isinstance(lines[0].get_xdata(), PeriodIndex)
+ _check_ticks_props(ax, xrot=30)
+
+ def test_xcompat_plot_period(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ ax = df.plot()
+ lines = ax.get_lines()
+ assert not isinstance(lines[0].get_xdata(), PeriodIndex)
+ msg = r"PeriodDtype\[B\] is deprecated "
+ with tm.assert_produces_warning(FutureWarning, match=msg):
+ assert isinstance(PeriodIndex(lines[0].get_xdata()), PeriodIndex)
+ _check_ticks_props(ax, xrot=0)
+
+ def test_period_compat(self):
+ # GH 9012
+ # period-array conversions
+ df = DataFrame(
+ np.random.default_rng(2).random((21, 2)),
+ index=bdate_range(datetime(2000, 1, 1), datetime(2000, 1, 31)),
+ columns=["a", "b"],
+ )
+
+ df.plot()
+ mpl.pyplot.axhline(y=0)
+
+ @pytest.mark.parametrize("index_dtype", [np.int64, np.float64])
+ def test_unsorted_index(self, index_dtype):
+ df = DataFrame(
+ {"y": np.arange(100)},
+ index=Index(np.arange(99, -1, -1), dtype=index_dtype),
+ dtype=np.int64,
+ )
+ ax = df.plot()
+ lines = ax.get_lines()[0]
+ rs = lines.get_xydata()
+ rs = Series(rs[:, 1], rs[:, 0], dtype=np.int64, name="y")
+ tm.assert_series_equal(rs, df.y, check_index_type=False)
+
+ @pytest.mark.parametrize(
+ "df",
+ [
+ DataFrame({"y": [0.0, 1.0, 2.0, 3.0]}, index=[1.0, 0.0, 3.0, 2.0]),
+ DataFrame(
+ {"y": [0.0, 1.0, np.nan, 3.0, 4.0, 5.0, 6.0]},
+ index=[1.0, 0.0, 3.0, 2.0, np.nan, 3.0, 2.0],
+ ),
+ ],
+ )
+ def test_unsorted_index_lims(self, df):
+ ax = df.plot()
+ xmin, xmax = ax.get_xlim()
+ lines = ax.get_lines()
+ assert xmin <= np.nanmin(lines[0].get_data()[0])
+ assert xmax >= np.nanmax(lines[0].get_data()[0])
+
+ def test_unsorted_index_lims_x_y(self):
+ df = DataFrame({"y": [0.0, 1.0, 2.0, 3.0], "z": [91.0, 90.0, 93.0, 92.0]})
+ ax = df.plot(x="z", y="y")
+ xmin, xmax = ax.get_xlim()
+ lines = ax.get_lines()
+ assert xmin <= np.nanmin(lines[0].get_data()[0])
+ assert xmax >= np.nanmax(lines[0].get_data()[0])
+
+ def test_negative_log(self):
+ df = -DataFrame(
+ np.random.default_rng(2).random((6, 4)),
+ index=list(string.ascii_letters[:6]),
+ columns=["x", "y", "z", "four"],
+ )
+ msg = "Log-y scales are not supported in area plot"
+ with pytest.raises(ValueError, match=msg):
+ df.plot.area(logy=True)
+ with pytest.raises(ValueError, match=msg):
+ df.plot.area(loglog=True)
+
+ def _compare_stacked_y_cood(self, normal_lines, stacked_lines):
+ base = np.zeros(len(normal_lines[0].get_data()[1]))
+ for nl, sl in zip(normal_lines, stacked_lines, strict=True):
+ base += nl.get_data()[1] # get y coordinates
+ sy = sl.get_data()[1]
+ tm.assert_numpy_array_equal(base, sy)
+
+ @pytest.mark.parametrize("kind", ["line", "area"])
+ @pytest.mark.parametrize("mult", [1, -1])
+ def test_line_area_stacked(self, kind, mult):
+ df = mult * DataFrame(
+ np.random.default_rng(2).random((6, 4)), columns=["w", "x", "y", "z"]
+ )
+
+ ax1 = _check_plot_works(df.plot, kind=kind, stacked=False)
+ ax2 = _check_plot_works(df.plot, kind=kind, stacked=True)
+ self._compare_stacked_y_cood(ax1.lines, ax2.lines)
+
+ @pytest.mark.parametrize("kind", ["line", "area"])
+ def test_line_area_stacked_sep_df(self, kind):
+ # each column has either positive or negative value
+ sep_df = DataFrame(
+ {
+ "w": np.random.default_rng(2).random(6),
+ "x": np.random.default_rng(2).random(6),
+ "y": -np.random.default_rng(2).random(6),
+ "z": -np.random.default_rng(2).random(6),
+ }
+ )
+ ax1 = _check_plot_works(sep_df.plot, kind=kind, stacked=False)
+ ax2 = _check_plot_works(sep_df.plot, kind=kind, stacked=True)
+ self._compare_stacked_y_cood(ax1.lines[:2], ax2.lines[:2])
+ self._compare_stacked_y_cood(ax1.lines[2:], ax2.lines[2:])
+
+ def test_line_area_stacked_mixed(self):
+ mixed_df = DataFrame(
+ np.random.default_rng(2).standard_normal((6, 4)),
+ index=list(string.ascii_letters[:6]),
+ columns=["w", "x", "y", "z"],
+ )
+ _check_plot_works(mixed_df.plot, stacked=False)
+
+ msg = (
+ "When stacked is True, each column must be either all positive or "
+ "all negative. Column 'w' contains both positive and negative "
+ "values"
+ )
+ with pytest.raises(ValueError, match=msg):
+ mixed_df.plot(stacked=True)
+
+ @pytest.mark.parametrize("kind", ["line", "area"])
+ def test_line_area_stacked_positive_idx(self, kind):
+ df = DataFrame(
+ np.random.default_rng(2).random((6, 4)), columns=["w", "x", "y", "z"]
+ )
+ # Use an index with strictly positive values, preventing
+ # matplotlib from warning about ignoring xlim
+ df2 = df.set_index(df.index + 1)
+ _check_plot_works(df2.plot, kind=kind, logx=True, stacked=True)
+
+ @pytest.mark.parametrize(
+ "idx", [range(4), date_range("2023-01-1", freq="D", periods=4)]
+ )
+ def test_line_area_nan_df(self, idx):
+ values1 = [1, 2, np.nan, 3]
+ values2 = [3, np.nan, 2, 1]
+ df = DataFrame({"a": values1, "b": values2}, index=idx)
+
+ ax = _check_plot_works(df.plot)
+ masked1 = ax.lines[0].get_ydata()
+ masked2 = ax.lines[1].get_ydata()
+ # remove nan for comparison purpose
+
+ exp = np.array([1, 2, 3], dtype=np.float64)
+ tm.assert_numpy_array_equal(np.delete(masked1.data, 2), exp)
+
+ exp = np.array([3, 2, 1], dtype=np.float64)
+ tm.assert_numpy_array_equal(np.delete(masked2.data, 1), exp)
+ tm.assert_numpy_array_equal(masked1.mask, np.array([False, False, True, False]))
+ tm.assert_numpy_array_equal(masked2.mask, np.array([False, True, False, False]))
+
+ @pytest.mark.parametrize(
+ "idx", [range(4), date_range("2023-01-1", freq="D", periods=4)]
+ )
+ def test_line_area_nan_df_stacked(self, idx):
+ values1 = [1, 2, np.nan, 3]
+ values2 = [3, np.nan, 2, 1]
+ df = DataFrame({"a": values1, "b": values2}, index=idx)
+
+ expected1 = np.array([1, 2, 0, 3], dtype=np.float64)
+ expected2 = np.array([3, 0, 2, 1], dtype=np.float64)
+
+ ax = _check_plot_works(df.plot, stacked=True)
+ tm.assert_numpy_array_equal(ax.lines[0].get_ydata(), expected1)
+ tm.assert_numpy_array_equal(ax.lines[1].get_ydata(), expected1 + expected2)
+
+ @pytest.mark.parametrize(
+ "idx", [range(4), date_range("2023-01-1", freq="D", periods=4)]
+ )
+ @pytest.mark.parametrize("kwargs", [{}, {"stacked": False}])
+ def test_line_area_nan_df_stacked_area(self, idx, kwargs):
+ values1 = [1, 2, np.nan, 3]
+ values2 = [3, np.nan, 2, 1]
+ df = DataFrame({"a": values1, "b": values2}, index=idx)
+
+ expected1 = np.array([1, 2, 0, 3], dtype=np.float64)
+ expected2 = np.array([3, 0, 2, 1], dtype=np.float64)
+
+ ax = _check_plot_works(df.plot.area, **kwargs)
+ tm.assert_numpy_array_equal(ax.lines[0].get_ydata(), expected1)
+ if kwargs:
+ tm.assert_numpy_array_equal(ax.lines[1].get_ydata(), expected2)
+ else:
+ tm.assert_numpy_array_equal(ax.lines[1].get_ydata(), expected1 + expected2)
+
+ ax = _check_plot_works(df.plot.area, stacked=False)
+ tm.assert_numpy_array_equal(ax.lines[0].get_ydata(), expected1)
+ tm.assert_numpy_array_equal(ax.lines[1].get_ydata(), expected2)
+
+ @pytest.mark.parametrize("kwargs", [{}, {"secondary_y": True}])
+ def test_line_lim(self, kwargs):
+ df = DataFrame(np.random.default_rng(2).random((6, 3)), columns=["x", "y", "z"])
+ ax = df.plot(**kwargs)
+ xmin, xmax = ax.get_xlim()
+ lines = ax.get_lines()
+ assert xmin <= lines[0].get_data()[0][0]
+ assert xmax >= lines[0].get_data()[0][-1]
+
+ def test_line_lim_subplots(self):
+ df = DataFrame(np.random.default_rng(2).random((6, 3)), columns=["x", "y", "z"])
+ axes = df.plot(secondary_y=True, subplots=True)
+ _check_axes_shape(axes, axes_num=3, layout=(3, 1))
+ for ax in axes:
+ assert hasattr(ax, "left_ax")
+ assert not hasattr(ax, "right_ax")
+ xmin, xmax = ax.get_xlim()
+ lines = ax.get_lines()
+ assert xmin <= lines[0].get_data()[0][0]
+ assert xmax >= lines[0].get_data()[0][-1]
+
+ @pytest.mark.xfail(
+ strict=False,
+ reason="2020-12-01 this has been failing periodically on the "
+ "ymin==0 assertion for a week or so.",
+ )
+ @pytest.mark.parametrize("stacked", [True, False])
+ def test_area_lim(self, stacked):
+ df = DataFrame(
+ np.random.default_rng(2).random((6, 4)), columns=["x", "y", "z", "four"]
+ )
+
+ neg_df = -df
+
+ ax = _check_plot_works(df.plot.area, stacked=stacked)
+ xmin, xmax = ax.get_xlim()
+ ymin, ymax = ax.get_ylim()
+ lines = ax.get_lines()
+ assert xmin <= lines[0].get_data()[0][0]
+ assert xmax >= lines[0].get_data()[0][-1]
+ assert ymin == 0
+
+ ax = _check_plot_works(neg_df.plot.area, stacked=stacked)
+ ymin, ymax = ax.get_ylim()
+ assert ymax == 0
+
+ def test_area_sharey_dont_overwrite(self):
+ # GH37942
+ df = DataFrame(np.random.default_rng(2).random((4, 2)), columns=["x", "y"])
+ fig, (ax1, ax2) = mpl.pyplot.subplots(1, 2, sharey=True)
+
+ df.plot(ax=ax1, kind="area")
+ df.plot(ax=ax2, kind="area")
+
+ assert get_y_axis(ax1).joined(ax1, ax2)
+ assert get_y_axis(ax2).joined(ax1, ax2)
+
+ @pytest.mark.parametrize("stacked", [True, False])
+ def test_bar_linewidth(self, stacked):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+
+ ax = df.plot.bar(stacked=stacked, linewidth=2)
+ for r in ax.patches:
+ assert r.get_linewidth() == 2
+
+ def test_bar_linewidth_subplots(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ # subplots
+ axes = df.plot.bar(linewidth=2, subplots=True)
+ _check_axes_shape(axes, axes_num=5, layout=(5, 1))
+ for ax in axes:
+ for r in ax.patches:
+ assert r.get_linewidth() == 2
+
+ @pytest.mark.parametrize(
+ "meth, dim", [("bar", "get_width"), ("barh", "get_height")]
+ )
+ @pytest.mark.parametrize("stacked", [True, False])
+ def test_bar_barwidth(self, meth, dim, stacked):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+
+ width = 0.9
+
+ ax = getattr(df.plot, meth)(stacked=stacked, width=width)
+ for r in ax.patches:
+ if not stacked:
+ assert getattr(r, dim)() == width / len(df.columns)
+ else:
+ assert getattr(r, dim)() == width
+
+ @pytest.mark.parametrize(
+ "meth, dim", [("bar", "get_width"), ("barh", "get_height")]
+ )
+ def test_barh_barwidth_subplots(self, meth, dim):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+
+ width = 0.9
+
+ axes = getattr(df.plot, meth)(width=width, subplots=True)
+ for ax in axes:
+ for r in ax.patches:
+ assert getattr(r, dim)() == width
+
+ def test_bar_bottom_left_bottom(self):
+ df = DataFrame(np.random.default_rng(2).random((5, 5)))
+ ax = df.plot.bar(stacked=False, bottom=1)
+ result = [p.get_y() for p in ax.patches]
+ assert result == [1] * 25
+
+ ax = df.plot.bar(stacked=True, bottom=[-1, -2, -3, -4, -5])
+ result = [p.get_y() for p in ax.patches[:5]]
+ assert result == [-1, -2, -3, -4, -5]
+
+ def test_bar_bottom_left_left(self):
+ df = DataFrame(np.random.default_rng(2).random((5, 5)))
+ ax = df.plot.barh(stacked=False, left=np.array([1, 1, 1, 1, 1]))
+ result = [p.get_x() for p in ax.patches]
+ assert result == [1] * 25
+
+ ax = df.plot.barh(stacked=True, left=[1, 2, 3, 4, 5])
+ result = [p.get_x() for p in ax.patches[:5]]
+ assert result == [1, 2, 3, 4, 5]
+
+ def test_bar_bottom_left_subplots(self):
+ df = DataFrame(np.random.default_rng(2).random((5, 5)))
+ axes = df.plot.bar(subplots=True, bottom=-1)
+ for ax in axes:
+ result = [p.get_y() for p in ax.patches]
+ assert result == [-1] * 5
+
+ axes = df.plot.barh(subplots=True, left=np.array([1, 1, 1, 1, 1]))
+ for ax in axes:
+ result = [p.get_x() for p in ax.patches]
+ assert result == [1] * 5
+
+ def test_bar_nan(self):
+ df = DataFrame({"A": [10, np.nan, 20], "B": [5, 10, 20], "C": [1, 2, 3]})
+ ax = df.plot.bar()
+ expected = [10, 0, 20, 5, 10, 20, 1, 2, 3]
+ result = [p.get_height() for p in ax.patches]
+ assert result == expected
+
+ def test_bar_nan_stacked(self):
+ df = DataFrame({"A": [10, np.nan, 20], "B": [5, 10, 20], "C": [1, 2, 3]})
+ ax = df.plot.bar(stacked=True)
+ expected = [10, 0, 20, 5, 10, 20, 1, 2, 3]
+ result = [p.get_height() for p in ax.patches]
+ assert result == expected
+
+ result = [p.get_y() for p in ax.patches]
+ expected = [0.0, 0.0, 0.0, 10.0, 0.0, 20.0, 15.0, 10.0, 40.0]
+ assert result == expected
+
+ def test_bar_stacked_label_position_with_zero_height(self):
+ # GH 59429
+ df = DataFrame({"A": [3, 0, 1], "B": [0, 2, 4], "C": [5, 0, 2]})
+ ax = df.plot.bar(stacked=True)
+ ax.bar_label(ax.containers[-1])
+ expected = [8.0, 2.0, 7.0]
+ result = [text.xy[1] for text in ax.texts]
+ tm.assert_almost_equal(result, expected)
+ plt.close("all")
+
+ @pytest.mark.parametrize("idx", [Index, pd.CategoricalIndex])
+ def test_bar_categorical(self, idx):
+ # GH 13019
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((6, 5)),
+ index=idx(list("ABCDEF")),
+ columns=idx(list("abcde")),
+ )
+
+ ax = df.plot.bar()
+ ticks = ax.xaxis.get_ticklocs()
+ tm.assert_numpy_array_equal(ticks, np.array([0, 1, 2, 3, 4, 5]))
+ assert ax.get_xlim() == (-0.5, 5.5)
+ # check left-edge of bars
+ assert ax.patches[0].get_x() == -0.25
+ assert ax.patches[-1].get_x() == 5.15
+
+ ax = df.plot.bar(stacked=True)
+ tm.assert_numpy_array_equal(ticks, np.array([0, 1, 2, 3, 4, 5]))
+ assert ax.get_xlim() == (-0.5, 5.5)
+ assert ax.patches[0].get_x() == -0.25
+ assert ax.patches[-1].get_x() == 4.75
+
+ @pytest.mark.parametrize("x, y", [("x", "y"), (1, 2)])
+ def test_plot_scatter(self, x, y):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((6, 4)),
+ index=list(string.ascii_letters[:6]),
+ columns=["x", "y", "z", "four"],
+ )
+
+ _check_plot_works(df.plot.scatter, x=x, y=y)
+
+ def test_plot_scatter_error(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((6, 4)),
+ index=list(string.ascii_letters[:6]),
+ columns=["x", "y", "z", "four"],
+ )
+ msg = re.escape("scatter() missing 1 required positional argument: 'y'")
+ with pytest.raises(TypeError, match=msg):
+ df.plot.scatter(x="x")
+ msg = re.escape("scatter() missing 1 required positional argument: 'x'")
+ with pytest.raises(TypeError, match=msg):
+ df.plot.scatter(y="y")
+
+ def test_plot_scatter_shape(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((6, 4)),
+ index=list(string.ascii_letters[:6]),
+ columns=["x", "y", "z", "four"],
+ )
+ # GH 6951
+ axes = df.plot(x="x", y="y", kind="scatter", subplots=True)
+ _check_axes_shape(axes, axes_num=1, layout=(1, 1))
+
+ def test_scatter_on_datetime_time_data(self):
+ # datetime.time type is now supported in scatter, since a converter
+ # is implemented in ScatterPlot
+ df = DataFrame(np.random.default_rng(2).standard_normal(10), columns=["a"])
+ df["dtime"] = date_range(start="2014-01-01", freq="h", periods=10).time
+ df.plot(kind="scatter", x="dtime", y="a")
+
+ def test_scatter_line_xticks(self):
+ # GH#61005
+ df = DataFrame(
+ [(datetime(year=2025, month=1, day=1, hour=n), n) for n in range(3)],
+ columns=["datetime", "y"],
+ )
+ fig, ax = plt.subplots(2, sharex=True)
+ df.plot.scatter(x="datetime", y="y", ax=ax[0])
+ scatter_xticks = ax[0].get_xticks()
+ df.plot(x="datetime", y="y", ax=ax[1])
+ line_xticks = ax[1].get_xticks()
+ assert scatter_xticks[0] == line_xticks[0]
+ assert scatter_xticks[-1] == line_xticks[-1]
+
+ @pytest.mark.parametrize("x, y", [("dates", "vals"), (0, 1)])
+ def test_scatterplot_datetime_data(self, x, y):
+ # GH 30391
+ dates = date_range(start=date(2019, 1, 1), periods=12, freq="W")
+ vals = np.random.default_rng(2).normal(0, 1, len(dates))
+ df = DataFrame({"dates": dates, "vals": vals})
+
+ _check_plot_works(df.plot.scatter, x=x, y=y)
+
+ @pytest.mark.parametrize(
+ "infer_string", [False, pytest.param(True, marks=td.skip_if_no("pyarrow"))]
+ )
+ @pytest.mark.parametrize("x, y", [("a", "b"), (0, 1)])
+ @pytest.mark.parametrize("b_col", [[2, 3, 4], ["a", "b", "c"]])
+ def test_scatterplot_object_data(self, b_col, x, y, infer_string):
+ # GH 18755
+ with option_context("future.infer_string", infer_string):
+ df = DataFrame({"a": ["A", "B", "C"], "b": b_col})
+
+ _check_plot_works(df.plot.scatter, x=x, y=y)
+
+ @pytest.mark.parametrize("ordered", [True, False])
+ @pytest.mark.parametrize(
+ "categories",
+ (["setosa", "versicolor", "virginica"], ["versicolor", "virginica", "setosa"]),
+ )
+ def test_scatterplot_color_by_categorical(self, ordered, categories):
+ df = DataFrame(
+ [[5.1, 3.5], [4.9, 3.0], [7.0, 3.2], [6.4, 3.2], [5.9, 3.0]],
+ columns=["length", "width"],
+ )
+ df["species"] = pd.Categorical(
+ ["setosa", "setosa", "virginica", "virginica", "versicolor"],
+ ordered=ordered,
+ categories=categories,
+ )
+ ax = df.plot.scatter(x=0, y=1, c="species")
+ (colorbar_collection,) = ax.collections
+ colorbar = colorbar_collection.colorbar
+
+ expected_ticks = np.array([0.5, 1.5, 2.5])
+ result_ticks = colorbar.get_ticks()
+ tm.assert_numpy_array_equal(result_ticks, expected_ticks)
+
+ expected_boundaries = np.array([0.0, 1.0, 2.0, 3.0])
+ result_boundaries = colorbar._boundaries
+ tm.assert_numpy_array_equal(result_boundaries, expected_boundaries)
+
+ expected_yticklabels = categories
+ result_yticklabels = [i.get_text() for i in colorbar.ax.get_ymajorticklabels()]
+ assert all(
+ i == j
+ for i, j in zip(result_yticklabels, expected_yticklabels, strict=True)
+ )
+
+ @pytest.mark.parametrize("x, y", [("x", "y"), ("y", "x"), ("y", "y")])
+ def test_plot_scatter_with_categorical_data(self, x, y):
+ # after fixing GH 18755, should be able to plot categorical data
+ df = DataFrame({"x": [1, 2, 3, 4], "y": pd.Categorical(["a", "b", "a", "c"])})
+
+ _check_plot_works(df.plot.scatter, x=x, y=y)
+
+ @pytest.mark.parametrize("x, y, c", [("x", "y", "z"), (0, 1, 2)])
+ def test_plot_scatter_with_c(self, x, y, c):
+ df = DataFrame(
+ np.random.default_rng(2).integers(low=0, high=100, size=(6, 4)),
+ index=list(string.ascii_letters[:6]),
+ columns=["x", "y", "z", "four"],
+ )
+
+ ax = df.plot.scatter(x=x, y=y, c=c)
+ # default to Greys
+ assert ax.collections[0].cmap.name == "Greys"
+
+ assert ax.collections[0].colorbar.ax.get_ylabel() == "z"
+
+ def test_plot_scatter_with_c_props(self):
+ df = DataFrame(
+ np.random.default_rng(2).integers(low=0, high=100, size=(6, 4)),
+ index=list(string.ascii_letters[:6]),
+ columns=["x", "y", "z", "four"],
+ )
+ cm = "cubehelix"
+ ax = df.plot.scatter(x="x", y="y", c="z", colormap=cm)
+ assert ax.collections[0].cmap.name == cm
+
+ # verify turning off colorbar works
+ ax = df.plot.scatter(x="x", y="y", c="z", colorbar=False)
+ assert ax.collections[0].colorbar is None
+
+ # verify that we can still plot a solid color
+ ax = df.plot.scatter(x=0, y=1, c="red")
+ assert ax.collections[0].colorbar is None
+ _check_colors(ax.collections, facecolors=["r"])
+
+ def test_plot_scatter_with_c_array(self):
+ # Ensure that we can pass an np.array straight through to matplotlib,
+ # this functionality was accidentally removed previously.
+ # See https://github.com/pandas-dev/pandas/issues/8852 for bug report
+ #
+ # Exercise colormap path and non-colormap path as they are independent
+ #
+ df = DataFrame({"A": [1, 2], "B": [3, 4]})
+ red_rgba = [1.0, 0.0, 0.0, 1.0]
+ green_rgba = [0.0, 1.0, 0.0, 1.0]
+ rgba_array = np.array([red_rgba, green_rgba])
+ ax = df.plot.scatter(x="A", y="B", c=rgba_array)
+ # expect the face colors of the points in the non-colormap path to be
+ # identical to the values we supplied, normally we'd be on shaky ground
+ # comparing floats for equality but here we expect them to be
+ # identical.
+ tm.assert_numpy_array_equal(ax.collections[0].get_facecolor(), rgba_array)
+ # we don't test the colors of the faces in this next plot because they
+ # are dependent on the spring colormap, which may change its colors
+ # later.
+ float_array = np.array([0.0, 1.0])
+ df.plot.scatter(x="A", y="B", c=float_array, cmap="spring")
+
+ def test_plot_scatter_with_s(self):
+ # this refers to GH 32904
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)) * 100, columns=["a", "b", "c"]
+ )
+
+ ax = df.plot.scatter(x="a", y="b", s="c")
+ tm.assert_numpy_array_equal(df["c"].values, right=ax.collections[0].get_sizes())
+
+ def test_plot_scatter_with_norm(self):
+ # added while fixing GH 45809
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)) * 100, columns=["a", "b", "c"]
+ )
+ norm = mpl.colors.LogNorm()
+ ax = df.plot.scatter(x="a", y="b", c="c", norm=norm)
+ assert ax.collections[0].norm is norm
+
+ def test_plot_scatter_without_norm(self):
+ # added while fixing GH 45809
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)) * 100, columns=["a", "b", "c"]
+ )
+ ax = df.plot.scatter(x="a", y="b", c="c")
+ plot_norm = ax.collections[0].norm
+ color_min_max = (df.c.min(), df.c.max())
+ default_norm = mpl.colors.Normalize(*color_min_max)
+ for value in df.c:
+ assert plot_norm(value) == default_norm(value)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "kwargs",
+ [
+ {},
+ {"legend": False},
+ {"default_axes": True, "subplots": True},
+ {"stacked": True},
+ ],
+ )
+ def test_plot_bar(self, kwargs):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((6, 4)),
+ index=list(string.ascii_letters[:6]),
+ columns=["one", "two", "three", "four"],
+ )
+
+ _check_plot_works(df.plot.bar, **kwargs)
+
+ @pytest.mark.slow
+ def test_plot_bar_int_col(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 15)),
+ index=list(string.ascii_letters[:10]),
+ columns=range(15),
+ )
+ _check_plot_works(df.plot.bar)
+
+ @pytest.mark.slow
+ def test_plot_bar_ticks(self):
+ df = DataFrame({"a": [0, 1], "b": [1, 0]})
+ ax = _check_plot_works(df.plot.bar)
+ _check_ticks_props(ax, xrot=90)
+
+ ax = df.plot.bar(rot=35, fontsize=10)
+ _check_ticks_props(ax, xrot=35, xlabelsize=10, ylabelsize=10)
+
+ @pytest.mark.slow
+ def test_plot_barh_ticks(self):
+ df = DataFrame({"a": [0, 1], "b": [1, 0]})
+ ax = _check_plot_works(df.plot.barh)
+ _check_ticks_props(ax, yrot=0)
+
+ ax = df.plot.barh(rot=55, fontsize=11)
+ _check_ticks_props(ax, yrot=55, ylabelsize=11, xlabelsize=11)
+
+ def test_boxplot(self, hist_df):
+ df = hist_df
+ numeric_cols = df._get_numeric_data().columns
+ labels = [pprint_thing(c) for c in numeric_cols]
+
+ ax = _check_plot_works(df.plot.box)
+ _check_text_labels(ax.get_xticklabels(), labels)
+ tm.assert_numpy_array_equal(
+ ax.xaxis.get_ticklocs(), np.arange(1, len(numeric_cols) + 1)
+ )
+ assert len(ax.lines) == 7 * len(numeric_cols)
+
+ def test_boxplot_series(self, hist_df):
+ df = hist_df
+ series = df["height"]
+ axes = series.plot.box(rot=40)
+ _check_ticks_props(axes, xrot=40, yrot=0)
+
+ _check_plot_works(series.plot.box)
+
+ def test_boxplot_series_positions(self, hist_df):
+ df = hist_df
+ positions = np.array([1, 6, 7])
+ ax = df.plot.box(positions=positions)
+ numeric_cols = df._get_numeric_data().columns
+ labels = [pprint_thing(c) for c in numeric_cols]
+ _check_text_labels(ax.get_xticklabels(), labels)
+ tm.assert_numpy_array_equal(ax.xaxis.get_ticklocs(), positions)
+ assert len(ax.lines) == 7 * len(numeric_cols)
+
+ @pytest.mark.filterwarnings("ignore:set_ticklabels:UserWarning")
+ @pytest.mark.xfail(
+ Version(mpl.__version__) >= Version("3.10"),
+ reason="Fails starting with matplotlib 3.10",
+ )
+ def test_boxplot_vertical(self, hist_df):
+ df = hist_df
+ numeric_cols = df._get_numeric_data().columns
+ labels = [pprint_thing(c) for c in numeric_cols]
+
+ # if horizontal, yticklabels are rotated
+ kwargs = (
+ {"vert": False}
+ if Version(mpl.__version__) < Version("3.10")
+ else {"orientation": "horizontal"}
+ )
+ ax = df.plot.box(rot=50, fontsize=8, **kwargs)
+ _check_ticks_props(ax, xrot=0, yrot=50, ylabelsize=8)
+ _check_text_labels(ax.get_yticklabels(), labels)
+ assert len(ax.lines) == 7 * len(numeric_cols)
+
+ @pytest.mark.filterwarnings("ignore::UserWarning")
+ @pytest.mark.xfail(
+ Version(mpl.__version__) >= Version("3.10"),
+ reason="Fails starting with matplotlib version 3.10",
+ )
+ def test_boxplot_vertical_subplots(self, hist_df):
+ df = hist_df
+ numeric_cols = df._get_numeric_data().columns
+ labels = [pprint_thing(c) for c in numeric_cols]
+ kwargs = (
+ {"vert": False}
+ if Version(mpl.__version__) < Version("3.10")
+ else {"orientation": "horizontal"}
+ )
+ axes = _check_plot_works(
+ df.plot.box, default_axes=True, subplots=True, logx=True, **kwargs
+ )
+ _check_axes_shape(axes, axes_num=3, layout=(1, 3))
+ _check_ax_scales(axes, xaxis="log")
+ for ax, label in zip(axes, labels, strict=True):
+ _check_text_labels(ax.get_yticklabels(), [label])
+ assert len(ax.lines) == 7
+
+ @pytest.mark.filterwarnings("ignore:set_ticklabels:UserWarning")
+ @pytest.mark.xfail(
+ Version(mpl.__version__) >= Version("3.10"),
+ reason="Fails starting with matplotlib 3.10",
+ )
+ def test_boxplot_vertical_positions(self, hist_df):
+ df = hist_df
+ numeric_cols = df._get_numeric_data().columns
+ labels = [pprint_thing(c) for c in numeric_cols]
+ positions = np.array([3, 2, 8])
+ kwargs = (
+ {"vert": False}
+ if Version(mpl.__version__) < Version("3.10")
+ else {"orientation": "horizontal"}
+ )
+ ax = df.plot.box(positions=positions, **kwargs)
+ _check_text_labels(ax.get_yticklabels(), labels)
+ tm.assert_numpy_array_equal(ax.yaxis.get_ticklocs(), positions)
+ assert len(ax.lines) == 7 * len(numeric_cols)
+
+ def test_boxplot_return_type_invalid(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((6, 4)),
+ index=list(string.ascii_letters[:6]),
+ columns=["one", "two", "three", "four"],
+ )
+ msg = "return_type must be {None, 'axes', 'dict', 'both'}"
+ with pytest.raises(ValueError, match=msg):
+ df.plot.box(return_type="not_a_type")
+
+ @pytest.mark.parametrize("return_type", ["dict", "axes", "both"])
+ def test_boxplot_return_type_invalid_type(self, return_type):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((6, 4)),
+ index=list(string.ascii_letters[:6]),
+ columns=["one", "two", "three", "four"],
+ )
+ result = df.plot.box(return_type=return_type)
+ _check_box_return_type(result, return_type)
+
+ def test_kde_df(self):
+ pytest.importorskip("scipy")
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 4)))
+ ax = _check_plot_works(df.plot, kind="kde")
+ expected = [pprint_thing(c) for c in df.columns]
+ _check_legend_labels(ax, labels=expected)
+ _check_ticks_props(ax, xrot=0)
+
+ def test_kde_df_rot(self):
+ pytest.importorskip("scipy")
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 4)))
+ ax = df.plot(kind="kde", rot=20, fontsize=5)
+ _check_ticks_props(ax, xrot=20, xlabelsize=5, ylabelsize=5)
+
+ def test_kde_df_subplots(self):
+ pytest.importorskip("scipy")
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 4)))
+ axes = _check_plot_works(
+ df.plot,
+ default_axes=True,
+ kind="kde",
+ subplots=True,
+ )
+ _check_axes_shape(axes, axes_num=4, layout=(4, 1))
+
+ def test_kde_df_logy(self):
+ pytest.importorskip("scipy")
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 4)))
+ axes = df.plot(kind="kde", logy=True, subplots=True)
+ _check_ax_scales(axes, yaxis="log")
+
+ def test_kde_missing_vals(self):
+ pytest.importorskip("scipy")
+ df = DataFrame(np.random.default_rng(2).uniform(size=(100, 4)))
+ df.loc[0, 0] = np.nan
+ _check_plot_works(df.plot, kind="kde")
+
+ def test_hist_df(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((100, 4)))
+
+ ax = _check_plot_works(df.plot.hist)
+ expected = [pprint_thing(c) for c in df.columns]
+ _check_legend_labels(ax, labels=expected)
+
+ axes = _check_plot_works(
+ df.plot.hist,
+ default_axes=True,
+ subplots=True,
+ logy=True,
+ )
+ _check_axes_shape(axes, axes_num=4, layout=(4, 1))
+ _check_ax_scales(axes, yaxis="log")
+
+ def test_hist_df_series(self):
+ series = Series(np.random.default_rng(2).random(10))
+ axes = series.plot.hist(rot=40)
+ _check_ticks_props(axes, xrot=40, yrot=0)
+
+ def test_hist_df_series_cumulative_density(self):
+ series = Series(np.random.default_rng(2).random(10))
+ ax = series.plot.hist(cumulative=True, bins=4, density=True)
+ # height of last bin (index 5) must be 1.0
+ rects = [x for x in ax.get_children() if isinstance(x, mpl.patches.Rectangle)]
+ tm.assert_almost_equal(rects[-1].get_height(), 1.0)
+
+ def test_hist_df_series_cumulative(self):
+ series = Series(np.random.default_rng(2).random(10))
+ ax = series.plot.hist(cumulative=True, bins=4)
+ rects = [x for x in ax.get_children() if isinstance(x, mpl.patches.Rectangle)]
+
+ tm.assert_almost_equal(rects[-2].get_height(), 10.0)
+
+ def test_hist_df_orientation(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 4)))
+ # if horizontal, yticklabels are rotated
+ axes = df.plot.hist(rot=50, fontsize=8, orientation="horizontal")
+ _check_ticks_props(axes, xrot=0, yrot=50, ylabelsize=8)
+
+ @pytest.mark.parametrize("weight_shape", [(100,), (100, 2)])
+ def test_hist_weights(self, weight_shape):
+ # GH 33173
+ weights = 0.1 * np.ones(shape=weight_shape)
+ df = DataFrame(
+ dict(
+ zip(
+ ["A", "B"],
+ np.random.default_rng(2).standard_normal((2, 100)),
+ strict=True,
+ )
+ )
+ )
+
+ ax1 = _check_plot_works(df.plot, kind="hist", weights=weights)
+ ax2 = _check_plot_works(df.plot, kind="hist")
+
+ patch_height_with_weights = [patch.get_height() for patch in ax1.patches]
+
+ # original heights with no weights, and we manually multiply with example
+ # weights, so after multiplication, they should be almost same
+ expected_patch_height = [0.1 * patch.get_height() for patch in ax2.patches]
+
+ tm.assert_almost_equal(patch_height_with_weights, expected_patch_height)
+
+ def _check_box_coord(
+ self,
+ patches,
+ expected_y=None,
+ expected_h=None,
+ expected_x=None,
+ expected_w=None,
+ ):
+ result_y = np.array([p.get_y() for p in patches])
+ result_height = np.array([p.get_height() for p in patches])
+ result_x = np.array([p.get_x() for p in patches])
+ result_width = np.array([p.get_width() for p in patches])
+ # dtype is depending on above values, no need to check
+
+ if expected_y is not None:
+ tm.assert_numpy_array_equal(result_y, expected_y, check_dtype=False)
+ if expected_h is not None:
+ tm.assert_numpy_array_equal(result_height, expected_h, check_dtype=False)
+ if expected_x is not None:
+ tm.assert_numpy_array_equal(result_x, expected_x, check_dtype=False)
+ if expected_w is not None:
+ tm.assert_numpy_array_equal(result_width, expected_w, check_dtype=False)
+
+ @pytest.mark.parametrize(
+ "data",
+ [
+ {
+ "A": np.repeat(np.array([1, 2, 3, 4, 5]), np.array([10, 9, 8, 7, 6])),
+ "B": np.repeat(np.array([1, 2, 3, 4, 5]), np.array([8, 8, 8, 8, 8])),
+ "C": np.repeat(np.array([1, 2, 3, 4, 5]), np.array([6, 7, 8, 9, 10])),
+ },
+ {
+ "A": np.repeat(
+ np.array([np.nan, 1, 2, 3, 4, 5]), np.array([3, 10, 9, 8, 7, 6])
+ ),
+ "B": np.repeat(
+ np.array([1, np.nan, 2, 3, 4, 5]), np.array([8, 3, 8, 8, 8, 8])
+ ),
+ "C": np.repeat(
+ np.array([1, 2, 3, np.nan, 4, 5]), np.array([6, 7, 8, 3, 9, 10])
+ ),
+ },
+ ],
+ )
+ def test_hist_df_coord(self, data):
+ df = DataFrame(data)
+
+ ax = df.plot.hist(bins=5)
+ self._check_box_coord(
+ ax.patches[:5],
+ expected_y=np.array([0, 0, 0, 0, 0]),
+ expected_h=np.array([10, 9, 8, 7, 6]),
+ )
+ self._check_box_coord(
+ ax.patches[5:10],
+ expected_y=np.array([0, 0, 0, 0, 0]),
+ expected_h=np.array([8, 8, 8, 8, 8]),
+ )
+ self._check_box_coord(
+ ax.patches[10:],
+ expected_y=np.array([0, 0, 0, 0, 0]),
+ expected_h=np.array([6, 7, 8, 9, 10]),
+ )
+
+ ax = df.plot.hist(bins=5, stacked=True)
+ self._check_box_coord(
+ ax.patches[:5],
+ expected_y=np.array([0, 0, 0, 0, 0]),
+ expected_h=np.array([10, 9, 8, 7, 6]),
+ )
+ self._check_box_coord(
+ ax.patches[5:10],
+ expected_y=np.array([10, 9, 8, 7, 6]),
+ expected_h=np.array([8, 8, 8, 8, 8]),
+ )
+ self._check_box_coord(
+ ax.patches[10:],
+ expected_y=np.array([18, 17, 16, 15, 14]),
+ expected_h=np.array([6, 7, 8, 9, 10]),
+ )
+
+ axes = df.plot.hist(bins=5, stacked=True, subplots=True)
+ self._check_box_coord(
+ axes[0].patches,
+ expected_y=np.array([0, 0, 0, 0, 0]),
+ expected_h=np.array([10, 9, 8, 7, 6]),
+ )
+ self._check_box_coord(
+ axes[1].patches,
+ expected_y=np.array([0, 0, 0, 0, 0]),
+ expected_h=np.array([8, 8, 8, 8, 8]),
+ )
+ self._check_box_coord(
+ axes[2].patches,
+ expected_y=np.array([0, 0, 0, 0, 0]),
+ expected_h=np.array([6, 7, 8, 9, 10]),
+ )
+
+ # horizontal
+ ax = df.plot.hist(bins=5, orientation="horizontal")
+ self._check_box_coord(
+ ax.patches[:5],
+ expected_x=np.array([0, 0, 0, 0, 0]),
+ expected_w=np.array([10, 9, 8, 7, 6]),
+ )
+ self._check_box_coord(
+ ax.patches[5:10],
+ expected_x=np.array([0, 0, 0, 0, 0]),
+ expected_w=np.array([8, 8, 8, 8, 8]),
+ )
+ self._check_box_coord(
+ ax.patches[10:],
+ expected_x=np.array([0, 0, 0, 0, 0]),
+ expected_w=np.array([6, 7, 8, 9, 10]),
+ )
+
+ ax = df.plot.hist(bins=5, stacked=True, orientation="horizontal")
+ self._check_box_coord(
+ ax.patches[:5],
+ expected_x=np.array([0, 0, 0, 0, 0]),
+ expected_w=np.array([10, 9, 8, 7, 6]),
+ )
+ self._check_box_coord(
+ ax.patches[5:10],
+ expected_x=np.array([10, 9, 8, 7, 6]),
+ expected_w=np.array([8, 8, 8, 8, 8]),
+ )
+ self._check_box_coord(
+ ax.patches[10:],
+ expected_x=np.array([18, 17, 16, 15, 14]),
+ expected_w=np.array([6, 7, 8, 9, 10]),
+ )
+
+ axes = df.plot.hist(
+ bins=5, stacked=True, subplots=True, orientation="horizontal"
+ )
+ self._check_box_coord(
+ axes[0].patches,
+ expected_x=np.array([0, 0, 0, 0, 0]),
+ expected_w=np.array([10, 9, 8, 7, 6]),
+ )
+ self._check_box_coord(
+ axes[1].patches,
+ expected_x=np.array([0, 0, 0, 0, 0]),
+ expected_w=np.array([8, 8, 8, 8, 8]),
+ )
+ self._check_box_coord(
+ axes[2].patches,
+ expected_x=np.array([0, 0, 0, 0, 0]),
+ expected_w=np.array([6, 7, 8, 9, 10]),
+ )
+
+ def test_plot_int_columns(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((100, 4))).cumsum()
+ _check_plot_works(df.plot, legend=True)
+
+ @pytest.mark.parametrize(
+ "markers",
+ [
+ {0: "^", 1: "+", 2: "o"},
+ {0: "^", 1: "+"},
+ ["^", "+", "o"],
+ ["^", "+"],
+ ],
+ )
+ def test_style_by_column(self, markers):
+ fig = plt.gcf()
+ fig.clf()
+ fig.add_subplot(111)
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 3)))
+ ax = df.plot(style=markers)
+ for idx, line in enumerate(ax.get_lines()[: len(markers)]):
+ assert line.get_marker() == markers[idx]
+
+ def test_line_label_none(self):
+ s = Series([1, 2])
+ ax = s.plot()
+ assert ax.get_legend() is None
+
+ ax = s.plot(legend=True)
+ assert ax.get_legend().get_texts()[0].get_text() == ""
+
+ @pytest.mark.parametrize(
+ "props, expected",
+ [
+ ("boxprops", "boxes"),
+ ("whiskerprops", "whiskers"),
+ ("capprops", "caps"),
+ ("medianprops", "medians"),
+ ],
+ )
+ def test_specified_props_kwd_plot_box(self, props, expected):
+ # GH 30346
+ df = DataFrame({k: np.random.default_rng(2).random(100) for k in "ABC"})
+ kwd = {props: {"color": "C1"}}
+ result = df.plot.box(return_type="dict", **kwd)
+
+ assert result[expected][0].get_color() == "C1"
+
+ def test_unordered_ts(self):
+ # GH#2609, GH#55906
+ index = [date(2012, 10, 1), date(2012, 9, 1), date(2012, 8, 1)]
+ values = [3.0, 2.0, 1.0]
+ df = DataFrame(
+ np.array(values),
+ index=index,
+ columns=["test"],
+ )
+ ax = df.plot()
+ xticks = ax.lines[0].get_xdata()
+ tm.assert_numpy_array_equal(xticks, np.array(index, dtype=object))
+ ydata = ax.lines[0].get_ydata()
+ tm.assert_numpy_array_equal(ydata, np.array(values))
+
+ # even though we don't sort the data before passing it to matplotlib,
+ # the ticks are sorted
+ xticks = ax.xaxis.get_ticklabels()
+ xlocs = [x.get_position()[0] for x in xticks]
+ assert Index(xlocs).is_monotonic_increasing
+ xlabels = [x.get_text() for x in xticks]
+ assert pd.to_datetime(xlabels, format="%Y-%m-%d").is_monotonic_increasing
+
+ @pytest.mark.parametrize("kind", plotting.PlotAccessor._common_kinds)
+ def test_kind_both_ways(self, kind):
+ pytest.importorskip("scipy")
+ df = DataFrame({"x": [1, 2, 3]})
+ df.plot(kind=kind)
+ getattr(df.plot, kind)()
+
+ @pytest.mark.parametrize("kind", ["scatter", "hexbin"])
+ def test_kind_both_ways_x_y(self, kind):
+ pytest.importorskip("scipy")
+ df = DataFrame({"x": [1, 2, 3]})
+ df.plot("x", "x", kind=kind)
+ getattr(df.plot, kind)("x", "x")
+
+ @pytest.mark.parametrize("kind", plotting.PlotAccessor._common_kinds)
+ def test_all_invalid_plot_data(self, kind):
+ df = DataFrame(list("abcd"))
+ msg = "no numeric data to plot"
+ with pytest.raises(TypeError, match=msg):
+ df.plot(kind=kind)
+
+ @pytest.mark.parametrize(
+ "kind", [*list(plotting.PlotAccessor._common_kinds), "area"]
+ )
+ def test_partially_invalid_plot_data_numeric(self, kind):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)),
+ dtype=object,
+ )
+ df[np.random.default_rng(2).random(df.shape[0]) > 0.5] = "a"
+ msg = "no numeric data to plot"
+ with pytest.raises(TypeError, match=msg):
+ df.plot(kind=kind)
+
+ def test_invalid_kind(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 2)))
+ msg = "invalid_plot_kind is not a valid plot kind"
+ with pytest.raises(ValueError, match=msg):
+ df.plot(kind="invalid_plot_kind")
+
+ @pytest.mark.parametrize(
+ "x,y,lbl",
+ [
+ (["B", "C"], "A", "a"),
+ (["A"], ["B", "C"], ["b", "c"]),
+ ],
+ )
+ def test_invalid_xy_args(self, x, y, lbl):
+ # GH 18671, 19699 allows y to be list-like but not x
+ df = DataFrame({"A": [1, 2], "B": [3, 4], "C": [5, 6]})
+ with pytest.raises(ValueError, match="x must be a label or position"):
+ df.plot(x=x, y=y, label=lbl)
+
+ def test_bad_label(self):
+ df = DataFrame({"A": [1, 2], "B": [3, 4], "C": [5, 6]})
+ msg = "label should be list-like and same length as y"
+ with pytest.raises(ValueError, match=msg):
+ df.plot(x="A", y=["B", "C"], label="bad_label")
+
+ @pytest.mark.parametrize("x,y", [("A", "B"), (["A"], "B")])
+ def test_invalid_xy_args_dup_cols(self, x, y):
+ # GH 18671, 19699 allows y to be list-like but not x
+ df = DataFrame([[1, 3, 5], [2, 4, 6]], columns=list("AAB"))
+ with pytest.raises(ValueError, match="x must be a label or position"):
+ df.plot(x=x, y=y)
+
+ @pytest.mark.parametrize(
+ "x,y,lbl,colors",
+ [
+ ("A", ["B"], ["b"], ["red"]),
+ ("A", ["B", "C"], ["b", "c"], ["red", "blue"]),
+ (0, [1, 2], ["bokeh", "cython"], ["green", "yellow"]),
+ ],
+ )
+ def test_y_listlike(self, x, y, lbl, colors):
+ # GH 19699: tests list-like y and verifies lbls & colors
+ df = DataFrame({"A": [1, 2], "B": [3, 4], "C": [5, 6]})
+ _check_plot_works(df.plot, x="A", y=y, label=lbl)
+
+ ax = df.plot(x=x, y=y, label=lbl, color=colors)
+ assert len(ax.lines) == len(y)
+ _check_colors(ax.get_lines(), linecolors=colors)
+
+ @pytest.mark.parametrize("x,y,colnames", [(0, 1, ["A", "B"]), (1, 0, [0, 1])])
+ def test_xy_args_integer(self, x, y, colnames):
+ # GH 20056: tests integer args for xy and checks col names
+ df = DataFrame({"A": [1, 2], "B": [3, 4]})
+ df.columns = colnames
+ _check_plot_works(df.plot, x=x, y=y)
+
+ def test_hexbin_basic(self):
+ df = DataFrame(
+ {
+ "A": np.random.default_rng(2).uniform(size=20),
+ "B": np.random.default_rng(2).uniform(size=20),
+ "C": np.arange(20) + np.random.default_rng(2).uniform(size=20),
+ }
+ )
+
+ ax = df.plot.hexbin(x="A", y="B", gridsize=10)
+ # TODO: need better way to test. This just does existence.
+ assert len(ax.collections) == 1
+
+ def test_hexbin_basic_subplots(self):
+ df = DataFrame(
+ {
+ "A": np.random.default_rng(2).uniform(size=20),
+ "B": np.random.default_rng(2).uniform(size=20),
+ "C": np.arange(20) + np.random.default_rng(2).uniform(size=20),
+ }
+ )
+ # GH 6951
+ axes = df.plot.hexbin(x="A", y="B", subplots=True)
+ # hexbin should have 2 axes in the figure, 1 for plotting and another
+ # is colorbar
+ assert len(axes[0].figure.axes) == 2
+ # return value is single axes
+ _check_axes_shape(axes, axes_num=1, layout=(1, 1))
+
+ @pytest.mark.parametrize("reduce_C", [None, np.std])
+ def test_hexbin_with_c(self, reduce_C):
+ df = DataFrame(
+ {
+ "A": np.random.default_rng(2).uniform(size=20),
+ "B": np.random.default_rng(2).uniform(size=20),
+ "C": np.arange(20) + np.random.default_rng(2).uniform(size=20),
+ }
+ )
+
+ ax = df.plot.hexbin(x="A", y="B", C="C", reduce_C_function=reduce_C)
+ assert len(ax.collections) == 1
+
+ @pytest.mark.parametrize(
+ "kwargs, expected",
+ [
+ ({}, "BuGn"), # default cmap
+ ({"colormap": "cubehelix"}, "cubehelix"),
+ ({"cmap": "YlGn"}, "YlGn"),
+ ],
+ )
+ def test_hexbin_cmap(self, kwargs, expected):
+ df = DataFrame(
+ {
+ "A": np.random.default_rng(2).uniform(size=20),
+ "B": np.random.default_rng(2).uniform(size=20),
+ "C": np.arange(20) + np.random.default_rng(2).uniform(size=20),
+ }
+ )
+ ax = df.plot.hexbin(x="A", y="B", **kwargs)
+ assert ax.collections[0].cmap.name == expected
+
+ def test_pie_df_err(self):
+ df = DataFrame(
+ np.random.default_rng(2).random((5, 3)),
+ columns=["X", "Y", "Z"],
+ index=["a", "b", "c", "d", "e"],
+ )
+ msg = "pie requires either y column or 'subplots=True'"
+ with pytest.raises(ValueError, match=msg):
+ df.plot.pie()
+
+ @pytest.mark.parametrize("y", ["Y", 2])
+ def test_pie_df(self, y):
+ df = DataFrame(
+ np.random.default_rng(2).random((5, 3)),
+ columns=["X", "Y", "Z"],
+ index=["a", "b", "c", "d", "e"],
+ )
+ ax = _check_plot_works(df.plot.pie, y=y)
+ _check_text_labels(ax.texts, df.index)
+
+ def test_pie_df_subplots(self):
+ df = DataFrame(
+ np.random.default_rng(2).random((5, 3)),
+ columns=["X", "Y", "Z"],
+ index=["a", "b", "c", "d", "e"],
+ )
+ axes = _check_plot_works(
+ df.plot.pie,
+ default_axes=True,
+ subplots=True,
+ )
+ assert len(axes) == len(df.columns)
+ for ax in axes:
+ _check_text_labels(ax.texts, df.index)
+ for ax, ylabel in zip(axes, df.columns, strict=True):
+ assert ax.get_ylabel() == ""
+
+ def test_pie_df_labels_colors(self):
+ df = DataFrame(
+ np.random.default_rng(2).random((5, 3)),
+ columns=["X", "Y", "Z"],
+ index=["a", "b", "c", "d", "e"],
+ )
+ labels = ["A", "B", "C", "D", "E"]
+ color_args = ["r", "g", "b", "c", "m"]
+ axes = _check_plot_works(
+ df.plot.pie,
+ default_axes=True,
+ subplots=True,
+ labels=labels,
+ colors=color_args,
+ )
+ assert len(axes) == len(df.columns)
+
+ for ax in axes:
+ _check_text_labels(ax.texts, labels)
+ _check_colors(ax.patches, facecolors=color_args)
+
+ def test_pie_df_nan(self):
+ df = DataFrame(np.random.default_rng(2).random((4, 4)))
+ for i in range(4):
+ df.iloc[i, i] = np.nan
+ _, axes = mpl.pyplot.subplots(ncols=4)
+
+ # GH 37668
+ kwargs = {"normalize": True}
+
+ with tm.assert_produces_warning(None):
+ df.plot.pie(subplots=True, ax=axes, legend=True, **kwargs)
+
+ base_expected = ["0", "1", "2", "3"]
+ for i, ax in enumerate(axes):
+ expected = list(base_expected) # force copy
+ expected[i] = ""
+ result = [x.get_text() for x in ax.texts]
+ assert result == expected
+
+ # legend labels
+ # NaN's not included in legend with subplots
+ # see https://github.com/pandas-dev/pandas/issues/8390
+ result_labels = [x.get_text() for x in ax.get_legend().get_texts()]
+ expected_labels = base_expected[:i] + base_expected[i + 1 :]
+ assert result_labels == expected_labels
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"logy": True},
+ {"logx": True, "logy": True},
+ {"loglog": True},
+ ],
+ )
+ def test_errorbar_plot(self, kwargs):
+ d = {"x": np.arange(12), "y": np.arange(12, 0, -1)}
+ df = DataFrame(d)
+ d_err = {"x": np.ones(12) * 0.2, "y": np.ones(12) * 0.4}
+ df_err = DataFrame(d_err)
+
+ # check line plots
+ ax = _check_plot_works(df.plot, yerr=df_err, **kwargs)
+ _check_has_errorbars(ax, xerr=0, yerr=2)
+
+ @pytest.mark.slow
+ def test_errorbar_plot_bar(self):
+ d = {"x": np.arange(12), "y": np.arange(12, 0, -1)}
+ df = DataFrame(d)
+ d_err = {"x": np.ones(12) * 0.2, "y": np.ones(12) * 0.4}
+ df_err = DataFrame(d_err)
+ ax = _check_plot_works(
+ (df + 1).plot, yerr=df_err, xerr=df_err, kind="bar", log=True
+ )
+ _check_has_errorbars(ax, xerr=2, yerr=2)
+
+ @pytest.mark.slow
+ def test_errorbar_plot_yerr_array(self):
+ d = {"x": np.arange(12), "y": np.arange(12, 0, -1)}
+ df = DataFrame(d)
+ # yerr is raw error values
+ ax = _check_plot_works(df["y"].plot, yerr=np.ones(12) * 0.4)
+ _check_has_errorbars(ax, xerr=0, yerr=1)
+
+ ax = _check_plot_works(df.plot, yerr=np.ones((2, 12)) * 0.4)
+ _check_has_errorbars(ax, xerr=0, yerr=2)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("yerr", ["yerr", "誤差"])
+ def test_errorbar_plot_column_name(self, yerr):
+ d = {"x": np.arange(12), "y": np.arange(12, 0, -1)}
+ df = DataFrame(d)
+ df[yerr] = np.ones(12) * 0.2
+
+ ax = _check_plot_works(df.plot, yerr=yerr)
+ _check_has_errorbars(ax, xerr=0, yerr=2)
+
+ ax = _check_plot_works(df.plot, y="y", x="x", yerr=yerr)
+ _check_has_errorbars(ax, xerr=0, yerr=1)
+
+ @pytest.mark.slow
+ def test_errorbar_plot_external_valueerror(self):
+ d = {"x": np.arange(12), "y": np.arange(12, 0, -1)}
+ df = DataFrame(d)
+ with tm.external_error_raised(ValueError):
+ df.plot(yerr=np.random.default_rng(2).standard_normal(11))
+
+ @pytest.mark.slow
+ def test_errorbar_plot_external_typeerror(self):
+ d = {"x": np.arange(12), "y": np.arange(12, 0, -1)}
+ df = DataFrame(d)
+ df_err = DataFrame({"x": ["zzz"] * 12, "y": ["zzz"] * 12})
+ with tm.external_error_raised(TypeError):
+ df.plot(yerr=df_err)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("kind", ["line", "bar", "barh"])
+ @pytest.mark.parametrize(
+ "y_err",
+ [
+ Series(np.ones(12) * 0.2, name="x"),
+ DataFrame({"x": np.ones(12) * 0.2, "y": np.ones(12) * 0.4}),
+ ],
+ )
+ def test_errorbar_plot_different_yerr(self, kind, y_err):
+ df = DataFrame({"x": np.arange(12), "y": np.arange(12, 0, -1)})
+
+ ax = _check_plot_works(df.plot, yerr=y_err, kind=kind)
+ _check_has_errorbars(ax, xerr=0, yerr=2)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("kind", ["line", "bar", "barh"])
+ @pytest.mark.parametrize(
+ "y_err, x_err",
+ [
+ (
+ DataFrame({"x": np.ones(12) * 0.2, "y": np.ones(12) * 0.4}),
+ DataFrame({"x": np.ones(12) * 0.2, "y": np.ones(12) * 0.4}),
+ ),
+ (Series(np.ones(12) * 0.2, name="x"), Series(np.ones(12) * 0.2, name="x")),
+ (0.2, 0.2),
+ ],
+ )
+ def test_errorbar_plot_different_yerr_xerr(self, kind, y_err, x_err):
+ df = DataFrame({"x": np.arange(12), "y": np.arange(12, 0, -1)})
+ ax = _check_plot_works(df.plot, yerr=y_err, xerr=x_err, kind=kind)
+ _check_has_errorbars(ax, xerr=2, yerr=2)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("kind", ["line", "bar", "barh"])
+ def test_errorbar_plot_different_yerr_xerr_subplots(self, kind):
+ df = DataFrame({"x": np.arange(12), "y": np.arange(12, 0, -1)})
+ df_err = DataFrame({"x": np.ones(12) * 0.2, "y": np.ones(12) * 0.4})
+ axes = _check_plot_works(
+ df.plot,
+ default_axes=True,
+ yerr=df_err,
+ xerr=df_err,
+ subplots=True,
+ kind=kind,
+ )
+ _check_has_errorbars(axes, xerr=1, yerr=1)
+
+ @pytest.mark.xfail(reason="Iterator is consumed", raises=ValueError)
+ def test_errorbar_plot_iterator(self):
+ d = {"x": np.arange(12), "y": np.arange(12, 0, -1)}
+ df = DataFrame(d)
+
+ # yerr is iterator
+ ax = _check_plot_works(df.plot, yerr=itertools.repeat(0.1, len(df)))
+ _check_has_errorbars(ax, xerr=0, yerr=2)
+
+ def test_errorbar_with_integer_column_names(self):
+ # test with integer column names
+ df = DataFrame(np.abs(np.random.default_rng(2).standard_normal((10, 2))))
+ df_err = DataFrame(np.abs(np.random.default_rng(2).standard_normal((10, 2))))
+ ax = _check_plot_works(df.plot, yerr=df_err)
+ _check_has_errorbars(ax, xerr=0, yerr=2)
+ ax = _check_plot_works(df.plot, y=0, yerr=1)
+ _check_has_errorbars(ax, xerr=0, yerr=1)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("kind", ["line", "bar"])
+ def test_errorbar_with_partial_columns_kind(self, kind):
+ df = DataFrame(np.abs(np.random.default_rng(2).standard_normal((10, 3))))
+ df_err = DataFrame(
+ np.abs(np.random.default_rng(2).standard_normal((10, 2))), columns=[0, 2]
+ )
+ ax = _check_plot_works(df.plot, yerr=df_err, kind=kind)
+ _check_has_errorbars(ax, xerr=0, yerr=2)
+
+ @pytest.mark.slow
+ def test_errorbar_with_partial_columns_dti(self):
+ df = DataFrame(np.abs(np.random.default_rng(2).standard_normal((10, 3))))
+ df_err = DataFrame(
+ np.abs(np.random.default_rng(2).standard_normal((10, 2))), columns=[0, 2]
+ )
+ ix = date_range("1/1/2000", periods=10, freq="ME")
+ df.set_index(ix, inplace=True)
+ df_err.set_index(ix, inplace=True)
+ ax = _check_plot_works(df.plot, yerr=df_err, kind="line")
+ _check_has_errorbars(ax, xerr=0, yerr=2)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("err_box", [lambda x: x, DataFrame])
+ def test_errorbar_with_partial_columns_box(self, err_box):
+ d = {"x": np.arange(12), "y": np.arange(12, 0, -1)}
+ df = DataFrame(d)
+ err = err_box({"x": np.ones(12) * 0.2, "z": np.ones(12) * 0.4})
+ ax = _check_plot_works(df.plot, yerr=err)
+ _check_has_errorbars(ax, xerr=0, yerr=1)
+
+ @pytest.mark.parametrize("kind", ["line", "bar", "barh"])
+ def test_errorbar_timeseries(self, kind):
+ d = {"x": np.arange(12), "y": np.arange(12, 0, -1)}
+ d_err = {"x": np.ones(12) * 0.2, "y": np.ones(12) * 0.4}
+
+ # check time-series plots
+ ix = date_range("1/1/2000", "1/1/2001", freq="ME")
+ tdf = DataFrame(d, index=ix)
+ tdf_err = DataFrame(d_err, index=ix)
+
+ ax = _check_plot_works(tdf.plot, yerr=tdf_err, kind=kind)
+ _check_has_errorbars(ax, xerr=0, yerr=2)
+
+ ax = _check_plot_works(tdf.plot, yerr=d_err, kind=kind)
+ _check_has_errorbars(ax, xerr=0, yerr=2)
+
+ ax = _check_plot_works(tdf.plot, y="y", yerr=tdf_err["x"], kind=kind)
+ _check_has_errorbars(ax, xerr=0, yerr=1)
+
+ ax = _check_plot_works(tdf.plot, y="y", yerr="x", kind=kind)
+ _check_has_errorbars(ax, xerr=0, yerr=1)
+
+ ax = _check_plot_works(tdf.plot, yerr=tdf_err, kind=kind)
+ _check_has_errorbars(ax, xerr=0, yerr=2)
+
+ axes = _check_plot_works(
+ tdf.plot,
+ default_axes=True,
+ kind=kind,
+ yerr=tdf_err,
+ subplots=True,
+ )
+ _check_has_errorbars(axes, xerr=0, yerr=1)
+
+ def test_errorbar_asymmetrical(self):
+ err = np.random.default_rng(2).random((3, 2, 5))
+
+ # each column is [0, 1, 2, 3, 4], [3, 4, 5, 6, 7]...
+ df = DataFrame(np.arange(15).reshape(3, 5)).T
+
+ ax = df.plot(yerr=err, xerr=err / 2)
+
+ yerr_0_0 = ax.collections[1].get_paths()[0].vertices[:, 1]
+ expected_0_0 = err[0, :, 0] * np.array([-1, 1])
+ tm.assert_almost_equal(yerr_0_0, expected_0_0)
+
+ msg = re.escape(
+ "Asymmetrical error bars should be provided with the shape (3, 2, 5)"
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.plot(yerr=err.T)
+
+ def test_table(self):
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=list(string.ascii_letters[:10]),
+ )
+ _check_plot_works(df.plot, table=True)
+ _check_plot_works(df.plot, table=df)
+
+ # GH 35945 UserWarning
+ with tm.assert_produces_warning(None):
+ ax = df.plot()
+ assert len(ax.tables) == 0
+ plotting.table(ax, df.T)
+ assert len(ax.tables) == 1
+
+ def test_errorbar_scatter(self):
+ df = DataFrame(
+ np.abs(np.random.default_rng(2).standard_normal((5, 2))),
+ index=range(5),
+ columns=["x", "y"],
+ )
+ df_err = DataFrame(
+ np.abs(np.random.default_rng(2).standard_normal((5, 2))) / 5,
+ index=range(5),
+ columns=["x", "y"],
+ )
+
+ ax = _check_plot_works(df.plot.scatter, x="x", y="y")
+ _check_has_errorbars(ax, xerr=0, yerr=0)
+ ax = _check_plot_works(df.plot.scatter, x="x", y="y", xerr=df_err)
+ _check_has_errorbars(ax, xerr=1, yerr=0)
+
+ ax = _check_plot_works(df.plot.scatter, x="x", y="y", yerr=df_err)
+ _check_has_errorbars(ax, xerr=0, yerr=1)
+ ax = _check_plot_works(df.plot.scatter, x="x", y="y", xerr=df_err, yerr=df_err)
+ _check_has_errorbars(ax, xerr=1, yerr=1)
+
+ def test_errorbar_scatter_color(self):
+ def _check_errorbar_color(containers, expected, has_err="has_xerr"):
+ lines = []
+ errs = next(c.lines for c in ax.containers if getattr(c, has_err, False))
+ for el in errs:
+ if is_list_like(el):
+ lines.extend(el)
+ else:
+ lines.append(el)
+ err_lines = [x for x in lines if x in ax.collections]
+ _check_colors(err_lines, linecolors=np.array([expected] * len(err_lines)))
+
+ # GH 8081
+ df = DataFrame(
+ np.abs(np.random.default_rng(2).standard_normal((10, 5))),
+ columns=["a", "b", "c", "d", "e"],
+ )
+ ax = df.plot.scatter(x="a", y="b", xerr="d", yerr="e", c="red")
+ _check_has_errorbars(ax, xerr=1, yerr=1)
+ _check_errorbar_color(ax.containers, "red", has_err="has_xerr")
+ _check_errorbar_color(ax.containers, "red", has_err="has_yerr")
+
+ ax = df.plot.scatter(x="a", y="b", yerr="e", color="green")
+ _check_has_errorbars(ax, xerr=0, yerr=1)
+ _check_errorbar_color(ax.containers, "green", has_err="has_yerr")
+
+ def test_scatter_unknown_colormap(self):
+ # GH#48726
+ df = DataFrame({"a": [1, 2, 3], "b": 4})
+ with pytest.raises((ValueError, KeyError), match="'unknown' is not a"):
+ df.plot(x="a", y="b", colormap="unknown", kind="scatter")
+
+ def test_sharex_and_ax(self):
+ # https://github.com/pandas-dev/pandas/issues/9737 using gridspec,
+ # the axis in fig.get_axis() are sorted differently than pandas
+ # expected them, so make sure that only the right ones are removed
+ gs, axes = _generate_4_axes_via_gridspec()
+
+ df = DataFrame(
+ {
+ "a": [1, 2, 3, 4, 5, 6],
+ "b": [1, 2, 3, 4, 5, 6],
+ "c": [1, 2, 3, 4, 5, 6],
+ "d": [1, 2, 3, 4, 5, 6],
+ }
+ )
+
+ def _check(axes):
+ for ax in axes:
+ assert len(ax.lines) == 1
+ _check_visible(ax.get_yticklabels(), visible=True)
+ for ax in [axes[0], axes[2]]:
+ _check_visible(ax.get_xticklabels(), visible=False)
+ _check_visible(ax.get_xticklabels(minor=True), visible=False)
+ for ax in [axes[1], axes[3]]:
+ _check_visible(ax.get_xticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(minor=True), visible=True)
+
+ for ax in axes:
+ df.plot(x="a", y="b", title="title", ax=ax, sharex=True)
+ gs.tight_layout(plt.gcf())
+ _check(axes)
+ plt.close("all")
+
+ gs, axes = _generate_4_axes_via_gridspec()
+ with tm.assert_produces_warning(UserWarning, match="sharex and sharey"):
+ axes = df.plot(subplots=True, ax=axes, sharex=True)
+ _check(axes)
+
+ def test_sharex_false_and_ax(self):
+ # https://github.com/pandas-dev/pandas/issues/9737 using gridspec,
+ # the axis in fig.get_axis() are sorted differently than pandas
+ # expected them, so make sure that only the right ones are removed
+ df = DataFrame(
+ {
+ "a": [1, 2, 3, 4, 5, 6],
+ "b": [1, 2, 3, 4, 5, 6],
+ "c": [1, 2, 3, 4, 5, 6],
+ "d": [1, 2, 3, 4, 5, 6],
+ }
+ )
+ gs, axes = _generate_4_axes_via_gridspec()
+ # without sharex, no labels should be touched!
+ for ax in axes:
+ df.plot(x="a", y="b", title="title", ax=ax)
+
+ gs.tight_layout(plt.gcf())
+ for ax in axes:
+ assert len(ax.lines) == 1
+ _check_visible(ax.get_yticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(minor=True), visible=True)
+
+ def test_sharey_and_ax(self):
+ # https://github.com/pandas-dev/pandas/issues/9737 using gridspec,
+ # the axis in fig.get_axis() are sorted differently than pandas
+ # expected them, so make sure that only the right ones are removed
+ gs, axes = _generate_4_axes_via_gridspec()
+
+ df = DataFrame(
+ {
+ "a": [1, 2, 3, 4, 5, 6],
+ "b": [1, 2, 3, 4, 5, 6],
+ "c": [1, 2, 3, 4, 5, 6],
+ "d": [1, 2, 3, 4, 5, 6],
+ }
+ )
+
+ def _check(axes):
+ for ax in axes:
+ assert len(ax.lines) == 1
+ _check_visible(ax.get_xticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(minor=True), visible=True)
+ for ax in [axes[0], axes[1]]:
+ _check_visible(ax.get_yticklabels(), visible=True)
+ for ax in [axes[2], axes[3]]:
+ _check_visible(ax.get_yticklabels(), visible=False)
+
+ for ax in axes:
+ df.plot(x="a", y="b", title="title", ax=ax, sharey=True)
+ gs.tight_layout(plt.gcf())
+ _check(axes)
+ plt.close("all")
+
+ gs, axes = _generate_4_axes_via_gridspec()
+ with tm.assert_produces_warning(UserWarning, match="sharex and sharey"):
+ axes = df.plot(subplots=True, ax=axes, sharey=True)
+
+ gs.tight_layout(plt.gcf())
+ _check(axes)
+
+ def test_sharey_and_ax_tight(self):
+ # https://github.com/pandas-dev/pandas/issues/9737 using gridspec,
+ df = DataFrame(
+ {
+ "a": [1, 2, 3, 4, 5, 6],
+ "b": [1, 2, 3, 4, 5, 6],
+ "c": [1, 2, 3, 4, 5, 6],
+ "d": [1, 2, 3, 4, 5, 6],
+ }
+ )
+ gs, axes = _generate_4_axes_via_gridspec()
+ # without sharex, no labels should be touched!
+ for ax in axes:
+ df.plot(x="a", y="b", title="title", ax=ax)
+
+ gs.tight_layout(plt.gcf())
+ for ax in axes:
+ assert len(ax.lines) == 1
+ _check_visible(ax.get_yticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(minor=True), visible=True)
+
+ @pytest.mark.parametrize("kind", plotting.PlotAccessor._all_kinds)
+ def test_memory_leak(self, kind):
+ """Check that every plot type gets properly collected."""
+ pytest.importorskip("scipy")
+ args = {}
+ if kind in ["hexbin", "scatter", "pie"]:
+ df = DataFrame(
+ {
+ "A": np.random.default_rng(2).uniform(size=20),
+ "B": np.random.default_rng(2).uniform(size=20),
+ "C": np.arange(20) + np.random.default_rng(2).uniform(size=20),
+ }
+ )
+ args = {"x": "A", "y": "B"}
+ elif kind == "area":
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ ).abs()
+ else:
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+
+ ax = df.plot(kind=kind, **args)
+ # https://github.com/pandas-dev/pandas/issues/9003#issuecomment-70544889
+ if kind in ["line", "area"]:
+ for i, (cached_data, _, _) in enumerate(ax._plot_data):
+ ser = df.iloc[:, i]
+ assert not tm.shares_memory(ser, cached_data)
+ tm.assert_numpy_array_equal(ser._values, cached_data._values)
+ else:
+ assert not hasattr(ax, "_plot_data")
+
+ def test_df_gridspec_patterns_vert_horiz(self):
+ # GH 10819
+ ts = Series(
+ np.random.default_rng(2).standard_normal(10),
+ index=date_range("1/1/2000", periods=10),
+ )
+
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)),
+ index=ts.index,
+ columns=list("AB"),
+ )
+
+ def _get_vertical_grid():
+ gs = mpl.gridspec.GridSpec(3, 1)
+ fig = plt.figure()
+ ax1 = fig.add_subplot(gs[:2, :])
+ ax2 = fig.add_subplot(gs[2, :])
+ return ax1, ax2
+
+ def _get_horizontal_grid():
+ gs = mpl.gridspec.GridSpec(1, 3)
+ fig = plt.figure()
+ ax1 = fig.add_subplot(gs[:, :2])
+ ax2 = fig.add_subplot(gs[:, 2])
+ return ax1, ax2
+
+ for ax1, ax2 in [_get_vertical_grid(), _get_horizontal_grid()]:
+ ax1 = ts.plot(ax=ax1)
+ assert len(ax1.lines) == 1
+ ax2 = df.plot(ax=ax2)
+ assert len(ax2.lines) == 2
+ for ax in [ax1, ax2]:
+ _check_visible(ax.get_yticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(minor=True), visible=True)
+ plt.close("all")
+
+ # subplots=True
+ for ax1, ax2 in [_get_vertical_grid(), _get_horizontal_grid()]:
+ axes = df.plot(subplots=True, ax=[ax1, ax2])
+ assert len(ax1.lines) == 1
+ assert len(ax2.lines) == 1
+ for ax in axes:
+ _check_visible(ax.get_yticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(minor=True), visible=True)
+ plt.close("all")
+
+ # vertical / subplots / sharex=True / sharey=True
+ ax1, ax2 = _get_vertical_grid()
+ with tm.assert_produces_warning(UserWarning, match="sharex and sharey"):
+ axes = df.plot(subplots=True, ax=[ax1, ax2], sharex=True, sharey=True)
+ assert len(axes[0].lines) == 1
+ assert len(axes[1].lines) == 1
+ for ax in [ax1, ax2]:
+ # yaxis are visible because there is only one column
+ _check_visible(ax.get_yticklabels(), visible=True)
+ # xaxis of axes0 (top) are hidden
+ _check_visible(axes[0].get_xticklabels(), visible=False)
+ _check_visible(axes[0].get_xticklabels(minor=True), visible=False)
+ _check_visible(axes[1].get_xticklabels(), visible=True)
+ _check_visible(axes[1].get_xticklabels(minor=True), visible=True)
+ plt.close("all")
+
+ # horizontal / subplots / sharex=True / sharey=True
+ ax1, ax2 = _get_horizontal_grid()
+ with tm.assert_produces_warning(UserWarning, match="sharex and sharey"):
+ axes = df.plot(subplots=True, ax=[ax1, ax2], sharex=True, sharey=True)
+ assert len(axes[0].lines) == 1
+ assert len(axes[1].lines) == 1
+ _check_visible(axes[0].get_yticklabels(), visible=True)
+ # yaxis of axes1 (right) are hidden
+ _check_visible(axes[1].get_yticklabels(), visible=False)
+ for ax in [ax1, ax2]:
+ # xaxis are visible because there is only one column
+ _check_visible(ax.get_xticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(minor=True), visible=True)
+ plt.close("all")
+
+ def test_df_gridspec_patterns_boxed(self):
+ # GH 10819
+ ts = Series(
+ np.random.default_rng(2).standard_normal(10),
+ index=date_range("1/1/2000", periods=10),
+ )
+
+ # boxed
+ def _get_boxed_grid():
+ gs = mpl.gridspec.GridSpec(3, 3)
+ fig = plt.figure()
+ ax1 = fig.add_subplot(gs[:2, :2])
+ ax2 = fig.add_subplot(gs[:2, 2])
+ ax3 = fig.add_subplot(gs[2, :2])
+ ax4 = fig.add_subplot(gs[2, 2])
+ return ax1, ax2, ax3, ax4
+
+ axes = _get_boxed_grid()
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ index=ts.index,
+ columns=list("ABCD"),
+ )
+ axes = df.plot(subplots=True, ax=axes)
+ for ax in axes:
+ assert len(ax.lines) == 1
+ # axis are visible because these are not shared
+ _check_visible(ax.get_yticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(minor=True), visible=True)
+ plt.close("all")
+
+ # subplots / sharex=True / sharey=True
+ axes = _get_boxed_grid()
+ with tm.assert_produces_warning(UserWarning, match="sharex and sharey"):
+ axes = df.plot(subplots=True, ax=axes, sharex=True, sharey=True)
+ for ax in axes:
+ assert len(ax.lines) == 1
+ for ax in [axes[0], axes[2]]: # left column
+ _check_visible(ax.get_yticklabels(), visible=True)
+ for ax in [axes[1], axes[3]]: # right column
+ _check_visible(ax.get_yticklabels(), visible=False)
+ for ax in [axes[0], axes[1]]: # top row
+ _check_visible(ax.get_xticklabels(), visible=False)
+ _check_visible(ax.get_xticklabels(minor=True), visible=False)
+ for ax in [axes[2], axes[3]]: # bottom row
+ _check_visible(ax.get_xticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(minor=True), visible=True)
+ plt.close("all")
+
+ def test_df_grid_settings(self):
+ # Make sure plot defaults to rcParams['axes.grid'] setting, GH 9792
+ _check_grid_settings(
+ DataFrame({"a": [1, 2, 3], "b": [2, 3, 4]}),
+ plotting.PlotAccessor._dataframe_kinds,
+ kws={"x": "a", "y": "b"},
+ )
+
+ def test_plain_axes(self):
+ # supplied ax itself is a SubplotAxes, but figure contains also
+ # a plain Axes object (GH11556)
+ fig, ax = mpl.pyplot.subplots()
+ fig.add_axes([0.2, 0.2, 0.2, 0.2])
+ Series(np.random.default_rng(2).random(10)).plot(ax=ax)
+
+ def test_plain_axes_df(self):
+ # supplied ax itself is a plain Axes, but because the cmap keyword
+ # a new ax is created for the colorbar -> also multiples axes (GH11520)
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).standard_normal(8),
+ "b": np.random.default_rng(2).standard_normal(8),
+ }
+ )
+ fig = mpl.pyplot.figure()
+ ax = fig.add_axes((0, 0, 1, 1))
+ df.plot(kind="scatter", ax=ax, x="a", y="b", c="a", cmap="hsv")
+
+ def test_plain_axes_make_axes_locatable(self):
+ # other examples
+ fig, ax = mpl.pyplot.subplots()
+ from mpl_toolkits.axes_grid1 import make_axes_locatable
+
+ divider = make_axes_locatable(ax)
+ cax = divider.append_axes("right", size="5%", pad=0.05)
+ Series(np.random.default_rng(2).random(10)).plot(ax=ax)
+ Series(np.random.default_rng(2).random(10)).plot(ax=cax)
+
+ def test_plain_axes_make_inset_axes(self):
+ fig, ax = mpl.pyplot.subplots()
+ from mpl_toolkits.axes_grid1.inset_locator import inset_axes
+
+ iax = inset_axes(ax, width="30%", height=1.0, loc=3)
+ Series(np.random.default_rng(2).random(10)).plot(ax=ax)
+ Series(np.random.default_rng(2).random(10)).plot(ax=iax)
+
+ @pytest.mark.parametrize("method", ["line", "barh", "bar"])
+ def test_secondary_axis_font_size(self, method):
+ # GH: 12565
+ df = (
+ DataFrame(
+ np.random.default_rng(2).standard_normal((15, 2)), columns=list("AB")
+ )
+ .assign(C=lambda df: df.B.cumsum())
+ .assign(D=lambda df: df.C * 1.1)
+ )
+
+ fontsize = 20
+ sy = ["C", "D"]
+
+ kwargs = {"secondary_y": sy, "fontsize": fontsize, "mark_right": True}
+ ax = getattr(df.plot, method)(**kwargs)
+ _check_ticks_props(axes=ax.right_ax, ylabelsize=fontsize)
+
+ def test_x_string_values_ticks(self):
+ # Test if string plot index have a fixed xtick position
+ # GH: 7612, GH: 22334
+ df = DataFrame(
+ {
+ "sales": [3, 2, 3],
+ "visits": [20, 42, 28],
+ "day": ["Monday", "Tuesday", "Wednesday"],
+ }
+ )
+ ax = df.plot.area(x="day")
+ ax.set_xlim(-1, 3)
+ xticklabels = [t.get_text() for t in ax.get_xticklabels()]
+ labels_position = dict(zip(xticklabels, ax.get_xticks(), strict=False))
+ # Testing if the label stayed at the right position
+ assert labels_position["Monday"] == 0.0
+ assert labels_position["Tuesday"] == 1.0
+ assert labels_position["Wednesday"] == 2.0
+
+ def test_x_multiindex_values_ticks(self):
+ # Test if multiindex plot index have a fixed xtick position
+ # GH: 15912
+ index = MultiIndex.from_product([[2012, 2013], [1, 2]])
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((4, 2)),
+ columns=["A", "B"],
+ index=index,
+ )
+ ax = df.plot()
+ ax.set_xlim(-1, 4)
+ xticklabels = [t.get_text() for t in ax.get_xticklabels()]
+ labels_position = dict(zip(xticklabels, ax.get_xticks(), strict=False))
+ # Testing if the label stayed at the right position
+ assert labels_position["(2012, 1)"] == 0.0
+ assert labels_position["(2012, 2)"] == 1.0
+ assert labels_position["(2013, 1)"] == 2.0
+ assert labels_position["(2013, 2)"] == 3.0
+
+ @pytest.mark.parametrize("kind", ["line", "area"])
+ def test_xlim_plot_line(self, kind):
+ # test if xlim is set correctly in plot.line and plot.area
+ # GH 27686
+ df = DataFrame([2, 4], index=[1, 2])
+ ax = df.plot(kind=kind)
+ xlims = ax.get_xlim()
+ assert xlims[0] < 1
+ assert xlims[1] > 2
+
+ def test_xlim_plot_line_correctly_in_mixed_plot_type(self):
+ # test if xlim is set correctly when ax contains multiple different kinds
+ # of plots, GH 27686
+ fig, ax = mpl.pyplot.subplots()
+
+ indexes = ["k1", "k2", "k3", "k4"]
+ df = DataFrame(
+ {
+ "s1": [1000, 2000, 1500, 2000],
+ "s2": [900, 1400, 2000, 3000],
+ "s3": [1500, 1500, 1600, 1200],
+ "secondary_y": [1, 3, 4, 3],
+ },
+ index=indexes,
+ )
+ df[["s1", "s2", "s3"]].plot.bar(ax=ax, stacked=False)
+ df[["secondary_y"]].plot(ax=ax, secondary_y=True)
+
+ xlims = ax.get_xlim()
+ assert xlims[0] < 0
+ assert xlims[1] > 3
+
+ # make sure axis labels are plotted correctly as well
+ xticklabels = [t.get_text() for t in ax.get_xticklabels()]
+ assert xticklabels == indexes
+
+ def test_plot_no_rows(self):
+ # GH 27758
+ df = DataFrame(columns=["foo"], dtype=int)
+ assert df.empty
+ ax = df.plot()
+ assert len(ax.get_lines()) == 1
+ line = ax.get_lines()[0]
+ assert len(line.get_xdata()) == 0
+ assert len(line.get_ydata()) == 0
+
+ def test_plot_no_numeric_data(self):
+ df = DataFrame(["a", "b", "c"])
+ with pytest.raises(TypeError, match="no numeric data to plot"):
+ df.plot()
+
+ @pytest.mark.parametrize(
+ "kind", ("line", "bar", "barh", "hist", "kde", "density", "area", "pie")
+ )
+ def test_group_subplot(self, kind):
+ pytest.importorskip("scipy")
+ d = {
+ "a": np.arange(10),
+ "b": np.arange(10) + 1,
+ "c": np.arange(10) + 1,
+ "d": np.arange(10),
+ "e": np.arange(10),
+ }
+ df = DataFrame(d)
+
+ axes = df.plot(subplots=[("b", "e"), ("c", "d")], kind=kind)
+ assert len(axes) == 3 # 2 groups + single column a
+
+ expected_labels = (["b", "e"], ["c", "d"], ["a"])
+ for ax, labels in zip(axes, expected_labels, strict=True):
+ if kind != "pie":
+ _check_legend_labels(ax, labels=labels)
+ if kind == "line":
+ assert len(ax.lines) == len(labels)
+
+ def test_group_subplot_series_notimplemented(self):
+ ser = Series(range(1))
+ msg = "An iterable subplots for a Series"
+ with pytest.raises(NotImplementedError, match=msg):
+ ser.plot(subplots=[("a",)])
+
+ def test_group_subplot_multiindex_notimplemented(self):
+ df = DataFrame(np.eye(2), columns=MultiIndex.from_tuples([(0, 1), (1, 2)]))
+ msg = "An iterable subplots for a DataFrame with a MultiIndex"
+ with pytest.raises(NotImplementedError, match=msg):
+ df.plot(subplots=[(0, 1)])
+
+ def test_group_subplot_nonunique_cols_notimplemented(self):
+ df = DataFrame(np.eye(2), columns=["a", "a"])
+ msg = "An iterable subplots for a DataFrame with non-unique"
+ with pytest.raises(NotImplementedError, match=msg):
+ df.plot(subplots=[("a",)])
+
+ @pytest.mark.parametrize(
+ "subplots, expected_msg",
+ [
+ (123, "subplots should be a bool or an iterable"),
+ ("a", "each entry should be a list/tuple"), # iterable of non-iterable
+ ((1,), "each entry should be a list/tuple"), # iterable of non-iterable
+ (("a",), "each entry should be a list/tuple"), # iterable of strings
+ ],
+ )
+ def test_group_subplot_bad_input(self, subplots, expected_msg):
+ # Make sure error is raised when subplots is not a properly
+ # formatted iterable. Only iterables of iterables are permitted, and
+ # entries should not be strings.
+ d = {"a": np.arange(10), "b": np.arange(10)}
+ df = DataFrame(d)
+
+ with pytest.raises(ValueError, match=expected_msg):
+ df.plot(subplots=subplots)
+
+ def test_group_subplot_invalid_column_name(self):
+ d = {"a": np.arange(10), "b": np.arange(10)}
+ df = DataFrame(d)
+
+ if Version(np.__version__) < Version("2.0.0"):
+ with pytest.raises(ValueError, match=r"Column label\(s\) \['bad_name'\]"):
+ df.plot(subplots=[("a", "bad_name")])
+ else:
+ with pytest.raises(
+ ValueError, match=r"Column label\(s\) \[np\.str\_\('bad_name'\)\]"
+ ):
+ df.plot(subplots=[("a", "bad_name")])
+
+ def test_group_subplot_duplicated_column(self):
+ d = {"a": np.arange(10), "b": np.arange(10), "c": np.arange(10)}
+ df = DataFrame(d)
+
+ with pytest.raises(ValueError, match="should be in only one subplot"):
+ df.plot(subplots=[("a", "b"), ("a", "c")])
+
+ @pytest.mark.parametrize("kind", ("box", "scatter", "hexbin"))
+ def test_group_subplot_invalid_kind(self, kind):
+ d = {"a": np.arange(10), "b": np.arange(10)}
+ df = DataFrame(d)
+ with pytest.raises(
+ ValueError, match="When subplots is an iterable, kind must be one of"
+ ):
+ df.plot(subplots=[("a", "b")], kind=kind)
+
+ @pytest.mark.parametrize(
+ "index_name, old_label, new_label",
+ [
+ (None, "", "new"),
+ ("old", "old", "new"),
+ (None, "", ""),
+ (None, "", 1),
+ (None, "", [1, 2]),
+ ],
+ )
+ @pytest.mark.parametrize("kind", ["line", "area", "bar"])
+ def test_xlabel_ylabel_dataframe_single_plot(
+ self, kind, index_name, old_label, new_label
+ ):
+ # GH 9093
+ df = DataFrame([[1, 2], [2, 5]], columns=["Type A", "Type B"])
+ df.index.name = index_name
+
+ # default is the ylabel is not shown and xlabel is index name
+ ax = df.plot(kind=kind)
+ assert ax.get_xlabel() == old_label
+ assert ax.get_ylabel() == ""
+
+ # old xlabel will be overridden and assigned ylabel will be used as ylabel
+ ax = df.plot(kind=kind, ylabel=new_label, xlabel=new_label)
+ assert ax.get_ylabel() == str(new_label)
+ assert ax.get_xlabel() == str(new_label)
+
+ @pytest.mark.parametrize(
+ "xlabel, ylabel",
+ [
+ (None, None),
+ ("X Label", None),
+ (None, "Y Label"),
+ ("X Label", "Y Label"),
+ ],
+ )
+ @pytest.mark.parametrize("kind", ["scatter", "hexbin"])
+ def test_xlabel_ylabel_dataframe_plane_plot(self, kind, xlabel, ylabel):
+ # GH 37001
+ xcol = "Type A"
+ ycol = "Type B"
+ df = DataFrame([[1, 2], [2, 5]], columns=[xcol, ycol])
+
+ # default is the labels are column names
+ ax = df.plot(kind=kind, x=xcol, y=ycol, xlabel=xlabel, ylabel=ylabel)
+ assert ax.get_xlabel() == (xcol if xlabel is None else xlabel)
+ assert ax.get_ylabel() == (ycol if ylabel is None else ylabel)
+
+ @pytest.mark.parametrize("secondary_y", (False, True))
+ def test_secondary_y(self, secondary_y):
+ ax_df = DataFrame([0]).plot(
+ secondary_y=secondary_y, ylabel="Y", ylim=(0, 100), yticks=[99]
+ )
+ for ax in ax_df.figure.axes:
+ if ax.yaxis.get_visible():
+ assert ax.get_ylabel() == "Y"
+ assert ax.get_ylim() == (0, 100)
+ assert ax.get_yticks()[0] == 99
+
+ @pytest.mark.slow
+ def test_plot_no_warning(self):
+ # GH 55138
+ # TODO(3.0): this can be removed once Period[B] deprecation is enforced
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ with tm.assert_produces_warning(False):
+ _ = df.plot()
+ _ = df.T.plot()
+
+ @pytest.mark.parametrize("freq", ["h", "7h", "60min", "120min", "3M"])
+ def test_plot_period_index_makes_no_right_shift(self, freq):
+ # GH#57587
+ idx = pd.period_range("01/01/2000", freq=freq, periods=4)
+ df = DataFrame(
+ np.array([0, 1, 0, 1]),
+ index=idx,
+ columns=["A"],
+ )
+ expected = idx.values
+
+ ax = df.plot()
+ result = ax.get_lines()[0].get_xdata()
+ assert all(str(result[i]) == str(expected[i]) for i in range(4))
+
+ def test_plot_display_xlabel_and_xticks(self):
+ # GH#44050
+ df = DataFrame(np.random.default_rng(2).random((10, 2)), columns=["a", "b"])
+ ax = df.plot.hexbin(x="a", y="b")
+
+ _check_visible([ax.xaxis.get_label()], visible=True)
+ _check_visible(ax.get_xticklabels(), visible=True)
+
+
+def _generate_4_axes_via_gridspec():
+ gs = mpl.gridspec.GridSpec(2, 2)
+ ax_tl = plt.subplot(gs[0, 0])
+ ax_ll = plt.subplot(gs[1, 0])
+ ax_tr = plt.subplot(gs[0, 1])
+ ax_lr = plt.subplot(gs[1, 1])
+
+ return gs, [ax_tl, ax_ll, ax_tr, ax_lr]
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame_color.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame_color.py
new file mode 100644
index 0000000000000000000000000000000000000000..96adf8e5bb8c91ce83f85088f648bdaf737b8e83
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame_color.py
@@ -0,0 +1,734 @@
+"""Test cases for DataFrame.plot"""
+
+import re
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import DataFrame
+import pandas._testing as tm
+from pandas.tests.plotting.common import (
+ _check_colors,
+ _check_plot_works,
+ _unpack_cycler,
+)
+
+mpl = pytest.importorskip("matplotlib")
+plt = pytest.importorskip("matplotlib.pyplot")
+cm = pytest.importorskip("matplotlib.cm")
+
+
+def _check_colors_box(bp, box_c, whiskers_c, medians_c, caps_c="k", fliers_c=None):
+ if fliers_c is None:
+ fliers_c = "k"
+ _check_colors(bp["boxes"], linecolors=[box_c] * len(bp["boxes"]))
+ _check_colors(bp["whiskers"], linecolors=[whiskers_c] * len(bp["whiskers"]))
+ _check_colors(bp["medians"], linecolors=[medians_c] * len(bp["medians"]))
+ _check_colors(bp["fliers"], linecolors=[fliers_c] * len(bp["fliers"]))
+ _check_colors(bp["caps"], linecolors=[caps_c] * len(bp["caps"]))
+
+
+class TestDataFrameColor:
+ @pytest.mark.parametrize("color", list(range(10)))
+ def test_mpl2_color_cycle_str(self, color):
+ # GH 15516
+ color = f"C{color}"
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 3)), columns=["a", "b", "c"]
+ )
+ _check_plot_works(df.plot, color=color)
+
+ def test_color_single_series_list(self):
+ # GH 3486
+ df = DataFrame({"A": [1, 2, 3]})
+ _check_plot_works(df.plot, color=["red"])
+
+ @pytest.mark.parametrize("color", [(1, 0, 0), (1, 0, 0, 0.5)])
+ def test_rgb_tuple_color(self, color):
+ # GH 16695
+ df = DataFrame({"x": [1, 2], "y": [3, 4]})
+ _check_plot_works(df.plot, x="x", y="y", color=color)
+
+ def test_color_empty_string(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 2)))
+ with pytest.raises(ValueError, match="Invalid color argument:"):
+ df.plot(color="")
+
+ def test_color_and_style_arguments(self):
+ df = DataFrame({"x": [1, 2], "y": [3, 4]})
+ # passing both 'color' and 'style' arguments should be allowed
+ # if there is no color symbol in the style strings:
+ ax = df.plot(color=["red", "black"], style=["-", "--"])
+ # check that the linestyles are correctly set:
+ linestyle = [line.get_linestyle() for line in ax.lines]
+ assert linestyle == ["-", "--"]
+ # check that the colors are correctly set:
+ color = [line.get_color() for line in ax.lines]
+ assert color == ["red", "black"]
+ # passing both 'color' and 'style' arguments should not be allowed
+ # if there is a color symbol in the style strings:
+ msg = (
+ "Cannot pass 'style' string with a color symbol and 'color' keyword "
+ "argument. Please use one or the other or pass 'style' without a color "
+ "symbol"
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.plot(color=["red", "black"], style=["k-", "r--"])
+
+ @pytest.mark.parametrize(
+ "color, expected",
+ [
+ ("green", ["green"] * 4),
+ (["yellow", "red", "green", "blue"], ["yellow", "red", "green", "blue"]),
+ ],
+ )
+ def test_color_and_marker(self, color, expected):
+ # GH 21003
+ df = DataFrame(np.random.default_rng(2).random((7, 4)))
+ ax = df.plot(color=color, style="d--")
+ # check colors
+ result = [i.get_color() for i in ax.lines]
+ assert result == expected
+ # check markers and linestyles
+ assert all(i.get_linestyle() == "--" for i in ax.lines)
+ assert all(i.get_marker() == "d" for i in ax.lines)
+
+ def test_color_and_style(self):
+ color = {"g": "black", "h": "brown"}
+ style = {"g": "-", "h": "--"}
+ expected_color = ["black", "brown"]
+ expected_style = ["-", "--"]
+ df = DataFrame({"g": [1, 2], "h": [2, 3]}, index=[1, 2])
+ ax = df.plot.line(color=color, style=style)
+ color = [i.get_color() for i in ax.lines]
+ style = [i.get_linestyle() for i in ax.lines]
+ assert color == expected_color
+ assert style == expected_style
+
+ def test_bar_colors(self):
+ default_colors = _unpack_cycler(plt.rcParams)
+
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ ax = df.plot.bar()
+ _check_colors(ax.patches[::5], facecolors=default_colors[:5])
+
+ def test_bar_colors_custom(self):
+ custom_colors = "rgcby"
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ ax = df.plot.bar(color=custom_colors)
+ _check_colors(ax.patches[::5], facecolors=custom_colors)
+
+ @pytest.mark.parametrize("colormap", ["jet", cm.jet])
+ def test_bar_colors_cmap(self, colormap):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+
+ ax = df.plot.bar(colormap=colormap)
+ rgba_colors = [cm.jet(n) for n in np.linspace(0, 1, 5)]
+ _check_colors(ax.patches[::5], facecolors=rgba_colors)
+
+ def test_bar_colors_single_col(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ ax = df.loc[:, [0]].plot.bar(color="DodgerBlue")
+ _check_colors([ax.patches[0]], facecolors=["DodgerBlue"])
+
+ def test_bar_colors_green(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ ax = df.plot(kind="bar", color="green")
+ _check_colors(ax.patches[::5], facecolors=["green"] * 5)
+
+ def test_bar_user_colors(self):
+ df = DataFrame(
+ {"A": range(4), "B": range(1, 5), "color": ["red", "blue", "blue", "red"]}
+ )
+ # This should *only* work when `y` is specified, else
+ # we use one color per column
+ ax = df.plot.bar(y="A", color=df["color"])
+ result = [p.get_facecolor() for p in ax.patches]
+ expected = [
+ (1.0, 0.0, 0.0, 1.0),
+ (0.0, 0.0, 1.0, 1.0),
+ (0.0, 0.0, 1.0, 1.0),
+ (1.0, 0.0, 0.0, 1.0),
+ ]
+ assert result == expected
+
+ def test_if_scatterplot_colorbar_affects_xaxis_visibility(self):
+ # addressing issue #10611, to ensure colobar does not
+ # interfere with x-axis label and ticklabels with
+ # ipython inline backend.
+ random_array = np.random.default_rng(2).random((10, 3))
+ df = DataFrame(random_array, columns=["A label", "B label", "C label"])
+
+ ax1 = df.plot.scatter(x="A label", y="B label")
+ ax2 = df.plot.scatter(x="A label", y="B label", c="C label")
+
+ vis1 = [vis.get_visible() for vis in ax1.xaxis.get_minorticklabels()]
+ vis2 = [vis.get_visible() for vis in ax2.xaxis.get_minorticklabels()]
+ assert vis1 == vis2
+
+ vis1 = [vis.get_visible() for vis in ax1.xaxis.get_majorticklabels()]
+ vis2 = [vis.get_visible() for vis in ax2.xaxis.get_majorticklabels()]
+ assert vis1 == vis2
+
+ assert (
+ ax1.xaxis.get_label().get_visible() == ax2.xaxis.get_label().get_visible()
+ )
+
+ def test_if_hexbin_xaxis_label_is_visible(self):
+ # addressing issue #10678, to ensure colobar does not
+ # interfere with x-axis label and ticklabels with
+ # ipython inline backend.
+ random_array = np.random.default_rng(2).random((10, 3))
+ df = DataFrame(random_array, columns=["A label", "B label", "C label"])
+
+ ax = df.plot.hexbin("A label", "B label", gridsize=12)
+ assert all(vis.get_visible() for vis in ax.xaxis.get_minorticklabels())
+ assert all(vis.get_visible() for vis in ax.xaxis.get_majorticklabels())
+ assert ax.xaxis.get_label().get_visible()
+
+ def test_if_scatterplot_colorbars_are_next_to_parent_axes(self):
+ random_array = np.random.default_rng(2).random((10, 3))
+ df = DataFrame(random_array, columns=["A label", "B label", "C label"])
+
+ fig, axes = plt.subplots(1, 2)
+ df.plot.scatter("A label", "B label", c="C label", ax=axes[0])
+ df.plot.scatter("A label", "B label", c="C label", ax=axes[1])
+ plt.tight_layout()
+
+ points = np.array([ax.get_position().get_points() for ax in fig.axes])
+ axes_x_coords = points[:, :, 0]
+ parent_distance = axes_x_coords[1, :] - axes_x_coords[0, :]
+ colorbar_distance = axes_x_coords[3, :] - axes_x_coords[2, :]
+ assert np.isclose(parent_distance, colorbar_distance, atol=1e-7).all()
+
+ @pytest.mark.parametrize("cmap", [None, "Greys"])
+ def test_scatter_with_c_column_name_with_colors(self, cmap):
+ # https://github.com/pandas-dev/pandas/issues/34316
+
+ df = DataFrame(
+ [[5.1, 3.5], [4.9, 3.0], [7.0, 3.2], [6.4, 3.2], [5.9, 3.0]],
+ columns=["length", "width"],
+ )
+ df["species"] = ["r", "r", "g", "g", "b"]
+ if cmap is not None:
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ ax = df.plot.scatter(x=0, y=1, cmap=cmap, c="species")
+ else:
+ ax = df.plot.scatter(x=0, y=1, c="species", cmap=cmap)
+
+ assert len(np.unique(ax.collections[0].get_facecolor(), axis=0)) == 3 # r/g/b
+ assert (
+ np.unique(ax.collections[0].get_facecolor(), axis=0)
+ == np.array(
+ [[0.0, 0.0, 1.0, 1.0], [0.0, 0.5, 0.0, 1.0], [1.0, 0.0, 0.0, 1.0]]
+ ) # r/g/b
+ ).all()
+ assert ax.collections[0].colorbar is None
+
+ def test_scatter_with_c_column_name_without_colors(self):
+ # Given
+ colors = ["NY", "MD", "MA", "CA"]
+ color_count = 4 # 4 unique colors
+
+ # When
+ df = DataFrame(
+ {
+ "dataX": range(100),
+ "dataY": range(100),
+ "color": (colors[i % len(colors)] for i in range(100)),
+ }
+ )
+
+ # Then
+ ax = df.plot.scatter("dataX", "dataY", c="color")
+ assert len(np.unique(ax.collections[0].get_facecolor(), axis=0)) == color_count
+
+ # Given
+ colors = ["r", "g", "not-a-color"]
+ color_count = 3
+ # Also, since not all are mpl-colors, points matching 'r' or 'g'
+ # are not necessarily red or green
+
+ # When
+ df = DataFrame(
+ {
+ "dataX": range(100),
+ "dataY": range(100),
+ "color": (colors[i % len(colors)] for i in range(100)),
+ }
+ )
+
+ # Then
+ ax = df.plot.scatter("dataX", "dataY", c="color")
+ assert len(np.unique(ax.collections[0].get_facecolor(), axis=0)) == color_count
+
+ def test_scatter_colors(self):
+ df = DataFrame({"a": [1, 2, 3], "b": [1, 2, 3], "c": [1, 2, 3]})
+ with pytest.raises(TypeError, match="Specify exactly one of `c` and `color`"):
+ df.plot.scatter(x="a", y="b", c="c", color="green")
+
+ def test_scatter_colors_not_raising_warnings(self):
+ # GH-53908. Do not raise UserWarning: No data for colormapping
+ # provided via 'c'. Parameters 'cmap' will be ignored
+ df = DataFrame({"x": [1, 2, 3], "y": [1, 2, 3]})
+ with tm.assert_produces_warning(None):
+ ax = df.plot.scatter(x="x", y="y", c="b")
+ assert (
+ len(np.unique(ax.collections[0].get_facecolor(), axis=0)) == 1
+ ) # blue
+ assert (
+ np.unique(ax.collections[0].get_facecolor(), axis=0)
+ == np.array([[0.0, 0.0, 1.0, 1.0]])
+ ).all() # blue
+
+ def test_scatter_colors_default(self):
+ df = DataFrame({"a": [1, 2, 3], "b": [1, 2, 3], "c": [1, 2, 3]})
+ default_colors = _unpack_cycler(mpl.pyplot.rcParams)
+
+ ax = df.plot.scatter(x="a", y="b", c="c")
+ tm.assert_numpy_array_equal(
+ ax.collections[0].get_facecolor()[0],
+ np.array(mpl.colors.ColorConverter.to_rgba(default_colors[0])),
+ )
+
+ def test_scatter_colors_white(self):
+ df = DataFrame({"a": [1, 2, 3], "b": [1, 2, 3], "c": [1, 2, 3]})
+ ax = df.plot.scatter(x="a", y="b", color="white")
+ tm.assert_numpy_array_equal(
+ ax.collections[0].get_facecolor()[0],
+ np.array([1, 1, 1, 1], dtype=np.float64),
+ )
+
+ def test_scatter_colorbar_different_cmap(self):
+ # GH 33389
+ df = DataFrame({"x": [1, 2, 3], "y": [1, 3, 2], "c": [1, 2, 3]})
+ df["x2"] = df["x"] + 1
+
+ _, ax = plt.subplots()
+ df.plot("x", "y", c="c", kind="scatter", cmap="cividis", ax=ax)
+ df.plot("x2", "y", c="c", kind="scatter", cmap="magma", ax=ax)
+
+ assert ax.collections[0].cmap.name == "cividis"
+ assert ax.collections[1].cmap.name == "magma"
+
+ def test_line_colors(self):
+ custom_colors = "rgcby"
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+
+ ax = df.plot(color=custom_colors)
+ _check_colors(ax.get_lines(), linecolors=custom_colors)
+
+ plt.close("all")
+
+ ax2 = df.plot(color=custom_colors)
+ lines2 = ax2.get_lines()
+
+ for l1, l2 in zip(ax.get_lines(), lines2, strict=True):
+ assert l1.get_color() == l2.get_color()
+
+ @pytest.mark.parametrize("colormap", ["jet", cm.jet])
+ def test_line_colors_cmap(self, colormap):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ ax = df.plot(colormap=colormap)
+ rgba_colors = [cm.jet(n) for n in np.linspace(0, 1, len(df))]
+ _check_colors(ax.get_lines(), linecolors=rgba_colors)
+
+ def test_line_colors_single_col(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ # make color a list if plotting one column frame
+ # handles cases like df.plot(color='DodgerBlue')
+ ax = df.loc[:, [0]].plot(color="DodgerBlue")
+ _check_colors(ax.lines, linecolors=["DodgerBlue"])
+
+ def test_line_colors_single_color(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ ax = df.plot(color="red")
+ _check_colors(ax.get_lines(), linecolors=["red"] * 5)
+
+ def test_line_colors_hex(self):
+ # GH 10299
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ custom_colors = ["#FF0000", "#0000FF", "#FFFF00", "#000000", "#FFFFFF"]
+ ax = df.plot(color=custom_colors)
+ _check_colors(ax.get_lines(), linecolors=custom_colors)
+
+ def test_dont_modify_colors(self):
+ colors = ["r", "g", "b"]
+ DataFrame(np.random.default_rng(2).random((10, 2))).plot(color=colors)
+ assert len(colors) == 3
+
+ def test_line_colors_and_styles_subplots(self):
+ # GH 9894
+ default_colors = _unpack_cycler(mpl.pyplot.rcParams)
+
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+
+ axes = df.plot(subplots=True)
+ for ax, c in zip(axes, list(default_colors)):
+ _check_colors(ax.get_lines(), linecolors=[c])
+
+ @pytest.mark.parametrize("color", ["k", "green"])
+ def test_line_colors_and_styles_subplots_single_color_str(self, color):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ axes = df.plot(subplots=True, color=color)
+ for ax in axes:
+ _check_colors(ax.get_lines(), linecolors=[color])
+
+ @pytest.mark.parametrize("color", ["rgcby", list("rgcby")])
+ def test_line_colors_and_styles_subplots_custom_colors(self, color):
+ # GH 9894
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ axes = df.plot(color=color, subplots=True)
+ for ax, c in zip(axes, list(color), strict=True):
+ _check_colors(ax.get_lines(), linecolors=[c])
+
+ def test_line_colors_and_styles_subplots_colormap_hex(self):
+ # GH 9894
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ # GH 10299
+ custom_colors = ["#FF0000", "#0000FF", "#FFFF00", "#000000", "#FFFFFF"]
+ axes = df.plot(color=custom_colors, subplots=True)
+ for ax, c in zip(axes, list(custom_colors), strict=True):
+ _check_colors(ax.get_lines(), linecolors=[c])
+
+ @pytest.mark.parametrize("cmap", ["jet", cm.jet])
+ def test_line_colors_and_styles_subplots_colormap_subplot(self, cmap):
+ # GH 9894
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ rgba_colors = [cm.jet(n) for n in np.linspace(0, 1, len(df))]
+ axes = df.plot(colormap=cmap, subplots=True)
+ for ax, c in zip(axes, rgba_colors, strict=True):
+ _check_colors(ax.get_lines(), linecolors=[c])
+
+ def test_line_colors_and_styles_subplots_single_col(self):
+ # GH 9894
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ # make color a list if plotting one column frame
+ # handles cases like df.plot(color='DodgerBlue')
+ axes = df.loc[:, [0]].plot(color="DodgerBlue", subplots=True)
+ _check_colors(axes[0].lines, linecolors=["DodgerBlue"])
+
+ def test_line_colors_and_styles_subplots_single_char(self):
+ # GH 9894
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ # single character style
+ axes = df.plot(style="r", subplots=True)
+ for ax in axes:
+ _check_colors(ax.get_lines(), linecolors=["r"])
+
+ def test_line_colors_and_styles_subplots_list_styles(self):
+ # GH 9894
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ # list of styles
+ styles = list("rgcby")
+ axes = df.plot(style=styles, subplots=True)
+ for ax, c in zip(axes, styles, strict=True):
+ _check_colors(ax.get_lines(), linecolors=[c])
+
+ def test_area_colors(self):
+ custom_colors = "rgcby"
+ df = DataFrame(np.random.default_rng(2).random((5, 5)))
+
+ ax = df.plot.area(color=custom_colors)
+ _check_colors(ax.get_lines(), linecolors=custom_colors)
+ poly = [
+ o
+ for o in ax.get_children()
+ if isinstance(o, mpl.collections.PolyCollection)
+ ]
+ _check_colors(poly, facecolors=custom_colors)
+
+ handles, _ = ax.get_legend_handles_labels()
+ _check_colors(handles, facecolors=custom_colors)
+
+ for h in handles:
+ assert h.get_alpha() is None
+
+ def test_area_colors_poly(self):
+ df = DataFrame(np.random.default_rng(2).random((5, 5)))
+ ax = df.plot.area(colormap="jet")
+ jet_colors = [mpl.cm.jet(n) for n in np.linspace(0, 1, len(df))]
+ _check_colors(ax.get_lines(), linecolors=jet_colors)
+ poly = [
+ o
+ for o in ax.get_children()
+ if isinstance(o, mpl.collections.PolyCollection)
+ ]
+ _check_colors(poly, facecolors=jet_colors)
+
+ handles, _ = ax.get_legend_handles_labels()
+ _check_colors(handles, facecolors=jet_colors)
+ for h in handles:
+ assert h.get_alpha() is None
+
+ def test_area_colors_stacked_false(self):
+ df = DataFrame(np.random.default_rng(2).random((5, 5)))
+ jet_colors = [mpl.cm.jet(n) for n in np.linspace(0, 1, len(df))]
+ # When stacked=False, alpha is set to 0.5
+ ax = df.plot.area(colormap=mpl.cm.jet, stacked=False)
+ _check_colors(ax.get_lines(), linecolors=jet_colors)
+ poly = [
+ o
+ for o in ax.get_children()
+ if isinstance(o, mpl.collections.PolyCollection)
+ ]
+ jet_with_alpha = [(c[0], c[1], c[2], 0.5) for c in jet_colors]
+ _check_colors(poly, facecolors=jet_with_alpha)
+
+ handles, _ = ax.get_legend_handles_labels()
+ linecolors = jet_with_alpha
+ _check_colors(handles[: len(jet_colors)], linecolors=linecolors)
+ for h in handles:
+ assert h.get_alpha() == 0.5
+
+ def test_hist_colors(self):
+ default_colors = _unpack_cycler(mpl.pyplot.rcParams)
+
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ ax = df.plot.hist()
+ _check_colors(ax.patches[::10], facecolors=default_colors[:5])
+
+ def test_hist_colors_single_custom(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ custom_colors = "rgcby"
+ ax = df.plot.hist(color=custom_colors)
+ _check_colors(ax.patches[::10], facecolors=custom_colors)
+
+ @pytest.mark.parametrize("colormap", ["jet", cm.jet])
+ def test_hist_colors_cmap(self, colormap):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ ax = df.plot.hist(colormap=colormap)
+ rgba_colors = [cm.jet(n) for n in np.linspace(0, 1, 5)]
+ _check_colors(ax.patches[::10], facecolors=rgba_colors)
+
+ def test_hist_colors_single_col(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ ax = df.loc[:, [0]].plot.hist(color="DodgerBlue")
+ _check_colors([ax.patches[0]], facecolors=["DodgerBlue"])
+
+ def test_hist_colors_single_color(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ ax = df.plot(kind="hist", color="green")
+ _check_colors(ax.patches[::10], facecolors=["green"] * 5)
+
+ def test_kde_colors(self):
+ pytest.importorskip("scipy")
+ custom_colors = "rgcby"
+ df = DataFrame(np.random.default_rng(2).random((5, 5)))
+
+ ax = df.plot.kde(color=custom_colors)
+ _check_colors(ax.get_lines(), linecolors=custom_colors)
+
+ @pytest.mark.parametrize("colormap", ["jet", cm.jet])
+ def test_kde_colors_cmap(self, colormap):
+ pytest.importorskip("scipy")
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ ax = df.plot.kde(colormap=colormap)
+ rgba_colors = [cm.jet(n) for n in np.linspace(0, 1, len(df))]
+ _check_colors(ax.get_lines(), linecolors=rgba_colors)
+
+ def test_kde_colors_and_styles_subplots(self):
+ pytest.importorskip("scipy")
+ default_colors = _unpack_cycler(mpl.pyplot.rcParams)
+
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+
+ axes = df.plot(kind="kde", subplots=True)
+ for ax, c in zip(axes, list(default_colors)):
+ _check_colors(ax.get_lines(), linecolors=[c])
+
+ @pytest.mark.parametrize("colormap", ["k", "red"])
+ def test_kde_colors_and_styles_subplots_single_col_str(self, colormap):
+ pytest.importorskip("scipy")
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ axes = df.plot(kind="kde", color=colormap, subplots=True)
+ for ax in axes:
+ _check_colors(ax.get_lines(), linecolors=[colormap])
+
+ def test_kde_colors_and_styles_subplots_custom_color(self):
+ pytest.importorskip("scipy")
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ custom_colors = "rgcby"
+ axes = df.plot(kind="kde", color=custom_colors, subplots=True)
+ for ax, c in zip(axes, list(custom_colors), strict=True):
+ _check_colors(ax.get_lines(), linecolors=[c])
+
+ @pytest.mark.parametrize("colormap", ["jet", cm.jet])
+ def test_kde_colors_and_styles_subplots_cmap(self, colormap):
+ pytest.importorskip("scipy")
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ rgba_colors = [cm.jet(n) for n in np.linspace(0, 1, len(df))]
+ axes = df.plot(kind="kde", colormap=colormap, subplots=True)
+ for ax, c in zip(axes, rgba_colors, strict=True):
+ _check_colors(ax.get_lines(), linecolors=[c])
+
+ def test_kde_colors_and_styles_subplots_single_col(self):
+ pytest.importorskip("scipy")
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ # make color a list if plotting one column frame
+ # handles cases like df.plot(color='DodgerBlue')
+ axes = df.loc[:, [0]].plot(kind="kde", color="DodgerBlue", subplots=True)
+ _check_colors(axes[0].lines, linecolors=["DodgerBlue"])
+
+ def test_kde_colors_and_styles_subplots_single_char(self):
+ pytest.importorskip("scipy")
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ # list of styles
+ # single character style
+ axes = df.plot(kind="kde", style="r", subplots=True)
+ for ax in axes:
+ _check_colors(ax.get_lines(), linecolors=["r"])
+
+ def test_kde_colors_and_styles_subplots_list(self):
+ pytest.importorskip("scipy")
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ # list of styles
+ styles = list("rgcby")
+ axes = df.plot(kind="kde", style=styles, subplots=True)
+ for ax, c in zip(axes, styles, strict=True):
+ _check_colors(ax.get_lines(), linecolors=[c])
+
+ def test_boxplot_colors(self):
+ default_colors = _unpack_cycler(mpl.pyplot.rcParams)
+
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ bp = df.plot.box(return_type="dict")
+ _check_colors_box(
+ bp,
+ default_colors[0],
+ default_colors[0],
+ default_colors[2],
+ default_colors[0],
+ )
+
+ def test_boxplot_colors_dict_colors(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ dict_colors = {
+ "boxes": "#572923",
+ "whiskers": "#982042",
+ "medians": "#804823",
+ "caps": "#123456",
+ }
+ bp = df.plot.box(color=dict_colors, sym="r+", return_type="dict")
+ _check_colors_box(
+ bp,
+ dict_colors["boxes"],
+ dict_colors["whiskers"],
+ dict_colors["medians"],
+ dict_colors["caps"],
+ "r",
+ )
+
+ def test_boxplot_colors_default_color(self):
+ default_colors = _unpack_cycler(mpl.pyplot.rcParams)
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ # partial colors
+ dict_colors = {"whiskers": "c", "medians": "m"}
+ bp = df.plot.box(color=dict_colors, return_type="dict")
+ _check_colors_box(bp, default_colors[0], "c", "m", default_colors[0])
+
+ @pytest.mark.parametrize("colormap", ["jet", cm.jet])
+ def test_boxplot_colors_cmap(self, colormap):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ bp = df.plot.box(colormap=colormap, return_type="dict")
+ jet_colors = [cm.jet(n) for n in np.linspace(0, 1, 3)]
+ _check_colors_box(
+ bp, jet_colors[0], jet_colors[0], jet_colors[2], jet_colors[0]
+ )
+
+ def test_boxplot_colors_single(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ # string color is applied to all artists except fliers
+ bp = df.plot.box(color="DodgerBlue", return_type="dict")
+ _check_colors_box(bp, "DodgerBlue", "DodgerBlue", "DodgerBlue", "DodgerBlue")
+
+ def test_boxplot_colors_tuple(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ # tuple is also applied to all artists except fliers
+ bp = df.plot.box(color=(0, 1, 0), sym="#123456", return_type="dict")
+ _check_colors_box(bp, (0, 1, 0), (0, 1, 0), (0, 1, 0), (0, 1, 0), "#123456")
+
+ def test_boxplot_colors_invalid(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ msg = re.escape(
+ "color dict contains invalid key 'xxxx'. The key must be either "
+ "['boxes', 'whiskers', 'medians', 'caps']"
+ )
+ with pytest.raises(ValueError, match=msg):
+ # Color contains invalid key results in ValueError
+ df.plot.box(color={"boxes": "red", "xxxx": "blue"})
+
+ def test_default_color_cycle(self):
+ import cycler
+
+ colors = list("rgbk")
+ plt.rcParams["axes.prop_cycle"] = cycler.cycler("color", colors)
+
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 3)))
+ ax = df.plot()
+
+ expected = _unpack_cycler(plt.rcParams)[:3]
+ _check_colors(ax.get_lines(), linecolors=expected)
+
+ def test_no_color_bar(self):
+ df = DataFrame(
+ {
+ "A": np.random.default_rng(2).uniform(size=20),
+ "B": np.random.default_rng(2).uniform(size=20),
+ "C": np.arange(20) + np.random.default_rng(2).uniform(size=20),
+ }
+ )
+ ax = df.plot.hexbin(x="A", y="B", colorbar=None)
+ assert ax.collections[0].colorbar is None
+
+ def test_mixing_cmap_and_colormap_raises(self):
+ df = DataFrame(
+ {
+ "A": np.random.default_rng(2).uniform(size=20),
+ "B": np.random.default_rng(2).uniform(size=20),
+ "C": np.arange(20) + np.random.default_rng(2).uniform(size=20),
+ }
+ )
+ msg = "Only specify one of `cmap` and `colormap`"
+ with pytest.raises(TypeError, match=msg):
+ df.plot.hexbin(x="A", y="B", cmap="YlGn", colormap="BuGn")
+
+ def test_passed_bar_colors(self):
+ color_tuples = [(0.9, 0, 0, 1), (0, 0.9, 0, 1), (0, 0, 0.9, 1)]
+ colormap = mpl.colors.ListedColormap(color_tuples)
+ barplot = DataFrame([[1, 2, 3]]).plot(kind="bar", cmap=colormap)
+ assert color_tuples == [c.get_facecolor() for c in barplot.patches]
+
+ def test_rcParams_bar_colors(self):
+ color_tuples = [(0.9, 0, 0, 1), (0, 0.9, 0, 1), (0, 0, 0.9, 1)]
+ with mpl.rc_context(rc={"axes.prop_cycle": mpl.cycler("color", color_tuples)}):
+ barplot = DataFrame([[1, 2, 3]]).plot(kind="bar")
+ assert color_tuples == [c.get_facecolor() for c in barplot.patches]
+
+ def test_colors_of_columns_with_same_name(self):
+ # ISSUE 11136 -> https://github.com/pandas-dev/pandas/issues/11136
+ # Creating a DataFrame with duplicate column labels and testing colors of them.
+ df = DataFrame({"b": [0, 1, 0], "a": [1, 2, 3]})
+ df1 = DataFrame({"a": [2, 4, 6]})
+ df_concat = pd.concat([df, df1], axis=1)
+ result = df_concat.plot()
+ legend = result.get_legend()
+ handles = legend.legend_handles
+ for legend, line in zip(handles, result.lines, strict=True):
+ assert legend.get_color() == line.get_color()
+
+ def test_invalid_colormap(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((3, 2)), columns=["A", "B"]
+ )
+ msg = "(is not a valid value)|(is not a known colormap)"
+ with pytest.raises((ValueError, KeyError), match=msg):
+ df.plot(colormap="invalid_colormap")
+
+ def test_dataframe_none_color(self):
+ # GH51953
+ df = DataFrame([[1, 2, 3]])
+ ax = df.plot(color=None)
+ expected = _unpack_cycler(mpl.pyplot.rcParams)[:3]
+ _check_colors(ax.get_lines(), linecolors=expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame_groupby.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame_groupby.py
new file mode 100644
index 0000000000000000000000000000000000000000..89589f5adbbc08f0bdd75d1752d5b732f935fb05
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame_groupby.py
@@ -0,0 +1,72 @@
+"""Test cases for DataFrame.plot"""
+
+import pytest
+
+from pandas import DataFrame
+from pandas.tests.plotting.common import _check_visible
+
+pytest.importorskip("matplotlib")
+
+
+class TestDataFramePlotsGroupby:
+ def _assert_ytickslabels_visibility(self, axes, expected):
+ for ax, exp in zip(axes, expected, strict=True):
+ _check_visible(ax.get_yticklabels(), visible=exp)
+
+ def _assert_xtickslabels_visibility(self, axes, expected):
+ for ax, exp in zip(axes, expected, strict=True):
+ _check_visible(ax.get_xticklabels(), visible=exp)
+
+ @pytest.mark.parametrize(
+ "kwargs, expected",
+ [
+ # behavior without keyword
+ ({}, [True, False, True, False]),
+ # set sharey=True should be identical
+ ({"sharey": True}, [True, False, True, False]),
+ # sharey=False, all yticklabels should be visible
+ ({"sharey": False}, [True, True, True, True]),
+ ],
+ )
+ def test_groupby_boxplot_sharey(self, kwargs, expected):
+ # https://github.com/pandas-dev/pandas/issues/20968
+ # sharey can now be switched check whether the right
+ # pair of axes is turned on or off
+ df = DataFrame(
+ {
+ "a": [-1.43, -0.15, -3.70, -1.43, -0.14],
+ "b": [0.56, 0.84, 0.29, 0.56, 0.85],
+ "c": [0, 1, 2, 3, 1],
+ },
+ index=[0, 1, 2, 3, 4],
+ )
+ axes = df.groupby("c").boxplot(**kwargs)
+ self._assert_ytickslabels_visibility(axes, expected)
+
+ @pytest.mark.parametrize(
+ "kwargs, expected",
+ [
+ # behavior without keyword
+ ({}, [True, True, True, True]),
+ # set sharex=False should be identical
+ ({"sharex": False}, [True, True, True, True]),
+ # sharex=True, xticklabels should be visible
+ # only for bottom plots
+ ({"sharex": True}, [False, False, True, True]),
+ ],
+ )
+ def test_groupby_boxplot_sharex(self, kwargs, expected):
+ # https://github.com/pandas-dev/pandas/issues/20968
+ # sharex can now be switched check whether the right
+ # pair of axes is turned on or off
+
+ df = DataFrame(
+ {
+ "a": [-1.43, -0.15, -3.70, -1.43, -0.14],
+ "b": [0.56, 0.84, 0.29, 0.56, 0.85],
+ "c": [0, 1, 2, 3, 1],
+ },
+ index=[0, 1, 2, 3, 4],
+ )
+ axes = df.groupby("c").boxplot(**kwargs)
+ self._assert_xtickslabels_visibility(axes, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame_legend.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame_legend.py
new file mode 100644
index 0000000000000000000000000000000000000000..755293e0bf6d753a1061c0a86ca18495e054e04e
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame_legend.py
@@ -0,0 +1,262 @@
+import numpy as np
+import pytest
+
+import pandas.util._test_decorators as td
+
+from pandas import (
+ DataFrame,
+ date_range,
+)
+from pandas.tests.plotting.common import (
+ _check_legend_labels,
+ _check_legend_marker,
+ _check_text_labels,
+)
+
+mpl = pytest.importorskip("matplotlib")
+
+
+class TestFrameLegend:
+ @pytest.mark.xfail(
+ reason=(
+ "Open bug in matplotlib "
+ "https://github.com/matplotlib/matplotlib/issues/11357"
+ )
+ )
+ def test_mixed_yerr(self):
+ # https://github.com/pandas-dev/pandas/issues/39522
+ df = DataFrame([{"x": 1, "a": 1, "b": 1}, {"x": 2, "a": 2, "b": 3}])
+
+ ax = df.plot("x", "a", c="orange", yerr=0.1, label="orange")
+ df.plot("x", "b", c="blue", yerr=None, ax=ax, label="blue")
+
+ legend = ax.get_legend()
+ result_handles = legend.legend_handles
+
+ assert isinstance(result_handles[0], mpl.collections.LineCollection)
+ assert isinstance(result_handles[1], mpl.lines.Line2D)
+
+ def test_legend_false(self):
+ # https://github.com/pandas-dev/pandas/issues/40044
+ df = DataFrame({"a": [1, 1], "b": [2, 3]})
+ df2 = DataFrame({"d": [2.5, 2.5]})
+
+ ax = df.plot(legend=True, color={"a": "blue", "b": "green"}, secondary_y="b")
+ df2.plot(legend=True, color={"d": "red"}, ax=ax)
+ legend = ax.get_legend()
+ handles = legend.legend_handles
+ result = [handle.get_color() for handle in handles]
+ expected = ["blue", "green", "red"]
+ assert result == expected
+
+ @pytest.mark.parametrize("kind", ["line", "bar", "barh", "kde", "area", "hist"])
+ def test_df_legend_labels(self, kind):
+ pytest.importorskip("scipy")
+ df = DataFrame(np.random.default_rng(2).random((3, 3)), columns=["a", "b", "c"])
+ df2 = DataFrame(
+ np.random.default_rng(2).random((3, 3)), columns=["d", "e", "f"]
+ )
+ df3 = DataFrame(
+ np.random.default_rng(2).random((3, 3)), columns=["g", "h", "i"]
+ )
+ df4 = DataFrame(
+ np.random.default_rng(2).random((3, 3)), columns=["j", "k", "l"]
+ )
+
+ ax = df.plot(kind=kind, legend=True)
+ _check_legend_labels(ax, labels=df.columns)
+
+ ax = df2.plot(kind=kind, legend=False, ax=ax)
+ _check_legend_labels(ax, labels=df.columns)
+
+ ax = df3.plot(kind=kind, legend=True, ax=ax)
+ _check_legend_labels(ax, labels=df.columns.union(df3.columns))
+
+ ax = df4.plot(kind=kind, legend="reverse", ax=ax)
+ expected = list(df.columns.union(df3.columns)) + list(reversed(df4.columns))
+ _check_legend_labels(ax, labels=expected)
+
+ def test_df_legend_labels_secondary_y(self):
+ pytest.importorskip("scipy")
+ df = DataFrame(np.random.default_rng(2).random((3, 3)), columns=["a", "b", "c"])
+ df2 = DataFrame(
+ np.random.default_rng(2).random((3, 3)), columns=["d", "e", "f"]
+ )
+ df3 = DataFrame(
+ np.random.default_rng(2).random((3, 3)), columns=["g", "h", "i"]
+ )
+ # Secondary Y
+ ax = df.plot(legend=True, secondary_y="b")
+ _check_legend_labels(ax, labels=["a", "b (right)", "c"])
+ ax = df2.plot(legend=False, ax=ax)
+ _check_legend_labels(ax, labels=["a", "b (right)", "c"])
+ ax = df3.plot(kind="bar", legend=True, secondary_y="h", ax=ax)
+ _check_legend_labels(ax, labels=["a", "b (right)", "c", "g", "h (right)", "i"])
+
+ def test_df_legend_labels_time_series(self):
+ # Time Series
+ pytest.importorskip("scipy")
+ ind = date_range("1/1/2014", periods=3)
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((3, 3)),
+ columns=["a", "b", "c"],
+ index=ind,
+ )
+ df2 = DataFrame(
+ np.random.default_rng(2).standard_normal((3, 3)),
+ columns=["d", "e", "f"],
+ index=ind,
+ )
+ df3 = DataFrame(
+ np.random.default_rng(2).standard_normal((3, 3)),
+ columns=["g", "h", "i"],
+ index=ind,
+ )
+ ax = df.plot(legend=True, secondary_y="b")
+ _check_legend_labels(ax, labels=["a", "b (right)", "c"])
+ ax = df2.plot(legend=False, ax=ax)
+ _check_legend_labels(ax, labels=["a", "b (right)", "c"])
+ ax = df3.plot(legend=True, ax=ax)
+ _check_legend_labels(ax, labels=["a", "b (right)", "c", "g", "h", "i"])
+
+ def test_df_legend_labels_time_series_scatter(self):
+ # Time Series
+ pytest.importorskip("scipy")
+ ind = date_range("1/1/2014", periods=3)
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((3, 3)),
+ columns=["a", "b", "c"],
+ index=ind,
+ )
+ df2 = DataFrame(
+ np.random.default_rng(2).standard_normal((3, 3)),
+ columns=["d", "e", "f"],
+ index=ind,
+ )
+ df3 = DataFrame(
+ np.random.default_rng(2).standard_normal((3, 3)),
+ columns=["g", "h", "i"],
+ index=ind,
+ )
+ # scatter
+ ax = df.plot.scatter(x="a", y="b", label="data1")
+ _check_legend_labels(ax, labels=["data1"])
+ ax = df2.plot.scatter(x="d", y="e", legend=False, label="data2", ax=ax)
+ _check_legend_labels(ax, labels=["data1"])
+ ax = df3.plot.scatter(x="g", y="h", label="data3", ax=ax)
+ _check_legend_labels(ax, labels=["data1", "data3"])
+
+ def test_df_legend_labels_time_series_no_mutate(self):
+ pytest.importorskip("scipy")
+ ind = date_range("1/1/2014", periods=3)
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((3, 3)),
+ columns=["a", "b", "c"],
+ index=ind,
+ )
+ # ensure label args pass through and
+ # index name does not mutate
+ # column names don't mutate
+ df5 = df.set_index("a")
+ ax = df5.plot(y="b")
+ _check_legend_labels(ax, labels=["b"])
+ ax = df5.plot(y="b", label="LABEL_b")
+ _check_legend_labels(ax, labels=["LABEL_b"])
+ _check_text_labels(ax.xaxis.get_label(), "a")
+ ax = df5.plot(y="c", label="LABEL_c", ax=ax)
+ _check_legend_labels(ax, labels=["LABEL_b", "LABEL_c"])
+ assert df5.columns.tolist() == ["b", "c"]
+
+ def test_missing_marker_multi_plots_on_same_ax(self):
+ # GH 18222
+ df = DataFrame(data=[[1, 1, 1, 1], [2, 2, 4, 8]], columns=["x", "r", "g", "b"])
+ _, ax = mpl.pyplot.subplots(nrows=1, ncols=3)
+ # Left plot
+ df.plot(x="x", y="r", linewidth=0, marker="o", color="r", ax=ax[0])
+ df.plot(x="x", y="g", linewidth=1, marker="x", color="g", ax=ax[0])
+ df.plot(x="x", y="b", linewidth=1, marker="o", color="b", ax=ax[0])
+ _check_legend_labels(ax[0], labels=["r", "g", "b"])
+ _check_legend_marker(ax[0], expected_markers=["o", "x", "o"])
+ # Center plot
+ df.plot(x="x", y="b", linewidth=1, marker="o", color="b", ax=ax[1])
+ df.plot(x="x", y="r", linewidth=0, marker="o", color="r", ax=ax[1])
+ df.plot(x="x", y="g", linewidth=1, marker="x", color="g", ax=ax[1])
+ _check_legend_labels(ax[1], labels=["b", "r", "g"])
+ _check_legend_marker(ax[1], expected_markers=["o", "o", "x"])
+ # Right plot
+ df.plot(x="x", y="g", linewidth=1, marker="x", color="g", ax=ax[2])
+ df.plot(x="x", y="b", linewidth=1, marker="o", color="b", ax=ax[2])
+ df.plot(x="x", y="r", linewidth=0, marker="o", color="r", ax=ax[2])
+ _check_legend_labels(ax[2], labels=["g", "b", "r"])
+ _check_legend_marker(ax[2], expected_markers=["x", "o", "o"])
+
+ def test_legend_name(self):
+ multi = DataFrame(
+ np.random.default_rng(2).standard_normal((4, 4)),
+ columns=[np.array(["a", "a", "b", "b"]), np.array(["x", "y", "x", "y"])],
+ )
+ multi.columns.names = ["group", "individual"]
+
+ ax = multi.plot()
+ leg_title = ax.legend_.get_title()
+ _check_text_labels(leg_title, "group,individual")
+
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ ax = df.plot(legend=True, ax=ax)
+ leg_title = ax.legend_.get_title()
+ _check_text_labels(leg_title, "group,individual")
+
+ df.columns.name = "new"
+ ax = df.plot(legend=False, ax=ax)
+ leg_title = ax.legend_.get_title()
+ _check_text_labels(leg_title, "group,individual")
+
+ ax = df.plot(legend=True, ax=ax)
+ leg_title = ax.legend_.get_title()
+ _check_text_labels(leg_title, "new")
+
+ @pytest.mark.parametrize(
+ "kind",
+ [
+ "line",
+ "bar",
+ "barh",
+ pytest.param("kde", marks=td.skip_if_no("scipy")),
+ "area",
+ "hist",
+ ],
+ )
+ def test_no_legend(self, kind):
+ df = DataFrame(np.random.default_rng(2).random((3, 3)), columns=["a", "b", "c"])
+ ax = df.plot(kind=kind, legend=False)
+ _check_legend_labels(ax, visible=False)
+
+ def test_missing_markers_legend(self):
+ # 14958
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((8, 3)), columns=["A", "B", "C"]
+ )
+ ax = df.plot(y=["A"], marker="x", linestyle="solid")
+ df.plot(y=["B"], marker="o", linestyle="dotted", ax=ax)
+ df.plot(y=["C"], marker="<", linestyle="dotted", ax=ax)
+
+ _check_legend_labels(ax, labels=["A", "B", "C"])
+ _check_legend_marker(ax, expected_markers=["x", "o", "<"])
+
+ def test_missing_markers_legend_using_style(self):
+ # 14563
+ df = DataFrame(
+ {
+ "A": [1, 2, 3, 4, 5, 6],
+ "B": [2, 4, 1, 3, 2, 4],
+ "C": [3, 3, 2, 6, 4, 2],
+ "X": [1, 2, 3, 4, 5, 6],
+ }
+ )
+
+ _, ax = mpl.pyplot.subplots()
+ for kind in "ABC":
+ df.plot("X", kind, label=kind, ax=ax, style=".")
+
+ _check_legend_labels(ax, labels=["A", "B", "C"])
+ _check_legend_marker(ax, expected_markers=[".", ".", "."])
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame_subplots.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame_subplots.py
new file mode 100644
index 0000000000000000000000000000000000000000..453bbc9d7846dabd8c65dcf2f057fa56a9f0b7c1
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_frame_subplots.py
@@ -0,0 +1,764 @@
+"""Test cases for DataFrame.plot"""
+
+import string
+
+import numpy as np
+import pytest
+
+from pandas.compat import is_platform_linux
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Series,
+ date_range,
+)
+import pandas._testing as tm
+from pandas.tests.plotting.common import (
+ _check_axes_shape,
+ _check_box_return_type,
+ _check_legend_labels,
+ _check_ticks_props,
+ _check_visible,
+ _flatten_visible,
+)
+from pandas.util.version import Version
+
+from pandas.io.formats.printing import pprint_thing
+
+mpl = pytest.importorskip("matplotlib")
+plt = pytest.importorskip("matplotlib.pyplot")
+
+
+class TestDataFramePlotsSubplots:
+ @pytest.mark.slow
+ @pytest.mark.parametrize("kind", ["bar", "barh", "line", "area"])
+ def test_subplots(self, kind):
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=list(string.ascii_letters[:10]),
+ )
+
+ axes = df.plot(kind=kind, subplots=True, sharex=True, legend=True)
+ _check_axes_shape(axes, axes_num=3, layout=(3, 1))
+ assert axes.shape == (3,)
+
+ for ax, column in zip(axes, df.columns, strict=True):
+ _check_legend_labels(ax, labels=[pprint_thing(column)])
+
+ for ax in axes[:-2]:
+ _check_visible(ax.xaxis) # xaxis must be visible for grid
+ _check_visible(ax.get_xticklabels(), visible=False)
+ if kind != "bar":
+ # change https://github.com/pandas-dev/pandas/issues/26714
+ _check_visible(ax.get_xticklabels(minor=True), visible=False)
+ _check_visible(ax.xaxis.get_label(), visible=False)
+ _check_visible(ax.get_yticklabels())
+
+ _check_visible(axes[-1].xaxis)
+ _check_visible(axes[-1].get_xticklabels())
+ _check_visible(axes[-1].get_xticklabels(minor=True))
+ _check_visible(axes[-1].xaxis.get_label())
+ _check_visible(axes[-1].get_yticklabels())
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("kind", ["bar", "barh", "line", "area"])
+ def test_subplots_no_share_x(self, kind):
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=list(string.ascii_letters[:10]),
+ )
+ axes = df.plot(kind=kind, subplots=True, sharex=False)
+ for ax in axes:
+ _check_visible(ax.xaxis)
+ _check_visible(ax.get_xticklabels())
+ _check_visible(ax.get_xticklabels(minor=True))
+ _check_visible(ax.xaxis.get_label())
+ _check_visible(ax.get_yticklabels())
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("kind", ["bar", "barh", "line", "area"])
+ def test_subplots_no_legend(self, kind):
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=list(string.ascii_letters[:10]),
+ )
+ axes = df.plot(kind=kind, subplots=True, legend=False)
+ for ax in axes:
+ assert ax.get_legend() is None
+
+ @pytest.mark.parametrize("kind", ["line", "area"])
+ def test_subplots_timeseries(self, kind):
+ idx = date_range(start="2014-07-01", freq="ME", periods=10)
+ df = DataFrame(np.random.default_rng(2).random((10, 3)), index=idx)
+
+ axes = df.plot(kind=kind, subplots=True, sharex=True)
+ _check_axes_shape(axes, axes_num=3, layout=(3, 1))
+
+ for ax in axes[:-2]:
+ # GH 7801
+ _check_visible(ax.xaxis) # xaxis must be visible for grid
+ _check_visible(ax.get_xticklabels(), visible=False)
+ _check_visible(ax.get_xticklabels(minor=True), visible=False)
+ _check_visible(ax.xaxis.get_label(), visible=False)
+ _check_visible(ax.get_yticklabels())
+
+ _check_visible(axes[-1].xaxis)
+ _check_visible(axes[-1].get_xticklabels())
+ _check_visible(axes[-1].get_xticklabels(minor=True))
+ _check_visible(axes[-1].xaxis.get_label())
+ _check_visible(axes[-1].get_yticklabels())
+ _check_ticks_props(axes, xrot=0)
+
+ @pytest.mark.parametrize("kind", ["line", "area"])
+ def test_subplots_timeseries_rot(self, kind):
+ idx = date_range(start="2014-07-01", freq="ME", periods=10)
+ df = DataFrame(np.random.default_rng(2).random((10, 3)), index=idx)
+ axes = df.plot(kind=kind, subplots=True, sharex=False, rot=45, fontsize=7)
+ for ax in axes:
+ _check_visible(ax.xaxis)
+ _check_visible(ax.get_xticklabels())
+ _check_visible(ax.get_xticklabels(minor=True))
+ _check_visible(ax.xaxis.get_label())
+ _check_visible(ax.get_yticklabels())
+ _check_ticks_props(ax, xlabelsize=7, xrot=45, ylabelsize=7)
+
+ @pytest.mark.parametrize(
+ "col", ["numeric", "timedelta", "datetime_no_tz", "datetime_all_tz"]
+ )
+ def test_subplots_timeseries_y_axis(self, col):
+ # GH16953
+ data = {
+ "numeric": np.array([1, 2, 5]),
+ "timedelta": [
+ pd.Timedelta(-10, unit="s"),
+ pd.Timedelta(10, unit="m"),
+ pd.Timedelta(10, unit="h"),
+ ],
+ "datetime_no_tz": [
+ pd.to_datetime("2017-08-01 00:00:00"),
+ pd.to_datetime("2017-08-01 02:00:00"),
+ pd.to_datetime("2017-08-02 00:00:00"),
+ ],
+ "datetime_all_tz": [
+ pd.to_datetime("2017-08-01 00:00:00", utc=True),
+ pd.to_datetime("2017-08-01 02:00:00", utc=True),
+ pd.to_datetime("2017-08-02 00:00:00", utc=True),
+ ],
+ "text": ["This", "should", "fail"],
+ }
+ testdata = DataFrame(data)
+
+ ax = testdata.plot(y=col)
+ result = ax.get_lines()[0].get_data()[1]
+ expected = testdata[col].values
+ assert (result == expected).all()
+
+ def test_subplots_timeseries_y_text_error(self):
+ # GH16953
+ data = {
+ "numeric": np.array([1, 2, 5]),
+ "text": ["This", "should", "fail"],
+ }
+ testdata = DataFrame(data)
+ msg = "no numeric data to plot"
+ with pytest.raises(TypeError, match=msg):
+ testdata.plot(y="text")
+
+ @pytest.mark.xfail(reason="not support for period, categorical, datetime_mixed_tz")
+ def test_subplots_timeseries_y_axis_not_supported(self):
+ """
+ This test will fail for:
+ period:
+ since period isn't yet implemented in ``select_dtypes``
+ and because it will need a custom value converter +
+ tick formatter (as was done for x-axis plots)
+
+ categorical:
+ because it will need a custom value converter +
+ tick formatter (also doesn't work for x-axis, as of now)
+
+ datetime_mixed_tz:
+ because of the way how pandas handles ``Series`` of
+ ``datetime`` objects with different timezone,
+ generally converting ``datetime`` objects in a tz-aware
+ form could help with this problem
+ """
+ data = {
+ "numeric": np.array([1, 2, 5]),
+ "period": [
+ pd.Period("2017-08-01 00:00:00", freq="h"),
+ pd.Period("2017-08-01 02:00", freq="h"),
+ pd.Period("2017-08-02 00:00:00", freq="h"),
+ ],
+ "categorical": pd.Categorical(
+ ["c", "b", "a"], categories=["a", "b", "c"], ordered=False
+ ),
+ "datetime_mixed_tz": [
+ pd.to_datetime("2017-08-01 00:00:00", utc=True),
+ pd.to_datetime("2017-08-01 02:00:00"),
+ pd.to_datetime("2017-08-02 00:00:00"),
+ ],
+ }
+ testdata = DataFrame(data)
+ ax_period = testdata.plot(x="numeric", y="period")
+ assert (
+ ax_period.get_lines()[0].get_data()[1] == testdata["period"].values
+ ).all()
+ ax_categorical = testdata.plot(x="numeric", y="categorical")
+ assert (
+ ax_categorical.get_lines()[0].get_data()[1]
+ == testdata["categorical"].values
+ ).all()
+ ax_datetime_mixed_tz = testdata.plot(x="numeric", y="datetime_mixed_tz")
+ assert (
+ ax_datetime_mixed_tz.get_lines()[0].get_data()[1]
+ == testdata["datetime_mixed_tz"].values
+ ).all()
+
+ @pytest.mark.parametrize(
+ "layout, exp_layout",
+ [
+ [(2, 2), (2, 2)],
+ [(-1, 2), (2, 2)],
+ [(2, -1), (2, 2)],
+ [(1, 4), (1, 4)],
+ [(-1, 4), (1, 4)],
+ [(4, -1), (4, 1)],
+ ],
+ )
+ def test_subplots_layout_multi_column(self, layout, exp_layout):
+ # GH 6667
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=list(string.ascii_letters[:10]),
+ )
+
+ axes = df.plot(subplots=True, layout=layout)
+ _check_axes_shape(axes, axes_num=3, layout=exp_layout)
+ assert axes.shape == exp_layout
+
+ def test_subplots_layout_multi_column_error(self):
+ # GH 6667
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=list(string.ascii_letters[:10]),
+ )
+ msg = "Layout of 1x1 must be larger than required size 3"
+
+ with pytest.raises(ValueError, match=msg):
+ df.plot(subplots=True, layout=(1, 1))
+
+ msg = "At least one dimension of layout must be positive"
+ with pytest.raises(ValueError, match=msg):
+ df.plot(subplots=True, layout=(-1, -1))
+
+ @pytest.mark.parametrize(
+ "kwargs, expected_axes_num, expected_layout, expected_shape",
+ [
+ ({}, 1, (1, 1), (1,)),
+ ({"layout": (3, 3)}, 1, (3, 3), (3, 3)),
+ ],
+ )
+ def test_subplots_layout_single_column(
+ self, kwargs, expected_axes_num, expected_layout, expected_shape
+ ):
+ # GH 6667
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 1)),
+ index=list(string.ascii_letters[:10]),
+ )
+ axes = df.plot(subplots=True, **kwargs)
+ _check_axes_shape(
+ axes,
+ axes_num=expected_axes_num,
+ layout=expected_layout,
+ )
+ assert axes.shape == expected_shape
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("idx", [range(5), date_range("1/1/2000", periods=5)])
+ def test_subplots_warnings(self, idx):
+ # GH 9464
+ with tm.assert_produces_warning(None):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 4)), index=idx)
+ df.plot(subplots=True, layout=(3, 2))
+
+ def test_subplots_multiple_axes(self):
+ # GH 5353, 6970, GH 7069
+ fig, axes = mpl.pyplot.subplots(2, 3)
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=list(string.ascii_letters[:10]),
+ )
+
+ returned = df.plot(subplots=True, ax=axes[0], sharex=False, sharey=False)
+ _check_axes_shape(returned, axes_num=3, layout=(1, 3))
+ assert returned.shape == (3,)
+ assert returned[0].figure is fig
+ # draw on second row
+ returned = df.plot(subplots=True, ax=axes[1], sharex=False, sharey=False)
+ _check_axes_shape(returned, axes_num=3, layout=(1, 3))
+ assert returned.shape == (3,)
+ assert returned[0].figure is fig
+ _check_axes_shape(axes, axes_num=6, layout=(2, 3))
+
+ def test_subplots_multiple_axes_error(self):
+ # GH 5353, 6970, GH 7069
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=list(string.ascii_letters[:10]),
+ )
+ msg = "The number of passed axes must be 3, the same as the output plot"
+ _, axes = mpl.pyplot.subplots(2, 3)
+
+ with pytest.raises(ValueError, match=msg):
+ # pass different number of axes from required
+ df.plot(subplots=True, ax=axes)
+
+ @pytest.mark.parametrize(
+ "layout, exp_layout",
+ [
+ [(2, 1), (2, 2)],
+ [(2, -1), (2, 2)],
+ [(-1, 2), (2, 2)],
+ ],
+ )
+ def test_subplots_multiple_axes_2_dim(self, layout, exp_layout):
+ # GH 5353, 6970, GH 7069
+ # pass 2-dim axes and invalid layout
+ # invalid layout should not affect to input and return value
+ # (show warning is tested in
+ # TestDataFrameGroupByPlots.test_grouped_box_multiple_axes
+ _, axes = mpl.pyplot.subplots(2, 2)
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 4)),
+ index=list(string.ascii_letters[:10]),
+ )
+ with tm.assert_produces_warning(UserWarning, match="layout keyword is ignored"):
+ returned = df.plot(
+ subplots=True, ax=axes, layout=layout, sharex=False, sharey=False
+ )
+ _check_axes_shape(returned, axes_num=4, layout=exp_layout)
+ assert returned.shape == (4,)
+
+ def test_subplots_multiple_axes_single_col(self):
+ # GH 5353, 6970, GH 7069
+ # single column
+ _, axes = mpl.pyplot.subplots(1, 1)
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 1)),
+ index=list(string.ascii_letters[:10]),
+ )
+
+ axes = df.plot(subplots=True, ax=[axes], sharex=False, sharey=False)
+ _check_axes_shape(axes, axes_num=1, layout=(1, 1))
+ assert axes.shape == (1,)
+
+ def test_subplots_ts_share_axes(self):
+ # GH 3964
+ _, axes = mpl.pyplot.subplots(3, 3, sharex=True, sharey=True)
+ mpl.pyplot.subplots_adjust(left=0.05, right=0.95, hspace=0.3, wspace=0.3)
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 9)),
+ index=date_range(start="2014-07-01", freq="ME", periods=10),
+ )
+ for i, ax in enumerate(axes.ravel()):
+ df[i].plot(ax=ax, fontsize=5)
+
+ # Rows other than bottom should not be visible
+ for ax in axes[0:-1].ravel():
+ _check_visible(ax.get_xticklabels(), visible=False)
+
+ # Bottom row should be visible
+ for ax in axes[-1].ravel():
+ _check_visible(ax.get_xticklabels(), visible=True)
+
+ # First column should be visible
+ for ax in axes[[0, 1, 2], [0]].ravel():
+ _check_visible(ax.get_yticklabels(), visible=True)
+
+ # Other columns should not be visible
+ for ax in axes[[0, 1, 2], [1]].ravel():
+ _check_visible(ax.get_yticklabels(), visible=False)
+ for ax in axes[[0, 1, 2], [2]].ravel():
+ _check_visible(ax.get_yticklabels(), visible=False)
+
+ def test_subplots_sharex_axes_existing_axes(self):
+ # GH 9158
+ d = {"A": [1.0, 2.0, 3.0, 4.0], "B": [4.0, 3.0, 2.0, 1.0], "C": [5, 1, 3, 4]}
+ df = DataFrame(d, index=date_range("2014 10 11", "2014 10 14"))
+
+ axes = df[["A", "B"]].plot(subplots=True)
+ df["C"].plot(ax=axes[0], secondary_y=True)
+
+ _check_visible(axes[0].get_xticklabels(), visible=False)
+ _check_visible(axes[1].get_xticklabels(), visible=True)
+ for ax in axes.ravel():
+ _check_visible(ax.get_yticklabels(), visible=True)
+
+ def test_subplots_dup_columns(self):
+ # GH 10962
+ df = DataFrame(np.random.default_rng(2).random((5, 5)), columns=list("aaaaa"))
+ axes = df.plot(subplots=True)
+ for ax in axes:
+ _check_legend_labels(ax, labels=["a"])
+ assert len(ax.lines) == 1
+
+ def test_subplots_dup_columns_secondary_y(self):
+ # GH 10962
+ df = DataFrame(np.random.default_rng(2).random((5, 5)), columns=list("aaaaa"))
+ axes = df.plot(subplots=True, secondary_y="a")
+ for ax in axes:
+ # (right) is only attached when subplots=False
+ _check_legend_labels(ax, labels=["a"])
+ assert len(ax.lines) == 1
+
+ def test_subplots_dup_columns_secondary_y_no_subplot(self):
+ # GH 10962
+ df = DataFrame(np.random.default_rng(2).random((5, 5)), columns=list("aaaaa"))
+ ax = df.plot(secondary_y="a")
+ _check_legend_labels(ax, labels=["a (right)"] * 5)
+ assert len(ax.lines) == 0
+ assert len(ax.right_ax.lines) == 5
+
+ @pytest.mark.xfail(
+ is_platform_linux(),
+ reason="Weird rounding problems",
+ strict=False,
+ )
+ def test_bar_log_no_subplots(self):
+ # GH3254, GH3298 matplotlib/matplotlib#1882, #1892
+ # regressions in 1.2.1
+ # matplotlib 3.11 no longer pads log-axis ticks a decade past the
+ # view limits, so the outermost tick is dropped (GH#65918)
+ expected = (
+ np.array([0.1, 1.0, 10.0, 100])
+ if Version(mpl.__version__) < Version("3.11.0rc1")
+ else np.array([0.1, 1.0, 10.0])
+ )
+
+ # no subplots
+ df = DataFrame({"A": [3] * 5, "B": list(range(1, 6))}, index=range(5))
+ ax = df.plot.bar(grid=True, log=True)
+ tm.assert_numpy_array_equal(ax.yaxis.get_ticklocs(), expected)
+
+ @pytest.mark.xfail(
+ is_platform_linux(),
+ reason="Weird rounding problems",
+ strict=False,
+ )
+ def test_bar_log_subplots(self):
+ # matplotlib 3.11 no longer pads log-axis ticks a decade past the
+ # view limits, so the outermost tick is dropped (GH#65918)
+ expected = (
+ np.array([0.1, 1.0, 10.0, 100.0, 1000.0, 1e4])
+ if Version(mpl.__version__) < Version("3.11.0rc1")
+ else np.array([0.1, 1.0, 10.0, 100.0, 1000.0])
+ )
+
+ ax = DataFrame([Series([200, 300]), Series([300, 500])]).plot.bar(
+ log=True, subplots=True
+ )
+
+ tm.assert_numpy_array_equal(ax[0].yaxis.get_ticklocs(), expected)
+ tm.assert_numpy_array_equal(ax[1].yaxis.get_ticklocs(), expected)
+
+ def test_boxplot_subplots_return_type_default(self, hist_df):
+ df = hist_df
+
+ # normal style: return_type=None
+ result = df.plot.box(subplots=True)
+ assert isinstance(result, Series)
+ _check_box_return_type(
+ result, None, expected_keys=["height", "weight", "category"]
+ )
+
+ @pytest.mark.parametrize("rt", ["dict", "axes", "both"])
+ def test_boxplot_subplots_return_type(self, hist_df, rt):
+ df = hist_df
+ returned = df.plot.box(return_type=rt, subplots=True)
+ _check_box_return_type(
+ returned,
+ rt,
+ expected_keys=["height", "weight", "category"],
+ check_ax_title=False,
+ )
+
+ def test_df_subplots_patterns_minorticks(self):
+ # GH 10657
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)),
+ index=date_range("1/1/2000", periods=10),
+ columns=list("AB"),
+ )
+
+ # shared subplots
+ _, axes = plt.subplots(2, 1, sharex=True)
+ axes = df.plot(subplots=True, ax=axes)
+ for ax in axes:
+ assert len(ax.lines) == 1
+ _check_visible(ax.get_yticklabels(), visible=True)
+ # xaxis of 1st ax must be hidden
+ _check_visible(axes[0].get_xticklabels(), visible=False)
+ _check_visible(axes[0].get_xticklabels(minor=True), visible=False)
+ _check_visible(axes[1].get_xticklabels(), visible=True)
+ _check_visible(axes[1].get_xticklabels(minor=True), visible=True)
+
+ def test_df_subplots_patterns_minorticks_1st_ax_hidden(self):
+ # GH 10657
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)),
+ index=date_range("1/1/2000", periods=10),
+ columns=list("AB"),
+ )
+ _, axes = plt.subplots(2, 1)
+ with tm.assert_produces_warning(UserWarning, match="sharex and sharey"):
+ axes = df.plot(subplots=True, ax=axes, sharex=True)
+ for ax in axes:
+ assert len(ax.lines) == 1
+ _check_visible(ax.get_yticklabels(), visible=True)
+ # xaxis of 1st ax must be hidden
+ _check_visible(axes[0].get_xticklabels(), visible=False)
+ _check_visible(axes[0].get_xticklabels(minor=True), visible=False)
+ _check_visible(axes[1].get_xticklabels(), visible=True)
+ _check_visible(axes[1].get_xticklabels(minor=True), visible=True)
+
+ def test_df_subplots_patterns_minorticks_not_shared(self):
+ # GH 10657
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)),
+ index=date_range("1/1/2000", periods=10),
+ columns=list("AB"),
+ )
+ # not shared
+ _, axes = plt.subplots(2, 1)
+ axes = df.plot(subplots=True, ax=axes)
+ for ax in axes:
+ assert len(ax.lines) == 1
+ _check_visible(ax.get_yticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(), visible=True)
+ _check_visible(ax.get_xticklabels(minor=True), visible=True)
+
+ def test_subplots_sharex_false(self):
+ # test when sharex is set to False, two plots should have different
+ # labels, GH 25160
+ df = DataFrame(np.random.default_rng(2).random((10, 2)))
+ df.iloc[5:, 1] = np.nan
+ df.iloc[:5, 0] = np.nan
+
+ _, axs = mpl.pyplot.subplots(2, 1)
+ df.plot.line(ax=axs, subplots=True, sharex=False)
+
+ expected_ax1 = np.arange(4.5, 10, 0.5)
+ expected_ax2 = np.arange(-0.5, 5, 0.5)
+
+ tm.assert_numpy_array_equal(axs[0].get_xticks(), expected_ax1)
+ tm.assert_numpy_array_equal(axs[1].get_xticks(), expected_ax2)
+
+ def test_subplots_constrained_layout(self, temp_file):
+ # GH 25261
+ idx = date_range(start="now", periods=10)
+ df = DataFrame(np.random.default_rng(2).random((10, 3)), index=idx)
+ kwargs = {}
+ if hasattr(mpl.pyplot.Figure, "get_constrained_layout"):
+ kwargs["constrained_layout"] = True
+ _, axes = mpl.pyplot.subplots(2, **kwargs)
+ with tm.assert_produces_warning(None):
+ df.plot(ax=axes[0])
+ with temp_file.open(mode="wb") as path:
+ mpl.pyplot.savefig(path)
+
+ @pytest.mark.parametrize(
+ "index_name, old_label, new_label",
+ [
+ (None, "", "new"),
+ ("old", "old", "new"),
+ (None, "", ""),
+ (None, "", 1),
+ (None, "", [1, 2]),
+ ],
+ )
+ @pytest.mark.parametrize("kind", ["line", "area", "bar"])
+ def test_xlabel_ylabel_dataframe_subplots(
+ self, kind, index_name, old_label, new_label
+ ):
+ # GH 9093
+ df = DataFrame([[1, 2], [2, 5]], columns=["Type A", "Type B"])
+ df.index.name = index_name
+
+ # default is the ylabel is not shown and xlabel is index name
+ axes = df.plot(kind=kind, subplots=True)
+ assert all(ax.get_ylabel() == "" for ax in axes)
+ assert all(ax.get_xlabel() == old_label for ax in axes)
+
+ # old xlabel will be overridden and assigned ylabel will be used as ylabel
+ axes = df.plot(kind=kind, ylabel=new_label, xlabel=new_label, subplots=True)
+ assert all(ax.get_ylabel() == str(new_label) for ax in axes)
+ assert all(ax.get_xlabel() == str(new_label) for ax in axes)
+
+ @pytest.mark.parametrize(
+ "kwargs",
+ [
+ # stacked center
+ {"kind": "bar", "stacked": True},
+ {"kind": "bar", "stacked": True, "width": 0.9},
+ {"kind": "barh", "stacked": True},
+ {"kind": "barh", "stacked": True, "width": 0.9},
+ # center
+ {"kind": "bar", "stacked": False},
+ {"kind": "bar", "stacked": False, "width": 0.9},
+ {"kind": "barh", "stacked": False},
+ {"kind": "barh", "stacked": False, "width": 0.9},
+ # subplots center
+ {"kind": "bar", "subplots": True},
+ {"kind": "bar", "subplots": True, "width": 0.9},
+ {"kind": "barh", "subplots": True},
+ {"kind": "barh", "subplots": True, "width": 0.9},
+ # align edge
+ {"kind": "bar", "stacked": True, "align": "edge"},
+ {"kind": "bar", "stacked": True, "width": 0.9, "align": "edge"},
+ {"kind": "barh", "stacked": True, "align": "edge"},
+ {"kind": "barh", "stacked": True, "width": 0.9, "align": "edge"},
+ {"kind": "bar", "stacked": False, "align": "edge"},
+ {"kind": "bar", "stacked": False, "width": 0.9, "align": "edge"},
+ {"kind": "barh", "stacked": False, "align": "edge"},
+ {"kind": "barh", "stacked": False, "width": 0.9, "align": "edge"},
+ {"kind": "bar", "subplots": True, "align": "edge"},
+ {"kind": "bar", "subplots": True, "width": 0.9, "align": "edge"},
+ {"kind": "barh", "subplots": True, "align": "edge"},
+ {"kind": "barh", "subplots": True, "width": 0.9, "align": "edge"},
+ ],
+ )
+ def test_bar_align_multiple_columns(self, kwargs):
+ # GH2157
+ df = DataFrame({"A": [3] * 5, "B": list(range(5))}, index=range(5))
+ self._check_bar_alignment(df, **kwargs)
+
+ @pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"kind": "bar", "stacked": False},
+ {"kind": "bar", "stacked": True},
+ {"kind": "barh", "stacked": False},
+ {"kind": "barh", "stacked": True},
+ {"kind": "bar", "subplots": True},
+ {"kind": "barh", "subplots": True},
+ ],
+ )
+ def test_bar_align_single_column(self, kwargs):
+ df = DataFrame(np.random.default_rng(2).standard_normal(5))
+ self._check_bar_alignment(df, **kwargs)
+
+ @pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"kind": "bar", "stacked": False},
+ {"kind": "bar", "stacked": True},
+ {"kind": "barh", "stacked": False},
+ {"kind": "barh", "stacked": True},
+ {"kind": "bar", "subplots": True},
+ {"kind": "barh", "subplots": True},
+ ],
+ )
+ def test_bar_barwidth_position(self, kwargs):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ self._check_bar_alignment(df, width=0.9, position=0.2, **kwargs)
+
+ @pytest.mark.parametrize("w", [1, 1.0])
+ def test_bar_barwidth_position_int(self, w):
+ # GH 12979
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ ax = df.plot.bar(stacked=True, width=w)
+ ticks = ax.xaxis.get_ticklocs()
+ tm.assert_numpy_array_equal(ticks, np.array([0, 1, 2, 3, 4]))
+ assert ax.get_xlim() == (-0.75, 4.75)
+ # check left-edge of bars
+ assert ax.patches[0].get_x() == -0.5
+ assert ax.patches[-1].get_x() == 3.5
+
+ @pytest.mark.parametrize(
+ "kind, kwargs",
+ [
+ ["bar", {"stacked": True}],
+ ["barh", {"stacked": False}],
+ ["barh", {"stacked": True}],
+ ["bar", {"subplots": True}],
+ ["barh", {"subplots": True}],
+ ],
+ )
+ def test_bar_barwidth_position_int_width_1(self, kind, kwargs):
+ # GH 12979
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ self._check_bar_alignment(df, kind=kind, width=1, **kwargs)
+
+ def _check_bar_alignment(
+ self,
+ df,
+ kind="bar",
+ stacked=False,
+ subplots=False,
+ align="center",
+ width=0.5,
+ position=0.5,
+ ):
+ axes = df.plot(
+ kind=kind,
+ stacked=stacked,
+ subplots=subplots,
+ align=align,
+ width=width,
+ position=position,
+ grid=True,
+ )
+
+ axes = _flatten_visible(axes)
+
+ for ax in axes:
+ if kind == "bar":
+ axis = ax.xaxis
+ ax_min, ax_max = ax.get_xlim()
+ min_edge = min(p.get_x() for p in ax.patches)
+ max_edge = max(p.get_x() + p.get_width() for p in ax.patches)
+ elif kind == "barh":
+ axis = ax.yaxis
+ ax_min, ax_max = ax.get_ylim()
+ min_edge = min(p.get_y() for p in ax.patches)
+ max_edge = max(p.get_y() + p.get_height() for p in ax.patches)
+ else:
+ raise ValueError
+
+ # GH 7498
+ # compare margins between lim and bar edges
+ tm.assert_almost_equal(ax_min, min_edge - 0.25)
+ tm.assert_almost_equal(ax_max, max_edge + 0.25)
+
+ p = ax.patches[0]
+ if kind == "bar" and (stacked is True or subplots is True):
+ edge = p.get_x()
+ center = edge + p.get_width() * position
+ elif kind == "bar" and stacked is False:
+ center = p.get_x() + p.get_width() * len(df.columns) * position
+ edge = p.get_x()
+ elif kind == "barh" and (stacked is True or subplots is True):
+ center = p.get_y() + p.get_height() * position
+ edge = p.get_y()
+ elif kind == "barh" and stacked is False:
+ center = p.get_y() + p.get_height() * len(df.columns) * position
+ edge = p.get_y()
+ else:
+ raise ValueError
+
+ # Check the ticks locates on integer
+ assert (axis.get_ticklocs() == np.arange(len(df))).all()
+
+ if align == "center":
+ # Check whether the bar locates on center
+ tm.assert_almost_equal(axis.get_ticklocs()[0], center)
+ elif align == "edge":
+ # Check whether the bar's edge starts from the tick
+ tm.assert_almost_equal(axis.get_ticklocs()[0], edge)
+ else:
+ raise ValueError
+
+ return axes
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_hist_box_by.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_hist_box_by.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9250fa8347cc04fa34c28b016e1fb27d837284f
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/frame/test_hist_box_by.py
@@ -0,0 +1,342 @@
+import re
+
+import numpy as np
+import pytest
+
+from pandas import DataFrame
+import pandas._testing as tm
+from pandas.tests.plotting.common import (
+ _check_axes_shape,
+ _check_plot_works,
+ get_x_axis,
+ get_y_axis,
+)
+
+pytest.importorskip("matplotlib")
+
+
+@pytest.fixture
+def hist_df():
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((30, 2)), columns=["A", "B"]
+ )
+ df["C"] = np.random.default_rng(2).choice(["a", "b", "c"], 30)
+ df["D"] = np.random.default_rng(2).choice(["a", "b", "c"], 30)
+ return df
+
+
+class TestHistWithBy:
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "by, column, titles, legends",
+ [
+ ("C", "A", ["a", "b", "c"], [["A"]] * 3),
+ ("C", ["A", "B"], ["a", "b", "c"], [["A", "B"]] * 3),
+ ("C", None, ["a", "b", "c"], [["A", "B"]] * 3),
+ (
+ ["C", "D"],
+ "A",
+ [
+ "(a, a)",
+ "(b, b)",
+ "(c, c)",
+ ],
+ [["A"]] * 3,
+ ),
+ (
+ ["C", "D"],
+ ["A", "B"],
+ [
+ "(a, a)",
+ "(b, b)",
+ "(c, c)",
+ ],
+ [["A", "B"]] * 3,
+ ),
+ (
+ ["C", "D"],
+ None,
+ [
+ "(a, a)",
+ "(b, b)",
+ "(c, c)",
+ ],
+ [["A", "B"]] * 3,
+ ),
+ ],
+ )
+ def test_hist_plot_by_argument(self, by, column, titles, legends, hist_df):
+ # GH 15079
+ axes = _check_plot_works(
+ hist_df.plot.hist, column=column, by=by, default_axes=True
+ )
+ result_titles = [ax.get_title() for ax in axes]
+ result_legends = [
+ [legend.get_text() for legend in ax.get_legend().texts] for ax in axes
+ ]
+
+ assert result_legends == legends
+ assert result_titles == titles
+
+ @pytest.mark.parametrize(
+ "by, column, titles, legends",
+ [
+ (0, "A", ["a", "b", "c"], [["A"]] * 3),
+ (0, None, ["a", "b", "c"], [["A", "B"]] * 3),
+ (
+ [0, "D"],
+ "A",
+ [
+ "(a, a)",
+ "(b, b)",
+ "(c, c)",
+ ],
+ [["A"]] * 3,
+ ),
+ ],
+ )
+ def test_hist_plot_by_0(self, by, column, titles, legends, hist_df):
+ # GH 15079
+ df = hist_df.copy()
+ df = df.rename(columns={"C": 0})
+
+ axes = _check_plot_works(df.plot.hist, default_axes=True, column=column, by=by)
+ result_titles = [ax.get_title() for ax in axes]
+ result_legends = [
+ [legend.get_text() for legend in ax.get_legend().texts] for ax in axes
+ ]
+
+ assert result_legends == legends
+ assert result_titles == titles
+
+ @pytest.mark.parametrize(
+ "by, column",
+ [
+ ([], ["A"]),
+ ([], ["A", "B"]),
+ ((), None),
+ ((), ["A", "B"]),
+ ],
+ )
+ def test_hist_plot_empty_list_string_tuple_by(self, by, column, hist_df):
+ # GH 15079
+ msg = "No group keys passed"
+ with pytest.raises(ValueError, match=msg):
+ _check_plot_works(
+ hist_df.plot.hist, default_axes=True, column=column, by=by
+ )
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "by, column, layout, axes_num",
+ [
+ (["C"], "A", (2, 2), 3),
+ ("C", "A", (2, 2), 3),
+ (["C"], ["A"], (1, 3), 3),
+ ("C", None, (3, 1), 3),
+ ("C", ["A", "B"], (3, 1), 3),
+ (["C", "D"], "A", (9, 1), 3),
+ (["C", "D"], "A", (3, 3), 3),
+ (["C", "D"], ["A"], (5, 2), 3),
+ (["C", "D"], ["A", "B"], (9, 1), 3),
+ (["C", "D"], None, (9, 1), 3),
+ (["C", "D"], ["A", "B"], (5, 2), 3),
+ ],
+ )
+ def test_hist_plot_layout_with_by(self, by, column, layout, axes_num, hist_df):
+ # GH 15079
+ # _check_plot_works adds an ax so catch warning. see GH #13188
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ axes = _check_plot_works(
+ hist_df.plot.hist, column=column, by=by, layout=layout
+ )
+ _check_axes_shape(axes, axes_num=axes_num, layout=layout)
+
+ @pytest.mark.parametrize(
+ "msg, by, layout",
+ [
+ ("larger than required size", ["C", "D"], (1, 1)),
+ (re.escape("Layout must be a tuple of (rows, columns)"), "C", (1,)),
+ ("At least one dimension of layout must be positive", "C", (-1, -1)),
+ ],
+ )
+ def test_hist_plot_invalid_layout_with_by_raises(self, msg, by, layout, hist_df):
+ # GH 15079, test if error is raised when invalid layout is given
+
+ with pytest.raises(ValueError, match=msg):
+ hist_df.plot.hist(column=["A", "B"], by=by, layout=layout)
+
+ @pytest.mark.slow
+ def test_axis_share_x_with_by(self, hist_df):
+ # GH 15079
+ ax1, ax2, ax3 = hist_df.plot.hist(column="A", by="C", sharex=True)
+
+ # share x
+ assert get_x_axis(ax1).joined(ax1, ax2)
+ assert get_x_axis(ax2).joined(ax1, ax2)
+ assert get_x_axis(ax3).joined(ax1, ax3)
+ assert get_x_axis(ax3).joined(ax2, ax3)
+
+ # don't share y
+ assert not get_y_axis(ax1).joined(ax1, ax2)
+ assert not get_y_axis(ax2).joined(ax1, ax2)
+ assert not get_y_axis(ax3).joined(ax1, ax3)
+ assert not get_y_axis(ax3).joined(ax2, ax3)
+
+ @pytest.mark.slow
+ def test_axis_share_y_with_by(self, hist_df):
+ # GH 15079
+ ax1, ax2, ax3 = hist_df.plot.hist(column="A", by="C", sharey=True)
+
+ # share y
+ assert get_y_axis(ax1).joined(ax1, ax2)
+ assert get_y_axis(ax2).joined(ax1, ax2)
+ assert get_y_axis(ax3).joined(ax1, ax3)
+ assert get_y_axis(ax3).joined(ax2, ax3)
+
+ # don't share x
+ assert not get_x_axis(ax1).joined(ax1, ax2)
+ assert not get_x_axis(ax2).joined(ax1, ax2)
+ assert not get_x_axis(ax3).joined(ax1, ax3)
+ assert not get_x_axis(ax3).joined(ax2, ax3)
+
+ @pytest.mark.parametrize("figsize", [(12, 8), (20, 10)])
+ def test_figure_shape_hist_with_by(self, figsize, hist_df):
+ # GH 15079
+ axes = hist_df.plot.hist(column="A", by="C", figsize=figsize)
+ _check_axes_shape(axes, axes_num=3, figsize=figsize)
+
+
+class TestBoxWithBy:
+ @pytest.mark.parametrize(
+ "by, column, titles, xticklabels",
+ [
+ ("C", "A", ["A"], [["a", "b", "c"]]),
+ (
+ ["C", "D"],
+ "A",
+ ["A"],
+ [
+ [
+ "(a, a)",
+ "(b, b)",
+ "(c, c)",
+ ]
+ ],
+ ),
+ ("C", ["A", "B"], ["A", "B"], [["a", "b", "c"]] * 2),
+ (
+ ["C", "D"],
+ ["A", "B"],
+ ["A", "B"],
+ [
+ [
+ "(a, a)",
+ "(b, b)",
+ "(c, c)",
+ ]
+ ]
+ * 2,
+ ),
+ (["C"], None, ["A", "B"], [["a", "b", "c"]] * 2),
+ ],
+ )
+ def test_box_plot_by_argument(self, by, column, titles, xticklabels, hist_df):
+ # GH 15079
+ axes = _check_plot_works(
+ hist_df.plot.box, default_axes=True, column=column, by=by
+ )
+ result_titles = [ax.get_title() for ax in axes]
+ result_xticklabels = [
+ [label.get_text() for label in ax.get_xticklabels()] for ax in axes
+ ]
+
+ assert result_xticklabels == xticklabels
+ assert result_titles == titles
+
+ @pytest.mark.parametrize(
+ "by, column, titles, xticklabels",
+ [
+ (0, "A", ["A"], [["a", "b", "c"]]),
+ (
+ [0, "D"],
+ "A",
+ ["A"],
+ [
+ [
+ "(a, a)",
+ "(b, b)",
+ "(c, c)",
+ ]
+ ],
+ ),
+ (0, None, ["A", "B"], [["a", "b", "c"]] * 2),
+ ],
+ )
+ def test_box_plot_by_0(self, by, column, titles, xticklabels, hist_df):
+ # GH 15079
+ df = hist_df.copy()
+ df = df.rename(columns={"C": 0})
+
+ axes = _check_plot_works(df.plot.box, default_axes=True, column=column, by=by)
+ result_titles = [ax.get_title() for ax in axes]
+ result_xticklabels = [
+ [label.get_text() for label in ax.get_xticklabels()] for ax in axes
+ ]
+
+ assert result_xticklabels == xticklabels
+ assert result_titles == titles
+
+ @pytest.mark.parametrize(
+ "by, column",
+ [
+ ([], ["A"]),
+ ((), "A"),
+ ([], None),
+ ((), ["A", "B"]),
+ ],
+ )
+ def test_box_plot_with_none_empty_list_by(self, by, column, hist_df):
+ # GH 15079
+ msg = "No group keys passed"
+ with pytest.raises(ValueError, match=msg):
+ _check_plot_works(hist_df.plot.box, default_axes=True, column=column, by=by)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "by, column, layout, axes_num",
+ [
+ (["C"], "A", (1, 1), 1),
+ ("C", "A", (1, 1), 1),
+ ("C", None, (2, 1), 2),
+ ("C", ["A", "B"], (1, 2), 2),
+ (["C", "D"], "A", (1, 1), 1),
+ (["C", "D"], None, (1, 2), 2),
+ ],
+ )
+ def test_box_plot_layout_with_by(self, by, column, layout, axes_num, hist_df):
+ # GH 15079
+ axes = _check_plot_works(
+ hist_df.plot.box, default_axes=True, column=column, by=by, layout=layout
+ )
+ _check_axes_shape(axes, axes_num=axes_num, layout=layout)
+
+ @pytest.mark.parametrize(
+ "msg, by, layout",
+ [
+ ("larger than required size", ["C", "D"], (1, 1)),
+ (re.escape("Layout must be a tuple of (rows, columns)"), "C", (1,)),
+ ("At least one dimension of layout must be positive", "C", (-1, -1)),
+ ],
+ )
+ def test_box_plot_invalid_layout_with_by_raises(self, msg, by, layout, hist_df):
+ # GH 15079, test if error is raised when invalid layout is given
+
+ with pytest.raises(ValueError, match=msg):
+ hist_df.plot.box(column=["A", "B"], by=by, layout=layout)
+
+ @pytest.mark.parametrize("figsize", [(12, 8), (20, 10)])
+ def test_figure_shape_hist_with_by(self, figsize, hist_df):
+ # GH 15079
+ axes = hist_df.plot.box(column="A", by="C", figsize=figsize)
+ _check_axes_shape(axes, axes_num=1, figsize=figsize)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_backend.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_backend.py
new file mode 100644
index 0000000000000000000000000000000000000000..683bfcfe54f3ed45567c77b636119973ac910f38
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_backend.py
@@ -0,0 +1,100 @@
+import sys
+import types
+
+import pytest
+
+import pandas.util._test_decorators as td
+
+import pandas
+
+pytestmark = pytest.mark.single_cpu
+
+
+@pytest.fixture
+def dummy_backend():
+ db = types.ModuleType("pandas_dummy_backend")
+ setattr(db, "plot", lambda *args, **kwargs: "used_dummy")
+ return db
+
+
+@pytest.fixture
+def restore_backend():
+ """Restore the plotting backend to matplotlib"""
+ with pandas.option_context("plotting.backend", "matplotlib"):
+ yield
+
+
+def test_backend_is_not_module():
+ msg = "Could not find plotting backend 'not_an_existing_module'."
+ with pytest.raises(ValueError, match=msg):
+ pandas.set_option("plotting.backend", "not_an_existing_module")
+
+ assert pandas.options.plotting.backend == "matplotlib"
+
+
+def test_backend_is_correct(monkeypatch, restore_backend, dummy_backend):
+ monkeypatch.setitem(sys.modules, "pandas_dummy_backend", dummy_backend)
+
+ pandas.set_option("plotting.backend", "pandas_dummy_backend")
+ assert pandas.get_option("plotting.backend") == "pandas_dummy_backend"
+ assert (
+ pandas.plotting._core._get_plot_backend("pandas_dummy_backend") is dummy_backend
+ )
+
+
+def test_backend_can_be_set_in_plot_call(monkeypatch, restore_backend, dummy_backend):
+ monkeypatch.setitem(sys.modules, "pandas_dummy_backend", dummy_backend)
+ df = pandas.DataFrame([1, 2, 3])
+
+ assert pandas.get_option("plotting.backend") == "matplotlib"
+ assert df.plot(backend="pandas_dummy_backend") == "used_dummy"
+
+
+def test_register_entrypoint(restore_backend, tmp_path, monkeypatch, dummy_backend):
+ monkeypatch.syspath_prepend(tmp_path)
+ monkeypatch.setitem(sys.modules, "pandas_dummy_backend", dummy_backend)
+
+ dist_info = tmp_path / "my_backend-0.0.0.dist-info"
+ dist_info.mkdir()
+ # entry_point name should not match module name - otherwise pandas will
+ # fall back to backend lookup by module name
+ (dist_info / "entry_points.txt").write_bytes(
+ b"[pandas_plotting_backends]\nmy_ep_backend = pandas_dummy_backend\n"
+ )
+
+ assert pandas.plotting._core._get_plot_backend("my_ep_backend") is dummy_backend
+
+ with pandas.option_context("plotting.backend", "my_ep_backend"):
+ assert pandas.plotting._core._get_plot_backend() is dummy_backend
+
+
+def test_setting_backend_without_plot_raises(monkeypatch):
+ # GH-28163
+ module = types.ModuleType("pandas_plot_backend")
+ monkeypatch.setitem(sys.modules, "pandas_plot_backend", module)
+
+ assert pandas.options.plotting.backend == "matplotlib"
+ with pytest.raises(
+ ValueError, match="Could not find plotting backend 'pandas_plot_backend'."
+ ):
+ pandas.set_option("plotting.backend", "pandas_plot_backend")
+
+ assert pandas.options.plotting.backend == "matplotlib"
+
+
+@td.skip_if_installed("matplotlib")
+def test_no_matplotlib_ok():
+ msg = (
+ 'matplotlib is required for plotting when the default backend "matplotlib" is '
+ "selected."
+ )
+ with pytest.raises(ImportError, match=msg):
+ pandas.plotting._core._get_plot_backend("matplotlib")
+
+
+def test_extra_kinds_ok(monkeypatch, restore_backend, dummy_backend):
+ # https://github.com/pandas-dev/pandas/pull/28647
+ monkeypatch.setitem(sys.modules, "pandas_dummy_backend", dummy_backend)
+ pandas.set_option("plotting.backend", "pandas_dummy_backend")
+ df = pandas.DataFrame({"A": [1, 2, 3]})
+ df.plot(kind="not a real kind")
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_boxplot_method.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_boxplot_method.py
new file mode 100644
index 0000000000000000000000000000000000000000..748153b63f0d5edb593fb68b81389d1c22430526
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_boxplot_method.py
@@ -0,0 +1,774 @@
+"""Test cases for .boxplot method"""
+
+from __future__ import annotations
+
+import itertools
+import string
+
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ MultiIndex,
+ Series,
+ date_range,
+ plotting,
+ timedelta_range,
+)
+import pandas._testing as tm
+from pandas.tests.plotting.common import (
+ _check_axes_shape,
+ _check_box_return_type,
+ _check_plot_works,
+ _check_ticks_props,
+ _check_visible,
+)
+from pandas.util.version import Version
+
+from pandas.io.formats.printing import pprint_thing
+
+mpl = pytest.importorskip("matplotlib")
+plt = pytest.importorskip("matplotlib.pyplot")
+
+
+def _check_ax_limits(col, ax):
+ y_min, y_max = ax.get_ylim()
+ assert y_min <= col.min()
+ assert y_max >= col.max()
+
+
+if Version(mpl.__version__) < Version("3.10"):
+ verts: list[dict[str, bool | str]] = [{"vert": False}, {"vert": True}]
+else:
+ verts = [{"orientation": "horizontal"}, {"orientation": "vertical"}]
+
+
+@pytest.fixture(params=verts)
+def vert(request):
+ return request.param
+
+
+class TestDataFramePlots:
+ def test_stacked_boxplot_set_axis(self):
+ # GH2980
+ n = 30
+ df = DataFrame(
+ {
+ "Clinical": np.random.default_rng(2).choice([0, 1, 2, 3], n),
+ "Confirmed": np.random.default_rng(2).choice([0, 1, 2, 3], n),
+ "Discarded": np.random.default_rng(2).choice([0, 1, 2, 3], n),
+ },
+ index=np.arange(0, n),
+ )
+ ax = df.plot(kind="bar", stacked=True)
+ assert [int(x.get_text()) for x in ax.get_xticklabels()] == df.index.to_list()
+ ax.set_xticks(np.arange(0, n, 10))
+ plt.draw() # Update changes
+ assert [int(x.get_text()) for x in ax.get_xticklabels()] == list(
+ np.arange(0, n, 10)
+ )
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "kwargs, warn",
+ [
+ [{"return_type": "dict"}, None],
+ [{"column": ["one", "two"]}, None],
+ [{"column": ["one", "two"], "by": "indic"}, UserWarning],
+ [{"column": ["one"], "by": ["indic", "indic2"]}, None],
+ [{"by": "indic"}, UserWarning],
+ [{"by": ["indic", "indic2"]}, UserWarning],
+ [{"notch": 1}, None],
+ [{"by": "indic", "notch": 1}, UserWarning],
+ ],
+ )
+ def test_boxplot_legacy1(self, kwargs, warn):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((6, 4)),
+ index=list(string.ascii_letters[:6]),
+ columns=["one", "two", "three", "four"],
+ )
+ df["indic"] = ["foo", "bar"] * 3
+ df["indic2"] = ["foo", "bar", "foo"] * 2
+
+ # _check_plot_works can add an ax so catch warning. see GH #13188
+ with tm.assert_produces_warning(warn, check_stacklevel=False):
+ _check_plot_works(df.boxplot, **kwargs)
+
+ def test_boxplot_legacy1_series(self):
+ ser = Series(np.random.default_rng(2).standard_normal(6))
+ _check_plot_works(plotting._core.boxplot, data=ser, return_type="dict")
+
+ def test_boxplot_legacy2(self):
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 2)), columns=["Col1", "Col2"]
+ )
+ df["X"] = Series(["A", "A", "A", "A", "A", "B", "B", "B", "B", "B"])
+ df["Y"] = Series(["A"] * 10)
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ _check_plot_works(df.boxplot, by="X")
+
+ def test_boxplot_legacy2_with_ax(self):
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 2)), columns=["Col1", "Col2"]
+ )
+ df["X"] = Series(["A", "A", "A", "A", "A", "B", "B", "B", "B", "B"])
+ df["Y"] = Series(["A"] * 10)
+ # When ax is supplied and required number of axes is 1,
+ # passed ax should be used:
+ _, ax = mpl.pyplot.subplots()
+ axes = df.boxplot("Col1", by="X", ax=ax)
+ ax_axes = ax.axes
+ assert ax_axes is axes
+
+ def test_boxplot_legacy2_with_ax_return_type(self):
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 2)), columns=["Col1", "Col2"]
+ )
+ df["X"] = Series(["A", "A", "A", "A", "A", "B", "B", "B", "B", "B"])
+ df["Y"] = Series(["A"] * 10)
+ fig, ax = mpl.pyplot.subplots()
+ axes = df.groupby("Y").boxplot(ax=ax, return_type="axes")
+ ax_axes = ax.axes
+ assert ax_axes is axes["A"]
+
+ def test_boxplot_legacy2_with_multi_col(self):
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 2)), columns=["Col1", "Col2"]
+ )
+ df["X"] = Series(["A", "A", "A", "A", "A", "B", "B", "B", "B", "B"])
+ df["Y"] = Series(["A"] * 10)
+ # Multiple columns with an ax argument should use same figure
+ fig, ax = mpl.pyplot.subplots()
+ msg = "the figure containing the passed axes is being cleared"
+ with tm.assert_produces_warning(UserWarning, match=msg):
+ axes = df.boxplot(
+ column=["Col1", "Col2"], by="X", ax=ax, return_type="axes"
+ )
+ assert axes["Col1"].get_figure() is fig
+
+ def test_boxplot_legacy2_by_none(self):
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 2)), columns=["Col1", "Col2"]
+ )
+ df["X"] = Series(["A", "A", "A", "A", "A", "B", "B", "B", "B", "B"])
+ df["Y"] = Series(["A"] * 10)
+ # When by is None, check that all relevant lines are present in the
+ # dict
+ _, ax = mpl.pyplot.subplots()
+ d = df.boxplot(ax=ax, return_type="dict")
+ lines = list(itertools.chain.from_iterable(d.values()))
+ assert len(ax.get_lines()) == len(lines)
+
+ def test_boxplot_return_type_none(self, hist_df):
+ # GH 12216; return_type=None & by=None -> axes
+ result = hist_df.boxplot()
+ assert isinstance(result, mpl.pyplot.Axes)
+
+ def test_boxplot_return_type_legacy(self):
+ # API change in https://github.com/pandas-dev/pandas/pull/7096
+
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((6, 4)),
+ index=list(string.ascii_letters[:6]),
+ columns=["one", "two", "three", "four"],
+ )
+ msg = "return_type must be {'axes', 'dict', 'both'}"
+ with pytest.raises(ValueError, match=msg):
+ df.boxplot(return_type="NOT_A_TYPE")
+
+ result = df.boxplot()
+ _check_box_return_type(result, "axes")
+
+ @pytest.mark.parametrize("return_type", ["dict", "axes", "both"])
+ def test_boxplot_return_type_legacy_return_type(self, return_type):
+ # API change in https://github.com/pandas-dev/pandas/pull/7096
+
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((6, 4)),
+ index=list(string.ascii_letters[:6]),
+ columns=["one", "two", "three", "four"],
+ )
+ with tm.assert_produces_warning(False):
+ result = df.boxplot(return_type=return_type)
+ _check_box_return_type(result, return_type)
+
+ def test_boxplot_axis_limits(self, hist_df):
+ df = hist_df.copy()
+ df["age"] = np.random.default_rng(2).integers(1, 20, df.shape[0])
+ # One full row
+ height_ax, weight_ax = df.boxplot(["height", "weight"], by="category")
+ _check_ax_limits(df["height"], height_ax)
+ _check_ax_limits(df["weight"], weight_ax)
+ assert weight_ax._sharey == height_ax
+
+ def test_boxplot_axis_limits_two_rows(self, hist_df):
+ df = hist_df.copy()
+ df["age"] = np.random.default_rng(2).integers(1, 20, df.shape[0])
+ # Two rows, one partial
+ p = df.boxplot(["height", "weight", "age"], by="category")
+ height_ax, weight_ax, age_ax = p[0, 0], p[0, 1], p[1, 0]
+ dummy_ax = p[1, 1]
+
+ _check_ax_limits(df["height"], height_ax)
+ _check_ax_limits(df["weight"], weight_ax)
+ _check_ax_limits(df["age"], age_ax)
+ assert weight_ax._sharey == height_ax
+ assert age_ax._sharey == height_ax
+ assert dummy_ax._sharey is None
+
+ def test_boxplot_empty_column(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((20, 4)))
+ df.loc[:, 0] = np.nan
+ _check_plot_works(df.boxplot, return_type="axes")
+
+ def test_figsize(self):
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 5)), columns=["A", "B", "C", "D", "E"]
+ )
+ result = df.boxplot(return_type="axes", figsize=(12, 8))
+ assert result.figure.bbox_inches.width == 12
+ assert result.figure.bbox_inches.height == 8
+
+ def test_fontsize(self):
+ df = DataFrame({"a": [1, 2, 3, 4, 5, 6]})
+ _check_ticks_props(df.boxplot("a", fontsize=16), xlabelsize=16, ylabelsize=16)
+
+ def test_boxplot_numeric_data(self):
+ # GH 22799
+ df = DataFrame(
+ {
+ "a": date_range("2012-01-01", periods=10),
+ "b": np.random.default_rng(2).standard_normal(10),
+ "c": np.random.default_rng(2).standard_normal(10) + 2,
+ "d": date_range("2012-01-01", periods=10).astype(str),
+ "e": date_range("2012-01-01", periods=10, tz="UTC"),
+ "f": timedelta_range("1 days", periods=10),
+ }
+ )
+ ax = df.plot(kind="box")
+ assert [x.get_text() for x in ax.get_xticklabels()] == ["b", "c"]
+
+ @pytest.mark.parametrize(
+ "colors_kwd, expected",
+ [
+ (
+ {"boxes": "r", "whiskers": "b", "medians": "g", "caps": "c"},
+ {"boxes": "r", "whiskers": "b", "medians": "g", "caps": "c"},
+ ),
+ ({"boxes": "r"}, {"boxes": "r"}),
+ ("r", {"boxes": "r", "whiskers": "r", "medians": "r", "caps": "r"}),
+ ],
+ )
+ def test_color_kwd(self, colors_kwd, expected):
+ # GH: 26214
+ df = DataFrame(np.random.default_rng(2).random((10, 2)))
+ result = df.boxplot(color=colors_kwd, return_type="dict")
+ for k, v in expected.items():
+ assert result[k][0].get_color() == v
+
+ @pytest.mark.parametrize(
+ "scheme,expected",
+ [
+ (
+ "dark_background",
+ {
+ "boxes": "#8dd3c7",
+ "whiskers": "#8dd3c7",
+ "medians": "#bfbbd9",
+ "caps": "#8dd3c7",
+ },
+ ),
+ (
+ "default",
+ {
+ "boxes": "#1f77b4",
+ "whiskers": "#1f77b4",
+ "medians": "#2ca02c",
+ "caps": "#1f77b4",
+ },
+ ),
+ ],
+ )
+ def test_colors_in_theme(self, scheme, expected):
+ # GH: 40769
+ df = DataFrame(np.random.default_rng(2).random((10, 2)))
+ plt.style.use(scheme)
+ result = df.plot.box(return_type="dict")
+ for k, v in expected.items():
+ assert mpl.colors.same_color(result[k][0].get_color(), v)
+
+ @pytest.mark.parametrize(
+ "dict_colors, msg",
+ [({"boxes": "r", "invalid_key": "r"}, "invalid key 'invalid_key'")],
+ )
+ def test_color_kwd_errors(self, dict_colors, msg):
+ # GH: 26214
+ df = DataFrame(np.random.default_rng(2).random((10, 2)))
+ with pytest.raises(ValueError, match=msg):
+ df.boxplot(color=dict_colors, return_type="dict")
+
+ @pytest.mark.parametrize(
+ "props, expected",
+ [
+ ("boxprops", "boxes"),
+ ("whiskerprops", "whiskers"),
+ ("capprops", "caps"),
+ ("medianprops", "medians"),
+ ],
+ )
+ def test_specified_props_kwd(self, props, expected):
+ # GH 30346
+ df = DataFrame({k: np.random.default_rng(2).random(10) for k in "ABC"})
+ kwd = {props: {"color": "C1"}}
+ result = df.boxplot(return_type="dict", **kwd)
+
+ assert result[expected][0].get_color() == "C1"
+
+ @pytest.mark.filterwarnings("ignore:set_ticklabels:UserWarning")
+ def test_plot_xlabel_ylabel(self, vert):
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).standard_normal(10),
+ "b": np.random.default_rng(2).standard_normal(10),
+ "group": np.random.default_rng(2).choice(["group1", "group2"], 10),
+ }
+ )
+ xlabel, ylabel = "x", "y"
+ ax = df.plot(kind="box", xlabel=xlabel, ylabel=ylabel, **vert)
+ assert ax.get_xlabel() == xlabel
+ assert ax.get_ylabel() == ylabel
+
+ @pytest.mark.filterwarnings("ignore:set_ticklabels:UserWarning")
+ def test_plot_box(self, vert):
+ # GH 54941
+ rng = np.random.default_rng(2)
+ df1 = DataFrame(rng.integers(0, 100, size=(10, 4)), columns=list("ABCD"))
+ df2 = DataFrame(rng.integers(0, 100, size=(10, 4)), columns=list("ABCD"))
+
+ xlabel, ylabel = "x", "y"
+ _, axs = plt.subplots(ncols=2, figsize=(10, 7), sharey=True)
+ df1.plot.box(ax=axs[0], xlabel=xlabel, ylabel=ylabel, **vert)
+ df2.plot.box(ax=axs[1], xlabel=xlabel, ylabel=ylabel, **vert)
+ for ax in axs:
+ assert ax.get_xlabel() == xlabel
+ assert ax.get_ylabel() == ylabel
+
+ @pytest.mark.filterwarnings("ignore:set_ticklabels:UserWarning")
+ def test_boxplot_xlabel_ylabel(self, vert):
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).standard_normal(10),
+ "b": np.random.default_rng(2).standard_normal(10),
+ "group": np.random.default_rng(2).choice(["group1", "group2"], 10),
+ }
+ )
+ xlabel, ylabel = "x", "y"
+ ax = df.boxplot(xlabel=xlabel, ylabel=ylabel, **vert)
+ assert ax.get_xlabel() == xlabel
+ assert ax.get_ylabel() == ylabel
+
+ @pytest.mark.filterwarnings("ignore:set_ticklabels:UserWarning")
+ def test_boxplot_group_xlabel_ylabel(self, vert):
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).standard_normal(10),
+ "b": np.random.default_rng(2).standard_normal(10),
+ "group": np.random.default_rng(2).choice(["group1", "group2"], 10),
+ }
+ )
+ xlabel, ylabel = "x", "y"
+ ax = df.boxplot(by="group", xlabel=xlabel, ylabel=ylabel, **vert)
+ for subplot in ax:
+ assert subplot.get_xlabel() == xlabel
+ assert subplot.get_ylabel() == ylabel
+
+ @pytest.mark.filterwarnings("ignore:set_ticklabels:UserWarning")
+ def test_boxplot_group_no_xlabel_ylabel(self, vert, request):
+ if Version(mpl.__version__) >= Version("3.10") and vert == {
+ "orientation": "horizontal"
+ }:
+ request.applymarker(
+ pytest.mark.xfail(reason=f"{vert} fails starting with matplotlib 3.10")
+ )
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).standard_normal(10),
+ "b": np.random.default_rng(2).standard_normal(10),
+ "group": np.random.default_rng(2).choice(["group1", "group2"], 10),
+ }
+ )
+ ax = df.boxplot(by="group", **vert)
+ for subplot in ax:
+ target_label = (
+ subplot.get_xlabel()
+ if vert in ({"vert": True}, {"orientation": "vertical"})
+ else subplot.get_ylabel()
+ )
+ assert target_label == pprint_thing(["group"])
+
+
+class TestDataFrameGroupByPlots:
+ def test_boxplot_legacy1(self, hist_df):
+ grouped = hist_df.groupby(by="gender")
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ axes = _check_plot_works(grouped.boxplot, return_type="axes")
+ _check_axes_shape(list(axes.values), axes_num=2, layout=(1, 2))
+
+ def test_boxplot_legacy1_return_type(self, hist_df):
+ grouped = hist_df.groupby(by="gender")
+ axes = _check_plot_works(grouped.boxplot, subplots=False, return_type="axes")
+ _check_axes_shape(axes, axes_num=1, layout=(1, 1))
+
+ @pytest.mark.slow
+ def test_boxplot_legacy2(self):
+ tuples = zip(string.ascii_letters[:10], range(10), strict=True)
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=MultiIndex.from_tuples(tuples),
+ )
+ grouped = df.groupby(level=1)
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ axes = _check_plot_works(grouped.boxplot, return_type="axes")
+ _check_axes_shape(list(axes.values), axes_num=10, layout=(4, 3))
+
+ @pytest.mark.slow
+ def test_boxplot_legacy2_return_type(self):
+ tuples = zip(string.ascii_letters[:10], range(10), strict=True)
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 3)),
+ index=MultiIndex.from_tuples(tuples),
+ )
+ grouped = df.groupby(level=1)
+ axes = _check_plot_works(grouped.boxplot, subplots=False, return_type="axes")
+ _check_axes_shape(axes, axes_num=1, layout=(1, 1))
+
+ def test_grouped_plot_fignums(self):
+ n = 10
+ weight = Series(np.random.default_rng(2).normal(166, 20, size=n))
+ height = Series(np.random.default_rng(2).normal(60, 10, size=n))
+ gender = np.random.default_rng(2).choice(["male", "female"], size=n)
+ df = DataFrame({"height": height, "weight": weight, "gender": gender})
+ gb = df.groupby("gender")
+
+ res = gb.plot()
+ assert len(mpl.pyplot.get_fignums()) == 2
+ assert len(res) == 2
+ plt.close("all")
+
+ res = gb.boxplot(return_type="axes")
+ assert len(mpl.pyplot.get_fignums()) == 1
+ assert len(res) == 2
+
+ def test_grouped_plot_fignums_excluded_col(self):
+ n = 10
+ weight = Series(np.random.default_rng(2).normal(166, 20, size=n))
+ height = Series(np.random.default_rng(2).normal(60, 10, size=n))
+ gender = np.random.default_rng(2).choice(["male", "female"], size=n)
+ df = DataFrame({"height": height, "weight": weight, "gender": gender})
+ # now works with GH 5610 as gender is excluded
+ df.groupby("gender").hist()
+
+ @pytest.mark.slow
+ def test_grouped_box_return_type(self, hist_df):
+ df = hist_df
+
+ # old style: return_type=None
+ result = df.boxplot(by="gender")
+ assert isinstance(result, np.ndarray)
+ _check_box_return_type(
+ result, None, expected_keys=["height", "weight", "category"]
+ )
+
+ @pytest.mark.slow
+ def test_grouped_box_return_type_groupby(self, hist_df):
+ df = hist_df
+ # now for groupby
+ result = df.groupby("gender").boxplot(return_type="dict")
+ _check_box_return_type(result, "dict", expected_keys=["Male", "Female"])
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("return_type", ["dict", "axes", "both"])
+ def test_grouped_box_return_type_arg(self, hist_df, return_type):
+ df = hist_df
+
+ returned = df.groupby("classroom").boxplot(return_type=return_type)
+ _check_box_return_type(returned, return_type, expected_keys=["A", "B", "C"])
+
+ returned = df.boxplot(by="classroom", return_type=return_type)
+ _check_box_return_type(
+ returned, return_type, expected_keys=["height", "weight", "category"]
+ )
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("return_type", ["dict", "axes", "both"])
+ def test_grouped_box_return_type_arg_duplcate_cats(self, return_type):
+ columns2 = "X B C D A".split()
+ df2 = DataFrame(
+ np.random.default_rng(2).standard_normal((6, 5)), columns=columns2
+ )
+ categories2 = "A B".split()
+ df2["category"] = categories2 * 3
+
+ returned = df2.groupby("category").boxplot(return_type=return_type)
+ _check_box_return_type(returned, return_type, expected_keys=categories2)
+
+ returned = df2.boxplot(by="category", return_type=return_type)
+ _check_box_return_type(returned, return_type, expected_keys=columns2)
+
+ @pytest.mark.slow
+ def test_grouped_box_layout_too_small(self, hist_df):
+ df = hist_df
+
+ msg = "Layout of 1x1 must be larger than required size 2"
+ with pytest.raises(ValueError, match=msg):
+ df.boxplot(column=["weight", "height"], by=df.gender, layout=(1, 1))
+
+ @pytest.mark.slow
+ def test_grouped_box_layout_needs_by(self, hist_df):
+ df = hist_df
+ msg = "The 'layout' keyword is not supported when 'by' is None"
+ with pytest.raises(ValueError, match=msg):
+ df.boxplot(
+ column=["height", "weight", "category"],
+ layout=(2, 1),
+ return_type="dict",
+ )
+
+ @pytest.mark.slow
+ def test_grouped_box_layout_positive_layout(self, hist_df):
+ df = hist_df
+ msg = "At least one dimension of layout must be positive"
+ with pytest.raises(ValueError, match=msg):
+ df.boxplot(column=["weight", "height"], by=df.gender, layout=(-1, -1))
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "gb_key, axes_num, rows",
+ [["gender", 2, 1], ["category", 4, 2], ["classroom", 3, 2]],
+ )
+ def test_grouped_box_layout_positive_layout_axes(
+ self, hist_df, gb_key, axes_num, rows
+ ):
+ df = hist_df
+ # _check_plot_works adds an ax so catch warning. see GH #13188 GH 6769
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ _check_plot_works(
+ df.groupby(gb_key).boxplot, column="height", return_type="dict"
+ )
+ _check_axes_shape(mpl.pyplot.gcf().axes, axes_num=axes_num, layout=(rows, 2))
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "col, visible", [["height", False], ["weight", True], ["category", True]]
+ )
+ def test_grouped_box_layout_visible(self, hist_df, col, visible):
+ df = hist_df
+ # GH 5897
+ axes = df.boxplot(
+ column=["height", "weight", "category"], by="gender", return_type="axes"
+ )
+ _check_axes_shape(mpl.pyplot.gcf().axes, axes_num=3, layout=(2, 2))
+ ax = axes[col]
+ _check_visible(ax.get_xticklabels(), visible=visible)
+ _check_visible([ax.xaxis.get_label()], visible=visible)
+
+ @pytest.mark.slow
+ def test_grouped_box_layout_shape(self, hist_df):
+ df = hist_df
+ df.groupby("classroom").boxplot(
+ column=["height", "weight", "category"], return_type="dict"
+ )
+ _check_axes_shape(mpl.pyplot.gcf().axes, axes_num=3, layout=(2, 2))
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("cols", [2, -1])
+ def test_grouped_box_layout_works(self, hist_df, cols):
+ df = hist_df
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ _check_plot_works(
+ df.groupby("category").boxplot,
+ column="height",
+ layout=(3, cols),
+ return_type="dict",
+ )
+ _check_axes_shape(mpl.pyplot.gcf().axes, axes_num=4, layout=(3, 2))
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("rows, res", [[4, 4], [-1, 3]])
+ def test_grouped_box_layout_axes_shape_rows(self, hist_df, rows, res):
+ df = hist_df
+ df.boxplot(
+ column=["height", "weight", "category"], by="gender", layout=(rows, 1)
+ )
+ _check_axes_shape(mpl.pyplot.gcf().axes, axes_num=3, layout=(res, 1))
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("cols, res", [[4, 4], [-1, 3]])
+ def test_grouped_box_layout_axes_shape_cols_groupby(self, hist_df, cols, res):
+ df = hist_df
+ df.groupby("classroom").boxplot(
+ column=["height", "weight", "category"],
+ layout=(1, cols),
+ return_type="dict",
+ )
+ _check_axes_shape(mpl.pyplot.gcf().axes, axes_num=3, layout=(1, res))
+
+ @pytest.mark.slow
+ def test_grouped_box_multiple_axes(self, hist_df):
+ # GH 6970, GH 7069
+ df = hist_df
+
+ # check warning to ignore sharex / sharey
+ # this check should be done in the first function which
+ # passes multiple axes to plot, hist or boxplot
+ # location should be changed if other test is added
+ # which has earlier alphabetical order
+ with tm.assert_produces_warning(UserWarning, match="sharex and sharey"):
+ _, axes = mpl.pyplot.subplots(2, 2)
+ df.groupby("category").boxplot(column="height", return_type="axes", ax=axes)
+ _check_axes_shape(mpl.pyplot.gcf().axes, axes_num=4, layout=(2, 2))
+
+ @pytest.mark.slow
+ def test_grouped_box_multiple_axes_on_fig(self, hist_df):
+ # GH 6970, GH 7069
+ df = hist_df
+ fig, axes = mpl.pyplot.subplots(2, 3)
+ with tm.assert_produces_warning(UserWarning, match="sharex and sharey"):
+ returned = df.boxplot(
+ column=["height", "weight", "category"],
+ by="gender",
+ return_type="axes",
+ ax=axes[0],
+ )
+ returned = np.array(list(returned.values))
+ _check_axes_shape(returned, axes_num=3, layout=(1, 3))
+ tm.assert_numpy_array_equal(returned, axes[0])
+ assert returned[0].figure is fig
+
+ # draw on second row
+ with tm.assert_produces_warning(UserWarning, match="sharex and sharey"):
+ returned = df.groupby("classroom").boxplot(
+ column=["height", "weight", "category"], return_type="axes", ax=axes[1]
+ )
+ returned = np.array(list(returned.values))
+ _check_axes_shape(returned, axes_num=3, layout=(1, 3))
+ tm.assert_numpy_array_equal(returned, axes[1])
+ assert returned[0].figure is fig
+
+ @pytest.mark.slow
+ def test_grouped_box_multiple_axes_ax_error(self, hist_df):
+ # GH 6970, GH 7069
+ df = hist_df
+ msg = "The number of passed axes must be 3, the same as the output plot"
+ _, axes = mpl.pyplot.subplots(2, 3)
+ with pytest.raises(ValueError, match=msg):
+ # pass different number of axes from required
+ with tm.assert_produces_warning(UserWarning, match="sharex and sharey"):
+ axes = df.groupby("classroom").boxplot(ax=axes)
+
+ def test_fontsize(self):
+ df = DataFrame({"a": [1, 2, 3, 4, 5, 6], "b": [0, 0, 0, 1, 1, 1]})
+ _check_ticks_props(
+ df.boxplot("a", by="b", fontsize=16), xlabelsize=16, ylabelsize=16
+ )
+
+ @pytest.mark.parametrize(
+ "col, expected_xticklabel",
+ [
+ ("v", ["(a, v)", "(b, v)", "(c, v)", "(d, v)", "(e, v)"]),
+ (["v"], ["(a, v)", "(b, v)", "(c, v)", "(d, v)", "(e, v)"]),
+ ("v1", ["(a, v1)", "(b, v1)", "(c, v1)", "(d, v1)", "(e, v1)"]),
+ (
+ ["v", "v1"],
+ [
+ "(a, v)",
+ "(a, v1)",
+ "(b, v)",
+ "(b, v1)",
+ "(c, v)",
+ "(c, v1)",
+ "(d, v)",
+ "(d, v1)",
+ "(e, v)",
+ "(e, v1)",
+ ],
+ ),
+ (
+ None,
+ [
+ "(a, v)",
+ "(a, v1)",
+ "(b, v)",
+ "(b, v1)",
+ "(c, v)",
+ "(c, v1)",
+ "(d, v)",
+ "(d, v1)",
+ "(e, v)",
+ "(e, v1)",
+ ],
+ ),
+ ],
+ )
+ def test_groupby_boxplot_subplots_false(self, col, expected_xticklabel):
+ # GH 16748
+ df = DataFrame(
+ {
+ "cat": np.random.default_rng(2).choice(list("abcde"), 100),
+ "v": np.random.default_rng(2).random(100),
+ "v1": np.random.default_rng(2).random(100),
+ }
+ )
+ grouped = df.groupby("cat")
+
+ axes = _check_plot_works(
+ grouped.boxplot, subplots=False, column=col, return_type="axes"
+ )
+
+ result_xticklabel = [x.get_text() for x in axes.get_xticklabels()]
+ assert expected_xticklabel == result_xticklabel
+
+ def test_groupby_boxplot_object(self, hist_df):
+ # GH 43480
+ df = hist_df.astype("object")
+ grouped = df.groupby("gender")
+ msg = "boxplot method requires numerical columns, nothing to plot"
+ with pytest.raises(ValueError, match=msg):
+ _check_plot_works(grouped.boxplot, subplots=False)
+
+ def test_boxplot_multiindex_column(self):
+ # GH 16748
+ arrays = [
+ ["bar", "bar", "baz", "baz", "foo", "foo", "qux", "qux"],
+ ["one", "two", "one", "two", "one", "two", "one", "two"],
+ ]
+ tuples = list(zip(*arrays, strict=True))
+ index = MultiIndex.from_tuples(tuples, names=["first", "second"])
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((3, 8)),
+ index=["A", "B", "C"],
+ columns=index,
+ )
+
+ col = [("bar", "one"), ("bar", "two")]
+ axes = _check_plot_works(df.boxplot, column=col, return_type="axes")
+
+ expected_xticklabel = ["(bar, one)", "(bar, two)"]
+ result_xticklabel = [x.get_text() for x in axes.get_xticklabels()]
+ assert expected_xticklabel == result_xticklabel
+
+ @pytest.mark.parametrize("group", ["X", ["X", "Y"]])
+ def test_boxplot_multi_groupby_groups(self, group):
+ # GH 14701
+ rows = 20
+ df = DataFrame(
+ np.random.default_rng(12).normal(size=(rows, 2)), columns=["Col1", "Col2"]
+ )
+ df["X"] = Series(np.repeat(["A", "B"], int(rows / 2)))
+ df["Y"] = Series(np.tile(["C", "D"], int(rows / 2)))
+ grouped = df.groupby(group)
+ _check_plot_works(df.boxplot, by=group, default_axes=True)
+ _check_plot_works(df.plot.box, by=group, default_axes=True)
+ _check_plot_works(grouped.boxplot, default_axes=True)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_common.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_common.py
new file mode 100644
index 0000000000000000000000000000000000000000..20daf5935624843af3224f991497f84fa6639a0d
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_common.py
@@ -0,0 +1,60 @@
+import pytest
+
+from pandas import DataFrame
+from pandas.tests.plotting.common import (
+ _check_plot_works,
+ _check_ticks_props,
+ _gen_two_subplots,
+)
+
+plt = pytest.importorskip("matplotlib.pyplot")
+
+
+class TestCommon:
+ def test__check_ticks_props(self):
+ # GH 34768
+ df = DataFrame({"b": [0, 1, 0], "a": [1, 2, 3]})
+ ax = _check_plot_works(df.plot, rot=30)
+ ax.yaxis.set_tick_params(rotation=30)
+ msg = "expected 0.00000 but got "
+ with pytest.raises(AssertionError, match=msg):
+ _check_ticks_props(ax, xrot=0)
+ with pytest.raises(AssertionError, match=msg):
+ _check_ticks_props(ax, xlabelsize=0)
+ with pytest.raises(AssertionError, match=msg):
+ _check_ticks_props(ax, yrot=0)
+ with pytest.raises(AssertionError, match=msg):
+ _check_ticks_props(ax, ylabelsize=0)
+
+ def test__gen_two_subplots_with_ax(self):
+ fig = plt.gcf()
+ gen = _gen_two_subplots(f=lambda **kwargs: None, fig=fig, ax="test")
+ # On the first yield, no subplot should be added since ax was passed
+ next(gen)
+ assert fig.get_axes() == []
+ # On the second, the one axis should match fig.subplot(2, 1, 2)
+ next(gen)
+ axes = fig.get_axes()
+ assert len(axes) == 1
+ subplot_geometry = list(axes[0].get_subplotspec().get_geometry()[:-1])
+ subplot_geometry[-1] += 1
+ assert subplot_geometry == [2, 1, 2]
+
+ def test_colorbar_layout(self):
+ fig = plt.figure()
+
+ axes = fig.subplot_mosaic(
+ """
+ AB
+ CC
+ """
+ )
+
+ x = [1, 2, 3]
+ y = [1, 2, 3]
+
+ cs0 = axes["A"].scatter(x, y)
+ axes["B"].scatter(x, y)
+
+ fig.colorbar(cs0, ax=[axes["A"], axes["B"]], location="right")
+ DataFrame(x).plot(ax=axes["C"])
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_converter.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_converter.py
new file mode 100644
index 0000000000000000000000000000000000000000..e33e91ccf6c6e2c508d4b528dbf1c194cd7918fa
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_converter.py
@@ -0,0 +1,391 @@
+from datetime import (
+ date,
+ datetime,
+)
+import subprocess
+import sys
+
+import numpy as np
+import pytest
+
+import pandas._config.config as cf
+
+from pandas._libs.tslibs import to_offset
+
+from pandas import (
+ Index,
+ Period,
+ PeriodIndex,
+ Series,
+ Timestamp,
+ arrays,
+ date_range,
+)
+import pandas._testing as tm
+
+from pandas.plotting import (
+ deregister_matplotlib_converters,
+ register_matplotlib_converters,
+)
+from pandas.tseries.offsets import (
+ Day,
+ Micro,
+ Milli,
+ Second,
+)
+
+plt = pytest.importorskip("matplotlib.pyplot")
+dates = pytest.importorskip("matplotlib.dates")
+units = pytest.importorskip("matplotlib.units")
+
+from pandas.plotting._matplotlib import converter
+
+
+@pytest.mark.single_cpu
+def test_registry_mpl_resets():
+ # Check that Matplotlib converters are properly reset (see issue #27481)
+ code = (
+ "import matplotlib.units as units; "
+ "import matplotlib.dates as mdates; "
+ "n_conv = len(units.registry); "
+ "import pandas as pd; "
+ "pd.plotting.register_matplotlib_converters(); "
+ "pd.plotting.deregister_matplotlib_converters(); "
+ "assert len(units.registry) == n_conv"
+ )
+ call = [sys.executable, "-c", code]
+ subprocess.check_output(call)
+
+
+def test_timtetonum_accepts_unicode():
+ assert converter.time2num("00:01") == converter.time2num("00:01")
+
+
+class TestRegistration:
+ @pytest.mark.single_cpu
+ def test_dont_register_by_default(self):
+ # Run in subprocess to ensure a clean state
+ code = (
+ "import matplotlib.units; "
+ "import pandas as pd; "
+ "units = dict(matplotlib.units.registry); "
+ "assert pd.Timestamp not in units"
+ )
+ call = [sys.executable, "-c", code]
+ assert subprocess.check_call(call) == 0
+
+ def test_registering_no_warning(self):
+ s = Series(range(12), index=date_range("2017", periods=12))
+ _, ax = plt.subplots()
+
+ # Set to the "warn" state, in case this isn't the first test run
+ register_matplotlib_converters()
+ ax.plot(s.index, s.values)
+
+ def test_pandas_plots_register(self):
+ s = Series(range(12), index=date_range("2017", periods=12))
+ # Set to the "warn" state, in case this isn't the first test run
+ with tm.assert_produces_warning(None) as w:
+ s.plot()
+
+ assert len(w) == 0
+
+ def test_matplotlib_formatters(self):
+ # Can't make any assertion about the start state.
+ # We we check that toggling converters off removes it, and toggling it
+ # on restores it.
+
+ with cf.option_context("plotting.matplotlib.register_converters", True):
+ with cf.option_context("plotting.matplotlib.register_converters", False):
+ assert Timestamp not in units.registry
+ assert Timestamp in units.registry
+
+ def test_option_no_warning(self):
+ s = Series(range(12), index=date_range("2017", periods=12))
+ _, ax = plt.subplots()
+
+ # Test without registering first, no warning
+ with cf.option_context("plotting.matplotlib.register_converters", False):
+ ax.plot(s.index, s.values)
+
+ # Now test with registering
+ register_matplotlib_converters()
+ with cf.option_context("plotting.matplotlib.register_converters", False):
+ ax.plot(s.index, s.values)
+
+ def test_registry_resets(self):
+ # make a copy, to reset to
+ original = dict(units.registry)
+
+ try:
+ # get to a known state
+ units.registry.clear()
+ date_converter = dates.DateConverter()
+ units.registry[datetime] = date_converter
+ units.registry[date] = date_converter
+
+ register_matplotlib_converters()
+ assert units.registry[date] is not date_converter
+ deregister_matplotlib_converters()
+ assert units.registry[date] is date_converter
+
+ finally:
+ # restore original stater
+ units.registry.clear()
+ for k, v in original.items():
+ units.registry[k] = v
+
+
+class TestDateTimeConverter:
+ @pytest.fixture
+ def dtc(self):
+ return converter.DatetimeConverter()
+
+ def test_convert_accepts_unicode(self, dtc):
+ r1 = dtc.convert("2000-01-01 12:22", None, None)
+ r2 = dtc.convert("2000-01-01 12:22", None, None)
+ assert r1 == r2, "DatetimeConverter.convert should accept unicode"
+
+ def test_conversion(self, dtc):
+ rs = dtc.convert(["2012-1-1"], None, None)[0]
+ xp = dates.date2num(datetime(2012, 1, 1))
+ assert rs == xp
+
+ rs = dtc.convert("2012-1-1", None, None)
+ assert rs == xp
+
+ rs = dtc.convert(date(2012, 1, 1), None, None)
+ assert rs == xp
+
+ rs = dtc.convert("2012-1-1", None, None)
+ assert rs == xp
+
+ rs = dtc.convert(Timestamp("2012-1-1"), None, None)
+ assert rs == xp
+
+ # also testing datetime64 dtype (GH8614)
+ rs = dtc.convert("2012-01-01", None, None)
+ assert rs == xp
+
+ rs = dtc.convert("2012-01-01 00:00:00+0000", None, None)
+ assert rs == xp
+
+ rs = dtc.convert(
+ np.array(["2012-01-01 00:00:00+0000", "2012-01-02 00:00:00+0000"]),
+ None,
+ None,
+ )
+ assert rs[0] == xp
+
+ # we have a tz-aware date (constructed to that when we turn to utc it
+ # is the same as our sample)
+ ts = Timestamp("2012-01-01").tz_localize("UTC").tz_convert("US/Eastern")
+ rs = dtc.convert(ts, None, None)
+ assert rs == xp
+
+ rs = dtc.convert(ts.to_pydatetime(), None, None)
+ assert rs == xp
+
+ rs = dtc.convert(Index([ts - Day(1), ts]), None, None)
+ assert rs[1] == xp
+
+ rs = dtc.convert(Index([ts - Day(1), ts]).to_pydatetime(), None, None)
+ assert rs[1] == xp
+
+ def test_conversion_float(self, dtc):
+ rtol = 0.5 * 10**-9
+
+ rs = dtc.convert(Timestamp("2012-1-1 01:02:03", tz="UTC"), None, None)
+ xp = dates.date2num(Timestamp("2012-1-1 01:02:03", tz="UTC"))
+ tm.assert_almost_equal(rs, xp, rtol=rtol)
+
+ rs = dtc.convert(
+ Timestamp("2012-1-1 09:02:03", tz="Asia/Hong_Kong"), None, None
+ )
+ tm.assert_almost_equal(rs, xp, rtol=rtol)
+
+ rs = dtc.convert(datetime(2012, 1, 1, 1, 2, 3), None, None)
+ tm.assert_almost_equal(rs, xp, rtol=rtol)
+
+ @pytest.mark.parametrize(
+ "values",
+ [
+ [date(1677, 1, 1), date(1677, 1, 2)],
+ [datetime(1677, 1, 1, 12), datetime(1677, 1, 2, 12)],
+ ],
+ )
+ def test_conversion_outofbounds_datetime(self, dtc, values):
+ # 2579
+ rs = dtc.convert(values, None, None)
+ xp = dates.date2num(values)
+ tm.assert_numpy_array_equal(rs, xp)
+ rs = dtc.convert(values[0], None, None)
+ xp = dates.date2num(values[0])
+ assert rs == xp
+
+ @pytest.mark.parametrize(
+ "time,format_expected",
+ [
+ (0, "00:00"), # time2num(datetime.time.min)
+ (86399.999999, "23:59:59.999999"), # time2num(datetime.time.max)
+ (90000, "01:00"),
+ (3723, "01:02:03"),
+ (39723.2, "11:02:03.200"),
+ ],
+ )
+ def test_time_formatter(self, time, format_expected):
+ # issue 18478
+ result = converter.TimeFormatter(None)(time)
+ assert result == format_expected
+
+ @pytest.mark.parametrize("freq", ("B", "ms", "s"))
+ def test_dateindex_conversion(self, freq, dtc):
+ rtol = 10**-9
+ dateindex = date_range("2020-01-01", periods=10, freq=freq)
+ rs = dtc.convert(dateindex, None, None)
+ xp = dates.date2num(dateindex._mpl_repr())
+ tm.assert_almost_equal(rs, xp, rtol=rtol)
+
+ @pytest.mark.parametrize("offset", [Second(), Milli(), Micro(50)])
+ def test_resolution(self, offset, dtc):
+ # Matplotlib's time representation using floats cannot distinguish
+ # intervals smaller than ~10 microsecond in the common range of years.
+ ts1 = Timestamp("2012-1-1")
+ ts2 = ts1 + offset
+ val1 = dtc.convert(ts1, None, None)
+ val2 = dtc.convert(ts2, None, None)
+ if not val1 < val2:
+ raise AssertionError(f"{val1} is not less than {val2}.")
+
+ def test_convert_nested(self, dtc):
+ inner = [Timestamp("2017-01-01"), Timestamp("2017-01-02")]
+ data = [inner, inner]
+ result = dtc.convert(data, None, None)
+ expected = [dtc.convert(x, None, None) for x in data]
+ assert (np.array(result) == expected).all()
+
+
+class TestPeriodConverter:
+ @pytest.fixture
+ def pc(self):
+ return converter.PeriodConverter()
+
+ @pytest.fixture
+ def axis(self):
+ class Axis:
+ pass
+
+ axis = Axis()
+ axis.freq = "D"
+ return axis
+
+ def test_convert_accepts_unicode(self, pc, axis):
+ r1 = pc.convert("2012-1-1", None, axis)
+ r2 = pc.convert("2012-1-1", None, axis)
+ assert r1 == r2
+
+ def test_conversion(self, pc, axis):
+ rs = pc.convert(["2012-1-1"], None, axis)[0]
+ xp = Period("2012-1-1").ordinal
+ assert rs == xp
+
+ rs = pc.convert("2012-1-1", None, axis)
+ assert rs == xp
+
+ rs = pc.convert([date(2012, 1, 1)], None, axis)[0]
+ assert rs == xp
+
+ rs = pc.convert(date(2012, 1, 1), None, axis)
+ assert rs == xp
+
+ rs = pc.convert([Timestamp("2012-1-1")], None, axis)[0]
+ assert rs == xp
+
+ rs = pc.convert(Timestamp("2012-1-1"), None, axis)
+ assert rs == xp
+
+ rs = pc.convert("2012-01-01", None, axis)
+ assert rs == xp
+
+ rs = pc.convert("2012-01-01 00:00:00+0000", None, axis)
+ assert rs == xp
+
+ rs = pc.convert(
+ np.array(
+ ["2012-01-01 00:00:00", "2012-01-02 00:00:00"],
+ dtype="datetime64[ns]",
+ ),
+ None,
+ axis,
+ )
+ assert rs[0] == xp
+
+ def test_integer_passthrough(self, pc, axis):
+ # GH9012
+ rs = pc.convert([0, 1], None, axis)
+ xp = [0, 1]
+ assert rs == xp
+
+ def test_convert_nested(self, pc, axis):
+ data = ["2012-1-1", "2012-1-2"]
+ r1 = pc.convert([data, data], None, axis)
+ r2 = [pc.convert(data, None, axis) for _ in range(2)]
+ assert r1 == r2
+
+
+class TestTimeDeltaConverter:
+ """Test timedelta converter"""
+
+ @pytest.mark.parametrize(
+ "x, decimal, format_expected",
+ [
+ (0.0, 0, "00:00:00"),
+ (3972320000000, 1, "01:06:12.3"),
+ (713233432000000, 2, "8 days 06:07:13.43"),
+ (32423432000000, 4, "09:00:23.4320"),
+ ],
+ )
+ def test_format_timedelta_ticks(self, x, decimal, format_expected):
+ tdc = converter.TimeSeries_TimedeltaFormatter
+ result = tdc.format_timedelta_ticks(x, pos=None, n_decimals=decimal, exp=9)
+ assert result == format_expected
+
+ @pytest.mark.parametrize("view_interval", [(1, 2), (2, 1)])
+ def test_call_w_different_view_intervals(self, view_interval, monkeypatch):
+ # previously broke on reversed xlmits; see GH37454
+ class mock_axis:
+ def get_view_interval(self):
+ return view_interval
+
+ tdc = converter.TimeSeries_TimedeltaFormatter()
+ monkeypatch.setattr(tdc, "axis", mock_axis())
+ tdc(0.0, 0)
+
+
+@pytest.mark.parametrize("year_span", [11.25, 30, 80, 150, 400, 800, 1500, 2500, 3500])
+# The range is limited to 11.25 at the bottom by if statements in
+# the _quarterly_finder() function
+def test_quarterly_finder(year_span):
+ vmin = -1000
+ vmax = vmin + year_span * 4
+ span = vmax - vmin + 1
+ if span < 45:
+ pytest.skip("the quarterly finder is only invoked if the span is >= 45")
+ nyears = span / 4
+ (min_anndef, maj_anndef) = converter._get_default_annual_spacing(nyears)
+ result = converter._quarterly_finder(vmin, vmax, to_offset("QE"))
+ quarters = PeriodIndex(
+ arrays.PeriodArray(np.array([x[0] for x in result]), dtype="period[Q]")
+ )
+ majors = np.array([x[1] for x in result])
+ minors = np.array([x[2] for x in result])
+ major_quarters = quarters[majors]
+ minor_quarters = quarters[minors]
+ check_major_years = major_quarters.year % maj_anndef == 0
+ check_minor_years = minor_quarters.year % min_anndef == 0
+ check_major_quarters = major_quarters.quarter == 1
+ check_minor_quarters = minor_quarters.quarter == 1
+ assert np.all(check_major_years)
+ assert np.all(check_minor_years)
+ assert np.all(check_major_quarters)
+ assert np.all(check_minor_quarters)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_datetimelike.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_datetimelike.py
new file mode 100644
index 0000000000000000000000000000000000000000..fb845c6e6d71d0895c4424b2b54eb333c7b320ce
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_datetimelike.py
@@ -0,0 +1,1721 @@
+"""Test cases for time series specific (freq conversion, etc)"""
+
+from datetime import (
+ date,
+ datetime,
+ time,
+ timedelta,
+)
+import pickle
+
+import numpy as np
+import pytest
+
+from pandas._libs.tslibs import (
+ BaseOffset,
+ to_offset,
+)
+
+from pandas.core.dtypes.dtypes import PeriodDtype
+
+from pandas import (
+ DataFrame,
+ Index,
+ NaT,
+ Series,
+ concat,
+ isna,
+ to_datetime,
+)
+import pandas._testing as tm
+from pandas.core.indexes.datetimes import (
+ DatetimeIndex,
+ bdate_range,
+ date_range,
+)
+from pandas.core.indexes.period import (
+ Period,
+ PeriodIndex,
+ period_range,
+)
+from pandas.core.indexes.timedeltas import timedelta_range
+from pandas.tests.plotting.common import _check_ticks_props
+
+from pandas.tseries.offsets import WeekOfMonth
+
+mpl = pytest.importorskip("matplotlib")
+plt = pytest.importorskip("matplotlib.pyplot")
+
+import pandas.plotting._matplotlib.converter as conv
+
+
+class TestTSPlot:
+ @pytest.mark.filterwarnings("ignore::UserWarning")
+ def test_ts_plot_with_tz(self, tz_aware_fixture):
+ # GH2877, GH17173, GH31205, GH31580
+ tz = tz_aware_fixture
+ index = date_range("1/1/2011", periods=2, freq="h", tz=tz)
+ ts = Series([188.5, 328.25], index=index)
+ _check_plot_works(ts.plot)
+ ax = ts.plot()
+ xdata = next(iter(ax.get_lines())).get_xdata()
+ # Check first and last points' labels are correct
+ assert (xdata[0].hour, xdata[0].minute) == (0, 0)
+ assert (xdata[-1].hour, xdata[-1].minute) == (1, 0)
+
+ def test_fontsize_set_correctly(self):
+ # For issue #8765
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 9)), index=range(10)
+ )
+ _, ax = mpl.pyplot.subplots()
+ df.plot(fontsize=2, ax=ax)
+ for label in ax.get_xticklabels() + ax.get_yticklabels():
+ assert label.get_fontsize() == 2
+
+ def test_frame_inferred(self):
+ # inferred freq
+ idx = date_range("1/1/1987", freq="MS", periods=10)
+ idx = DatetimeIndex(idx.values, freq=None)
+
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((len(idx), 3)), index=idx
+ )
+ _check_plot_works(df.plot)
+
+ # axes freq
+ idx = idx[0:4].union(idx[6:])
+ df2 = DataFrame(
+ np.random.default_rng(2).standard_normal((len(idx), 3)), index=idx
+ )
+ _check_plot_works(df2.plot)
+
+ def test_frame_inferred_n_gt_1(self):
+ # N > 1
+ idx = date_range("2008-1-1 00:15:00", freq="15min", periods=10)
+ idx = DatetimeIndex(idx.values, freq=None)
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((len(idx), 3)), index=idx
+ )
+ _check_plot_works(df.plot)
+
+ def test_is_error_nozeroindex(self):
+ # GH11858
+ i = np.array([1, 2, 3])
+ a = DataFrame(i, index=i)
+ _check_plot_works(a.plot, xerr=a)
+ _check_plot_works(a.plot, yerr=a)
+
+ def test_nonnumeric_exclude(self):
+ idx = date_range("1/1/1987", freq="YE", periods=3)
+ df = DataFrame({"A": ["x", "y", "z"], "B": [1, 2, 3]}, idx)
+
+ fig, ax = mpl.pyplot.subplots()
+ df.plot(ax=ax) # it works
+ assert len(ax.get_lines()) == 1 # B was plotted
+
+ def test_nonnumeric_exclude_error(self):
+ idx = date_range("1/1/1987", freq="YE", periods=3)
+ df = DataFrame({"A": ["x", "y", "z"], "B": [1, 2, 3]}, idx)
+ msg = "no numeric data to plot"
+ with pytest.raises(TypeError, match=msg):
+ df["A"].plot()
+
+ @pytest.mark.parametrize("freq", ["s", "min", "h", "D", "W", "M", "Q", "Y"])
+ def test_tsplot_period(self, freq):
+ idx = period_range("12/31/1999", freq=freq, periods=10)
+ ser = Series(np.random.default_rng(2).standard_normal(len(idx)), idx)
+ _, ax = mpl.pyplot.subplots()
+ _check_plot_works(ser.plot, ax=ax)
+
+ @pytest.mark.parametrize(
+ "freq", ["s", "min", "h", "D", "W", "ME", "QE-DEC", "YE", "1B30Min"]
+ )
+ def test_tsplot_datetime(self, freq):
+ idx = date_range("12/31/1999", freq=freq, periods=10)
+ ser = Series(np.random.default_rng(2).standard_normal(len(idx)), idx)
+ _, ax = mpl.pyplot.subplots()
+ _check_plot_works(ser.plot, ax=ax)
+
+ def test_tsplot(self):
+ ts = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ _, ax = mpl.pyplot.subplots()
+ ts.plot(style="k", ax=ax)
+ color = (0.0, 0.0, 0.0, 1)
+ assert color == ax.get_lines()[0].get_color()
+
+ @pytest.mark.parametrize("index", [None, date_range("2020-01-01", periods=10)])
+ def test_both_style_and_color(self, index):
+ ts = Series(np.arange(10, dtype=np.float64), index=index)
+ msg = (
+ "Cannot pass 'style' string with a color symbol and 'color' "
+ "keyword argument. Please use one or the other or pass 'style' "
+ "without a color symbol"
+ )
+ with pytest.raises(ValueError, match=msg):
+ ts.plot(style="b-", color="#000099")
+
+ @pytest.mark.parametrize("freq", ["ms", "us"])
+ def test_high_freq(self, freq):
+ _, ax = mpl.pyplot.subplots()
+ rng = date_range("1/1/2012", periods=10, freq=freq)
+ ser = Series(np.random.default_rng(2).standard_normal(len(rng)), rng)
+ _check_plot_works(ser.plot, ax=ax)
+
+ def test_get_datevalue(self):
+ assert conv._get_datevalue(None, "D") is None
+ assert conv._get_datevalue(1987, "Y") == 1987
+ assert (
+ conv._get_datevalue(Period(1987, "Y"), "M")
+ == Period("1987-12", "M").ordinal
+ )
+ assert conv._get_datevalue("1/1/1987", "D") == Period("1987-1-1", "D").ordinal
+
+ @pytest.mark.parametrize(
+ "freq, expected_string",
+ [["YE-DEC", "t = 2014 y = 1.000000"], ["D", "t = 2014-01-01 y = 1.000000"]],
+ )
+ def test_ts_plot_format_coord(self, freq, expected_string):
+ ser = Series(1, index=date_range("2014-01-01", periods=3, freq=freq))
+ _, ax = mpl.pyplot.subplots()
+ ser.plot(ax=ax)
+ first_line = ax.get_lines()[0]
+ first_x = first_line.get_xdata()[0].ordinal
+ first_y = first_line.get_ydata()[0]
+ assert expected_string == ax.format_coord(first_x, first_y)
+
+ @pytest.mark.parametrize("freq", ["s", "min", "h", "D", "W", "M", "Q", "Y"])
+ def test_line_plot_period_series(self, freq):
+ idx = period_range("12/31/1999", freq=freq, periods=10)
+ ser = Series(np.random.default_rng(2).standard_normal(len(idx)), idx)
+ _check_plot_works(ser.plot, ser.index.freq)
+
+ @pytest.mark.parametrize(
+ "frqncy", ["1s", "3s", "5min", "7h", "4D", "8W", "11M", "3Y"]
+ )
+ def test_line_plot_period_mlt_series(self, frqncy):
+ # test period index line plot for series with multiples (`mlt`) of the
+ # frequency (`frqncy`) rule code. tests resolution of issue #14763
+ idx = period_range("12/31/1999", freq=frqncy, periods=10)
+ s = Series(np.random.default_rng(2).standard_normal(len(idx)), idx)
+ _check_plot_works(s.plot, s.index.freq.rule_code)
+
+ @pytest.mark.parametrize(
+ "freq", ["s", "min", "h", "D", "W", "ME", "QE-DEC", "YE", "1B30Min"]
+ )
+ def test_line_plot_datetime_series(self, freq):
+ idx = date_range("12/31/1999", freq=freq, periods=10)
+ ser = Series(np.random.default_rng(2).standard_normal(len(idx)), idx)
+ _check_plot_works(ser.plot, ser.index.freq.rule_code)
+
+ @pytest.mark.parametrize("freq", ["s", "min", "h", "D", "W", "ME", "QE", "YE"])
+ def test_line_plot_period_frame(self, freq):
+ idx = date_range("12/31/1999", freq=freq, periods=10)
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((len(idx), 3)),
+ index=idx,
+ columns=["A", "B", "C"],
+ )
+ _check_plot_works(df.plot, df.index.freq)
+
+ @pytest.mark.parametrize(
+ "frqncy", ["1s", "3s", "5min", "7h", "4D", "8W", "11M", "3Y"]
+ )
+ def test_line_plot_period_mlt_frame(self, frqncy):
+ # test period index line plot for DataFrames with multiples (`mlt`)
+ # of the frequency (`frqncy`) rule code. tests resolution of issue
+ # #14763
+ idx = period_range("12/31/1999", freq=frqncy, periods=10)
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((len(idx), 3)),
+ index=idx,
+ columns=["A", "B", "C"],
+ )
+ freq = df.index.freq.rule_code
+ _check_plot_works(df.plot, freq)
+
+ @pytest.mark.filterwarnings(r"ignore:PeriodDtype\[B\] is deprecated:FutureWarning")
+ @pytest.mark.parametrize(
+ "freq", ["s", "min", "h", "D", "W", "ME", "QE-DEC", "YE", "1B30Min"]
+ )
+ def test_line_plot_datetime_frame(self, freq):
+ idx = date_range("12/31/1999", freq=freq, periods=10)
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((len(idx), 3)),
+ index=idx,
+ columns=["A", "B", "C"],
+ )
+ freq = PeriodDtype(df.index.freq)._freqstr
+ freq = df.index.to_period(freq).freq
+ _check_plot_works(df.plot, freq)
+
+ @pytest.mark.parametrize(
+ "freq", ["s", "min", "h", "D", "W", "ME", "QE-DEC", "YE", "1B30Min"]
+ )
+ def test_line_plot_inferred_freq(self, freq):
+ idx = date_range("12/31/1999", freq=freq, periods=10)
+ ser = Series(np.random.default_rng(2).standard_normal(len(idx)), idx)
+ ser = Series(ser.values, Index(np.asarray(ser.index)))
+ _check_plot_works(ser.plot, ser.index.inferred_freq)
+
+ ser = ser.iloc[[0, 3, 5, 6]]
+ _check_plot_works(ser.plot)
+
+ def test_fake_inferred_business(self):
+ _, ax = mpl.pyplot.subplots()
+ rng = date_range("2001-1-1", "2001-1-10")
+ ts = Series(range(len(rng)), index=rng)
+ ts = concat([ts[:3], ts[5:]])
+ ts.plot(ax=ax)
+ assert not hasattr(ax, "freq")
+
+ def test_plot_offset_freq(self):
+ ser = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ _check_plot_works(ser.plot)
+
+ def test_plot_offset_freq_business(self):
+ dr = date_range("2023-01-01", freq="BQS", periods=10)
+ ser = Series(np.random.default_rng(2).standard_normal(len(dr)), index=dr)
+ _check_plot_works(ser.plot)
+
+ def test_plot_multiple_inferred_freq(self):
+ dr = Index([datetime(2000, 1, 1), datetime(2000, 1, 6), datetime(2000, 1, 11)])
+ ser = Series(np.random.default_rng(2).standard_normal(len(dr)), index=dr)
+ _check_plot_works(ser.plot)
+
+ def test_irreg_hf(self):
+ idx = date_range("2012-6-22 21:59:51", freq="s", periods=10)
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((len(idx), 2)), index=idx
+ )
+
+ irreg = df.iloc[[0, 1, 3, 4]]
+ _, ax = mpl.pyplot.subplots()
+ irreg.plot(ax=ax)
+ diffs = Series(ax.get_lines()[0].get_xydata()[:, 0]).diff()
+
+ sec = 1.0 / 24 / 60 / 60
+ assert (np.fabs(diffs[1:] - [sec, sec * 2, sec]) < 1e-8).all()
+
+ def test_irreg_hf_object(self):
+ idx = date_range("2012-6-22 21:59:51", freq="s", periods=10)
+ df2 = DataFrame(
+ np.random.default_rng(2).standard_normal((len(idx), 2)), index=idx
+ )
+ _, ax = mpl.pyplot.subplots()
+ df2.index = df2.index.astype(object)
+ df2.plot(ax=ax)
+ diffs = Series(ax.get_lines()[0].get_xydata()[:, 0]).diff()
+ sec = 1.0 / 24 / 60 / 60
+ assert (np.fabs(diffs[1:] - sec) < 1e-8).all()
+
+ def test_irregular_datetime64_repr_bug(self):
+ ser = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ ser = ser.iloc[[0, 1, 2, 7]]
+
+ _, ax = mpl.pyplot.subplots()
+
+ ret = ser.plot(ax=ax)
+ assert ret is not None
+
+ for rs, xp in zip(ax.get_lines()[0].get_xdata(), ser.index, strict=True):
+ assert rs == xp
+
+ def test_business_freq(self):
+ bts = Series(range(5), period_range("2020-01-01", periods=5))
+ msg = r"PeriodDtype\[B\] is deprecated"
+ dt = bts.index[0].to_timestamp()
+ with tm.assert_produces_warning(FutureWarning, match=msg):
+ bts.index = period_range(start=dt, periods=len(bts), freq="B")
+ _, ax = mpl.pyplot.subplots()
+ bts.plot(ax=ax)
+ assert ax.get_lines()[0].get_xydata()[0, 0] == bts.index[0].ordinal
+ idx = ax.get_lines()[0].get_xdata()
+ with tm.assert_produces_warning(FutureWarning, match=msg):
+ assert PeriodIndex(data=idx).freqstr == "B"
+
+ def test_business_freq_convert(self):
+ bts = Series(
+ np.arange(50, dtype=np.float64),
+ index=date_range("2020-01-01", periods=50, freq="B"),
+ ).asfreq("BME")
+ ts = bts.to_period("M")
+ _, ax = mpl.pyplot.subplots()
+ bts.plot(ax=ax)
+ assert ax.get_lines()[0].get_xydata()[0, 0] == ts.index[0].ordinal
+ idx = ax.get_lines()[0].get_xdata()
+ assert PeriodIndex(data=idx).freqstr == "M"
+
+ def test_freq_with_no_period_alias(self):
+ # GH34487
+ freq = WeekOfMonth()
+ bts = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ ).asfreq(freq)
+ _, ax = mpl.pyplot.subplots()
+ bts.plot(ax=ax)
+
+ idx = ax.get_lines()[0].get_xdata()
+ msg = "freq not specified and cannot be inferred"
+ with pytest.raises(ValueError, match=msg):
+ PeriodIndex(data=idx)
+
+ def test_nonzero_base(self):
+ # GH2571
+ idx = date_range("2012-12-20", periods=24, freq="h") + timedelta(minutes=30)
+ df = DataFrame(np.arange(24), index=idx)
+ _, ax = mpl.pyplot.subplots()
+ df.plot(ax=ax)
+ rs = ax.get_lines()[0].get_xdata()
+ assert not Index(rs).is_normalized
+
+ def test_dataframe(self):
+ bts = DataFrame(
+ {
+ "a": Series(
+ np.arange(10, dtype=np.float64),
+ index=date_range("2020-01-01", periods=10),
+ )
+ }
+ )
+ _, ax = mpl.pyplot.subplots()
+ bts.plot(ax=ax)
+ idx = ax.get_lines()[0].get_xdata()
+ tm.assert_index_equal(bts.index.to_period(), PeriodIndex(idx))
+
+ @pytest.mark.filterwarnings(
+ "ignore:Period with BDay freq is deprecated:FutureWarning"
+ )
+ @pytest.mark.parametrize(
+ "obj",
+ [
+ Series(
+ np.arange(10, dtype=np.float64),
+ index=date_range("2020-01-01", periods=10),
+ ),
+ DataFrame(
+ {
+ "a": Series(
+ np.arange(10, dtype=np.float64),
+ index=date_range("2020-01-01", periods=10),
+ ),
+ "b": Series(
+ np.arange(10, dtype=np.float64),
+ index=date_range("2020-01-01", periods=10),
+ )
+ + 1,
+ }
+ ),
+ ],
+ )
+ def test_axis_limits(self, obj):
+ _, ax = mpl.pyplot.subplots()
+ obj.plot(ax=ax)
+ xlim = ax.get_xlim()
+ ax.set_xlim(xlim[0] - 5, xlim[1] + 10)
+ result = ax.get_xlim()
+ assert result[0] == xlim[0] - 5
+ assert result[1] == xlim[1] + 10
+
+ # string
+ expected = (Period("1/1/2000", ax.freq), Period("4/1/2000", ax.freq))
+ ax.set_xlim("1/1/2000", "4/1/2000")
+ result = ax.get_xlim()
+ assert int(result[0]) == expected[0].ordinal
+ assert int(result[1]) == expected[1].ordinal
+
+ # datetime
+ expected = (Period("1/1/2000", ax.freq), Period("4/1/2000", ax.freq))
+ ax.set_xlim(datetime(2000, 1, 1), datetime(2000, 4, 1))
+ result = ax.get_xlim()
+ assert int(result[0]) == expected[0].ordinal
+ assert int(result[1]) == expected[1].ordinal
+
+ def test_get_finder(self):
+ assert conv.get_finder(to_offset("B")) == conv._daily_finder
+ assert conv.get_finder(to_offset("D")) == conv._daily_finder
+ assert conv.get_finder(to_offset("ME")) == conv._monthly_finder
+ assert conv.get_finder(to_offset("QE")) == conv._quarterly_finder
+ assert conv.get_finder(to_offset("YE")) == conv._annual_finder
+ assert conv.get_finder(to_offset("W")) == conv._daily_finder
+
+ def test_finder_daily(self):
+ day_lst = [10, 40, 252, 400, 950, 2750, 10000]
+
+ msg = "Period with BDay freq is deprecated"
+ with tm.assert_produces_warning(FutureWarning, match=msg):
+ xpl1 = xpl2 = [Period("1999-1-1", freq="B").ordinal] * len(day_lst)
+ rs1 = []
+ rs2 = []
+ for n in day_lst:
+ rng = bdate_range("1999-1-1", periods=n)
+ ser = Series(np.random.default_rng(2).standard_normal(len(rng)), rng)
+ _, ax = mpl.pyplot.subplots()
+ ser.plot(ax=ax)
+ xaxis = ax.get_xaxis()
+ rs1.append(xaxis.get_majorticklocs()[0])
+
+ vmin, vmax = ax.get_xlim()
+ ax.set_xlim(vmin + 0.9, vmax)
+ rs2.append(xaxis.get_majorticklocs()[0])
+ mpl.pyplot.close(ax.get_figure())
+
+ assert rs1 == xpl1
+ assert rs2 == xpl2
+
+ def test_finder_quarterly(self):
+ yrs = [3.5, 11]
+
+ xpl1 = xpl2 = [Period("1988Q1").ordinal] * len(yrs)
+ rs1 = []
+ rs2 = []
+ for n in yrs:
+ rng = period_range("1987Q2", periods=int(n * 4), freq="Q")
+ ser = Series(np.random.default_rng(2).standard_normal(len(rng)), rng)
+ _, ax = mpl.pyplot.subplots()
+ ser.plot(ax=ax)
+ xaxis = ax.get_xaxis()
+ rs1.append(xaxis.get_majorticklocs()[0])
+
+ (vmin, vmax) = ax.get_xlim()
+ ax.set_xlim(vmin + 0.9, vmax)
+ rs2.append(xaxis.get_majorticklocs()[0])
+ mpl.pyplot.close(ax.get_figure())
+
+ assert rs1 == xpl1
+ assert rs2 == xpl2
+
+ def test_finder_monthly(self):
+ yrs = [1.15, 2.5, 4, 11]
+
+ xpl1 = xpl2 = [Period("Jan 1988").ordinal] * len(yrs)
+ rs1 = []
+ rs2 = []
+ for n in yrs:
+ rng = period_range("1987Q2", periods=int(n * 12), freq="M")
+ ser = Series(np.random.default_rng(2).standard_normal(len(rng)), rng)
+ _, ax = mpl.pyplot.subplots()
+ ser.plot(ax=ax)
+ xaxis = ax.get_xaxis()
+ rs1.append(xaxis.get_majorticklocs()[0])
+
+ vmin, vmax = ax.get_xlim()
+ ax.set_xlim(vmin + 0.9, vmax)
+ rs2.append(xaxis.get_majorticklocs()[0])
+ mpl.pyplot.close(ax.get_figure())
+
+ assert rs1 == xpl1
+ assert rs2 == xpl2
+
+ def test_finder_monthly_long(self):
+ rng = period_range("1988Q1", periods=24 * 12, freq="M")
+ ser = Series(np.random.default_rng(2).standard_normal(len(rng)), rng)
+ _, ax = mpl.pyplot.subplots()
+ ser.plot(ax=ax)
+ xaxis = ax.get_xaxis()
+ rs = xaxis.get_majorticklocs()[0]
+ xp = Period("1989Q1", "M").ordinal
+ assert rs == xp
+
+ def test_finder_annual(self):
+ xp = [1987, 1988, 1990, 1990, 1995, 2020, 2070, 2170]
+ xp = [Period(x, freq="Y").ordinal for x in xp]
+ rs = []
+ for nyears in [5, 10, 19, 49, 99, 199, 599, 1001]:
+ rng = period_range("1987", periods=nyears, freq="Y")
+ ser = Series(np.random.default_rng(2).standard_normal(len(rng)), rng)
+ _, ax = mpl.pyplot.subplots()
+ ser.plot(ax=ax)
+ xaxis = ax.get_xaxis()
+ rs.append(xaxis.get_majorticklocs()[0])
+ mpl.pyplot.close(ax.get_figure())
+
+ assert rs == xp
+
+ @pytest.mark.slow
+ def test_finder_minutely(self):
+ nminutes = 1 * 24 * 60
+ rng = date_range("1/1/1999", freq="Min", periods=nminutes)
+ ser = Series(np.random.default_rng(2).standard_normal(len(rng)), rng)
+ _, ax = mpl.pyplot.subplots()
+ ser.plot(ax=ax)
+ xaxis = ax.get_xaxis()
+ rs = xaxis.get_majorticklocs()[0]
+ xp = Period("1/1/1999", freq="Min").ordinal
+
+ assert rs == xp
+
+ def test_finder_hourly(self):
+ nhours = 23
+ rng = date_range("1/1/1999", freq="h", periods=nhours)
+ ser = Series(np.random.default_rng(2).standard_normal(len(rng)), rng)
+ _, ax = mpl.pyplot.subplots()
+ ser.plot(ax=ax)
+ xaxis = ax.get_xaxis()
+ rs = xaxis.get_majorticklocs()[0]
+ xp = Period("1/1/1999", freq="h").ordinal
+
+ assert rs == xp
+
+ def test_gaps(self):
+ ts = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ ts.iloc[5:7] = np.nan
+ _, ax = mpl.pyplot.subplots()
+ ts.plot(ax=ax)
+ lines = ax.get_lines()
+ assert len(lines) == 1
+ line = lines[0]
+ data = line.get_xydata()
+
+ data = np.ma.MaskedArray(data, mask=isna(data), fill_value=np.nan)
+
+ assert isinstance(data, np.ma.core.MaskedArray)
+ mask = data.mask
+ assert mask[5:7, 1].all()
+
+ def test_gaps_irregular(self):
+ # irregular
+ ts = Series(
+ np.arange(30, dtype=np.float64), index=date_range("2020-01-01", periods=30)
+ )
+ ts = ts.iloc[[0, 1, 2, 5, 7, 9, 12, 15, 20]]
+ ts.iloc[2:5] = np.nan
+ _, ax = mpl.pyplot.subplots()
+ ax = ts.plot(ax=ax)
+ lines = ax.get_lines()
+ assert len(lines) == 1
+ line = lines[0]
+ data = line.get_xydata()
+
+ data = np.ma.MaskedArray(data, mask=isna(data), fill_value=np.nan)
+
+ assert isinstance(data, np.ma.core.MaskedArray)
+ mask = data.mask
+ assert mask[2:5, 1].all()
+
+ def test_gaps_non_ts(self):
+ # non-ts
+ idx = [0, 1, 2, 5, 7, 9, 12, 15, 20]
+ ser = Series(np.random.default_rng(2).standard_normal(len(idx)), idx)
+ ser.iloc[2:5] = np.nan
+ _, ax = mpl.pyplot.subplots()
+ ser.plot(ax=ax)
+ lines = ax.get_lines()
+ assert len(lines) == 1
+ line = lines[0]
+ data = line.get_xydata()
+ data = np.ma.MaskedArray(data, mask=isna(data), fill_value=np.nan)
+
+ assert isinstance(data, np.ma.core.MaskedArray)
+ mask = data.mask
+ assert mask[2:5, 1].all()
+
+ def test_gap_upsample(self):
+ low = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ low.iloc[5:7] = np.nan
+ _, ax = mpl.pyplot.subplots()
+ low.plot(ax=ax)
+
+ idxh = date_range(low.index[0], low.index[-1], freq="12h")
+ s = Series(np.random.default_rng(2).standard_normal(len(idxh)), idxh)
+ s.plot(secondary_y=True)
+ lines = ax.get_lines()
+ assert len(lines) == 1
+ assert len(ax.right_ax.get_lines()) == 1
+
+ line = lines[0]
+ data = line.get_xydata()
+ data = np.ma.MaskedArray(data, mask=isna(data), fill_value=np.nan)
+
+ assert isinstance(data, np.ma.core.MaskedArray)
+ mask = data.mask
+ assert mask[5:7, 1].all()
+
+ def test_secondary_y(self):
+ ser = Series(np.random.default_rng(2).standard_normal(10))
+ fig, _ = mpl.pyplot.subplots()
+ ax = ser.plot(secondary_y=True)
+ assert hasattr(ax, "left_ax")
+ assert not hasattr(ax, "right_ax")
+ axes = fig.get_axes()
+ line = ax.get_lines()[0]
+ xp = Series(line.get_ydata(), line.get_xdata())
+ tm.assert_series_equal(ser, xp)
+ assert ax.get_yaxis().get_ticks_position() == "right"
+ assert not axes[0].get_yaxis().get_visible()
+
+ def test_secondary_y_yaxis(self):
+ Series(np.random.default_rng(2).standard_normal(10))
+ ser2 = Series(np.random.default_rng(2).standard_normal(10))
+ _, ax2 = mpl.pyplot.subplots()
+ ser2.plot(ax=ax2)
+ assert ax2.get_yaxis().get_ticks_position() == "left"
+
+ def test_secondary_both(self):
+ ser = Series(np.random.default_rng(2).standard_normal(10))
+ ser2 = Series(np.random.default_rng(2).standard_normal(10))
+ ax = ser2.plot()
+ ax2 = ser.plot(secondary_y=True)
+ assert ax.get_yaxis().get_visible()
+ assert not hasattr(ax, "left_ax")
+ assert hasattr(ax, "right_ax")
+ assert hasattr(ax2, "left_ax")
+ assert not hasattr(ax2, "right_ax")
+
+ def test_secondary_y_ts(self):
+ idx = date_range("1/1/2000", periods=10, unit="ns")
+ ser = Series(np.random.default_rng(2).standard_normal(10), idx)
+ fig, _ = mpl.pyplot.subplots()
+ ax = ser.plot(secondary_y=True)
+ assert hasattr(ax, "left_ax")
+ assert not hasattr(ax, "right_ax")
+ axes = fig.get_axes()
+ line = ax.get_lines()[0]
+ xp = Series(line.get_ydata(), line.get_xdata()).to_timestamp()
+ xp.index = xp.index.as_unit("ns")
+ tm.assert_series_equal(ser, xp)
+ assert ax.get_yaxis().get_ticks_position() == "right"
+ assert not axes[0].get_yaxis().get_visible()
+
+ def test_secondary_y_ts_yaxis(self):
+ idx = date_range("1/1/2000", periods=10)
+ ser2 = Series(np.random.default_rng(2).standard_normal(10), idx)
+ _, ax2 = mpl.pyplot.subplots()
+ ser2.plot(ax=ax2)
+ assert ax2.get_yaxis().get_ticks_position() == "left"
+
+ def test_secondary_y_ts_visible(self):
+ idx = date_range("1/1/2000", periods=10)
+ ser2 = Series(np.random.default_rng(2).standard_normal(10), idx)
+ ax = ser2.plot()
+ assert ax.get_yaxis().get_visible()
+
+ def test_secondary_kde(self):
+ pytest.importorskip("scipy")
+ ser = Series(np.random.default_rng(2).standard_normal(10))
+ fig, ax = mpl.pyplot.subplots()
+ ax = ser.plot(secondary_y=True, kind="density", ax=ax)
+ assert hasattr(ax, "left_ax")
+ assert not hasattr(ax, "right_ax")
+ axes = fig.get_axes()
+ assert axes[1].get_yaxis().get_ticks_position() == "right"
+
+ def test_secondary_bar(self):
+ ser = Series(np.random.default_rng(2).standard_normal(10))
+ fig, ax = mpl.pyplot.subplots()
+ ser.plot(secondary_y=True, kind="bar", ax=ax)
+ axes = fig.get_axes()
+ assert axes[1].get_yaxis().get_ticks_position() == "right"
+
+ def test_secondary_frame(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((5, 3)), columns=["a", "b", "c"]
+ )
+ axes = df.plot(secondary_y=["a", "c"], subplots=True)
+ assert axes[0].get_yaxis().get_ticks_position() == "right"
+ assert axes[1].get_yaxis().get_ticks_position() == "left"
+ assert axes[2].get_yaxis().get_ticks_position() == "right"
+
+ def test_secondary_bar_frame(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((5, 3)), columns=["a", "b", "c"]
+ )
+ axes = df.plot(kind="bar", secondary_y=["a", "c"], subplots=True)
+ assert axes[0].get_yaxis().get_ticks_position() == "right"
+ assert axes[1].get_yaxis().get_ticks_position() == "left"
+ assert axes[2].get_yaxis().get_ticks_position() == "right"
+
+ def test_mixed_freq_regular_first(self):
+ # TODO
+ s1 = Series(
+ np.arange(20, dtype=np.float64),
+ index=date_range("2020-01-01", periods=20, freq="B"),
+ )
+ s2 = s1.iloc[[0, 5, 10, 11, 12, 13, 14, 15]]
+
+ # it works!
+ _, ax = mpl.pyplot.subplots()
+ s1.plot(ax=ax)
+
+ ax2 = s2.plot(style="g", ax=ax)
+ lines = ax2.get_lines()
+ msg = r"PeriodDtype\[B\] is deprecated"
+ with tm.assert_produces_warning(FutureWarning, match=msg):
+ idx1 = PeriodIndex(lines[0].get_xdata())
+ idx2 = PeriodIndex(lines[1].get_xdata())
+
+ tm.assert_index_equal(idx1, s1.index.to_period("B"))
+ tm.assert_index_equal(idx2, s2.index.to_period("B"))
+
+ left, right = ax2.get_xlim()
+ pidx = s1.index.to_period()
+ assert left <= pidx[0].ordinal
+ assert right >= pidx[-1].ordinal
+
+ def test_mixed_freq_irregular_first(self):
+ s1 = Series(
+ np.arange(20, dtype=np.float64), index=date_range("2020-01-01", periods=20)
+ )
+ s2 = s1.iloc[[0, 5, 10, 11, 12, 13, 14, 15]]
+ _, ax = mpl.pyplot.subplots()
+ s2.plot(style="g", ax=ax)
+ s1.plot(ax=ax)
+ assert not hasattr(ax, "freq")
+ lines = ax.get_lines()
+ x1 = lines[0].get_xdata()
+ tm.assert_numpy_array_equal(x1, s2.index.astype(object).values)
+ x2 = lines[1].get_xdata()
+ tm.assert_numpy_array_equal(x2, s1.index.astype(object).values)
+
+ def test_mixed_freq_regular_first_df(self):
+ # GH 9852
+ s1 = Series(
+ np.arange(20, dtype=np.float64),
+ index=date_range("2020-01-01", periods=20, freq="B"),
+ ).to_frame()
+ s2 = s1.iloc[[0, 5, 10, 11, 12, 13, 14, 15], :]
+ _, ax = mpl.pyplot.subplots()
+ s1.plot(ax=ax)
+ ax2 = s2.plot(style="g", ax=ax)
+ lines = ax2.get_lines()
+ msg = r"PeriodDtype\[B\] is deprecated"
+ with tm.assert_produces_warning(FutureWarning, match=msg):
+ idx1 = PeriodIndex(lines[0].get_xdata())
+ idx2 = PeriodIndex(lines[1].get_xdata())
+ assert idx1.equals(s1.index.to_period("B"))
+ assert idx2.equals(s2.index.to_period("B"))
+ left, right = ax2.get_xlim()
+ pidx = s1.index.to_period()
+ assert left <= pidx[0].ordinal
+ assert right >= pidx[-1].ordinal
+
+ def test_mixed_freq_irregular_first_df(self):
+ # GH 9852
+ s1 = Series(
+ np.arange(20, dtype=np.float64), index=date_range("2020-01-01", periods=20)
+ ).to_frame()
+ s2 = s1.iloc[[0, 5, 10, 11, 12, 13, 14, 15], :]
+ _, ax = mpl.pyplot.subplots()
+ s2.plot(style="g", ax=ax)
+ s1.plot(ax=ax)
+ assert not hasattr(ax, "freq")
+ lines = ax.get_lines()
+ x1 = lines[0].get_xdata()
+ tm.assert_numpy_array_equal(x1, s2.index.astype(object).values)
+ x2 = lines[1].get_xdata()
+ tm.assert_numpy_array_equal(x2, s1.index.astype(object).values)
+
+ def test_mixed_freq_hf_first(self):
+ idxh = date_range("1/1/1999", periods=365, freq="D")
+ idxl = date_range("1/1/1999", periods=12, freq="ME")
+ high = Series(np.random.default_rng(2).standard_normal(len(idxh)), idxh)
+ low = Series(np.random.default_rng(2).standard_normal(len(idxl)), idxl)
+ _, ax = mpl.pyplot.subplots()
+ high.plot(ax=ax)
+ low.plot(ax=ax)
+ for line in ax.get_lines():
+ assert PeriodIndex(data=line.get_xdata()).freq == "D"
+
+ def test_mixed_freq_alignment(self):
+ ts_ind = date_range("2012-01-01 13:00", "2012-01-02", freq="h")
+ ts_data = np.random.default_rng(2).standard_normal(12)
+
+ ts = Series(ts_data, index=ts_ind)
+ ts2 = ts.asfreq("min").interpolate()
+
+ _, ax = mpl.pyplot.subplots()
+ ax = ts.plot(ax=ax)
+ ts2.plot(style="r", ax=ax)
+
+ assert ax.lines[0].get_xdata()[0] == ax.lines[1].get_xdata()[0]
+
+ def test_mixed_freq_lf_first(self):
+ idxh = date_range("1/1/1999", periods=365, freq="D")
+ idxl = date_range("1/1/1999", periods=12, freq="ME")
+ high = Series(np.random.default_rng(2).standard_normal(len(idxh)), idxh)
+ low = Series(np.random.default_rng(2).standard_normal(len(idxl)), idxl)
+ _, ax = mpl.pyplot.subplots()
+ low.plot(legend=True, ax=ax)
+ high.plot(legend=True, ax=ax)
+ for line in ax.get_lines():
+ assert PeriodIndex(data=line.get_xdata()).freq == "D"
+ leg = ax.get_legend()
+ assert len(leg.texts) == 2
+ mpl.pyplot.close(ax.get_figure())
+
+ def test_mixed_freq_lf_first_hourly(self):
+ idxh = date_range("1/1/1999", periods=240, freq="min")
+ idxl = date_range("1/1/1999", periods=4, freq="h")
+ high = Series(np.random.default_rng(2).standard_normal(len(idxh)), idxh)
+ low = Series(np.random.default_rng(2).standard_normal(len(idxl)), idxl)
+ _, ax = mpl.pyplot.subplots()
+ low.plot(ax=ax)
+ high.plot(ax=ax)
+ for line in ax.get_lines():
+ assert PeriodIndex(data=line.get_xdata()).freq == "min"
+
+ @pytest.mark.filterwarnings(r"ignore:PeriodDtype\[B\] is deprecated:FutureWarning")
+ def test_mixed_freq_irreg_period(self):
+ ts = Series(
+ np.arange(30, dtype=np.float64), index=date_range("2020-01-01", periods=30)
+ )
+ irreg = ts.iloc[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 17, 18, 29]]
+ msg = r"PeriodDtype\[B\] is deprecated"
+ with tm.assert_produces_warning(FutureWarning, match=msg):
+ rng = period_range("1/3/2000", periods=30, freq="B")
+ ps = Series(np.random.default_rng(2).standard_normal(len(rng)), rng)
+ _, ax = mpl.pyplot.subplots()
+ irreg.plot(ax=ax)
+ ps.plot(ax=ax)
+
+ def test_mixed_freq_shared_ax(self):
+ # GH13341, using sharex=True
+ idx1 = date_range("2015-01-01", periods=3, freq="ME")
+ idx2 = idx1[:1].union(idx1[2:])
+ s1 = Series(range(len(idx1)), idx1)
+ s2 = Series(range(len(idx2)), idx2)
+
+ _, (ax1, ax2) = mpl.pyplot.subplots(nrows=2, sharex=True)
+ s1.plot(ax=ax1)
+ s2.plot(ax=ax2)
+
+ assert ax1.freq == "M"
+ assert ax2.freq == "M"
+ assert ax1.lines[0].get_xydata()[0, 0] == ax2.lines[0].get_xydata()[0, 0]
+
+ def test_mixed_freq_shared_ax_twin_x(self):
+ # GH13341, using sharex=True
+ idx1 = date_range("2015-01-01", periods=3, freq="ME")
+ idx2 = idx1[:1].union(idx1[2:])
+ s1 = Series(range(len(idx1)), idx1)
+ s2 = Series(range(len(idx2)), idx2)
+ # using twinx
+ _, ax1 = mpl.pyplot.subplots()
+ ax2 = ax1.twinx()
+ s1.plot(ax=ax1)
+ s2.plot(ax=ax2)
+
+ assert ax1.lines[0].get_xydata()[0, 0] == ax2.lines[0].get_xydata()[0, 0]
+
+ @pytest.mark.xfail(reason="TODO (GH14330, GH14322)")
+ def test_mixed_freq_shared_ax_twin_x_irregular_first(self):
+ # GH13341, using sharex=True
+ idx1 = date_range("2015-01-01", periods=3, freq="ME")
+ idx2 = idx1[:1].union(idx1[2:])
+ s1 = Series(range(len(idx1)), idx1)
+ s2 = Series(range(len(idx2)), idx2)
+ _, ax1 = mpl.pyplot.subplots()
+ ax2 = ax1.twinx()
+ s2.plot(ax=ax1)
+ s1.plot(ax=ax2)
+ assert ax1.lines[0].get_xydata()[0, 0] == ax2.lines[0].get_xydata()[0, 0]
+
+ def test_nat_handling(self):
+ _, ax = mpl.pyplot.subplots()
+
+ dti = DatetimeIndex(["2015-01-01", NaT, "2015-01-03"])
+ s = Series(range(len(dti)), dti)
+ s.plot(ax=ax)
+ xdata = ax.get_lines()[0].get_xdata()
+ # plot x data is bounded by index values
+ assert s.index.min() <= Series(xdata).min()
+ assert Series(xdata).max() <= s.index.max()
+
+ def test_to_weekly_resampling_disallow_how_kwd(self):
+ idxh = date_range("1/1/1999", periods=52, freq="W")
+ idxl = date_range("1/1/1999", periods=12, freq="ME")
+ high = Series(np.random.default_rng(2).standard_normal(len(idxh)), idxh)
+ low = Series(np.random.default_rng(2).standard_normal(len(idxl)), idxl)
+ _, ax = mpl.pyplot.subplots()
+ high.plot(ax=ax)
+
+ msg = (
+ "'how' is not a valid keyword for plotting functions. If plotting "
+ "multiple objects on shared axes, resample manually first."
+ )
+ with pytest.raises(ValueError, match=msg):
+ low.plot(ax=ax, how="foo")
+
+ def test_to_weekly_resampling(self):
+ idxh = date_range("1/1/1999", periods=52, freq="W")
+ idxl = date_range("1/1/1999", periods=12, freq="ME")
+ high = Series(np.random.default_rng(2).standard_normal(len(idxh)), idxh)
+ low = Series(np.random.default_rng(2).standard_normal(len(idxl)), idxl)
+ _, ax = mpl.pyplot.subplots()
+ high.plot(ax=ax)
+ low.plot(ax=ax)
+ for line in ax.get_lines():
+ assert PeriodIndex(data=line.get_xdata()).freq == idxh.freq
+
+ def test_from_weekly_resampling(self):
+ idxh = date_range("1/1/1999", periods=52, freq="W")
+ idxl = date_range("1/1/1999", periods=12, freq="ME")
+ high = Series(np.random.default_rng(2).standard_normal(len(idxh)), idxh)
+ low = Series(np.random.default_rng(2).standard_normal(len(idxl)), idxl)
+ _, ax = mpl.pyplot.subplots()
+ low.plot(ax=ax)
+ high.plot(ax=ax)
+
+ expected_h = idxh.to_period().asi8.astype(np.float64)
+ expected_l = np.array(
+ [1514, 1519, 1523, 1527, 1531, 1536, 1540, 1544, 1549, 1553, 1558, 1562],
+ dtype=np.float64,
+ )
+ for line in ax.get_lines():
+ assert PeriodIndex(data=line.get_xdata()).freq == idxh.freq
+ xdata = line.get_xdata(orig=False)
+ if len(xdata) == 12: # idxl lines
+ tm.assert_numpy_array_equal(xdata, expected_l)
+ else:
+ tm.assert_numpy_array_equal(xdata, expected_h)
+
+ @pytest.mark.parametrize("kind1, kind2", [("line", "area"), ("area", "line")])
+ def test_from_resampling_area_line_mixed(self, kind1, kind2):
+ idxh = date_range("1/1/1999", periods=52, freq="W")
+ idxl = date_range("1/1/1999", periods=12, freq="ME")
+ high = DataFrame(
+ np.random.default_rng(2).random((len(idxh), 3)),
+ index=idxh,
+ columns=[0, 1, 2],
+ )
+ low = DataFrame(
+ np.random.default_rng(2).random((len(idxl), 3)),
+ index=idxl,
+ columns=[0, 1, 2],
+ )
+
+ _, ax = mpl.pyplot.subplots()
+ low.plot(kind=kind1, stacked=True, ax=ax)
+ high.plot(kind=kind2, stacked=True, ax=ax)
+
+ # check low dataframe result
+ expected_x = np.array(
+ [
+ 1514,
+ 1519,
+ 1523,
+ 1527,
+ 1531,
+ 1536,
+ 1540,
+ 1544,
+ 1549,
+ 1553,
+ 1558,
+ 1562,
+ ],
+ dtype=np.float64,
+ )
+ expected_y = np.zeros(len(expected_x), dtype=np.float64)
+ for i in range(3):
+ line = ax.lines[i]
+ assert PeriodIndex(line.get_xdata()).freq == idxh.freq
+ tm.assert_numpy_array_equal(line.get_xdata(orig=False), expected_x)
+ # check stacked values are correct
+ expected_y += low[i].values
+ tm.assert_numpy_array_equal(line.get_ydata(orig=False), expected_y)
+
+ # check high dataframe result
+ expected_x = idxh.to_period().asi8.astype(np.float64)
+ expected_y = np.zeros(len(expected_x), dtype=np.float64)
+ for i in range(3):
+ line = ax.lines[3 + i]
+ assert PeriodIndex(data=line.get_xdata()).freq == idxh.freq
+ tm.assert_numpy_array_equal(line.get_xdata(orig=False), expected_x)
+ expected_y += high[i].values
+ tm.assert_numpy_array_equal(line.get_ydata(orig=False), expected_y)
+
+ @pytest.mark.parametrize("kind1, kind2", [("line", "area"), ("area", "line")])
+ def test_from_resampling_area_line_mixed_high_to_low(self, kind1, kind2):
+ idxh = date_range("1/1/1999", periods=52, freq="W")
+ idxl = date_range("1/1/1999", periods=12, freq="ME")
+ high = DataFrame(
+ np.random.default_rng(2).random((len(idxh), 3)),
+ index=idxh,
+ columns=[0, 1, 2],
+ )
+ low = DataFrame(
+ np.random.default_rng(2).random((len(idxl), 3)),
+ index=idxl,
+ columns=[0, 1, 2],
+ )
+ _, ax = mpl.pyplot.subplots()
+ high.plot(kind=kind1, stacked=True, ax=ax)
+ low.plot(kind=kind2, stacked=True, ax=ax)
+
+ # check high dataframe result
+ expected_x = idxh.to_period().asi8.astype(np.float64)
+ expected_y = np.zeros(len(expected_x), dtype=np.float64)
+ for i in range(3):
+ line = ax.lines[i]
+ assert PeriodIndex(data=line.get_xdata()).freq == idxh.freq
+ tm.assert_numpy_array_equal(line.get_xdata(orig=False), expected_x)
+ expected_y += high[i].values
+ tm.assert_numpy_array_equal(line.get_ydata(orig=False), expected_y)
+
+ # check low dataframe result
+ expected_x = np.array(
+ [
+ 1514,
+ 1519,
+ 1523,
+ 1527,
+ 1531,
+ 1536,
+ 1540,
+ 1544,
+ 1549,
+ 1553,
+ 1558,
+ 1562,
+ ],
+ dtype=np.float64,
+ )
+ expected_y = np.zeros(len(expected_x), dtype=np.float64)
+ for i in range(3):
+ lines = ax.lines[3 + i]
+ assert PeriodIndex(data=lines.get_xdata()).freq == idxh.freq
+ tm.assert_numpy_array_equal(lines.get_xdata(orig=False), expected_x)
+ expected_y += low[i].values
+ tm.assert_numpy_array_equal(lines.get_ydata(orig=False), expected_y)
+
+ def test_mixed_freq_second_millisecond(self):
+ # GH 7772, GH 7760
+ idxh = date_range("2014-07-01 09:00", freq="s", periods=5)
+ idxl = date_range("2014-07-01 09:00", freq="100ms", periods=50)
+ high = Series(np.random.default_rng(2).standard_normal(len(idxh)), idxh)
+ low = Series(np.random.default_rng(2).standard_normal(len(idxl)), idxl)
+ # high to low
+ _, ax = mpl.pyplot.subplots()
+ high.plot(ax=ax)
+ low.plot(ax=ax)
+ assert len(ax.get_lines()) == 2
+ for line in ax.get_lines():
+ assert PeriodIndex(data=line.get_xdata()).freq == "ms"
+
+ def test_mixed_freq_second_millisecond_low_to_high(self):
+ # GH 7772, GH 7760
+ idxh = date_range("2014-07-01 09:00", freq="s", periods=5)
+ idxl = date_range("2014-07-01 09:00", freq="100ms", periods=50)
+ high = Series(np.random.default_rng(2).standard_normal(len(idxh)), idxh)
+ low = Series(np.random.default_rng(2).standard_normal(len(idxl)), idxl)
+ # low to high
+ _, ax = mpl.pyplot.subplots()
+ low.plot(ax=ax)
+ high.plot(ax=ax)
+ assert len(ax.get_lines()) == 2
+ for line in ax.get_lines():
+ assert PeriodIndex(data=line.get_xdata()).freq == "ms"
+
+ def test_irreg_dtypes(self):
+ # date
+ idx = [date(2000, 1, 1), date(2000, 1, 5), date(2000, 1, 20)]
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((len(idx), 3)),
+ Index(idx, dtype=object),
+ )
+ _check_plot_works(df.plot)
+
+ def test_irreg_dtypes_dt64(self):
+ # np.datetime64
+ idx = date_range("1/1/2000", periods=10)
+ idx = idx[[0, 2, 5, 9]].astype(object)
+ df = DataFrame(np.random.default_rng(2).standard_normal((len(idx), 3)), idx)
+ _, ax = mpl.pyplot.subplots()
+ _check_plot_works(df.plot, ax=ax)
+
+ def test_time(self):
+ t = datetime(1, 1, 1, 3, 30, 0)
+ deltas = np.random.default_rng(2).integers(1, 20, 3).cumsum()
+ ts = np.array([(t + timedelta(minutes=int(x))).time() for x in deltas])
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).standard_normal(len(ts)),
+ "b": np.random.default_rng(2).standard_normal(len(ts)),
+ },
+ index=ts,
+ )
+ _, ax = mpl.pyplot.subplots()
+ df.plot(ax=ax)
+
+ # verify tick labels
+ ticks = ax.get_xticks()
+ labels = ax.get_xticklabels()
+ for _tick, _label in zip(ticks, labels, strict=True):
+ m, s = divmod(int(_tick), 60)
+ h, m = divmod(m, 60)
+ rs = _label.get_text()
+ if len(rs) > 0:
+ if s != 0:
+ xp = time(h, m, s).strftime("%H:%M:%S")
+ else:
+ xp = time(h, m, s).strftime("%H:%M")
+ assert xp == rs
+
+ def test_time_change_xlim(self):
+ t = datetime(1, 1, 1, 3, 30, 0)
+ deltas = np.random.default_rng(2).integers(1, 20, 3).cumsum()
+ ts = np.array([(t + timedelta(minutes=int(x))).time() for x in deltas])
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).standard_normal(len(ts)),
+ "b": np.random.default_rng(2).standard_normal(len(ts)),
+ },
+ index=ts,
+ )
+ _, ax = mpl.pyplot.subplots()
+ df.plot(ax=ax)
+
+ # verify tick labels
+ ticks = ax.get_xticks()
+ labels = ax.get_xticklabels()
+ for _tick, _label in zip(ticks, labels, strict=True):
+ m, s = divmod(int(_tick), 60)
+ h, m = divmod(m, 60)
+ rs = _label.get_text()
+ if len(rs) > 0:
+ if s != 0:
+ xp = time(h, m, s).strftime("%H:%M:%S")
+ else:
+ xp = time(h, m, s).strftime("%H:%M")
+ assert xp == rs
+
+ # change xlim
+ ax.set_xlim("1:30", "5:00")
+
+ # check tick labels again
+ ticks = ax.get_xticks()
+ labels = ax.get_xticklabels()
+ for _tick, _label in zip(ticks, labels, strict=True):
+ m, s = divmod(int(_tick), 60)
+ h, m = divmod(m, 60)
+ rs = _label.get_text()
+ if len(rs) > 0:
+ if s != 0:
+ xp = time(h, m, s).strftime("%H:%M:%S")
+ else:
+ xp = time(h, m, s).strftime("%H:%M")
+ assert xp == rs
+
+ def test_time_musec(self):
+ t = datetime(1, 1, 1, 3, 30, 0)
+ deltas = np.random.default_rng(2).integers(1, 20, 3).cumsum()
+ ts = np.array([(t + timedelta(microseconds=int(x))).time() for x in deltas])
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).standard_normal(len(ts)),
+ "b": np.random.default_rng(2).standard_normal(len(ts)),
+ },
+ index=ts,
+ )
+ _, ax = mpl.pyplot.subplots()
+ ax = df.plot(ax=ax)
+
+ # verify tick labels
+ ticks = ax.get_xticks()
+ labels = ax.get_xticklabels()
+ for _tick, _label in zip(ticks, labels, strict=True):
+ m, s = divmod(int(_tick), 60)
+
+ us = round((_tick - int(_tick)) * 1e6)
+
+ h, m = divmod(m, 60)
+ rs = _label.get_text()
+ if len(rs) > 0:
+ if (us % 1000) != 0:
+ xp = time(h, m, s, us).strftime("%H:%M:%S.%f")
+ elif (us // 1000) != 0:
+ xp = time(h, m, s, us).strftime("%H:%M:%S.%f")[:-3]
+ elif s != 0:
+ xp = time(h, m, s, us).strftime("%H:%M:%S")
+ else:
+ xp = time(h, m, s, us).strftime("%H:%M")
+ assert xp == rs
+
+ def test_secondary_upsample(self):
+ idxh = date_range("1/1/1999", periods=365, freq="D")
+ idxl = date_range("1/1/1999", periods=12, freq="ME")
+ high = Series(np.random.default_rng(2).standard_normal(len(idxh)), idxh)
+ low = Series(np.random.default_rng(2).standard_normal(len(idxl)), idxl)
+ _, ax = mpl.pyplot.subplots()
+ low.plot(ax=ax)
+ ax = high.plot(secondary_y=True, ax=ax)
+ for line in ax.get_lines():
+ assert PeriodIndex(line.get_xdata()).freq == "D"
+ assert hasattr(ax, "left_ax")
+ assert not hasattr(ax, "right_ax")
+ for line in ax.left_ax.get_lines():
+ assert PeriodIndex(line.get_xdata()).freq == "D"
+
+ def test_secondary_legend(self):
+ fig = mpl.pyplot.figure()
+ ax = fig.add_subplot(211)
+
+ # ts
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ df.plot(secondary_y=["A", "B"], ax=ax)
+ leg = ax.get_legend()
+ assert len(leg.get_lines()) == 4
+ assert leg.get_texts()[0].get_text() == "A (right)"
+ assert leg.get_texts()[1].get_text() == "B (right)"
+ assert leg.get_texts()[2].get_text() == "C"
+ assert leg.get_texts()[3].get_text() == "D"
+ assert ax.right_ax.get_legend() is None
+ colors = set()
+ for line in leg.get_lines():
+ colors.add(line.get_color())
+
+ # TODO: color cycle problems
+ assert len(colors) == 4
+
+ def test_secondary_legend_right(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ fig = mpl.pyplot.figure()
+ ax = fig.add_subplot(211)
+ df.plot(secondary_y=["A", "C"], mark_right=False, ax=ax)
+ leg = ax.get_legend()
+ assert len(leg.get_lines()) == 4
+ assert leg.get_texts()[0].get_text() == "A"
+ assert leg.get_texts()[1].get_text() == "B"
+ assert leg.get_texts()[2].get_text() == "C"
+ assert leg.get_texts()[3].get_text() == "D"
+
+ def test_secondary_legend_bar(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ fig, ax = mpl.pyplot.subplots()
+ df.plot(kind="bar", secondary_y=["A"], ax=ax)
+ leg = ax.get_legend()
+ assert leg.get_texts()[0].get_text() == "A (right)"
+ assert leg.get_texts()[1].get_text() == "B"
+
+ def test_secondary_legend_bar_right(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ fig, ax = mpl.pyplot.subplots()
+ df.plot(kind="bar", secondary_y=["A"], mark_right=False, ax=ax)
+ leg = ax.get_legend()
+ assert leg.get_texts()[0].get_text() == "A"
+ assert leg.get_texts()[1].get_text() == "B"
+
+ def test_secondary_legend_multi_col(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ fig = mpl.pyplot.figure()
+ ax = fig.add_subplot(211)
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ ax = df.plot(secondary_y=["C", "D"], ax=ax)
+ leg = ax.get_legend()
+ assert len(leg.get_lines()) == 4
+ assert ax.right_ax.get_legend() is None
+ colors = set()
+ for line in leg.get_lines():
+ colors.add(line.get_color())
+
+ # TODO: color cycle problems
+ assert len(colors) == 4
+
+ def test_secondary_legend_nonts(self):
+ # non-ts
+ df = DataFrame(
+ 1.1 * np.arange(40).reshape((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(10)], dtype=object),
+ )
+ fig = mpl.pyplot.figure()
+ ax = fig.add_subplot(211)
+ ax = df.plot(secondary_y=["A", "B"], ax=ax)
+ leg = ax.get_legend()
+ assert len(leg.get_lines()) == 4
+ assert ax.right_ax.get_legend() is None
+ colors = set()
+ for line in leg.get_lines():
+ colors.add(line.get_color())
+
+ # TODO: color cycle problems
+ assert len(colors) == 4
+
+ def test_secondary_legend_nonts_multi_col(self):
+ # non-ts
+ df = DataFrame(
+ 1.1 * np.arange(40).reshape((10, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=Index([f"i-{i}" for i in range(10)], dtype=object),
+ )
+ fig = mpl.pyplot.figure()
+ ax = fig.add_subplot(211)
+ ax = df.plot(secondary_y=["C", "D"], ax=ax)
+ leg = ax.get_legend()
+ assert len(leg.get_lines()) == 4
+ assert ax.right_ax.get_legend() is None
+ colors = set()
+ for line in leg.get_lines():
+ colors.add(line.get_color())
+
+ # TODO: color cycle problems
+ assert len(colors) == 4
+
+ @pytest.mark.xfail(reason="Api changed in 3.6.0")
+ def test_format_date_axis(self):
+ rng = date_range("1/1/2012", periods=12, freq="ME")
+ df = DataFrame(np.random.default_rng(2).standard_normal((len(rng), 3)), rng)
+ _, ax = mpl.pyplot.subplots()
+ ax = df.plot(ax=ax)
+ xaxis = ax.get_xaxis()
+ for line in xaxis.get_ticklabels():
+ if len(line.get_text()) > 0:
+ assert line.get_rotation() == 30
+
+ def test_ax_plot(self):
+ x = date_range(start="2012-01-02", periods=10, freq="D")
+ y = list(range(len(x)))
+ _, ax = mpl.pyplot.subplots()
+ lines = ax.plot(x, y, label="Y")
+ tm.assert_index_equal(DatetimeIndex(lines[0].get_xdata()), x)
+
+ def test_mpl_nopandas(self):
+ dates = [date(2008, 12, 31), date(2009, 1, 31)]
+ values1 = np.arange(10.0, 11.0, 0.5)
+ values2 = np.arange(11.0, 12.0, 0.5)
+
+ _, ax = mpl.pyplot.subplots()
+ (
+ line1,
+ line2,
+ ) = ax.plot(
+ [x.toordinal() for x in dates],
+ values1,
+ "-",
+ [x.toordinal() for x in dates],
+ values2,
+ "-",
+ linewidth=4,
+ )
+
+ exp = np.array([x.toordinal() for x in dates], dtype=np.float64)
+ tm.assert_numpy_array_equal(line1.get_xydata()[:, 0], exp)
+ tm.assert_numpy_array_equal(line2.get_xydata()[:, 0], exp)
+
+ def test_irregular_ts_shared_ax_xlim(self):
+ # GH 2960
+ ts = Series(
+ np.arange(20, dtype=np.float64), index=date_range("2020-01-01", periods=20)
+ )
+ ts_irregular = ts.iloc[[1, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 17, 18]]
+
+ # plot the left section of the irregular series, then the right section
+ _, ax = mpl.pyplot.subplots()
+ ts_irregular[:5].plot(ax=ax)
+ ts_irregular[5:].plot(ax=ax)
+
+ # check that axis limits are correct
+ left, right = ax.get_xlim()
+ assert left <= conv.DatetimeConverter.convert(ts_irregular.index.min(), "", ax)
+ assert right >= conv.DatetimeConverter.convert(ts_irregular.index.max(), "", ax)
+
+ def test_secondary_y_non_ts_xlim(self):
+ # GH 3490 - non-timeseries with secondary y
+ index_1 = [1, 2, 3, 4]
+ index_2 = [5, 6, 7, 8]
+ s1 = Series(1, index=index_1)
+ s2 = Series(2, index=index_2)
+
+ _, ax = mpl.pyplot.subplots()
+ s1.plot(ax=ax)
+ left_before, right_before = ax.get_xlim()
+ s2.plot(secondary_y=True, ax=ax)
+ left_after, right_after = ax.get_xlim()
+
+ assert left_before >= left_after
+ assert right_before < right_after
+
+ def test_secondary_y_regular_ts_xlim(self):
+ # GH 3490 - regular-timeseries with secondary y
+ index_1 = date_range(start="2000-01-01", periods=4, freq="D")
+ index_2 = date_range(start="2000-01-05", periods=4, freq="D")
+ s1 = Series(1, index=index_1)
+ s2 = Series(2, index=index_2)
+
+ _, ax = mpl.pyplot.subplots()
+ s1.plot(ax=ax)
+ left_before, right_before = ax.get_xlim()
+ s2.plot(secondary_y=True, ax=ax)
+ left_after, right_after = ax.get_xlim()
+
+ assert left_before >= left_after
+ assert right_before < right_after
+
+ def test_secondary_y_mixed_freq_ts_xlim(self):
+ # GH 3490 - mixed frequency timeseries with secondary y
+ rng = date_range("2000-01-01", periods=10, freq="min")
+ ts = Series(1, index=rng)
+
+ _, ax = mpl.pyplot.subplots()
+ ts.plot(ax=ax)
+ left_before, right_before = ax.get_xlim()
+ ts.resample("D").mean().plot(secondary_y=True, ax=ax)
+ left_after, right_after = ax.get_xlim()
+
+ # a downsample should not have changed either limit
+ assert left_before == left_after
+ assert right_before == right_after
+
+ def test_secondary_y_irregular_ts_xlim(self):
+ # GH 3490 - irregular-timeseries with secondary y
+ ts = Series(
+ np.arange(20, dtype=np.float64), index=date_range("2020-01-01", periods=20)
+ )
+ ts_irregular = ts.iloc[[1, 4, 5, 6, 8, 9, 10, 12, 13, 14, 15, 17, 18]]
+
+ _, ax = mpl.pyplot.subplots()
+ ts_irregular[:5].plot(ax=ax)
+ # plot higher-x values on secondary axis
+ ts_irregular[5:].plot(secondary_y=True, ax=ax)
+ # ensure secondary limits aren't overwritten by plot on primary
+ ts_irregular[:5].plot(ax=ax)
+
+ left, right = ax.get_xlim()
+ assert left <= conv.DatetimeConverter.convert(ts_irregular.index.min(), "", ax)
+ assert right >= conv.DatetimeConverter.convert(ts_irregular.index.max(), "", ax)
+
+ def test_plot_outofbounds_datetime(self):
+ # 2579 - checking this does not raise
+ values = [date(1677, 1, 1), date(1677, 1, 2)]
+ _, ax = mpl.pyplot.subplots()
+ ax.plot(values)
+
+ values = [datetime(1677, 1, 1, 12), datetime(1677, 1, 2, 12)]
+ ax.plot(values)
+
+ def test_format_timedelta_ticks_narrow(self):
+ expected_labels = [f"00:00:00.0000000{i:0>2d}" for i in np.arange(10)]
+
+ rng = timedelta_range("0", periods=10, freq="ns")
+ df = DataFrame(np.random.default_rng(2).standard_normal((len(rng), 3)), rng)
+ _, ax = mpl.pyplot.subplots()
+ df.plot(fontsize=2, ax=ax)
+ mpl.pyplot.draw()
+ labels = ax.get_xticklabels()
+
+ result_labels = [x.get_text() for x in labels]
+ assert len(result_labels) == len(expected_labels)
+ assert result_labels == expected_labels
+
+ def test_format_timedelta_ticks_wide(self, unit):
+ expected_labels = [
+ "00:00:00",
+ "1 days 03:46:40",
+ "2 days 07:33:20",
+ "3 days 11:20:00",
+ "4 days 15:06:40",
+ "5 days 18:53:20",
+ "6 days 22:40:00",
+ "8 days 02:26:40",
+ "9 days 06:13:20",
+ ]
+
+ rng = timedelta_range("0", periods=10, freq="1 D", unit=unit)
+ df = DataFrame(np.random.default_rng(2).standard_normal((len(rng), 3)), rng)
+ _, ax = mpl.pyplot.subplots()
+ ax = df.plot(fontsize=2, ax=ax)
+ mpl.pyplot.draw()
+ labels = ax.get_xticklabels()
+
+ result_labels = [x.get_text() for x in labels]
+ assert len(result_labels) == len(expected_labels)
+ assert result_labels == expected_labels
+
+ def test_timedelta_plot(self):
+ # test issue #8711
+ s = Series(range(5), timedelta_range("1day", periods=5))
+ _, ax = mpl.pyplot.subplots()
+ _check_plot_works(s.plot, ax=ax)
+
+ def test_timedelta_long_period(self):
+ # test long period
+ index = timedelta_range("1 day 2 hr 30 min 10 s", periods=10, freq="1 D")
+ s = Series(np.random.default_rng(2).standard_normal(len(index)), index)
+ _, ax = mpl.pyplot.subplots()
+ _check_plot_works(s.plot, ax=ax)
+
+ def test_timedelta_short_period(self):
+ # test short period
+ index = timedelta_range("1 day 2 hr 30 min 10 s", periods=10, freq="1 ns")
+ s = Series(np.random.default_rng(2).standard_normal(len(index)), index)
+ _, ax = mpl.pyplot.subplots()
+ _check_plot_works(s.plot, ax=ax)
+
+ def test_hist(self):
+ # https://github.com/matplotlib/matplotlib/issues/8459
+ rng = date_range("1/1/2011", periods=10, freq="h")
+ x = rng
+ w1 = np.arange(0, 1, 0.1)
+ w2 = np.arange(0, 1, 0.1)[::-1]
+ _, ax = mpl.pyplot.subplots()
+ ax.hist([x, x], weights=[w1, w2])
+
+ def test_overlapping_datetime(self):
+ # GB 6608
+ s1 = Series(
+ [1, 2, 3],
+ index=[
+ datetime(1995, 12, 31),
+ datetime(2000, 12, 31),
+ datetime(2005, 12, 31),
+ ],
+ )
+ s2 = Series(
+ [1, 2, 3],
+ index=[
+ datetime(1997, 12, 31),
+ datetime(2003, 12, 31),
+ datetime(2008, 12, 31),
+ ],
+ )
+
+ # plot first series, then add the second series to those axes,
+ # then try adding the first series again
+ _, ax = mpl.pyplot.subplots()
+ s1.plot(ax=ax)
+ s2.plot(ax=ax)
+ s1.plot(ax=ax)
+
+ @pytest.mark.xfail(reason="GH9053 matplotlib does not use ax.xaxis.converter")
+ def test_add_matplotlib_datetime64(self):
+ # GH9053 - ensure that a plot with PeriodConverter still understands
+ # datetime64 data. This still fails because matplotlib overrides the
+ # ax.xaxis.converter with a DatetimeConverter
+ s = Series(
+ np.random.default_rng(2).standard_normal(10),
+ index=date_range("1970-01-02", periods=10),
+ )
+ ax = s.plot()
+ with tm.assert_produces_warning(DeprecationWarning):
+ # multi-dimensional indexing
+ ax.plot(s.index, s.values, color="g")
+ l1, l2 = ax.lines
+ tm.assert_numpy_array_equal(l1.get_xydata(), l2.get_xydata())
+
+ def test_matplotlib_scatter_datetime64(self):
+ # https://github.com/matplotlib/matplotlib/issues/11391
+ df = DataFrame(np.random.default_rng(2).random((10, 2)), columns=["x", "y"])
+ df["time"] = date_range("2018-01-01", periods=10, freq="D")
+ _, ax = mpl.pyplot.subplots()
+ ax.scatter(x="time", y="y", data=df)
+ mpl.pyplot.draw()
+ label = ax.get_xticklabels()[0]
+ expected = "2018-01-01"
+ assert label.get_text() == expected
+
+ def test_check_xticks_rot(self):
+ # https://github.com/pandas-dev/pandas/issues/29460
+ # regular time series
+ x = to_datetime(["2020-05-01", "2020-05-02", "2020-05-03"])
+ df = DataFrame({"x": x, "y": [1, 2, 3]})
+ axes = df.plot(x="x", y="y")
+ _check_ticks_props(axes, xrot=0)
+
+ def test_check_xticks_rot_irregular(self):
+ # irregular time series
+ x = to_datetime(["2020-05-01", "2020-05-02", "2020-05-04"])
+ df = DataFrame({"x": x, "y": [1, 2, 3]})
+ axes = df.plot(x="x", y="y")
+ _check_ticks_props(axes, xrot=30)
+
+ def test_check_xticks_rot_use_idx(self):
+ # irregular time series
+ x = to_datetime(["2020-05-01", "2020-05-02", "2020-05-04"])
+ df = DataFrame({"x": x, "y": [1, 2, 3]})
+ # use timeseries index or not
+ axes = df.set_index("x").plot(y="y", use_index=True)
+ _check_ticks_props(axes, xrot=30)
+ axes = df.set_index("x").plot(y="y", use_index=False)
+ _check_ticks_props(axes, xrot=0)
+
+ def test_check_xticks_rot_sharex(self):
+ # irregular time series
+ x = to_datetime(["2020-05-01", "2020-05-02", "2020-05-04"])
+ df = DataFrame({"x": x, "y": [1, 2, 3]})
+ # separate subplots
+ axes = df.plot(x="x", y="y", subplots=True, sharex=True)
+ _check_ticks_props(axes, xrot=30)
+ axes = df.plot(x="x", y="y", subplots=True, sharex=False)
+ _check_ticks_props(axes, xrot=0)
+
+ @pytest.mark.parametrize(
+ "idx",
+ [
+ date_range("2020-01-01", periods=5),
+ date_range("2020-01-01", periods=5, tz="UTC"),
+ timedelta_range("1 day", periods=5, freq="D"),
+ period_range("2020-01-01", periods=5, freq="D"),
+ Index([date(2000, 1, i) for i in [1, 3, 6, 20, 22]], dtype=object),
+ range(5),
+ ],
+ )
+ def test_pickle_fig(self, temp_file, frame_or_series, idx):
+ # GH18439, GH#24088, statsmodels#4772
+ df = frame_or_series(range(5), index=idx)
+ fig, ax = plt.subplots(1, 1)
+ df.plot(ax=ax)
+ with temp_file.open(mode="wb") as path:
+ pickle.dump(fig, path)
+
+
+def _check_plot_works(f, freq=None, series=None, *args, **kwargs):
+ fig = plt.gcf()
+
+ fig.clf()
+ ax = fig.add_subplot(211)
+ orig_ax = kwargs.pop("ax", plt.gca())
+ orig_axfreq = getattr(orig_ax, "freq", None)
+
+ ret = f(*args, **kwargs)
+ assert ret is not None # do something more intelligent
+
+ ax = kwargs.pop("ax", plt.gca())
+ if series is not None:
+ dfreq = series.index.freq
+ if isinstance(dfreq, BaseOffset):
+ dfreq = dfreq.rule_code
+ if orig_axfreq is None:
+ assert ax.freq == dfreq
+
+ if freq is not None and orig_axfreq is None:
+ assert to_offset(ax.freq, is_period=True) == freq
+
+ ax = fig.add_subplot(212)
+ kwargs["ax"] = ax
+ ret = f(*args, **kwargs)
+ assert ret is not None # TODO: do something more intelligent
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_groupby.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_groupby.py
new file mode 100644
index 0000000000000000000000000000000000000000..e86c4e9838d2459f2ad87fcf97a4c46bad2e820d
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_groupby.py
@@ -0,0 +1,154 @@
+"""Test cases for GroupBy.plot"""
+
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ Index,
+ Series,
+)
+from pandas.tests.plotting.common import (
+ _check_axes_shape,
+ _check_legend_labels,
+)
+
+pytest.importorskip("matplotlib")
+
+
+class TestDataFrameGroupByPlots:
+ def test_series_groupby_plotting_nominally_works(self):
+ n = 10
+ weight = Series(np.random.default_rng(2).normal(166, 20, size=n))
+ gender = np.random.default_rng(2).choice(["male", "female"], size=n)
+
+ weight.groupby(gender).plot()
+
+ def test_series_groupby_plotting_nominally_works_hist(self):
+ n = 10
+ height = Series(np.random.default_rng(2).normal(60, 10, size=n))
+ gender = np.random.default_rng(2).choice(["male", "female"], size=n)
+ height.groupby(gender).hist()
+
+ def test_series_groupby_plotting_nominally_works_alpha(self):
+ n = 10
+ height = Series(np.random.default_rng(2).normal(60, 10, size=n))
+ gender = np.random.default_rng(2).choice(["male", "female"], size=n)
+ # Regression test for GH8733
+ height.groupby(gender).plot(alpha=0.5)
+
+ def test_plotting_with_float_index_works(self):
+ # GH 7025
+ df = DataFrame(
+ {
+ "def": [1, 1, 1, 2, 2, 2, 3, 3, 3],
+ "val": np.random.default_rng(2).standard_normal(9),
+ },
+ index=[1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0],
+ )
+
+ df.groupby("def")["val"].plot()
+
+ def test_plotting_with_float_index_works_apply(self):
+ # GH 7025
+ df = DataFrame(
+ {
+ "def": [1, 1, 1, 2, 2, 2, 3, 3, 3],
+ "val": np.random.default_rng(2).standard_normal(9),
+ },
+ index=[1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0],
+ )
+ df.groupby("def")["val"].apply(lambda x: x.plot())
+
+ def test_hist_single_row(self):
+ # GH10214
+ bins = np.arange(80, 100 + 2, 1)
+ df = DataFrame({"Name": ["AAA", "BBB"], "ByCol": [1, 2], "Mark": [85, 89]})
+ df["Mark"].hist(by=df["ByCol"], bins=bins)
+
+ def test_hist_single_row_single_bycol(self):
+ # GH10214
+ bins = np.arange(80, 100 + 2, 1)
+ df = DataFrame({"Name": ["AAA"], "ByCol": [1], "Mark": [85]})
+ df["Mark"].hist(by=df["ByCol"], bins=bins)
+
+ def test_plot_submethod_works(self):
+ df = DataFrame({"x": [1, 2, 3, 4, 5], "y": [1, 2, 3, 2, 1], "z": list("ababa")})
+ df.groupby("z").plot.scatter("x", "y")
+
+ def test_plot_submethod_works_line(self):
+ df = DataFrame({"x": [1, 2, 3, 4, 5], "y": [1, 2, 3, 2, 1], "z": list("ababa")})
+ df.groupby("z")["x"].plot.line()
+
+ def test_plot_kwargs(self):
+ df = DataFrame({"x": [1, 2, 3, 4, 5], "y": [1, 2, 3, 2, 1], "z": list("ababa")})
+
+ res = df.groupby("z").plot(kind="scatter", x="x", y="y")
+ # check that a scatter plot is effectively plotted: the axes should
+ # contain a PathCollection from the scatter plot (GH11805)
+ assert len(res["a"].collections) == 1
+
+ def test_plot_kwargs_scatter(self):
+ df = DataFrame({"x": [1, 2, 3, 4, 5], "y": [1, 2, 3, 2, 1], "z": list("ababa")})
+ res = df.groupby("z").plot.scatter(x="x", y="y")
+ assert len(res["a"].collections) == 1
+
+ @pytest.mark.parametrize("column, expected_axes_num", [(None, 2), ("b", 1)])
+ def test_groupby_hist_frame_with_legend(self, column, expected_axes_num):
+ # GH 6279 - DataFrameGroupBy histogram can have a legend
+ expected_layout = (1, expected_axes_num)
+ expected_labels = column or [["a"], ["b"]]
+
+ index = Index(15 * ["1"] + 15 * ["2"], name="c")
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((30, 2)),
+ index=index,
+ columns=["a", "b"],
+ )
+ g = df.groupby("c")
+
+ for axes in g.hist(legend=True, column=column):
+ _check_axes_shape(axes, axes_num=expected_axes_num, layout=expected_layout)
+ for ax, expected_label in zip(axes[0], expected_labels, strict=True):
+ _check_legend_labels(ax, expected_label)
+
+ @pytest.mark.parametrize("column", [None, "b"])
+ def test_groupby_hist_frame_with_legend_raises(self, column):
+ # GH 6279 - DataFrameGroupBy histogram with legend and label raises
+ index = Index(15 * ["1"] + 15 * ["2"], name="c")
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((30, 2)),
+ index=index,
+ columns=["a", "b"],
+ )
+ g = df.groupby("c")
+
+ with pytest.raises(ValueError, match="Cannot use both legend and label"):
+ g.hist(legend=True, column=column, label="d")
+
+ def test_groupby_hist_series_with_legend(self):
+ # GH 6279 - SeriesGroupBy histogram can have a legend
+ index = Index(15 * ["1"] + 15 * ["2"], name="c")
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((30, 2)),
+ index=index,
+ columns=["a", "b"],
+ )
+ g = df.groupby("c")
+
+ for ax in g["a"].hist(legend=True):
+ _check_axes_shape(ax, axes_num=1, layout=(1, 1))
+ _check_legend_labels(ax, ["1", "2"])
+
+ def test_groupby_hist_series_with_legend_raises(self):
+ # GH 6279 - SeriesGroupBy histogram with legend and label raises
+ index = Index(15 * ["1"] + 15 * ["2"], name="c")
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((30, 2)),
+ index=index,
+ columns=["a", "b"],
+ )
+ g = df.groupby("c")
+
+ with pytest.raises(ValueError, match="Cannot use both legend and label"):
+ g.hist(legend=True, label="d")
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_hist_method.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_hist_method.py
new file mode 100644
index 0000000000000000000000000000000000000000..e71d4ce5475a89dd686f831afcd1e4c4d726851d
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_hist_method.py
@@ -0,0 +1,957 @@
+"""Test cases for .hist method"""
+
+import re
+
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ Index,
+ Series,
+ date_range,
+ to_datetime,
+)
+import pandas._testing as tm
+from pandas.tests.plotting.common import (
+ _check_ax_scales,
+ _check_axes_shape,
+ _check_colors,
+ _check_legend_labels,
+ _check_patches_all_filled,
+ _check_plot_works,
+ _check_text_labels,
+ _check_ticks_props,
+ get_x_axis,
+ get_y_axis,
+)
+
+mpl = pytest.importorskip("matplotlib")
+plt = pytest.importorskip("matplotlib.pyplot")
+
+from pandas.plotting._matplotlib.hist import _grouped_hist
+
+
+@pytest.fixture
+def ts():
+ return Series(
+ np.arange(30, dtype=np.float64),
+ index=date_range("2020-01-01", periods=30, freq="B"),
+ name="ts",
+ )
+
+
+class TestSeriesPlots:
+ @pytest.mark.parametrize("kwargs", [{}, {"grid": False}, {"figsize": (8, 10)}])
+ def test_hist_legacy_kwargs(self, ts, kwargs):
+ _check_plot_works(ts.hist, **kwargs)
+
+ @pytest.mark.parametrize("kwargs", [{}, {"bins": 5}])
+ def test_hist_legacy_kwargs_warning(self, ts, kwargs):
+ # _check_plot_works adds an ax so catch warning. see GH #13188
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ _check_plot_works(ts.hist, by=ts.index.month, **kwargs)
+
+ def test_hist_legacy_ax(self, ts):
+ fig, ax = mpl.pyplot.subplots(1, 1)
+ _check_plot_works(ts.hist, ax=ax, default_axes=True)
+
+ def test_hist_legacy_ax_and_fig(self, ts):
+ fig, ax = mpl.pyplot.subplots(1, 1)
+ _check_plot_works(ts.hist, ax=ax, figure=fig, default_axes=True)
+
+ def test_hist_legacy_fig(self, ts):
+ fig, _ = mpl.pyplot.subplots(1, 1)
+ _check_plot_works(ts.hist, figure=fig, default_axes=True)
+
+ def test_hist_legacy_multi_ax(self, ts):
+ fig, (ax1, ax2) = mpl.pyplot.subplots(1, 2)
+ _check_plot_works(ts.hist, figure=fig, ax=ax1, default_axes=True)
+ _check_plot_works(ts.hist, figure=fig, ax=ax2, default_axes=True)
+
+ def test_hist_legacy_by_fig_error(self, ts):
+ fig, _ = mpl.pyplot.subplots(1, 1)
+ msg = (
+ "Cannot pass 'figure' when using the 'by' argument, since a new 'Figure' "
+ "instance will be created"
+ )
+ with pytest.raises(ValueError, match=msg):
+ ts.hist(by=ts.index, figure=fig)
+
+ def test_hist_bins_legacy(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 2)))
+ ax = df.hist(bins=2)[0][0]
+ assert len(ax.patches) == 2
+
+ def test_hist_layout(self, hist_df):
+ df = hist_df
+ msg = "The 'layout' keyword is not supported when 'by' is None"
+ with pytest.raises(ValueError, match=msg):
+ df.height.hist(layout=(1, 1))
+
+ with pytest.raises(ValueError, match=msg):
+ df.height.hist(layout=[1, 1])
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "by, layout, axes_num, res_layout",
+ [
+ ["gender", (2, 1), 2, (2, 1)],
+ ["gender", (3, -1), 2, (3, 1)],
+ ["category", (4, 1), 4, (4, 1)],
+ ["category", (2, -1), 4, (2, 2)],
+ ["category", (3, -1), 4, (3, 2)],
+ ["category", (-1, 4), 4, (1, 4)],
+ ["classroom", (2, 2), 3, (2, 2)],
+ ],
+ )
+ def test_hist_layout_with_by(self, hist_df, by, layout, axes_num, res_layout):
+ df = hist_df
+
+ # _check_plot_works adds an `ax` kwarg to the method call
+ # so we get a warning about an axis being cleared, even
+ # though we don't explicitly pass one, see GH #13188
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ axes = _check_plot_works(df.height.hist, by=getattr(df, by), layout=layout)
+ _check_axes_shape(axes, axes_num=axes_num, layout=res_layout)
+
+ def test_hist_layout_with_by_shape(self, hist_df):
+ df = hist_df
+
+ axes = df.height.hist(by=df.category, layout=(4, 2), figsize=(12, 7))
+ _check_axes_shape(axes, axes_num=4, layout=(4, 2), figsize=(12, 7))
+
+ def test_hist_no_overlap(self):
+ x = Series(np.random.default_rng(2).standard_normal(2))
+ y = Series(np.random.default_rng(2).standard_normal(2))
+ plt.subplot(121)
+ x.hist()
+ plt.subplot(122)
+ y.hist()
+ fig = plt.gcf()
+ axes = fig.axes
+ assert len(axes) == 2
+
+ def test_hist_by_no_extra_plots(self, hist_df):
+ df = hist_df
+ df.height.hist(by=df.gender)
+ assert len(mpl.pyplot.get_fignums()) == 1
+
+ def test_plot_fails_when_ax_differs_from_figure(self, ts):
+ fig1 = plt.figure(1)
+ fig2 = plt.figure(2)
+ ax1 = fig1.add_subplot(111)
+ msg = "passed axis not bound to passed figure"
+ with pytest.raises(AssertionError, match=msg):
+ ts.hist(ax=ax1, figure=fig2)
+
+ @pytest.mark.parametrize(
+ "histtype, expected",
+ [
+ ("bar", True),
+ ("barstacked", True),
+ ("step", False),
+ ("stepfilled", True),
+ ],
+ )
+ def test_histtype_argument(self, histtype, expected):
+ # GH23992 Verify functioning of histtype argument
+ ser = Series(np.random.default_rng(2).integers(1, 10))
+ ax = ser.hist(histtype=histtype)
+ _check_patches_all_filled(ax, filled=expected)
+
+ @pytest.mark.parametrize(
+ "by, expected_axes_num, expected_layout", [(None, 1, (1, 1)), ("b", 2, (1, 2))]
+ )
+ def test_hist_with_legend(self, by, expected_axes_num, expected_layout):
+ # GH 6279 - Series histogram can have a legend
+ index = 5 * ["1"] + 5 * ["2"]
+ s = Series(np.random.default_rng(2).standard_normal(10), index=index, name="a")
+ s.index.name = "b"
+
+ # Use default_axes=True when plotting method generate subplots itself
+ axes = _check_plot_works(s.hist, default_axes=True, legend=True, by=by)
+ _check_axes_shape(axes, axes_num=expected_axes_num, layout=expected_layout)
+ _check_legend_labels(axes, "a")
+
+ @pytest.mark.parametrize("by", [None, "b"])
+ def test_hist_with_legend_raises(self, by):
+ # GH 6279 - Series histogram with legend and label raises
+ index = 5 * ["1"] + 5 * ["2"]
+ s = Series(np.random.default_rng(2).standard_normal(10), index=index, name="a")
+ s.index.name = "b"
+
+ with pytest.raises(ValueError, match="Cannot use both legend and label"):
+ s.hist(legend=True, by=by, label="c")
+
+ def test_hist_kwargs(self, ts):
+ _, ax = mpl.pyplot.subplots()
+ ax = ts.plot.hist(bins=5, ax=ax)
+ assert len(ax.patches) == 5
+ _check_text_labels(ax.yaxis.get_label(), "Frequency")
+
+ def test_hist_kwargs_horizontal(self, ts):
+ _, ax = mpl.pyplot.subplots()
+ ax = ts.plot.hist(bins=5, ax=ax)
+ ax = ts.plot.hist(orientation="horizontal", ax=ax)
+ _check_text_labels(ax.xaxis.get_label(), "Frequency")
+
+ def test_hist_kwargs_align(self, ts):
+ _, ax = mpl.pyplot.subplots()
+ ax = ts.plot.hist(bins=5, ax=ax)
+ ax = ts.plot.hist(align="left", stacked=True, ax=ax)
+
+ @pytest.mark.xfail(reason="Api changed in 3.6.0")
+ def test_hist_kde(self, ts):
+ pytest.importorskip("scipy")
+ _, ax = mpl.pyplot.subplots()
+ ax = ts.plot.hist(logy=True, ax=ax)
+ _check_ax_scales(ax, yaxis="log")
+ xlabels = ax.get_xticklabels()
+ # ticks are values, thus ticklabels are blank
+ _check_text_labels(xlabels, [""] * len(xlabels))
+ ylabels = ax.get_yticklabels()
+ _check_text_labels(ylabels, [""] * len(ylabels))
+
+ def test_hist_kde_plot_works(self, ts):
+ pytest.importorskip("scipy")
+ _check_plot_works(ts.plot.kde)
+
+ def test_hist_kde_density_works(self, ts):
+ pytest.importorskip("scipy")
+ _check_plot_works(ts.plot.density)
+
+ @pytest.mark.xfail(reason="Api changed in 3.6.0")
+ def test_hist_kde_logy(self, ts):
+ pytest.importorskip("scipy")
+ _, ax = mpl.pyplot.subplots()
+ ax = ts.plot.kde(logy=True, ax=ax)
+ _check_ax_scales(ax, yaxis="log")
+ xlabels = ax.get_xticklabels()
+ _check_text_labels(xlabels, [""] * len(xlabels))
+ ylabels = ax.get_yticklabels()
+ _check_text_labels(ylabels, [""] * len(ylabels))
+
+ def test_hist_kde_color_bins(self, ts):
+ pytest.importorskip("scipy")
+ _, ax = mpl.pyplot.subplots()
+ ax = ts.plot.hist(logy=True, bins=10, color="b", ax=ax)
+ _check_ax_scales(ax, yaxis="log")
+ assert len(ax.patches) == 10
+ _check_colors(ax.patches, facecolors=["b"] * 10)
+
+ def test_hist_kde_color(self, ts):
+ pytest.importorskip("scipy")
+ _, ax = mpl.pyplot.subplots()
+ ax = ts.plot.kde(logy=True, color="r", ax=ax)
+ _check_ax_scales(ax, yaxis="log")
+ lines = ax.get_lines()
+ assert len(lines) == 1
+ _check_colors(lines, ["r"])
+
+
+class TestDataFramePlots:
+ @pytest.mark.slow
+ def test_hist_df_legacy(self, hist_df):
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ _check_plot_works(hist_df.hist)
+
+ @pytest.mark.slow
+ def test_hist_df_legacy_layout(self):
+ # make sure layout is handled
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 2)))
+ df[2] = to_datetime(
+ np.random.default_rng(2).integers(
+ 812419200000000000,
+ 819331200000000000,
+ size=10,
+ dtype=np.int64,
+ )
+ )
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ axes = _check_plot_works(df.hist, grid=False)
+ _check_axes_shape(axes, axes_num=3, layout=(2, 2))
+ assert not axes[1, 1].get_visible()
+
+ _check_plot_works(df[[2]].hist)
+
+ @pytest.mark.slow
+ def test_hist_df_legacy_layout2(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 1)))
+ _check_plot_works(df.hist)
+
+ @pytest.mark.slow
+ def test_hist_df_legacy_layout3(self):
+ # make sure layout is handled
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 5)))
+ df[5] = to_datetime(
+ np.random.default_rng(2).integers(
+ 812419200000000000,
+ 819331200000000000,
+ size=10,
+ dtype=np.int64,
+ )
+ )
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ axes = _check_plot_works(df.hist, layout=(4, 2))
+ _check_axes_shape(axes, axes_num=6, layout=(4, 2))
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "kwargs", [{"sharex": True, "sharey": True}, {"figsize": (8, 10)}, {"bins": 5}]
+ )
+ def test_hist_df_legacy_layout_kwargs(self, kwargs):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 5)))
+ df[5] = to_datetime(
+ np.random.default_rng(2).integers(
+ 812419200000000000,
+ 819331200000000000,
+ size=10,
+ dtype=np.int64,
+ )
+ )
+ # make sure sharex, sharey is handled
+ # handle figsize arg
+ # check bins argument
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ _check_plot_works(df.hist, **kwargs)
+
+ @pytest.mark.slow
+ def test_hist_df_legacy_layout_labelsize_rot(self, frame_or_series):
+ # make sure xlabelsize and xrot are handled
+ obj = frame_or_series(range(10))
+ xf, yf = 20, 18
+ xrot, yrot = 30, 40
+ axes = obj.hist(xlabelsize=xf, xrot=xrot, ylabelsize=yf, yrot=yrot)
+ _check_ticks_props(axes, xlabelsize=xf, xrot=xrot, ylabelsize=yf, yrot=yrot)
+
+ @pytest.mark.slow
+ def test_hist_df_legacy_rectangles(self):
+ ser = Series(range(10))
+ ax = ser.hist(cumulative=True, bins=4, density=True)
+ # height of last bin (index 5) must be 1.0
+ rects = [x for x in ax.get_children() if isinstance(x, mpl.patches.Rectangle)]
+ tm.assert_almost_equal(rects[-1].get_height(), 1.0)
+
+ @pytest.mark.slow
+ def test_hist_df_legacy_scale(self):
+ ser = Series(range(10))
+ ax = ser.hist(log=True)
+ # scale of y must be 'log'
+ _check_ax_scales(ax, yaxis="log")
+
+ @pytest.mark.slow
+ def test_hist_df_legacy_external_error(self):
+ ser = Series(range(10))
+ # propagate attr exception from matplotlib.Axes.hist
+ with tm.external_error_raised(AttributeError):
+ ser.hist(foo="bar")
+
+ def test_hist_non_numerical_or_datetime_raises(self):
+ # gh-10444, GH32590
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).random(10),
+ "b": np.random.default_rng(2).integers(0, 10, 10),
+ "c": to_datetime(
+ np.random.default_rng(2).integers(
+ 1582800000000000000, 1583500000000000000, 10, dtype=np.int64
+ )
+ ),
+ "d": to_datetime(
+ np.random.default_rng(2).integers(
+ 1582800000000000000, 1583500000000000000, 10, dtype=np.int64
+ ),
+ utc=True,
+ ),
+ }
+ )
+ df_o = df.astype(object)
+
+ msg = "hist method requires numerical or datetime columns, nothing to plot."
+ with pytest.raises(ValueError, match=msg):
+ df_o.hist()
+
+ @pytest.mark.parametrize(
+ "layout_test",
+ (
+ {"layout": None, "expected_size": (2, 2)}, # default is 2x2
+ {"layout": (2, 2), "expected_size": (2, 2)},
+ {"layout": (4, 1), "expected_size": (4, 1)},
+ {"layout": (1, 4), "expected_size": (1, 4)},
+ {"layout": (3, 3), "expected_size": (3, 3)},
+ {"layout": (-1, 4), "expected_size": (1, 4)},
+ {"layout": (4, -1), "expected_size": (4, 1)},
+ {"layout": (-1, 2), "expected_size": (2, 2)},
+ {"layout": (2, -1), "expected_size": (2, 2)},
+ ),
+ )
+ def test_hist_layout(self, layout_test):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 2)))
+ df[2] = to_datetime(
+ np.random.default_rng(2).integers(
+ 812419200000000000,
+ 819331200000000000,
+ size=10,
+ dtype=np.int64,
+ )
+ )
+ axes = df.hist(layout=layout_test["layout"])
+ expected = layout_test["expected_size"]
+ _check_axes_shape(axes, axes_num=3, layout=expected)
+
+ def test_hist_layout_error(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 2)))
+ df[2] = to_datetime(
+ np.random.default_rng(2).integers(
+ 812419200000000000,
+ 819331200000000000,
+ size=10,
+ dtype=np.int64,
+ )
+ )
+ # layout too small for all 4 plots
+ msg = "Layout of 1x1 must be larger than required size 3"
+ with pytest.raises(ValueError, match=msg):
+ df.hist(layout=(1, 1))
+
+ # invalid format for layout
+ msg = re.escape("Layout must be a tuple of (rows, columns)")
+ with pytest.raises(ValueError, match=msg):
+ df.hist(layout=(1,))
+ msg = "At least one dimension of layout must be positive"
+ with pytest.raises(ValueError, match=msg):
+ df.hist(layout=(-1, -1))
+
+ # GH 9351
+ def test_tight_layout(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 2)))
+ df[2] = to_datetime(
+ np.random.default_rng(2).integers(
+ 812419200000000000,
+ 819331200000000000,
+ size=10,
+ dtype=np.int64,
+ )
+ )
+ # Use default_axes=True when plotting method generate subplots itself
+ _check_plot_works(df.hist, default_axes=True)
+ mpl.pyplot.tight_layout()
+
+ def test_hist_subplot_xrot(self):
+ # GH 30288
+ df = DataFrame(
+ {
+ "length": [1.5, 0.5, 1.2, 0.9, 3],
+ "animal": ["pig", "rabbit", "pig", "pig", "rabbit"],
+ }
+ )
+ # Use default_axes=True when plotting method generate subplots itself
+ axes = _check_plot_works(
+ df.hist,
+ default_axes=True,
+ column="length",
+ by="animal",
+ bins=5,
+ xrot=0,
+ )
+ _check_ticks_props(axes, xrot=0)
+
+ @pytest.mark.parametrize(
+ "column, expected",
+ [
+ (None, ["width", "length", "height"]),
+ (["length", "width", "height"], ["length", "width", "height"]),
+ ],
+ )
+ def test_hist_column_order_unchanged(self, column, expected):
+ # GH29235
+
+ df = DataFrame(
+ {
+ "width": [0.7, 0.2, 0.15, 0.2, 1.1],
+ "length": [1.5, 0.5, 1.2, 0.9, 3],
+ "height": [3, 0.5, 3.4, 2, 1],
+ },
+ index=["pig", "rabbit", "duck", "chicken", "horse"],
+ )
+
+ # Use default_axes=True when plotting method generate subplots itself
+ axes = _check_plot_works(
+ df.hist,
+ default_axes=True,
+ column=column,
+ layout=(1, 3),
+ )
+ result = [axes[0, i].get_title() for i in range(3)]
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "histtype, expected",
+ [
+ ("bar", True),
+ ("barstacked", True),
+ ("step", False),
+ ("stepfilled", True),
+ ],
+ )
+ def test_histtype_argument(self, histtype, expected):
+ # GH23992 Verify functioning of histtype argument
+ df = DataFrame(
+ np.random.default_rng(2).integers(1, 10, size=(10, 2)), columns=["a", "b"]
+ )
+ ax = df.hist(histtype=histtype)
+ _check_patches_all_filled(ax, filled=expected)
+
+ @pytest.mark.parametrize("by", [None, "c"])
+ @pytest.mark.parametrize("column", [None, "b"])
+ def test_hist_with_legend(self, by, column):
+ # GH 6279 - DataFrame histogram can have a legend
+ expected_axes_num = 1 if by is None and column is not None else 2
+ expected_layout = (1, expected_axes_num)
+ expected_labels = column or ["a", "b"]
+ if by is not None:
+ expected_labels = [expected_labels] * 2
+
+ index = Index(5 * ["1"] + 5 * ["2"], name="c")
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)),
+ index=index,
+ columns=["a", "b"],
+ )
+
+ # Use default_axes=True when plotting method generate subplots itself
+ axes = _check_plot_works(
+ df.hist,
+ default_axes=True,
+ legend=True,
+ by=by,
+ column=column,
+ )
+
+ _check_axes_shape(axes, axes_num=expected_axes_num, layout=expected_layout)
+ if by is None and column is None:
+ axes = axes[0]
+ for expected_label, ax in zip(expected_labels, axes, strict=True):
+ _check_legend_labels(ax, expected_label)
+
+ @pytest.mark.parametrize("by", [None, "c"])
+ @pytest.mark.parametrize("column", [None, "b"])
+ def test_hist_with_legend_raises(self, by, column):
+ # GH 6279 - DataFrame histogram with legend and label raises
+ index = Index(5 * ["1"] + 5 * ["2"], name="c")
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)),
+ index=index,
+ columns=["a", "b"],
+ )
+
+ with pytest.raises(ValueError, match="Cannot use both legend and label"):
+ df.hist(legend=True, by=by, column=column, label="d")
+
+ def test_hist_df_kwargs(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 2)))
+ _, ax = mpl.pyplot.subplots()
+ ax = df.plot.hist(bins=5, ax=ax)
+ assert len(ax.patches) == 10
+
+ def test_hist_df_with_nonnumerics(self):
+ # GH 9853
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=["A", "B", "C", "D"],
+ )
+ df["E"] = ["x", "y"] * 5
+ _, ax = mpl.pyplot.subplots()
+ ax = df.plot.hist(bins=5, ax=ax)
+ assert len(ax.patches) == 20
+
+ def test_hist_df_with_nonnumerics_no_bins(self):
+ # GH 9853
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=["A", "B", "C", "D"],
+ )
+ df["E"] = ["x", "y"] * 5
+ _, ax = mpl.pyplot.subplots()
+ ax = df.plot.hist(ax=ax) # bins=10
+ assert len(ax.patches) == 40
+
+ def test_hist_secondary_legend(self):
+ # GH 9610
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)), columns=list("abcd")
+ )
+
+ # primary -> secondary
+ _, ax = mpl.pyplot.subplots()
+ ax = df["a"].plot.hist(legend=True, ax=ax)
+ df["b"].plot.hist(ax=ax, legend=True, secondary_y=True)
+ # both legends are drawn on left ax
+ # left and right axis must be visible
+ _check_legend_labels(ax, labels=["a", "b (right)"])
+ assert ax.get_yaxis().get_visible()
+ assert ax.right_ax.get_yaxis().get_visible()
+
+ def test_hist_secondary_secondary(self):
+ # GH 9610
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)), columns=list("abcd")
+ )
+ # secondary -> secondary
+ _, ax = mpl.pyplot.subplots()
+ ax = df["a"].plot.hist(legend=True, secondary_y=True, ax=ax)
+ df["b"].plot.hist(ax=ax, legend=True, secondary_y=True)
+ # both legends are draw on left ax
+ # left axis must be invisible, right axis must be visible
+ _check_legend_labels(ax.left_ax, labels=["a (right)", "b (right)"])
+ assert not ax.left_ax.get_yaxis().get_visible()
+ assert ax.get_yaxis().get_visible()
+
+ def test_hist_secondary_primary(self):
+ # GH 9610
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)), columns=list("abcd")
+ )
+ # secondary -> primary
+ _, ax = mpl.pyplot.subplots()
+ ax = df["a"].plot.hist(legend=True, secondary_y=True, ax=ax)
+ # right axes is returned
+ df["b"].plot.hist(ax=ax, legend=True)
+ # both legends are draw on left ax
+ # left and right axis must be visible
+ _check_legend_labels(ax.left_ax, labels=["a (right)", "b"])
+ assert ax.left_ax.get_yaxis().get_visible()
+ assert ax.get_yaxis().get_visible()
+
+ def test_hist_with_nans_and_weights(self):
+ # GH 48884
+ df = DataFrame(
+ [[np.nan, 0.2, 0.3], [0.4, np.nan, np.nan], [0.7, 0.8, 0.9]],
+ columns=list("abc"),
+ )
+ weights = np.array([0.25, 0.3, 0.45])
+ no_nan_df = DataFrame([[0.4, 0.2, 0.3], [0.7, 0.8, 0.9]], columns=list("abc"))
+ no_nan_weights = np.array([[0.3, 0.25, 0.25], [0.45, 0.45, 0.45]])
+
+ _, ax0 = mpl.pyplot.subplots()
+ df.plot.hist(ax=ax0, weights=weights)
+ rects = [x for x in ax0.get_children() if isinstance(x, mpl.patches.Rectangle)]
+ heights = [rect.get_height() for rect in rects]
+ _, ax1 = mpl.pyplot.subplots()
+ no_nan_df.plot.hist(ax=ax1, weights=no_nan_weights)
+ no_nan_rects = [
+ x for x in ax1.get_children() if isinstance(x, mpl.patches.Rectangle)
+ ]
+ no_nan_heights = [rect.get_height() for rect in no_nan_rects]
+ assert all(h0 == h1 for h0, h1 in zip(heights, no_nan_heights, strict=True))
+
+ idxerror_weights = np.array([[0.3, 0.25], [0.45, 0.45]])
+
+ msg = "weights must have the same shape as data, or be a single column"
+ _, ax2 = mpl.pyplot.subplots()
+ with pytest.raises(ValueError, match=msg):
+ no_nan_df.plot.hist(ax=ax2, weights=idxerror_weights)
+
+
+class TestDataFrameGroupByPlots:
+ def test_grouped_hist_legacy(self):
+ rs = np.random.default_rng(10)
+ df = DataFrame(rs.standard_normal((10, 1)), columns=["A"])
+ df["B"] = to_datetime(
+ rs.integers(
+ 812419200000000000,
+ 819331200000000000,
+ size=10,
+ dtype=np.int64,
+ )
+ )
+ df["C"] = rs.integers(0, 4, 10)
+ df["D"] = ["X"] * 10
+
+ axes = _grouped_hist(df.A, by=df.C)
+ _check_axes_shape(axes, axes_num=4, layout=(2, 2))
+
+ def test_grouped_hist_legacy_axes_shape_no_col(self):
+ rs = np.random.default_rng(10)
+ df = DataFrame(rs.standard_normal((10, 1)), columns=["A"])
+ df["B"] = to_datetime(
+ rs.integers(
+ 812419200000000000,
+ 819331200000000000,
+ size=10,
+ dtype=np.int64,
+ )
+ )
+ df["C"] = rs.integers(0, 4, 10)
+ df["D"] = ["X"] * 10
+ axes = df.hist(by=df.C)
+ _check_axes_shape(axes, axes_num=4, layout=(2, 2))
+
+ def test_grouped_hist_legacy_single_key(self):
+ rs = np.random.default_rng(2)
+ df = DataFrame(rs.standard_normal((10, 1)), columns=["A"])
+ df["B"] = to_datetime(
+ rs.integers(
+ 812419200000000000,
+ 819331200000000000,
+ size=10,
+ dtype=np.int64,
+ )
+ )
+ df["C"] = rs.integers(0, 4, 10)
+ df["D"] = ["X"] * 10
+ # group by a key with single value
+ axes = df.hist(by="D", rot=30)
+ _check_axes_shape(axes, axes_num=1, layout=(1, 1))
+ _check_ticks_props(axes, xrot=30)
+
+ def test_grouped_hist_legacy_grouped_hist_kwargs(self):
+ rs = np.random.default_rng(2)
+ df = DataFrame(rs.standard_normal((10, 1)), columns=["A"])
+ df["B"] = to_datetime(
+ rs.integers(
+ 812419200000000000,
+ 819331200000000000,
+ size=10,
+ dtype=np.int64,
+ )
+ )
+ df["C"] = rs.integers(0, 4, 10)
+ # make sure kwargs to hist are handled
+ xf, yf = 20, 18
+ xrot, yrot = 30, 40
+
+ axes = _grouped_hist(
+ df.A,
+ by=df.C,
+ cumulative=True,
+ bins=4,
+ xlabelsize=xf,
+ xrot=xrot,
+ ylabelsize=yf,
+ yrot=yrot,
+ density=True,
+ )
+ # height of last bin (index 5) must be 1.0
+ for ax in axes.ravel():
+ rects = [
+ x for x in ax.get_children() if isinstance(x, mpl.patches.Rectangle)
+ ]
+ height = rects[-1].get_height()
+ tm.assert_almost_equal(height, 1.0)
+ _check_ticks_props(axes, xlabelsize=xf, xrot=xrot, ylabelsize=yf, yrot=yrot)
+
+ def test_grouped_hist_legacy_grouped_hist(self):
+ rs = np.random.default_rng(2)
+ df = DataFrame(rs.standard_normal((10, 1)), columns=["A"])
+ df["B"] = to_datetime(
+ rs.integers(
+ 812419200000000000,
+ 819331200000000000,
+ size=10,
+ dtype=np.int64,
+ )
+ )
+ df["C"] = rs.integers(0, 4, 10)
+ df["D"] = ["X"] * 10
+ axes = _grouped_hist(df.A, by=df.C, log=True)
+ # scale of y must be 'log'
+ _check_ax_scales(axes, yaxis="log")
+
+ def test_grouped_hist_legacy_external_err(self):
+ rs = np.random.default_rng(2)
+ df = DataFrame(rs.standard_normal((10, 1)), columns=["A"])
+ df["B"] = to_datetime(
+ rs.integers(
+ 812419200000000000,
+ 819331200000000000,
+ size=10,
+ dtype=np.int64,
+ )
+ )
+ df["C"] = rs.integers(0, 4, 10)
+ df["D"] = ["X"] * 10
+ # propagate attr exception from matplotlib.Axes.hist
+ with tm.external_error_raised(AttributeError):
+ _grouped_hist(df.A, by=df.C, foo="bar")
+
+ def test_grouped_hist_legacy_figsize_err(self):
+ rs = np.random.default_rng(2)
+ df = DataFrame(rs.standard_normal((10, 1)), columns=["A"])
+ df["B"] = to_datetime(
+ rs.integers(
+ 812419200000000000,
+ 819331200000000000,
+ size=10,
+ dtype=np.int64,
+ )
+ )
+ df["C"] = rs.integers(0, 4, 10)
+ df["D"] = ["X"] * 10
+ msg = "Specify figure size by tuple instead"
+ with pytest.raises(ValueError, match=msg):
+ df.hist(by="C", figsize="default")
+
+ def test_grouped_hist_legacy2(self):
+ n = 10
+ weight = Series(np.random.default_rng(2).normal(166, 20, size=n))
+ height = Series(np.random.default_rng(2).normal(60, 10, size=n))
+ gender_int = np.random.default_rng(2).choice([0, 1], size=n)
+ df_int = DataFrame({"height": height, "weight": weight, "gender": gender_int})
+ gb = df_int.groupby("gender")
+ axes = gb.hist()
+ assert len(axes) == 2
+ assert len(mpl.pyplot.get_fignums()) == 2
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "msg, plot_col, by_col, layout",
+ [
+ [
+ "Layout of 1x1 must be larger than required size 2",
+ "weight",
+ "gender",
+ (1, 1),
+ ],
+ [
+ "Layout of 1x3 must be larger than required size 4",
+ "height",
+ "category",
+ (1, 3),
+ ],
+ [
+ "At least one dimension of layout must be positive",
+ "height",
+ "category",
+ (-1, -1),
+ ],
+ ],
+ )
+ def test_grouped_hist_layout_error(self, hist_df, msg, plot_col, by_col, layout):
+ df = hist_df
+ with pytest.raises(ValueError, match=msg):
+ df.hist(column=plot_col, by=getattr(df, by_col), layout=layout)
+
+ @pytest.mark.slow
+ def test_grouped_hist_layout_warning(self, hist_df):
+ df = hist_df
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ axes = _check_plot_works(
+ df.hist, column="height", by=df.gender, layout=(2, 1)
+ )
+ _check_axes_shape(axes, axes_num=2, layout=(2, 1))
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "layout, check_layout, figsize",
+ [[(4, 1), (4, 1), None], [(-1, 1), (4, 1), None], [(4, 2), (4, 2), (12, 8)]],
+ )
+ def test_grouped_hist_layout_figsize(self, hist_df, layout, check_layout, figsize):
+ df = hist_df
+ axes = df.hist(column="height", by=df.category, layout=layout, figsize=figsize)
+ _check_axes_shape(axes, axes_num=4, layout=check_layout, figsize=figsize)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("kwargs", [{}, {"column": "height", "layout": (2, 2)}])
+ def test_grouped_hist_layout_by_warning(self, hist_df, kwargs):
+ df = hist_df
+ # GH 6769
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ axes = _check_plot_works(df.hist, by="classroom", **kwargs)
+ _check_axes_shape(axes, axes_num=3, layout=(2, 2))
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "kwargs, axes_num, layout",
+ [
+ [{"by": "gender", "layout": (3, 5)}, 2, (3, 5)],
+ [{"column": ["height", "weight", "category"]}, 3, (2, 2)],
+ ],
+ )
+ def test_grouped_hist_layout_axes(self, hist_df, kwargs, axes_num, layout):
+ df = hist_df
+ axes = df.hist(**kwargs)
+ _check_axes_shape(axes, axes_num=axes_num, layout=layout)
+
+ def test_grouped_hist_multiple_axes(self, hist_df):
+ # GH 6970, GH 7069
+ df = hist_df
+
+ fig, axes = mpl.pyplot.subplots(2, 3)
+ returned = df.hist(column=["height", "weight", "category"], ax=axes[0])
+ _check_axes_shape(returned, axes_num=3, layout=(1, 3))
+ tm.assert_numpy_array_equal(returned, axes[0])
+ assert returned[0].figure is fig
+
+ def test_grouped_hist_multiple_axes_no_cols(self, hist_df):
+ # GH 6970, GH 7069
+ df = hist_df
+
+ fig, axes = mpl.pyplot.subplots(2, 3)
+ returned = df.hist(by="classroom", ax=axes[1])
+ _check_axes_shape(returned, axes_num=3, layout=(1, 3))
+ tm.assert_numpy_array_equal(returned, axes[1])
+ assert returned[0].figure is fig
+
+ def test_grouped_hist_multiple_axes_error(self, hist_df):
+ # GH 6970, GH 7069
+ df = hist_df
+ fig, axes = mpl.pyplot.subplots(2, 3)
+ # pass different number of axes from required
+ msg = "The number of passed axes must be 1, the same as the output plot"
+ with pytest.raises(ValueError, match=msg):
+ axes = df.hist(column="height", ax=axes)
+
+ def test_axis_share_x(self, hist_df):
+ df = hist_df
+ # GH4089
+ ax1, ax2 = df.hist(column="height", by=df.gender, sharex=True)
+
+ # share x
+ assert get_x_axis(ax1).joined(ax1, ax2)
+ assert get_x_axis(ax2).joined(ax1, ax2)
+
+ # don't share y
+ assert not get_y_axis(ax1).joined(ax1, ax2)
+ assert not get_y_axis(ax2).joined(ax1, ax2)
+
+ def test_axis_share_y(self, hist_df):
+ df = hist_df
+ ax1, ax2 = df.hist(column="height", by=df.gender, sharey=True)
+
+ # share y
+ assert get_y_axis(ax1).joined(ax1, ax2)
+ assert get_y_axis(ax2).joined(ax1, ax2)
+
+ # don't share x
+ assert not get_x_axis(ax1).joined(ax1, ax2)
+ assert not get_x_axis(ax2).joined(ax1, ax2)
+
+ def test_axis_share_xy(self, hist_df):
+ df = hist_df
+ ax1, ax2 = df.hist(column="height", by=df.gender, sharex=True, sharey=True)
+
+ # share both x and y
+ assert get_x_axis(ax1).joined(ax1, ax2)
+ assert get_x_axis(ax2).joined(ax1, ax2)
+
+ assert get_y_axis(ax1).joined(ax1, ax2)
+ assert get_y_axis(ax2).joined(ax1, ax2)
+
+ @pytest.mark.parametrize(
+ "histtype, expected",
+ [
+ ("bar", True),
+ ("barstacked", True),
+ ("step", False),
+ ("stepfilled", True),
+ ],
+ )
+ def test_histtype_argument(self, histtype, expected):
+ # GH23992 Verify functioning of histtype argument
+ df = DataFrame(
+ np.random.default_rng(2).integers(1, 10, size=(10, 2)), columns=["a", "b"]
+ )
+ ax = df.hist(by="a", histtype=histtype)
+ _check_patches_all_filled(ax, filled=expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_misc.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_misc.py
new file mode 100644
index 0000000000000000000000000000000000000000..8b6d2499787c02347b0f152fe0d4e0231568e544
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_misc.py
@@ -0,0 +1,866 @@
+"""Test cases for misc plot functions"""
+
+import os
+
+import numpy as np
+import pytest
+
+import pandas.util._test_decorators as td
+
+from pandas import (
+ DataFrame,
+ Index,
+ Series,
+ Timestamp,
+ date_range,
+ interval_range,
+ period_range,
+ plotting,
+ read_csv,
+)
+import pandas._testing as tm
+from pandas.tests.plotting.common import (
+ _check_colors,
+ _check_legend_labels,
+ _check_plot_works,
+ _check_text_labels,
+ _check_ticks_props,
+)
+
+mpl = pytest.importorskip("matplotlib")
+plt = pytest.importorskip("matplotlib.pyplot")
+cm = pytest.importorskip("matplotlib.cm")
+
+import re
+
+from pandas.plotting._matplotlib.style import get_standard_colors
+
+
+@pytest.fixture
+def iris(datapath) -> DataFrame:
+ """
+ The iris dataset as a DataFrame.
+ """
+ return read_csv(datapath("io", "data", "csv", "iris.csv"))
+
+
+@td.skip_if_installed("matplotlib")
+def test_import_error_message():
+ # GH-19810
+ df = DataFrame({"A": [1, 2]})
+
+ with pytest.raises(ImportError, match="matplotlib is required for plotting"):
+ df.plot()
+
+
+def test_get_accessor_args():
+ func = plotting._core.PlotAccessor._get_call_args
+
+ msg = "Called plot accessor for type list, expected Series or DataFrame"
+ with pytest.raises(TypeError, match=msg):
+ func(backend_name="", data=[], args=[], kwargs={})
+
+ msg = "should not be called with positional arguments"
+ with pytest.raises(TypeError, match=msg):
+ func(backend_name="", data=Series(dtype=object), args=["line", None], kwargs={})
+
+ x, y, kind, kwargs = func(
+ backend_name="",
+ data=DataFrame(),
+ args=["x"],
+ kwargs={"y": "y", "kind": "bar", "grid": False},
+ )
+ assert x == "x"
+ assert y == "y"
+ assert kind == "bar"
+ assert kwargs == {"grid": False}
+
+ x, y, kind, kwargs = func(
+ backend_name="pandas.plotting._matplotlib",
+ data=Series(dtype=object),
+ args=[],
+ kwargs={},
+ )
+ assert x is None
+ assert y is None
+ assert kind == "line"
+ assert len(kwargs) == 24
+
+
+@pytest.mark.parametrize("kind", plotting.PlotAccessor._all_kinds)
+@pytest.mark.parametrize(
+ "data", [DataFrame(np.arange(15).reshape(5, 3)), Series(range(5))]
+)
+@pytest.mark.parametrize(
+ "index",
+ [
+ Index(range(5)),
+ date_range("2020-01-01", periods=5),
+ period_range("2020-01-01", periods=5),
+ ],
+)
+def test_savefig(kind, data, index):
+ fig, ax = plt.subplots()
+ data.index = index
+ kwargs = {}
+ if kind in ["hexbin", "scatter", "pie"]:
+ if isinstance(data, Series):
+ pytest.skip(f"{kind} not supported with Series")
+ kwargs = {"x": 0, "y": 1}
+ data.plot(kind=kind, ax=ax, **kwargs)
+ fig.savefig(os.devnull)
+
+
+class TestSeriesPlots:
+ def test_autocorrelation_plot(self):
+ ser = Series(
+ np.arange(10, dtype=np.float64),
+ index=date_range("2020-01-01", periods=10),
+ name="ts",
+ )
+ # Ensure no UserWarning when making plot
+ with tm.assert_produces_warning(None):
+ _check_plot_works(plotting.autocorrelation_plot, series=ser)
+ _check_plot_works(plotting.autocorrelation_plot, series=ser.values)
+
+ ax = plotting.autocorrelation_plot(ser, label="Test")
+ _check_legend_labels(ax, labels=["Test"])
+
+ @pytest.mark.parametrize("kwargs", [{}, {"lag": 5}])
+ def test_lag_plot(self, kwargs):
+ ser = Series(
+ np.arange(10, dtype=np.float64),
+ index=date_range("2020-01-01", periods=10),
+ name="ts",
+ )
+ _check_plot_works(plotting.lag_plot, series=ser, **kwargs)
+
+ def test_bootstrap_plot(self):
+ ser = Series(
+ np.arange(10, dtype=np.float64),
+ index=date_range("2020-01-01", periods=10),
+ name="ts",
+ )
+ _check_plot_works(plotting.bootstrap_plot, series=ser, size=10)
+
+
+class TestDataFramePlots:
+ @pytest.mark.parametrize("pass_axis", [False, True])
+ def test_scatter_matrix_axis(self, pass_axis):
+ pytest.importorskip("scipy")
+ scatter_matrix = plotting.scatter_matrix
+
+ ax = None
+ if pass_axis:
+ _, ax = mpl.pyplot.subplots(3, 3)
+
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 3)))
+
+ # we are plotting multiples on a sub-plot
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ axes = _check_plot_works(
+ scatter_matrix,
+ frame=df,
+ range_padding=0.1,
+ ax=ax,
+ )
+ axes0_labels = axes[0][0].yaxis.get_majorticklabels()
+ # GH 5662
+ expected = ["-2", "-1", "0"]
+ _check_text_labels(axes0_labels, expected)
+ _check_ticks_props(axes, xlabelsize=8, xrot=90, ylabelsize=8, yrot=0)
+
+ @pytest.mark.parametrize("pass_axis", [False, True])
+ def test_scatter_matrix_axis_smaller(self, pass_axis):
+ pytest.importorskip("scipy")
+ scatter_matrix = plotting.scatter_matrix
+
+ ax = None
+ if pass_axis:
+ _, ax = mpl.pyplot.subplots(3, 3)
+
+ df = DataFrame(np.random.default_rng(11).standard_normal((10, 3)))
+ df[0] = (df[0] - 2) / 3
+
+ # we are plotting multiples on a sub-plot
+ with tm.assert_produces_warning(UserWarning, check_stacklevel=False):
+ axes = _check_plot_works(
+ scatter_matrix,
+ frame=df,
+ range_padding=0.1,
+ ax=ax,
+ )
+ axes0_labels = axes[0][0].yaxis.get_majorticklabels()
+ expected = ["-1.25", "-1.0", "-0.75", "-0.5"]
+ _check_text_labels(axes0_labels, expected)
+ _check_ticks_props(axes, xlabelsize=8, xrot=90, ylabelsize=8, yrot=0)
+
+ @pytest.mark.slow
+ def test_andrews_curves_no_warning(self, iris):
+ # Ensure no UserWarning when making plot
+ with tm.assert_produces_warning(None):
+ _check_plot_works(plotting.andrews_curves, frame=iris, class_column="Name")
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "linecolors",
+ [
+ ("#556270", "#4ECDC4", "#C7F464"),
+ ["dodgerblue", "aquamarine", "seagreen"],
+ ],
+ )
+ @pytest.mark.parametrize(
+ "df",
+ [
+ "iris",
+ DataFrame(
+ {
+ "A": np.random.default_rng(2).standard_normal(10),
+ "B": np.random.default_rng(2).standard_normal(10),
+ "C": np.random.default_rng(2).standard_normal(10),
+ "Name": ["A"] * 10,
+ }
+ ),
+ ],
+ )
+ def test_andrews_curves_linecolors(self, request, df, linecolors):
+ if isinstance(df, str):
+ df = request.getfixturevalue(df)
+ ax = _check_plot_works(
+ plotting.andrews_curves, frame=df, class_column="Name", color=linecolors
+ )
+ _check_colors(
+ ax.get_lines()[:10], linecolors=linecolors, mapping=df["Name"][:10]
+ )
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "df",
+ [
+ "iris",
+ DataFrame(
+ {
+ "A": np.random.default_rng(2).standard_normal(10),
+ "B": np.random.default_rng(2).standard_normal(10),
+ "C": np.random.default_rng(2).standard_normal(10),
+ "Name": ["A"] * 10,
+ }
+ ),
+ ],
+ )
+ def test_andrews_curves_cmap(self, request, df):
+ if isinstance(df, str):
+ df = request.getfixturevalue(df)
+ cmaps = [cm.jet(n) for n in np.linspace(0, 1, df["Name"].nunique())]
+ ax = _check_plot_works(
+ plotting.andrews_curves, frame=df, class_column="Name", color=cmaps
+ )
+ _check_colors(ax.get_lines()[:10], linecolors=cmaps, mapping=df["Name"][:10])
+
+ @pytest.mark.slow
+ def test_andrews_curves_handle(self):
+ colors = ["b", "g", "r"]
+ df = DataFrame({"A": [1, 2, 3], "B": [1, 2, 3], "C": [1, 2, 3], "Name": colors})
+ ax = plotting.andrews_curves(df, "Name", color=colors)
+ handles, _ = ax.get_legend_handles_labels()
+ _check_colors(handles, linecolors=colors)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "color",
+ [("#556270", "#4ECDC4", "#C7F464"), ["dodgerblue", "aquamarine", "seagreen"]],
+ )
+ def test_parallel_coordinates_colors(self, iris, color):
+ df = iris
+
+ ax = _check_plot_works(
+ plotting.parallel_coordinates, frame=df, class_column="Name", color=color
+ )
+ _check_colors(ax.get_lines()[:10], linecolors=color, mapping=df["Name"][:10])
+
+ @pytest.mark.slow
+ def test_parallel_coordinates_cmap(self, iris):
+ df = iris
+
+ ax = _check_plot_works(
+ plotting.parallel_coordinates,
+ frame=df,
+ class_column="Name",
+ colormap=cm.jet,
+ )
+ cmaps = [mpl.cm.jet(n) for n in np.linspace(0, 1, df["Name"].nunique())]
+ _check_colors(ax.get_lines()[:10], linecolors=cmaps, mapping=df["Name"][:10])
+
+ @pytest.mark.slow
+ def test_parallel_coordinates_line_diff(self, iris):
+ df = iris
+
+ ax = _check_plot_works(
+ plotting.parallel_coordinates, frame=df, class_column="Name"
+ )
+ nlines = len(ax.get_lines())
+ nxticks = len(ax.xaxis.get_ticklabels())
+
+ ax = _check_plot_works(
+ plotting.parallel_coordinates, frame=df, class_column="Name", axvlines=False
+ )
+ assert len(ax.get_lines()) == (nlines - nxticks)
+
+ @pytest.mark.slow
+ def test_parallel_coordinates_handles(self, iris):
+ df = iris
+ colors = ["b", "g", "r"]
+ df = DataFrame({"A": [1, 2, 3], "B": [1, 2, 3], "C": [1, 2, 3], "Name": colors})
+ ax = plotting.parallel_coordinates(df, "Name", color=colors)
+ handles, _ = ax.get_legend_handles_labels()
+ _check_colors(handles, linecolors=colors)
+
+ # not sure if this is indicative of a problem
+ @pytest.mark.filterwarnings("ignore:Attempting to set:UserWarning")
+ def test_parallel_coordinates_with_sorted_labels(self):
+ # GH 15908
+ df = DataFrame(
+ {
+ "feat": list(range(30)),
+ "class": [2 for _ in range(10)]
+ + [3 for _ in range(10)]
+ + [1 for _ in range(10)],
+ }
+ )
+ ax = plotting.parallel_coordinates(df, "class", sort_labels=True)
+ polylines, labels = ax.get_legend_handles_labels()
+ color_label_tuples = zip(
+ [polyline.get_color() for polyline in polylines], labels, strict=True
+ )
+ ordered_color_label_tuples = sorted(color_label_tuples, key=lambda x: x[1])
+ prev_next_tupels = zip(
+ list(ordered_color_label_tuples[0:-1]),
+ list(ordered_color_label_tuples[1:]),
+ strict=True,
+ )
+ for prev, nxt in prev_next_tupels:
+ # labels and colors are ordered strictly increasing
+ assert prev[1] < nxt[1] and prev[0] < nxt[0]
+
+ def test_radviz_no_warning(self, iris):
+ # Ensure no UserWarning when making plot
+ with tm.assert_produces_warning(None):
+ _check_plot_works(plotting.radviz, frame=iris, class_column="Name")
+
+ @pytest.mark.parametrize(
+ "color",
+ [("#556270", "#4ECDC4", "#C7F464"), ["dodgerblue", "aquamarine", "seagreen"]],
+ )
+ def test_radviz_color(self, iris, color):
+ df = iris
+ ax = _check_plot_works(
+ plotting.radviz, frame=df, class_column="Name", color=color
+ )
+ # skip Circle drawn as ticks
+ patches = [p for p in ax.patches[:20] if p.get_label() != ""]
+ _check_colors(patches[:10], facecolors=color, mapping=df["Name"][:10])
+
+ def test_radviz_color_cmap(self, iris):
+ df = iris
+ ax = _check_plot_works(
+ plotting.radviz, frame=df, class_column="Name", colormap=cm.jet
+ )
+ cmaps = [mpl.cm.jet(n) for n in np.linspace(0, 1, df["Name"].nunique())]
+ patches = [p for p in ax.patches[:20] if p.get_label() != ""]
+ _check_colors(patches, facecolors=cmaps, mapping=df["Name"][:10])
+
+ def test_radviz_colors_handles(self):
+ colors = [[0.0, 0.0, 1.0, 1.0], [0.0, 0.5, 1.0, 1.0], [1.0, 0.0, 0.0, 1.0]]
+ df = DataFrame(
+ {"A": [1, 2, 3], "B": [2, 1, 3], "C": [3, 2, 1], "Name": ["b", "g", "r"]}
+ )
+ ax = plotting.radviz(df, "Name", color=colors)
+ handles, _ = ax.get_legend_handles_labels()
+ _check_colors(handles, facecolors=colors)
+
+ def test_subplot_titles(self, iris):
+ df = iris.drop("Name", axis=1).head()
+ # Use the column names as the subplot titles
+ title = list(df.columns)
+
+ # Case len(title) == len(df)
+ plot = df.plot(subplots=True, title=title)
+ assert [p.get_title() for p in plot] == title
+
+ def test_subplot_titles_too_much(self, iris):
+ df = iris.drop("Name", axis=1).head()
+ # Use the column names as the subplot titles
+ title = list(df.columns)
+ # Case len(title) > len(df)
+ msg = (
+ "The length of `title` must equal the number of columns if "
+ "using `title` of type `list` and `subplots=True`"
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.plot(subplots=True, title=[*title, "kittens > puppies"])
+
+ def test_subplot_titles_too_little(self, iris):
+ df = iris.drop("Name", axis=1).head()
+ # Use the column names as the subplot titles
+ title = list(df.columns)
+ msg = (
+ "The length of `title` must equal the number of columns if "
+ "using `title` of type `list` and `subplots=True`"
+ )
+ # Case len(title) < len(df)
+ with pytest.raises(ValueError, match=msg):
+ df.plot(subplots=True, title=title[:2])
+
+ def test_subplot_titles_subplots_false(self, iris):
+ df = iris.drop("Name", axis=1).head()
+ # Use the column names as the subplot titles
+ title = list(df.columns)
+ # Case subplots=False and title is of type list
+ msg = (
+ "Using `title` of type `list` is not supported unless "
+ "`subplots=True` is passed"
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.plot(subplots=False, title=title)
+
+ def test_subplot_titles_numeric_square_layout(self, iris):
+ df = iris.drop("Name", axis=1).head()
+ # Use the column names as the subplot titles
+ title = list(df.columns)
+ # Case df with 3 numeric columns but layout of (2,2)
+ plot = df.drop("SepalWidth", axis=1).plot(
+ subplots=True, layout=(2, 2), title=title[:-1]
+ )
+ title_list = [ax.get_title() for sublist in plot for ax in sublist]
+ assert title_list == [*title[:3], ""]
+
+ def test_get_standard_colors_random_seed(self):
+ # GH17525
+ df = DataFrame(np.zeros((10, 10)))
+
+ # Make sure that the random seed isn't reset by get_standard_colors
+ plotting.parallel_coordinates(df, 0)
+ rand1 = np.random.default_rng(None).random()
+ plotting.parallel_coordinates(df, 0)
+ rand2 = np.random.default_rng(None).random()
+ assert rand1 != rand2
+
+ def test_get_standard_colors_consistency(self):
+ # GH17525
+ # Make sure it produces the same colors every time it's called
+ color1 = get_standard_colors(1, color_type="random")
+ color2 = get_standard_colors(1, color_type="random")
+ assert color1 == color2
+
+ def test_get_standard_colors_default_num_colors(self):
+ # Make sure the default color_types returns the specified amount
+ color1 = get_standard_colors(1, color_type="default")
+ color2 = get_standard_colors(9, color_type="default")
+ color3 = get_standard_colors(20, color_type="default")
+ assert len(color1) == 1
+ assert len(color2) == 9
+ assert len(color3) == 20
+
+ def test_plot_single_color(self):
+ # Example from #20585. All 3 bars should have the same color
+ df = DataFrame(
+ {
+ "account-start": ["2017-02-03", "2017-03-03", "2017-01-01"],
+ "client": ["Alice Anders", "Bob Baker", "Charlie Chaplin"],
+ "balance": [-1432.32, 10.43, 30000.00],
+ "db-id": [1234, 2424, 251],
+ "proxy-id": [525, 1525, 2542],
+ "rank": [52, 525, 32],
+ }
+ )
+ ax = df.client.value_counts().plot.bar()
+ colors = [rect.get_facecolor() for rect in ax.get_children()[0:3]]
+ assert all(color == colors[0] for color in colors)
+
+ def test_get_standard_colors_no_appending(self):
+ # GH20726
+
+ # Make sure not to add more colors so that matplotlib can cycle
+ # correctly.
+ color_before = mpl.cm.gnuplot(range(5))
+ color_after = get_standard_colors(1, color=color_before)
+ assert len(color_after) == len(color_before)
+
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((48, 4)), columns=list("ABCD")
+ )
+
+ color_list = mpl.cm.gnuplot(np.linspace(0, 1, 16))
+ p = df.A.plot.bar(figsize=(16, 7), color=color_list)
+ assert p.patches[1].get_facecolor() == p.patches[17].get_facecolor()
+
+ @pytest.mark.parametrize("kind", ["bar", "line"])
+ def test_dictionary_color(self, kind):
+ # issue-8193
+ # Test plot color dictionary format
+ data_files = ["a", "b"]
+
+ expected = [(0.5, 0.24, 0.6), (0.3, 0.7, 0.7)]
+
+ df1 = DataFrame(np.random.default_rng(2).random((2, 2)), columns=data_files)
+ dic_color = {"b": (0.3, 0.7, 0.7), "a": (0.5, 0.24, 0.6)}
+
+ ax = df1.plot(kind=kind, color=dic_color)
+ if kind == "bar":
+ colors = [rect.get_facecolor()[0:-1] for rect in ax.get_children()[0:3:2]]
+ else:
+ colors = [rect.get_color() for rect in ax.get_lines()[0:2]]
+ assert all(color == expected[index] for index, color in enumerate(colors))
+
+ def test_bar_plot(self):
+ # GH38947
+ # Test bar plot with string and int index
+ expected = [mpl.text.Text(0, 0, "0"), mpl.text.Text(1, 0, "Total")]
+
+ df = DataFrame(
+ {
+ "a": [1, 2],
+ },
+ index=Index([0, "Total"]),
+ )
+ plot_bar = df.plot.bar()
+ assert all(
+ (a.get_text() == b.get_text())
+ for a, b in zip(plot_bar.get_xticklabels(), expected, strict=True)
+ )
+
+ def test_barh_plot_labels_mixed_integer_string(self):
+ # GH39126
+ # Test barh plot with string and integer at the same column
+ df = DataFrame([{"word": 1, "value": 0}, {"word": "knowledge", "value": 2}])
+ plot_barh = df.plot.barh(x="word", legend=None)
+ expected_yticklabels = [
+ mpl.text.Text(0, 0, "1"),
+ mpl.text.Text(0, 1, "knowledge"),
+ ]
+ assert all(
+ actual.get_text() == expected.get_text()
+ for actual, expected in zip(
+ plot_barh.get_yticklabels(), expected_yticklabels, strict=True
+ )
+ )
+
+ def test_has_externally_shared_axis_x_axis(self):
+ # GH33819
+ # Test _has_externally_shared_axis() works for x-axis
+ func = plotting._matplotlib.tools._has_externally_shared_axis
+
+ fig = mpl.pyplot.figure()
+ plots = fig.subplots(2, 4)
+
+ # Create *externally* shared axes for first and third columns
+ plots[0][0] = fig.add_subplot(231, sharex=plots[1][0])
+ plots[0][2] = fig.add_subplot(233, sharex=plots[1][2])
+
+ # Create *internally* shared axes for second and third columns
+ plots[0][1].twinx()
+ plots[0][2].twinx()
+
+ # First column is only externally shared
+ # Second column is only internally shared
+ # Third column is both
+ # Fourth column is neither
+ assert func(plots[0][0], "x")
+ assert not func(plots[0][1], "x")
+ assert func(plots[0][2], "x")
+ assert not func(plots[0][3], "x")
+
+ def test_has_externally_shared_axis_y_axis(self):
+ # GH33819
+ # Test _has_externally_shared_axis() works for y-axis
+ func = plotting._matplotlib.tools._has_externally_shared_axis
+
+ fig = mpl.pyplot.figure()
+ plots = fig.subplots(4, 2)
+
+ # Create *externally* shared axes for first and third rows
+ plots[0][0] = fig.add_subplot(321, sharey=plots[0][1])
+ plots[2][0] = fig.add_subplot(325, sharey=plots[2][1])
+
+ # Create *internally* shared axes for second and third rows
+ plots[1][0].twiny()
+ plots[2][0].twiny()
+
+ # First row is only externally shared
+ # Second row is only internally shared
+ # Third row is both
+ # Fourth row is neither
+ assert func(plots[0][0], "y")
+ assert not func(plots[1][0], "y")
+ assert func(plots[2][0], "y")
+ assert not func(plots[3][0], "y")
+
+ def test_has_externally_shared_axis_invalid_compare_axis(self):
+ # GH33819
+ # Test _has_externally_shared_axis() raises an exception when
+ # passed an invalid value as compare_axis parameter
+ func = plotting._matplotlib.tools._has_externally_shared_axis
+
+ fig = mpl.pyplot.figure()
+ plots = fig.subplots(4, 2)
+
+ # Create arbitrary axes
+ plots[0][0] = fig.add_subplot(321, sharey=plots[0][1])
+
+ # Check that an invalid compare_axis value triggers the expected exception
+ msg = "needs 'x' or 'y' as a second parameter"
+ with pytest.raises(ValueError, match=msg):
+ func(plots[0][0], "z")
+
+ def test_externally_shared_axes(self):
+ # Example from GH33819
+ # Create data
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).standard_normal(10),
+ "b": np.random.default_rng(2).standard_normal(10),
+ }
+ )
+
+ # Create figure
+ fig = mpl.pyplot.figure()
+ plots = fig.subplots(2, 3)
+
+ # Create *externally* shared axes
+ plots[0][0] = fig.add_subplot(231, sharex=plots[1][0])
+ # note: no plots[0][1] that's the twin only case
+ plots[0][2] = fig.add_subplot(233, sharex=plots[1][2])
+
+ # Create *internally* shared axes
+ # note: no plots[0][0] that's the external only case
+ twin_ax1 = plots[0][1].twinx()
+ twin_ax2 = plots[0][2].twinx()
+
+ # Plot data to primary axes
+ df["a"].plot(ax=plots[0][0], title="External share only").set_xlabel(
+ "this label should never be visible"
+ )
+ df["a"].plot(ax=plots[1][0])
+
+ df["a"].plot(ax=plots[0][1], title="Internal share (twin) only").set_xlabel(
+ "this label should always be visible"
+ )
+ df["a"].plot(ax=plots[1][1])
+
+ df["a"].plot(ax=plots[0][2], title="Both").set_xlabel(
+ "this label should never be visible"
+ )
+ df["a"].plot(ax=plots[1][2])
+
+ # Plot data to twinned axes
+ df["b"].plot(ax=twin_ax1, color="green")
+ df["b"].plot(ax=twin_ax2, color="yellow")
+
+ assert not plots[0][0].xaxis.get_label().get_visible()
+ assert plots[0][1].xaxis.get_label().get_visible()
+ assert not plots[0][2].xaxis.get_label().get_visible()
+
+ def test_plot_bar_axis_units_timestamp_conversion(self):
+ # GH 38736
+ # Ensure string x-axis from the second plot will not be converted to datetime
+ # due to axis data from first plot
+ df = DataFrame(
+ [1.0],
+ index=[Timestamp("2022-02-22 22:22:22")],
+ )
+ _check_plot_works(df.plot)
+ s = Series({"A": 1.0})
+ _check_plot_works(s.plot.bar)
+
+ def test_bar_plt_xaxis_intervalrange(self):
+ # GH 38969
+ # Ensure IntervalIndex x-axis produces a bar plot as expected
+ expected = [mpl.text.Text(0, 0, "([0, 1],)"), mpl.text.Text(1, 0, "([1, 2],)")]
+ s = Series(
+ [1, 2],
+ index=[interval_range(0, 2, closed="both")],
+ )
+ _check_plot_works(s.plot.bar)
+ assert all(
+ (a.get_text() == b.get_text())
+ for a, b in zip(s.plot.bar().get_xticklabels(), expected, strict=True)
+ )
+
+
+@pytest.fixture
+def df_bar_data():
+ return np.random.default_rng(3).integers(0, 100, 5)
+
+
+@pytest.fixture
+def df_bar_df(df_bar_data) -> DataFrame:
+ df_bar_df = DataFrame(
+ {
+ "A": df_bar_data,
+ "B": df_bar_data[::-1],
+ "C": df_bar_data[0],
+ "D": df_bar_data[-1],
+ }
+ )
+ return df_bar_df
+
+
+def _df_bar_xyheight_from_ax_helper(df_bar_data, ax, subplot_division):
+ subplot_data_df_list = []
+
+ # get xy and height of squares representing data, separated by subplots
+ for i in range(len(subplot_division)):
+ subplot_data = np.array(
+ [
+ (x.get_x(), x.get_y(), x.get_height())
+ for x in ax[i].findobj(plt.Rectangle)
+ if x.get_height() in df_bar_data
+ ]
+ )
+ subplot_data_df_list.append(
+ DataFrame(data=subplot_data, columns=["x_coord", "y_coord", "height"])
+ )
+
+ return subplot_data_df_list
+
+
+def _df_bar_subplot_checker(df_bar_data, df_bar_df, subplot_data_df, subplot_columns):
+ subplot_sliced_by_source = [
+ subplot_data_df.iloc[
+ len(df_bar_data) * i : len(df_bar_data) * (i + 1)
+ ].reset_index()
+ for i in range(len(subplot_columns))
+ ]
+
+ if len(subplot_columns) == 1:
+ expected_total_height = df_bar_df.loc[:, subplot_columns[0]]
+ else:
+ expected_total_height = df_bar_df.loc[:, subplot_columns].sum(axis=1)
+
+ for i in range(len(subplot_columns)):
+ sliced_df = subplot_sliced_by_source[i]
+ if i == 0:
+ # Checks that the bar chart starts y=0
+ assert (sliced_df["y_coord"] == 0).all()
+ height_iter = sliced_df["y_coord"].add(sliced_df["height"])
+ else:
+ height_iter = height_iter + sliced_df["height"]
+
+ if i + 1 == len(subplot_columns):
+ # Checks final height matches what is expected
+ tm.assert_series_equal(
+ height_iter, expected_total_height, check_names=False, check_dtype=False
+ )
+ else:
+ # Checks each preceding bar ends where the next one starts
+ next_start_coord = subplot_sliced_by_source[i + 1]["y_coord"]
+ tm.assert_series_equal(
+ height_iter, next_start_coord, check_names=False, check_dtype=False
+ )
+
+
+# GH Issue 61018
+@pytest.mark.parametrize("columns_used", [["A", "B"], ["C", "D"], ["D", "A"]])
+def test_bar_1_subplot_1_double_stacked(df_bar_data, df_bar_df, columns_used):
+ df_bar_df_trimmed = df_bar_df[columns_used]
+ subplot_division = [columns_used]
+ ax = df_bar_df_trimmed.plot(subplots=subplot_division, kind="bar", stacked=True)
+ subplot_data_df_list = _df_bar_xyheight_from_ax_helper(
+ df_bar_data, ax, subplot_division
+ )
+ for i in range(len(subplot_data_df_list)):
+ _df_bar_subplot_checker(
+ df_bar_data, df_bar_df_trimmed, subplot_data_df_list[i], subplot_division[i]
+ )
+
+
+@pytest.mark.parametrize(
+ "columns_used", [["A", "B", "C"], ["A", "C", "B"], ["D", "A", "C"]]
+)
+def test_bar_2_subplot_1_double_stacked(df_bar_data, df_bar_df, columns_used):
+ df_bar_df_trimmed = df_bar_df[columns_used]
+ subplot_division = [(columns_used[0], columns_used[1]), (columns_used[2],)]
+ ax = df_bar_df_trimmed.plot(subplots=subplot_division, kind="bar", stacked=True)
+ subplot_data_df_list = _df_bar_xyheight_from_ax_helper(
+ df_bar_data, ax, subplot_division
+ )
+ for i in range(len(subplot_data_df_list)):
+ _df_bar_subplot_checker(
+ df_bar_data, df_bar_df_trimmed, subplot_data_df_list[i], subplot_division[i]
+ )
+
+
+@pytest.mark.parametrize(
+ "subplot_division",
+ [
+ [("A", "B"), ("C", "D")],
+ [("A", "D"), ("C", "B")],
+ [("B", "C"), ("D", "A")],
+ [("B", "D"), ("C", "A")],
+ ],
+)
+def test_bar_2_subplot_2_double_stacked(df_bar_data, df_bar_df, subplot_division):
+ ax = df_bar_df.plot(subplots=subplot_division, kind="bar", stacked=True)
+ subplot_data_df_list = _df_bar_xyheight_from_ax_helper(
+ df_bar_data, ax, subplot_division
+ )
+ for i in range(len(subplot_data_df_list)):
+ _df_bar_subplot_checker(
+ df_bar_data, df_bar_df, subplot_data_df_list[i], subplot_division[i]
+ )
+
+
+@pytest.mark.parametrize(
+ "subplot_division",
+ [[("A", "B", "C")], [("A", "D", "B")], [("C", "A", "D")], [("D", "C", "A")]],
+)
+def test_bar_2_subplots_1_triple_stacked(df_bar_data, df_bar_df, subplot_division):
+ ax = df_bar_df.plot(subplots=subplot_division, kind="bar", stacked=True)
+ subplot_data_df_list = _df_bar_xyheight_from_ax_helper(
+ df_bar_data, ax, subplot_division
+ )
+ for i in range(len(subplot_data_df_list)):
+ _df_bar_subplot_checker(
+ df_bar_data, df_bar_df, subplot_data_df_list[i], subplot_division[i]
+ )
+
+
+def test_bar_subplots_stacking_bool(df_bar_data, df_bar_df):
+ subplot_division = [("A"), ("B"), ("C"), ("D")]
+ ax = df_bar_df.plot(subplots=True, kind="bar", stacked=True)
+ subplot_data_df_list = _df_bar_xyheight_from_ax_helper(
+ df_bar_data, ax, subplot_division
+ )
+ for i in range(len(subplot_data_df_list)):
+ _df_bar_subplot_checker(
+ df_bar_data, df_bar_df, subplot_data_df_list[i], subplot_division[i]
+ )
+
+
+def test_plot_bar_label_count_default():
+ df = DataFrame(
+ [(30, 10, 10, 10), (20, 20, 20, 20), (10, 30, 30, 10)], columns=list("ABCD")
+ )
+ df.plot(subplots=True, kind="bar", title=["A", "B", "C", "D"])
+
+
+def test_plot_bar_label_count_expected_fail():
+ df = DataFrame(
+ [(30, 10, 10, 10), (20, 20, 20, 20), (10, 30, 30, 10)], columns=list("ABCD")
+ )
+ error_regex = re.escape(
+ "The number of titles (4) must equal the number of subplots (3)."
+ )
+ with pytest.raises(ValueError, match=error_regex):
+ df.plot(
+ subplots=[("A", "B")],
+ kind="bar",
+ title=["A&B", "C", "D", "Extra Title"],
+ )
+
+
+def test_plot_bar_label_count_expected_success():
+ df = DataFrame(
+ [(30, 10, 10, 10), (20, 20, 20, 20), (10, 30, 30, 10)], columns=list("ABCD")
+ )
+ df.plot(subplots=[("A", "B", "D")], kind="bar", title=["A&B&D", "C"])
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_series.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_series.py
new file mode 100644
index 0000000000000000000000000000000000000000..065725c65773ae273e2857fde14c558d44e40f5d
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_series.py
@@ -0,0 +1,1022 @@
+"""Test cases for Series.plot"""
+
+from datetime import datetime
+from itertools import chain
+
+import numpy as np
+import pytest
+
+from pandas.compat import is_platform_linux
+import pandas.util._test_decorators as td
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Series,
+ date_range,
+ period_range,
+ plotting,
+)
+import pandas._testing as tm
+from pandas.tests.plotting.common import (
+ _check_ax_scales,
+ _check_axes_shape,
+ _check_colors,
+ _check_grid_settings,
+ _check_has_errorbars,
+ _check_legend_labels,
+ _check_plot_works,
+ _check_text_labels,
+ _check_ticks_props,
+ _unpack_cycler,
+ get_y_axis,
+)
+from pandas.util.version import Version
+
+from pandas.tseries.offsets import CustomBusinessDay
+
+mpl = pytest.importorskip("matplotlib")
+plt = pytest.importorskip("matplotlib.pyplot")
+
+from pandas.plotting._matplotlib.converter import DatetimeConverter
+from pandas.plotting._matplotlib.style import get_standard_colors
+
+pytestmark = [
+ pytest.mark.filterwarnings(
+ "ignore:divide by zero encountered in scalar divide:RuntimeWarning"
+ ),
+ pytest.mark.filterwarnings(
+ "ignore:invalid value encountered in scalar multiply:RuntimeWarning"
+ ),
+]
+
+
+@pytest.fixture
+def ts():
+ return Series(
+ np.arange(10, dtype=np.float64),
+ index=date_range("2020-01-01", periods=10),
+ name="ts",
+ )
+
+
+@pytest.fixture
+def series():
+ return Series(
+ range(10), dtype=np.float64, name="series", index=[f"i_{i}" for i in range(10)]
+ )
+
+
+class TestSeriesPlots:
+ @pytest.mark.slow
+ @pytest.mark.parametrize("kwargs", [{"label": "foo"}, {"use_index": False}])
+ def test_plot(self, ts, kwargs):
+ _check_plot_works(ts.plot, **kwargs)
+
+ @pytest.mark.slow
+ def test_plot_tick_props(self, ts):
+ axes = _check_plot_works(ts.plot, rot=0)
+ _check_ticks_props(axes, xrot=0)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "scale, exp_scale",
+ [
+ [{"logy": True}, {"yaxis": "log"}],
+ [{"logx": True}, {"xaxis": "log"}],
+ [{"loglog": True}, {"xaxis": "log", "yaxis": "log"}],
+ ],
+ )
+ def test_plot_scales(self, ts, scale, exp_scale):
+ ax = _check_plot_works(ts.plot, style=".", **scale)
+ _check_ax_scales(ax, **exp_scale)
+
+ @pytest.mark.slow
+ def test_plot_ts_bar(self, ts):
+ _check_plot_works(ts[:10].plot.bar)
+
+ @pytest.mark.slow
+ def test_plot_ts_area_stacked(self, ts):
+ _check_plot_works(ts.plot.area, stacked=False)
+
+ def test_plot_iseries(self):
+ ser = Series(range(5), period_range("2020-01-01", periods=5))
+ _check_plot_works(ser.plot)
+
+ @pytest.mark.parametrize(
+ "kind",
+ [
+ "line",
+ "bar",
+ "barh",
+ pytest.param("kde", marks=td.skip_if_no("scipy")),
+ "hist",
+ "box",
+ ],
+ )
+ def test_plot_series_kinds(self, series, kind):
+ _check_plot_works(series[:5].plot, kind=kind)
+
+ def test_plot_series_barh(self, series):
+ _check_plot_works(series[:10].plot.barh)
+
+ def test_plot_series_bar_ax(self):
+ ax = _check_plot_works(
+ Series(np.random.default_rng(2).standard_normal(10)).plot.bar, color="black"
+ )
+ _check_colors([ax.patches[0]], facecolors=["black"])
+
+ @pytest.mark.parametrize("kwargs", [{}, {"layout": (-1, 1)}, {"layout": (1, -1)}])
+ def test_plot_6951(self, ts, kwargs):
+ # GH 6951
+ ax = _check_plot_works(ts.plot, subplots=True, **kwargs)
+ _check_axes_shape(ax, axes_num=1, layout=(1, 1))
+
+ def test_plot_figsize_and_title(self, series):
+ # figsize and title
+ _, ax = mpl.pyplot.subplots()
+ ax = series.plot(title="Test", figsize=(16, 8), ax=ax)
+ _check_text_labels(ax.title, "Test")
+ _check_axes_shape(ax, axes_num=1, layout=(1, 1), figsize=(16, 8))
+
+ def test_dont_modify_rcParams(self):
+ # GH 8242
+ key = "axes.prop_cycle"
+ colors = mpl.pyplot.rcParams[key]
+ _, ax = mpl.pyplot.subplots()
+ Series([1, 2, 3]).plot(ax=ax)
+ assert colors == mpl.pyplot.rcParams[key]
+
+ @pytest.mark.parametrize("kwargs", [{}, {"secondary_y": True}])
+ def test_ts_line_lim(self, ts, kwargs):
+ _, ax = mpl.pyplot.subplots()
+ ax = ts.plot(ax=ax, **kwargs)
+ xmin, xmax = ax.get_xlim()
+ lines = ax.get_lines()
+ assert xmin <= lines[0].get_data(orig=False)[0][0]
+ assert xmax >= lines[0].get_data(orig=False)[0][-1]
+
+ def test_ts_area_lim(self, ts):
+ _, ax = mpl.pyplot.subplots()
+ ax = ts.plot.area(stacked=False, ax=ax)
+ xmin, xmax = ax.get_xlim()
+ line = ax.get_lines()[0].get_data(orig=False)[0]
+ assert xmin <= line[0]
+ assert xmax >= line[-1]
+ _check_ticks_props(ax, xrot=0)
+
+ def test_ts_area_lim_xcompat(self, ts):
+ # GH 7471
+ _, ax = mpl.pyplot.subplots()
+ ax = ts.plot.area(stacked=False, x_compat=True, ax=ax)
+ xmin, xmax = ax.get_xlim()
+ line = ax.get_lines()[0].get_data(orig=False)[0]
+ assert xmin <= line[0]
+ assert xmax >= line[-1]
+ _check_ticks_props(ax, xrot=30)
+
+ def test_ts_tz_area_lim_xcompat(self, ts):
+ tz_ts = ts.copy()
+ tz_ts.index = tz_ts.tz_localize("GMT").tz_convert("CET")
+ _, ax = mpl.pyplot.subplots()
+ ax = tz_ts.plot.area(stacked=False, x_compat=True, ax=ax)
+ xmin, xmax = ax.get_xlim()
+ line = ax.get_lines()[0].get_data(orig=False)[0]
+ assert xmin <= line[0]
+ assert xmax >= line[-1]
+ _check_ticks_props(ax, xrot=0)
+
+ def test_ts_tz_area_lim_xcompat_secondary_y(self, ts):
+ tz_ts = ts.copy()
+ tz_ts.index = tz_ts.tz_localize("GMT").tz_convert("CET")
+ _, ax = mpl.pyplot.subplots()
+ ax = tz_ts.plot.area(stacked=False, secondary_y=True, ax=ax)
+ xmin, xmax = ax.get_xlim()
+ line = ax.get_lines()[0].get_data(orig=False)[0]
+ assert xmin <= line[0]
+ assert xmax >= line[-1]
+ _check_ticks_props(ax, xrot=0)
+
+ def test_area_sharey_dont_overwrite(self, ts):
+ # GH37942
+ fig, (ax1, ax2) = mpl.pyplot.subplots(1, 2, sharey=True)
+
+ abs(ts).plot(ax=ax1, kind="area")
+ abs(ts).plot(ax=ax2, kind="area")
+
+ assert get_y_axis(ax1).joined(ax1, ax2)
+ assert get_y_axis(ax2).joined(ax1, ax2)
+
+ def test_label(self):
+ s = Series([1, 2])
+ _, ax = mpl.pyplot.subplots()
+ ax = s.plot(label="LABEL", legend=True, ax=ax)
+ _check_legend_labels(ax, labels=["LABEL"])
+
+ def test_label_none(self):
+ s = Series([1, 2])
+ _, ax = mpl.pyplot.subplots()
+ ax = s.plot(legend=True, ax=ax)
+ _check_legend_labels(ax, labels=[""])
+
+ def test_label_ser_name(self):
+ s = Series([1, 2], name="NAME")
+ _, ax = mpl.pyplot.subplots()
+ ax = s.plot(legend=True, ax=ax)
+ _check_legend_labels(ax, labels=["NAME"])
+
+ def test_label_ser_name_override(self):
+ s = Series([1, 2], name="NAME")
+ # override the default
+ _, ax = mpl.pyplot.subplots()
+ ax = s.plot(legend=True, label="LABEL", ax=ax)
+ _check_legend_labels(ax, labels=["LABEL"])
+
+ def test_label_ser_name_override_dont_draw(self):
+ s = Series([1, 2], name="NAME")
+ # Add lebel info, but don't draw
+ _, ax = mpl.pyplot.subplots()
+ ax = s.plot(legend=False, label="LABEL", ax=ax)
+ assert ax.get_legend() is None # Hasn't been drawn
+ ax.legend() # draw it
+ _check_legend_labels(ax, labels=["LABEL"])
+
+ def test_boolean(self):
+ # GH 23719
+ s = Series([False, False, True])
+ _check_plot_works(s.plot, include_bool=True)
+
+ msg = "no numeric data to plot"
+ with pytest.raises(TypeError, match=msg):
+ _check_plot_works(s.plot)
+
+ @pytest.mark.parametrize("index", [None, date_range("2020-01-01", periods=4)])
+ def test_line_area_nan_series(self, index):
+ values = [1, 2, np.nan, 3]
+ d = Series(values, index=index)
+ ax = _check_plot_works(d.plot)
+ masked = ax.lines[0].get_ydata()
+ # remove nan for comparison purpose
+ exp = np.array([1, 2, 3], dtype=np.float64)
+ tm.assert_numpy_array_equal(np.delete(masked.data, 2), exp)
+ tm.assert_numpy_array_equal(masked.mask, np.array([False, False, True, False]))
+
+ expected = np.array([1, 2, 0, 3], dtype=np.float64)
+ ax = _check_plot_works(d.plot, stacked=True)
+ tm.assert_numpy_array_equal(ax.lines[0].get_ydata(), expected)
+ ax = _check_plot_works(d.plot.area)
+ tm.assert_numpy_array_equal(ax.lines[0].get_ydata(), expected)
+ ax = _check_plot_works(d.plot.area, stacked=False)
+ tm.assert_numpy_array_equal(ax.lines[0].get_ydata(), expected)
+
+ def test_line_use_index_false(self):
+ s = Series([1, 2, 3], index=["a", "b", "c"])
+ s.index.name = "The Index"
+ _, ax = mpl.pyplot.subplots()
+ ax = s.plot(use_index=False, ax=ax)
+ label = ax.get_xlabel()
+ assert label == ""
+
+ def test_line_use_index_false_diff_var(self):
+ s = Series([1, 2, 3], index=["a", "b", "c"])
+ s.index.name = "The Index"
+ _, ax = mpl.pyplot.subplots()
+ ax2 = s.plot.bar(use_index=False, ax=ax)
+ label2 = ax2.get_xlabel()
+ assert label2 == ""
+
+ @pytest.mark.xfail(
+ is_platform_linux(),
+ reason="Weird rounding problems",
+ strict=False,
+ )
+ @pytest.mark.parametrize("axis, meth", [("yaxis", "bar"), ("xaxis", "barh")])
+ def test_bar_log(self, axis, meth):
+ # matplotlib 3.11 no longer pads log-axis ticks a decade past the
+ # view limits, so the outermost tick is dropped (GH#65918)
+ expected = (
+ np.array([1e-1, 1e0, 1e1, 1e2, 1e3, 1e4])
+ if Version(mpl.__version__) < Version("3.11.0rc1")
+ else np.array([1e-1, 1e0, 1e1, 1e2, 1e3])
+ )
+
+ _, ax = mpl.pyplot.subplots()
+ ax = getattr(Series([200, 500]).plot, meth)(log=True, ax=ax)
+ tm.assert_numpy_array_equal(getattr(ax, axis).get_ticklocs(), expected)
+
+ @pytest.mark.xfail(
+ is_platform_linux(),
+ reason="Weird rounding problems",
+ strict=False,
+ )
+ @pytest.mark.parametrize(
+ "axis, kind, res_meth",
+ [["yaxis", "bar", "get_ylim"], ["xaxis", "barh", "get_xlim"]],
+ )
+ def test_bar_log_kind_bar(self, axis, kind, res_meth):
+ # GH 9905
+ # matplotlib 3.11 no longer pads log-axis ticks a decade past the
+ # view limits, so the outermost ticks are dropped (GH#65918)
+ expected = (
+ np.array([1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e0, 1e1])
+ if Version(mpl.__version__) < Version("3.11.0rc1")
+ else np.array([1e-4, 1e-3, 1e-2, 1e-1, 1e0])
+ )
+
+ _, ax = mpl.pyplot.subplots()
+ ax = Series([0.1, 0.01, 0.001]).plot(log=True, kind=kind, ax=ax)
+ ymin = 0.0007943282347242822
+ ymax = 0.12589254117941673
+ res = getattr(ax, res_meth)()
+ tm.assert_almost_equal(res[0], ymin)
+ tm.assert_almost_equal(res[1], ymax)
+ tm.assert_numpy_array_equal(getattr(ax, axis).get_ticklocs(), expected)
+
+ def test_bar_ignore_index(self):
+ df = Series([1, 2, 3, 4], index=["a", "b", "c", "d"])
+ _, ax = mpl.pyplot.subplots()
+ ax = df.plot.bar(use_index=False, ax=ax)
+ _check_text_labels(ax.get_xticklabels(), ["0", "1", "2", "3"])
+
+ def test_bar_user_colors(self):
+ s = Series([1, 2, 3, 4])
+ ax = s.plot.bar(color=["red", "blue", "blue", "red"])
+ result = [p.get_facecolor() for p in ax.patches]
+ expected = [
+ (1.0, 0.0, 0.0, 1.0),
+ (0.0, 0.0, 1.0, 1.0),
+ (0.0, 0.0, 1.0, 1.0),
+ (1.0, 0.0, 0.0, 1.0),
+ ]
+ assert result == expected
+
+ def test_rotation_default(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ # Default rot 0
+ _, ax = mpl.pyplot.subplots()
+ axes = df.plot(ax=ax)
+ _check_ticks_props(axes, xrot=0)
+
+ def test_rotation_30(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((5, 5)))
+ _, ax = mpl.pyplot.subplots()
+ axes = df.plot(rot=30, ax=ax)
+ _check_ticks_props(axes, xrot=30)
+
+ def test_irregular_datetime(self):
+ rng = date_range("1/1/2000", "1/15/2000")
+ rng = rng[[0, 1, 2, 3, 5, 9, 10, 11, 12]]
+ ser = Series(np.random.default_rng(2).standard_normal(len(rng)), rng)
+ _, ax = mpl.pyplot.subplots()
+ ax = ser.plot(ax=ax)
+ xp = DatetimeConverter.convert(datetime(1999, 1, 1), "", ax)
+ ax.set_xlim("1/1/1999", "1/1/2001")
+ assert xp == ax.get_xlim()[0]
+ _check_ticks_props(ax, xrot=30)
+
+ def test_unsorted_index_xlim(self):
+ ser = Series(
+ [0.0, 1.0, np.nan, 3.0, 4.0, 5.0, 6.0],
+ index=[1.0, 0.0, 3.0, 2.0, np.nan, 3.0, 2.0],
+ )
+ _, ax = mpl.pyplot.subplots()
+ ax = ser.plot(ax=ax)
+ xmin, xmax = ax.get_xlim()
+ lines = ax.get_lines()
+ assert xmin <= np.nanmin(lines[0].get_data(orig=False)[0])
+ assert xmax >= np.nanmax(lines[0].get_data(orig=False)[0])
+
+ def test_pie_series(self):
+ # if sum of values is less than 1.0, pie handle them as rate and draw
+ # semicircle.
+ series = Series(
+ np.random.default_rng(2).integers(1, 5),
+ index=["a", "b", "c", "d", "e"],
+ name="YLABEL",
+ )
+ ax = _check_plot_works(series.plot.pie)
+ _check_text_labels(ax.texts, series.index)
+ assert ax.get_ylabel() == ""
+
+ def test_pie_arrow_type(self):
+ # GH 59192
+ pytest.importorskip("pyarrow")
+ ser = Series([1, 2, 3, 4], dtype="int32[pyarrow]")
+ _check_plot_works(ser.plot.pie)
+
+ def test_pie_series_no_label(self):
+ series = Series(
+ np.random.default_rng(2).integers(1, 5),
+ index=["a", "b", "c", "d", "e"],
+ name="YLABEL",
+ )
+ ax = _check_plot_works(series.plot.pie, labels=None)
+ _check_text_labels(ax.texts, [""] * 5)
+
+ def test_pie_series_less_colors_than_elements(self):
+ series = Series(
+ np.random.default_rng(2).integers(1, 5),
+ index=["a", "b", "c", "d", "e"],
+ name="YLABEL",
+ )
+ color_args = ["r", "g", "b"]
+ ax = _check_plot_works(series.plot.pie, colors=color_args)
+
+ color_expected = ["r", "g", "b", "r", "g"]
+ _check_colors(ax.patches, facecolors=color_expected)
+
+ def test_pie_series_labels_and_colors(self):
+ series = Series(
+ np.random.default_rng(2).integers(1, 5),
+ index=["a", "b", "c", "d", "e"],
+ name="YLABEL",
+ )
+ # with labels and colors
+ labels = ["A", "B", "C", "D", "E"]
+ color_args = ["r", "g", "b", "c", "m"]
+ ax = _check_plot_works(series.plot.pie, labels=labels, colors=color_args)
+ _check_text_labels(ax.texts, labels)
+ _check_colors(ax.patches, facecolors=color_args)
+
+ def test_pie_series_autopct_and_fontsize(self):
+ series = Series(
+ np.random.default_rng(2).integers(1, 5),
+ index=["a", "b", "c", "d", "e"],
+ name="YLABEL",
+ )
+ color_args = ["r", "g", "b", "c", "m"]
+ ax = _check_plot_works(
+ series.plot.pie, colors=color_args, autopct="%.2f", fontsize=7
+ )
+ pcts = [f"{s * 100:.2f}" for s in series.values / series.sum()]
+ expected_texts = (
+ list(chain.from_iterable(zip(series.index, pcts, strict=True)))
+ if Version(mpl.__version__) < Version("3.11.0rc1")
+ else list(chain.from_iterable((series.index, pcts)))
+ )
+ _check_text_labels(ax.texts, expected_texts)
+ for t in ax.texts:
+ assert t.get_fontsize() == 7
+
+ def test_pie_series_negative_raises(self):
+ # includes negative value
+ series = Series([1, 2, 0, 4, -1], index=["a", "b", "c", "d", "e"])
+ with pytest.raises(ValueError, match="pie plot doesn't allow negative values"):
+ series.plot.pie()
+
+ def test_pie_series_nan(self):
+ # includes nan
+ series = Series([1, 2, np.nan, 4], index=["a", "b", "c", "d"], name="YLABEL")
+ ax = _check_plot_works(series.plot.pie)
+ _check_text_labels(ax.texts, ["a", "b", "", "d"])
+
+ def test_pie_nan(self):
+ s = Series([1, np.nan, 1, 1])
+ _, ax = mpl.pyplot.subplots()
+ ax = s.plot.pie(legend=True, ax=ax)
+ expected = ["0", "", "2", "3"]
+ result = [x.get_text() for x in ax.texts]
+ assert result == expected
+
+ def test_df_series_secondary_legend(self):
+ # GH 9779
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 3)), columns=list("abc")
+ )
+ s = Series(np.random.default_rng(2).standard_normal(10), name="x")
+
+ # primary -> secondary (without passing ax)
+ _, ax = mpl.pyplot.subplots()
+ ax = df.plot(ax=ax)
+ s.plot(legend=True, secondary_y=True, ax=ax)
+ # both legends are drawn on left ax
+ # left and right axis must be visible
+ _check_legend_labels(ax, labels=["a", "b", "c", "x (right)"])
+ assert ax.get_yaxis().get_visible()
+ assert ax.right_ax.get_yaxis().get_visible()
+
+ def test_df_series_secondary_legend_both(self):
+ # GH 9779
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 3)), columns=list("abc")
+ )
+ s = Series(np.random.default_rng(2).standard_normal(10), name="x")
+ # secondary -> secondary (without passing ax)
+ _, ax = mpl.pyplot.subplots()
+ ax = df.plot(secondary_y=True, ax=ax)
+ s.plot(legend=True, secondary_y=True, ax=ax)
+ # both legends are drawn on left ax
+ # left axis must be invisible and right axis must be visible
+ expected = ["a (right)", "b (right)", "c (right)", "x (right)"]
+ _check_legend_labels(ax.left_ax, labels=expected)
+ assert not ax.left_ax.get_yaxis().get_visible()
+ assert ax.get_yaxis().get_visible()
+
+ def test_df_series_secondary_legend_both_with_axis_2(self):
+ # GH 9779
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 3)), columns=list("abc")
+ )
+ s = Series(np.random.default_rng(2).standard_normal(10), name="x")
+ # secondary -> secondary (with passing ax)
+ _, ax = mpl.pyplot.subplots()
+ ax = df.plot(secondary_y=True, mark_right=False, ax=ax)
+ s.plot(ax=ax, legend=True, secondary_y=True)
+ # both legends are drawn on left ax
+ # left axis must be invisible and right axis must be visible
+ expected = ["a", "b", "c", "x (right)"]
+ _check_legend_labels(ax.left_ax, expected)
+ assert not ax.left_ax.get_yaxis().get_visible()
+ assert ax.get_yaxis().get_visible()
+
+ @pytest.mark.parametrize(
+ "input_logy, expected_scale", [(True, "log"), ("sym", "symlog")]
+ )
+ @pytest.mark.parametrize("secondary_kwarg", [{}, {"secondary_y": True}])
+ def test_secondary_logy(self, input_logy, expected_scale, secondary_kwarg):
+ # GH 25545, GH 24980
+ s1 = Series(np.random.default_rng(2).standard_normal(10))
+ ax1 = s1.plot(logy=input_logy, **secondary_kwarg)
+ assert ax1.get_yscale() == expected_scale
+
+ def test_plot_fails_with_dupe_color_and_style(self):
+ x = Series(np.random.default_rng(2).standard_normal(2))
+ _, ax = mpl.pyplot.subplots()
+ msg = (
+ "Cannot pass 'style' string with a color symbol and 'color' keyword "
+ "argument. Please use one or the other or pass 'style' without a color "
+ "symbol"
+ )
+ with pytest.raises(ValueError, match=msg):
+ x.plot(style="k--", color="k", ax=ax)
+
+ @pytest.mark.parametrize(
+ "bw_method, ind",
+ [
+ ["scott", 20],
+ [None, 20],
+ [None, np.int_(20)],
+ [0.5, np.linspace(-100, 100, 20)],
+ ],
+ )
+ def test_kde_kwargs(self, ts, bw_method, ind):
+ pytest.importorskip("scipy")
+ _check_plot_works(ts.plot.kde, bw_method=bw_method, ind=ind)
+
+ @pytest.mark.parametrize(
+ "bw_method, ind, weights",
+ [
+ ["scott", 20, None],
+ [None, 20, None],
+ [None, np.int_(20), None],
+ [0.5, np.linspace(-100, 100, 20), None],
+ ["scott", 40, np.linspace(0.0, 2.0, 50)],
+ ],
+ )
+ def test_kde_kwargs_weights(self, bw_method, ind, weights):
+ # GH59337
+ pytest.importorskip("scipy")
+ s = Series(np.random.default_rng(2).uniform(size=50))
+ _check_plot_works(s.plot.kde, bw_method=bw_method, ind=ind, weights=weights)
+
+ def test_density_kwargs(self, ts):
+ pytest.importorskip("scipy")
+ sample_points = np.linspace(-100, 100, 20)
+ _check_plot_works(ts.plot.density, bw_method=0.5, ind=sample_points)
+
+ def test_kde_kwargs_check_axes(self, ts):
+ pytest.importorskip("scipy")
+ _, ax = mpl.pyplot.subplots()
+ sample_points = np.linspace(-100, 100, 20)
+ ax = ts.plot.kde(logy=True, bw_method=0.5, ind=sample_points, ax=ax)
+ _check_ax_scales(ax, yaxis="log")
+ _check_text_labels(ax.yaxis.get_label(), "Density")
+
+ def test_kde_missing_vals(self):
+ pytest.importorskip("scipy")
+ s = Series(np.random.default_rng(2).uniform(size=50))
+ s[0] = np.nan
+ axes = _check_plot_works(s.plot.kde)
+
+ # gh-14821: check if the values have any missing values
+ assert any(~np.isnan(axes.lines[0].get_xdata()))
+
+ @pytest.mark.xfail(reason="Api changed in 3.6.0")
+ def test_boxplot_series(self, ts):
+ _, ax = mpl.pyplot.subplots()
+ ax = ts.plot.box(logy=True, ax=ax)
+ _check_ax_scales(ax, yaxis="log")
+ xlabels = ax.get_xticklabels()
+ _check_text_labels(xlabels, [ts.name])
+ ylabels = ax.get_yticklabels()
+ _check_text_labels(ylabels, [""] * len(ylabels))
+
+ @pytest.mark.parametrize(
+ "kind",
+ plotting.PlotAccessor._common_kinds + plotting.PlotAccessor._series_kinds,
+ )
+ def test_kind_kwarg(self, kind):
+ pytest.importorskip("scipy")
+ s = Series(range(3))
+ _, ax = mpl.pyplot.subplots()
+ s.plot(kind=kind, ax=ax)
+ mpl.pyplot.close()
+
+ @pytest.mark.parametrize(
+ "kind",
+ plotting.PlotAccessor._common_kinds + plotting.PlotAccessor._series_kinds,
+ )
+ def test_kind_attr(self, kind):
+ pytest.importorskip("scipy")
+ s = Series(range(3))
+ _, ax = mpl.pyplot.subplots()
+ getattr(s.plot, kind)()
+ mpl.pyplot.close()
+
+ @pytest.mark.parametrize("kind", plotting.PlotAccessor._common_kinds)
+ def test_invalid_plot_data(self, kind):
+ s = Series(list("abcd"))
+ _, ax = mpl.pyplot.subplots()
+ msg = "no numeric data to plot"
+ with pytest.raises(TypeError, match=msg):
+ s.plot(kind=kind, ax=ax)
+
+ @pytest.mark.parametrize("kind", plotting.PlotAccessor._common_kinds)
+ def test_valid_object_plot(self, kind):
+ pytest.importorskip("scipy")
+ s = Series(range(10), dtype=object)
+ _check_plot_works(s.plot, kind=kind)
+
+ @pytest.mark.parametrize("kind", plotting.PlotAccessor._common_kinds)
+ def test_partially_invalid_plot_data(self, kind):
+ s = Series(["a", "b", 1.0, 2])
+ _, ax = mpl.pyplot.subplots()
+ msg = "no numeric data to plot"
+ with pytest.raises(TypeError, match=msg):
+ s.plot(kind=kind, ax=ax)
+
+ def test_invalid_kind(self):
+ s = Series([1, 2])
+ with pytest.raises(ValueError, match="invalid_kind is not a valid plot kind"):
+ s.plot(kind="invalid_kind")
+
+ def test_dup_datetime_index_plot(self):
+ dr1 = date_range("1/1/2009", periods=4)
+ dr2 = date_range("1/2/2009", periods=4)
+ index = dr1.append(dr2)
+ values = np.random.default_rng(2).standard_normal(index.size)
+ s = Series(values, index=index)
+ _check_plot_works(s.plot)
+
+ def test_errorbar_asymmetrical(self):
+ # GH9536
+ s = Series(np.arange(10), name="x")
+ err = np.random.default_rng(2).random((2, 10))
+
+ ax = s.plot(yerr=err, xerr=err)
+
+ result = np.vstack([i.vertices[:, 1] for i in ax.collections[1].get_paths()])
+ expected = (err.T * np.array([-1, 1])) + s.to_numpy().reshape(-1, 1)
+ tm.assert_numpy_array_equal(result, expected)
+
+ def test_errorbar_asymmetrical_error(self):
+ # GH9536
+ s = Series(np.arange(10), name="x")
+ msg = (
+ "Asymmetrical error bars should be provided "
+ f"with the shape \\(2, {len(s)}\\)"
+ )
+ with pytest.raises(ValueError, match=msg):
+ s.plot(yerr=np.random.default_rng(2).random((2, 11)))
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize("kind", ["line", "bar"])
+ @pytest.mark.parametrize(
+ "yerr",
+ [
+ Series(np.abs(np.random.default_rng(2).standard_normal(10))),
+ np.abs(np.random.default_rng(2).standard_normal(10)),
+ list(np.abs(np.random.default_rng(2).standard_normal(10))),
+ DataFrame(
+ np.abs(np.random.default_rng(2).standard_normal((10, 2))),
+ columns=["x", "y"],
+ ),
+ ],
+ )
+ def test_errorbar_plot(self, kind, yerr):
+ s = Series(np.arange(10), name="x")
+ ax = _check_plot_works(s.plot, yerr=yerr, kind=kind)
+ _check_has_errorbars(ax, xerr=0, yerr=1)
+
+ @pytest.mark.slow
+ def test_errorbar_plot_yerr_0(self):
+ s = Series(np.arange(10), name="x")
+ s_err = np.abs(np.random.default_rng(2).standard_normal(10))
+ ax = _check_plot_works(s.plot, xerr=s_err)
+ _check_has_errorbars(ax, xerr=1, yerr=0)
+
+ @pytest.mark.slow
+ @pytest.mark.parametrize(
+ "yerr",
+ [
+ Series(np.abs(np.random.default_rng(2).standard_normal(12))),
+ DataFrame(
+ np.abs(np.random.default_rng(2).standard_normal((12, 2))),
+ columns=["x", "y"],
+ ),
+ ],
+ )
+ def test_errorbar_plot_ts(self, yerr):
+ # test time series plotting
+ ix = date_range("1/1/2000", "1/1/2001", freq="ME")
+ ts = Series(np.arange(12), index=ix, name="x")
+ yerr.index = ix
+
+ ax = _check_plot_works(ts.plot, yerr=yerr)
+ _check_has_errorbars(ax, xerr=0, yerr=1)
+
+ @pytest.mark.slow
+ def test_errorbar_plot_invalid_yerr_shape(self):
+ s = Series(np.arange(10), name="x")
+ # check incorrect lengths and types
+ with tm.external_error_raised(ValueError):
+ s.plot(yerr=np.arange(11))
+
+ @pytest.mark.slow
+ def test_errorbar_plot_invalid_yerr(self):
+ s = Series(np.arange(10), name="x")
+ s_err = ["zzz"] * 10
+ with tm.external_error_raised(TypeError):
+ s.plot(yerr=s_err)
+
+ @pytest.mark.slow
+ def test_table_true(self, series):
+ _check_plot_works(series.plot, table=True)
+
+ @pytest.mark.slow
+ def test_table_self(self, series):
+ _check_plot_works(series.plot, table=series)
+
+ @pytest.mark.slow
+ def test_series_grid_settings(self):
+ # Make sure plot defaults to rcParams['axes.grid'] setting, GH 9792
+ pytest.importorskip("scipy")
+ _check_grid_settings(
+ Series([1, 2, 3]),
+ plotting.PlotAccessor._series_kinds + plotting.PlotAccessor._common_kinds,
+ )
+
+ @pytest.mark.parametrize("c", ["r", "red", "green", "#FF0000"])
+ def test_standard_colors(self, c):
+ result = get_standard_colors(1, color=c)
+ assert result == [c]
+
+ result = get_standard_colors(1, color=[c])
+ assert result == [c]
+
+ result = get_standard_colors(3, color=c)
+ assert result == [c] * 3
+
+ result = get_standard_colors(3, color=[c])
+ assert result == [c] * 3
+
+ def test_standard_colors_all(self):
+ # multiple colors like mediumaquamarine
+ for c in mpl.colors.cnames:
+ result = get_standard_colors(num_colors=1, color=c)
+ assert result == [c]
+
+ result = get_standard_colors(num_colors=1, color=[c])
+ assert result == [c]
+
+ result = get_standard_colors(num_colors=3, color=c)
+ assert result == [c] * 3
+
+ result = get_standard_colors(num_colors=3, color=[c])
+ assert result == [c] * 3
+
+ # single letter colors like k
+ for c in mpl.colors.ColorConverter.colors:
+ result = get_standard_colors(num_colors=1, color=c)
+ assert result == [c]
+
+ result = get_standard_colors(num_colors=1, color=[c])
+ assert result == [c]
+
+ result = get_standard_colors(num_colors=3, color=c)
+ assert result == [c] * 3
+
+ result = get_standard_colors(num_colors=3, color=[c])
+ assert result == [c] * 3
+
+ def test_series_plot_color_kwargs(self):
+ # GH1890
+ _, ax = mpl.pyplot.subplots()
+ ax = Series(np.arange(12) + 1).plot(color="green", ax=ax)
+ _check_colors(ax.get_lines(), linecolors=["green"])
+
+ def test_time_series_plot_color_kwargs(self):
+ # #1890
+ _, ax = mpl.pyplot.subplots()
+ ax = Series(np.arange(12) + 1, index=date_range("1/1/2000", periods=12)).plot(
+ color="green", ax=ax
+ )
+ _check_colors(ax.get_lines(), linecolors=["green"])
+
+ def test_time_series_plot_color_with_empty_kwargs(self):
+ def_colors = _unpack_cycler(mpl.rcParams)
+ index = date_range("1/1/2000", periods=12)
+ s = Series(np.arange(1, 13), index=index)
+
+ ncolors = 3
+
+ _, ax = mpl.pyplot.subplots()
+ for i in range(ncolors):
+ ax = s.plot(ax=ax)
+ _check_colors(ax.get_lines(), linecolors=def_colors[:ncolors])
+
+ def test_xticklabels(self):
+ # GH11529
+ s = Series(np.arange(10), index=[f"P{i:02d}" for i in range(10)])
+ _, ax = mpl.pyplot.subplots()
+ ax = s.plot(xticks=[0, 3, 5, 9], ax=ax)
+ exp = [f"P{i:02d}" for i in [0, 3, 5, 9]]
+ _check_text_labels(ax.get_xticklabels(), exp)
+
+ def test_xtick_barPlot(self):
+ # GH28172
+ s = Series(range(10), index=[f"P{i:02d}" for i in range(10)])
+ ax = s.plot.bar(xticks=range(0, 11, 2))
+ exp = np.array(list(range(0, 11, 2)))
+ tm.assert_numpy_array_equal(exp, ax.get_xticks())
+
+ def test_custom_business_day_freq(self):
+ # GH7222
+ s = Series(
+ range(100, 121),
+ index=pd.bdate_range(
+ start="2014-05-01",
+ end="2014-06-01",
+ freq=CustomBusinessDay(holidays=["2014-05-26"]),
+ ),
+ )
+
+ _check_plot_works(s.plot)
+
+ @pytest.mark.xfail(
+ reason="GH#24426, see also "
+ "github.com/pandas-dev/pandas/commit/"
+ "ef1bd69fa42bbed5d09dd17f08c44fc8bfc2b685#r61470674"
+ )
+ def test_plot_accessor_updates_on_inplace(self):
+ ser = Series([1, 2, 3, 4])
+ _, ax = mpl.pyplot.subplots()
+ ax = ser.plot(ax=ax)
+ before = ax.xaxis.get_ticklocs()
+
+ ser.drop([0, 1], inplace=True)
+ _, ax = mpl.pyplot.subplots()
+ after = ax.xaxis.get_ticklocs()
+ tm.assert_numpy_array_equal(before, after)
+
+ @pytest.mark.parametrize("kind", ["line", "area"])
+ def test_plot_xlim_for_series(self, kind):
+ # test if xlim is also correctly plotted in Series for line and area
+ # GH 27686
+ s = Series([2, 3])
+ _, ax = mpl.pyplot.subplots()
+ s.plot(kind=kind, ax=ax)
+ xlims = ax.get_xlim()
+
+ assert xlims[0] < 0
+ assert xlims[1] > 1
+
+ def test_plot_no_rows(self):
+ # GH 27758
+ df = Series(dtype=int)
+ assert df.empty
+ ax = df.plot()
+ assert len(ax.get_lines()) == 1
+ line = ax.get_lines()[0]
+ assert len(line.get_xdata()) == 0
+ assert len(line.get_ydata()) == 0
+
+ def test_plot_no_numeric_data(self):
+ df = Series(["a", "b", "c"])
+ with pytest.raises(TypeError, match="no numeric data to plot"):
+ df.plot()
+
+ @pytest.mark.parametrize(
+ "data, index",
+ [
+ ([1, 2, 3, 4], [3, 2, 1, 0]),
+ ([10, 50, 20, 30], [1910, 1920, 1980, 1950]),
+ ],
+ )
+ def test_plot_order(self, data, index):
+ # GH38865 Verify plot order of a Series
+ ser = Series(data=data, index=index)
+ ax = ser.plot(kind="bar")
+
+ expected = ser.tolist()
+ result = [
+ patch.get_bbox().ymax
+ for patch in sorted(ax.patches, key=lambda patch: patch.get_bbox().xmax)
+ ]
+ assert expected == result
+
+ def test_style_single_ok(self):
+ s = Series([1, 2])
+ ax = s.plot(style="s", color="C3")
+ assert ax.lines[0].get_color() == "C3"
+
+ @pytest.mark.parametrize(
+ "index_name, old_label, new_label",
+ [(None, "", "new"), ("old", "old", "new"), (None, "", "")],
+ )
+ @pytest.mark.parametrize("kind", ["line", "area", "bar", "barh", "hist"])
+ def test_xlabel_ylabel_series(self, kind, index_name, old_label, new_label):
+ # GH 9093
+ ser = Series([1, 2, 3, 4])
+ ser.index.name = index_name
+
+ # default is the ylabel is not shown and xlabel is index name (reverse for barh)
+ ax = ser.plot(kind=kind)
+ if kind == "barh":
+ assert ax.get_xlabel() == ""
+ assert ax.get_ylabel() == old_label
+ elif kind == "hist":
+ assert ax.get_xlabel() == ""
+ assert ax.get_ylabel() == "Frequency"
+ else:
+ assert ax.get_ylabel() == ""
+ assert ax.get_xlabel() == old_label
+
+ # old xlabel will be overridden and assigned ylabel will be used as ylabel
+ ax = ser.plot(kind=kind, ylabel=new_label, xlabel=new_label)
+ assert ax.get_ylabel() == new_label
+ assert ax.get_xlabel() == new_label
+
+ @pytest.mark.parametrize(
+ "index",
+ [
+ pd.timedelta_range(start=0, periods=2, freq="D"),
+ [pd.Timedelta(days=1), pd.Timedelta(days=2)],
+ ],
+ )
+ def test_timedelta_index(self, index):
+ # GH37454
+ xlims = (3, 1)
+ ax = Series([1, 2], index=index).plot(xlim=(xlims))
+ assert ax.get_xlim() == (3, 1)
+
+ def test_series_none_color(self):
+ # GH51953
+ series = Series([1, 2, 3])
+ ax = series.plot(color=None)
+ expected = _unpack_cycler(mpl.pyplot.rcParams)[:1]
+ _check_colors(ax.get_lines(), linecolors=expected)
+
+ @pytest.mark.slow
+ def test_plot_no_warning(self, ts):
+ # GH 55138
+ # TODO(3.0): this can be removed once Period[B] deprecation is enforced
+ with tm.assert_produces_warning(False):
+ _ = ts.plot()
+
+ def test_secondary_y_subplot_axis_labels(self):
+ # GH#14102
+ s1 = Series([5, 7, 6, 8, 7], index=[1, 2, 3, 4, 5])
+ s2 = Series([6, 4, 5, 3, 4], index=[1, 2, 3, 4, 5])
+
+ ax = plt.subplot(2, 1, 1)
+ s1.plot(ax=ax)
+ s2.plot(ax=ax, secondary_y=True)
+ ax2 = plt.subplot(2, 1, 2)
+ s1.plot(ax=ax2)
+ assert len(ax.xaxis.get_minor_ticks()) == 0
+ assert len(ax.get_xticklabels()) > 0
+
+ def test_bar_line_plot(self):
+ """
+ Test that bar and line plots with the same x values are superposed
+ and that the x limits are set such that the plots are visible.
+ """
+ # GH61161
+ index = period_range("2023", periods=3, freq="Y")
+ years = set(index.year.astype(str))
+ s = Series([1, 2, 3], index=index)
+ ax = plt.subplot()
+ s.plot(kind="bar", ax=ax)
+ bar_xticks = [
+ label for label in ax.get_xticklabels() if label.get_text() in years
+ ]
+ s.plot(kind="line", ax=ax, color="r")
+ line_xticks = [
+ label for label in ax.get_xticklabels() if label.get_text() in years
+ ]
+ assert len(bar_xticks) == len(index)
+ assert bar_xticks == line_xticks
+ x_limits = ax.get_xlim()
+ assert x_limits[0] <= bar_xticks[0].get_position()[0]
+ assert x_limits[1] >= bar_xticks[-1].get_position()[0]
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_style.py b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_style.py
new file mode 100644
index 0000000000000000000000000000000000000000..04dbf75e06004de49057b05796c20a712674d66e
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/plotting/test_style.py
@@ -0,0 +1,148 @@
+import pytest
+
+from pandas import Series
+
+mpl = pytest.importorskip("matplotlib")
+plt = pytest.importorskip("matplotlib.pyplot")
+from pandas.plotting._matplotlib.style import get_standard_colors
+
+
+class TestGetStandardColors:
+ @pytest.mark.parametrize(
+ "num_colors, expected",
+ [
+ (3, ["red", "green", "blue"]),
+ (5, ["red", "green", "blue", "red", "green"]),
+ (7, ["red", "green", "blue", "red", "green", "blue", "red"]),
+ (2, ["red", "green"]),
+ (1, ["red"]),
+ ],
+ )
+ def test_default_colors_named_from_prop_cycle(self, num_colors, expected):
+ mpl_params = {
+ "axes.prop_cycle": plt.cycler(color=["red", "green", "blue"]),
+ }
+ with mpl.rc_context(rc=mpl_params):
+ result = get_standard_colors(num_colors=num_colors)
+ assert mpl.colors.same_color(result, expected)
+
+ @pytest.mark.parametrize(
+ "num_colors, expected",
+ [
+ (1, ["b"]),
+ (3, ["b", "g", "r"]),
+ (4, ["b", "g", "r", "y"]),
+ (5, ["b", "g", "r", "y", "b"]),
+ (7, ["b", "g", "r", "y", "b", "g", "r"]),
+ ],
+ )
+ def test_default_colors_named_from_prop_cycle_string(self, num_colors, expected):
+ mpl_params = {
+ "axes.prop_cycle": plt.cycler(color="bgry"),
+ }
+ with mpl.rc_context(rc=mpl_params):
+ result = get_standard_colors(num_colors=num_colors)
+ assert mpl.colors.same_color(result, expected)
+
+ @pytest.mark.parametrize(
+ "num_colors, expected",
+ [
+ (1, ["C0"]),
+ (3, ["C0", "C1", "C2"]),
+ (
+ 12,
+ [
+ "C0",
+ "C1",
+ "C2",
+ "C3",
+ "C4",
+ "C5",
+ "C6",
+ "C7",
+ "C8",
+ "C9",
+ "C0",
+ "C1",
+ ],
+ ),
+ ],
+ )
+ def test_default_colors_named_undefined_prop_cycle(self, num_colors, expected):
+ with mpl.rc_context(rc={}):
+ result = get_standard_colors(num_colors=num_colors)
+ assert mpl.colors.same_color(result, expected)
+
+ @pytest.mark.parametrize(
+ "num_colors, expected",
+ [
+ (1, ["red", "green", (0.1, 0.2, 0.3)]),
+ (2, ["red", "green", (0.1, 0.2, 0.3)]),
+ (3, ["red", "green", (0.1, 0.2, 0.3)]),
+ (4, ["red", "green", (0.1, 0.2, 0.3), "red"]),
+ ],
+ )
+ def test_user_input_color_sequence(self, num_colors, expected):
+ color = ["red", "green", (0.1, 0.2, 0.3)]
+ result = get_standard_colors(color=color, num_colors=num_colors)
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "num_colors, expected",
+ [
+ (1, ["r", "g", "b", "k"]),
+ (2, ["r", "g", "b", "k"]),
+ (3, ["r", "g", "b", "k"]),
+ (4, ["r", "g", "b", "k"]),
+ (5, ["r", "g", "b", "k", "r"]),
+ (6, ["r", "g", "b", "k", "r", "g"]),
+ ],
+ )
+ def test_user_input_color_string(self, num_colors, expected):
+ color = "rgbk"
+ result = get_standard_colors(color=color, num_colors=num_colors)
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "num_colors, expected",
+ [
+ (1, [(0.1, 0.2, 0.3)]),
+ (2, [(0.1, 0.2, 0.3), (0.1, 0.2, 0.3)]),
+ (3, [(0.1, 0.2, 0.3), (0.1, 0.2, 0.3), (0.1, 0.2, 0.3)]),
+ ],
+ )
+ def test_user_input_color_floats(self, num_colors, expected):
+ color = (0.1, 0.2, 0.3)
+ result = get_standard_colors(color=color, num_colors=num_colors)
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "color, num_colors, expected",
+ [
+ ("Crimson", 1, ["Crimson"]),
+ ("DodgerBlue", 2, ["DodgerBlue", "DodgerBlue"]),
+ ("firebrick", 3, ["firebrick", "firebrick", "firebrick"]),
+ ],
+ )
+ def test_user_input_named_color_string(self, color, num_colors, expected):
+ result = get_standard_colors(color=color, num_colors=num_colors)
+ assert result == expected
+
+ @pytest.mark.parametrize("color", ["", [], (), Series([], dtype="object")])
+ def test_empty_color_raises(self, color):
+ with pytest.raises(ValueError, match="Invalid color argument"):
+ get_standard_colors(color=color, num_colors=1)
+
+ @pytest.mark.parametrize(
+ "color",
+ [
+ "bad_color",
+ ("red", "green", "bad_color"),
+ (0.1,),
+ (0.1, 0.2),
+ (0.1, 0.2, 0.3, 0.4, 0.5), # must be either 3 or 4 floats
+ ],
+ )
+ def test_bad_color_raises(self, color):
+ with pytest.raises(ValueError, match="Invalid color"):
+ get_standard_colors(color=color, num_colors=5)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reductions/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/reductions/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e3851753b67421842a0d3d9fd5f88e7eb72734dd
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reductions/__init__.py
@@ -0,0 +1,4 @@
+"""
+Tests for reductions where we want to test for matching behavior across
+Array, Index, Series, and DataFrame methods.
+"""
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reductions/test_reductions.py b/.venv/lib/python3.13/site-packages/pandas/tests/reductions/test_reductions.py
new file mode 100644
index 0000000000000000000000000000000000000000..e7f6554a019a40fb8c41f4c272de6304e63aec0e
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reductions/test_reductions.py
@@ -0,0 +1,1686 @@
+from datetime import (
+ datetime,
+ timedelta,
+)
+from decimal import Decimal
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ Categorical,
+ DataFrame,
+ DatetimeIndex,
+ Index,
+ NaT,
+ Period,
+ PeriodIndex,
+ RangeIndex,
+ Series,
+ Timedelta,
+ TimedeltaIndex,
+ Timestamp,
+ date_range,
+ isna,
+ period_range,
+ timedelta_range,
+ to_timedelta,
+)
+import pandas._testing as tm
+from pandas.core import nanops
+from pandas.core.arrays.string_arrow import ArrowStringArray
+
+
+def get_objs():
+ indexes = [
+ Index([True, False] * 5, name="a"),
+ Index(np.arange(10), dtype=np.int64, name="a"),
+ Index(np.arange(10), dtype=np.float64, name="a"),
+ DatetimeIndex(date_range("2020-01-01", periods=10, unit="ns"), name="a"),
+ DatetimeIndex(
+ date_range("2020-01-01", periods=10, unit="ns"), name="a"
+ ).tz_localize(tz="US/Eastern"),
+ PeriodIndex(period_range("2020-01-01", periods=10, freq="D"), name="a"),
+ Index([str(i) for i in range(10)], name="a"),
+ ]
+
+ arr = np.random.default_rng(2).standard_normal(10)
+ series = [Series(arr, index=idx, name="a") for idx in indexes]
+
+ objs = indexes + series
+ return objs
+
+
+class TestReductions:
+ @pytest.mark.filterwarnings(
+ "ignore:Period with BDay freq is deprecated:FutureWarning"
+ )
+ @pytest.mark.parametrize("opname", ["max", "min"])
+ @pytest.mark.parametrize("obj", get_objs())
+ def test_ops(self, opname, obj):
+ result = getattr(obj, opname)()
+ if not isinstance(obj, PeriodIndex):
+ if isinstance(obj.values, ArrowStringArray):
+ # max not on the interface
+ expected = getattr(np.array(obj.values), opname)()
+ else:
+ expected = getattr(obj.values, opname)()
+ else:
+ expected = Period(ordinal=getattr(obj.asi8, opname)(), freq=obj.freq)
+
+ if getattr(obj, "tz", None) is not None:
+ # We need to de-localize before comparing to the numpy-produced result
+ expected = expected.astype("M8[ns]").astype("int64")
+ assert result._value == expected
+ else:
+ assert result == expected
+
+ @pytest.mark.parametrize("opname", ["max", "min"])
+ @pytest.mark.parametrize(
+ "dtype, val",
+ [
+ ("object", 2.0),
+ ("float64", 2.0),
+ ("datetime64[ns]", datetime(2011, 11, 1)),
+ ("Int64", 2),
+ ("boolean", True),
+ ],
+ )
+ def test_nanminmax(self, opname, dtype, val, index_or_series):
+ # GH#7261
+ klass = index_or_series
+
+ def check_missing(res):
+ if dtype == "datetime64[ns]":
+ return res is NaT
+ elif dtype in ["Int64", "boolean"]:
+ return res is pd.NA
+ else:
+ return isna(res)
+
+ obj = klass([None], dtype=dtype)
+ assert check_missing(getattr(obj, opname)())
+ assert check_missing(getattr(obj, opname)(skipna=False))
+
+ obj = klass([], dtype=dtype)
+ assert check_missing(getattr(obj, opname)())
+ assert check_missing(getattr(obj, opname)(skipna=False))
+
+ if dtype == "object":
+ # generic test with object only works for empty / all NaN
+ return
+
+ obj = klass([None, val], dtype=dtype)
+ assert getattr(obj, opname)() == val
+ assert check_missing(getattr(obj, opname)(skipna=False))
+
+ obj = klass([None, val, None], dtype=dtype)
+ assert getattr(obj, opname)() == val
+ assert check_missing(getattr(obj, opname)(skipna=False))
+
+ @pytest.mark.parametrize("opname", ["max", "min"])
+ def test_nanargminmax(self, opname, index_or_series):
+ # GH#7261
+ klass = index_or_series
+ arg_op = "arg" + opname if klass is Index else "idx" + opname
+
+ obj = klass([NaT, datetime(2011, 11, 1)])
+ assert getattr(obj, arg_op)() == 1
+
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ getattr(obj, arg_op)(skipna=False)
+
+ obj = klass([NaT, datetime(2011, 11, 1), NaT])
+ # check DatetimeIndex non-monotonic path
+ assert getattr(obj, arg_op)() == 1
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ getattr(obj, arg_op)(skipna=False)
+
+ @pytest.mark.parametrize("opname", ["max", "min"])
+ @pytest.mark.parametrize("dtype", ["M8[ns]", "datetime64[ns, UTC]"])
+ def test_nanops_empty_object(self, opname, index_or_series, dtype):
+ klass = index_or_series
+ arg_op = "arg" + opname if klass is Index else "idx" + opname
+
+ obj = klass([], dtype=dtype)
+
+ assert getattr(obj, opname)() is NaT
+ assert getattr(obj, opname)(skipna=False) is NaT
+
+ with pytest.raises(ValueError, match="empty sequence"):
+ getattr(obj, arg_op)()
+ with pytest.raises(ValueError, match="empty sequence"):
+ getattr(obj, arg_op)(skipna=False)
+
+ def test_argminmax(self):
+ obj = Index(np.arange(5, dtype="int64"))
+ assert obj.argmin() == 0
+ assert obj.argmax() == 4
+
+ obj = Index([np.nan, 1, np.nan, 2])
+ assert obj.argmin() == 1
+ assert obj.argmax() == 3
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ obj.argmin(skipna=False)
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ obj.argmax(skipna=False)
+
+ obj = Index([np.nan])
+ with pytest.raises(ValueError, match="Encountered all NA values"):
+ obj.argmin()
+ with pytest.raises(ValueError, match="Encountered all NA values"):
+ obj.argmax()
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ obj.argmin(skipna=False)
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ obj.argmax(skipna=False)
+
+ obj = Index([NaT, datetime(2011, 11, 1), datetime(2011, 11, 2), NaT])
+ assert obj.argmin() == 1
+ assert obj.argmax() == 2
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ obj.argmin(skipna=False)
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ obj.argmax(skipna=False)
+
+ obj = Index([NaT])
+ with pytest.raises(ValueError, match="Encountered all NA values"):
+ obj.argmin()
+ with pytest.raises(ValueError, match="Encountered all NA values"):
+ obj.argmax()
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ obj.argmin(skipna=False)
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ obj.argmax(skipna=False)
+
+ @pytest.mark.parametrize("op, expected_col", [["max", "a"], ["min", "b"]])
+ def test_same_tz_min_max_axis_1(self, op, expected_col):
+ # GH 10390
+ df = DataFrame(
+ date_range("2016-01-01 00:00:00", periods=3, tz="UTC", unit="ns"),
+ columns=["a"],
+ )
+ df["b"] = df.a.subtract(Timedelta(seconds=3600))
+ result = getattr(df, op)(axis=1)
+ expected = df[expected_col].rename(None)
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize("func", ["maximum", "minimum"])
+ def test_numpy_reduction_with_tz_aware_dtype(self, tz_aware_fixture, func):
+ # GH 15552
+ tz = tz_aware_fixture
+ arg = pd.to_datetime(["2019"]).tz_localize(tz)
+ expected = Series(arg)
+ result = getattr(np, func)(expected, expected)
+ tm.assert_series_equal(result, expected)
+
+ def test_nan_int_timedelta_sum(self):
+ # GH 27185
+ df = DataFrame(
+ {
+ "A": Series([1, 2, NaT], dtype="timedelta64[ns]"),
+ "B": Series([1, 2, pd.NA], dtype="Int64"),
+ }
+ )
+ expected = Series({"A": Timedelta(3), "B": 3})
+ result = df.sum()
+ tm.assert_series_equal(result, expected)
+
+
+class TestIndexReductions:
+ # Note: the name TestIndexReductions indicates these tests
+ # were moved from an Index-specific test file, _not_ that these tests are
+ # intended long-term to be Index-specific
+
+ @pytest.mark.parametrize(
+ "start,stop,step",
+ [
+ (0, 400, 3),
+ (500, 0, -6),
+ (-(10**6), 10**6, 4),
+ (10**6, -(10**6), -4),
+ (0, 10, 20),
+ ],
+ )
+ def test_max_min_range(self, start, stop, step):
+ # GH#17607
+ idx = RangeIndex(start, stop, step)
+ expected = idx._values.max()
+ result = idx.max()
+ assert result == expected
+
+ # skipna should be irrelevant since RangeIndex should never have NAs
+ result2 = idx.max(skipna=False)
+ assert result2 == expected
+
+ expected = idx._values.min()
+ result = idx.min()
+ assert result == expected
+
+ # skipna should be irrelevant since RangeIndex should never have NAs
+ result2 = idx.min(skipna=False)
+ assert result2 == expected
+
+ # empty
+ idx = RangeIndex(start, stop, -step)
+ assert isna(idx.max())
+ assert isna(idx.min())
+
+ def test_minmax_timedelta64(self):
+ # monotonic
+ idx1 = TimedeltaIndex(["1 days", "2 days", "3 days"])
+ assert idx1.is_monotonic_increasing
+
+ # non-monotonic
+ idx2 = TimedeltaIndex(["1 days", np.nan, "3 days", "NaT"])
+ assert not idx2.is_monotonic_increasing
+
+ for idx in [idx1, idx2]:
+ assert idx.min() == Timedelta("1 days")
+ assert idx.max() == Timedelta("3 days")
+ assert idx.argmin() == 0
+ assert idx.argmax() == 2
+
+ @pytest.mark.parametrize("op", ["min", "max"])
+ def test_minmax_timedelta_empty_or_na(self, op):
+ # Return NaT
+ obj = TimedeltaIndex([])
+ assert getattr(obj, op)() is NaT
+
+ obj = TimedeltaIndex([NaT])
+ assert getattr(obj, op)() is NaT
+
+ obj = TimedeltaIndex([NaT, NaT, NaT])
+ assert getattr(obj, op)() is NaT
+
+ def test_numpy_minmax_timedelta64(self):
+ td = timedelta_range("16815 days", "16820 days", freq="D")
+
+ assert np.min(td) == Timedelta("16815 days")
+ assert np.max(td) == Timedelta("16820 days")
+
+ errmsg = "the 'out' parameter is not supported"
+ with pytest.raises(ValueError, match=errmsg):
+ np.min(td, out=0)
+ with pytest.raises(ValueError, match=errmsg):
+ np.max(td, out=0)
+
+ assert np.argmin(td) == 0
+ assert np.argmax(td) == 5
+
+ errmsg = "the 'out' parameter is not supported"
+ with pytest.raises(ValueError, match=errmsg):
+ np.argmin(td, out=0)
+ with pytest.raises(ValueError, match=errmsg):
+ np.argmax(td, out=0)
+
+ def test_timedelta_ops(self):
+ # GH#4984
+ # make sure ops return Timedelta
+ s = Series(
+ [Timestamp("20130101") + timedelta(seconds=i * i) for i in range(10)]
+ )
+ td = s.diff()
+
+ result = td.mean()
+ expected = to_timedelta(timedelta(seconds=9))
+ assert result == expected
+
+ result = td.to_frame().mean()
+ assert result[0] == expected
+
+ result = td.quantile(0.1)
+ expected = Timedelta(np.timedelta64(2600, "ms"))
+ assert result == expected
+
+ result = td.median()
+ expected = to_timedelta("00:00:09")
+ assert result == expected
+
+ result = td.to_frame().median()
+ assert result[0] == expected
+
+ # GH#6462
+ # consistency in returned values for sum
+ result = td.sum()
+ expected = to_timedelta("00:01:21")
+ assert result == expected
+
+ result = td.to_frame().sum()
+ assert result[0] == expected
+
+ # std
+ result = td.std()
+ expected = to_timedelta(Series(td.dropna().values).std())
+ assert result == expected
+
+ result = td.to_frame().std()
+ assert result[0] == expected
+
+ # GH#10040
+ # make sure NaT is properly handled by median()
+ s = Series([Timestamp("2015-02-03"), Timestamp("2015-02-07")])
+ assert s.diff().median() == timedelta(days=4)
+
+ s = Series(
+ [Timestamp("2015-02-03"), Timestamp("2015-02-07"), Timestamp("2015-02-15")]
+ )
+ assert s.diff().median() == timedelta(days=6)
+
+ @pytest.mark.parametrize("opname", ["skew", "kurt", "sem", "prod", "var"])
+ def test_invalid_td64_reductions(self, opname):
+ s = Series(
+ [Timestamp("20130101") + timedelta(seconds=i * i) for i in range(10)]
+ )
+ td = s.diff()
+
+ msg = "|".join(
+ [
+ f"reduction operation '{opname}' not allowed for this dtype",
+ rf"cannot perform {opname} with type timedelta64\[ns\]",
+ f"does not support operation '{opname}'",
+ ]
+ )
+
+ with pytest.raises(TypeError, match=msg):
+ getattr(td, opname)()
+
+ with pytest.raises(TypeError, match=msg):
+ getattr(td.to_frame(), opname)(numeric_only=False)
+
+ def test_minmax_tz(self, tz_naive_fixture):
+ tz = tz_naive_fixture
+ # monotonic
+ idx1 = DatetimeIndex(["2011-01-01", "2011-01-02", "2011-01-03"], tz=tz)
+ assert idx1.is_monotonic_increasing
+
+ # non-monotonic
+ idx2 = DatetimeIndex(
+ ["2011-01-01", NaT, "2011-01-03", "2011-01-02", NaT], tz=tz
+ )
+ assert not idx2.is_monotonic_increasing
+
+ for idx in [idx1, idx2]:
+ assert idx.min() == Timestamp("2011-01-01", tz=tz)
+ assert idx.max() == Timestamp("2011-01-03", tz=tz)
+ assert idx.argmin() == 0
+ assert idx.argmax() == 2
+
+ @pytest.mark.parametrize("op", ["min", "max"])
+ def test_minmax_nat_datetime64(self, op):
+ # Return NaT
+ obj = DatetimeIndex([])
+ assert isna(getattr(obj, op)())
+
+ obj = DatetimeIndex([NaT])
+ assert isna(getattr(obj, op)())
+
+ obj = DatetimeIndex([NaT, NaT, NaT])
+ assert isna(getattr(obj, op)())
+
+ def test_numpy_minmax_integer(self):
+ # GH#26125
+ idx = Index([1, 2, 3])
+
+ expected = idx.values.max()
+ result = np.max(idx)
+ assert result == expected
+
+ expected = idx.values.min()
+ result = np.min(idx)
+ assert result == expected
+
+ errmsg = "the 'out' parameter is not supported"
+ with pytest.raises(ValueError, match=errmsg):
+ np.min(idx, out=0)
+ with pytest.raises(ValueError, match=errmsg):
+ np.max(idx, out=0)
+
+ expected = idx.values.argmax()
+ result = np.argmax(idx)
+ assert result == expected
+
+ expected = idx.values.argmin()
+ result = np.argmin(idx)
+ assert result == expected
+
+ errmsg = "the 'out' parameter is not supported"
+ with pytest.raises(ValueError, match=errmsg):
+ np.argmin(idx, out=0)
+ with pytest.raises(ValueError, match=errmsg):
+ np.argmax(idx, out=0)
+
+ def test_numpy_minmax_range(self):
+ # GH#26125
+ idx = RangeIndex(0, 10, 3)
+
+ result = np.max(idx)
+ assert result == 9
+
+ result = np.min(idx)
+ assert result == 0
+
+ errmsg = "the 'out' parameter is not supported"
+ with pytest.raises(ValueError, match=errmsg):
+ np.min(idx, out=0)
+ with pytest.raises(ValueError, match=errmsg):
+ np.max(idx, out=0)
+
+ # No need to test again argmax/argmin compat since the implementation
+ # is the same as basic integer index
+
+ def test_numpy_minmax_datetime64(self):
+ dr = date_range(start="2016-01-15", end="2016-01-20")
+
+ assert np.min(dr) == Timestamp("2016-01-15 00:00:00")
+ assert np.max(dr) == Timestamp("2016-01-20 00:00:00")
+
+ errmsg = "the 'out' parameter is not supported"
+ with pytest.raises(ValueError, match=errmsg):
+ np.min(dr, out=0)
+
+ with pytest.raises(ValueError, match=errmsg):
+ np.max(dr, out=0)
+
+ assert np.argmin(dr) == 0
+ assert np.argmax(dr) == 5
+
+ errmsg = "the 'out' parameter is not supported"
+ with pytest.raises(ValueError, match=errmsg):
+ np.argmin(dr, out=0)
+
+ with pytest.raises(ValueError, match=errmsg):
+ np.argmax(dr, out=0)
+
+ def test_minmax_period(self):
+ # monotonic
+ idx1 = PeriodIndex([NaT, "2011-01-01", "2011-01-02", "2011-01-03"], freq="D")
+ assert not idx1.is_monotonic_increasing
+ assert idx1[1:].is_monotonic_increasing
+
+ # non-monotonic
+ idx2 = PeriodIndex(
+ ["2011-01-01", NaT, "2011-01-03", "2011-01-02", NaT], freq="D"
+ )
+ assert not idx2.is_monotonic_increasing
+
+ for idx in [idx1, idx2]:
+ assert idx.min() == Period("2011-01-01", freq="D")
+ assert idx.max() == Period("2011-01-03", freq="D")
+ assert idx1.argmin() == 1
+ assert idx2.argmin() == 0
+ assert idx1.argmax() == 3
+ assert idx2.argmax() == 2
+
+ @pytest.mark.parametrize("op", ["min", "max"])
+ @pytest.mark.parametrize("data", [[], [NaT], [NaT, NaT, NaT]])
+ def test_minmax_period_empty_nat(self, op, data):
+ # Return NaT
+ obj = PeriodIndex(data, freq="M")
+ result = getattr(obj, op)()
+ assert result is NaT
+
+ def test_numpy_minmax_period(self):
+ pr = period_range(start="2016-01-15", end="2016-01-20")
+
+ assert np.min(pr) == Period("2016-01-15", freq="D")
+ assert np.max(pr) == Period("2016-01-20", freq="D")
+
+ errmsg = "the 'out' parameter is not supported"
+ with pytest.raises(ValueError, match=errmsg):
+ np.min(pr, out=0)
+ with pytest.raises(ValueError, match=errmsg):
+ np.max(pr, out=0)
+
+ assert np.argmin(pr) == 0
+ assert np.argmax(pr) == 5
+
+ errmsg = "the 'out' parameter is not supported"
+ with pytest.raises(ValueError, match=errmsg):
+ np.argmin(pr, out=0)
+ with pytest.raises(ValueError, match=errmsg):
+ np.argmax(pr, out=0)
+
+ def test_min_max_categorical(self):
+ ci = pd.CategoricalIndex(list("aabbca"), categories=list("cab"), ordered=False)
+ msg = (
+ r"Categorical is not ordered for operation min\n"
+ r"you can use .as_ordered\(\) to change the Categorical to an ordered one\n"
+ )
+ with pytest.raises(TypeError, match=msg):
+ ci.min()
+ msg = (
+ r"Categorical is not ordered for operation max\n"
+ r"you can use .as_ordered\(\) to change the Categorical to an ordered one\n"
+ )
+ with pytest.raises(TypeError, match=msg):
+ ci.max()
+
+ ci = pd.CategoricalIndex(list("aabbca"), categories=list("cab"), ordered=True)
+ assert ci.min() == "c"
+ assert ci.max() == "b"
+
+
+class TestSeriesReductions:
+ # Note: the name TestSeriesReductions indicates these tests
+ # were moved from a series-specific test file, _not_ that these tests are
+ # intended long-term to be series-specific
+
+ def test_sum_inf(self):
+ s = Series(np.random.default_rng(2).standard_normal(10))
+ s2 = s.copy()
+
+ s[5:8] = np.inf
+ s2[5:8] = np.nan
+
+ assert np.isinf(s.sum())
+
+ arr = np.random.default_rng(2).standard_normal((100, 100)).astype("f4")
+ arr[:, 2] = np.inf
+
+ res = nanops.nansum(arr, axis=1)
+ assert np.isinf(res).all()
+
+ @pytest.mark.parametrize(
+ "dtype", ["float64", "Float32", "Int64", "boolean", "object"]
+ )
+ @pytest.mark.parametrize("use_bottleneck", [True, False])
+ @pytest.mark.parametrize("method, unit", [("sum", 0.0), ("prod", 1.0)])
+ def test_empty(self, method, unit, use_bottleneck, dtype):
+ item = pd.NA if dtype in ["Float32", "Int64"] else np.nan
+ with pd.option_context("use_bottleneck", use_bottleneck):
+ # GH#9422 / GH#18921
+ # Entirely empty
+ s = Series([], dtype=dtype)
+ # NA by default
+ result = getattr(s, method)()
+ assert result == unit
+
+ # Explicit
+ result = getattr(s, method)(min_count=0)
+ assert result == unit
+
+ result = getattr(s, method)(min_count=1)
+ assert isna(result)
+
+ # Skipna, default
+ result = getattr(s, method)(skipna=True)
+ result == unit
+
+ # Skipna, explicit
+ result = getattr(s, method)(skipna=True, min_count=0)
+ assert result == unit
+
+ result = getattr(s, method)(skipna=True, min_count=1)
+ assert isna(result)
+
+ result = getattr(s, method)(skipna=False, min_count=0)
+ assert result == unit
+
+ result = getattr(s, method)(skipna=False, min_count=1)
+ assert isna(result)
+
+ # All-NA
+ s = Series([item], dtype=dtype)
+ # NA by default
+ result = getattr(s, method)()
+ assert result == unit
+
+ # Explicit
+ result = getattr(s, method)(min_count=0)
+ assert result == unit
+
+ result = getattr(s, method)(min_count=1)
+ assert isna(result)
+
+ # Skipna, default
+ result = getattr(s, method)(skipna=True)
+ result == unit
+
+ # skipna, explicit
+ result = getattr(s, method)(skipna=True, min_count=0)
+ assert result == unit
+
+ result = getattr(s, method)(skipna=True, min_count=1)
+ assert isna(result)
+
+ # Mix of valid, empty
+ s = Series([item, 1], dtype=dtype)
+ # Default
+ result = getattr(s, method)()
+ assert result == 1.0
+
+ # Explicit
+ result = getattr(s, method)(min_count=0)
+ assert result == 1.0
+
+ result = getattr(s, method)(min_count=1)
+ assert result == 1.0
+
+ # Skipna
+ result = getattr(s, method)(skipna=True)
+ assert result == 1.0
+
+ result = getattr(s, method)(skipna=True, min_count=0)
+ assert result == 1.0
+
+ # GH#844 (changed in GH#9422)
+ df = DataFrame(np.empty((10, 0)), dtype=dtype)
+ assert (getattr(df, method)(axis=1) == unit).all()
+
+ s = Series([1], dtype=dtype)
+ result = getattr(s, method)(min_count=2)
+ assert isna(result)
+
+ result = getattr(s, method)(skipna=False, min_count=2)
+ assert isna(result)
+
+ s = Series([item], dtype=dtype)
+ result = getattr(s, method)(min_count=2)
+ assert isna(result)
+
+ s = Series([item, 1], dtype=dtype)
+ result = getattr(s, method)(min_count=2)
+ assert isna(result)
+
+ @pytest.mark.parametrize("method", ["mean", "var"])
+ @pytest.mark.parametrize("dtype", ["Float64", "Int64", "boolean"])
+ def test_ops_consistency_on_empty_nullable(self, method, dtype):
+ # GH#34814
+ # consistency for nullable dtypes on empty or ALL-NA mean
+
+ # empty series
+ eser = Series([], dtype=dtype)
+ result = getattr(eser, method)()
+ assert result is pd.NA
+
+ # ALL-NA series
+ nser = Series([pd.NA], dtype=dtype)
+ result = getattr(nser, method)()
+ assert result is pd.NA
+
+ @pytest.mark.parametrize("method", ["mean", "median", "std", "var"])
+ def test_ops_consistency_on_empty(self, method):
+ # GH#7869
+ # consistency on empty
+
+ # float
+ result = getattr(Series(dtype=float), method)()
+ assert isna(result)
+
+ # timedelta64[ns]
+ tdser = Series([], dtype="m8[ns]")
+ if method == "var":
+ msg = "|".join(
+ [
+ "operation 'var' not allowed",
+ r"cannot perform var with type timedelta64\[ns\]",
+ "does not support operation 'var'",
+ ]
+ )
+ with pytest.raises(TypeError, match=msg):
+ getattr(tdser, method)()
+ else:
+ result = getattr(tdser, method)()
+ assert result is NaT
+
+ def test_nansum_buglet(self):
+ ser = Series([1.0, np.nan], index=[0, 1])
+ result = np.nansum(ser)
+ tm.assert_almost_equal(result, 1)
+
+ @pytest.mark.parametrize("use_bottleneck", [True, False])
+ @pytest.mark.parametrize("dtype", ["int32", "int64"])
+ def test_sum_overflow_int(self, use_bottleneck, dtype):
+ with pd.option_context("use_bottleneck", use_bottleneck):
+ # GH#6915
+ # overflowing on the smaller int dtypes
+ v = np.arange(5000000, dtype=dtype)
+ s = Series(v)
+
+ result = s.sum(skipna=False)
+ assert int(result) == v.sum(dtype="int64")
+ result = s.min(skipna=False)
+ assert int(result) == 0
+ result = s.max(skipna=False)
+ assert int(result) == v[-1]
+
+ @pytest.mark.parametrize("use_bottleneck", [True, False])
+ @pytest.mark.parametrize("dtype", ["float32", "float64"])
+ def test_sum_overflow_float(self, use_bottleneck, dtype):
+ with pd.option_context("use_bottleneck", use_bottleneck):
+ v = np.arange(5000000, dtype=dtype)
+ s = Series(v)
+
+ result = s.sum(skipna=False)
+ assert result == v.sum(dtype=dtype)
+ result = s.min(skipna=False)
+ assert np.allclose(float(result), 0.0)
+ result = s.max(skipna=False)
+ assert np.allclose(float(result), v[-1])
+
+ def test_mean_masked_overflow(self):
+ # GH#48378
+ val = 100_000_000_000_000_000
+ n_elements = 100
+ na = np.array([val] * n_elements)
+ ser = Series([val] * n_elements, dtype="Int64")
+
+ result_numpy = np.mean(na)
+ result_masked = ser.mean()
+ assert result_masked - result_numpy == 0
+ assert result_masked == 1e17
+
+ @pytest.mark.parametrize("ddof, exp", [(1, 2.5), (0, 2.0)])
+ def test_var_masked_array(self, ddof, exp):
+ # GH#48379
+ ser = Series([1, 2, 3, 4, 5], dtype="Int64")
+ ser_numpy_dtype = Series([1, 2, 3, 4, 5], dtype="int64")
+ result = ser.var(ddof=ddof)
+ result_numpy_dtype = ser_numpy_dtype.var(ddof=ddof)
+ assert result == result_numpy_dtype
+ assert result == exp
+
+ @pytest.mark.parametrize(
+ "values,expected", [([-1j, 0j, 1j], 1.0), ([1 + 2j, 2 + 3j, 3 + 4j], 2.0)]
+ )
+ def test_var_complex_array(self, values, expected):
+ # GH 61645, 62421
+ ser = Series(values, dtype=complex)
+ tm.assert_almost_equal(ser.var(ddof=1), expected)
+ tm.assert_almost_equal(ser.std(ddof=1), np.sqrt(expected))
+ tm.assert_almost_equal(ser.sem(ddof=1), np.sqrt(expected / len(values)))
+
+ @pytest.mark.parametrize("dtype", ("m8[ns]", "M8[ns]", "M8[ns, UTC]"))
+ def test_empty_timeseries_reductions_return_nat(self, dtype, skipna):
+ # covers GH#11245
+ assert Series([], dtype=dtype).min(skipna=skipna) is NaT
+ assert Series([], dtype=dtype).max(skipna=skipna) is NaT
+
+ def test_numpy_argmin(self):
+ # See GH#16830
+ data = np.arange(1, 11)
+
+ s = Series(data, index=data)
+ result = np.argmin(s)
+
+ expected = np.argmin(data)
+ assert result == expected
+
+ result = s.argmin()
+
+ assert result == expected
+
+ msg = "the 'out' parameter is not supported"
+ with pytest.raises(ValueError, match=msg):
+ np.argmin(s, out=data)
+
+ def test_numpy_argmax(self):
+ # See GH#16830
+ data = np.arange(1, 11)
+
+ ser = Series(data, index=data)
+ result = np.argmax(ser)
+ expected = np.argmax(data)
+ assert result == expected
+
+ result = ser.argmax()
+
+ assert result == expected
+
+ msg = "the 'out' parameter is not supported"
+ with pytest.raises(ValueError, match=msg):
+ np.argmax(ser, out=data)
+
+ def test_idxmin_dt64index(self, unit):
+ # GH#43587 should have NaT instead of NaN
+ dti = DatetimeIndex(["NaT", "2015-02-08", "NaT"]).as_unit(unit)
+ ser = Series([1.0, 2.0, np.nan], index=dti)
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ ser.idxmin(skipna=False)
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ ser.idxmax(skipna=False)
+
+ df = ser.to_frame()
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ df.idxmin(skipna=False)
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ df.idxmax(skipna=False)
+
+ def test_idxmin(self):
+ # test idxmin
+ # _check_stat_op approach can not be used here because of isna check.
+ string_series = Series(range(20), dtype=np.float64, name="series")
+
+ # add some NaNs
+ string_series[5:15] = np.nan
+
+ # skipna or no
+ assert string_series[string_series.idxmin()] == string_series.min()
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ string_series.idxmin(skipna=False)
+
+ # no NaNs
+ nona = string_series.dropna()
+ assert nona[nona.idxmin()] == nona.min()
+ assert nona.index.values.tolist().index(nona.idxmin()) == nona.values.argmin()
+
+ # all NaNs
+ allna = string_series * np.nan
+ msg = "Encountered all NA values"
+ with pytest.raises(ValueError, match=msg):
+ allna.idxmin()
+
+ # datetime64[ns]
+ s = Series(date_range("20130102", periods=6))
+ result = s.idxmin()
+ assert result == 0
+
+ s[0] = np.nan
+ result = s.idxmin()
+ assert result == 1
+
+ def test_idxmax(self):
+ # test idxmax
+ # _check_stat_op approach can not be used here because of isna check.
+ string_series = Series(range(20), dtype=np.float64, name="series")
+
+ # add some NaNs
+ string_series[5:15] = np.nan
+
+ # skipna or no
+ assert string_series[string_series.idxmax()] == string_series.max()
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ assert isna(string_series.idxmax(skipna=False))
+
+ # no NaNs
+ nona = string_series.dropna()
+ assert nona[nona.idxmax()] == nona.max()
+ assert nona.index.values.tolist().index(nona.idxmax()) == nona.values.argmax()
+
+ # all NaNs
+ allna = string_series * np.nan
+ msg = "Encountered all NA values"
+ with pytest.raises(ValueError, match=msg):
+ allna.idxmax()
+
+ s = Series(date_range("20130102", periods=6))
+ result = s.idxmax()
+ assert result == 5
+
+ s[5] = np.nan
+ result = s.idxmax()
+ assert result == 4
+
+ # Index with float64 dtype
+ # GH#5914
+ s = Series([1, 2, 3], [1.1, 2.1, 3.1])
+ result = s.idxmax()
+ assert result == 3.1
+ result = s.idxmin()
+ assert result == 1.1
+
+ s = Series(s.index, s.index)
+ result = s.idxmax()
+ assert result == 3.1
+ result = s.idxmin()
+ assert result == 1.1
+
+ def test_all_any(self):
+ ts = Series(
+ np.arange(10, dtype=np.float64),
+ index=date_range("2020-01-01", periods=10),
+ name="ts",
+ )
+ bool_series = ts > 0
+ assert not bool_series.all()
+ assert bool_series.any()
+
+ # Alternative types, with implicit 'object' dtype.
+ s = Series(["abc", True])
+ assert s.any()
+
+ def test_numpy_all_any(self, index_or_series):
+ # GH#40180
+ idx = index_or_series([0, 1, 2])
+ assert not np.all(idx)
+ assert np.any(idx)
+ idx = Index([1, 2, 3])
+ assert np.all(idx)
+
+ def test_all_any_skipna(self):
+ # Check skipna, with implicit 'object' dtype.
+ s1 = Series([np.nan, True])
+ s2 = Series([np.nan, False])
+ assert s1.all(skipna=False) # nan && True => True
+ assert s1.all(skipna=True)
+ assert s2.any(skipna=False)
+ assert not s2.any(skipna=True)
+
+ def test_all_any_bool_only(self):
+ s = Series([False, False, True, True, False, True], index=[0, 0, 1, 1, 2, 2])
+
+ # GH#47500 - test bool_only works
+ assert s.any(bool_only=True)
+ assert not s.all(bool_only=True)
+
+ def test_any_all_object_dtype(self, all_boolean_reductions, skipna):
+ # GH#12863
+ ser = Series(["a", "b", "c", "d", "e"], dtype=object)
+ result = getattr(ser, all_boolean_reductions)(skipna=skipna)
+ expected = True
+
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "data", [[False, None], [None, False], [False, np.nan], [np.nan, False]]
+ )
+ def test_any_all_object_dtype_missing(self, data, all_boolean_reductions):
+ # GH#27709
+ ser = Series(data)
+ result = getattr(ser, all_boolean_reductions)(skipna=False)
+
+ # None is treated is False, but np.nan is treated as True
+ expected = all_boolean_reductions == "any" and None not in data
+ assert result == expected
+
+ @pytest.mark.parametrize("dtype", ["boolean", "Int64", "UInt64", "Float64"])
+ @pytest.mark.parametrize(
+ # expected_data indexed as [[skipna=False/any, skipna=False/all],
+ # [skipna=True/any, skipna=True/all]]
+ "data,expected_data",
+ [
+ ([0, 0, 0], [[False, False], [False, False]]),
+ ([1, 1, 1], [[True, True], [True, True]]),
+ ([pd.NA, pd.NA, pd.NA], [[pd.NA, pd.NA], [False, True]]),
+ ([0, pd.NA, 0], [[pd.NA, False], [False, False]]),
+ ([1, pd.NA, 1], [[True, pd.NA], [True, True]]),
+ ([1, pd.NA, 0], [[True, False], [True, False]]),
+ ],
+ )
+ def test_any_all_nullable_kleene_logic(
+ self, all_boolean_reductions, skipna, data, dtype, expected_data
+ ):
+ # GH-37506, GH-41967
+ ser = Series(data, dtype=dtype)
+ expected = expected_data[skipna][all_boolean_reductions == "all"]
+
+ result = getattr(ser, all_boolean_reductions)(skipna=skipna)
+ assert (result is pd.NA and expected is pd.NA) or result == expected
+
+ def test_any_axis1_bool_only(self):
+ # GH#32432
+ df = DataFrame({"A": [True, False], "B": [1, 2]})
+ result = df.any(axis=1, bool_only=True)
+ expected = Series([True, False])
+ tm.assert_series_equal(result, expected)
+
+ def test_any_all_datetimelike(self):
+ # GH#38723 these may not be the desired long-term behavior (GH#34479)
+ # but in the interim should be internally consistent
+ dta = date_range("1995-01-02", periods=3)._data
+ ser = Series(dta)
+ df = DataFrame(ser)
+
+ # GH#34479
+ msg = "datetime64 type does not support operation '(any|all)'"
+ with pytest.raises(TypeError, match=msg):
+ dta.all()
+ with pytest.raises(TypeError, match=msg):
+ dta.any()
+
+ with pytest.raises(TypeError, match=msg):
+ ser.all()
+ with pytest.raises(TypeError, match=msg):
+ ser.any()
+
+ with pytest.raises(TypeError, match=msg):
+ df.any().all()
+ with pytest.raises(TypeError, match=msg):
+ df.all().all()
+
+ dta = dta.tz_localize("UTC")
+ ser = Series(dta)
+ df = DataFrame(ser)
+ # GH#34479
+ with pytest.raises(TypeError, match=msg):
+ dta.all()
+ with pytest.raises(TypeError, match=msg):
+ dta.any()
+
+ with pytest.raises(TypeError, match=msg):
+ ser.all()
+ with pytest.raises(TypeError, match=msg):
+ ser.any()
+
+ with pytest.raises(TypeError, match=msg):
+ df.any().all()
+ with pytest.raises(TypeError, match=msg):
+ df.all().all()
+
+ tda = dta - dta[0]
+ ser = Series(tda)
+ df = DataFrame(ser)
+
+ assert tda.any()
+ assert not tda.all()
+
+ assert ser.any()
+ assert not ser.all()
+
+ assert df.any().all()
+ assert not df.all().any()
+
+ def test_any_all_string_dtype(self, any_string_dtype):
+ # GH#54591
+ if (
+ isinstance(any_string_dtype, pd.StringDtype)
+ and any_string_dtype.na_value is pd.NA
+ ):
+ # the nullable string dtype currently still raise an error
+ # https://github.com/pandas-dev/pandas/issues/51939
+ ser = Series(["a", "b"], dtype=any_string_dtype)
+ with pytest.raises(TypeError):
+ ser.any()
+ with pytest.raises(TypeError):
+ ser.all()
+ return
+
+ ser = Series(["", "a"], dtype=any_string_dtype)
+ assert ser.any()
+ assert not ser.all()
+ assert ser.any(skipna=False)
+ assert not ser.all(skipna=False)
+
+ ser = Series([np.nan, "a"], dtype=any_string_dtype)
+ assert ser.any()
+ assert ser.all()
+ assert ser.any(skipna=False)
+ assert ser.all(skipna=False) # NaN is considered truthy
+
+ ser = Series([np.nan, ""], dtype=any_string_dtype)
+ assert not ser.any()
+ assert not ser.all()
+ assert ser.any(skipna=False) # NaN is considered truthy
+ assert not ser.all(skipna=False)
+
+ ser = Series(["a", "b"], dtype=any_string_dtype)
+ assert ser.any()
+ assert ser.all()
+ assert ser.any(skipna=False)
+ assert ser.all(skipna=False)
+
+ ser = Series([], dtype=any_string_dtype)
+ assert not ser.any()
+ assert ser.all()
+ assert not ser.any(skipna=False)
+ assert ser.all(skipna=False)
+
+ ser = Series([""], dtype=any_string_dtype)
+ assert not ser.any()
+ assert not ser.all()
+ assert not ser.any(skipna=False)
+ assert not ser.all(skipna=False)
+
+ ser = Series([np.nan], dtype=any_string_dtype)
+ assert not ser.any()
+ assert ser.all()
+ assert ser.any(skipna=False) # NaN is considered truthy
+ assert ser.all(skipna=False) # NaN is considered truthy
+
+ def test_timedelta64_analytics(self):
+ # index min/max
+ dti = date_range("2012-1-1", periods=3, freq="D")
+ td = Series(dti) - Timestamp("20120101")
+
+ result = td.idxmin()
+ assert result == 0
+
+ result = td.idxmax()
+ assert result == 2
+
+ # GH#2982
+ # with NaT
+ td[0] = np.nan
+
+ result = td.idxmin()
+ assert result == 1
+
+ result = td.idxmax()
+ assert result == 2
+
+ # abs
+ s1 = Series(date_range("20120101", periods=3))
+ s2 = Series(date_range("20120102", periods=3))
+ expected = Series(s2 - s1)
+
+ result = np.abs(s1 - s2)
+ tm.assert_series_equal(result, expected)
+
+ result = (s1 - s2).abs()
+ tm.assert_series_equal(result, expected)
+
+ # max/min
+ result = td.max()
+ expected = Timedelta("2 days")
+ assert result == expected
+
+ result = td.min()
+ expected = Timedelta("1 days")
+ assert result == expected
+
+ def test_assert_idxminmax_empty_raises(self):
+ """
+ Cases where ``Series.argmax`` and related should raise an exception
+ """
+ test_input = Series([], dtype="float64")
+ msg = "attempt to get argmin of an empty sequence"
+ with pytest.raises(ValueError, match=msg):
+ test_input.idxmin()
+ with pytest.raises(ValueError, match=msg):
+ test_input.idxmin(skipna=False)
+ msg = "attempt to get argmax of an empty sequence"
+ with pytest.raises(ValueError, match=msg):
+ test_input.idxmax()
+ with pytest.raises(ValueError, match=msg):
+ test_input.idxmax(skipna=False)
+
+ def test_idxminmax_object_dtype(self, using_infer_string):
+ # pre-2.1 object-dtype was disallowed for argmin/max
+ ser = Series(["foo", "bar", "baz"])
+ assert ser.idxmax() == 0
+ assert ser.idxmax(skipna=False) == 0
+ assert ser.idxmin() == 1
+ assert ser.idxmin(skipna=False) == 1
+
+ ser2 = Series([(1,), (2,)])
+ assert ser2.idxmax() == 1
+ assert ser2.idxmax(skipna=False) == 1
+ assert ser2.idxmin() == 0
+ assert ser2.idxmin(skipna=False) == 0
+
+ if not using_infer_string:
+ # attempting to compare np.nan with string raises
+ ser3 = Series(["foo", "foo", "bar", "bar", None, np.nan, "baz"])
+ msg = "'>' not supported between instances of 'float' and 'str'"
+ with pytest.raises(TypeError, match=msg):
+ ser3.idxmax()
+ with pytest.raises(TypeError, match=msg):
+ ser3.idxmax(skipna=False)
+ msg = "'<' not supported between instances of 'float' and 'str'"
+ with pytest.raises(TypeError, match=msg):
+ ser3.idxmin()
+ with pytest.raises(TypeError, match=msg):
+ ser3.idxmin(skipna=False)
+
+ def test_idxminmax_object_frame(self):
+ # GH#4279
+ df = DataFrame([["zimm", 2.5], ["biff", 1.0], ["bid", 12.0]])
+ res = df.idxmax()
+ exp = Series([0, 2])
+ tm.assert_series_equal(res, exp)
+
+ def test_idxminmax_object_tuples(self):
+ # GH#43697
+ ser = Series([(1, 3), (2, 2), (3, 1)])
+ assert ser.idxmax() == 2
+ assert ser.idxmin() == 0
+ assert ser.idxmax(skipna=False) == 2
+ assert ser.idxmin(skipna=False) == 0
+
+ def test_idxminmax_object_decimals(self):
+ # GH#40685
+ df = DataFrame(
+ {
+ "idx": [0, 1],
+ "x": [Decimal("8.68"), Decimal("42.23")],
+ "y": [Decimal("7.11"), Decimal("79.61")],
+ }
+ )
+ res = df.idxmax()
+ exp = Series({"idx": 1, "x": 1, "y": 1})
+ tm.assert_series_equal(res, exp)
+
+ res2 = df.idxmin()
+ exp2 = exp - 1
+ tm.assert_series_equal(res2, exp2)
+
+ def test_argminmax_object_ints(self):
+ # GH#18021
+ ser = Series([0, 1], dtype="object")
+ assert ser.argmax() == 1
+ assert ser.argmin() == 0
+ assert ser.argmax(skipna=False) == 1
+ assert ser.argmin(skipna=False) == 0
+
+ def test_idxminmax_with_inf(self):
+ # For numeric data with NA and Inf (GH #13595)
+ s = Series([0, -np.inf, np.inf, np.nan])
+
+ assert s.idxmin() == 1
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ s.idxmin(skipna=False)
+
+ assert s.idxmax() == 2
+ with pytest.raises(ValueError, match="Encountered an NA value"):
+ s.idxmax(skipna=False)
+
+ def test_sum_uint64(self):
+ # GH 53401
+ s = Series([10000000000000000000], dtype="uint64")
+ result = s.sum()
+ expected = np.uint64(10000000000000000000)
+ tm.assert_almost_equal(result, expected)
+
+ def test_signedness_preserved_after_sum(self, using_python_scalars):
+ # GH 37491
+ ser = Series([1, 2, 3, 4])
+
+ result = ser.astype("uint8").sum()
+ if using_python_scalars:
+ assert isinstance(result, int)
+ else:
+ assert isinstance(result, np.uint64)
+
+
+class TestDatetime64SeriesReductions:
+ # Note: the name TestDatetime64SeriesReductions indicates these tests
+ # were moved from a series-specific test file, _not_ that these tests are
+ # intended long-term to be series-specific
+
+ @pytest.mark.parametrize(
+ "nat_ser",
+ [
+ Series([NaT, NaT]),
+ Series([NaT, Timedelta("nat")]),
+ Series([Timedelta("nat"), Timedelta("nat")]),
+ ],
+ )
+ def test_minmax_nat_series(self, nat_ser):
+ # GH#23282
+ assert nat_ser.min() is NaT
+ assert nat_ser.max() is NaT
+ assert nat_ser.min(skipna=False) is NaT
+ assert nat_ser.max(skipna=False) is NaT
+
+ @pytest.mark.parametrize(
+ "nat_df",
+ [
+ [NaT, NaT],
+ [NaT, Timedelta("nat")],
+ [Timedelta("nat"), Timedelta("nat")],
+ ],
+ )
+ def test_minmax_nat_dataframe(self, nat_df):
+ # GH#23282
+ nat_df = DataFrame(nat_df)
+ assert nat_df.min()[0] is NaT
+ assert nat_df.max()[0] is NaT
+ assert nat_df.min(skipna=False)[0] is NaT
+ assert nat_df.max(skipna=False)[0] is NaT
+
+ def test_min_max(self):
+ rng = date_range("1/1/2000", "12/31/2000")
+ rng2 = rng.take(np.random.default_rng(2).permutation(len(rng)))
+
+ the_min = rng2.min()
+ the_max = rng2.max()
+ assert isinstance(the_min, Timestamp)
+ assert isinstance(the_max, Timestamp)
+ assert the_min == rng[0]
+ assert the_max == rng[-1]
+
+ assert rng.min() == rng[0]
+ assert rng.max() == rng[-1]
+
+ def test_min_max_series(self):
+ rng = date_range("1/1/2000", periods=10, freq="4h")
+ lvls = ["A", "A", "A", "B", "B", "B", "C", "C", "C", "C"]
+ df = DataFrame(
+ {
+ "TS": rng,
+ "V": np.random.default_rng(2).standard_normal(len(rng)),
+ "L": lvls,
+ }
+ )
+
+ result = df.TS.max()
+ exp = Timestamp(df.TS.iat[-1])
+ assert isinstance(result, Timestamp)
+ assert result == exp
+
+ result = df.TS.min()
+ exp = Timestamp(df.TS.iat[0])
+ assert isinstance(result, Timestamp)
+ assert result == exp
+
+
+class TestCategoricalSeriesReductions:
+ # Note: the name TestCategoricalSeriesReductions indicates these tests
+ # were moved from a series-specific test file, _not_ that these tests are
+ # intended long-term to be series-specific
+
+ @pytest.mark.parametrize("function", ["min", "max"])
+ def test_min_max_unordered_raises(self, function):
+ # unordered cats have no min/max
+ cat = Series(Categorical(["a", "b", "c", "d"], ordered=False))
+ msg = f"Categorical is not ordered for operation {function}"
+ with pytest.raises(TypeError, match=msg):
+ getattr(cat, function)()
+
+ @pytest.mark.parametrize(
+ "values, categories",
+ [
+ (list("abc"), list("abc")),
+ (list("abc"), list("cba")),
+ ([*list("abc"), np.nan], list("cba")),
+ ([1, 2, 3], [3, 2, 1]),
+ ([1, 2, 3, np.nan], [3, 2, 1]),
+ ],
+ )
+ @pytest.mark.parametrize("function", ["min", "max"])
+ def test_min_max_ordered(self, values, categories, function):
+ # GH 25303
+ cat = Series(Categorical(values, categories=categories, ordered=True))
+ result = getattr(cat, function)(skipna=True)
+ expected = categories[0] if function == "min" else categories[2]
+ assert result == expected
+
+ @pytest.mark.parametrize("function", ["min", "max"])
+ def test_min_max_ordered_with_nan_only(self, function, skipna):
+ # https://github.com/pandas-dev/pandas/issues/33450
+ cat = Series(Categorical([np.nan], categories=[1, 2], ordered=True))
+ result = getattr(cat, function)(skipna=skipna)
+ assert result is np.nan
+
+ @pytest.mark.parametrize("function", ["min", "max"])
+ def test_min_max_skipna(self, function, skipna):
+ cat = Series(
+ Categorical(["a", "b", np.nan, "a"], categories=["b", "a"], ordered=True)
+ )
+ result = getattr(cat, function)(skipna=skipna)
+
+ if skipna is True:
+ expected = "b" if function == "min" else "a"
+ assert result == expected
+ else:
+ assert result is np.nan
+
+
+class TestSeriesMode:
+ # Note: the name TestSeriesMode indicates these tests
+ # were moved from a series-specific test file, _not_ that these tests are
+ # intended long-term to be series-specific
+
+ def test_mode_empty(self, dropna):
+ s = Series([], dtype=np.float64)
+ result = s.mode(dropna)
+ tm.assert_series_equal(result, s)
+
+ @pytest.mark.parametrize(
+ "dropna, data, expected",
+ [
+ (True, [1, 1, 1, 2], [1]),
+ (True, [1, 1, 1, 2, 3, 3, 3], [1, 3]),
+ (False, [1, 1, 1, 2], [1]),
+ (False, [1, 1, 1, 2, 3, 3, 3], [1, 3]),
+ ],
+ )
+ def test_mode_numerical(self, dropna, data, expected, any_real_numpy_dtype):
+ s = Series(data, dtype=any_real_numpy_dtype)
+ result = s.mode(dropna)
+ expected = Series(expected, dtype=any_real_numpy_dtype)
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize("dropna, expected", [(True, [1.0]), (False, [1, np.nan])])
+ def test_mode_numerical_nan(self, dropna, expected):
+ s = Series([1, 1, 2, np.nan, np.nan])
+ result = s.mode(dropna)
+ expected = Series(expected)
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "dropna, expected1, expected2",
+ [
+ (True, ["b"], ["bar"]),
+ (False, ["b"], [np.nan]),
+ ],
+ )
+ def test_mode_object(self, dropna, expected1, expected2):
+ # Test string and object types.
+ data = ["a"] * 2 + ["b"] * 3
+
+ s = Series(data, dtype="c")
+ result = s.mode(dropna)
+ expected1 = Series(expected1, dtype="c")
+ tm.assert_series_equal(result, expected1)
+
+ data = ["foo", "bar", "bar", np.nan, np.nan, np.nan]
+
+ s = Series(data, dtype=object)
+ result = s.mode(dropna)
+ expected2 = Series(expected2, dtype=object)
+ tm.assert_series_equal(result, expected2)
+
+ @pytest.mark.parametrize(
+ "dropna, expected1, expected2",
+ [
+ (True, ["b"], ["bar"]),
+ (False, ["b"], [np.nan]),
+ ],
+ )
+ def test_mode_string(self, dropna, expected1, expected2, any_string_dtype):
+ # Test string and object types.
+ data = ["a"] * 2 + ["b"] * 3
+
+ s = Series(data, dtype=any_string_dtype)
+ result = s.mode(dropna)
+ expected1 = Series(expected1, dtype=any_string_dtype)
+ tm.assert_series_equal(result, expected1)
+
+ data = ["foo", "bar", "bar", np.nan, np.nan, np.nan]
+
+ s = Series(data, dtype=any_string_dtype)
+ result = s.mode(dropna)
+ expected2 = Series(expected2, dtype=any_string_dtype)
+ tm.assert_series_equal(result, expected2)
+
+ @pytest.mark.parametrize(
+ "dropna, expected1, expected2",
+ [(True, ["foo"], ["foo"]), (False, ["foo"], [np.nan])],
+ )
+ def test_mode_mixeddtype(self, dropna, expected1, expected2):
+ s = Series([1, "foo", "foo"])
+ result = s.mode(dropna)
+ expected = Series(expected1, dtype=object)
+ tm.assert_series_equal(result, expected)
+
+ s = Series([1, "foo", "foo", np.nan, np.nan, np.nan])
+ result = s.mode(dropna)
+ expected = Series(expected2, dtype=object)
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "dropna, expected1, expected2",
+ [
+ (
+ True,
+ ["1900-05-03", "2011-01-03", "2013-01-02"],
+ ["2011-01-03", "2013-01-02"],
+ ),
+ (False, [np.nan], [np.nan, "2011-01-03", "2013-01-02"]),
+ ],
+ )
+ def test_mode_datetime(self, dropna, expected1, expected2):
+ s = Series(
+ ["2011-01-03", "2013-01-02", "1900-05-03", "nan", "nan"], dtype="M8[ns]"
+ )
+ result = s.mode(dropna)
+ expected1 = Series(expected1, dtype="M8[ns]")
+ tm.assert_series_equal(result, expected1)
+
+ s = Series(
+ [
+ "2011-01-03",
+ "2013-01-02",
+ "1900-05-03",
+ "2011-01-03",
+ "2013-01-02",
+ "nan",
+ "nan",
+ ],
+ dtype="M8[ns]",
+ )
+ result = s.mode(dropna)
+ expected2 = Series(expected2, dtype="M8[ns]")
+ tm.assert_series_equal(result, expected2)
+
+ @pytest.mark.parametrize(
+ "dropna, expected1, expected2",
+ [
+ (True, ["-1 days", "0 days", "1 days"], ["2 min", "1 day"]),
+ (False, [np.nan], [np.nan, "2 min", "1 day"]),
+ ],
+ )
+ def test_mode_timedelta(self, dropna, expected1, expected2):
+ # gh-5986: Test timedelta types.
+
+ s = Series(
+ ["1 days", "-1 days", "0 days", "nan", "nan"], dtype="timedelta64[ns]"
+ )
+ result = s.mode(dropna)
+ expected1 = Series(expected1, dtype="timedelta64[ns]")
+ tm.assert_series_equal(result, expected1)
+
+ s = Series(
+ [
+ "1 day",
+ "1 day",
+ "-1 day",
+ "-1 day 2 min",
+ "2 min",
+ "2 min",
+ "nan",
+ "nan",
+ ],
+ dtype="timedelta64[ns]",
+ )
+ result = s.mode(dropna)
+ expected2 = Series(expected2, dtype="timedelta64[ns]")
+ tm.assert_series_equal(result, expected2)
+
+ @pytest.mark.parametrize(
+ "dropna, expected1, expected2, expected3",
+ [
+ (
+ True,
+ Categorical([1, 2], categories=[1, 2]),
+ Categorical(["a"], categories=[1, "a"]),
+ Categorical([3, 1], categories=[3, 2, 1], ordered=True),
+ ),
+ (
+ False,
+ Categorical([np.nan], categories=[1, 2]),
+ Categorical([np.nan, "a"], categories=[1, "a"]),
+ Categorical([np.nan, 3, 1], categories=[3, 2, 1], ordered=True),
+ ),
+ ],
+ )
+ def test_mode_category(self, dropna, expected1, expected2, expected3):
+ s = Series(Categorical([1, 2, np.nan, np.nan]))
+ result = s.mode(dropna)
+ expected1 = Series(expected1, dtype="category")
+ tm.assert_series_equal(result, expected1)
+
+ s = Series(Categorical([1, "a", "a", np.nan, np.nan]))
+ result = s.mode(dropna)
+ expected2 = Series(expected2, dtype="category")
+ tm.assert_series_equal(result, expected2)
+
+ s = Series(
+ Categorical(
+ [1, 1, 2, 3, 3, np.nan, np.nan], categories=[3, 2, 1], ordered=True
+ )
+ )
+ result = s.mode(dropna)
+ expected3 = Series(expected3, dtype="category")
+ tm.assert_series_equal(result, expected3)
+
+ @pytest.mark.parametrize(
+ "dropna, expected1, expected2",
+ [(True, [2**63], [1, 2**63]), (False, [2**63], [1, 2**63])],
+ )
+ def test_mode_intoverflow(self, dropna, expected1, expected2):
+ # Test for uint64 overflow.
+ s = Series([1, 2**63, 2**63], dtype=np.uint64)
+ result = s.mode(dropna)
+ expected1 = Series(expected1, dtype=np.uint64)
+ tm.assert_series_equal(result, expected1)
+
+ s = Series([1, 2**63], dtype=np.uint64)
+ result = s.mode(dropna)
+ expected2 = Series(expected2, dtype=np.uint64)
+ tm.assert_series_equal(result, expected2)
+
+ def test_mode_sort_with_na(self):
+ s = Series([1, "foo", "foo", np.nan, np.nan])
+ expected = Series(["foo", np.nan], dtype=object)
+ result = s.mode(dropna=False)
+ tm.assert_series_equal(result, expected)
+
+ def test_mode_boolean_with_na(self):
+ # GH#42107
+ ser = Series([True, False, True, pd.NA], dtype="boolean")
+ result = ser.mode()
+ expected = Series([True], dtype="boolean")
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "array,expected,dtype",
+ [
+ (
+ [0, 1j, 1, 1, 1 + 1j, 1 + 2j],
+ [1],
+ np.complex128,
+ ),
+ (
+ [0, 1j, 1, 1, 1 + 1j, 1 + 2j],
+ [1],
+ np.complex64,
+ ),
+ (
+ [1 + 1j, 2j, 1 + 1j],
+ [1 + 1j],
+ np.complex128,
+ ),
+ ],
+ )
+ def test_single_mode_value_complex(self, array, expected, dtype):
+ result = Series(array, dtype=dtype).mode()
+ expected = Series(expected, dtype=dtype)
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "array,expected,dtype",
+ [
+ (
+ # no modes
+ [0, 1j, 1, 1 + 1j, 1 + 2j],
+ [0j, 1j, 1 + 0j, 1 + 1j, 1 + 2j],
+ np.complex128,
+ ),
+ (
+ [1 + 1j, 2j, 1 + 1j, 2j, 3],
+ [2j, 1 + 1j],
+ np.complex64,
+ ),
+ ],
+ )
+ def test_multimode_complex(self, array, expected, dtype):
+ # GH 17927
+ # mode tries to sort multimodal series.
+ # Complex numbers are sorted by their magnitude
+ result = Series(array, dtype=dtype).mode()
+ expected = Series(expected, dtype=dtype)
+ tm.assert_series_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reductions/test_stat_reductions.py b/.venv/lib/python3.13/site-packages/pandas/tests/reductions/test_stat_reductions.py
new file mode 100644
index 0000000000000000000000000000000000000000..604bd7efa8f7d4555af6292aae70f7460e4d192a
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reductions/test_stat_reductions.py
@@ -0,0 +1,280 @@
+"""
+Tests for statistical reductions of 2nd moment or higher: var, skew, kurt, ...
+"""
+
+import inspect
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Series,
+ date_range,
+)
+import pandas._testing as tm
+
+
+class TestDatetimeLikeStatReductions:
+ def test_dt64_mean(self, tz_naive_fixture, index_or_series_or_array):
+ tz = tz_naive_fixture
+
+ dti = date_range("2001-01-01", periods=11, tz=tz)
+ # shuffle so that we are not just working with monotone-increasing
+ dti = dti.take([4, 1, 3, 10, 9, 7, 8, 5, 0, 2, 6])
+ dtarr = dti._data
+
+ obj = index_or_series_or_array(dtarr)
+ assert obj.mean() == pd.Timestamp("2001-01-06", tz=tz)
+ assert obj.mean(skipna=False) == pd.Timestamp("2001-01-06", tz=tz)
+
+ # dtarr[-2] will be the first date 2001-01-1
+ dtarr[-2] = pd.NaT
+
+ obj = index_or_series_or_array(dtarr)
+ assert obj.mean() == pd.Timestamp("2001-01-06 07:12:00", tz=tz)
+ assert obj.mean(skipna=False) is pd.NaT
+
+ @pytest.mark.parametrize("freq", ["s", "h", "D", "W", "B"])
+ def test_period_mean(self, index_or_series_or_array, freq):
+ # GH#24757
+ dti = date_range("2001-01-01", periods=11)
+ # shuffle so that we are not just working with monotone-increasing
+ dti = dti.take([4, 1, 3, 10, 9, 7, 8, 5, 0, 2, 6])
+
+ warn = FutureWarning if freq == "B" else None
+ msg = r"PeriodDtype\[B\] is deprecated"
+ with tm.assert_produces_warning(warn, match=msg):
+ parr = dti._data.to_period(freq)
+ obj = index_or_series_or_array(parr)
+ with pytest.raises(TypeError, match="ambiguous"):
+ obj.mean()
+ with pytest.raises(TypeError, match="ambiguous"):
+ obj.mean(skipna=True)
+
+ # parr[-2] will be the first date 2001-01-1
+ parr[-2] = pd.NaT
+
+ with pytest.raises(TypeError, match="ambiguous"):
+ obj.mean()
+ with pytest.raises(TypeError, match="ambiguous"):
+ obj.mean(skipna=True)
+
+ def test_td64_mean(self, index_or_series_or_array):
+ m8values = np.array([0, 3, -2, -7, 1, 2, -1, 3, 5, -2, 4], "m8[D]")
+ tdi = pd.TimedeltaIndex(m8values).as_unit("ns")
+
+ tdarr = tdi._data
+ obj = index_or_series_or_array(tdarr, copy=False)
+
+ result = obj.mean()
+ expected = np.array(tdarr).mean()
+ assert result == expected
+
+ tdarr[0] = pd.NaT
+ assert obj.mean(skipna=False) is pd.NaT
+
+ result2 = obj.mean(skipna=True)
+ assert result2 == tdi[1:].mean()
+
+ # exact equality fails by 1 nanosecond
+ assert result2.round("us") == (result * 11.0 / 10).round("us")
+
+
+class TestSeriesStatReductions:
+ # Note: the name TestSeriesStatReductions indicates these tests
+ # were moved from a series-specific test file, _not_ that these tests are
+ # intended long-term to be series-specific
+
+ def _check_stat_op(
+ self, name, alternate, string_series_, check_objects=False, check_allna=False
+ ):
+ with pd.option_context("use_bottleneck", False):
+ f = getattr(Series, name)
+
+ # add some NaNs
+ string_series_[5:15] = np.nan
+
+ # mean, idxmax, idxmin, min, and max are valid for dates
+ if name not in ["max", "min", "mean", "median", "std"]:
+ ds = Series(date_range("1/1/2001", periods=10))
+ msg = f"does not support operation '{name}'"
+ with pytest.raises(TypeError, match=msg):
+ f(ds)
+
+ # skipna or no
+ assert pd.notna(f(string_series_))
+ assert pd.isna(f(string_series_, skipna=False))
+
+ # check the result is correct
+ nona = string_series_.dropna()
+ tm.assert_almost_equal(f(nona), alternate(nona.values))
+ tm.assert_almost_equal(f(string_series_), alternate(nona.values))
+
+ allna = string_series_ * np.nan
+
+ if check_allna:
+ assert np.isnan(f(allna))
+
+ # dtype=object with None, it works!
+ s = Series([1, 2, 3, None, 5])
+ f(s)
+
+ # GH#2888
+ items = [0]
+ items.extend(range(2**40, 2**40 + 1000))
+ s = Series(items, dtype="int64")
+ tm.assert_almost_equal(float(f(s)), float(alternate(s.values)))
+
+ # check date range
+ if check_objects:
+ s = Series(pd.bdate_range("1/1/2000", periods=10))
+ res = f(s)
+ exp = alternate(s)
+ assert res == exp
+
+ # check on string data
+ if name not in ["sum", "min", "max"]:
+ with pytest.raises(TypeError, match=None):
+ f(Series(list("abc")))
+
+ # Invalid axis.
+ msg = "No axis named 1 for object type Series"
+ with pytest.raises(ValueError, match=msg):
+ f(string_series_, axis=1)
+
+ if "numeric_only" in inspect.getfullargspec(f).args:
+ # only the index is string; dtype is float
+ f(string_series_, numeric_only=True)
+
+ def test_sum(self):
+ string_series = Series(range(20), dtype=np.float64, name="series")
+ self._check_stat_op("sum", np.sum, string_series, check_allna=False)
+
+ def test_mean(self):
+ string_series = Series(range(20), dtype=np.float64, name="series")
+ self._check_stat_op("mean", np.mean, string_series)
+
+ def test_median(self):
+ string_series = Series(range(20), dtype=np.float64, name="series")
+ self._check_stat_op("median", np.median, string_series)
+
+ # test with integers, test failure
+ int_ts = Series(np.ones(10, dtype=int), index=range(10))
+ tm.assert_almost_equal(np.median(int_ts), int_ts.median())
+
+ def test_prod(self):
+ string_series = Series(range(20), dtype=np.float64, name="series")
+ self._check_stat_op("prod", np.prod, string_series)
+
+ def test_min(self):
+ string_series = Series(range(20), dtype=np.float64, name="series")
+ self._check_stat_op("min", np.min, string_series, check_objects=True)
+
+ def test_max(self):
+ string_series = Series(range(20), dtype=np.float64, name="series")
+ self._check_stat_op("max", np.max, string_series, check_objects=True)
+
+ def test_var_std(self):
+ string_series = Series(range(20), dtype=np.float64, name="series")
+ datetime_series = Series(
+ np.arange(10, dtype=np.float64),
+ index=date_range("2020-01-01", periods=10),
+ name="ts",
+ )
+
+ alt = lambda x: np.std(x, ddof=1)
+ self._check_stat_op("std", alt, string_series)
+
+ alt = lambda x: np.var(x, ddof=1)
+ self._check_stat_op("var", alt, string_series)
+
+ result = datetime_series.std(ddof=4)
+ expected = np.std(datetime_series.values, ddof=4)
+ tm.assert_almost_equal(result, expected)
+
+ result = datetime_series.var(ddof=4)
+ expected = np.var(datetime_series.values, ddof=4)
+ tm.assert_almost_equal(result, expected)
+
+ # 1 - element series with ddof=1
+ s = datetime_series.iloc[[0]]
+ result = s.var(ddof=1)
+ assert pd.isna(result)
+
+ result = s.std(ddof=1)
+ assert pd.isna(result)
+
+ def test_sem(self):
+ string_series = Series(range(20), dtype=np.float64, name="series")
+ datetime_series = Series(
+ np.arange(10, dtype=np.float64),
+ index=date_range("2020-01-01", periods=10),
+ name="ts",
+ )
+
+ alt = lambda x: np.std(x, ddof=1) / np.sqrt(len(x))
+ self._check_stat_op("sem", alt, string_series)
+
+ result = datetime_series.sem(ddof=4)
+ expected = np.std(datetime_series.values, ddof=4) / np.sqrt(
+ len(datetime_series.values)
+ )
+ tm.assert_almost_equal(result, expected)
+
+ # 1 - element series with ddof=1
+ s = datetime_series.iloc[[0]]
+ result = s.sem(ddof=1)
+ assert pd.isna(result)
+
+ def test_skew(self, using_python_scalars):
+ sp_stats = pytest.importorskip("scipy.stats")
+
+ string_series = Series(range(20), dtype=np.float64, name="series")
+
+ alt = lambda x: sp_stats.skew(x, bias=False)
+ self._check_stat_op("skew", alt, string_series)
+
+ # test corner cases, skew() returns NaN unless there's at least 3
+ # values
+ min_N = 3
+ for i in range(1, min_N + 1):
+ s = Series(np.ones(i))
+ df = DataFrame(np.ones((i, i)))
+ if i < min_N:
+ assert np.isnan(s.skew())
+ assert np.isnan(df.skew()).all()
+ else:
+ assert 0 == s.skew()
+ if using_python_scalars:
+ assert type(s.skew()) == float
+ else:
+ assert isinstance(s.skew(), np.float64) # GH53482
+ assert (df.skew() == 0).all()
+
+ def test_kurt(self):
+ sp_stats = pytest.importorskip("scipy.stats")
+
+ string_series = Series(range(20), dtype=np.float64, name="series")
+
+ alt = lambda x: sp_stats.kurtosis(x, bias=False)
+ self._check_stat_op("kurt", alt, string_series)
+
+ def test_kurt_corner(self, using_python_scalars):
+ # test corner cases, kurt() returns NaN unless there's at least 4
+ # values
+ min_N = 4
+ for i in range(1, min_N + 1):
+ s = Series(np.ones(i))
+ df = DataFrame(np.ones((i, i)))
+ if i < min_N:
+ assert np.isnan(s.kurt())
+ assert np.isnan(df.kurt()).all()
+ else:
+ assert 0 == s.kurt()
+ if using_python_scalars:
+ assert type(s.kurt()) == float
+ else:
+ assert isinstance(s.kurt(), np.float64) # GH53482
+ assert (df.kurt() == 0).all()
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/resample/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/resample/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/resample/conftest.py b/.venv/lib/python3.13/site-packages/pandas/tests/resample/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..6c45ece5d8fb963d32945877b4af6521ea7b9450
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/resample/conftest.py
@@ -0,0 +1,33 @@
+import pytest
+
+# The various methods we support
+downsample_methods = [
+ "min",
+ "max",
+ "first",
+ "last",
+ "sum",
+ "mean",
+ "sem",
+ "median",
+ "prod",
+ "var",
+ "std",
+ "ohlc",
+ "quantile",
+]
+upsample_methods = ["count", "size"]
+series_methods = ["nunique"]
+resample_methods = downsample_methods + upsample_methods + series_methods
+
+
+@pytest.fixture(params=downsample_methods)
+def downsample_method(request):
+ """Fixture for parametrization of Grouper downsample methods."""
+ return request.param
+
+
+@pytest.fixture(params=resample_methods)
+def resample_method(request):
+ """Fixture for parametrization of Grouper resample methods."""
+ return request.param
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_base.py b/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_base.py
new file mode 100644
index 0000000000000000000000000000000000000000..359ad72bd67f32d472d502550dfb28285b33a4af
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_base.py
@@ -0,0 +1,554 @@
+from datetime import datetime
+
+import numpy as np
+import pytest
+
+from pandas.errors import Pandas4Warning
+
+from pandas.core.dtypes.common import is_extension_array_dtype
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ DatetimeIndex,
+ Index,
+ MultiIndex,
+ NaT,
+ PeriodIndex,
+ Series,
+ TimedeltaIndex,
+)
+import pandas._testing as tm
+from pandas.core.groupby.groupby import DataError
+from pandas.core.groupby.grouper import Grouper
+from pandas.core.indexes.datetimes import date_range
+from pandas.core.indexes.period import period_range
+from pandas.core.indexes.timedeltas import timedelta_range
+from pandas.core.resample import _asfreq_compat
+
+
+@pytest.fixture(
+ params=[
+ "linear",
+ "time",
+ "index",
+ "values",
+ "nearest",
+ "zero",
+ "slinear",
+ "quadratic",
+ "cubic",
+ "barycentric",
+ "krogh",
+ "from_derivatives",
+ "piecewise_polynomial",
+ "pchip",
+ "akima",
+ ],
+)
+def all_1d_no_arg_interpolation_methods(request):
+ return request.param
+
+
+@pytest.mark.parametrize("freq", ["2D", "1h"])
+@pytest.mark.parametrize(
+ "index",
+ [
+ timedelta_range("1 day", "10 day", freq="D"),
+ date_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="D"),
+ ],
+)
+def test_asfreq(frame_or_series, index, freq):
+ obj = frame_or_series(range(len(index)), index=index)
+ idx_range = date_range if isinstance(index, DatetimeIndex) else timedelta_range
+
+ result = obj.resample(freq).asfreq()
+ new_index = idx_range(obj.index[0], obj.index[-1], freq=freq)
+ expected = obj.reindex(new_index)
+ tm.assert_almost_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "index",
+ [
+ timedelta_range("1 day", "10 day", freq="D"),
+ date_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="D"),
+ ],
+)
+def test_asfreq_fill_value(index):
+ # test for fill value during resampling, issue 3715
+
+ ser = Series(range(len(index)), index=index, name="a")
+ idx_range = date_range if isinstance(index, DatetimeIndex) else timedelta_range
+
+ result = ser.resample("1h").asfreq()
+ new_index = idx_range(ser.index[0], ser.index[-1], freq="1h")
+ expected = ser.reindex(new_index)
+ tm.assert_series_equal(result, expected)
+
+ # Explicit cast to float to avoid implicit cast when setting None
+ frame = ser.astype("float").to_frame("value")
+ frame.iloc[1] = None
+ result = frame.resample("1h").asfreq(fill_value=4.0)
+ new_index = idx_range(frame.index[0], frame.index[-1], freq="1h")
+ expected = frame.reindex(new_index, fill_value=4.0)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "index",
+ [
+ timedelta_range("1 day", "3 day", freq="D"),
+ date_range(datetime(2005, 1, 1), datetime(2005, 1, 3), freq="D"),
+ period_range(datetime(2005, 1, 1), datetime(2005, 1, 3), freq="D"),
+ ],
+)
+def test_resample_interpolate(index):
+ # GH#12925
+ df = DataFrame(range(len(index)), index=index)
+ result = df.resample("1min").asfreq().interpolate()
+ expected = df.resample("1min").interpolate()
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_interpolate_inplace_deprecated():
+ # GH#58690
+ dti = date_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="D")
+
+ df = DataFrame(range(len(dti)), index=dti)
+ rs = df.resample("1min")
+ msg = "The 'inplace' keyword in DatetimeIndexResampler.interpolate"
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ rs.interpolate(inplace=False)
+
+ msg2 = "Cannot interpolate inplace on a resampled object"
+ with pytest.raises(ValueError, match=msg2):
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ rs.interpolate(inplace=True)
+
+
+def test_resample_interpolate_regular_sampling_off_grid(
+ all_1d_no_arg_interpolation_methods,
+):
+ pytest.importorskip("scipy")
+ # GH#21351
+ index = date_range("2000-01-01 00:01:00", periods=5, freq="2h")
+ ser = Series(np.arange(5.0), index)
+
+ method = all_1d_no_arg_interpolation_methods
+ result = ser.resample("1h").interpolate(method)
+
+ if method == "linear":
+ values = np.repeat(np.arange(0.0, 4.0), 2) + np.tile([1 / 3, 2 / 3], 4)
+ elif method == "nearest":
+ values = np.repeat(np.arange(0.0, 5.0), 2)[1:-1]
+ elif method == "zero":
+ values = np.repeat(np.arange(0.0, 4.0), 2)
+ else:
+ values = 0.491667 + np.arange(0.0, 4.0, 0.5)
+ values = np.insert(values, 0, np.nan)
+ index = date_range("2000-01-01 00:00:00", periods=9, freq="1h")
+ expected = Series(values, index=index)
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_interpolate_irregular_sampling(all_1d_no_arg_interpolation_methods):
+ pytest.importorskip("scipy")
+ # GH#21351
+ ser = Series(
+ np.linspace(0.0, 1.0, 5),
+ index=DatetimeIndex(
+ [
+ "2000-01-01 00:00:03",
+ "2000-01-01 00:00:22",
+ "2000-01-01 00:00:24",
+ "2000-01-01 00:00:31",
+ "2000-01-01 00:00:39",
+ ]
+ ),
+ )
+
+ # Resample to 5 second sampling and interpolate with the given method
+ ser_resampled = ser.resample("5s").interpolate(all_1d_no_arg_interpolation_methods)
+
+ # Check that none of the resampled values are NaN, except the first one
+ # which lies 3 seconds before the first actual data point
+ assert np.isnan(ser_resampled.iloc[0])
+ assert not ser_resampled.iloc[1:].isna().any()
+
+
+def test_raises_on_non_datetimelike_index():
+ # this is a non datetimelike index
+ xp = DataFrame()
+ msg = (
+ "Only valid with DatetimeIndex, TimedeltaIndex or PeriodIndex, "
+ "but got an instance of 'RangeIndex'"
+ )
+ with pytest.raises(TypeError, match=msg):
+ xp.resample("YE")
+
+
+@pytest.mark.parametrize(
+ "index",
+ [
+ PeriodIndex([], freq="D", name="a"),
+ DatetimeIndex([], name="a"),
+ TimedeltaIndex([], name="a"),
+ ],
+)
+@pytest.mark.parametrize("freq", ["ME", "D", "h"])
+def test_resample_empty_series(freq, index, resample_method):
+ # GH12771 & GH12868
+
+ ser = Series(index=index, dtype=float)
+ if freq == "ME" and isinstance(ser.index, TimedeltaIndex):
+ msg = (
+ "Resampling on a TimedeltaIndex requires fixed-duration `freq`, "
+ "e.g. '24h' or '3D', not "
+ )
+ with pytest.raises(ValueError, match=msg):
+ ser.resample(freq)
+ return
+ elif freq == "ME" and isinstance(ser.index, PeriodIndex):
+ # index is PeriodIndex, so convert to corresponding Period freq
+ freq = "M"
+ rs = ser.resample(freq)
+ result = getattr(rs, resample_method)()
+
+ if resample_method == "ohlc":
+ expected = DataFrame(
+ [], index=ser.index[:0], columns=["open", "high", "low", "close"]
+ )
+ expected.index = _asfreq_compat(ser.index, freq)
+ tm.assert_frame_equal(result, expected, check_dtype=False)
+ else:
+ expected = ser.copy()
+ expected.index = _asfreq_compat(ser.index, freq)
+ tm.assert_series_equal(result, expected, check_dtype=False)
+
+ tm.assert_index_equal(result.index, expected.index)
+ assert result.index.freq == expected.index.freq
+
+
+@pytest.mark.parametrize("min_count", [0, 1])
+def test_resample_empty_sum_string(string_dtype_no_object, min_count):
+ # https://github.com/pandas-dev/pandas/issues/60229
+ dtype = string_dtype_no_object
+ ser = Series(
+ pd.NA,
+ index=DatetimeIndex(
+ [
+ "2000-01-01 00:00:00",
+ "2000-01-01 00:00:10",
+ "2000-01-01 00:00:20",
+ "2000-01-01 00:00:30",
+ ]
+ ),
+ dtype=dtype,
+ )
+ rs = ser.resample("20s")
+ result = rs.sum(min_count=min_count)
+
+ value = "" if min_count == 0 else pd.NA
+ index = date_range(start="2000-01-01", freq="20s", periods=2, unit="us")
+ expected = Series(value, index=index, dtype=dtype)
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "freq",
+ [
+ pytest.param("ME", marks=pytest.mark.xfail(reason="Don't know why this fails")),
+ "D",
+ "h",
+ ],
+)
+def test_resample_nat_index_series(freq, resample_method):
+ # GH39227
+
+ ser = Series(range(5), index=PeriodIndex([NaT] * 5, freq=freq))
+
+ rs = ser.resample(freq)
+ result = getattr(rs, resample_method)()
+
+ if resample_method == "ohlc":
+ expected = DataFrame(
+ [], index=ser.index[:0], columns=["open", "high", "low", "close"]
+ )
+ tm.assert_frame_equal(result, expected, check_dtype=False)
+ else:
+ expected = ser[:0].copy()
+ tm.assert_series_equal(result, expected, check_dtype=False)
+ tm.assert_index_equal(result.index, expected.index)
+ assert result.index.freq == expected.index.freq
+
+
+@pytest.mark.parametrize(
+ "index",
+ [
+ PeriodIndex([], freq="D", name="a"),
+ DatetimeIndex([], name="a"),
+ TimedeltaIndex([], name="a"),
+ ],
+)
+@pytest.mark.parametrize("freq", ["ME", "D", "h"])
+@pytest.mark.parametrize("resample_method", ["count", "size"])
+def test_resample_count_empty_series(freq, index, resample_method):
+ # GH28427
+ ser = Series(index=index)
+ if freq == "ME" and isinstance(ser.index, TimedeltaIndex):
+ msg = (
+ "Resampling on a TimedeltaIndex requires fixed-duration `freq`, "
+ "e.g. '24h' or '3D', not "
+ )
+ with pytest.raises(ValueError, match=msg):
+ ser.resample(freq)
+ return
+ elif freq == "ME" and isinstance(ser.index, PeriodIndex):
+ # index is PeriodIndex, so convert to corresponding Period freq
+ freq = "M"
+ rs = ser.resample(freq)
+
+ result = getattr(rs, resample_method)()
+
+ index = _asfreq_compat(ser.index, freq)
+
+ expected = Series([], dtype="int64", index=index, name=ser.name)
+
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "index", [DatetimeIndex([]), TimedeltaIndex([]), PeriodIndex([], freq="D")]
+)
+@pytest.mark.parametrize("freq", ["ME", "D", "h"])
+def test_resample_empty_dataframe(index, freq, resample_method):
+ # GH13212
+ df = DataFrame(index=index)
+ # count retains dimensions too
+ if freq == "ME" and isinstance(df.index, TimedeltaIndex):
+ msg = (
+ "Resampling on a TimedeltaIndex requires fixed-duration `freq`, "
+ "e.g. '24h' or '3D', not "
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.resample(freq, group_keys=False)
+ return
+ elif freq == "ME" and isinstance(df.index, PeriodIndex):
+ # index is PeriodIndex, so convert to corresponding Period freq
+ freq = "M"
+ rs = df.resample(freq, group_keys=False)
+ result = getattr(rs, resample_method)()
+ if resample_method == "ohlc":
+ # TODO: no tests with len(df.columns) > 0
+ mi = MultiIndex.from_product([df.columns, ["open", "high", "low", "close"]])
+ expected = DataFrame([], index=df.index[:0], columns=mi, dtype=np.float64)
+ expected.index = _asfreq_compat(df.index, freq)
+
+ elif resample_method != "size":
+ expected = df.copy()
+ else:
+ # GH14962
+ expected = Series([], dtype=np.int64)
+
+ expected.index = _asfreq_compat(df.index, freq)
+
+ tm.assert_index_equal(result.index, expected.index)
+ assert result.index.freq == expected.index.freq
+ tm.assert_almost_equal(result, expected)
+
+ # test size for GH13212 (currently stays as df)
+
+
+@pytest.mark.parametrize(
+ "index", [DatetimeIndex([]), TimedeltaIndex([]), PeriodIndex([], freq="D")]
+)
+@pytest.mark.parametrize("freq", ["ME", "D", "h"])
+def test_resample_count_empty_dataframe(freq, index):
+ # GH28427
+ empty_frame_dti = DataFrame(index=index, columns=Index(["a"], dtype=object))
+
+ if freq == "ME" and isinstance(empty_frame_dti.index, TimedeltaIndex):
+ msg = (
+ "Resampling on a TimedeltaIndex requires fixed-duration `freq`, "
+ "e.g. '24h' or '3D', not "
+ )
+ with pytest.raises(ValueError, match=msg):
+ empty_frame_dti.resample(freq)
+ return
+ elif freq == "ME" and isinstance(empty_frame_dti.index, PeriodIndex):
+ # index is PeriodIndex, so convert to corresponding Period freq
+ freq = "M"
+ result = empty_frame_dti.resample(freq).count()
+
+ index = _asfreq_compat(empty_frame_dti.index, freq)
+
+ expected = DataFrame(dtype="int64", index=index, columns=Index(["a"], dtype=object))
+
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "index", [DatetimeIndex([]), TimedeltaIndex([]), PeriodIndex([], freq="D")]
+)
+@pytest.mark.parametrize("freq", ["ME", "D", "h"])
+def test_resample_size_empty_dataframe(freq, index):
+ # GH28427
+
+ empty_frame_dti = DataFrame(index=index, columns=Index(["a"], dtype=object))
+
+ if freq == "ME" and isinstance(empty_frame_dti.index, TimedeltaIndex):
+ msg = (
+ "Resampling on a TimedeltaIndex requires fixed-duration `freq`, "
+ "e.g. '24h' or '3D', not "
+ )
+ with pytest.raises(ValueError, match=msg):
+ empty_frame_dti.resample(freq)
+ return
+ elif freq == "ME" and isinstance(empty_frame_dti.index, PeriodIndex):
+ # index is PeriodIndex, so convert to corresponding Period freq
+ freq = "M"
+ result = empty_frame_dti.resample(freq).size()
+
+ index = _asfreq_compat(empty_frame_dti.index, freq)
+
+ expected = Series([], dtype="int64", index=index)
+
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("index", [DatetimeIndex([]), TimedeltaIndex([])])
+@pytest.mark.parametrize("freq", ["D", "h"])
+@pytest.mark.parametrize(
+ "method", ["ffill", "bfill", "nearest", "asfreq", "interpolate", "mean"]
+)
+def test_resample_apply_empty_dataframe(index, freq, method):
+ # GH#55572
+ empty_frame_dti = DataFrame(index=index)
+
+ rs = empty_frame_dti.resample(freq)
+ result = rs.apply(getattr(rs, method))
+
+ expected_index = _asfreq_compat(empty_frame_dti.index, freq)
+ expected = DataFrame([], index=expected_index)
+
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "index",
+ [
+ PeriodIndex([], freq="M", name="a"),
+ DatetimeIndex([], name="a"),
+ TimedeltaIndex([], name="a"),
+ ],
+)
+@pytest.mark.parametrize("dtype", [float, int, object, "datetime64[ns]"])
+def test_resample_empty_dtypes(index, dtype, resample_method):
+ # Empty series were sometimes causing a segfault (for the functions
+ # with Cython bounds-checking disabled) or an IndexError. We just run
+ # them to ensure they no longer do. (GH #10228)
+ empty_series_dti = Series([], index, dtype)
+ rs = empty_series_dti.resample("D", group_keys=False)
+ try:
+ getattr(rs, resample_method)()
+ except DataError:
+ # Ignore these since some combinations are invalid
+ # (ex: doing mean with dtype of np.object_)
+ pass
+
+
+@pytest.mark.parametrize(
+ "index",
+ [
+ PeriodIndex([], freq="D", name="a"),
+ DatetimeIndex([], name="a"),
+ TimedeltaIndex([], name="a"),
+ ],
+)
+@pytest.mark.parametrize("freq", ["ME", "D", "h"])
+def test_apply_to_empty_series(index, freq):
+ # GH 14313
+ ser = Series(index=index)
+
+ if freq == "ME" and isinstance(ser.index, TimedeltaIndex):
+ msg = (
+ "Resampling on a TimedeltaIndex requires fixed-duration `freq`, "
+ "e.g. '24h' or '3D', not "
+ )
+ with pytest.raises(ValueError, match=msg):
+ ser.resample(freq)
+ return
+ elif freq == "ME" and isinstance(ser.index, PeriodIndex):
+ # index is PeriodIndex, so convert to corresponding Period freq
+ freq = "M"
+ result = ser.resample(freq, group_keys=False).apply(lambda x: 1)
+ expected = ser.resample(freq).apply("sum")
+
+ tm.assert_series_equal(result, expected, check_dtype=False)
+
+
+@pytest.mark.parametrize(
+ "index",
+ [
+ timedelta_range("1 day", "10 day", freq="D"),
+ date_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="D"),
+ period_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="D"),
+ ],
+)
+def test_resampler_is_iterable(index):
+ # GH 15314
+ series = Series(range(len(index)), index=index)
+ freq = "h"
+ tg = Grouper(freq=freq, convention="start")
+ grouped = series.groupby(tg)
+ resampled = series.resample(freq)
+ for (rk, rv), (gk, gv) in zip(resampled, grouped):
+ assert rk == gk
+ tm.assert_series_equal(rv, gv)
+
+
+@pytest.mark.parametrize(
+ "index",
+ [
+ timedelta_range("1 day", "10 day", freq="D"),
+ date_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="D"),
+ period_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="D"),
+ ],
+)
+def test_resample_quantile(index):
+ # GH 15023
+ ser = Series(range(len(index)), index=index)
+ q = 0.75
+ freq = "h"
+
+ result = ser.resample(freq).quantile(q)
+ expected = ser.resample(freq).agg(lambda x: x.quantile(q)).rename(ser.name)
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("how", ["first", "last"])
+def test_first_last_skipna(any_real_nullable_dtype, skipna, how):
+ # GH#57019
+ if is_extension_array_dtype(any_real_nullable_dtype):
+ na_value = Series(dtype=any_real_nullable_dtype).dtype.na_value
+ else:
+ na_value = np.nan
+ df = DataFrame(
+ {
+ "a": [2, 1, 1, 2],
+ "b": [na_value, 3.0, na_value, 4.0],
+ "c": [na_value, 3.0, na_value, 4.0],
+ },
+ index=date_range("2020-01-01", periods=4, freq="D", unit="ns"),
+ dtype=any_real_nullable_dtype,
+ )
+ rs = df.resample("ME")
+ method = getattr(rs, how)
+ result = method(skipna=skipna)
+
+ ts = pd.to_datetime("2020-01-31").as_unit("ns")
+ gb = df.groupby(df.shape[0] * [ts])
+ expected = getattr(gb, how)(skipna=skipna)
+ expected.index.freq = "ME"
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_datetime_index.py b/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_datetime_index.py
new file mode 100644
index 0000000000000000000000000000000000000000..6867b6cf9927142888a066717a15278fbfee9013
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_datetime_index.py
@@ -0,0 +1,2190 @@
+from datetime import datetime
+from functools import partial
+import zoneinfo
+
+import numpy as np
+import pytest
+
+from pandas._libs import lib
+from pandas._libs.tslibs import Day
+from pandas._typing import DatetimeNaTType
+from pandas.compat import is_platform_windows
+from pandas.compat.pyarrow import pa_version_under22p0
+from pandas.errors import Pandas4Warning
+import pandas.util._test_decorators as td
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ Series,
+ Timedelta,
+ Timestamp,
+ isna,
+ notna,
+)
+import pandas._testing as tm
+from pandas.core.groupby.grouper import Grouper
+from pandas.core.indexes.datetimes import date_range
+from pandas.core.indexes.period import (
+ Period,
+ period_range,
+)
+from pandas.core.resample import (
+ DatetimeIndex,
+ _get_timestamp_range_edges,
+)
+
+from pandas.tseries import offsets
+from pandas.tseries.frequencies import to_offset
+from pandas.tseries.offsets import Minute
+
+
+@pytest.fixture
+def simple_date_range_series():
+ """
+ Series with date range index and random data for test purposes.
+ """
+
+ def _simple_date_range_series(start, end, freq="D"):
+ rng = date_range(start, end, freq=freq)
+ return Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+
+ return _simple_date_range_series
+
+
+def test_custom_grouper(unit):
+ index = date_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="Min")
+ dti = index.as_unit(unit)
+ s = Series(np.array([1] * len(dti)), index=dti, dtype="int64")
+
+ b = Grouper(freq=Minute(5))
+ g = s.groupby(b)
+
+ # check all cython functions work
+ g.ohlc() # doesn't use _cython_agg_general
+ funcs = ["sum", "mean", "prod", "min", "max", "var"]
+ for f in funcs:
+ g._cython_agg_general(f, alt=None, numeric_only=True)
+
+ b = Grouper(freq=Minute(5), closed="right", label="right")
+ g = s.groupby(b)
+ # check all cython functions work
+ g.ohlc() # doesn't use _cython_agg_general
+ funcs = ["sum", "mean", "prod", "min", "max", "var"]
+ for f in funcs:
+ g._cython_agg_general(f, alt=None, numeric_only=True)
+
+ assert g.ngroups == 2593
+ assert notna(g.mean()).all()
+
+ # construct expected val
+ arr = [1] + [5] * 2592
+ idx = dti[0:-1:5]
+ idx = idx.append(dti[-1:])
+ idx = DatetimeIndex(idx, freq="5min").as_unit(unit)
+ expect = Series(arr, index=idx)
+
+ # GH2763 - return input dtype if we can
+ result = g.agg("sum")
+ tm.assert_series_equal(result, expect)
+
+
+def test_custom_grouper_df(unit):
+ index = date_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="D")
+ b = Grouper(freq=Minute(5), closed="right", label="right")
+ dti = index.as_unit(unit)
+ df = DataFrame(
+ np.random.default_rng(2).random((len(dti), 10)), index=dti, dtype="float64"
+ )
+ r = df.groupby(b).agg("sum")
+
+ assert len(r.columns) == 10
+ assert len(r.index) == 2593
+
+
+@pytest.mark.parametrize(
+ "closed, expected",
+ [
+ (
+ "right",
+ lambda s: Series(
+ [s.iloc[0], s[1:6].mean(), s[6:11].mean(), s[11:].mean()],
+ index=date_range("1/1/2000", periods=4, freq="5min", name="index"),
+ ),
+ ),
+ (
+ "left",
+ lambda s: Series(
+ [s[:5].mean(), s[5:10].mean(), s[10:].mean()],
+ index=date_range(
+ "1/1/2000 00:05", periods=3, freq="5min", name="index"
+ ),
+ ),
+ ),
+ ],
+)
+def test_resample_basic(closed, expected, unit):
+ index = date_range("1/1/2000 00:00:00", "1/1/2000 00:13:00", freq="Min")
+ s = Series(range(len(index)), index=index)
+ s.index.name = "index"
+ s.index = s.index.as_unit(unit)
+ expected = expected(s)
+ expected.index = expected.index.as_unit(unit)
+ result = s.resample("5min", closed=closed, label="right").mean()
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_integerarray(unit):
+ # GH 25580, resample on IntegerArray
+ ts = Series(
+ range(9),
+ index=date_range("1/1/2000", periods=9, freq="min").as_unit(unit),
+ dtype="Int64",
+ )
+ result = ts.resample("3min").sum()
+ expected = Series(
+ [3, 12, 21],
+ index=date_range("1/1/2000", periods=3, freq="3min").as_unit(unit),
+ dtype="Int64",
+ )
+ tm.assert_series_equal(result, expected)
+
+ result = ts.resample("3min").mean()
+ expected = Series(
+ [1, 4, 7],
+ index=date_range("1/1/2000", periods=3, freq="3min").as_unit(unit),
+ dtype="Float64",
+ )
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_basic_grouper(unit):
+ index = date_range("1/1/2000 00:00:00", "1/1/2000 00:13:00", freq="Min")
+ s = Series(range(len(index)), index=index)
+ s.index.name = "index"
+ s.index = s.index.as_unit(unit)
+ result = s.resample("5Min").last()
+ grouper = Grouper(freq=Minute(5), closed="left", label="left")
+ expected = s.groupby(grouper).agg(lambda x: x.iloc[-1])
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "keyword,value",
+ [("label", "righttt"), ("closed", "righttt"), ("convention", "starttt")],
+)
+def test_resample_string_kwargs(keyword, value, unit):
+ # see gh-19303
+ # Check that wrong keyword argument strings raise an error
+ index = date_range("1/1/2000 00:00:00", "1/1/2000 00:13:00", freq="Min")
+ series = Series(range(len(index)), index=index)
+ series.index.name = "index"
+ series.index = series.index.as_unit(unit)
+ msg = f"Unsupported value {value} for `{keyword}`"
+ with pytest.raises(ValueError, match=msg):
+ series.resample("5min", **({keyword: value}))
+
+
+def test_resample_how(downsample_method, unit):
+ if downsample_method == "ohlc":
+ pytest.skip("covered by test_resample_how_ohlc")
+ index = date_range("1/1/2000 00:00:00", "1/1/2000 00:13:00", freq="Min")
+ s = Series(range(len(index)), index=index)
+ s.index.name = "index"
+ s.index = s.index.as_unit(unit)
+ grouplist = np.ones_like(s)
+ grouplist[0] = 0
+ grouplist[1:6] = 1
+ grouplist[6:11] = 2
+ grouplist[11:] = 3
+ expected = s.groupby(grouplist).agg(downsample_method)
+ expected.index = date_range(
+ "1/1/2000", periods=4, freq="5min", name="index"
+ ).as_unit(unit)
+
+ result = getattr(
+ s.resample("5min", closed="right", label="right"), downsample_method
+ )()
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_how_ohlc(unit):
+ index = date_range("1/1/2000 00:00:00", "1/1/2000 00:13:00", freq="Min")
+ s = Series(range(len(index)), index=index)
+ s.index.name = "index"
+ s.index = s.index.as_unit(unit)
+ grouplist = np.ones_like(s)
+ grouplist[0] = 0
+ grouplist[1:6] = 1
+ grouplist[6:11] = 2
+ grouplist[11:] = 3
+
+ def _ohlc(group):
+ if isna(group).all():
+ return np.repeat(np.nan, 4)
+ return [group.iloc[0], group.max(), group.min(), group.iloc[-1]]
+
+ expected = DataFrame(
+ s.groupby(grouplist).agg(_ohlc).values.tolist(),
+ index=date_range("1/1/2000", periods=4, freq="5min", name="index").as_unit(
+ unit
+ ),
+ columns=["open", "high", "low", "close"],
+ )
+
+ result = s.resample("5min", closed="right", label="right").ohlc()
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_how_callables(unit):
+ # GH#7929
+ data = np.arange(5, dtype=np.int64)
+ msg = "'d' is deprecated and will be removed in a future version."
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ ind = date_range(start="2014-01-01", periods=len(data), freq="d").as_unit(unit)
+ df = DataFrame({"A": data, "B": data}, index=ind)
+
+ def fn(x, a=1):
+ return str(type(x))
+
+ class FnClass:
+ def __call__(self, x):
+ return str(type(x))
+
+ df_standard = df.resample("ME").apply(fn)
+ df_lambda = df.resample("ME").apply(lambda x: str(type(x)))
+ df_partial = df.resample("ME").apply(partial(fn))
+ df_partial2 = df.resample("ME").apply(partial(fn, a=2))
+ df_class = df.resample("ME").apply(FnClass())
+
+ tm.assert_frame_equal(df_standard, df_lambda)
+ tm.assert_frame_equal(df_standard, df_partial)
+ tm.assert_frame_equal(df_standard, df_partial2)
+ tm.assert_frame_equal(df_standard, df_class)
+
+
+def test_resample_rounding(unit):
+ # GH 8371
+ # odd results when rounding is needed
+
+ ts = [
+ "2014-11-08 00:00:01",
+ "2014-11-08 00:00:02",
+ "2014-11-08 00:00:02",
+ "2014-11-08 00:00:03",
+ "2014-11-08 00:00:07",
+ "2014-11-08 00:00:07",
+ "2014-11-08 00:00:08",
+ "2014-11-08 00:00:08",
+ "2014-11-08 00:00:08",
+ "2014-11-08 00:00:09",
+ "2014-11-08 00:00:10",
+ "2014-11-08 00:00:11",
+ "2014-11-08 00:00:11",
+ "2014-11-08 00:00:13",
+ "2014-11-08 00:00:14",
+ "2014-11-08 00:00:15",
+ "2014-11-08 00:00:17",
+ "2014-11-08 00:00:20",
+ "2014-11-08 00:00:21",
+ ]
+ df = DataFrame({"value": [1] * 19}, index=pd.to_datetime(ts))
+ df.index = df.index.as_unit(unit)
+
+ result = df.resample("6s").sum()
+ expected = DataFrame(
+ {"value": [4, 9, 4, 2]},
+ index=date_range("2014-11-08", freq="6s", periods=4).as_unit(unit),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = df.resample("7s").sum()
+ expected = DataFrame(
+ {"value": [4, 10, 4, 1]},
+ index=date_range("2014-11-08", freq="7s", periods=4).as_unit(unit),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = df.resample("11s").sum()
+ expected = DataFrame(
+ {"value": [11, 8]},
+ index=date_range("2014-11-08", freq="11s", periods=2).as_unit(unit),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = df.resample("13s").sum()
+ expected = DataFrame(
+ {"value": [13, 6]},
+ index=date_range("2014-11-08", freq="13s", periods=2).as_unit(unit),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = df.resample("17s").sum()
+ expected = DataFrame(
+ {"value": [16, 3]},
+ index=date_range("2014-11-08", freq="17s", periods=2).as_unit(unit),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_basic_from_daily(unit):
+ # from daily
+ dti = date_range(
+ start=datetime(2005, 1, 1), end=datetime(2005, 1, 10), freq="D", name="index"
+ ).as_unit(unit)
+
+ s = Series(np.random.default_rng(2).random(len(dti)), dti)
+
+ # to weekly
+ msg = "'w-sun' is deprecated and will be removed in a future version."
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ result = s.resample("w-sun").last()
+
+ assert len(result) == 3
+ assert (result.index.dayofweek == [6, 6, 6]).all()
+ assert result.iloc[0] == s["1/2/2005"]
+ assert result.iloc[1] == s["1/9/2005"]
+ assert result.iloc[2] == s.iloc[-1]
+
+ result = s.resample("W-MON").last()
+ assert len(result) == 2
+ assert (result.index.dayofweek == [0, 0]).all()
+ assert result.iloc[0] == s["1/3/2005"]
+ assert result.iloc[1] == s["1/10/2005"]
+
+ result = s.resample("W-TUE").last()
+ assert len(result) == 2
+ assert (result.index.dayofweek == [1, 1]).all()
+ assert result.iloc[0] == s["1/4/2005"]
+ assert result.iloc[1] == s["1/10/2005"]
+
+ result = s.resample("W-WED").last()
+ assert len(result) == 2
+ assert (result.index.dayofweek == [2, 2]).all()
+ assert result.iloc[0] == s["1/5/2005"]
+ assert result.iloc[1] == s["1/10/2005"]
+
+ result = s.resample("W-THU").last()
+ assert len(result) == 2
+ assert (result.index.dayofweek == [3, 3]).all()
+ assert result.iloc[0] == s["1/6/2005"]
+ assert result.iloc[1] == s["1/10/2005"]
+
+ result = s.resample("W-FRI").last()
+ assert len(result) == 2
+ assert (result.index.dayofweek == [4, 4]).all()
+ assert result.iloc[0] == s["1/7/2005"]
+ assert result.iloc[1] == s["1/10/2005"]
+
+ # to biz day
+ result = s.resample("B").last()
+ assert len(result) == 7
+ assert (result.index.dayofweek == [4, 0, 1, 2, 3, 4, 0]).all()
+
+ assert result.iloc[0] == s["1/2/2005"]
+ assert result.iloc[1] == s["1/3/2005"]
+ assert result.iloc[5] == s["1/9/2005"]
+ assert result.index.name == "index"
+
+
+def test_resample_upsampling_picked_but_not_correct(unit):
+ # Test for issue #3020
+ dates = date_range("01-Jan-2014", "05-Jan-2014", freq="D").as_unit(unit)
+ series = Series(1, index=dates)
+
+ result = series.resample("D").mean()
+ assert result.index[0] == dates[0]
+
+ # GH 5955
+ # incorrect deciding to upsample when the axis frequency matches the
+ # resample frequency
+
+ s = Series(
+ np.arange(1.0, 6), index=[datetime(1975, 1, i, 12, 0) for i in range(1, 6)]
+ )
+ s.index = s.index.as_unit(unit)
+ expected = Series(
+ np.arange(1.0, 6),
+ index=date_range("19750101", periods=5, freq="D").as_unit(unit),
+ )
+
+ result = s.resample("D").count()
+ tm.assert_series_equal(result, Series(1, index=expected.index))
+
+ result1 = s.resample("D").sum()
+ result2 = s.resample("D").mean()
+ tm.assert_series_equal(result1, expected)
+ tm.assert_series_equal(result2, expected)
+
+
+@pytest.mark.parametrize("f", ["sum", "mean", "prod", "min", "max", "var"])
+def test_resample_frame_basic_cy_funcs(f, unit):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((50, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=50, freq="B"),
+ )
+ df.index = df.index.as_unit(unit)
+
+ b = Grouper(freq="ME")
+ g = df.groupby(b)
+
+ # check all cython functions work
+ g._cython_agg_general(f, alt=None, numeric_only=True)
+
+
+@pytest.mark.parametrize("freq", ["YE", "ME"])
+def test_resample_frame_basic_M_A(freq, unit):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((50, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=50, freq="B"),
+ )
+ df.index = df.index.as_unit(unit)
+ result = df.resample(freq).mean()
+ tm.assert_series_equal(result["A"], df["A"].resample(freq).mean())
+
+
+def test_resample_upsample(unit):
+ # from daily
+ dti = date_range(
+ start=datetime(2005, 1, 1), end=datetime(2005, 1, 10), freq="D", name="index"
+ ).as_unit(unit)
+
+ s = Series(np.random.default_rng(2).random(len(dti)), dti)
+
+ # to minutely, by padding
+ result = s.resample("Min").ffill()
+ assert len(result) == 12961
+ assert result.iloc[0] == s.iloc[0]
+ assert result.iloc[-1] == s.iloc[-1]
+
+ assert result.index.name == "index"
+
+
+def test_resample_how_method(unit):
+ # GH9915
+ s = Series(
+ [11, 22],
+ index=[
+ Timestamp("2015-03-31 21:48:52.672000"),
+ Timestamp("2015-03-31 21:49:52.739000"),
+ ],
+ )
+ s.index = s.index.as_unit(unit)
+ expected = Series(
+ [11, np.nan, np.nan, np.nan, np.nan, np.nan, 22],
+ index=DatetimeIndex(
+ [
+ Timestamp("2015-03-31 21:48:50"),
+ Timestamp("2015-03-31 21:49:00"),
+ Timestamp("2015-03-31 21:49:10"),
+ Timestamp("2015-03-31 21:49:20"),
+ Timestamp("2015-03-31 21:49:30"),
+ Timestamp("2015-03-31 21:49:40"),
+ Timestamp("2015-03-31 21:49:50"),
+ ],
+ freq="10s",
+ ),
+ )
+ expected.index = expected.index.as_unit(unit)
+ tm.assert_series_equal(s.resample("10s").mean(), expected)
+
+
+def test_resample_extra_index_point(unit):
+ # GH#9756
+ index = date_range(start="20150101", end="20150331", freq="BME").as_unit(unit)
+ expected = DataFrame({"A": Series([21, 41, 63], index=index)})
+
+ index = date_range(start="20150101", end="20150331", freq="B").as_unit(unit)
+ df = DataFrame({"A": Series(range(len(index)), index=index)}, dtype="int64")
+ result = df.resample("BME").last()
+ tm.assert_frame_equal(result, expected)
+
+
+def test_upsample_with_limit(unit):
+ rng = date_range("1/1/2000", periods=3, freq="5min").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), rng)
+
+ result = ts.resample("min").ffill(limit=2)
+ expected = ts.reindex(result.index, method="ffill", limit=2)
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("freq", ["1D", "10h", "5Min", "10s"])
+@pytest.mark.parametrize("rule", ["YE", "3ME", "15D", "30h", "15Min", "30s"])
+def test_nearest_upsample_with_limit(tz_aware_fixture, freq, rule, unit):
+ # GH 33939
+ rng = date_range("1/1/2000", periods=3, freq=freq, tz=tz_aware_fixture).as_unit(
+ unit
+ )
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), rng)
+
+ result = ts.resample(rule).nearest(limit=2)
+ expected = ts.reindex(result.index, method="nearest", limit=2)
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_ohlc(unit):
+ index = date_range(datetime(2005, 1, 1), datetime(2005, 1, 2), freq="Min")
+ s = Series(range(len(index)), index=index)
+ s.index.name = "index"
+ s.index = s.index.as_unit(unit)
+
+ grouper = Grouper(freq=Minute(5))
+ expect = s.groupby(grouper).agg(lambda x: x.iloc[-1])
+ result = s.resample("5Min").ohlc()
+
+ assert len(result) == len(expect)
+ assert len(result.columns) == 4
+
+ xs = result.iloc[-2]
+ assert xs["open"] == s.iloc[-6]
+ assert xs["high"] == s[-6:-1].max()
+ assert xs["low"] == s[-6:-1].min()
+ assert xs["close"] == s.iloc[-2]
+
+ xs = result.iloc[0]
+ assert xs["open"] == s.iloc[0]
+ assert xs["high"] == s[:5].max()
+ assert xs["low"] == s[:5].min()
+ assert xs["close"] == s.iloc[4]
+
+
+def test_resample_ohlc_result(unit):
+ # GH 12332
+ index = date_range("1-1-2000", "2-15-2000", freq="h").as_unit(unit)
+ index = index.union(date_range("4-15-2000", "5-15-2000", freq="h").as_unit(unit))
+ s = Series(range(len(index)), index=index)
+
+ a = s.loc[:"4-15-2000"].resample("30min").ohlc()
+ assert isinstance(a, DataFrame)
+
+ b = s.loc[:"4-14-2000"].resample("30min").ohlc()
+ assert isinstance(b, DataFrame)
+
+
+def test_resample_ohlc_result_odd_period(unit):
+ # GH12348
+ # raising on odd period
+ rng = date_range("2013-12-30", "2014-01-07").as_unit(unit)
+ index = rng.drop(
+ [
+ Timestamp("2014-01-01"),
+ Timestamp("2013-12-31"),
+ Timestamp("2014-01-04"),
+ Timestamp("2014-01-05"),
+ ]
+ )
+ df = DataFrame(data=np.arange(len(index)), index=index)
+ result = df.resample("B").mean()
+ expected = df.reindex(index=date_range(rng[0], rng[-1], freq="B").as_unit(unit))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_ohlc_dataframe(unit):
+ df = (
+ DataFrame(
+ {
+ "PRICE": {
+ Timestamp("2011-01-06 10:59:05", tz=None): 24990,
+ Timestamp("2011-01-06 12:43:33", tz=None): 25499,
+ Timestamp("2011-01-06 12:54:09", tz=None): 25499,
+ },
+ "VOLUME": {
+ Timestamp("2011-01-06 10:59:05", tz=None): 1500000000,
+ Timestamp("2011-01-06 12:43:33", tz=None): 5000000000,
+ Timestamp("2011-01-06 12:54:09", tz=None): 100000000,
+ },
+ }
+ )
+ ).reindex(["VOLUME", "PRICE"], axis=1)
+ df.index = df.index.as_unit(unit)
+ df.columns.name = "Cols"
+ res = df.resample("h").ohlc()
+ exp = pd.concat(
+ [df["VOLUME"].resample("h").ohlc(), df["PRICE"].resample("h").ohlc()],
+ axis=1,
+ keys=df.columns,
+ )
+ assert exp.columns.names[0] == "Cols"
+ tm.assert_frame_equal(exp, res)
+
+ df.columns = [["a", "b"], ["c", "d"]]
+ res = df.resample("h").ohlc()
+ exp.columns = pd.MultiIndex.from_tuples(
+ [
+ ("a", "c", "open"),
+ ("a", "c", "high"),
+ ("a", "c", "low"),
+ ("a", "c", "close"),
+ ("b", "d", "open"),
+ ("b", "d", "high"),
+ ("b", "d", "low"),
+ ("b", "d", "close"),
+ ]
+ )
+ tm.assert_frame_equal(exp, res)
+
+ # dupe columns fail atm
+ # df.columns = ['PRICE', 'PRICE']
+
+
+def test_resample_reresample(unit):
+ dti = date_range(
+ start=datetime(2005, 1, 1), end=datetime(2005, 1, 10), freq="D"
+ ).as_unit(unit)
+ s = Series(np.random.default_rng(2).random(len(dti)), dti)
+ bs = s.resample("B", closed="right", label="right").mean()
+ result = bs.resample("8h").mean()
+ assert len(result) == 25
+ assert isinstance(result.index.freq, offsets.DateOffset)
+ assert result.index.freq == offsets.Hour(8)
+
+
+@pytest.mark.parametrize(
+ "freq, expected_kwargs",
+ [
+ ["YE-DEC", {"start": "1990", "end": "2000", "freq": "Y-DEC"}],
+ ["YE-JUN", {"start": "1990", "end": "2000", "freq": "Y-JUN"}],
+ ["ME", {"start": "1990-01", "end": "2000-01", "freq": "M"}],
+ ],
+)
+def test_resample_timestamp_to_period(
+ simple_date_range_series, freq, expected_kwargs, unit
+):
+ ts = simple_date_range_series("1/1/1990", "1/1/2000")
+ ts.index = ts.index.as_unit(unit)
+
+ result = ts.resample(freq).mean().to_period()
+ expected = ts.resample(freq).mean()
+ expected.index = period_range(**expected_kwargs)
+ tm.assert_series_equal(result, expected)
+
+
+def test_ohlc_5min(unit):
+ def _ohlc(group):
+ if isna(group).all():
+ return np.repeat(np.nan, 4)
+ return [group.iloc[0], group.max(), group.min(), group.iloc[-1]]
+
+ rng = date_range("1/1/2000 00:00:00", "1/1/2000 5:59:50", freq="10s").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+
+ resampled = ts.resample("5min", closed="right", label="right").ohlc()
+
+ assert (resampled.loc["1/1/2000 00:00"] == ts.iloc[0]).all()
+
+ exp = _ohlc(ts[1:31])
+ assert (resampled.loc["1/1/2000 00:05"] == exp).all()
+
+ exp = _ohlc(ts["1/1/2000 5:55:01":])
+ assert (resampled.loc["1/1/2000 6:00:00"] == exp).all()
+
+
+def test_downsample_non_unique(unit):
+ rng = date_range("1/1/2000", "2/29/2000").as_unit(unit)
+ rng2 = rng.repeat(5).values
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng2)), index=rng2)
+
+ result = ts.resample("ME").mean()
+
+ expected = ts.groupby(lambda x: x.month).mean()
+ assert len(result) == 2
+ tm.assert_almost_equal(result.iloc[0], expected[1])
+ tm.assert_almost_equal(result.iloc[1], expected[2])
+
+
+def test_asfreq_non_unique(unit):
+ # GH #1077
+ rng = date_range("1/1/2000", "2/29/2000").as_unit(unit)
+ rng2 = rng.repeat(2).values
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng2)), index=rng2)
+
+ msg = "cannot reindex on an axis with duplicate labels"
+ with pytest.raises(ValueError, match=msg):
+ ts.asfreq("B")
+
+
+@pytest.mark.parametrize("freq", ["min", "5min", "15min", "30min", "4h", "12h"])
+def test_resample_anchored_ticks(freq, unit):
+ # If a fixed delta (5 minute, 4 hour) evenly divides a day, we should
+ # "anchor" the origin at midnight so we get regular intervals rather
+ # than starting from the first timestamp which might start in the
+ # middle of a desired interval
+
+ rng = date_range("1/1/2000 04:00:00", periods=86400, freq="s").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+ ts[:2] = np.nan # so results are the same
+ result = ts[2:].resample(freq, closed="left", label="left").mean()
+ expected = ts.resample(freq, closed="left", label="left").mean()
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("end", [1, 2])
+def test_resample_single_group(end, unit):
+ mysum = lambda x: x.sum()
+
+ rng = date_range("2000-1-1", f"2000-{end}-10", freq="D").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+ tm.assert_series_equal(ts.resample("ME").sum(), ts.resample("ME").apply(mysum))
+
+
+def test_resample_single_group_std(unit):
+ # GH 3849
+ s = Series(
+ [30.1, 31.6],
+ index=[Timestamp("20070915 15:30:00"), Timestamp("20070915 15:40:00")],
+ )
+ s.index = s.index.as_unit(unit)
+ expected = Series(
+ [0.75], index=DatetimeIndex([Timestamp("20070915")], freq="D").as_unit(unit)
+ )
+ result = s.resample("D").apply(lambda x: np.std(x))
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_offset(unit):
+ # GH 31809
+
+ rng = date_range("1/1/2000 00:00:00", "1/1/2000 02:00", freq="s").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+
+ resampled = ts.resample("5min", offset="2min").mean()
+ exp_rng = date_range("12/31/1999 23:57:00", "1/1/2000 01:57", freq="5min").as_unit(
+ unit
+ )
+ tm.assert_index_equal(resampled.index, exp_rng)
+
+
+@pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"origin": "1999-12-31 23:57:00"},
+ {"origin": Timestamp("1970-01-01 00:02:00")},
+ {"origin": "epoch", "offset": "2m"},
+ # origin of '1999-31-12 12:02:00' should be equivalent for this case
+ {"origin": "1999-12-31 12:02:00"},
+ {"offset": "-3m"},
+ ],
+)
+def test_resample_origin(kwargs, unit):
+ # GH 31809
+ rng = date_range("2000-01-01 00:00:00", "2000-01-01 02:00", freq="s").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+
+ exp_rng = date_range(
+ "1999-12-31 23:57:00", "2000-01-01 01:57", freq="5min"
+ ).as_unit(unit)
+
+ resampled = ts.resample("5min", **kwargs).mean()
+ tm.assert_index_equal(resampled.index, exp_rng)
+
+
+@pytest.mark.parametrize(
+ "origin", ["invalid_value", "epch", "startday", "startt", "2000-30-30", object()]
+)
+def test_resample_bad_origin(origin, unit):
+ rng = date_range("2000-01-01 00:00:00", "2000-01-01 02:00", freq="s").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+ msg = (
+ "'origin' should be equal to 'epoch', 'start', 'start_day', "
+ "'end', 'end_day' or should be a Timestamp convertible type. Got "
+ f"'{origin}' instead."
+ )
+ with pytest.raises(ValueError, match=msg):
+ ts.resample("5min", origin=origin)
+
+
+@pytest.mark.parametrize("offset", ["invalid_value", "12dayys", "2000-30-30", object()])
+def test_resample_bad_offset(offset, unit):
+ rng = date_range("2000-01-01 00:00:00", "2000-01-01 02:00", freq="s").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+ msg = f"'offset' should be a Timedelta convertible type. Got '{offset}' instead."
+ with pytest.raises(ValueError, match=msg):
+ ts.resample("5min", offset=offset)
+
+
+def test_resample_origin_prime_freq(unit):
+ # GH 31809
+ start, end = "2000-10-01 23:30:00", "2000-10-02 00:30:00"
+ rng = date_range(start, end, freq="7min").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+
+ exp_rng = date_range(
+ "2000-10-01 23:14:00", "2000-10-02 00:22:00", freq="17min"
+ ).as_unit(unit)
+ resampled = ts.resample("17min").mean()
+ tm.assert_index_equal(resampled.index, exp_rng)
+ resampled = ts.resample("17min", origin="start_day").mean()
+ tm.assert_index_equal(resampled.index, exp_rng)
+
+ exp_rng = date_range(
+ "2000-10-01 23:30:00", "2000-10-02 00:21:00", freq="17min"
+ ).as_unit(unit)
+ resampled = ts.resample("17min", origin="start").mean()
+ tm.assert_index_equal(resampled.index, exp_rng)
+ resampled = ts.resample("17min", offset="23h30min").mean()
+ tm.assert_index_equal(resampled.index, exp_rng)
+ resampled = ts.resample("17min", origin="start_day", offset="23h30min").mean()
+ tm.assert_index_equal(resampled.index, exp_rng)
+
+ exp_rng = date_range(
+ "2000-10-01 23:18:00", "2000-10-02 00:26:00", freq="17min"
+ ).as_unit(unit)
+ resampled = ts.resample("17min", origin="epoch").mean()
+ tm.assert_index_equal(resampled.index, exp_rng)
+
+ exp_rng = date_range(
+ "2000-10-01 23:24:00", "2000-10-02 00:15:00", freq="17min"
+ ).as_unit(unit)
+ resampled = ts.resample("17min", origin="2000-01-01").mean()
+ tm.assert_index_equal(resampled.index, exp_rng)
+
+
+def test_resample_origin_with_tz(unit):
+ # GH 31809
+ msg = "The origin must have the same timezone as the index."
+
+ tz = "Europe/Paris"
+ rng = date_range(
+ "2000-01-01 00:00:00", "2000-01-01 02:00", freq="s", tz=tz
+ ).as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+
+ exp_rng = date_range(
+ "1999-12-31 23:57:00", "2000-01-01 01:57", freq="5min", tz=tz
+ ).as_unit(unit)
+ resampled = ts.resample("5min", origin="1999-12-31 23:57:00+00:00").mean()
+ tm.assert_index_equal(resampled.index, exp_rng)
+
+ # origin of '1999-31-12 12:02:00+03:00' should be equivalent for this case
+ resampled = ts.resample("5min", origin="1999-12-31 12:02:00+03:00").mean()
+ tm.assert_index_equal(resampled.index, exp_rng)
+
+ resampled = ts.resample("5min", origin="epoch", offset="2m").mean()
+ tm.assert_index_equal(resampled.index, exp_rng)
+
+ with pytest.raises(ValueError, match=msg):
+ ts.resample("5min", origin="12/31/1999 23:57:00").mean()
+
+ # if the series is not tz aware, origin should not be tz aware
+ rng = date_range("2000-01-01 00:00:00", "2000-01-01 02:00", freq="s").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+ with pytest.raises(ValueError, match=msg):
+ ts.resample("5min", origin="12/31/1999 23:57:00+03:00").mean()
+
+
+def test_resample_origin_epoch_with_tz_day_vs_24h(unit):
+ # GH 34474
+ start, end = "2000-10-01 23:30:00+0500", "2000-12-02 00:30:00+0500"
+ rng = date_range(start, end, freq="7min").as_unit(unit)
+ random_values = np.random.default_rng(2).standard_normal(len(rng))
+ ts_1 = Series(random_values, index=rng)
+
+ result_1 = ts_1.resample("D").mean()
+ result_2 = ts_1.resample("24h", origin="epoch").mean()
+ tm.assert_series_equal(result_1, result_2, check_freq=False)
+ # GH#41943 check_freq=False bc Day and Hour(24) no longer compare as equal
+
+ # check that we have the same behavior with epoch even if we are not timezone aware
+ ts_no_tz = ts_1.tz_localize(None)
+ result_3 = ts_no_tz.resample("D").mean()
+ result_4 = ts_no_tz.resample("24h", origin="epoch").mean()
+ tm.assert_series_equal(result_1, result_3.tz_localize(rng.tz), check_freq=False)
+ tm.assert_series_equal(result_1, result_4.tz_localize(rng.tz), check_freq=False)
+
+ # check that we have the similar results with two different timezones (+2H and +5H)
+ start, end = "2000-10-01 23:30:00+0200", "2000-12-02 00:30:00+0200"
+ rng = date_range(start, end, freq="7min").as_unit(unit)
+ ts_2 = Series(random_values, index=rng)
+ result_5 = ts_2.resample("D").mean()
+ result_6 = ts_2.resample("24h", origin="epoch").mean()
+ tm.assert_series_equal(result_1.tz_localize(None), result_5.tz_localize(None))
+ tm.assert_series_equal(result_1.tz_localize(None), result_6.tz_localize(None))
+
+
+def test_resample_origin_with_day_freq_on_dst(unit):
+ # GH 31809
+ tz = "America/Chicago"
+ msg = "The '(origin|offset)' keyword does not take effect"
+
+ def _create_series(values, timestamps, freq="D"):
+ return Series(
+ values,
+ index=DatetimeIndex(
+ [Timestamp(t, tz=tz) for t in timestamps], freq=freq, ambiguous=True
+ ).as_unit(unit),
+ )
+
+ # test classical behavior of origin in a DST context
+ start = Timestamp("2013-11-02", tz=tz)
+ end = Timestamp("2013-11-03 23:59", tz=tz)
+ rng = date_range(start, end, freq="1h").as_unit(unit)
+ ts = Series(np.ones(len(rng)), index=rng)
+
+ expected = _create_series([24.0, 25.0], ["2013-11-02", "2013-11-03"])
+ for origin in ["epoch", "start", "start_day", start, None]:
+ warn = RuntimeWarning if origin != "start_day" else None
+ with tm.assert_produces_warning(warn, match=msg):
+ result = ts.resample("D", origin=origin).sum()
+ tm.assert_series_equal(result, expected)
+
+ # test complex behavior of origin/offset in a DST context
+ start = Timestamp("2013-11-03", tz=tz)
+ end = Timestamp("2013-11-03 23:59", tz=tz)
+ rng = date_range(start, end, freq="1h").as_unit(unit)
+ ts = Series(np.ones(len(rng)), index=rng)
+
+ # GH#61985 changed this to behave like "B" rather than "24h"
+ expected_ts = ["2013-11-03 00:00-05:00"]
+ expected = _create_series([25.0], expected_ts)
+ with tm.assert_produces_warning(RuntimeWarning, match=msg):
+ result = ts.resample("D", origin="start", offset="-2h").sum()
+ tm.assert_series_equal(result, expected)
+
+ expected_ts = ["2013-11-02 22:00-05:00", "2013-11-03 21:00-06:00"]
+ expected = _create_series([22.0, 3.0], expected_ts, freq="24h")
+ result = ts.resample("24h", origin="start", offset="-2h").sum()
+ tm.assert_series_equal(result, expected)
+
+ # GH#61985 changed this to behave like "B" rather than "24h"
+ expected_ts = ["2013-11-03 00:00-05:00"]
+ expected = _create_series([25.0], expected_ts)
+ with tm.assert_produces_warning(RuntimeWarning, match=msg):
+ result = ts.resample("D", origin="start", offset="2h").sum()
+ tm.assert_series_equal(result, expected)
+
+ expected_ts = ["2013-11-03 00:00-05:00"]
+ expected = _create_series([25.0], expected_ts)
+ with tm.assert_produces_warning(RuntimeWarning, match=msg):
+ result = ts.resample("D", origin="start", offset="-1h").sum()
+ tm.assert_series_equal(result, expected)
+
+ expected_ts = ["2013-11-03 00:00-05:00"]
+ expected = _create_series([25.0], expected_ts)
+ with tm.assert_produces_warning(RuntimeWarning, match=msg):
+ result = ts.resample("D", origin="start", offset="1h").sum()
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_dst_midnight_last_nonexistent():
+ # GH 58380
+ ts = Series(
+ 1,
+ date_range("2024-04-19", "2024-04-20", tz="Africa/Cairo", freq="15min"),
+ )
+
+ expected = Series([len(ts)], index=DatetimeIndex([ts.index[0]], freq="7D"))
+
+ result = ts.resample("7D").sum()
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_daily_anchored(unit):
+ rng = date_range("1/1/2000 0:00:00", periods=10000, freq="min").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+ ts[:2] = np.nan # so results are the same
+
+ result = ts[2:].resample("D", closed="left", label="left").mean()
+ expected = ts.resample("D", closed="left", label="left").mean()
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_to_period_monthly_buglet(unit):
+ # GH #1259
+
+ rng = date_range("1/1/2000", "12/31/2000").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+
+ result = ts.resample("ME").mean().to_period()
+ exp_index = period_range("Jan-2000", "Dec-2000", freq="M")
+ tm.assert_index_equal(result.index, exp_index)
+
+
+def test_period_with_agg():
+ # aggregate a period resampler with a lambda
+ s2 = Series(
+ np.random.default_rng(2).integers(0, 5, 50),
+ index=period_range("2012-01-01", freq="h", periods=50),
+ dtype="float64",
+ )
+
+ expected = s2.to_timestamp().resample("D").mean().to_period()
+ result = s2.resample("D").agg(lambda x: x.mean())
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_segfault(unit):
+ # GH 8573
+ # segfaulting in older versions
+ all_wins_and_wagers = [
+ (1, datetime(2013, 10, 1, 16, 20), 1, 0),
+ (2, datetime(2013, 10, 1, 16, 10), 1, 0),
+ (2, datetime(2013, 10, 1, 18, 15), 1, 0),
+ (2, datetime(2013, 10, 1, 16, 10, 31), 1, 0),
+ ]
+
+ df = DataFrame.from_records(
+ all_wins_and_wagers, columns=("ID", "timestamp", "A", "B")
+ ).set_index("timestamp")
+ df.index = df.index.as_unit(unit)
+ result = df.groupby("ID").resample("5min").sum()
+ expected = df.groupby("ID").apply(lambda x: x.resample("5min").sum())
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_dtype_preservation(unit):
+ # GH 12202
+ # validation tests for dtype preservation
+
+ df = DataFrame(
+ {
+ "date": date_range(start="2016-01-01", periods=4, freq="W").as_unit(unit),
+ "group": [1, 1, 2, 2],
+ "val": Series([5, 6, 7, 8], dtype="int32"),
+ }
+ ).set_index("date")
+
+ result = df.resample("1D").ffill()
+ assert result.val.dtype == np.int32
+
+ result = df.groupby("group").resample("1D").ffill()
+ assert result.val.dtype == np.int32
+
+
+def test_resample_dtype_coercion(unit):
+ pytest.importorskip("scipy.interpolate")
+
+ # GH 16361
+ df = {"a": [1, 3, 1, 4]}
+ df = DataFrame(df, index=date_range("2017-01-01", "2017-01-04").as_unit(unit))
+
+ expected = df.astype("float64").resample("h").mean()["a"].interpolate("cubic")
+
+ result = df.resample("h")["a"].mean().interpolate("cubic")
+ tm.assert_series_equal(result, expected)
+
+ result = df.resample("h").mean()["a"].interpolate("cubic")
+ tm.assert_series_equal(result, expected)
+
+
+def test_weekly_resample_buglet(unit):
+ # #1327
+ rng = date_range("1/1/2000", freq="B", periods=20).as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+
+ resampled = ts.resample("W").mean()
+ expected = ts.resample("W-SUN").mean()
+ tm.assert_series_equal(resampled, expected)
+
+
+def test_monthly_resample_error(unit):
+ # #1451
+ dates = date_range("4/16/2012 20:00", periods=5000, freq="h").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(dates)), index=dates)
+ # it works!
+ ts.resample("ME")
+
+
+def test_nanosecond_resample_error():
+ # GH 12307 - Values falls after last bin when
+ # Resampling using pd.tseries.offsets.Nano as period
+ start = 1443707890427
+ exp_start = 1443707890400
+ indx = date_range(start=pd.to_datetime(start), periods=10, freq="100ns")
+ ts = Series(range(len(indx)), index=indx)
+ r = ts.resample(pd.tseries.offsets.Nano(100))
+ result = r.agg("mean")
+
+ exp_indx = date_range(start=pd.to_datetime(exp_start), periods=10, freq="100ns")
+ exp = Series(range(len(exp_indx)), index=exp_indx, dtype=float)
+
+ tm.assert_series_equal(result, exp)
+
+
+def test_resample_anchored_intraday(unit):
+ # #1471, #1458
+
+ rng = date_range("1/1/2012", "4/1/2012", freq="100min").as_unit(unit)
+ df = DataFrame(rng.month, index=rng)
+
+ result = df.resample("ME").mean()
+ expected = df.resample("ME").mean().to_period()
+ expected = expected.to_timestamp(how="end")
+ expected.index += Timedelta(1, unit="us") - Timedelta(1, unit="D")
+ expected.index = expected.index.as_unit(unit)._with_freq("infer")
+ assert expected.index.freq == "ME"
+ tm.assert_frame_equal(result, expected)
+
+ result = df.resample("ME", closed="left").mean()
+ exp = df.shift(1, freq="D").resample("ME").mean().to_period()
+ exp = exp.to_timestamp(how="end")
+
+ exp.index = exp.index + Timedelta(1, unit="us") - Timedelta(1, unit="D")
+ exp.index = exp.index.as_unit(unit)._with_freq("infer")
+ assert exp.index.freq == "ME"
+ tm.assert_frame_equal(result, exp)
+
+
+def test_resample_anchored_intraday2(unit):
+ rng = date_range("1/1/2012", "4/1/2012", freq="100min").as_unit(unit)
+ df = DataFrame(rng.month, index=rng)
+
+ result = df.resample("QE").mean()
+ expected = df.resample("QE").mean().to_period()
+ expected = expected.to_timestamp(how="end")
+ expected.index += Timedelta(1, unit="us") - Timedelta(1, unit="D")
+ expected.index._data.freq = "QE"
+ expected.index._freq = lib.no_default
+ expected.index = expected.index.as_unit(unit)
+ tm.assert_frame_equal(result, expected)
+
+ result = df.resample("QE", closed="left").mean()
+ expected = df.shift(1, freq="D").resample("QE").mean()
+ expected = expected.to_period()
+ expected = expected.to_timestamp(how="end")
+ expected.index += Timedelta(1, unit="us") - Timedelta(1, unit="D")
+ expected.index._data.freq = "QE"
+ expected.index._freq = lib.no_default
+ expected.index = expected.index.as_unit(unit)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_anchored_intraday3(simple_date_range_series, unit):
+ ts = simple_date_range_series("2012-04-29 23:00", "2012-04-30 5:00", freq="h")
+ ts.index = ts.index.as_unit(unit)
+ resampled = ts.resample("ME").mean()
+ assert len(resampled) == 1
+
+
+@pytest.mark.parametrize("freq", ["MS", "BMS", "QS-MAR", "YS-DEC", "YS-JUN"])
+def test_resample_anchored_monthstart(simple_date_range_series, freq, unit):
+ ts = simple_date_range_series("1/1/2000", "12/31/2002")
+ ts.index = ts.index.as_unit(unit)
+ ts.resample(freq).mean()
+
+
+@pytest.mark.parametrize("label, sec", [[None, 2.0], ["right", "4.2"]])
+def test_resample_anchored_multiday(label, sec):
+ # When resampling a range spanning multiple days, ensure that the
+ # start date gets used to determine the offset. Fixes issue where
+ # a one day period is not a multiple of the frequency.
+ #
+ # See: https://github.com/pandas-dev/pandas/issues/8683
+
+ index1 = date_range("2014-10-14 23:06:23.206", periods=3, freq="400ms")
+ index2 = date_range("2014-10-15 23:00:00", periods=2, freq="2200ms")
+ index = index1.union(index2)
+
+ s = Series(np.random.default_rng(2).standard_normal(5), index=index)
+
+ # Ensure left closing works
+ result = s.resample("2200ms", label=label).mean()
+ assert result.index[-1] == Timestamp(f"2014-10-15 23:00:{sec}00")
+
+
+def test_corner_cases(unit):
+ # miscellaneous test coverage
+
+ rng = date_range("1/1/2000", periods=12, freq="min").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+
+ result = ts.resample("5min", closed="right", label="left").mean()
+ ex_index = date_range("1999-12-31 23:55", periods=4, freq="5min").as_unit(unit)
+ tm.assert_index_equal(result.index, ex_index)
+
+
+def test_corner_cases_date(simple_date_range_series, unit):
+ # resample to periods
+ ts = simple_date_range_series("2000-04-28", "2000-04-30 11:00", freq="h")
+ ts.index = ts.index.as_unit(unit)
+ result = ts.resample("ME").mean().to_period()
+ assert len(result) == 1
+ assert result.index[0] == Period("2000-04", freq="M")
+
+
+def test_anchored_lowercase_buglet(unit):
+ dates = date_range("4/16/2012 20:00", periods=50000, freq="s").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(dates)), index=dates)
+ # it works!
+ msg = "'d' is deprecated and will be removed in a future version."
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ ts.resample("d").mean()
+
+
+def test_upsample_apply_functions(unit):
+ # #1596
+ rng = date_range("2012-06-12", periods=4, freq="h").as_unit(unit)
+
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+
+ result = ts.resample("20min").aggregate(["mean", "sum"])
+ assert isinstance(result, DataFrame)
+
+
+def test_resample_not_monotonic(unit):
+ rng = date_range("2012-06-12", periods=200, freq="h").as_unit(unit)
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+
+ ts = ts.take(np.random.default_rng(2).permutation(len(ts)))
+
+ result = ts.resample("D").sum()
+ exp = ts.sort_index().resample("D").sum()
+ tm.assert_series_equal(result, exp)
+
+
+@pytest.mark.parametrize(
+ "dtype",
+ [
+ "int64",
+ "int32",
+ "float64",
+ "float32",
+ ],
+)
+def test_resample_median_bug_1688(dtype, unit):
+ # GH#55958
+ dti = DatetimeIndex(
+ [datetime(2012, 1, 1, 0, 0, 0), datetime(2012, 1, 1, 0, 5, 0)]
+ ).as_unit(unit)
+ df = DataFrame(
+ [1, 2],
+ index=dti,
+ dtype=dtype,
+ )
+
+ result = df.resample("min").apply(lambda x: x.mean())
+ exp = df.asfreq("min")
+ tm.assert_frame_equal(result, exp)
+
+ result = df.resample("min").median()
+ exp = df.asfreq("min")
+ tm.assert_frame_equal(result, exp)
+
+
+def test_how_lambda_functions(simple_date_range_series, unit):
+ ts = simple_date_range_series("1/1/2000", "4/1/2000")
+ ts.index = ts.index.as_unit(unit)
+
+ result = ts.resample("ME").apply(lambda x: x.mean())
+ exp = ts.resample("ME").mean()
+ tm.assert_series_equal(result, exp)
+
+ foo_exp = ts.resample("ME").mean()
+ foo_exp.name = "foo"
+ bar_exp = ts.resample("ME").std()
+ bar_exp.name = "bar"
+
+ result = ts.resample("ME").apply([lambda x: x.mean(), lambda x: x.std(ddof=1)])
+ result.columns = ["foo", "bar"]
+ tm.assert_series_equal(result["foo"], foo_exp)
+ tm.assert_series_equal(result["bar"], bar_exp)
+
+ # this is a MI Series, so comparing the names of the results
+ # doesn't make sense
+ result = ts.resample("ME").aggregate(
+ {"foo": lambda x: x.mean(), "bar": lambda x: x.std(ddof=1)}
+ )
+ tm.assert_series_equal(result["foo"], foo_exp, check_names=False)
+ tm.assert_series_equal(result["bar"], bar_exp, check_names=False)
+
+
+def test_resample_unequal_times(unit):
+ # #1772
+ start = datetime(1999, 3, 1, 5)
+ # end hour is less than start
+ end = datetime(2012, 7, 31, 4)
+ bad_ind = date_range(start, end, freq="30min").as_unit(unit)
+ df = DataFrame({"close": 1}, index=bad_ind)
+
+ # it works!
+ df.resample("YS").sum()
+
+
+def test_resample_consistency(unit):
+ # GH 6418
+ # resample with bfill / limit / reindex consistency
+
+ i30 = date_range("2002-02-02", periods=4, freq="30min").as_unit(unit)
+ s = Series(np.arange(4.0), index=i30)
+ s.iloc[2] = np.nan
+
+ # Upsample by factor 3 with reindex() and resample() methods:
+ i10 = date_range(i30[0], i30[-1], freq="10min").as_unit(unit)
+
+ s10 = s.reindex(index=i10, method="bfill")
+ s10_2 = s.reindex(index=i10, method="bfill", limit=2)
+ with tm.assert_produces_warning(Pandas4Warning):
+ rl = s.reindex_like(s10, method="bfill", limit=2)
+ r10_2 = s.resample("10Min").bfill(limit=2)
+ r10 = s.resample("10Min").bfill()
+
+ # s10_2, r10, r10_2, rl should all be equal
+ tm.assert_series_equal(s10_2, r10)
+ tm.assert_series_equal(s10_2, r10_2)
+ tm.assert_series_equal(s10_2, rl)
+
+
+dates1: list[DatetimeNaTType] = [
+ datetime(2014, 10, 1),
+ datetime(2014, 9, 3),
+ datetime(2014, 11, 5),
+ datetime(2014, 9, 5),
+ datetime(2014, 10, 8),
+ datetime(2014, 7, 15),
+]
+
+dates2: list[DatetimeNaTType] = [*dates1[:2], pd.NaT, *dates1[2:4], pd.NaT, *dates1[4:]]
+dates3 = [pd.NaT, *dates1, pd.NaT]
+
+
+@pytest.mark.parametrize("dates", [dates1, dates2, dates3])
+def test_resample_timegrouper(dates, unit):
+ # GH 7227
+ dates = DatetimeIndex(dates).as_unit(unit)
+ df = DataFrame({"A": dates, "B": np.arange(len(dates))})
+ result = df.set_index("A").resample("ME").count()
+ exp_idx = DatetimeIndex(
+ ["2014-07-31", "2014-08-31", "2014-09-30", "2014-10-31", "2014-11-30"],
+ freq="ME",
+ name="A",
+ ).as_unit(unit)
+ expected = DataFrame({"B": [1, 0, 2, 2, 1]}, index=exp_idx)
+ if df["A"].isna().any():
+ expected.index = expected.index._with_freq(None)
+ tm.assert_frame_equal(result, expected)
+
+ result = df.groupby(Grouper(freq="ME", key="A")).count()
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("dates", [dates1, dates2, dates3])
+def test_resample_timegrouper2(dates, unit):
+ dates = DatetimeIndex(dates).as_unit(unit)
+
+ df = DataFrame({"A": dates, "B": np.arange(len(dates)), "C": np.arange(len(dates))})
+ result = df.set_index("A").resample("ME").count()
+
+ exp_idx = DatetimeIndex(
+ ["2014-07-31", "2014-08-31", "2014-09-30", "2014-10-31", "2014-11-30"],
+ freq="ME",
+ name="A",
+ ).as_unit(unit)
+ expected = DataFrame(
+ {"B": [1, 0, 2, 2, 1], "C": [1, 0, 2, 2, 1]},
+ index=exp_idx,
+ columns=["B", "C"],
+ )
+ if df["A"].isna().any():
+ expected.index = expected.index._with_freq(None)
+ tm.assert_frame_equal(result, expected)
+
+ result = df.groupby(Grouper(freq="ME", key="A")).count()
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_nunique(unit):
+ # GH 12352
+ df = DataFrame(
+ {
+ "ID": {
+ Timestamp("2015-06-05 00:00:00"): "0010100903",
+ Timestamp("2015-06-08 00:00:00"): "0010150847",
+ },
+ "DATE": {
+ Timestamp("2015-06-05 00:00:00"): "2015-06-05",
+ Timestamp("2015-06-08 00:00:00"): "2015-06-08",
+ },
+ }
+ )
+ df.index = df.index.as_unit(unit)
+ r = df.resample("D")
+ g = df.groupby(Grouper(freq="D"))
+ expected = df.groupby(Grouper(freq="D")).ID.apply(lambda x: x.nunique())
+ assert expected.name == "ID"
+
+ for t in [r, g]:
+ result = t.ID.nunique()
+ tm.assert_series_equal(result, expected)
+
+ result = df.ID.resample("D").nunique()
+ tm.assert_series_equal(result, expected)
+
+ result = df.ID.groupby(Grouper(freq="D")).nunique()
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_nunique_preserves_column_level_names(unit):
+ # see gh-23222
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((5, 4)),
+ columns=Index(list("ABCD"), dtype=object),
+ index=date_range("2000-01-01", periods=5, freq="D"),
+ ).abs()
+ df.index = df.index.as_unit(unit)
+ df.columns = pd.MultiIndex.from_arrays(
+ [df.columns.tolist()] * 2, names=["lev0", "lev1"]
+ )
+ result = df.resample("1h").nunique()
+ tm.assert_index_equal(df.columns, result.columns)
+
+
+@pytest.mark.parametrize(
+ "func",
+ [
+ lambda x: x.nunique(),
+ lambda x: x.agg(Series.nunique),
+ lambda x: x.agg("nunique"),
+ ],
+ ids=["nunique", "series_nunique", "nunique_str"],
+)
+def test_resample_nunique_with_date_gap(func, unit):
+ # GH 13453
+ # Since all elements are unique, these should all be the same
+ index = date_range("1-1-2000", "2-15-2000", freq="h").as_unit(unit)
+ index2 = date_range("4-15-2000", "5-15-2000", freq="h").as_unit(unit)
+ index3 = index.append(index2)
+ s = Series(range(len(index3)), index=index3, dtype="int64")
+ r = s.resample("ME")
+ result = r.count()
+ expected = func(r)
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_group_info(unit):
+ # GH10914
+
+ # use a fixed seed to always have the same uniques
+ n = 100
+ k = 10
+ prng = np.random.default_rng(2)
+
+ dr = date_range(start="2015-08-27", periods=n // 10, freq="min").as_unit(unit)
+ ts = Series(prng.integers(0, n // k, n).astype("int64"), index=prng.choice(dr, n))
+
+ left = ts.resample("30min").nunique()
+ ix = date_range(start=ts.index.min(), end=ts.index.max(), freq="30min").as_unit(
+ unit
+ )
+
+ vals = ts.values
+ bins = np.searchsorted(ix.values, ts.index, side="right")
+
+ sorter = np.lexsort((vals, bins))
+ vals, bins = vals[sorter], bins[sorter]
+
+ mask = np.r_[True, vals[1:] != vals[:-1]]
+ mask |= np.r_[True, bins[1:] != bins[:-1]]
+
+ arr = np.bincount(bins[mask] - 1, minlength=len(ix)).astype("int64", copy=False)
+ right = Series(arr, index=ix)
+
+ tm.assert_series_equal(left, right)
+
+
+def test_resample_size(unit):
+ n = 10000
+ dr = date_range("2015-09-19", periods=n, freq="min").as_unit(unit)
+ ts = Series(
+ np.random.default_rng(2).standard_normal(n),
+ index=np.random.default_rng(2).choice(dr, n),
+ )
+
+ left = ts.resample("7min").size()
+ ix = date_range(start=left.index.min(), end=ts.index.max(), freq="7min").as_unit(
+ unit
+ )
+
+ bins = np.searchsorted(ix.values, ts.index.values, side="right")
+ val = np.bincount(bins, minlength=len(ix) + 1)[1:].astype("int64", copy=False)
+
+ right = Series(val, index=ix)
+ tm.assert_series_equal(left, right)
+
+
+def test_resample_across_dst():
+ # The test resamples a DatetimeIndex with values before and after a
+ # DST change
+ # Issue: 14682
+
+ # The DatetimeIndex we will start with
+ # (note that DST happens at 03:00+02:00 -> 02:00+01:00)
+ # 2016-10-30 02:23:00+02:00, 2016-10-30 02:23:00+01:00
+ df1 = DataFrame([1477786980, 1477790580], columns=["ts"])
+ dti1 = DatetimeIndex(
+ pd.to_datetime(df1.ts, unit="s")
+ .dt.tz_localize("UTC")
+ .dt.tz_convert("Europe/Madrid")
+ )
+
+ # The expected DatetimeIndex after resampling.
+ # 2016-10-30 02:00:00+02:00, 2016-10-30 02:00:00+01:00
+ df2 = DataFrame([1477785600, 1477789200], columns=["ts"])
+ dti2 = DatetimeIndex(
+ pd.to_datetime(df2.ts, unit="s")
+ .dt.tz_localize("UTC")
+ .dt.tz_convert("Europe/Madrid"),
+ freq="h",
+ )
+ df = DataFrame([5, 5], index=dti1)
+
+ result = df.resample(rule="h").sum()
+ expected = DataFrame([5, 5], index=dti2)
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_groupby_with_dst_time_change(unit):
+ # GH 24972
+ index = (
+ DatetimeIndex([1478064900001000000, 1480037118776792000], tz="UTC")
+ .tz_convert("America/Chicago")
+ .as_unit(unit)
+ )
+
+ df = DataFrame([1, 2], index=index)
+ result = df.groupby(Grouper(freq="1D")).last()
+ expected_index_values = date_range(
+ "2016-11-02", "2016-11-24", freq="D", tz="America/Chicago"
+ ).as_unit(unit)
+
+ index = DatetimeIndex(expected_index_values)
+ expected = DataFrame([1.0] + ([np.nan] * 21) + [2.0], index=index)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_dst_anchor(unit):
+ # 5172
+ dti = DatetimeIndex([datetime(2012, 11, 4, 23)], tz="US/Eastern").as_unit(unit)
+ df = DataFrame([5], index=dti)
+
+ dti = DatetimeIndex(df.index.normalize(), freq="D").as_unit(unit)
+ expected = DataFrame([5], index=dti)
+ tm.assert_frame_equal(df.resample(rule="D").sum(), expected)
+ df.resample(rule="MS").sum()
+ tm.assert_frame_equal(
+ df.resample(rule="MS").sum(),
+ DataFrame(
+ [5],
+ index=DatetimeIndex(
+ [datetime(2012, 11, 1)], tz="US/Eastern", freq="MS"
+ ).as_unit(unit),
+ ),
+ )
+
+
+def test_resample_dst_anchor2(unit):
+ dti = date_range(
+ "2013-09-30", "2013-11-02", freq="30Min", tz="Europe/Paris"
+ ).as_unit(unit)
+ values = range(dti.size)
+ df = DataFrame({"a": values, "b": values, "c": values}, index=dti, dtype="int64")
+ how = {"a": "min", "b": "max", "c": "count"}
+
+ rs = df.resample("W-MON")
+ result = rs.agg(how)[["a", "b", "c"]]
+ expected = DataFrame(
+ {
+ "a": [0, 48, 384, 720, 1056, 1394],
+ "b": [47, 383, 719, 1055, 1393, 1586],
+ "c": [48, 336, 336, 336, 338, 193],
+ },
+ index=date_range(
+ "9/30/2013", "11/4/2013", freq="W-MON", tz="Europe/Paris"
+ ).as_unit(unit),
+ )
+ tm.assert_frame_equal(
+ result,
+ expected,
+ "W-MON Frequency",
+ )
+
+ rs2 = df.resample("2W-MON")
+ result2 = rs2.agg(how)[["a", "b", "c"]]
+ expected2 = DataFrame(
+ {
+ "a": [0, 48, 720, 1394],
+ "b": [47, 719, 1393, 1586],
+ "c": [48, 672, 674, 193],
+ },
+ index=date_range(
+ "9/30/2013", "11/11/2013", freq="2W-MON", tz="Europe/Paris"
+ ).as_unit(unit),
+ )
+ tm.assert_frame_equal(
+ result2,
+ expected2,
+ "2W-MON Frequency",
+ )
+
+ rs3 = df.resample("MS")
+ result3 = rs3.agg(how)[["a", "b", "c"]]
+ expected3 = DataFrame(
+ {"a": [0, 48, 1538], "b": [47, 1537, 1586], "c": [48, 1490, 49]},
+ index=date_range("9/1/2013", "11/1/2013", freq="MS", tz="Europe/Paris").as_unit(
+ unit
+ ),
+ )
+ tm.assert_frame_equal(
+ result3,
+ expected3,
+ "MS Frequency",
+ )
+
+ rs4 = df.resample("2MS")
+ result4 = rs4.agg(how)[["a", "b", "c"]]
+ expected4 = DataFrame(
+ {"a": [0, 1538], "b": [1537, 1586], "c": [1538, 49]},
+ index=date_range(
+ "9/1/2013", "11/1/2013", freq="2MS", tz="Europe/Paris"
+ ).as_unit(unit),
+ )
+ tm.assert_frame_equal(
+ result4,
+ expected4,
+ "2MS Frequency",
+ )
+
+ df_daily = df["10/26/2013":"10/29/2013"]
+ rs_d = df_daily.resample("D")
+ result_d = rs_d.agg({"a": "min", "b": "max", "c": "count"})[["a", "b", "c"]]
+ expected_d = DataFrame(
+ {
+ "a": [1248, 1296, 1346, 1394],
+ "b": [1295, 1345, 1393, 1441],
+ "c": [48, 50, 48, 48],
+ },
+ index=date_range(
+ "10/26/2013", "10/29/2013", freq="D", tz="Europe/Paris"
+ ).as_unit(unit),
+ )
+ tm.assert_frame_equal(
+ result_d,
+ expected_d,
+ "D Frequency",
+ )
+
+
+def test_downsample_across_dst(unit):
+ # GH 8531
+ tz = zoneinfo.ZoneInfo("Europe/Berlin")
+ dt = datetime(2014, 10, 26)
+ dates = date_range(dt.astimezone(tz), periods=4, freq="2h").as_unit(unit)
+ result = Series(5, index=dates).resample("h").mean()
+ expected = Series(
+ [5.0, np.nan] * 3 + [5.0],
+ index=date_range(dt.astimezone(tz), periods=7, freq="h").as_unit(unit),
+ )
+ tm.assert_series_equal(result, expected)
+
+
+def test_downsample_across_dst_weekly(unit):
+ # GH 9119, GH 21459
+ df = DataFrame(
+ index=DatetimeIndex(
+ ["2017-03-25", "2017-03-26", "2017-03-27", "2017-03-28", "2017-03-29"],
+ tz="Europe/Amsterdam",
+ ).as_unit(unit),
+ data=[11, 12, 13, 14, 15],
+ )
+ result = df.resample("1W").sum()
+ expected = DataFrame(
+ [23, 42],
+ index=DatetimeIndex(
+ ["2017-03-26", "2017-04-02"], tz="Europe/Amsterdam", freq="W"
+ ).as_unit(unit),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_downsample_across_dst_weekly_2(unit):
+ # GH 9119, GH 21459
+ idx = date_range("2013-04-01", "2013-05-01", tz="Europe/London", freq="h").as_unit(
+ unit
+ )
+ s = Series(index=idx, dtype=np.float64)
+ result = s.resample("W").mean()
+ expected = Series(
+ index=date_range("2013-04-07", freq="W", periods=5, tz="Europe/London").as_unit(
+ unit
+ ),
+ dtype=np.float64,
+ )
+ tm.assert_series_equal(result, expected)
+
+
+def test_downsample_dst_at_midnight(unit):
+ # GH 25758
+ start = datetime(2018, 11, 3, 12)
+ end = datetime(2018, 11, 5, 12)
+ index = date_range(start, end, freq="1h").as_unit(unit)
+ index = index.tz_localize("UTC").tz_convert("America/Havana")
+ data = list(range(len(index)))
+ dataframe = DataFrame(data, index=index)
+ result = dataframe.groupby(Grouper(freq="1D")).mean()
+
+ dti = date_range("2018-11-03", periods=3).tz_localize(
+ "America/Havana", ambiguous=True
+ )
+ dti = DatetimeIndex(dti, freq="D").as_unit(unit)
+ expected = DataFrame([7.5, 28.0, 44.5], index=dti)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_with_nat(unit):
+ # GH 13020
+ index = DatetimeIndex(
+ [
+ pd.NaT,
+ "1970-01-01 00:00:00",
+ pd.NaT,
+ "1970-01-01 00:00:01",
+ "1970-01-01 00:00:02",
+ ]
+ ).as_unit(unit)
+ frame = DataFrame([2, 3, 5, 7, 11], index=index)
+
+ index_1s = DatetimeIndex(
+ ["1970-01-01 00:00:00", "1970-01-01 00:00:01", "1970-01-01 00:00:02"]
+ ).as_unit(unit)
+ frame_1s = DataFrame([3.0, 7.0, 11.0], index=index_1s)
+ tm.assert_frame_equal(frame.resample("1s").mean(), frame_1s)
+
+ index_2s = DatetimeIndex(["1970-01-01 00:00:00", "1970-01-01 00:00:02"]).as_unit(
+ unit
+ )
+ frame_2s = DataFrame([5.0, 11.0], index=index_2s)
+ tm.assert_frame_equal(frame.resample("2s").mean(), frame_2s)
+
+ index_3s = DatetimeIndex(["1970-01-01 00:00:00"]).as_unit(unit)
+ frame_3s = DataFrame([7.0], index=index_3s)
+ tm.assert_frame_equal(frame.resample("3s").mean(), frame_3s)
+
+ tm.assert_frame_equal(frame.resample("60s").mean(), frame_3s)
+
+
+def test_resample_datetime_values(unit):
+ # GH 13119
+ # check that datetime dtype is preserved when NaT values are
+ # introduced by the resampling
+
+ dates = [datetime(2016, 1, 15), datetime(2016, 1, 19)]
+ df = DataFrame({"timestamp": dates}, index=dates)
+ df.index = df.index.as_unit(unit)
+
+ exp = Series(
+ [datetime(2016, 1, 15), pd.NaT, datetime(2016, 1, 19)],
+ index=date_range("2016-01-15", periods=3, freq="2D").as_unit(unit),
+ name="timestamp",
+ )
+
+ res = df.resample("2D").first()["timestamp"]
+ tm.assert_series_equal(res, exp)
+ res = df["timestamp"].resample("2D").first()
+ tm.assert_series_equal(res, exp)
+
+
+def test_resample_apply_with_additional_args(unit):
+ # GH 14615
+ index = date_range("1/1/2000 00:00:00", "1/1/2000 00:13:00", freq="Min")
+ series = Series(range(len(index)), index=index)
+ series.index.name = "index"
+
+ def f(data, add_arg):
+ return np.mean(data) * add_arg
+
+ series.index = series.index.as_unit(unit)
+
+ multiplier = 10
+ result = series.resample("D").apply(f, multiplier)
+ expected = series.resample("D").mean().multiply(multiplier)
+ tm.assert_series_equal(result, expected)
+
+ # Testing as kwarg
+ result = series.resample("D").apply(f, add_arg=multiplier)
+ expected = series.resample("D").mean().multiply(multiplier)
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_apply_with_additional_args2():
+ # Testing dataframe
+ def f(data, add_arg):
+ return np.mean(data) * add_arg
+
+ multiplier = 10
+
+ df = DataFrame({"A": 1, "B": 2}, index=date_range("2017", periods=10))
+ result = df.groupby("A").resample("D").agg(f, multiplier).astype(float)
+ expected = df.groupby("A").resample("D").mean().multiply(multiplier)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("k", [1, 2, 3])
+@pytest.mark.parametrize(
+ "n1, freq1, n2, freq2",
+ [
+ (30, "s", 0.5, "Min"),
+ (60, "s", 1, "Min"),
+ (3600, "s", 1, "h"),
+ (60, "Min", 1, "h"),
+ (21600, "s", 0.25, "D"),
+ (86400, "s", 1, "D"),
+ (43200, "s", 0.5, "D"),
+ (1440, "Min", 1, "D"),
+ (12, "h", 0.5, "D"),
+ (24, "h", 1, "D"),
+ ],
+)
+def test_resample_equivalent_offsets(n1, freq1, n2, freq2, k, unit):
+ # GH 24127
+ n1_ = n1 * k
+ n2_ = n2 * k
+ dti = date_range("1991-09-05", "1991-09-06", freq=freq1).as_unit(unit)
+ ser = Series(range(len(dti)), index=dti)
+
+ result1 = ser.resample(str(n1_) + freq1).mean()
+ result2 = ser.resample(str(n2_) + freq2).mean()
+ if freq2 == "D" and isinstance(result2.index.freq, Day):
+ # GH#55502 Day is no longer a Tick so no longer compares as equivalent,
+ # but the actual values we expect should still match
+ result2.index.freq = to_offset(Timedelta(days=result2.index.freq.n))
+ tm.assert_series_equal(result1, result2)
+
+
+@pytest.mark.parametrize(
+ "first,last,freq,exp_first,exp_last",
+ [
+ ("19910905", "19920406", "D", "19910905", "19920407"),
+ ("19910905 00:00", "19920406 06:00", "D", "19910905", "19920407"),
+ ("19910905 06:00", "19920406 06:00", "h", "19910905 06:00", "19920406 07:00"),
+ ("19910906", "19920406", "ME", "19910831", "19920430"),
+ ("19910831", "19920430", "ME", "19910831", "19920531"),
+ ("1991-08", "1992-04", "ME", "19910831", "19920531"),
+ ],
+)
+def test_get_timestamp_range_edges(first, last, freq, exp_first, exp_last, unit):
+ first = Period(first)
+ first = first.to_timestamp(first.freq).as_unit(unit)
+ last = Period(last)
+ last = last.to_timestamp(last.freq).as_unit(unit)
+
+ exp_first = Timestamp(exp_first)
+ exp_last = Timestamp(exp_last)
+
+ freq = pd.tseries.frequencies.to_offset(freq)
+ result = _get_timestamp_range_edges(first, last, freq, unit="ns")
+ expected = (exp_first, exp_last)
+ assert result == expected
+
+
+@pytest.mark.parametrize("duplicates", [True, False])
+def test_resample_apply_product(duplicates, unit):
+ # GH 5586
+ index = date_range(start="2012-01-31", freq="ME", periods=12).as_unit(unit)
+
+ ts = Series(range(12), index=index)
+ df = DataFrame({"A": ts, "B": ts + 2})
+ if duplicates:
+ df.columns = ["A", "A"]
+
+ result = df.resample("QE").apply(np.prod)
+ expected = DataFrame(
+ np.array([[0, 24], [60, 210], [336, 720], [990, 1716]], dtype=np.int64),
+ index=DatetimeIndex(
+ ["2012-03-31", "2012-06-30", "2012-09-30", "2012-12-31"], freq="QE-DEC"
+ ).as_unit(unit),
+ columns=df.columns,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "first,last,freq_in,freq_out,exp_last",
+ [
+ (
+ "2020-03-28",
+ "2020-03-31",
+ "D",
+ "24h",
+ "2020-03-30 01:00",
+ ), # includes transition into DST
+ (
+ "2020-03-28",
+ "2020-10-27",
+ "D",
+ "24h",
+ "2020-10-27 00:00",
+ ), # includes transition into and out of DST
+ (
+ "2020-10-25",
+ "2020-10-27",
+ "D",
+ "24h",
+ "2020-10-26 23:00",
+ ), # includes transition out of DST
+ (
+ "2020-03-28",
+ "2020-03-31",
+ "24h",
+ "D",
+ "2020-03-30 00:00",
+ ), # same as above, but from 24H to D
+ ("2020-03-28", "2020-10-27", "24h", "D", "2020-10-27 00:00"),
+ ("2020-10-25", "2020-10-27", "24h", "D", "2020-10-26 00:00"),
+ ],
+)
+def test_resample_calendar_day_with_dst(
+ first: str, last: str, freq_in: str, freq_out: str, exp_last: str, unit
+):
+ # GH 35219
+ ts = Series(
+ 1.0, date_range(first, last, freq=freq_in, tz="Europe/Amsterdam").as_unit(unit)
+ )
+ result = ts.resample(freq_out).ffill()
+ expected = Series(
+ 1.0,
+ date_range(first, exp_last, freq=freq_out, tz="Europe/Amsterdam").as_unit(unit),
+ )
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("func", ["min", "max", "first", "last"])
+def test_resample_aggregate_functions_min_count(func, unit):
+ # GH#37768
+ index = date_range(start="2020", freq="ME", periods=3).as_unit(unit)
+ ser = Series([1, np.nan, np.nan], index)
+ result = getattr(ser.resample("QE"), func)(min_count=2)
+ expected = Series(
+ [np.nan],
+ index=DatetimeIndex(["2020-03-31"], freq="QE-DEC").as_unit(unit),
+ )
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_unsigned_int(any_unsigned_int_numpy_dtype, unit):
+ # gh-43329
+ df = DataFrame(
+ index=date_range(start="2000-01-01", end="2000-01-03 23", freq="12h").as_unit(
+ unit
+ ),
+ columns=["x"],
+ data=[0, 1, 0] * 2,
+ dtype=any_unsigned_int_numpy_dtype,
+ )
+ df = df.loc[(df.index < "2000-01-02") | (df.index > "2000-01-03"), :]
+
+ result = df.resample("D").max()
+
+ expected = DataFrame(
+ [1, np.nan, 0],
+ columns=["x"],
+ index=date_range(start="2000-01-01", end="2000-01-03 23", freq="D").as_unit(
+ unit
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_long_rule_non_nano():
+ # https://github.com/pandas-dev/pandas/issues/51024
+ idx = date_range("0300-01-01", "2000-01-01", unit="s", freq="100YE")
+ ser = Series([1, 4, 2, 8, 5, 7, 1, 4, 2, 8, 5, 7, 1, 4, 2, 8, 5], index=idx)
+ result = ser.resample("200YE").mean()
+ expected_idx = DatetimeIndex(
+ np.array(
+ [
+ "0300-12-31",
+ "0500-12-31",
+ "0700-12-31",
+ "0900-12-31",
+ "1100-12-31",
+ "1300-12-31",
+ "1500-12-31",
+ "1700-12-31",
+ "1900-12-31",
+ ]
+ ).astype("datetime64[s]"),
+ freq="200YE-DEC",
+ )
+ expected = Series([1.0, 3.0, 6.5, 4.0, 3.0, 6.5, 4.0, 3.0, 6.5], index=expected_idx)
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_empty_series_with_tz():
+ # GH#53664
+ df = DataFrame({"ts": [], "values": []}).astype(
+ {"ts": "datetime64[ns, Atlantic/Faroe]"}
+ )
+ rs = df.resample("2MS", on="ts", closed="left", label="left")
+ result = rs["values"].sum()
+
+ expected_idx = DatetimeIndex(
+ [], freq="2MS", name="ts", dtype="datetime64[ns, Atlantic/Faroe]"
+ )
+ expected = Series([], index=expected_idx, name="values", dtype="float64")
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("freq", ["2M", "2m", "2Q", "2Q-SEP", "2q-sep", "1Y", "2Y-MAR"])
+def test_resample_M_Q_Y_raises(freq):
+ msg = f"Invalid frequency: {freq}"
+
+ s = Series(range(10), index=date_range("20130101", freq="D", periods=10))
+ with pytest.raises(ValueError, match=msg):
+ s.resample(freq).mean()
+
+
+@pytest.mark.parametrize("freq", ["2BM", "1bm", "1BQ", "2BQ-MAR", "2bq=-mar"])
+def test_resample_BM_BQ_raises(freq):
+ msg = f"Invalid frequency: {freq}"
+
+ s = Series(range(10), index=date_range("20130101", freq="D", periods=10))
+ with pytest.raises(ValueError, match=msg):
+ s.resample(freq).mean()
+
+
+@pytest.mark.parametrize(
+ "freq,freq_depr,data",
+ [
+ ("1W-SUN", "1w-sun", ["2013-01-06"]),
+ ("1D", "1d", ["2013-01-01"]),
+ ("1B", "1b", ["2013-01-01"]),
+ ("1C", "1c", ["2013-01-01"]),
+ ],
+)
+def test_resample_depr_lowercase_frequency(freq, freq_depr, data):
+ msg = f"'{freq_depr[1:]}' is deprecated and will be removed in a future version."
+
+ s = Series(range(5), index=date_range("20130101", freq="h", periods=5, unit="ns"))
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ result = s.resample(freq_depr).mean()
+
+ exp_dti = DatetimeIndex(data=data, dtype="datetime64[ns]", freq=freq)
+ expected = Series(2.0, index=exp_dti)
+ tm.assert_series_equal(result, expected, check_freq=False)
+ # GH#41943 check_freq=False bc 24H and D no longer compare as equal
+
+
+def test_resample_ms_closed_right(unit):
+ # https://github.com/pandas-dev/pandas/issues/55271
+ dti = date_range(start="2020-01-31", freq="1min", periods=6000, unit=unit)
+ df = DataFrame({"ts": dti}, index=dti)
+ grouped = df.resample("MS", closed="right")
+ result = grouped.last()
+ exp_dti = DatetimeIndex(
+ [datetime(2020, 1, 1), datetime(2020, 2, 1)], freq="MS"
+ ).as_unit(unit)
+ expected = DataFrame(
+ {"ts": [datetime(2020, 2, 1), datetime(2020, 2, 4, 3, 59)]},
+ index=exp_dti,
+ ).astype(f"M8[{unit}]")
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("freq", ["B", "C"])
+def test_resample_c_b_closed_right(freq: str, unit):
+ # https://github.com/pandas-dev/pandas/issues/55281
+ dti = date_range(start="2020-01-31", freq="1min", periods=6000, unit=unit)
+ df = DataFrame({"ts": dti}, index=dti)
+ grouped = df.resample(freq, closed="right")
+ result = grouped.last()
+
+ exp_dti = DatetimeIndex(
+ [
+ datetime(2020, 1, 30),
+ datetime(2020, 1, 31),
+ datetime(2020, 2, 3),
+ datetime(2020, 2, 4),
+ ],
+ freq=freq,
+ ).as_unit(unit)
+ expected = DataFrame(
+ {
+ "ts": [
+ datetime(2020, 1, 31),
+ datetime(2020, 2, 3),
+ datetime(2020, 2, 4),
+ datetime(2020, 2, 4, 3, 59),
+ ]
+ },
+ index=exp_dti,
+ ).astype(f"M8[{unit}]")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_b_55282(unit):
+ # https://github.com/pandas-dev/pandas/issues/55282
+ dti = date_range("2023-09-26", periods=6, freq="12h", unit=unit)
+ ser = Series([1, 2, 3, 4, 5, 6], index=dti)
+ result = ser.resample("B", closed="right", label="right").mean()
+
+ exp_dti = DatetimeIndex(
+ [
+ datetime(2023, 9, 26),
+ datetime(2023, 9, 27),
+ datetime(2023, 9, 28),
+ datetime(2023, 9, 29),
+ ],
+ freq="B",
+ ).as_unit(unit)
+ expected = Series(
+ [1.0, 2.5, 4.5, 6.0],
+ index=exp_dti,
+ )
+ tm.assert_series_equal(result, expected)
+
+
+@td.skip_if_no("pyarrow")
+@pytest.mark.parametrize(
+ "tz",
+ [
+ None,
+ pytest.param(
+ "UTC",
+ marks=pytest.mark.xfail(
+ condition=is_platform_windows() and pa_version_under22p0,
+ reason="TODO: Set ARROW_TIMEZONE_DATABASE env var in CI",
+ ),
+ ),
+ ],
+)
+def test_arrow_timestamp_resample(tz):
+ # GH 56371
+ idx = Series(date_range("2020-01-01", periods=5), dtype="timestamp[ns][pyarrow]")
+ if tz is not None:
+ idx = idx.dt.tz_localize(tz)
+ expected = Series(np.arange(5, dtype=np.float64), index=idx)
+ result = expected.resample("1D").mean()
+ tm.assert_series_equal(result, expected)
+
+
+@td.skip_if_no("pyarrow")
+def test_arrow_timestamp_resample_keep_index_name():
+ # https://github.com/pandas-dev/pandas/issues/61222
+ idx = Series(date_range("2020-01-01", periods=5), dtype="timestamp[ns][pyarrow]")
+ expected = Series(np.arange(5, dtype=np.float64), index=idx)
+ expected.index.name = "index_name"
+ result = expected.resample("1D").mean()
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_unit_second_large_years():
+ # GH#57427
+ index = DatetimeIndex(
+ date_range(start=Timestamp("1950-01-01"), periods=10, freq="1000YS", unit="s")
+ )
+ ser = Series(1, index=index)
+ result = ser.resample("2000YS").sum()
+ expected = Series(2, index=index[::2])
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("freq", ["1A", "2A-MAR"])
+def test_resample_A_raises(freq):
+ msg = f"Invalid frequency: {freq[1:]}"
+
+ s = Series(range(10), index=date_range("20130101", freq="D", periods=10))
+ with pytest.raises(ValueError, match=msg):
+ s.resample(freq).mean()
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_period_index.py b/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_period_index.py
new file mode 100644
index 0000000000000000000000000000000000000000..d7e2613b823f75f83bd1fdb840d2e25df26c8123
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_period_index.py
@@ -0,0 +1,1032 @@
+from datetime import (
+ datetime,
+ timezone,
+)
+import re
+import warnings
+import zoneinfo
+
+import dateutil
+import numpy as np
+import pytest
+
+from pandas._libs.tslibs.ccalendar import (
+ DAYS,
+ MONTHS,
+)
+from pandas._libs.tslibs.period import IncompatibleFrequency
+from pandas.errors import InvalidIndexError
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Series,
+ Timestamp,
+)
+import pandas._testing as tm
+from pandas.core.indexes.datetimes import date_range
+from pandas.core.indexes.period import (
+ Period,
+ PeriodIndex,
+ period_range,
+)
+from pandas.core.resample import _get_period_range_edges
+
+from pandas.tseries import offsets
+
+
+@pytest.fixture
+def simple_period_range_series():
+ """
+ Series with period range index and random data for test purposes.
+ """
+
+ def _simple_period_range_series(start, end, freq="D"):
+ with warnings.catch_warnings():
+ # suppress Period[B] deprecation warning
+ msg = "|".join(["Period with BDay freq", r"PeriodDtype\[B\] is deprecated"])
+ warnings.filterwarnings(
+ "ignore",
+ msg,
+ category=FutureWarning,
+ )
+ rng = period_range(start, end, freq=freq)
+ return Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+
+ return _simple_period_range_series
+
+
+class TestPeriodIndex:
+ @pytest.mark.parametrize("freq", ["2D", "1h", "2h"])
+ def test_asfreq(self, frame_or_series, freq):
+ # GH 12884, 15944
+
+ obj = frame_or_series(range(5), index=period_range("2020-01-01", periods=5))
+
+ expected = obj.to_timestamp().resample(freq).asfreq()
+ result = obj.to_timestamp().resample(freq).asfreq()
+ tm.assert_almost_equal(result, expected)
+
+ start = obj.index[0].to_timestamp(how="start")
+ end = (obj.index[-1] + obj.index.freq).to_timestamp(how="start")
+ new_index = date_range(start=start, end=end, freq=freq, inclusive="left")
+ expected = obj.to_timestamp().reindex(new_index).to_period(freq)
+
+ result = obj.resample(freq).asfreq()
+ tm.assert_almost_equal(result, expected)
+
+ result = obj.resample(freq).asfreq().to_timestamp().to_period()
+ tm.assert_almost_equal(result, expected)
+
+ def test_asfreq_fill_value(self):
+ # test for fill value during resampling, issue 3715
+
+ index = period_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="D")
+ s = Series(range(len(index)), index=index)
+ new_index = date_range(
+ s.index[0].to_timestamp(how="start"),
+ (s.index[-1]).to_timestamp(how="start"),
+ freq="1h",
+ )
+ expected = s.to_timestamp().reindex(new_index, fill_value=4.0)
+ result = s.to_timestamp().resample("1h").asfreq(fill_value=4.0)
+ tm.assert_series_equal(result, expected)
+
+ frame = s.to_frame("value")
+ new_index = date_range(
+ frame.index[0].to_timestamp(how="start"),
+ (frame.index[-1]).to_timestamp(how="start"),
+ freq="1h",
+ )
+ expected = frame.to_timestamp().reindex(new_index, fill_value=3.0)
+ result = frame.to_timestamp().resample("1h").asfreq(fill_value=3.0)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("freq", ["h", "12h", "2D", "W"])
+ @pytest.mark.parametrize("kwargs", [{"on": "date"}, {"level": "d"}])
+ def test_selection(self, freq, kwargs):
+ # This is a bug, these should be implemented
+ # GH 14008
+ index = period_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="D")
+ rng = np.arange(len(index), dtype=np.int64)
+ df = DataFrame(
+ {"date": index, "a": rng},
+ index=pd.MultiIndex.from_arrays([rng, index], names=["v", "d"]),
+ )
+ msg = (
+ "Resampling from level= or on= selection with a PeriodIndex is "
+ r"not currently supported, use \.set_index\(\.\.\.\) to "
+ "explicitly set index"
+ )
+ with pytest.raises(NotImplementedError, match=msg):
+ df.resample(freq, **kwargs)
+
+ @pytest.mark.parametrize("month", MONTHS)
+ @pytest.mark.parametrize("meth", ["ffill", "bfill"])
+ @pytest.mark.parametrize("conv", ["start", "end"])
+ @pytest.mark.parametrize(
+ ("offset", "period"), [("D", "D"), ("B", "B"), ("ME", "M"), ("QE", "Q")]
+ )
+ def test_annual_upsample_cases(
+ self, offset, period, conv, meth, month, simple_period_range_series
+ ):
+ ts = simple_period_range_series("1/1/1990", "12/31/1990", freq=f"Y-{month}")
+ warn = FutureWarning if period == "B" else None
+ msg = r"PeriodDtype\[B\] is deprecated"
+ with tm.assert_produces_warning(warn, match=msg):
+ result = getattr(ts.resample(period, convention=conv), meth)()
+ expected = result.to_timestamp(period, how=conv)
+ expected = expected.asfreq(offset, meth).to_period()
+ tm.assert_series_equal(result, expected)
+
+ def test_basic_downsample(self, simple_period_range_series):
+ ts = simple_period_range_series("1/1/1990", "6/30/1995", freq="M")
+ result = ts.resample("Y-DEC").mean()
+
+ expected = ts.groupby(ts.index.year).mean()
+ expected.index = period_range("1/1/1990", "6/30/1995", freq="Y-DEC")
+ tm.assert_series_equal(result, expected)
+
+ # this is ok
+ tm.assert_series_equal(ts.resample("Y-DEC").mean(), result)
+ tm.assert_series_equal(ts.resample("Y").mean(), result)
+
+ @pytest.mark.parametrize(
+ "rule,expected_error_msg",
+ [
+ ("Y-DEC", ""),
+ ("Q-MAR", ""),
+ ("M", ""),
+ ("W-THU", ""),
+ ],
+ )
+ def test_not_subperiod(self, simple_period_range_series, rule, expected_error_msg):
+ # These are incompatible period rules for resampling
+ ts = simple_period_range_series("1/1/1990", "6/30/1995", freq="W-WED")
+ msg = (
+ "Frequency cannot be resampled to "
+ f"{expected_error_msg}, as they are not sub or super periods"
+ )
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ ts.resample(rule).mean()
+
+ @pytest.mark.parametrize("freq", ["D", "2D"])
+ def test_basic_upsample(self, freq, simple_period_range_series):
+ ts = simple_period_range_series("1/1/1990", "6/30/1995", freq="M")
+ result = ts.resample("Y-DEC").mean()
+
+ resampled = result.resample(freq, convention="end").ffill()
+ expected = result.to_timestamp(freq, how="end")
+ expected = expected.asfreq(freq, "ffill").to_period(freq)
+ tm.assert_series_equal(resampled, expected)
+
+ def test_upsample_with_limit(self):
+ rng = period_range("1/1/2000", periods=5, freq="Y")
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), rng)
+
+ result = ts.resample("M", convention="end").ffill(limit=2)
+ expected = ts.asfreq("M").reindex(result.index, method="ffill", limit=2)
+ tm.assert_series_equal(result, expected)
+
+ def test_annual_upsample(self, simple_period_range_series):
+ ts = simple_period_range_series("1/1/1990", "12/31/1995", freq="Y-DEC")
+ df = DataFrame({"a": ts})
+ rdf = df.resample("D").ffill()
+ exp = df["a"].resample("D").ffill()
+ tm.assert_series_equal(rdf["a"], exp)
+
+ def test_annual_upsample2(self):
+ rng = period_range("2000", "2003", freq="Y-DEC")
+ ts = Series([1, 2, 3, 4], index=rng)
+
+ result = ts.resample("M").ffill()
+ ex_index = period_range("2000-01", "2003-12", freq="M")
+
+ expected = ts.asfreq("M", how="start").reindex(ex_index, method="ffill")
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize("month", MONTHS)
+ @pytest.mark.parametrize("convention", ["start", "end"])
+ @pytest.mark.parametrize(
+ ("offset", "period"), [("D", "D"), ("B", "B"), ("ME", "M")]
+ )
+ def test_quarterly_upsample(
+ self, month, offset, period, convention, simple_period_range_series
+ ):
+ freq = f"Q-{month}"
+ ts = simple_period_range_series("1/1/1990", "12/31/1991", freq=freq)
+ warn = FutureWarning if period == "B" else None
+ msg = r"PeriodDtype\[B\] is deprecated"
+ with tm.assert_produces_warning(warn, match=msg):
+ result = ts.resample(period, convention=convention).ffill()
+ expected = result.to_timestamp(period, how=convention)
+ expected = expected.asfreq(offset, "ffill").to_period()
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize("target", ["D", "B"])
+ @pytest.mark.parametrize("convention", ["start", "end"])
+ def test_monthly_upsample(self, target, convention, simple_period_range_series):
+ ts = simple_period_range_series("1/1/1990", "12/31/1995", freq="M")
+
+ warn = None if target == "D" else FutureWarning
+ msg = r"PeriodDtype\[B\] is deprecated"
+ with tm.assert_produces_warning(warn, match=msg):
+ result = ts.resample(target, convention=convention).ffill()
+ expected = result.to_timestamp(target, how=convention)
+ expected = expected.asfreq(target, "ffill").to_period()
+ tm.assert_series_equal(result, expected)
+
+ def test_resample_basic(self):
+ # GH3609
+ s = Series(
+ range(100),
+ index=date_range("20130101", freq="s", periods=100, name="idx"),
+ dtype="float",
+ )
+ s[10:30] = np.nan
+ index = PeriodIndex(
+ [Period("2013-01-01 00:00", "min"), Period("2013-01-01 00:01", "min")],
+ name="idx",
+ )
+ expected = Series([34.5, 79.5], index=index)
+ result = s.to_period().resample("min").mean()
+ tm.assert_series_equal(result, expected)
+ result2 = s.resample("min").mean().to_period()
+ tm.assert_series_equal(result2, expected)
+
+ @pytest.mark.parametrize(
+ "freq,expected_vals", [("M", [31, 29, 31, 9]), ("2M", [31 + 29, 31 + 9])]
+ )
+ def test_resample_count(self, freq, expected_vals):
+ # GH12774
+ series = Series(1, index=period_range(start="2000", periods=100))
+ result = series.resample(freq).count()
+ expected_index = period_range(
+ start="2000", freq=freq, periods=len(expected_vals)
+ )
+ expected = Series(expected_vals, index=expected_index)
+ tm.assert_series_equal(result, expected)
+
+ def test_resample_same_freq(self, resample_method):
+ # GH12770
+ series = Series(range(3), index=period_range(start="2000", periods=3, freq="M"))
+ expected = series
+
+ result = getattr(series.resample("M"), resample_method)()
+ tm.assert_series_equal(result, expected)
+
+ def test_resample_incompat_freq(self):
+ msg = (
+ "Frequency cannot be resampled to , "
+ "as they are not sub or super periods"
+ )
+ pi = period_range(start="2000", periods=3, freq="M")
+ ser = Series(range(3), index=pi)
+ rs = ser.resample("W")
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ # TODO: should this raise at the resample call instead of at the mean call?
+ rs.mean()
+
+ @pytest.mark.parametrize(
+ "tz",
+ [
+ zoneinfo.ZoneInfo("America/Los_Angeles"),
+ dateutil.tz.gettz("America/Los_Angeles"),
+ ],
+ )
+ def test_with_local_timezone(self, tz):
+ # see gh-5430
+ local_timezone = tz
+
+ start = datetime(
+ year=2013, month=11, day=1, hour=0, minute=0, tzinfo=timezone.utc
+ )
+ # 1 day later
+ end = datetime(
+ year=2013, month=11, day=2, hour=0, minute=0, tzinfo=timezone.utc
+ )
+
+ index = date_range(start, end, freq="h", name="idx")
+
+ series = Series(1, index=index)
+ series = series.tz_convert(local_timezone)
+ msg = "Converting to PeriodArray/Index representation will drop timezone"
+ with tm.assert_produces_warning(UserWarning, match=msg):
+ result = series.resample("D").mean().to_period()
+
+ # Create the expected series
+ # Index is moved back a day with the timezone conversion from UTC to
+ # Pacific
+ expected_index = (
+ period_range(start=start, end=end, freq="D", name="idx") - offsets.Day()
+ )
+ expected = Series(1.0, index=expected_index)
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "tz",
+ [
+ zoneinfo.ZoneInfo("America/Los_Angeles"),
+ dateutil.tz.gettz("America/Los_Angeles"),
+ ],
+ )
+ def test_resample_with_tz(self, tz, unit):
+ # GH 13238
+ dti = date_range("2017-01-01", periods=48, freq="h", tz=tz, unit=unit)
+ ser = Series(2, index=dti)
+ result = ser.resample("D").mean()
+ exp_dti = pd.DatetimeIndex(
+ ["2017-01-01", "2017-01-02"], tz=tz, freq="D"
+ ).as_unit(unit)
+ expected = Series(
+ 2.0,
+ index=exp_dti,
+ )
+ tm.assert_series_equal(result, expected)
+
+ def test_resample_nonexistent_time_bin_edge(self):
+ # GH 19375
+ index = date_range("2017-03-12", "2017-03-12 1:45:00", freq="15min")
+ s = Series(np.zeros(len(index)), index=index)
+ expected = s.tz_localize("US/Pacific")
+ expected.index = pd.DatetimeIndex(expected.index, freq="900s")
+ result = expected.resample("900s").mean()
+ tm.assert_series_equal(result, expected)
+
+ def test_resample_nonexistent_time_bin_edge2(self):
+ # GH 23742
+ index = date_range(start="2017-10-10", end="2017-10-20", freq="1h")
+ index = index.tz_localize("UTC").tz_convert("America/Sao_Paulo")
+ df = DataFrame(data=list(range(len(index))), index=index)
+ result = df.groupby(pd.Grouper(freq="1D")).count()
+ expected = date_range(
+ start="2017-10-09",
+ end="2017-10-20",
+ freq="D",
+ tz="America/Sao_Paulo",
+ nonexistent="shift_forward",
+ inclusive="left",
+ )
+ tm.assert_index_equal(result.index, expected)
+
+ def test_resample_ambiguous_time_bin_edge(self):
+ # GH 10117
+ idx = date_range(
+ "2014-10-25 22:00:00",
+ "2014-10-26 00:30:00",
+ freq="30min",
+ tz="Europe/London",
+ )
+ expected = Series(np.zeros(len(idx)), index=idx)
+ result = expected.resample("30min").mean()
+ tm.assert_series_equal(result, expected)
+
+ def test_fill_method_and_how_upsample(self):
+ # GH2073
+ s = Series(
+ np.arange(9, dtype="int64"),
+ index=date_range("2010-01-01", periods=9, freq="QE"),
+ )
+ last = s.resample("ME").ffill()
+ both = s.resample("ME").ffill().resample("ME").last().astype("int64")
+ tm.assert_series_equal(last, both)
+
+ @pytest.mark.parametrize("day", DAYS)
+ @pytest.mark.parametrize("target", ["D", "B"])
+ @pytest.mark.parametrize("convention", ["start", "end"])
+ def test_weekly_upsample(self, day, target, convention, simple_period_range_series):
+ freq = f"W-{day}"
+ ts = simple_period_range_series("1/1/1990", "07/31/1990", freq=freq)
+ warn = None if target == "D" else FutureWarning
+ msg = r"PeriodDtype\[B\] is deprecated"
+ with tm.assert_produces_warning(warn, match=msg):
+ result = ts.resample(target, convention=convention).ffill()
+ expected = result.to_timestamp(target, how=convention)
+ expected = expected.asfreq(target, "ffill").to_period()
+ tm.assert_series_equal(result, expected)
+
+ def test_resample_to_timestamps(self, simple_period_range_series):
+ ts = simple_period_range_series("1/1/1990", "12/31/1995", freq="M")
+
+ result = ts.resample("Y-DEC").mean().to_timestamp()
+ expected = ts.resample("Y-DEC").mean().to_timestamp(how="start")
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize("month", MONTHS)
+ def test_resample_to_quarterly(self, simple_period_range_series, month):
+ ts = simple_period_range_series("1990", "1992", freq=f"Y-{month}")
+ quar_ts = ts.resample(f"Q-{month}").ffill()
+
+ stamps = ts.to_timestamp("D", how="start")
+ qdates = period_range(
+ ts.index[0].asfreq("D", "start"),
+ ts.index[-1].asfreq("D", "end"),
+ freq=f"Q-{month}",
+ )
+
+ expected = stamps.reindex(qdates.to_timestamp("D", "s"), method="ffill")
+ expected.index = qdates
+
+ tm.assert_series_equal(quar_ts, expected)
+
+ @pytest.mark.parametrize("how", ["start", "end"])
+ def test_resample_to_quarterly_start_end(self, simple_period_range_series, how):
+ # conforms, but different month
+ ts = simple_period_range_series("1990", "1992", freq="Y-JUN")
+ result = ts.resample("Q-MAR", convention=how).ffill()
+ expected = ts.asfreq("Q-MAR", how=how)
+ expected = expected.reindex(result.index, method="ffill")
+
+ # FIXME: don't leave commented-out
+ # .to_timestamp('D')
+ # expected = expected.resample('Q-MAR').ffill()
+
+ tm.assert_series_equal(result, expected)
+
+ def test_resample_fill_missing(self):
+ rng = PeriodIndex([2000, 2005, 2007, 2009], freq="Y")
+
+ s = Series(np.random.default_rng(2).standard_normal(4), index=rng)
+
+ stamps = s.to_timestamp()
+ filled = s.resample("Y").ffill()
+ expected = stamps.resample("YE").ffill().to_period("Y")
+ tm.assert_series_equal(filled, expected)
+
+ def test_cant_fill_missing_dups(self):
+ rng = PeriodIndex([2000, 2005, 2005, 2007, 2007], freq="Y")
+ s = Series(np.random.default_rng(2).standard_normal(5), index=rng)
+ msg = "Reindexing only valid with uniquely valued Index objects"
+ with pytest.raises(InvalidIndexError, match=msg):
+ s.resample("Y").ffill()
+
+ def test_resample_5minute(self):
+ rng = period_range("1/1/2000", "1/5/2000", freq="min")
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+ expected = ts.to_timestamp().resample("5min").mean()
+ result = ts.resample("5min").mean().to_timestamp()
+ tm.assert_series_equal(result, expected)
+
+ expected = expected.to_period("5min")
+ result = ts.resample("5min").mean()
+ tm.assert_series_equal(result, expected)
+ result = ts.resample("5min").mean().to_timestamp().to_period()
+ tm.assert_series_equal(result, expected)
+
+ def test_upsample_daily_business_daily(self, simple_period_range_series):
+ ts = simple_period_range_series("1/1/2000", "2/1/2000", freq="B")
+
+ result = ts.resample("D").asfreq()
+ expected = ts.asfreq("D").reindex(period_range("1/3/2000", "2/1/2000"))
+ tm.assert_series_equal(result, expected)
+
+ ts = simple_period_range_series("1/1/2000", "2/1/2000")
+ result = ts.resample("h", convention="s").asfreq()
+ exp_rng = period_range("1/1/2000", "2/1/2000 23:00", freq="h")
+ expected = ts.asfreq("h", how="s").reindex(exp_rng)
+ tm.assert_series_equal(result, expected)
+
+ def test_resample_irregular_sparse(self):
+ dr = date_range(start="1/1/2012", freq="5min", periods=1000)
+ s = Series(np.array(100), index=dr)
+ # subset the data.
+ subset = s[:"2012-01-04 06:55"]
+
+ result = subset.resample("10min").apply(len)
+ expected = s.resample("10min").apply(len).loc[result.index]
+ tm.assert_series_equal(result, expected)
+
+ def test_resample_weekly_all_na(self):
+ rng = date_range("1/1/2000", periods=10, freq="W-WED")
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+
+ result = ts.resample("W-THU").asfreq()
+
+ assert result.isna().all()
+
+ result = ts.resample("W-THU").asfreq().ffill()[:-1]
+ expected = ts.asfreq("W-THU").ffill()
+ tm.assert_series_equal(result, expected)
+
+ def test_resample_tz_localized(self, unit):
+ dr = date_range(start="2012-4-13", end="2012-5-1", unit=unit)
+ ts = Series(range(len(dr)), index=dr)
+
+ ts_utc = ts.tz_localize("UTC")
+ ts_local = ts_utc.tz_convert("America/Los_Angeles")
+
+ result = ts_local.resample("W").mean()
+
+ ts_local_naive = ts_local.copy()
+ ts_local_naive.index = ts_local_naive.index.tz_localize(None)
+
+ exp = ts_local_naive.resample("W").mean().tz_localize("America/Los_Angeles")
+ exp.index = pd.DatetimeIndex(exp.index, freq="W")
+
+ tm.assert_series_equal(result, exp)
+
+ # it works
+ result = ts_local.resample("D").mean()
+
+ def test_resample_tz_localized2(self):
+ # #2245
+ idx = date_range(
+ "2001-09-20 15:59", "2001-09-20 16:00", freq="min", tz="Australia/Sydney"
+ )
+ s = Series([1, 2], index=idx)
+
+ # GH#61985 changed this to behave like "B" rather than "24h"
+ result = s.resample("D", closed="right", label="right").mean()
+ ex_index = date_range("2001-09-20", periods=2, freq="D", tz="Australia/Sydney")
+ expected = Series([np.nan, 1.5], index=ex_index)
+
+ tm.assert_series_equal(result, expected)
+
+ # for good measure
+ msg = "Converting to PeriodArray/Index representation will drop timezone "
+ with tm.assert_produces_warning(UserWarning, match=msg):
+ result = s.resample("D").mean().to_period()
+ ex_index = period_range("2001-09-20", periods=1, freq="D")
+ expected = Series([1.5], index=ex_index)
+ tm.assert_series_equal(result, expected)
+
+ def test_resample_tz_localized3(self):
+ # GH 6397
+ # comparing an offset that doesn't propagate tz's
+ rng = date_range("1/1/2011", periods=20000, freq="h")
+ rng = rng.tz_localize("EST")
+ ts = DataFrame(index=rng)
+ ts["first"] = np.random.default_rng(2).standard_normal(len(rng))
+ ts["second"] = np.cumsum(np.random.default_rng(2).standard_normal(len(rng)))
+ expected = DataFrame(
+ {
+ "first": ts.resample("YE").sum()["first"],
+ "second": ts.resample("YE").mean()["second"],
+ },
+ columns=["first", "second"],
+ )
+ result = (
+ ts.resample("YE")
+ .agg({"first": "sum", "second": "mean"})
+ .reindex(columns=["first", "second"])
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_closed_left_corner(self):
+ # #1465
+ s = Series(
+ np.random.default_rng(2).standard_normal(21),
+ index=date_range(start="1/1/2012 9:30", freq="1min", periods=21),
+ )
+ s.iloc[0] = np.nan
+
+ result = s.resample("10min", closed="left", label="right").mean()
+ exp = s[1:].resample("10min", closed="left", label="right").mean()
+ tm.assert_series_equal(result, exp)
+
+ result = s.resample("10min", closed="left", label="left").mean()
+ exp = s[1:].resample("10min", closed="left", label="left").mean()
+
+ ex_index = date_range(start="1/1/2012 9:30", freq="10min", periods=3)
+
+ tm.assert_index_equal(result.index, ex_index)
+ tm.assert_series_equal(result, exp)
+
+ def test_quarterly_resampling(self):
+ rng = period_range("2000Q1", periods=10, freq="Q-DEC")
+ ts = Series(np.arange(10), index=rng)
+
+ result = ts.resample("Y").mean()
+ exp = ts.to_timestamp().resample("YE").mean().to_period()
+ tm.assert_series_equal(result, exp)
+
+ def test_resample_weekly_bug_1726(self):
+ # 8/6/12 is a Monday
+ ind = date_range(start="8/6/2012", end="8/26/2012", freq="D")
+ n = len(ind)
+ data = [[x] * 5 for x in range(n)]
+ df = DataFrame(data, columns=["open", "high", "low", "close", "vol"], index=ind)
+
+ # it works!
+ df.resample("W-MON", closed="left", label="left").first()
+
+ def test_resample_with_dst_time_change(self):
+ # GH 15549
+ index = (
+ pd.DatetimeIndex([1457537600000000000, 1458059600000000000])
+ .tz_localize("UTC")
+ .tz_convert("America/Chicago")
+ )
+ df = DataFrame([1, 2], index=index)
+ result = df.resample("12h", closed="right", label="right").last().ffill()
+
+ expected_index_values = [
+ "2016-03-09 12:00:00-06:00",
+ "2016-03-10 00:00:00-06:00",
+ "2016-03-10 12:00:00-06:00",
+ "2016-03-11 00:00:00-06:00",
+ "2016-03-11 12:00:00-06:00",
+ "2016-03-12 00:00:00-06:00",
+ "2016-03-12 12:00:00-06:00",
+ "2016-03-13 00:00:00-06:00",
+ "2016-03-13 13:00:00-05:00",
+ "2016-03-14 01:00:00-05:00",
+ "2016-03-14 13:00:00-05:00",
+ "2016-03-15 01:00:00-05:00",
+ "2016-03-15 13:00:00-05:00",
+ ]
+ index = (
+ pd.to_datetime(expected_index_values, utc=True)
+ .tz_convert("America/Chicago")
+ .as_unit(index.unit)
+ )
+ index = pd.DatetimeIndex(index, freq="12h")
+ expected = DataFrame(
+ [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0],
+ index=index,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_resample_bms_2752(self):
+ # GH2753
+ timeseries = Series(
+ index=pd.bdate_range("20000101", "20000201"), dtype=np.float64
+ )
+ res1 = timeseries.resample("BMS").mean()
+ res2 = timeseries.resample("BMS").mean().resample("B").mean()
+ assert res1.index[0] == Timestamp("20000103")
+ assert res1.index[0] == res2.index[0]
+
+ @pytest.mark.xfail(reason="Commented out for more than 3 years. Should this work?")
+ def test_monthly_convention_span(self):
+ rng = period_range("2000-01", periods=3, freq="ME")
+ ts = Series(np.arange(3), index=rng)
+
+ # hacky way to get same thing
+ exp_index = period_range("2000-01-01", "2000-03-31", freq="D")
+ expected = ts.asfreq("D", how="end").reindex(exp_index)
+ expected = expected.fillna(method="bfill")
+
+ result = ts.resample("D").mean()
+
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "from_freq, to_freq", [("D", "ME"), ("QE", "YE"), ("ME", "QE"), ("D", "W")]
+ )
+ def test_default_right_closed_label(self, from_freq, to_freq):
+ idx = date_range(start="8/15/2012", periods=100, freq=from_freq)
+ df = DataFrame(np.random.default_rng(2).standard_normal((len(idx), 2)), idx)
+
+ resampled = df.resample(to_freq).mean()
+ tm.assert_frame_equal(
+ resampled, df.resample(to_freq, closed="right", label="right").mean()
+ )
+
+ @pytest.mark.parametrize(
+ "from_freq, to_freq",
+ [("D", "MS"), ("QE", "YS"), ("ME", "QS"), ("h", "D"), ("min", "h")],
+ )
+ def test_default_left_closed_label(self, from_freq, to_freq):
+ idx = date_range(start="8/15/2012", periods=100, freq=from_freq)
+ df = DataFrame(np.random.default_rng(2).standard_normal((len(idx), 2)), idx)
+
+ resampled = df.resample(to_freq).mean()
+ tm.assert_frame_equal(
+ resampled, df.resample(to_freq, closed="left", label="left").mean()
+ )
+
+ def test_all_values_single_bin(self):
+ # GH#2070
+ index = period_range(start="2012-01-01", end="2012-12-31", freq="M")
+ ser = Series(np.random.default_rng(2).standard_normal(len(index)), index=index)
+
+ result = ser.resample("Y").mean()
+ tm.assert_almost_equal(result.iloc[0], ser.mean())
+
+ def test_evenly_divisible_with_no_extra_bins(self):
+ # GH#4076
+ # when the frequency is evenly divisible, sometimes extra bins
+
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((9, 3)),
+ index=date_range("2000-1-1", periods=9, unit="ns"),
+ )
+ result = df.resample("5D").mean()
+ expected = pd.concat([df.iloc[0:5].mean(), df.iloc[5:].mean()], axis=1).T
+ expected.index = pd.DatetimeIndex(
+ [Timestamp("2000-1-1"), Timestamp("2000-1-6")], dtype="M8[ns]", freq="5D"
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_evenly_divisible_with_no_extra_bins2(self):
+ index = date_range(start="2001-5-4", periods=28)
+ df = DataFrame(
+ [
+ {
+ "REST_KEY": 1,
+ "DLY_TRN_QT": 80,
+ "DLY_SLS_AMT": 90,
+ "COOP_DLY_TRN_QT": 30,
+ "COOP_DLY_SLS_AMT": 20,
+ }
+ ]
+ * 28
+ + [
+ {
+ "REST_KEY": 2,
+ "DLY_TRN_QT": 70,
+ "DLY_SLS_AMT": 10,
+ "COOP_DLY_TRN_QT": 50,
+ "COOP_DLY_SLS_AMT": 20,
+ }
+ ]
+ * 28,
+ index=index.append(index),
+ ).sort_index()
+
+ index = date_range("2001-5-4", periods=4, freq="7D")
+ expected = DataFrame(
+ [
+ {
+ "REST_KEY": 14,
+ "DLY_TRN_QT": 14,
+ "DLY_SLS_AMT": 14,
+ "COOP_DLY_TRN_QT": 14,
+ "COOP_DLY_SLS_AMT": 14,
+ }
+ ]
+ * 4,
+ index=index,
+ )
+ result = df.resample("7D").count()
+ tm.assert_frame_equal(result, expected)
+
+ expected = DataFrame(
+ [
+ {
+ "REST_KEY": 21,
+ "DLY_TRN_QT": 1050,
+ "DLY_SLS_AMT": 700,
+ "COOP_DLY_TRN_QT": 560,
+ "COOP_DLY_SLS_AMT": 280,
+ }
+ ]
+ * 4,
+ index=index,
+ )
+ result = df.resample("7D").sum()
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("freq, period_mult", [("h", 24), ("12h", 2)])
+ def test_upsampling_ohlc(self, freq, period_mult):
+ # GH 13083
+ pi = period_range(start="2000", freq="D", periods=10)
+ s = Series(range(len(pi)), index=pi)
+ expected = s.to_timestamp().resample(freq).ohlc().to_period(freq)
+
+ # timestamp-based resampling doesn't include all sub-periods
+ # of the last original period, so extend accordingly:
+ new_index = period_range(start="2000", freq=freq, periods=period_mult * len(pi))
+ expected = expected.reindex(new_index)
+ result = s.resample(freq).ohlc()
+ tm.assert_frame_equal(result, expected)
+
+ result = s.resample(freq).ohlc().to_timestamp().to_period()
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "periods, values",
+ [
+ (
+ [
+ pd.NaT,
+ "1970-01-01 00:00:00",
+ pd.NaT,
+ "1970-01-01 00:00:02",
+ "1970-01-01 00:00:03",
+ ],
+ [2, 3, 5, 7, 11],
+ ),
+ (
+ [
+ pd.NaT,
+ pd.NaT,
+ "1970-01-01 00:00:00",
+ pd.NaT,
+ pd.NaT,
+ pd.NaT,
+ "1970-01-01 00:00:02",
+ "1970-01-01 00:00:03",
+ pd.NaT,
+ pd.NaT,
+ ],
+ [1, 2, 3, 5, 6, 8, 7, 11, 12, 13],
+ ),
+ ],
+ )
+ @pytest.mark.parametrize(
+ "freq, expected_values",
+ [
+ ("1s", [3, np.nan, 7, 11]),
+ ("2s", [3, (7 + 11) / 2]),
+ ("3s", [(3 + 7) / 2, 11]),
+ ],
+ )
+ def test_resample_with_nat(self, periods, values, freq, expected_values):
+ # GH 13224
+ index = PeriodIndex(periods, freq="s")
+ frame = DataFrame(values, index=index)
+
+ expected_index = period_range(
+ "1970-01-01 00:00:00", periods=len(expected_values), freq=freq
+ )
+ expected = DataFrame(expected_values, index=expected_index)
+ result = frame.resample(freq).mean()
+ tm.assert_frame_equal(result, expected)
+
+ def test_resample_with_only_nat(self):
+ # GH 13224
+ pi = PeriodIndex([pd.NaT] * 3, freq="s")
+ frame = DataFrame([2, 3, 5], index=pi, columns=["a"])
+ expected_index = PeriodIndex(data=[], freq=pi.freq)
+ expected = DataFrame(index=expected_index, columns=["a"], dtype="float64")
+ result = frame.resample("1s").mean()
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "start,end,start_freq,end_freq,offset",
+ [
+ ("19910905", "19910909 03:00", "h", "24h", "10h"),
+ ("19910905", "19910909 12:00", "h", "24h", "10h"),
+ ("19910905", "19910909 23:00", "h", "24h", "10h"),
+ ("19910905 10:00", "19910909", "h", "24h", "10h"),
+ ("19910905 10:00", "19910909 10:00", "h", "24h", "10h"),
+ ("19910905", "19910909 10:00", "h", "24h", "10h"),
+ ("19910905 12:00", "19910909", "h", "24h", "10h"),
+ ("19910905 12:00", "19910909 03:00", "h", "24h", "10h"),
+ ("19910905 12:00", "19910909 12:00", "h", "24h", "10h"),
+ ("19910905 12:00", "19910909 12:00", "h", "24h", "34h"),
+ ("19910905 12:00", "19910909 12:00", "h", "17h", "10h"),
+ ("19910905 12:00", "19910909 12:00", "h", "17h", "3h"),
+ ("19910905", "19910913 06:00", "2h", "24h", "10h"),
+ ("19910905", "19910905 01:39", "Min", "5Min", "3Min"),
+ ("19910905", "19910905 03:18", "2Min", "5Min", "3Min"),
+ ],
+ )
+ def test_resample_with_offset(self, start, end, start_freq, end_freq, offset):
+ # GH 23882 & 31809
+ pi = period_range(start, end, freq=start_freq)
+ ser = Series(np.arange(len(pi)), index=pi)
+ result = ser.resample(end_freq, offset=offset).mean()
+ result = result.to_timestamp(end_freq)
+
+ expected = ser.to_timestamp().resample(end_freq, offset=offset).mean()
+ tm.assert_series_equal(result, expected)
+
+ def test_resample_with_offset_month(self):
+ # GH 23882 & 31809
+ pi = period_range("19910905 12:00", "19910909 1:00", freq="h")
+ ser = Series(np.arange(len(pi)), index=pi)
+ result = ser.resample("M").mean()
+ result = result.to_timestamp("M")
+ expected = ser.to_timestamp().resample("ME").mean()
+ # TODO: is non-tick the relevant characteristic? (GH 33815)
+ expected.index = expected.index._with_freq(None)
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "first,last,freq,freq_to_offset,exp_first,exp_last",
+ [
+ ("19910905", "19920406", "D", "D", "19910905", "19920406"),
+ ("19910905 00:00", "19920406 06:00", "D", "D", "19910905", "19920406"),
+ (
+ "19910905 06:00",
+ "19920406 06:00",
+ "h",
+ "h",
+ "19910905 06:00",
+ "19920406 06:00",
+ ),
+ ("19910906", "19920406", "M", "ME", "1991-09", "1992-04"),
+ ("19910831", "19920430", "M", "ME", "1991-08", "1992-04"),
+ ("1991-08", "1992-04", "M", "ME", "1991-08", "1992-04"),
+ ],
+ )
+ def test_get_period_range_edges(
+ self, first, last, freq, freq_to_offset, exp_first, exp_last
+ ):
+ first = Period(first)
+ last = Period(last)
+
+ exp_first = Period(exp_first, freq=freq)
+ exp_last = Period(exp_last, freq=freq)
+
+ freq = pd.tseries.frequencies.to_offset(freq_to_offset)
+ result = _get_period_range_edges(first, last, freq)
+ expected = (exp_first, exp_last)
+ assert result == expected
+
+ def test_sum_min_count(self):
+ # GH 19974
+ index = date_range(start="2018", freq="ME", periods=6)
+ data = np.ones(6)
+ data[3:6] = np.nan
+ s = Series(data, index).to_period()
+ result = s.resample("Q").sum(min_count=1)
+ expected = Series(
+ [3.0, np.nan], index=PeriodIndex(["2018Q1", "2018Q2"], freq="Q-DEC")
+ )
+ tm.assert_series_equal(result, expected)
+
+ def test_resample_t_l_deprecated(self):
+ # GH#52536
+ msg_t = "Invalid frequency: T"
+ msg_l = "Invalid frequency: L"
+
+ with pytest.raises(ValueError, match=msg_l):
+ period_range(
+ "2020-01-01 00:00:00 00:00", "2020-01-01 00:00:00 00:01", freq="L"
+ )
+ rng_l = period_range(
+ "2020-01-01 00:00:00 00:00", "2020-01-01 00:00:00 00:01", freq="ms"
+ )
+ ser = Series(np.arange(len(rng_l)), index=rng_l)
+
+ with pytest.raises(ValueError, match=msg_t):
+ ser.resample("T").mean()
+
+ @pytest.mark.parametrize(
+ "freq, freq_depr, freq_depr_res",
+ [
+ ("2Q", "2q", "2y"),
+ ("2M", "2m", "2q"),
+ ],
+ )
+ def test_resample_lowercase_frequency_raises(self, freq, freq_depr, freq_depr_res):
+ msg = f"Invalid frequency: {freq_depr}"
+ with pytest.raises(ValueError, match=msg):
+ period_range("2020-01-01", "2020-08-01", freq=freq_depr)
+
+ msg = f"Invalid frequency: {freq_depr_res}"
+ rng = period_range("2020-01-01", "2020-08-01", freq=freq)
+ ser = Series(np.arange(len(rng)), index=rng)
+ with pytest.raises(ValueError, match=msg):
+ ser.resample(freq_depr_res).mean()
+
+ @pytest.mark.parametrize(
+ "offset",
+ [
+ offsets.MonthBegin(),
+ offsets.BYearBegin(2),
+ offsets.BusinessHour(2),
+ ],
+ )
+ def test_asfreq_invalid_period_offset(self, offset, frame_or_series):
+ # GH#55785
+ msg = re.escape(f"{offset} is not supported as period frequency")
+
+ obj = frame_or_series(range(5), index=period_range("2020-01-01", periods=5))
+ with pytest.raises(ValueError, match=msg):
+ obj.asfreq(freq=offset)
+
+
+@pytest.mark.parametrize(
+ "freq",
+ [
+ ("2ME"),
+ ("2QE"),
+ ("2QE-FEB"),
+ ("2YE"),
+ ("2YE-MAR"),
+ ("2me"),
+ ("2qe"),
+ ("2ye-mar"),
+ ],
+)
+def test_resample_frequency_ME_QE_YE_raises(frame_or_series, freq):
+ # GH#9586
+ msg = f"{freq[1:]} is not supported as period frequency"
+
+ obj = frame_or_series(range(5), index=period_range("2020-01-01", periods=5))
+ msg = f"Invalid frequency: {freq}"
+ with pytest.raises(ValueError, match=msg):
+ obj.resample(freq)
+
+
+def test_corner_cases_period(simple_period_range_series):
+ # miscellaneous test coverage
+ len0pts = simple_period_range_series("2007-01", "2010-05", freq="M")[:0]
+ # it works
+ result = len0pts.resample("Y-DEC").mean()
+ assert len(result) == 0
+
+
+@pytest.mark.parametrize("freq", ["2BME", "2CBME", "2SME", "2BQE-FEB", "2BYE-MAR"])
+def test_resample_frequency_invalid_freq(frame_or_series, freq):
+ # GH#9586
+ msg = f"Invalid frequency: {freq}"
+
+ obj = frame_or_series(range(5), index=period_range("2020-01-01", periods=5))
+ with pytest.raises(ValueError, match=msg):
+ obj.resample(freq)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_resample_api.py b/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_resample_api.py
new file mode 100644
index 0000000000000000000000000000000000000000..36ef01178b3bc905bc4099fe6cc3e271f3872b5f
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_resample_api.py
@@ -0,0 +1,1018 @@
+from datetime import datetime
+import re
+
+import numpy as np
+import pytest
+
+from pandas._libs import lib
+from pandas._libs.tslibs import Day
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ NamedAgg,
+ Series,
+)
+import pandas._testing as tm
+from pandas.core.indexes.datetimes import date_range
+
+
+@pytest.fixture
+def dti():
+ return date_range(start=datetime(2005, 1, 1), end=datetime(2005, 1, 10), freq="Min")
+
+
+@pytest.fixture
+def _test_series(dti):
+ return Series(np.random.default_rng(2).random(len(dti)), dti)
+
+
+@pytest.fixture
+def test_frame(dti, _test_series):
+ return DataFrame({"A": _test_series, "B": _test_series, "C": np.arange(len(dti))})
+
+
+def test_str(_test_series):
+ r = _test_series.resample("h")
+ assert (
+ "DatetimeIndexResampler [freq=, closed=left, "
+ "label=left, convention=start, origin=start_day]" in str(r)
+ )
+
+ r = _test_series.resample("h", origin="2000-01-01")
+ assert (
+ "DatetimeIndexResampler [freq=, closed=left, "
+ "label=left, convention=start, origin=2000-01-01 00:00:00]" in str(r)
+ )
+
+
+def test_api(_test_series):
+ r = _test_series.resample("h")
+ result = r.mean()
+ assert isinstance(result, Series)
+ assert len(result) == 217
+
+ r = _test_series.to_frame().resample("h")
+ result = r.mean()
+ assert isinstance(result, DataFrame)
+ assert len(result) == 217
+
+
+def test_groupby_resample_api():
+ # GH 12448
+ # .groupby(...).resample(...) hitting warnings
+ # when appropriate
+ df = DataFrame(
+ {
+ "date": date_range(start="2016-01-01", periods=4, freq="W"),
+ "group": [1, 1, 2, 2],
+ "val": [5, 6, 7, 8],
+ }
+ ).set_index("date")
+
+ # replication step
+ i = (
+ date_range("2016-01-03", periods=8).tolist()
+ + date_range("2016-01-17", periods=8).tolist()
+ )
+ index = pd.MultiIndex.from_arrays([[1] * 8 + [2] * 8, i], names=["group", "date"])
+ expected = DataFrame({"val": [5] * 7 + [6] + [7] * 7 + [8]}, index=index)
+ result = df.groupby("group").apply(lambda x: x.resample("1D").ffill())[["val"]]
+ tm.assert_frame_equal(result, expected)
+
+
+def test_groupby_resample_on_api():
+ # GH 15021
+ # .groupby(...).resample(on=...) results in an unexpected
+ # keyword warning.
+ df = DataFrame(
+ {
+ "key": ["A", "B"] * 5,
+ "dates": date_range("2016-01-01", periods=10),
+ "values": np.random.default_rng(2).standard_normal(10),
+ }
+ )
+
+ expected = df.set_index("dates").groupby("key").resample("D").mean()
+ result = df.groupby("key").resample("D", on="dates").mean()
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_group_keys():
+ df = DataFrame({"A": 1, "B": 2}, index=date_range("2000", periods=10, unit="ns"))
+ expected = df.copy()
+
+ # group_keys=False
+ g = df.resample("5D", group_keys=False)
+ result = g.apply(lambda x: x)
+ tm.assert_frame_equal(result, expected)
+
+ # group_keys defaults to False
+ g = df.resample("5D")
+ result = g.apply(lambda x: x)
+ tm.assert_frame_equal(result, expected)
+
+ # group_keys=True
+ expected.index = pd.MultiIndex.from_arrays(
+ [
+ pd.to_datetime(["2000-01-01", "2000-01-06"]).as_unit("ns").repeat(5),
+ expected.index,
+ ]
+ )
+ g = df.resample("5D", group_keys=True)
+ result = g.apply(lambda x: x)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_pipe(test_frame, _test_series):
+ # GH17905
+
+ # series
+ r = _test_series.resample("h")
+ expected = r.max() - r.mean()
+ result = r.pipe(lambda x: x.max() - x.mean())
+ tm.assert_series_equal(result, expected)
+
+ # dataframe
+ r = test_frame.resample("h")
+ expected = r.max() - r.mean()
+ result = r.pipe(lambda x: x.max() - x.mean())
+ tm.assert_frame_equal(result, expected)
+
+
+def test_getitem(test_frame):
+ r = test_frame.resample("h")
+ tm.assert_index_equal(r._selected_obj.columns, test_frame.columns)
+
+ r = test_frame.resample("h")["B"]
+ assert r._selected_obj.name == test_frame.columns[1]
+
+ # technically this is allowed
+ r = test_frame.resample("h")["A", "B"]
+ tm.assert_index_equal(r._selected_obj.columns, test_frame.columns[[0, 1]])
+
+ r = test_frame.resample("h")["A", "B"]
+ tm.assert_index_equal(r._selected_obj.columns, test_frame.columns[[0, 1]])
+
+
+@pytest.mark.parametrize("key", [["D"], ["A", "D"]])
+def test_select_bad_cols(key, test_frame):
+ g = test_frame.resample("h")
+ # 'A' should not be referenced as a bad column...
+ # will have to rethink regex if you change message!
+ msg = r"^\"Columns not found: 'D'\"$"
+ with pytest.raises(KeyError, match=msg):
+ g[key]
+
+
+def test_attribute_access(test_frame):
+ r = test_frame.resample("h")
+ tm.assert_series_equal(r.A.sum(), r["A"].sum())
+
+
+@pytest.mark.parametrize("attr", ["groups", "ngroups", "indices"])
+def test_api_compat_before_use(attr):
+ # make sure that we are setting the binner
+ # on these attributes
+ rng = date_range("1/1/2012", periods=100, freq="s")
+ ts = Series(np.arange(len(rng)), index=rng)
+ rs = ts.resample("30s")
+
+ # before use
+ getattr(rs, attr)
+
+ # after grouper is initialized is ok
+ rs.mean()
+ getattr(rs, attr)
+
+
+def tests_raises_on_nuisance(test_frame, using_infer_string):
+ df = test_frame
+ df["D"] = "foo"
+ r = df.resample("h")
+ result = r[["A", "B"]].mean()
+ expected = pd.concat([r.A.mean(), r.B.mean()], axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ expected = r[["A", "B", "C"]].mean()
+ msg = re.escape("agg function failed [how->mean,dtype->")
+ if using_infer_string:
+ msg = "dtype 'str' does not support operation 'mean'"
+ with pytest.raises(TypeError, match=msg):
+ r.mean()
+ result = r.mean(numeric_only=True)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_downsample_but_actually_upsampling():
+ # this is reindex / asfreq
+ rng = date_range("1/1/2012", periods=100, freq="s")
+ ts = Series(np.arange(len(rng), dtype="int64"), index=rng)
+ result = ts.resample("20s").asfreq()
+ expected = Series(
+ [0, 20, 40, 60, 80],
+ index=date_range("2012-01-01 00:00:00", freq="20s", periods=5),
+ )
+ tm.assert_series_equal(result, expected)
+
+
+def test_combined_up_downsampling_of_irregular():
+ # since we are really doing an operation like this
+ # ts2.resample('2s').mean().ffill()
+ # preserve these semantics
+
+ rng = date_range("1/1/2012", periods=100, freq="s", unit="ns")
+ ts = Series(np.arange(len(rng)), index=rng)
+ ts2 = ts.iloc[[0, 1, 2, 3, 5, 7, 11, 15, 16, 25, 30]]
+
+ result = ts2.resample("2s").mean().ffill()
+ expected = Series(
+ [
+ 0.5,
+ 2.5,
+ 5.0,
+ 7.0,
+ 7.0,
+ 11.0,
+ 11.0,
+ 15.0,
+ 16.0,
+ 16.0,
+ 16.0,
+ 16.0,
+ 25.0,
+ 25.0,
+ 25.0,
+ 30.0,
+ ],
+ index=pd.DatetimeIndex(
+ [
+ "2012-01-01 00:00:00",
+ "2012-01-01 00:00:02",
+ "2012-01-01 00:00:04",
+ "2012-01-01 00:00:06",
+ "2012-01-01 00:00:08",
+ "2012-01-01 00:00:10",
+ "2012-01-01 00:00:12",
+ "2012-01-01 00:00:14",
+ "2012-01-01 00:00:16",
+ "2012-01-01 00:00:18",
+ "2012-01-01 00:00:20",
+ "2012-01-01 00:00:22",
+ "2012-01-01 00:00:24",
+ "2012-01-01 00:00:26",
+ "2012-01-01 00:00:28",
+ "2012-01-01 00:00:30",
+ ],
+ dtype="datetime64[ns]",
+ freq="2s",
+ ),
+ )
+ tm.assert_series_equal(result, expected)
+
+
+def test_transform_series(_test_series):
+ r = _test_series.resample("20min")
+ expected = _test_series.groupby(pd.Grouper(freq="20min")).transform("mean")
+ result = r.transform("mean")
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("on", [None, "date"])
+def test_transform_frame(on):
+ # GH#47079
+ index = date_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="D")
+ index.name = "date"
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 2)), columns=list("AB"), index=index
+ )
+ expected = df.groupby(pd.Grouper(freq="20min")).transform("mean")
+ if on == "date":
+ # Move date to being a column; result will then have a RangeIndex
+ expected = expected.reset_index(drop=True)
+ df = df.reset_index()
+
+ r = df.resample("20min", on=on)
+ result = r.transform("mean")
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "func",
+ [
+ lambda x: x.resample("20min", group_keys=False),
+ lambda x: x.groupby(pd.Grouper(freq="20min"), group_keys=False),
+ ],
+ ids=["resample", "groupby"],
+)
+def test_apply_without_aggregation(func, _test_series):
+ # both resample and groupby should work w/o aggregation
+ t = func(_test_series)
+ result = t.apply(lambda x: x)
+ tm.assert_series_equal(result, _test_series)
+
+
+def test_apply_without_aggregation2(_test_series):
+ grouped = _test_series.to_frame(name="foo").resample("20min", group_keys=False)
+ result = grouped["foo"].apply(lambda x: x)
+ tm.assert_series_equal(result, _test_series.rename("foo"))
+
+
+def test_agg_consistency():
+ # make sure that we are consistent across
+ # similar aggregations with and w/o selection list
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((1000, 3)),
+ index=date_range("1/1/2012", freq="s", periods=1000),
+ columns=["A", "B", "C"],
+ )
+
+ r = df.resample("3min")
+
+ msg = r"Label\(s\) \['r1', 'r2'\] do not exist"
+ with pytest.raises(KeyError, match=msg):
+ r.agg({"r1": "mean", "r2": "sum"})
+
+
+def test_agg_consistency_int_str_column_mix():
+ # GH#39025
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((1000, 2)),
+ index=date_range("1/1/2012", freq="s", periods=1000),
+ columns=[1, "a"],
+ )
+
+ r = df.resample("3min")
+
+ msg = r"Label\(s\) \[2, 'b'\] do not exist"
+ with pytest.raises(KeyError, match=msg):
+ r.agg({2: "mean", "b": "sum"})
+
+
+# TODO(GH#14008): once GH 14008 is fixed, move these tests into
+# `Base` test class
+
+
+@pytest.fixture
+def index():
+ index = date_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="D", unit="ns")
+ index.name = "date"
+ return index
+
+
+@pytest.fixture
+def df(index):
+ frame = DataFrame(
+ np.random.default_rng(2).random((10, 2)), columns=list("AB"), index=index
+ )
+ return frame
+
+
+@pytest.fixture
+def df_col(df):
+ return df.reset_index()
+
+
+@pytest.fixture
+def df_mult(df_col, index):
+ df_mult = df_col.copy()
+ df_mult.index = pd.MultiIndex.from_arrays(
+ [range(10), index], names=["index", "date"]
+ )
+ return df_mult
+
+
+@pytest.fixture
+def a_mean(df):
+ return df.resample("2D")["A"].mean()
+
+
+@pytest.fixture
+def a_std(df):
+ return df.resample("2D")["A"].std()
+
+
+@pytest.fixture
+def a_sum(df):
+ return df.resample("2D")["A"].sum()
+
+
+@pytest.fixture
+def b_mean(df):
+ return df.resample("2D")["B"].mean()
+
+
+@pytest.fixture
+def b_std(df):
+ return df.resample("2D")["B"].std()
+
+
+@pytest.fixture
+def b_sum(df):
+ return df.resample("2D")["B"].sum()
+
+
+@pytest.fixture
+def df_resample(df):
+ return df.resample("2D")
+
+
+@pytest.fixture
+def df_col_resample(df_col):
+ return df_col.resample("2D", on="date")
+
+
+@pytest.fixture
+def df_mult_resample(df_mult):
+ return df_mult.resample("2D", level="date")
+
+
+@pytest.fixture
+def df_grouper_resample(df):
+ return df.groupby(pd.Grouper(freq="2D"))
+
+
+@pytest.fixture(
+ params=["df_resample", "df_col_resample", "df_mult_resample", "df_grouper_resample"]
+)
+def cases(request):
+ return request.getfixturevalue(request.param)
+
+
+def test_agg_mixed_column_aggregation(cases, a_mean, a_std, b_mean, b_std, request):
+ expected = pd.concat([a_mean, a_std, b_mean, b_std], axis=1)
+ expected.columns = pd.MultiIndex.from_product([["A", "B"], ["mean", ""]])
+ # "date" is an index and a column, so get included in the agg
+ if "df_mult" in request.node.callspec.id:
+ date_mean = cases["date"].mean()
+ date_std = cases["date"].std()
+ expected = pd.concat([date_mean, date_std, expected], axis=1)
+ expected.columns = pd.MultiIndex.from_product(
+ [["date", "A", "B"], ["mean", ""]]
+ )
+ result = cases.aggregate([np.mean, lambda x: np.std(x, ddof=1)])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "agg",
+ [
+ {"func": {"A": np.mean, "B": lambda x: np.std(x, ddof=1)}},
+ {"A": ("A", np.mean), "B": ("B", lambda x: np.std(x, ddof=1))},
+ {"A": NamedAgg("A", np.mean), "B": NamedAgg("B", lambda x: np.std(x, ddof=1))},
+ ],
+)
+def test_agg_both_mean_std_named_result(cases, a_mean, b_std, agg):
+ expected = pd.concat([a_mean, b_std], axis=1)
+ result = cases.aggregate(**agg)
+ tm.assert_frame_equal(result, expected, check_like=True)
+
+
+def test_agg_both_mean_std_dict_of_list(cases, a_mean, a_std):
+ expected = pd.concat([a_mean, a_std], axis=1)
+ expected.columns = pd.MultiIndex.from_tuples([("A", "mean"), ("A", "std")])
+ result = cases.aggregate({"A": ["mean", "std"]})
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "agg", [{"func": ["mean", "sum"]}, {"mean": "mean", "sum": "sum"}]
+)
+def test_agg_both_mean_sum(cases, a_mean, a_sum, agg):
+ expected = pd.concat([a_mean, a_sum], axis=1)
+ expected.columns = ["mean", "sum"]
+ result = cases["A"].aggregate(**agg)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "agg",
+ [
+ {"A": {"mean": "mean", "sum": "sum"}},
+ {
+ "A": {"mean": "mean", "sum": "sum"},
+ "B": {"mean2": "mean", "sum2": "sum"},
+ },
+ ],
+)
+def test_agg_dict_of_dict_specificationerror(cases, agg):
+ msg = "nested renamer is not supported"
+ with pytest.raises(pd.errors.SpecificationError, match=msg):
+ cases.aggregate(agg)
+
+
+def test_agg_dict_of_lists(cases, a_mean, a_std, b_mean, b_std):
+ expected = pd.concat([a_mean, a_std, b_mean, b_std], axis=1)
+ expected.columns = pd.MultiIndex.from_tuples(
+ [("A", "mean"), ("A", "std"), ("B", "mean"), ("B", "std")]
+ )
+ result = cases.aggregate({"A": ["mean", "std"], "B": ["mean", "std"]})
+ tm.assert_frame_equal(result, expected, check_like=True)
+
+
+@pytest.mark.parametrize(
+ "agg",
+ [
+ {"func": {"A": np.sum, "B": lambda x: np.std(x, ddof=1)}},
+ {"A": ("A", np.sum), "B": ("B", lambda x: np.std(x, ddof=1))},
+ {"A": NamedAgg("A", np.sum), "B": NamedAgg("B", lambda x: np.std(x, ddof=1))},
+ ],
+)
+def test_agg_with_lambda(cases, agg):
+ # passed lambda
+ rcustom = cases["B"].apply(lambda x: np.std(x, ddof=1))
+ expected = pd.concat([cases["A"].sum(), rcustom], axis=1)
+ result = cases.agg(**agg)
+ tm.assert_frame_equal(result, expected, check_like=True)
+
+
+@pytest.mark.parametrize(
+ "agg",
+ [
+ {"func": {"result1": np.sum, "result2": np.mean}},
+ {"A": ("result1", np.sum), "B": ("result2", np.mean)},
+ {"A": NamedAgg("result1", np.sum), "B": NamedAgg("result2", np.mean)},
+ ],
+)
+def test_agg_no_column(cases, agg):
+ msg = r"Label\(s\) \['result1', 'result2'\] do not exist"
+ with pytest.raises(KeyError, match=msg):
+ cases[["A", "B"]].agg(**agg)
+
+
+@pytest.mark.parametrize(
+ "cols, agg",
+ [
+ [None, {"A": ["sum", "std"], "B": ["mean", "std"]}],
+ [
+ [
+ "A",
+ "B",
+ ],
+ {"A": ["sum", "std"], "B": ["mean", "std"]},
+ ],
+ ],
+)
+def test_agg_specificationerror_nested(cases, cols, agg, a_sum, a_std, b_mean, b_std):
+ # agg with different hows
+ # equivalent of using a selection list / or not
+ expected = pd.concat([a_sum, a_std, b_mean, b_std], axis=1)
+ expected.columns = pd.MultiIndex.from_tuples(
+ [("A", "sum"), ("A", "std"), ("B", "mean"), ("B", "std")]
+ )
+ if cols is not None:
+ obj = cases[cols]
+ else:
+ obj = cases
+
+ result = obj.agg(agg)
+ tm.assert_frame_equal(result, expected, check_like=True)
+
+
+@pytest.mark.parametrize(
+ "agg", [{"A": ["sum", "std"]}, {"A": ["sum", "std"], "B": ["mean", "std"]}]
+)
+def test_agg_specificationerror_series(cases, agg):
+ msg = "nested renamer is not supported"
+
+ # series like aggs
+ with pytest.raises(pd.errors.SpecificationError, match=msg):
+ cases["A"].agg(agg)
+
+
+def test_agg_specificationerror_invalid_names(cases):
+ # errors
+ # invalid names in the agg specification
+ msg = r"Label\(s\) \['B'\] do not exist"
+ with pytest.raises(KeyError, match=msg):
+ cases[["A"]].agg({"A": ["sum", "std"], "B": ["mean", "std"]})
+
+
+def test_agg_nested_dicts():
+ index = date_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="D")
+ index.name = "date"
+ df = DataFrame(
+ np.random.default_rng(2).random((10, 2)), columns=list("AB"), index=index
+ )
+ df_col = df.reset_index()
+ df_mult = df_col.copy()
+ df_mult.index = pd.MultiIndex.from_arrays(
+ [range(10), df.index], names=["index", "date"]
+ )
+ r = df.resample("2D")
+ cases = [
+ r,
+ df_col.resample("2D", on="date"),
+ df_mult.resample("2D", level="date"),
+ df.groupby(pd.Grouper(freq="2D")),
+ ]
+
+ msg = "nested renamer is not supported"
+ for t in cases:
+ with pytest.raises(pd.errors.SpecificationError, match=msg):
+ t.aggregate({"r1": {"A": ["mean", "sum"]}, "r2": {"B": ["mean", "sum"]}})
+
+ for t in cases:
+ with pytest.raises(pd.errors.SpecificationError, match=msg):
+ t[["A", "B"]].agg(
+ {"A": {"ra": ["mean", "std"]}, "B": {"rb": ["mean", "std"]}}
+ )
+
+ with pytest.raises(pd.errors.SpecificationError, match=msg):
+ t.agg({"A": {"ra": ["mean", "std"]}, "B": {"rb": ["mean", "std"]}})
+
+
+def test_try_aggregate_non_existing_column():
+ # GH 16766
+ data = [
+ {"dt": datetime(2017, 6, 1, 0), "x": 1.0, "y": 2.0},
+ {"dt": datetime(2017, 6, 1, 1), "x": 2.0, "y": 2.0},
+ {"dt": datetime(2017, 6, 1, 2), "x": 3.0, "y": 1.5},
+ ]
+ df = DataFrame(data).set_index("dt")
+
+ # Error as we don't have 'z' column
+ msg = r"Label\(s\) \['z'\] do not exist"
+ with pytest.raises(KeyError, match=msg):
+ df.resample("30min").agg({"x": ["mean"], "y": ["median"], "z": ["sum"]})
+
+
+def test_agg_list_like_func_with_args():
+ # 50624
+ df = DataFrame(
+ {"x": [1, 2, 3]}, index=date_range("2020-01-01", periods=3, freq="D")
+ )
+
+ def foo1(x, a=1, c=0):
+ return x + a + c
+
+ def foo2(x, b=2, c=0):
+ return x + b + c
+
+ msg = r"foo1\(\) got an unexpected keyword argument 'b'"
+ with pytest.raises(TypeError, match=msg):
+ df.resample("D").agg([foo1, foo2], 3, b=3, c=4)
+
+ result = df.resample("D").agg([foo1, foo2], 3, c=4)
+ expected = DataFrame(
+ [[8, 8], [9, 9], [10, 10]],
+ index=date_range("2020-01-01", periods=3, freq="D"),
+ columns=pd.MultiIndex.from_tuples([("x", "foo1"), ("x", "foo2")]),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_selection_api_validation():
+ # GH 13500
+ index = date_range(datetime(2005, 1, 1), datetime(2005, 1, 10), freq="D")
+
+ rng = np.arange(len(index), dtype=np.int64)
+ df = DataFrame(
+ {"date": index, "a": rng},
+ index=pd.MultiIndex.from_arrays([rng, index], names=["v", "d"]),
+ )
+ df_exp = DataFrame({"a": rng}, index=index)
+
+ # non DatetimeIndex
+ msg = (
+ "Only valid with DatetimeIndex, TimedeltaIndex or PeriodIndex, "
+ "but got an instance of 'Index'"
+ )
+ with pytest.raises(TypeError, match=msg):
+ df.resample("2D", level="v")
+
+ msg = "The Grouper cannot specify both a key and a level!"
+ with pytest.raises(ValueError, match=msg):
+ df.resample("2D", on="date", level="d")
+
+ msg = "unhashable type: 'list'"
+ with pytest.raises(TypeError, match=msg):
+ df.resample("2D", on=["a", "date"])
+
+ msg = r"\"Level \['a', 'date'\] not found\""
+ with pytest.raises(KeyError, match=msg):
+ df.resample("2D", level=["a", "date"])
+
+ # upsampling not allowed
+ msg = (
+ "Upsampling from level= or on= selection is not supported, use "
+ r"\.set_index\(\.\.\.\) to explicitly set index to datetime-like"
+ )
+ with pytest.raises(ValueError, match=msg):
+ df.resample("2D", level="d").asfreq()
+ with pytest.raises(ValueError, match=msg):
+ df.resample("2D", on="date").asfreq()
+
+ exp = df_exp.resample("2D").sum()
+ exp.index.name = "date"
+ result = df.resample("2D", on="date").sum()
+ tm.assert_frame_equal(exp, result)
+
+ exp.index.name = "d"
+ with pytest.raises(
+ TypeError, match="datetime64 type does not support operation 'sum'"
+ ):
+ df.resample("2D", level="d").sum()
+ result = df.resample("2D", level="d").sum(numeric_only=True)
+ tm.assert_frame_equal(exp, result)
+
+
+@pytest.mark.parametrize(
+ "col_name", ["t2", "t2x", "t2q", "T_2M", "t2p", "t2m", "t2m1", "T2M"]
+)
+def test_agg_with_datetime_index_list_agg_func(col_name):
+ # GH 22660
+ # The parametrized column names would get converted to dates by our
+ # date parser. Some would result in OutOfBoundsError (ValueError) while
+ # others would result in OverflowError when passed into Timestamp.
+ # We catch these errors and move on to the correct branch.
+ df = DataFrame(
+ list(range(200)),
+ index=date_range(
+ start="2017-01-01", freq="15min", periods=200, tz="Europe/Berlin"
+ ),
+ columns=[col_name],
+ )
+ result = df.resample("1D").aggregate(["mean"])
+ expected = DataFrame(
+ [47.5, 143.5, 195.5],
+ index=date_range(start="2017-01-01", freq="D", periods=3, tz="Europe/Berlin"),
+ columns=pd.MultiIndex(levels=[[col_name], ["mean"]], codes=[[0], [0]]),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_agg_readonly():
+ # GH#31710 cython needs to allow readonly data
+ index = date_range("2020-01-01", "2020-01-02", freq="1h", unit="ns")
+ arr = np.zeros_like(index)
+ arr.setflags(write=False)
+
+ ser = Series(arr, index=index)
+ rs = ser.resample("1D")
+
+ expected = Series([pd.Timestamp(0), pd.Timestamp(0)], index=index[::24])
+ expected.index.freq = Day(1) # GH#41943 no longer equivalent to 24h
+
+ result = rs.agg("last")
+ tm.assert_series_equal(result, expected)
+
+ result = rs.agg("first")
+ tm.assert_series_equal(result, expected)
+
+ result = rs.agg("max")
+ tm.assert_series_equal(result, expected)
+
+ result = rs.agg("min")
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "start,end,freq,data,resample_freq,origin,closed,exp_data,exp_end,exp_periods",
+ [
+ (
+ "2000-10-01 23:30:00",
+ "2000-10-02 00:26:00",
+ "7min",
+ [0, 3, 6, 9, 12, 15, 18, 21, 24],
+ "17min",
+ "end",
+ None,
+ [0, 18, 27, 63],
+ "20001002 00:26:00",
+ 4,
+ ),
+ (
+ "20200101 8:26:35",
+ "20200101 9:31:58",
+ "77s",
+ [1] * 51,
+ "7min",
+ "end",
+ "right",
+ [1, 6, 5, 6, 5, 6, 5, 6, 5, 6],
+ "2020-01-01 09:30:45",
+ 10,
+ ),
+ (
+ "2000-10-01 23:30:00",
+ "2000-10-02 00:26:00",
+ "7min",
+ [0, 3, 6, 9, 12, 15, 18, 21, 24],
+ "17min",
+ "end",
+ "left",
+ [0, 18, 27, 39, 24],
+ "20001002 00:43:00",
+ 5,
+ ),
+ (
+ "2000-10-01 23:30:00",
+ "2000-10-02 00:26:00",
+ "7min",
+ [0, 3, 6, 9, 12, 15, 18, 21, 24],
+ "17min",
+ "end_day",
+ None,
+ [3, 15, 45, 45],
+ "2000-10-02 00:29:00",
+ 4,
+ ),
+ ],
+)
+def test_end_and_end_day_origin(
+ start,
+ end,
+ freq,
+ data,
+ resample_freq,
+ origin,
+ closed,
+ exp_data,
+ exp_end,
+ exp_periods,
+):
+ rng = date_range(start, end, freq=freq)
+ ts = Series(data, index=rng)
+
+ res = ts.resample(resample_freq, origin=origin, closed=closed).sum()
+ expected = Series(
+ exp_data,
+ index=date_range(end=exp_end, freq=resample_freq, periods=exp_periods),
+ )
+
+ tm.assert_series_equal(res, expected)
+
+
+@pytest.mark.parametrize(
+ # expected_data is a string when op raises a ValueError
+ "method, numeric_only, expected_data",
+ [
+ ("sum", True, {"num": [25]}),
+ ("sum", False, {"cat": ["cat_1cat_2"], "num": [25]}),
+ ("sum", lib.no_default, {"cat": ["cat_1cat_2"], "num": [25]}),
+ ("prod", True, {"num": [100]}),
+ ("prod", False, "can't multiply sequence"),
+ ("prod", lib.no_default, "can't multiply sequence"),
+ ("min", True, {"num": [5]}),
+ ("min", False, {"cat": ["cat_1"], "num": [5]}),
+ ("min", lib.no_default, {"cat": ["cat_1"], "num": [5]}),
+ ("max", True, {"num": [20]}),
+ ("max", False, {"cat": ["cat_2"], "num": [20]}),
+ ("max", lib.no_default, {"cat": ["cat_2"], "num": [20]}),
+ ("first", True, {"num": [5]}),
+ ("first", False, {"cat": ["cat_1"], "num": [5]}),
+ ("first", lib.no_default, {"cat": ["cat_1"], "num": [5]}),
+ ("last", True, {"num": [20]}),
+ ("last", False, {"cat": ["cat_2"], "num": [20]}),
+ ("last", lib.no_default, {"cat": ["cat_2"], "num": [20]}),
+ ("mean", True, {"num": [12.5]}),
+ ("mean", False, "Could not convert"),
+ ("mean", lib.no_default, "Could not convert"),
+ ("median", True, {"num": [12.5]}),
+ ("median", False, r"Cannot convert \['cat_1' 'cat_2'\] to numeric"),
+ ("median", lib.no_default, r"Cannot convert \['cat_1' 'cat_2'\] to numeric"),
+ ("std", True, {"num": [10.606601717798213]}),
+ ("std", False, "could not convert string to float"),
+ ("std", lib.no_default, "could not convert string to float"),
+ ("var", True, {"num": [112.5]}),
+ ("var", False, "could not convert string to float"),
+ ("var", lib.no_default, "could not convert string to float"),
+ ("sem", True, {"num": [7.5]}),
+ ("sem", False, "could not convert string to float"),
+ ("sem", lib.no_default, "could not convert string to float"),
+ ],
+)
+def test_frame_downsample_method(
+ method, numeric_only, expected_data, using_infer_string
+):
+ # GH#46442 test if `numeric_only` behave as expected for DataFrameGroupBy
+
+ index = date_range("2018-01-01", periods=2, freq="D")
+ expected_index = date_range("2018-12-31", periods=1, freq="YE")
+ df = DataFrame({"cat": ["cat_1", "cat_2"], "num": [5, 20]}, index=index)
+ resampled = df.resample("YE")
+ if numeric_only is lib.no_default:
+ kwargs = {}
+ else:
+ kwargs = {"numeric_only": numeric_only}
+
+ func = getattr(resampled, method)
+ if isinstance(expected_data, str):
+ if method in ("var", "mean", "median", "prod"):
+ klass = TypeError
+ msg = re.escape(f"agg function failed [how->{method},dtype->")
+ if using_infer_string:
+ msg = f"dtype 'str' does not support operation '{method}'"
+ elif method in ["sum", "std", "sem"] and using_infer_string:
+ klass = TypeError
+ msg = f"dtype 'str' does not support operation '{method}'"
+ else:
+ klass = ValueError
+ msg = expected_data
+ with pytest.raises(klass, match=msg):
+ _ = func(**kwargs)
+ else:
+ result = func(**kwargs)
+ expected = DataFrame(expected_data, index=expected_index)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "method, numeric_only, expected_data",
+ [
+ ("sum", True, ()),
+ ("sum", False, ["cat_1cat_2"]),
+ ("sum", lib.no_default, ["cat_1cat_2"]),
+ ("prod", True, ()),
+ ("prod", False, ()),
+ ("prod", lib.no_default, ()),
+ ("min", True, ()),
+ ("min", False, ["cat_1"]),
+ ("min", lib.no_default, ["cat_1"]),
+ ("max", True, ()),
+ ("max", False, ["cat_2"]),
+ ("max", lib.no_default, ["cat_2"]),
+ ("first", True, ()),
+ ("first", False, ["cat_1"]),
+ ("first", lib.no_default, ["cat_1"]),
+ ("last", True, ()),
+ ("last", False, ["cat_2"]),
+ ("last", lib.no_default, ["cat_2"]),
+ ],
+)
+def test_series_downsample_method(
+ method, numeric_only, expected_data, using_infer_string
+):
+ # GH#46442 test if `numeric_only` behave as expected for SeriesGroupBy
+
+ index = date_range("2018-01-01", periods=2, freq="D")
+ expected_index = date_range("2018-12-31", periods=1, freq="YE")
+ df = Series(["cat_1", "cat_2"], index=index)
+ resampled = df.resample("YE")
+ kwargs = {} if numeric_only is lib.no_default else {"numeric_only": numeric_only}
+
+ func = getattr(resampled, method)
+ if numeric_only and numeric_only is not lib.no_default:
+ msg = rf"Cannot use numeric_only=True with SeriesGroupBy\.{method}"
+ with pytest.raises(TypeError, match=msg):
+ func(**kwargs)
+ elif method == "prod":
+ msg = re.escape("agg function failed [how->prod,dtype->")
+ if using_infer_string:
+ msg = "dtype 'str' does not support operation 'prod'"
+ with pytest.raises(TypeError, match=msg):
+ func(**kwargs)
+
+ else:
+ result = func(**kwargs)
+ expected = Series(expected_data, index=expected_index)
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_empty():
+ # GH#52484
+ df = DataFrame(
+ index=pd.to_datetime(
+ ["2018-01-01 00:00:00", "2018-01-01 12:00:00", "2018-01-02 00:00:00"]
+ )
+ )
+ expected = DataFrame(
+ index=pd.to_datetime(
+ [
+ "2018-01-01 00:00:00",
+ "2018-01-01 08:00:00",
+ "2018-01-01 16:00:00",
+ "2018-01-02 00:00:00",
+ ]
+ )
+ )
+ result = df.resample("8h").mean()
+ tm.assert_frame_equal(result, expected)
+
+
+def test_asfreq_respects_origin_with_fixed_freq_all_seconds_equal():
+ # GH#62725: Ensure Resampler.asfreq respects origin="start_day"
+ # when all datetimes share identical seconds values.
+ idx = [
+ datetime(2025, 10, 17, 17, 15, 10),
+ datetime(2025, 10, 17, 17, 16, 10),
+ datetime(2025, 10, 17, 17, 17, 10),
+ ]
+ df = DataFrame({"value": [0, 1, 2]}, index=idx)
+
+ result = df.resample("1min", origin="start_day").asfreq()
+
+ # Expected index: list of Timestamps, matching dtype
+ exp_idx = pd.DatetimeIndex(
+ [
+ pd.Timestamp("2025-10-17 17:15:00"),
+ pd.Timestamp("2025-10-17 17:16:00"),
+ pd.Timestamp("2025-10-17 17:17:00"),
+ ],
+ dtype=result.index.dtype,
+ freq="min",
+ )
+
+ exp = DataFrame({"value": [np.nan, np.nan, np.nan]}, index=exp_idx)
+ tm.assert_frame_equal(result, exp)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_resampler_grouper.py b/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_resampler_grouper.py
new file mode 100644
index 0000000000000000000000000000000000000000..862578decb782af4e056f0f076b1f4cba894c200
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_resampler_grouper.py
@@ -0,0 +1,671 @@
+from textwrap import dedent
+
+import numpy as np
+import pytest
+
+from pandas.compat import is_platform_windows
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ Series,
+ TimedeltaIndex,
+ Timestamp,
+)
+import pandas._testing as tm
+from pandas.core.indexes.datetimes import date_range
+
+
+@pytest.fixture
+def test_frame():
+ return DataFrame(
+ {"A": [1] * 20 + [2] * 12 + [3] * 8, "B": np.arange(40)},
+ index=date_range("1/1/2000", freq="s", periods=40, unit="ns"),
+ )
+
+
+def test_tab_complete_ipython6_warning(ip):
+ from IPython.core.completer import provisionalcompleter
+
+ code = dedent(
+ """\
+ import numpy as np
+ from pandas import Series, date_range
+ data = np.arange(10, dtype=np.float64)
+ index = date_range("2020-01-01", periods=len(data))
+ s = Series(data, index=index)
+ rs = s.resample("D")
+ """
+ )
+ ip.run_cell(code)
+
+ # GH 31324 newer jedi version raises Deprecation warning;
+ # appears resolved 2021-02-02
+ with tm.assert_produces_warning(None, raise_on_extra_warnings=False):
+ with provisionalcompleter("ignore"):
+ list(ip.Completer.completions("rs.", 1))
+
+
+def test_deferred_with_groupby():
+ # GH 12486
+ # support deferred resample ops with groupby
+ data = [
+ ["2010-01-01", "A", 2],
+ ["2010-01-02", "A", 3],
+ ["2010-01-05", "A", 8],
+ ["2010-01-10", "A", 7],
+ ["2010-01-13", "A", 3],
+ ["2010-01-01", "B", 5],
+ ["2010-01-03", "B", 2],
+ ["2010-01-04", "B", 1],
+ ["2010-01-11", "B", 7],
+ ["2010-01-14", "B", 3],
+ ]
+
+ df = DataFrame(data, columns=["date", "id", "score"])
+ df.date = pd.to_datetime(df.date)
+
+ def f_0(x):
+ return x.set_index("date").resample("D").asfreq()
+
+ expected = df.groupby("id").apply(f_0)
+ result = df.set_index("date").groupby("id").resample("D").asfreq()
+ tm.assert_frame_equal(result, expected)
+
+ df = DataFrame(
+ {
+ "date": date_range(start="2016-01-01", periods=4, freq="W"),
+ "group": [1, 1, 2, 2],
+ "val": [5, 6, 7, 8],
+ }
+ ).set_index("date")
+
+ def f_1(x):
+ return x.resample("1D").ffill()
+
+ expected = df.groupby("group").apply(f_1)
+ result = df.groupby("group").resample("1D").ffill()
+ tm.assert_frame_equal(result, expected)
+
+
+def test_getitem(test_frame):
+ g = test_frame.groupby("A")
+
+ expected = g.B.apply(lambda x: x.resample("2s").mean())
+
+ result = g.resample("2s").B.mean()
+ tm.assert_series_equal(result, expected)
+
+ result = g.B.resample("2s").mean()
+ tm.assert_series_equal(result, expected)
+
+ result = g.resample("2s").mean().B
+ tm.assert_series_equal(result, expected)
+
+
+def test_getitem_multiple():
+ # GH 13174
+ # multiple calls after selection causing an issue with aliasing
+ data = [{"id": 1, "buyer": "A"}, {"id": 2, "buyer": "B"}]
+ df = DataFrame(data, index=date_range("2016-01-01", periods=2))
+ r = df.groupby("id").resample("1D")
+ result = r["buyer"].count()
+
+ exp_mi = pd.MultiIndex.from_arrays([[1, 2], df.index], names=("id", None))
+ expected = Series(
+ [1, 1],
+ index=exp_mi,
+ name="buyer",
+ )
+ tm.assert_series_equal(result, expected)
+
+ result = r["buyer"].count()
+ tm.assert_series_equal(result, expected)
+
+
+def test_groupby_resample_on_api_with_getitem():
+ # GH 17813
+ df = DataFrame(
+ {"id": list("aabbb"), "date": date_range("1-1-2016", periods=5), "data": 1}
+ )
+ exp = df.set_index("date").groupby("id").resample("2D")["data"].sum()
+ result = df.groupby("id").resample("2D", on="date")["data"].sum()
+ tm.assert_series_equal(result, exp)
+
+
+def test_groupby_with_origin():
+ # GH 31809
+
+ freq = "1399min" # prime number that is smaller than 24h
+ start, end = "1/1/2000 00:00:00", "1/31/2000 00:00"
+ middle = "1/15/2000 00:00:00"
+
+ rng = date_range(start, end, freq="1231min") # prime number
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+ ts2 = ts[middle:end]
+
+ # proves that grouper without a fixed origin does not work
+ # when dealing with unusual frequencies
+ simple_grouper = pd.Grouper(freq=freq)
+ count_ts = ts.groupby(simple_grouper).agg("count")
+ count_ts = count_ts[middle:end]
+ count_ts2 = ts2.groupby(simple_grouper).agg("count")
+ with pytest.raises(AssertionError, match="Index are different"):
+ tm.assert_index_equal(count_ts.index, count_ts2.index)
+
+ # test origin on 1970-01-01 00:00:00
+ origin = Timestamp(0)
+ adjusted_grouper = pd.Grouper(freq=freq, origin=origin)
+ adjusted_count_ts = ts.groupby(adjusted_grouper).agg("count")
+ adjusted_count_ts = adjusted_count_ts[middle:end]
+ adjusted_count_ts2 = ts2.groupby(adjusted_grouper).agg("count")
+ tm.assert_series_equal(adjusted_count_ts, adjusted_count_ts2)
+
+ # test origin on 2049-10-18 20:00:00
+ origin_future = Timestamp(0) + pd.Timedelta("1399min") * 30_000
+ adjusted_grouper2 = pd.Grouper(freq=freq, origin=origin_future)
+ adjusted2_count_ts = ts.groupby(adjusted_grouper2).agg("count")
+ adjusted2_count_ts = adjusted2_count_ts[middle:end]
+ adjusted2_count_ts2 = ts2.groupby(adjusted_grouper2).agg("count")
+ tm.assert_series_equal(adjusted2_count_ts, adjusted2_count_ts2)
+
+ # both grouper use an adjusted timestamp that is a multiple of 1399 min
+ # they should be equals even if the adjusted_timestamp is in the future
+ tm.assert_series_equal(adjusted_count_ts, adjusted2_count_ts2)
+
+
+def test_nearest():
+ # GH 17496
+ # Resample nearest
+ index = date_range("1/1/2000", periods=3, freq="min", unit="ns")
+ result = Series(range(3), index=index).resample("20s").nearest()
+
+ expected = Series(
+ [0, 0, 1, 1, 1, 2, 2],
+ index=pd.DatetimeIndex(
+ [
+ "2000-01-01 00:00:00",
+ "2000-01-01 00:00:20",
+ "2000-01-01 00:00:40",
+ "2000-01-01 00:01:00",
+ "2000-01-01 00:01:20",
+ "2000-01-01 00:01:40",
+ "2000-01-01 00:02:00",
+ ],
+ dtype="datetime64[ns]",
+ freq="20s",
+ ),
+ )
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "f",
+ [
+ "first",
+ "last",
+ "median",
+ "sem",
+ "sum",
+ "mean",
+ "min",
+ "max",
+ "size",
+ "count",
+ "nearest",
+ "bfill",
+ "ffill",
+ "asfreq",
+ "ohlc",
+ ],
+)
+def test_methods(f, test_frame):
+ g = test_frame.groupby("A")
+ r = g.resample("2s")
+
+ result = getattr(r, f)()
+ expected = g.apply(lambda x: getattr(x.resample("2s"), f)())
+ tm.assert_equal(result, expected)
+
+
+def test_methods_nunique(test_frame):
+ # series only
+ g = test_frame.groupby("A")
+ r = g.resample("2s")
+ result = r.B.nunique()
+ expected = g.B.apply(lambda x: x.resample("2s").nunique())
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("f", ["std", "var"])
+def test_methods_std_var(f, test_frame):
+ g = test_frame.groupby("A")
+ r = g.resample("2s")
+ result = getattr(r, f)(ddof=1)
+ expected = g.apply(lambda x: getattr(x.resample("2s"), f)(ddof=1))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_apply(test_frame):
+ g = test_frame.groupby("A")
+ r = g.resample("2s")
+
+ # reduction
+ expected = g.resample("2s").sum()
+
+ def f_0(x):
+ return x.resample("2s").sum()
+
+ result = r.apply(f_0)
+ tm.assert_frame_equal(result, expected)
+
+ def f_1(x):
+ return x.resample("2s").apply(lambda y: y.sum())
+
+ result = g.apply(f_1)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_apply_with_mutated_index():
+ # GH 15169
+ index = date_range("1-1-2015", "12-31-15", freq="D")
+ df = DataFrame(
+ data={"col1": np.random.default_rng(2).random(len(index))}, index=index
+ )
+
+ def f(x):
+ s = Series([1, 2], index=["a", "b"])
+ return s
+
+ expected = df.groupby(pd.Grouper(freq="ME")).apply(f)
+
+ result = df.resample("ME").apply(f)
+ tm.assert_frame_equal(result, expected)
+
+ # A case for series
+ expected = df["col1"].groupby(pd.Grouper(freq="ME"), group_keys=False).apply(f)
+ result = df["col1"].resample("ME").apply(f)
+ tm.assert_series_equal(result, expected)
+
+
+def test_apply_columns_multilevel():
+ # GH 16231
+ cols = pd.MultiIndex.from_tuples([("A", "a", "", "one"), ("B", "b", "i", "two")])
+ ind = date_range(start="2017-01-01", freq="15Min", periods=8)
+ df = DataFrame(
+ np.array([0] * 16, dtype=np.int64).reshape(8, 2), index=ind, columns=cols
+ )
+ agg_dict = {col: (np.sum if col[3] == "one" else np.mean) for col in df.columns}
+ result = df.resample("h").apply(lambda x: agg_dict[x.name](x))
+ expected = DataFrame(
+ 2 * [[0, 0.0]],
+ index=date_range(start="2017-01-01", freq="1h", periods=2),
+ columns=pd.MultiIndex.from_tuples(
+ [("A", "a", "", "one"), ("B", "b", "i", "two")]
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_apply_non_naive_index():
+ def weighted_quantile(series, weights, q):
+ series = series.sort_values()
+ cumsum = weights.reindex(series.index).fillna(0).cumsum()
+ cutoff = cumsum.iloc[-1] * q
+ return series[cumsum >= cutoff].iloc[0]
+
+ times = date_range("2017-6-23 18:00", periods=8, freq="15min", tz="UTC")
+ data = Series([1.0, 1, 1, 1, 1, 2, 2, 0], index=times)
+ weights = Series([160.0, 91, 65, 43, 24, 10, 1, 0], index=times)
+
+ result = data.resample("D").apply(weighted_quantile, weights=weights, q=0.5)
+ ind = date_range(
+ "2017-06-23 00:00:00+00:00", "2017-06-23 00:00:00+00:00", freq="D", tz="UTC"
+ )
+ expected = Series([1.0], index=ind)
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_groupby_with_label(unit):
+ # GH 13235
+ index = date_range("2000-01-01", freq="2D", periods=5, unit=unit)
+ df = DataFrame(index=index, data={"col0": [0, 0, 1, 1, 2], "col1": [1, 1, 1, 1, 1]})
+ result = df.groupby("col0").resample("1W", label="left").sum()
+
+ mi = [
+ np.array([0, 0, 1, 2], dtype=np.int64),
+ np.array(
+ ["1999-12-26", "2000-01-02", "2000-01-02", "2000-01-02"],
+ dtype=f"M8[{unit}]",
+ ),
+ ]
+ mindex = pd.MultiIndex.from_arrays(mi, names=["col0", None])
+ expected = DataFrame(data={"col1": [1, 1, 2, 1]}, index=mindex)
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_consistency_with_window(test_frame):
+ # consistent return values with window
+ df = test_frame
+ expected = Index([1, 2, 3], name="A")
+ result = df.groupby("A").resample("2s").mean()
+ assert result.index.nlevels == 2
+ tm.assert_index_equal(result.index.levels[0], expected)
+
+ result = df.groupby("A").rolling(20).mean()
+ assert result.index.nlevels == 2
+ tm.assert_index_equal(result.index.levels[0], expected)
+
+
+def test_median_duplicate_columns():
+ # GH 14233
+
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((20, 3)),
+ columns=list("aaa"),
+ index=date_range("2012-01-01", periods=20, freq="s"),
+ )
+ result = df.resample("5s").median()
+ df.columns = ["a", "b", "c"]
+ expected = df.resample("5s").median()
+ expected.columns = result.columns
+ tm.assert_frame_equal(result, expected)
+
+
+def test_apply_to_one_column_of_df():
+ # GH: 36951
+ df = DataFrame(
+ {"col": range(10), "col1": range(10, 20)},
+ index=date_range("2012-01-01", periods=10, freq="20min"),
+ )
+
+ # access "col" via getattr -> make sure we handle AttributeError
+ result = df.resample("h").apply(lambda group: group.col.sum())
+ expected = Series(
+ [3, 12, 21, 9], index=date_range("2012-01-01", periods=4, freq="h")
+ )
+ tm.assert_series_equal(result, expected)
+
+ # access "col" via _getitem__ -> make sure we handle KeyErrpr
+ result = df.resample("h").apply(lambda group: group["col"].sum())
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_groupby_agg():
+ # GH: 33548
+ df = DataFrame(
+ {
+ "cat": [
+ "cat_1",
+ "cat_1",
+ "cat_2",
+ "cat_1",
+ "cat_2",
+ "cat_1",
+ "cat_2",
+ "cat_1",
+ ],
+ "num": [5, 20, 22, 3, 4, 30, 10, 50],
+ "date": [
+ "2019-2-1",
+ "2018-02-03",
+ "2020-3-11",
+ "2019-2-2",
+ "2019-2-2",
+ "2018-12-4",
+ "2020-3-11",
+ "2020-12-12",
+ ],
+ }
+ )
+ df["date"] = pd.to_datetime(df["date"])
+
+ resampled = df.groupby("cat").resample("YE", on="date")
+ expected = resampled[["num"]].sum()
+ result = resampled.agg({"num": "sum"})
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_groupby_agg_listlike():
+ # GH 42905
+ ts = Timestamp("2021-02-28 00:00:00")
+ df = DataFrame({"class": ["beta"], "value": [69]}, index=Index([ts], name="date"))
+ resampled = df.groupby("class").resample("ME")["value"]
+ result = resampled.agg(["sum", "size"])
+ expected = DataFrame(
+ [[69, 1]],
+ index=pd.MultiIndex.from_tuples([("beta", ts)], names=["class", "date"]),
+ columns=["sum", "size"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("keys", [["a"], ["a", "b"]])
+def test_empty(keys):
+ # GH 26411
+ df = DataFrame([], columns=["a", "b"], index=TimedeltaIndex([]))
+ result = df.groupby(keys).resample(rule=pd.to_timedelta("00:00:01")).mean()
+ expected_columns = ["b"] if keys == ["a"] else []
+ expected = (
+ DataFrame(columns=["a", "b"])
+ .set_index(keys, drop=False)
+ .set_index(TimedeltaIndex([]), append=True)[expected_columns]
+ )
+ if len(keys) == 1:
+ expected.index.name = keys[0]
+
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("consolidate", [True, False])
+def test_resample_groupby_agg_object_dtype_all_nan(consolidate):
+ # https://github.com/pandas-dev/pandas/issues/39329
+
+ dates = date_range("2020-01-01", periods=15, freq="D", unit="ns")
+ df1 = DataFrame({"key": "A", "date": dates, "col1": range(15), "col_object": "val"})
+ df2 = DataFrame({"key": "B", "date": dates, "col1": range(15)})
+ df = pd.concat([df1, df2], ignore_index=True)
+ if consolidate:
+ df = df._consolidate()
+
+ result = df.groupby(["key"]).resample("W", on="date").min()
+ idx = pd.MultiIndex.from_arrays(
+ [
+ ["A"] * 3 + ["B"] * 3,
+ pd.to_datetime(["2020-01-05", "2020-01-12", "2020-01-19"] * 2).as_unit(
+ "ns"
+ ),
+ ],
+ names=["key", "date"],
+ )
+ expected = DataFrame(
+ {
+ "col1": [0, 5, 12] * 2,
+ "col_object": ["val"] * 3 + [np.nan] * 3,
+ },
+ index=idx,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("min_count", [0, 1])
+def test_groupby_resample_empty_sum_string(
+ string_dtype_no_object, test_frame, min_count
+):
+ # https://github.com/pandas-dev/pandas/issues/60229
+ dtype = string_dtype_no_object
+ test_frame = test_frame.assign(B=pd.array([pd.NA] * len(test_frame), dtype=dtype))
+ gbrs = test_frame.groupby("A").resample("40s")
+ result = gbrs.sum(min_count=min_count)
+
+ index = pd.MultiIndex(
+ levels=[[1, 2, 3], [pd.to_datetime("2000-01-01", unit="ns").as_unit("ns")]],
+ codes=[[0, 1, 2], [0, 0, 0]],
+ names=["A", None],
+ )
+ value = "" if min_count == 0 else pd.NA
+ expected = DataFrame({"B": value}, index=index, dtype=dtype)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_groupby_resample_with_list_of_keys():
+ # GH 47362
+ df = DataFrame(
+ data={
+ "date": date_range(start="2016-01-01", periods=8),
+ "group": [0, 0, 0, 0, 1, 1, 1, 1],
+ "val": [1, 7, 5, 2, 3, 10, 5, 1],
+ }
+ )
+ result = df.groupby("group").resample("2D", on="date")[["val"]].mean()
+
+ mi_exp = pd.MultiIndex.from_arrays(
+ [[0, 0, 1, 1], df["date"]._values[::2]], names=["group", "date"]
+ )
+ expected = DataFrame(
+ data={
+ "val": [4.0, 3.5, 6.5, 3.0],
+ },
+ index=mi_exp,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("keys", [["a"], ["a", "b"]])
+def test_resample_no_index(keys):
+ # GH 47705
+ df = DataFrame([], columns=["a", "b", "date"])
+ df["date"] = pd.to_datetime(df["date"])
+ df = df.set_index("date")
+ result = df.groupby(keys).resample(rule=pd.to_timedelta("00:00:01")).mean()
+ expected_columns = ["b"] if keys == ["a"] else []
+ expected = DataFrame(columns=["a", "b", "date"]).set_index(keys, drop=False)
+ expected["date"] = pd.to_datetime(expected["date"])
+ expected = expected.set_index("date", append=True, drop=True)[expected_columns]
+ if len(keys) == 1:
+ expected.index.name = keys[0]
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_no_columns():
+ # GH#52484
+ df = DataFrame(
+ index=Index(
+ pd.to_datetime(
+ ["2018-01-01 00:00:00", "2018-01-01 12:00:00", "2018-01-02 00:00:00"]
+ ),
+ name="date",
+ )
+ )
+ result = df.groupby([0, 0, 1]).resample(rule=pd.to_timedelta("06:00:00")).mean()
+ index = pd.to_datetime(
+ [
+ "2018-01-01 00:00:00",
+ "2018-01-01 06:00:00",
+ "2018-01-01 12:00:00",
+ "2018-01-02 00:00:00",
+ ]
+ )
+ expected = DataFrame(
+ index=pd.MultiIndex(
+ levels=[np.array([0, 1], dtype=np.intp), index],
+ codes=[[0, 0, 0, 1], [0, 1, 2, 3]],
+ names=[None, "date"],
+ )
+ )
+
+ # GH#52710 - Index comes out as 32-bit on 64-bit Windows
+ tm.assert_frame_equal(result, expected, check_index_type=not is_platform_windows())
+
+
+def test_groupby_resample_size_all_index_same():
+ # GH 46826
+ df = DataFrame(
+ {"A": [1] * 3 + [2] * 3 + [1] * 3 + [2] * 3, "B": np.arange(12)},
+ index=date_range("31/12/2000 18:00", freq="h", periods=12, unit="ns"),
+ )
+ result = df.groupby("A").resample("D").size()
+
+ mi_exp = pd.MultiIndex.from_arrays(
+ [
+ [1, 1, 2, 2],
+ pd.DatetimeIndex(["2000-12-31", "2001-01-01"] * 2, dtype="M8[ns]"),
+ ],
+ names=["A", None],
+ )
+ expected = Series(
+ 3,
+ index=mi_exp,
+ )
+ tm.assert_series_equal(result, expected)
+
+
+def test_groupby_resample_on_index_with_list_of_keys():
+ # GH 50840
+ df = DataFrame(
+ data={
+ "group": [0, 0, 0, 0, 1, 1, 1, 1],
+ "val": [3, 1, 4, 1, 5, 9, 2, 6],
+ },
+ index=date_range(start="2016-01-01", periods=8, name="date"),
+ )
+ result = df.groupby("group").resample("2D")[["val"]].mean()
+
+ mi_exp = pd.MultiIndex.from_arrays(
+ [[0, 0, 1, 1], df.index[::2]], names=["group", "date"]
+ )
+ expected = DataFrame(
+ data={
+ "val": [2.0, 2.5, 7.0, 4.0],
+ },
+ index=mi_exp,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_groupby_resample_on_index_with_list_of_keys_multi_columns():
+ # GH 50876
+ df = DataFrame(
+ data={
+ "group": [0, 0, 0, 0, 1, 1, 1, 1],
+ "first_val": [3, 1, 4, 1, 5, 9, 2, 6],
+ "second_val": [2, 7, 1, 8, 2, 8, 1, 8],
+ "third_val": [1, 4, 1, 4, 2, 1, 3, 5],
+ },
+ index=date_range(start="2016-01-01", periods=8, name="date"),
+ )
+ result = df.groupby("group").resample("2D")[["first_val", "second_val"]].mean()
+
+ mi_exp = pd.MultiIndex.from_arrays(
+ [[0, 0, 1, 1], df.index[::2]], names=["group", "date"]
+ )
+ expected = DataFrame(
+ data={
+ "first_val": [2.0, 2.5, 7.0, 4.0],
+ "second_val": [4.5, 4.5, 5.0, 4.5],
+ },
+ index=mi_exp,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_groupby_resample_on_index_with_list_of_keys_missing_column():
+ # GH 50876
+ df = DataFrame(
+ data={
+ "group": [0, 0, 0, 0, 1, 1, 1, 1],
+ "val": [3, 1, 4, 1, 5, 9, 2, 6],
+ },
+ index=Series(
+ date_range(start="2016-01-01", periods=8),
+ name="date",
+ ),
+ )
+ gb = df.groupby("group")
+ rs = gb.resample("2D")
+ with pytest.raises(KeyError, match="Columns not found"):
+ rs[["val_not_in_dataframe"]]
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_time_grouper.py b/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_time_grouper.py
new file mode 100644
index 0000000000000000000000000000000000000000..e214a9f17824dbfc4dc00896b651d9113fdac472
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_time_grouper.py
@@ -0,0 +1,439 @@
+from datetime import datetime
+from operator import methodcaller
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ Series,
+ Timestamp,
+)
+import pandas._testing as tm
+from pandas.core.groupby.grouper import Grouper
+from pandas.core.indexes.datetimes import date_range
+
+
+@pytest.fixture
+def test_series():
+ return Series(
+ np.random.default_rng(2).standard_normal(1000),
+ index=date_range("1/1/2000", periods=1000),
+ )
+
+
+def test_apply(test_series):
+ grouper = Grouper(freq="YE", label="right", closed="right")
+
+ grouped = test_series.groupby(grouper)
+
+ def f(x):
+ return x.sort_values()[-3:]
+
+ applied = grouped.apply(f)
+ expected = test_series.groupby(lambda x: x.year).apply(f)
+
+ applied.index = applied.index.droplevel(0)
+ expected.index = expected.index.droplevel(0)
+ tm.assert_series_equal(applied, expected)
+
+
+def test_count(test_series):
+ test_series[::3] = np.nan
+
+ expected = test_series.groupby(lambda x: x.year).count()
+
+ grouper = Grouper(freq="YE", label="right", closed="right")
+ result = test_series.groupby(grouper).count()
+ expected.index = result.index
+ tm.assert_series_equal(result, expected)
+
+ result = test_series.resample("YE").count()
+ expected.index = result.index
+ tm.assert_series_equal(result, expected)
+
+
+def test_numpy_reduction(test_series):
+ result = test_series.resample("YE", closed="right").prod()
+ expected = test_series.groupby(lambda x: x.year).agg(np.prod)
+ expected.index = result.index
+ tm.assert_series_equal(result, expected)
+
+
+def test_apply_iteration():
+ # #2300
+ N = 1000
+ ind = date_range(start="2000-01-01", freq="D", periods=N)
+ df = DataFrame({"open": 1, "close": 2}, index=ind)
+ tg = Grouper(freq="ME")
+
+ grouper, _ = tg._get_grouper(df)
+
+ # Errors
+ grouped = df.groupby(grouper, group_keys=False)
+
+ def f(df):
+ return df["close"] / df["open"]
+
+ # it works!
+ result = grouped.apply(f)
+ tm.assert_index_equal(result.index, df.index)
+
+
+@pytest.mark.parametrize(
+ "index",
+ [
+ Index([1, 2]),
+ Index(["a", "b"]),
+ Index([1.1, 2.2]),
+ pd.MultiIndex.from_arrays([[1, 2], ["a", "b"]]),
+ ],
+)
+def test_fails_on_no_datetime_index(index):
+ name = type(index).__name__
+ df = DataFrame({"a": range(len(index))}, index=index)
+
+ msg = (
+ "Only valid with DatetimeIndex, TimedeltaIndex "
+ f"or PeriodIndex, but got an instance of '{name}'"
+ )
+ with pytest.raises(TypeError, match=msg):
+ df.groupby(Grouper(freq="D"))
+
+
+def test_aaa_group_order():
+ # GH 12840
+ # check TimeGrouper perform stable sorts
+ n = 20
+ data = np.random.default_rng(2).standard_normal((n, 4))
+ df = DataFrame(data, columns=["A", "B", "C", "D"])
+ df["key"] = [
+ datetime(2013, 1, 1),
+ datetime(2013, 1, 2),
+ datetime(2013, 1, 3),
+ datetime(2013, 1, 4),
+ datetime(2013, 1, 5),
+ ] * 4
+ grouped = df.groupby(Grouper(key="key", freq="D"))
+
+ tm.assert_frame_equal(grouped.get_group(datetime(2013, 1, 1)), df[::5])
+ tm.assert_frame_equal(grouped.get_group(datetime(2013, 1, 2)), df[1::5])
+ tm.assert_frame_equal(grouped.get_group(datetime(2013, 1, 3)), df[2::5])
+ tm.assert_frame_equal(grouped.get_group(datetime(2013, 1, 4)), df[3::5])
+ tm.assert_frame_equal(grouped.get_group(datetime(2013, 1, 5)), df[4::5])
+
+
+def test_aggregate_normal(resample_method):
+ """Check TimeGrouper's aggregation is identical as normal groupby."""
+
+ data = np.random.default_rng(2).standard_normal((20, 4))
+ normal_df = DataFrame(data, columns=["A", "B", "C", "D"])
+ normal_df["key"] = [1, 2, 3, 4, 5] * 4
+
+ dt_df = DataFrame(data, columns=["A", "B", "C", "D"])
+ dt_df["key"] = Index(
+ [
+ datetime(2013, 1, 1),
+ datetime(2013, 1, 2),
+ datetime(2013, 1, 3),
+ datetime(2013, 1, 4),
+ datetime(2013, 1, 5),
+ ]
+ * 4,
+ dtype="M8[ns]",
+ )
+
+ normal_grouped = normal_df.groupby("key")
+ dt_grouped = dt_df.groupby(Grouper(key="key", freq="D"))
+
+ expected = getattr(normal_grouped, resample_method)()
+ dt_result = getattr(dt_grouped, resample_method)()
+ expected.index = date_range(
+ start="2013-01-01", freq="D", periods=5, unit="ns", name="key"
+ )
+ tm.assert_equal(expected, dt_result)
+
+
+@pytest.mark.xfail(reason="if TimeGrouper is used included, 'nth' doesn't work yet")
+def test_aggregate_nth():
+ """Check TimeGrouper's aggregation is identical as normal groupby."""
+
+ data = np.random.default_rng(2).standard_normal((20, 4))
+ normal_df = DataFrame(data, columns=["A", "B", "C", "D"])
+ normal_df["key"] = [1, 2, 3, 4, 5] * 4
+
+ dt_df = DataFrame(data, columns=["A", "B", "C", "D"])
+ dt_df["key"] = [
+ datetime(2013, 1, 1),
+ datetime(2013, 1, 2),
+ datetime(2013, 1, 3),
+ datetime(2013, 1, 4),
+ datetime(2013, 1, 5),
+ ] * 4
+
+ normal_grouped = normal_df.groupby("key")
+ dt_grouped = dt_df.groupby(Grouper(key="key", freq="D"))
+
+ expected = normal_grouped.nth(3)
+ expected.index = date_range(start="2013-01-01", freq="D", periods=5, name="key")
+ dt_result = dt_grouped.nth(3)
+ tm.assert_frame_equal(expected, dt_result)
+
+
+@pytest.mark.parametrize(
+ "method, method_args, unit",
+ [
+ ("sum", {}, 0),
+ ("sum", {"min_count": 0}, 0),
+ ("sum", {"min_count": 1}, np.nan),
+ ("prod", {}, 1),
+ ("prod", {"min_count": 0}, 1),
+ ("prod", {"min_count": 1}, np.nan),
+ ],
+)
+def test_resample_entirely_nat_window(method, method_args, unit):
+ ser = Series([0] * 2 + [np.nan] * 2, index=date_range("2017", periods=4, unit="ns"))
+ result = methodcaller(method, **method_args)(ser.resample("2D"))
+
+ exp_dti = pd.DatetimeIndex(["2017-01-01", "2017-01-03"], dtype="M8[ns]", freq="2D")
+ expected = Series([0.0, unit], index=exp_dti)
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "func, fill_value",
+ [("min", np.nan), ("max", np.nan), ("sum", 0), ("prod", 1), ("count", 0)],
+)
+def test_aggregate_with_nat(func, fill_value):
+ # check TimeGrouper's aggregation is identical as normal groupby
+ # if NaT is included, 'var', 'std', 'mean', 'first','last'
+ # and 'nth' doesn't work yet
+
+ n = 20
+ data = np.random.default_rng(2).standard_normal((n, 4)).astype("int64")
+ normal_df = DataFrame(data, columns=["A", "B", "C", "D"])
+ normal_df["key"] = [1, 2, np.nan, 4, 5] * 4
+
+ dt_df = DataFrame(data, columns=["A", "B", "C", "D"])
+ dt_df["key"] = Index(
+ [
+ datetime(2013, 1, 1),
+ datetime(2013, 1, 2),
+ pd.NaT,
+ datetime(2013, 1, 4),
+ datetime(2013, 1, 5),
+ ]
+ * 4,
+ dtype="M8[ns]",
+ )
+
+ normal_grouped = normal_df.groupby("key")
+ dt_grouped = dt_df.groupby(Grouper(key="key", freq="D"))
+
+ normal_result = getattr(normal_grouped, func)()
+ dt_result = getattr(dt_grouped, func)()
+
+ pad = DataFrame([[fill_value] * 4], index=[3], columns=["A", "B", "C", "D"])
+ expected = pd.concat([normal_result, pad])
+ expected = expected.sort_index()
+ dti = date_range(
+ start="2013-01-01",
+ freq="D",
+ periods=5,
+ name="key",
+ unit=dt_df["key"]._values.unit,
+ )
+ expected.index = dti._with_freq(None) # TODO: is this desired?
+ tm.assert_frame_equal(expected, dt_result)
+ assert dt_result.index.name == "key"
+
+
+def test_aggregate_with_nat_size():
+ # GH 9925
+ n = 20
+ data = np.random.default_rng(2).standard_normal((n, 4)).astype("int64")
+ normal_df = DataFrame(data, columns=["A", "B", "C", "D"])
+ normal_df["key"] = [1, 2, np.nan, 4, 5] * 4
+
+ dt_df = DataFrame(data, columns=["A", "B", "C", "D"])
+ dt_df["key"] = Index(
+ [
+ datetime(2013, 1, 1),
+ datetime(2013, 1, 2),
+ pd.NaT,
+ datetime(2013, 1, 4),
+ datetime(2013, 1, 5),
+ ]
+ * 4,
+ dtype="M8[ns]",
+ )
+
+ normal_grouped = normal_df.groupby("key")
+ dt_grouped = dt_df.groupby(Grouper(key="key", freq="D"))
+
+ normal_result = normal_grouped.size()
+ dt_result = dt_grouped.size()
+
+ pad = Series([0], index=[3])
+ expected = pd.concat([normal_result, pad])
+ expected = expected.sort_index()
+ expected.index = date_range(
+ start="2013-01-01",
+ freq="D",
+ periods=5,
+ name="key",
+ unit=dt_df["key"]._values.unit,
+ )._with_freq(None)
+ tm.assert_series_equal(expected, dt_result)
+ assert dt_result.index.name == "key"
+
+
+def test_repr():
+ # GH18203
+ result = repr(Grouper(key="A", freq="h"))
+ expected = (
+ "TimeGrouper(key='A', freq=, sort=True, dropna=True, "
+ "closed='left', label='left', how='mean', "
+ "convention='e', origin='start_day')"
+ )
+ assert result == expected
+
+ result = repr(Grouper(key="A", freq="h", origin="2000-01-01"))
+ expected = (
+ "TimeGrouper(key='A', freq=, sort=True, dropna=True, "
+ "closed='left', label='left', how='mean', "
+ "convention='e', origin=Timestamp('2000-01-01 00:00:00'))"
+ )
+ assert result == expected
+
+
+@pytest.mark.parametrize(
+ "method, method_args, expected_values",
+ [
+ ("sum", {}, [1, 0, 1]),
+ ("sum", {"min_count": 0}, [1, 0, 1]),
+ ("sum", {"min_count": 1}, [1, np.nan, 1]),
+ ("sum", {"min_count": 2}, [np.nan, np.nan, np.nan]),
+ ("prod", {}, [1, 1, 1]),
+ ("prod", {"min_count": 0}, [1, 1, 1]),
+ ("prod", {"min_count": 1}, [1, np.nan, 1]),
+ ("prod", {"min_count": 2}, [np.nan, np.nan, np.nan]),
+ ],
+)
+def test_upsample_sum(method, method_args, expected_values):
+ ser = Series(1, index=date_range("2017", periods=2, freq="h", unit="ns"))
+ resampled = ser.resample("30min")
+ index = pd.DatetimeIndex(
+ ["2017-01-01T00:00:00", "2017-01-01T00:30:00", "2017-01-01T01:00:00"],
+ dtype="M8[ns]",
+ freq="30min",
+ )
+ result = methodcaller(method, **method_args)(resampled)
+ expected = Series(expected_values, index=index)
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.fixture
+def groupy_test_df():
+ return DataFrame(
+ {"price": [10, 11, 9], "volume": [50, 60, 50]},
+ index=date_range("01/01/2018", periods=3, freq="W", unit="ns"),
+ )
+
+
+def test_groupby_resample_interpolate_raises(groupy_test_df):
+ # GH 35325
+
+ # Make a copy of the test data frame that has index.name=None
+ groupy_test_df_without_index_name = groupy_test_df.copy()
+ groupy_test_df_without_index_name.index.name = None
+
+ dfs = [groupy_test_df, groupy_test_df_without_index_name]
+
+ for df in dfs:
+ with pytest.raises(
+ NotImplementedError,
+ match="Direct interpolation of MultiIndex data frames is not supported",
+ ):
+ df.groupby("volume").resample("1D").interpolate(method="linear")
+
+
+def test_groupby_resample_interpolate_with_apply_syntax(groupy_test_df):
+ # GH 35325
+
+ # Make a copy of the test data frame that has index.name=None
+ groupy_test_df_without_index_name = groupy_test_df.copy()
+ groupy_test_df_without_index_name.index.name = None
+
+ dfs = [groupy_test_df, groupy_test_df_without_index_name]
+
+ for df in dfs:
+ result = df.groupby("volume").apply(
+ lambda x: x.resample("1D").interpolate(method="linear"),
+ )
+
+ volume = [50] * 15 + [60]
+ week_starting = [
+ *list(date_range("2018-01-07", "2018-01-21", unit="ns")),
+ Timestamp("2018-01-14"),
+ ]
+ expected_ind = pd.MultiIndex.from_arrays(
+ [volume, week_starting],
+ names=["volume", df.index.name],
+ )
+
+ expected = DataFrame(
+ data={
+ "price": [
+ 10.0,
+ 9.928571428571429,
+ 9.857142857142858,
+ 9.785714285714286,
+ 9.714285714285714,
+ 9.642857142857142,
+ 9.571428571428571,
+ 9.5,
+ 9.428571428571429,
+ 9.357142857142858,
+ 9.285714285714286,
+ 9.214285714285714,
+ 9.142857142857142,
+ 9.071428571428571,
+ 9.0,
+ 11.0,
+ ]
+ },
+ index=expected_ind,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_groupby_resample_interpolate_with_apply_syntax_off_grid(groupy_test_df):
+ """Similar test as test_groupby_resample_interpolate_with_apply_syntax but
+ with resampling that results in missing anchor points when interpolating.
+ See GH#21351."""
+ # GH#21351
+ result = groupy_test_df.groupby("volume").apply(
+ lambda x: x.resample("265h").interpolate(method="linear")
+ )
+
+ volume = [50, 50, 60]
+ week_starting = pd.DatetimeIndex(
+ [
+ Timestamp("2018-01-07"),
+ Timestamp("2018-01-18 01:00:00"),
+ Timestamp("2018-01-14"),
+ ]
+ ).as_unit("ns")
+ expected_ind = pd.MultiIndex.from_arrays(
+ [volume, week_starting],
+ names=["volume", "week_starting"],
+ )
+
+ expected = DataFrame(
+ data={"price": [10.0, 9.5, 11.0]},
+ index=expected_ind,
+ )
+ tm.assert_frame_equal(result, expected, check_names=False)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_timedelta.py b/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_timedelta.py
new file mode 100644
index 0000000000000000000000000000000000000000..9fdaea50c7430465bf50c80c14568b1e4deffbad
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/resample/test_timedelta.py
@@ -0,0 +1,218 @@
+from datetime import timedelta
+
+import numpy as np
+import pytest
+
+import pandas.util._test_decorators as td
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Series,
+)
+import pandas._testing as tm
+from pandas.core.indexes.timedeltas import timedelta_range
+
+
+def test_asfreq_bug():
+ df = DataFrame(data=[1, 3], index=[timedelta(), timedelta(minutes=3)])
+ result = df.resample("1min").asfreq()
+ expected = DataFrame(
+ data=[1, np.nan, np.nan, 3],
+ index=timedelta_range("0 day", periods=4, freq="1min", unit="us"),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_with_nat():
+ # GH 13223
+ index = pd.to_timedelta(["0s", pd.NaT, "2s"])
+ result = DataFrame({"value": [2, 3, 5]}, index).resample("1s").mean()
+ expected = DataFrame(
+ {"value": [2.5, np.nan, 5.0]},
+ index=timedelta_range("0 day", periods=3, freq="1s"),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_as_freq_with_subperiod():
+ # GH 13022
+ index = timedelta_range("00:00:00", "00:10:00", freq="5min")
+ df = DataFrame(data={"value": [1, 5, 10]}, index=index)
+ result = df.resample("2min").asfreq()
+ expected_data = {"value": [1, np.nan, np.nan, np.nan, np.nan, 10]}
+ expected = DataFrame(
+ data=expected_data, index=timedelta_range("00:00:00", "00:10:00", freq="2min")
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_with_timedeltas():
+ expected = DataFrame({"A": np.arange(1480)})
+ expected = expected.groupby(expected.index // 30).sum()
+ expected.index = timedelta_range("0 days", freq="30min", periods=50)
+
+ df = DataFrame(
+ {"A": np.arange(1480)},
+ index=pd.to_timedelta(np.arange(1480), unit="min").as_unit("us"),
+ )
+ result = df.resample("30min").sum()
+
+ tm.assert_frame_equal(result, expected)
+
+ s = df["A"]
+ result = s.resample("30min").sum()
+ tm.assert_series_equal(result, expected["A"])
+
+
+def test_resample_single_period_timedelta():
+ s = Series(list(range(5)), index=timedelta_range("1 day", freq="s", periods=5))
+ result = s.resample("2s").sum()
+ expected = Series([1, 5, 4], index=timedelta_range("1 day", freq="2s", periods=3))
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_timedelta_idempotency():
+ # GH 12072
+ index = timedelta_range("0", periods=9, freq="10ms")
+ series = Series(range(9), index=index)
+ result = series.resample("10ms").mean()
+ expected = series.astype(float)
+ tm.assert_series_equal(result, expected)
+
+
+def test_resample_offset_with_timedeltaindex():
+ # GH 10530 & 31809
+ rng = timedelta_range(start="0s", periods=25, freq="s")
+ ts = Series(np.random.default_rng(2).standard_normal(len(rng)), index=rng)
+
+ with_base = ts.resample("2s", offset="5s").mean()
+ without_base = ts.resample("2s").mean()
+
+ exp_without_base = timedelta_range(start="0s", end="25s", freq="2s")
+ exp_with_base = timedelta_range(start="5s", end="29s", freq="2s")
+
+ tm.assert_index_equal(without_base.index, exp_without_base)
+ tm.assert_index_equal(with_base.index, exp_with_base)
+
+
+def test_resample_categorical_data_with_timedeltaindex():
+ # GH #12169
+ df = DataFrame({"Group_obj": "A"}, index=pd.to_timedelta(list(range(20)), unit="s"))
+ df["Group"] = df["Group_obj"].astype("category")
+ result = df.resample("10s").agg(lambda x: x.value_counts().index[0])
+ exp_tdi = pd.TimedeltaIndex(np.array([0, 10], dtype="m8[s]"), freq="10s")
+ expected = DataFrame(
+ {"Group_obj": ["A", "A"], "Group": ["A", "A"]},
+ index=exp_tdi,
+ )
+ expected = expected.reindex(["Group_obj", "Group"], axis=1)
+ expected["Group"] = expected["Group_obj"].astype("category")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_timedelta_values():
+ # GH 13119
+ # check that timedelta dtype is preserved when NaT values are
+ # introduced by the resampling
+
+ times = timedelta_range("1 day", "6 day", freq="4D")
+ df = DataFrame({"time": times}, index=times)
+
+ times2 = timedelta_range("1 day", "6 day", freq="2D")
+ exp = Series(times2, index=times2, name="time")
+ exp.iloc[1] = pd.NaT
+
+ res = df.resample("2D").first()["time"]
+ tm.assert_series_equal(res, exp)
+ res = df["time"].resample("2D").first()
+ tm.assert_series_equal(res, exp)
+
+
+@pytest.mark.parametrize(
+ "start, end, freq, resample_freq",
+ [
+ ("8h", "21h59min50s", "10s", "3h"), # GH 30353 example
+ ("3h", "22h", "1h", "5h"),
+ ("527D", "5006D", "3D", "10D"),
+ ("1D", "10D", "1D", "2D"), # GH 13022 example
+ # tests that worked before GH 33498:
+ ("8h", "21h59min50s", "10s", "2h"),
+ ("0h", "21h59min50s", "10s", "3h"),
+ ("10D", "85D", "D", "2D"),
+ ],
+)
+def test_resample_timedelta_edge_case(start, end, freq, resample_freq):
+ # GH 33498
+ # check that the timedelta bins does not contains an extra bin
+ idx = timedelta_range(start=start, end=end, freq=freq)
+ s = Series(np.arange(len(idx)), index=idx)
+ result = s.resample(resample_freq).min()
+ expected_index = timedelta_range(freq=resample_freq, start=start, end=end)
+ tm.assert_index_equal(result.index, expected_index)
+ assert result.index.freq == expected_index.freq
+ assert not np.isnan(result.iloc[-1])
+
+
+@pytest.mark.parametrize("duplicates", [True, False])
+def test_resample_with_timedelta_yields_no_empty_groups(duplicates):
+ # GH 10603
+ df = DataFrame(
+ np.random.default_rng(2).normal(size=(10000, 4)),
+ index=timedelta_range(start="0s", periods=10000, freq="3906250ns"),
+ )
+ if duplicates:
+ # case with non-unique columns
+ df.columns = ["A", "B", "A", "C"]
+
+ result = df.loc["1s":, :].resample("3s").apply(lambda x: len(x))
+
+ expected = DataFrame(
+ [[768] * 4] * 12 + [[528] * 4],
+ index=timedelta_range(start="1s", periods=13, freq="3s", unit="ns"),
+ )
+ expected.columns = df.columns
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_quantile_timedelta(unit):
+ # GH: 29485
+ dtype = np.dtype(f"m8[{unit}]")
+ df = DataFrame(
+ {"value": pd.to_timedelta(np.arange(4), unit="s").astype(dtype)},
+ index=pd.date_range("20200101", periods=4, tz="UTC"),
+ )
+ result = df.resample("2D").quantile(0.99)
+ expected = DataFrame(
+ {
+ "value": [
+ pd.Timedelta("0 days 00:00:00.990000"),
+ pd.Timedelta("0 days 00:00:02.990000"),
+ ]
+ },
+ index=pd.date_range("20200101", periods=2, tz="UTC", freq="2D"),
+ ).astype(dtype)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_resample_closed_right():
+ # GH#45414
+ idx = pd.Index([pd.Timedelta(seconds=120 + i * 30) for i in range(10)])
+ ser = Series(range(10), index=idx)
+ result = ser.resample("min", closed="right", label="right").sum()
+ expected = Series(
+ [0, 3, 7, 11, 15, 9],
+ index=pd.TimedeltaIndex(
+ [pd.Timedelta(seconds=120 + i * 60) for i in range(6)], freq="min"
+ ),
+ )
+ tm.assert_series_equal(result, expected)
+
+
+@td.skip_if_no("pyarrow")
+def test_arrow_duration_resample():
+ # GH 56371
+ idx = pd.Index(timedelta_range("1 day", periods=5), dtype="duration[ns][pyarrow]")
+ expected = Series(np.arange(5, dtype=np.float64), index=idx)
+ result = expected.resample("1D").mean()
+ tm.assert_series_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_append.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_append.py
new file mode 100644
index 0000000000000000000000000000000000000000..258d0e89fda7aabf84520b039d8a4a043ed290c3
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_append.py
@@ -0,0 +1,351 @@
+import datetime as dt
+from itertools import combinations
+
+import dateutil
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ Series,
+ Timestamp,
+ concat,
+ isna,
+)
+import pandas._testing as tm
+
+
+class TestAppend:
+ def test_append(self, sort, float_frame):
+ mixed_frame = float_frame.copy()
+ mixed_frame["foo"] = "bar"
+
+ begin_index = float_frame.index[:5]
+ end_index = float_frame.index[5:]
+
+ begin_frame = float_frame.reindex(begin_index)
+ end_frame = float_frame.reindex(end_index)
+
+ appended = concat([begin_frame, end_frame])
+ tm.assert_almost_equal(appended["A"], float_frame["A"])
+
+ del end_frame["A"]
+ partial_appended = concat([begin_frame, end_frame], sort=sort)
+ assert "A" in partial_appended
+
+ partial_appended = concat([end_frame, begin_frame], sort=sort)
+ assert "A" in partial_appended
+
+ # mixed type handling
+ appended = concat([mixed_frame[:5], mixed_frame[5:]])
+ tm.assert_frame_equal(appended, mixed_frame)
+
+ # what to test here
+ mixed_appended = concat([mixed_frame[:5], float_frame[5:]], sort=sort)
+ mixed_appended2 = concat([float_frame[:5], mixed_frame[5:]], sort=sort)
+
+ # all equal except 'foo' column
+ tm.assert_frame_equal(
+ mixed_appended.reindex(columns=["A", "B", "C", "D"]),
+ mixed_appended2.reindex(columns=["A", "B", "C", "D"]),
+ )
+
+ def test_append_empty(self, float_frame):
+ empty = DataFrame()
+
+ appended = concat([float_frame, empty])
+ tm.assert_frame_equal(float_frame, appended)
+ assert appended is not float_frame
+
+ appended = concat([empty, float_frame])
+ tm.assert_frame_equal(float_frame, appended)
+ assert appended is not float_frame
+
+ def test_append_overlap_raises(self, float_frame):
+ msg = "Indexes have overlapping values"
+ with pytest.raises(ValueError, match=msg):
+ concat([float_frame, float_frame], verify_integrity=True)
+
+ def test_append_new_columns(self):
+ # see gh-6129: new columns
+ df = DataFrame({"a": {"x": 1, "y": 2}, "b": {"x": 3, "y": 4}})
+ row = Series([5, 6, 7], index=["a", "b", "c"], name="z")
+ expected = DataFrame(
+ {
+ "a": {"x": 1, "y": 2, "z": 5},
+ "b": {"x": 3, "y": 4, "z": 6},
+ "c": {"z": 7},
+ }
+ )
+ result = df._append_internal(row)
+ tm.assert_frame_equal(result, expected)
+
+ def test_append_length0_frame(self, sort):
+ df = DataFrame(columns=["A", "B", "C"])
+ df3 = DataFrame(index=[0, 1], columns=["A", "B"])
+ df5 = concat([df, df3], sort=sort)
+
+ expected = DataFrame(index=[0, 1], columns=["A", "B", "C"])
+ tm.assert_frame_equal(df5, expected)
+
+ def test_append_records(self):
+ arr1 = np.zeros((2,), dtype=("i4,f4,S10"))
+ arr1[:] = [(1, 2.0, "Hello"), (2, 3.0, "World")]
+
+ arr2 = np.zeros((3,), dtype=("i4,f4,S10"))
+ arr2[:] = [(3, 4.0, "foo"), (5, 6.0, "bar"), (7.0, 8.0, "baz")]
+
+ df1 = DataFrame(arr1)
+ df2 = DataFrame(arr2)
+
+ result = concat([df1, df2], ignore_index=True)
+ expected = DataFrame(np.concatenate((arr1, arr2)))
+ tm.assert_frame_equal(result, expected)
+
+ # rewrite sort fixture, since we also want to test default of None
+ def test_append_sorts(self, sort):
+ df1 = DataFrame({"a": [1, 2], "b": [1, 2]}, columns=["b", "a"])
+ df2 = DataFrame({"a": [1, 2], "c": [3, 4]}, index=[2, 3])
+
+ result = concat([df1, df2], sort=sort)
+
+ # for None / True
+ expected = DataFrame(
+ {"b": [1, 2, None, None], "a": [1, 2, 1, 2], "c": [None, None, 3, 4]},
+ columns=["a", "b", "c"],
+ )
+ if sort is False:
+ expected = expected[["b", "a", "c"]]
+ tm.assert_frame_equal(result, expected)
+
+ def test_append_different_columns(self, sort):
+ df = DataFrame(
+ {
+ "bools": np.random.default_rng(2).standard_normal(10) > 0,
+ "ints": np.random.default_rng(2).integers(0, 10, 10),
+ "floats": np.random.default_rng(2).standard_normal(10),
+ "strings": ["foo", "bar"] * 5,
+ }
+ )
+
+ a = df[:5].loc[:, ["bools", "ints", "floats"]]
+ b = df[5:].loc[:, ["strings", "ints", "floats"]]
+
+ appended = concat([a, b], sort=sort)
+ assert isna(appended["strings"][0:4]).all()
+ assert isna(appended["bools"][5:]).all()
+
+ def test_append_preserve_index_name(self):
+ # #980
+ df1 = DataFrame(columns=["A", "B", "C"])
+ df1 = df1.set_index(["A"])
+ df2 = DataFrame(data=[[1, 4, 7], [2, 5, 8], [3, 6, 9]], columns=["A", "B", "C"])
+ df2 = df2.set_index(["A"])
+
+ result = concat([df1, df2])
+ assert result.index.name == "A"
+
+ indexes_can_append = [
+ pd.RangeIndex(3),
+ Index([4, 5, 6]),
+ Index([4.5, 5.5, 6.5]),
+ Index(list("abc")),
+ pd.CategoricalIndex("A B C".split()),
+ pd.CategoricalIndex("D E F".split(), ordered=True),
+ pd.IntervalIndex.from_breaks([7, 8, 9, 10]),
+ pd.DatetimeIndex(
+ [
+ dt.datetime(2013, 1, 3, 0, 0),
+ dt.datetime(2013, 1, 3, 6, 10),
+ dt.datetime(2013, 1, 3, 7, 12),
+ ]
+ ),
+ pd.MultiIndex.from_arrays(["A B C".split(), "D E F".split()]),
+ ]
+
+ @pytest.mark.parametrize(
+ "index", indexes_can_append, ids=lambda x: type(x).__name__
+ )
+ def test_append_same_columns_type(self, index):
+ # GH18359
+
+ # df wider than ser
+ df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=index)
+ ser_index = index[:2]
+ ser = Series([7, 8], index=ser_index, name=2)
+ result = df._append_internal(ser)
+ expected = DataFrame(
+ [[1, 2, 3.0], [4, 5, 6], [7, 8, np.nan]], index=[0, 1, 2], columns=index
+ )
+ # integer dtype is preserved for columns present in ser.index
+ assert expected.dtypes.iloc[0].kind == "i"
+ assert expected.dtypes.iloc[1].kind == "i"
+
+ tm.assert_frame_equal(result, expected)
+
+ # ser wider than df
+ ser_index = index
+ index = index[:2]
+ df = DataFrame([[1, 2], [4, 5]], columns=index)
+ ser = Series([7, 8, 9], index=ser_index, name=2)
+ result = df._append_internal(ser)
+ expected = DataFrame(
+ [[1, 2, np.nan], [4, 5, np.nan], [7, 8, 9]],
+ index=[0, 1, 2],
+ columns=ser_index,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "df_columns, series_index",
+ combinations(indexes_can_append, r=2),
+ ids=lambda x: type(x).__name__,
+ )
+ def test_append_different_columns_types(self, df_columns, series_index):
+ # GH18359
+ # See also test 'test_append_different_columns_types_raises' below
+ # for errors raised when appending
+
+ df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=df_columns)
+ ser = Series([7, 8, 9], index=series_index, name=2)
+
+ result = df._append_internal(ser)
+ idx_diff = ser.index.difference(df_columns)
+ combined_columns = Index(df_columns.tolist()).append(idx_diff)
+ expected = DataFrame(
+ [
+ [1.0, 2.0, 3.0, np.nan, np.nan, np.nan],
+ [4, 5, 6, np.nan, np.nan, np.nan],
+ [np.nan, np.nan, np.nan, 7, 8, 9],
+ ],
+ index=[0, 1, 2],
+ columns=combined_columns,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_append_dtype_coerce(self, sort):
+ # GH 4993
+ # appending with datetime will incorrectly convert datetime64
+
+ df1 = DataFrame(
+ index=[1, 2],
+ data=[dt.datetime(2013, 1, 1, 0, 0), dt.datetime(2013, 1, 2, 0, 0)],
+ columns=["start_time"],
+ )
+ df2 = DataFrame(
+ index=[4, 5],
+ data=[
+ [dt.datetime(2013, 1, 3, 0, 0), dt.datetime(2013, 1, 3, 6, 10)],
+ [dt.datetime(2013, 1, 4, 0, 0), dt.datetime(2013, 1, 4, 7, 10)],
+ ],
+ columns=["start_time", "end_time"],
+ )
+
+ expected = concat(
+ [
+ Series(
+ [
+ pd.NaT,
+ pd.NaT,
+ dt.datetime(2013, 1, 3, 6, 10),
+ dt.datetime(2013, 1, 4, 7, 10),
+ ],
+ name="end_time",
+ ),
+ Series(
+ [
+ dt.datetime(2013, 1, 1, 0, 0),
+ dt.datetime(2013, 1, 2, 0, 0),
+ dt.datetime(2013, 1, 3, 0, 0),
+ dt.datetime(2013, 1, 4, 0, 0),
+ ],
+ name="start_time",
+ ),
+ ],
+ axis=1,
+ sort=sort,
+ )
+ result = concat([df1, df2], ignore_index=True, sort=sort)
+ if sort:
+ expected = expected[["end_time", "start_time"]]
+ else:
+ expected = expected[["start_time", "end_time"]]
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_append_missing_column_proper_upcast(self, sort):
+ df1 = DataFrame({"A": np.array([1, 2, 3, 4], dtype="i8")})
+ df2 = DataFrame({"B": np.array([True, False, True, False], dtype=bool)})
+
+ appended = concat([df1, df2], sort=sort)
+ assert appended["A"].dtype == "f8"
+ assert appended["B"].dtype == "O"
+
+ def test_append_empty_frame_to_series_with_dateutil_tz(self):
+ # GH 23682
+ date = Timestamp("2018-10-24 07:30:00", tz=dateutil.tz.tzutc())
+ ser = Series({"a": 1.0, "b": 2.0, "date": date})
+ df = DataFrame(columns=["c", "d"])
+ result_a = df._append_internal(ser, ignore_index=True)
+ expected = DataFrame(
+ [[np.nan, np.nan, 1.0, 2.0, date]], columns=["c", "d", "a", "b", "date"]
+ )
+ # These columns get cast to object after append
+ expected["c"] = expected["c"].astype(object)
+ expected["d"] = expected["d"].astype(object)
+ tm.assert_frame_equal(result_a, expected)
+
+ expected = DataFrame(
+ [[np.nan, np.nan, 1.0, 2.0, date]] * 2, columns=["c", "d", "a", "b", "date"]
+ )
+ expected["c"] = expected["c"].astype(object)
+ expected["d"] = expected["d"].astype(object)
+ result_b = result_a._append_internal(ser, ignore_index=True)
+ tm.assert_frame_equal(result_b, expected)
+
+ def test_append_empty_tz_frame_with_datetime64ns(self):
+ # https://github.com/pandas-dev/pandas/issues/35460
+ df = DataFrame(columns=["a"]).astype("datetime64[ns, UTC]")
+
+ # also test with typed value to append
+ df = DataFrame(columns=["a"]).astype("datetime64[ns, UTC]")
+ other = Series({"a": pd.NaT}, dtype="datetime64[ns]").to_frame().T
+ result = concat([df, other], ignore_index=True)
+ expected = DataFrame({"a": [pd.NaT]}, dtype=object)
+ tm.assert_frame_equal(result, expected)
+
+ # mismatched tz
+ other = Series({"a": pd.NaT}, dtype="datetime64[ns, US/Pacific]").to_frame().T
+ result = concat([df, other], ignore_index=True)
+ expected = DataFrame({"a": [pd.NaT]}).astype(object)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "dtype_str", ["datetime64[ns, UTC]", "datetime64[ns]", "Int64", "int64"]
+ )
+ @pytest.mark.parametrize("val", [1, "NaT"])
+ def test_append_empty_frame_with_timedelta64ns_nat(self, dtype_str, val):
+ # https://github.com/pandas-dev/pandas/issues/35460
+ df = DataFrame(columns=["a"]).astype(dtype_str)
+
+ other = DataFrame({"a": [np.timedelta64(val, "ns")]})
+ result = concat([df, other])
+
+ expected = other.astype(object)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "dtype_str", ["datetime64[ns, UTC]", "datetime64[ns]", "Int64", "int64"]
+ )
+ @pytest.mark.parametrize("val", [1, "NaT"])
+ def test_append_frame_with_timedelta64ns_nat(self, dtype_str, val):
+ # https://github.com/pandas-dev/pandas/issues/35460
+ df = DataFrame({"a": pd.array([1], dtype=dtype_str)})
+
+ other = DataFrame({"a": [np.timedelta64(val, "ns")]})
+ result = concat([df, other], ignore_index=True)
+
+ expected = DataFrame({"a": [df.iloc[0, 0], other.iloc[0, 0]]}, dtype=object)
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_append_common.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_append_common.py
new file mode 100644
index 0000000000000000000000000000000000000000..d47fd3c743f956e5d29d01d8dbc6a9fdd768dd32
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_append_common.py
@@ -0,0 +1,715 @@
+import zoneinfo
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ Categorical,
+ DataFrame,
+ Index,
+ Series,
+)
+import pandas._testing as tm
+
+
+@pytest.fixture(
+ params=list(
+ {
+ "bool": [True, False, True],
+ "int64": [1, 2, 3],
+ "float64": [1.1, np.nan, 3.3],
+ "category": Categorical(["X", "Y", "Z"]),
+ "object": ["a", "b", "c"],
+ "datetime64[s]": [
+ pd.Timestamp("2011-01-01").as_unit("s"),
+ pd.Timestamp("2011-01-02").as_unit("s"),
+ pd.Timestamp("2011-01-03").as_unit("s"),
+ ],
+ "datetime64[s, US/Eastern]": [
+ pd.Timestamp("2011-01-01", tz="US/Eastern").as_unit("s"),
+ pd.Timestamp("2011-01-02", tz="US/Eastern").as_unit("s"),
+ pd.Timestamp("2011-01-03", tz="US/Eastern").as_unit("s"),
+ ],
+ "timedelta64[us]": [
+ pd.Timedelta("1 days"),
+ pd.Timedelta("2 days"),
+ pd.Timedelta("3 days"),
+ ],
+ "period[M]": [
+ pd.Period("2011-01", freq="M"),
+ pd.Period("2011-02", freq="M"),
+ pd.Period("2011-03", freq="M"),
+ ],
+ }.items()
+ )
+)
+def item(request):
+ key, data = request.param
+ return key, data
+
+
+@pytest.fixture
+def item2(item):
+ return item
+
+
+class TestConcatAppendCommon:
+ """
+ Test common dtype coercion rules between concat and append.
+ """
+
+ def test_dtypes(self, item, index_or_series, using_infer_string):
+ # to confirm test case covers intended dtypes
+ typ, vals = item
+ obj = index_or_series(vals)
+ if typ == "object" and using_infer_string:
+ typ = "string"
+ if isinstance(obj, Index):
+ assert obj.dtype == typ
+ elif isinstance(obj, Series):
+ if typ.startswith("period"):
+ assert obj.dtype == "Period[M]"
+ else:
+ assert obj.dtype == typ
+
+ def test_concatlike_same_dtypes(self, item):
+ # GH 13660
+ typ1, vals1 = item
+
+ vals2 = vals1
+ vals3 = vals1
+
+ if typ1 == "category":
+ exp_data = Categorical(list(vals1) + list(vals2))
+ exp_data3 = Categorical(list(vals1) + list(vals2) + list(vals3))
+ else:
+ exp_data = vals1 + vals2
+ exp_data3 = vals1 + vals2 + vals3
+
+ # ----- Index ----- #
+
+ # index.append
+ res = Index(vals1).append(Index(vals2))
+ exp = Index(exp_data)
+ tm.assert_index_equal(res, exp)
+
+ # 3 elements
+ res = Index(vals1).append([Index(vals2), Index(vals3)])
+ exp = Index(exp_data3)
+ tm.assert_index_equal(res, exp)
+
+ # index.append name mismatch
+ i1 = Index(vals1, name="x")
+ i2 = Index(vals2, name="y")
+ res = i1.append(i2)
+ exp = Index(exp_data)
+ tm.assert_index_equal(res, exp)
+
+ # index.append name match
+ i1 = Index(vals1, name="x")
+ i2 = Index(vals2, name="x")
+ res = i1.append(i2)
+ exp = Index(exp_data, name="x")
+ tm.assert_index_equal(res, exp)
+
+ # cannot append non-index
+ with pytest.raises(TypeError, match="all inputs must be Index"):
+ Index(vals1).append(vals2)
+
+ with pytest.raises(TypeError, match="all inputs must be Index"):
+ Index(vals1).append([Index(vals2), vals3])
+
+ # ----- Series ----- #
+
+ # series.append
+ res = Series(vals1)._append_internal(Series(vals2), ignore_index=True)
+ exp = Series(exp_data)
+ tm.assert_series_equal(res, exp, check_index_type=True)
+
+ # concat
+ res = pd.concat([Series(vals1), Series(vals2)], ignore_index=True)
+ tm.assert_series_equal(res, exp, check_index_type=True)
+
+ # 3 elements
+ exp = Series(exp_data3)
+ res = pd.concat(
+ [Series(vals1), Series(vals2), Series(vals3)],
+ ignore_index=True,
+ )
+ tm.assert_series_equal(res, exp)
+
+ # name mismatch
+ s1 = Series(vals1, name="x")
+ s2 = Series(vals2, name="y")
+ res = s1._append_internal(s2, ignore_index=True)
+ exp = Series(exp_data)
+ tm.assert_series_equal(res, exp, check_index_type=True)
+
+ res = pd.concat([s1, s2], ignore_index=True)
+ tm.assert_series_equal(res, exp, check_index_type=True)
+
+ # name match
+ s1 = Series(vals1, name="x")
+ s2 = Series(vals2, name="x")
+ res = s1._append_internal(s2, ignore_index=True)
+ exp = Series(exp_data, name="x")
+ tm.assert_series_equal(res, exp, check_index_type=True)
+
+ res = pd.concat([s1, s2], ignore_index=True)
+ tm.assert_series_equal(res, exp, check_index_type=True)
+
+ # cannot append non-index
+ msg = (
+ r"cannot concatenate object of type '.+'; "
+ "only Series and DataFrame objs are valid"
+ )
+ with pytest.raises(TypeError, match=msg):
+ pd.concat([Series(vals1), vals2])
+
+ with pytest.raises(TypeError, match=msg):
+ pd.concat([Series(vals1), Series(vals2), vals3])
+
+ def test_concatlike_dtypes_coercion(self, item, item2, request):
+ # GH 13660
+ typ1, vals1 = item
+ typ2, vals2 = item2
+
+ vals3 = vals2
+
+ # basically infer
+ exp_index_dtype = None
+ exp_series_dtype = None
+
+ if typ1 == typ2:
+ pytest.skip("same dtype is tested in test_concatlike_same_dtypes")
+ elif typ1 == "category" or typ2 == "category":
+ pytest.skip("categorical type tested elsewhere")
+
+ # specify expected dtype
+ if typ1 == "bool" and typ2 in ("int64", "float64"):
+ # series coerces to numeric based on numpy rule
+ # index doesn't because bool is object dtype
+ exp_series_dtype = typ2
+ mark = pytest.mark.xfail(reason="GH#39187 casting to object")
+ request.applymarker(mark)
+ elif typ2 == "bool" and typ1 in ("int64", "float64"):
+ exp_series_dtype = typ1
+ mark = pytest.mark.xfail(reason="GH#39187 casting to object")
+ request.applymarker(mark)
+ elif typ1 in {"datetime64[ns, US/Eastern]", "timedelta64[ns]"} or typ2 in {
+ "datetime64[ns, US/Eastern]",
+ "timedelta64[ns]",
+ }:
+ exp_index_dtype = object
+ exp_series_dtype = object
+
+ exp_data = vals1 + vals2
+ exp_data3 = vals1 + vals2 + vals3
+
+ # ----- Index ----- #
+
+ # index.append
+ # GH#39817
+ res = Index(vals1).append(Index(vals2))
+ exp = Index(exp_data, dtype=exp_index_dtype)
+ tm.assert_index_equal(res, exp)
+
+ # 3 elements
+ res = Index(vals1).append([Index(vals2), Index(vals3)])
+ exp = Index(exp_data3, dtype=exp_index_dtype)
+ tm.assert_index_equal(res, exp)
+
+ # ----- Series ----- #
+
+ # series._append
+ # GH#39817
+ res = Series(vals1)._append_internal(Series(vals2), ignore_index=True)
+ exp = Series(exp_data, dtype=exp_series_dtype)
+ tm.assert_series_equal(res, exp, check_index_type=True)
+
+ # concat
+ # GH#39817
+ res = pd.concat([Series(vals1), Series(vals2)], ignore_index=True)
+ tm.assert_series_equal(res, exp, check_index_type=True)
+
+ # 3 elements
+ # GH#39817
+ exp = Series(exp_data3, dtype=exp_series_dtype)
+ res = pd.concat(
+ [Series(vals1), Series(vals2), Series(vals3)],
+ ignore_index=True,
+ )
+ tm.assert_series_equal(res, exp)
+
+ def test_concatlike_common_coerce_to_pandas_object(self):
+ # GH 13626
+ # result must be Timestamp/Timedelta, not datetime.datetime/timedelta
+ dti = pd.DatetimeIndex(["2011-01-01", "2011-01-02"])
+ tdi = pd.TimedeltaIndex(["1 days", "2 days"])
+
+ exp = Index(
+ [
+ pd.Timestamp("2011-01-01"),
+ pd.Timestamp("2011-01-02"),
+ pd.Timedelta("1 days"),
+ pd.Timedelta("2 days"),
+ ]
+ )
+
+ res = dti.append(tdi)
+ tm.assert_index_equal(res, exp)
+ assert isinstance(res[0], pd.Timestamp)
+ assert isinstance(res[-1], pd.Timedelta)
+
+ dts = Series(dti)
+ tds = Series(tdi)
+ res = dts._append_internal(tds)
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+ assert isinstance(res.iloc[0], pd.Timestamp)
+ assert isinstance(res.iloc[-1], pd.Timedelta)
+
+ res = pd.concat([dts, tds])
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+ assert isinstance(res.iloc[0], pd.Timestamp)
+ assert isinstance(res.iloc[-1], pd.Timedelta)
+
+ def test_concatlike_datetimetz(self, tz_aware_fixture):
+ tz = tz_aware_fixture
+ # GH 7795
+ dti1 = pd.DatetimeIndex(["2011-01-01", "2011-01-02"], tz=tz)
+ dti2 = pd.DatetimeIndex(["2012-01-01", "2012-01-02"], tz=tz)
+
+ exp = pd.DatetimeIndex(
+ ["2011-01-01", "2011-01-02", "2012-01-01", "2012-01-02"], tz=tz
+ )
+
+ res = dti1.append(dti2)
+ tm.assert_index_equal(res, exp)
+
+ dts1 = Series(dti1)
+ dts2 = Series(dti2)
+ res = dts1._append_internal(dts2)
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+
+ res = pd.concat([dts1, dts2])
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+
+ @pytest.mark.parametrize("tz", ["UTC", "US/Eastern", "Asia/Tokyo", "EST5EDT"])
+ def test_concatlike_datetimetz_short(self, tz):
+ # GH#7795
+ ix1 = pd.date_range(
+ start="2014-07-15", end="2014-07-17", freq="D", tz=tz, unit="ns"
+ )
+ ix2 = pd.DatetimeIndex(["2014-07-11", "2014-07-21"], tz=tz)
+ df1 = DataFrame(0, index=ix1, columns=["A", "B"])
+ df2 = DataFrame(0, index=ix2, columns=["A", "B"])
+
+ exp_idx = pd.DatetimeIndex(
+ ["2014-07-15", "2014-07-16", "2014-07-17", "2014-07-11", "2014-07-21"],
+ tz=tz,
+ ).as_unit("ns")
+ exp = DataFrame(0, index=exp_idx, columns=["A", "B"])
+
+ tm.assert_frame_equal(pd.concat([df1, df2]), exp)
+
+ def test_concatlike_datetimetz_to_object(self, tz_aware_fixture):
+ tz = tz_aware_fixture
+ # GH 13660
+
+ # different tz coerces to object
+ dti1 = pd.DatetimeIndex(["2011-01-01", "2011-01-02"], tz=tz)
+ dti2 = pd.DatetimeIndex(["2012-01-01", "2012-01-02"])
+
+ exp = Index(
+ [
+ pd.Timestamp("2011-01-01", tz=tz),
+ pd.Timestamp("2011-01-02", tz=tz),
+ pd.Timestamp("2012-01-01"),
+ pd.Timestamp("2012-01-02"),
+ ],
+ dtype=object,
+ )
+
+ res = dti1.append(dti2)
+ tm.assert_index_equal(res, exp)
+
+ dts1 = Series(dti1)
+ dts2 = Series(dti2)
+ res = dts1._append_internal(dts2)
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+
+ res = pd.concat([dts1, dts2])
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+
+ # different tz
+ tz_diff = zoneinfo.ZoneInfo("US/Hawaii")
+ dti3 = pd.DatetimeIndex(["2012-01-01", "2012-01-02"], tz=tz_diff)
+
+ exp = Index(
+ [
+ pd.Timestamp("2011-01-01", tz=tz),
+ pd.Timestamp("2011-01-02", tz=tz),
+ pd.Timestamp("2012-01-01", tz=tz_diff),
+ pd.Timestamp("2012-01-02", tz=tz_diff),
+ ],
+ dtype=object,
+ )
+
+ res = dti1.append(dti3)
+ tm.assert_index_equal(res, exp)
+
+ dts1 = Series(dti1)
+ dts3 = Series(dti3)
+ res = dts1._append_internal(dts3)
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+
+ res = pd.concat([dts1, dts3])
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+
+ def test_concatlike_common_period(self):
+ # GH 13660
+ pi1 = pd.PeriodIndex(["2011-01", "2011-02"], freq="M")
+ pi2 = pd.PeriodIndex(["2012-01", "2012-02"], freq="M")
+
+ exp = pd.PeriodIndex(["2011-01", "2011-02", "2012-01", "2012-02"], freq="M")
+
+ res = pi1.append(pi2)
+ tm.assert_index_equal(res, exp)
+
+ ps1 = Series(pi1)
+ ps2 = Series(pi2)
+ res = ps1._append_internal(ps2)
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+
+ res = pd.concat([ps1, ps2])
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+
+ def test_concatlike_common_period_diff_freq_to_object(self):
+ # GH 13221
+ pi1 = pd.PeriodIndex(["2011-01", "2011-02"], freq="M")
+ pi2 = pd.PeriodIndex(["2012-01-01", "2012-02-01"], freq="D")
+
+ exp = Index(
+ [
+ pd.Period("2011-01", freq="M"),
+ pd.Period("2011-02", freq="M"),
+ pd.Period("2012-01-01", freq="D"),
+ pd.Period("2012-02-01", freq="D"),
+ ],
+ dtype=object,
+ )
+
+ res = pi1.append(pi2)
+ tm.assert_index_equal(res, exp)
+
+ ps1 = Series(pi1)
+ ps2 = Series(pi2)
+ res = ps1._append_internal(ps2)
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+
+ res = pd.concat([ps1, ps2])
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+
+ def test_concatlike_common_period_mixed_dt_to_object(self):
+ # GH 13221
+ # different datetimelike
+ pi1 = pd.PeriodIndex(["2011-01", "2011-02"], freq="M")
+ tdi = pd.TimedeltaIndex(["1 days", "2 days"])
+ exp = Index(
+ [
+ pd.Period("2011-01", freq="M"),
+ pd.Period("2011-02", freq="M"),
+ pd.Timedelta("1 days"),
+ pd.Timedelta("2 days"),
+ ],
+ dtype=object,
+ )
+
+ res = pi1.append(tdi)
+ tm.assert_index_equal(res, exp)
+
+ ps1 = Series(pi1)
+ tds = Series(tdi)
+ res = ps1._append_internal(tds)
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+
+ res = pd.concat([ps1, tds])
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+
+ # inverse
+ exp = Index(
+ [
+ pd.Timedelta("1 days"),
+ pd.Timedelta("2 days"),
+ pd.Period("2011-01", freq="M"),
+ pd.Period("2011-02", freq="M"),
+ ],
+ dtype=object,
+ )
+
+ res = tdi.append(pi1)
+ tm.assert_index_equal(res, exp)
+
+ ps1 = Series(pi1)
+ tds = Series(tdi)
+ res = tds._append_internal(ps1)
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+
+ res = pd.concat([tds, ps1])
+ tm.assert_series_equal(res, Series(exp, index=[0, 1, 0, 1]))
+
+ def test_concat_categorical(self):
+ # GH 13524
+
+ # same categories -> category
+ s1 = Series([1, 2, np.nan], dtype="category")
+ s2 = Series([2, 1, 2], dtype="category")
+
+ exp = Series([1, 2, np.nan, 2, 1, 2], dtype="category")
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+ tm.assert_series_equal(s1._append_internal(s2, ignore_index=True), exp)
+
+ # partially different categories => not-category
+ s1 = Series([3, 2], dtype="category")
+ s2 = Series([2, 1], dtype="category")
+
+ exp = Series([3, 2, 2, 1])
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+ tm.assert_series_equal(s1._append_internal(s2, ignore_index=True), exp)
+
+ # completely different categories (same dtype) => not-category
+ s1 = Series([10, 11, np.nan], dtype="category")
+ s2 = Series([np.nan, 1, 3, 2], dtype="category")
+
+ exp = Series([10, 11, np.nan, np.nan, 1, 3, 2], dtype=np.float64)
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+ tm.assert_series_equal(s1._append_internal(s2, ignore_index=True), exp)
+
+ def test_union_categorical_same_categories_different_order(self):
+ # https://github.com/pandas-dev/pandas/issues/19096
+ a = Series(Categorical(["a", "b", "c"], categories=["a", "b", "c"]))
+ b = Series(Categorical(["a", "b", "c"], categories=["b", "a", "c"]))
+ result = pd.concat([a, b], ignore_index=True)
+ expected = Series(
+ Categorical(["a", "b", "c", "a", "b", "c"], categories=["a", "b", "c"])
+ )
+ tm.assert_series_equal(result, expected)
+
+ def test_concat_categorical_coercion(self):
+ # GH 13524
+
+ # category + not-category => not-category
+ s1 = Series([1, 2, np.nan], dtype="category")
+ s2 = Series([2, 1, 2])
+
+ exp = Series([1, 2, np.nan, 2, 1, 2], dtype=np.float64)
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+ tm.assert_series_equal(s1._append_internal(s2, ignore_index=True), exp)
+
+ # result shouldn't be affected by 1st elem dtype
+ exp = Series([2, 1, 2, 1, 2, np.nan], dtype=np.float64)
+ tm.assert_series_equal(pd.concat([s2, s1], ignore_index=True), exp)
+ tm.assert_series_equal(s2._append_internal(s1, ignore_index=True), exp)
+
+ # all values are not in category => not-category
+ s1 = Series([3, 2], dtype="category")
+ s2 = Series([2, 1])
+
+ exp = Series([3, 2, 2, 1])
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+ tm.assert_series_equal(s1._append_internal(s2, ignore_index=True), exp)
+
+ exp = Series([2, 1, 3, 2])
+ tm.assert_series_equal(pd.concat([s2, s1], ignore_index=True), exp)
+ tm.assert_series_equal(s2._append_internal(s1, ignore_index=True), exp)
+
+ # completely different categories => not-category
+ s1 = Series([10, 11, np.nan], dtype="category")
+ s2 = Series([1, 3, 2])
+
+ exp = Series([10, 11, np.nan, 1, 3, 2], dtype=np.float64)
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+ tm.assert_series_equal(s1._append_internal(s2, ignore_index=True), exp)
+
+ exp = Series([1, 3, 2, 10, 11, np.nan], dtype=np.float64)
+ tm.assert_series_equal(pd.concat([s2, s1], ignore_index=True), exp)
+ tm.assert_series_equal(s2._append_internal(s1, ignore_index=True), exp)
+
+ # different dtype => not-category
+ s1 = Series([10, 11, np.nan], dtype="category")
+ s2 = Series(["a", "b", "c"])
+
+ exp = Series([10, 11, np.nan, "a", "b", "c"])
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+ tm.assert_series_equal(s1._append_internal(s2, ignore_index=True), exp)
+
+ exp = Series(["a", "b", "c", 10, 11, np.nan])
+ tm.assert_series_equal(pd.concat([s2, s1], ignore_index=True), exp)
+ tm.assert_series_equal(s2._append_internal(s1, ignore_index=True), exp)
+
+ # if normal series only contains NaN-likes => not-category
+ s1 = Series([10, 11], dtype="category")
+ s2 = Series([np.nan, np.nan, np.nan])
+
+ exp = Series([10, 11, np.nan, np.nan, np.nan])
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+ tm.assert_series_equal(s1._append_internal(s2, ignore_index=True), exp)
+
+ exp = Series([np.nan, np.nan, np.nan, 10, 11])
+ tm.assert_series_equal(pd.concat([s2, s1], ignore_index=True), exp)
+ tm.assert_series_equal(s2._append_internal(s1, ignore_index=True), exp)
+
+ def test_concat_categorical_3elem_coercion(self):
+ # GH 13524
+
+ # mixed dtypes => not-category
+ s1 = Series([1, 2, np.nan], dtype="category")
+ s2 = Series([2, 1, 2], dtype="category")
+ s3 = Series([1, 2, 1, 2, np.nan])
+
+ exp = Series([1, 2, np.nan, 2, 1, 2, 1, 2, 1, 2, np.nan], dtype="float")
+ tm.assert_series_equal(pd.concat([s1, s2, s3], ignore_index=True), exp)
+
+ exp = Series([1, 2, 1, 2, np.nan, 1, 2, np.nan, 2, 1, 2], dtype="float")
+ tm.assert_series_equal(pd.concat([s3, s1, s2], ignore_index=True), exp)
+
+ # values are all in either category => not-category
+ s1 = Series([4, 5, 6], dtype="category")
+ s2 = Series([1, 2, 3], dtype="category")
+ s3 = Series([1, 3, 4])
+
+ exp = Series([4, 5, 6, 1, 2, 3, 1, 3, 4])
+ tm.assert_series_equal(pd.concat([s1, s2, s3], ignore_index=True), exp)
+
+ exp = Series([1, 3, 4, 4, 5, 6, 1, 2, 3])
+ tm.assert_series_equal(pd.concat([s3, s1, s2], ignore_index=True), exp)
+
+ # values are all in either category => not-category
+ s1 = Series([4, 5, 6], dtype="category")
+ s2 = Series([1, 2, 3], dtype="category")
+ s3 = Series([10, 11, 12])
+
+ exp = Series([4, 5, 6, 1, 2, 3, 10, 11, 12])
+ tm.assert_series_equal(pd.concat([s1, s2, s3], ignore_index=True), exp)
+
+ exp = Series([10, 11, 12, 4, 5, 6, 1, 2, 3])
+ tm.assert_series_equal(pd.concat([s3, s1, s2], ignore_index=True), exp)
+
+ def test_concat_categorical_multi_coercion(self):
+ # GH 13524
+
+ s1 = Series([1, 3], dtype="category")
+ s2 = Series([3, 4], dtype="category")
+ s3 = Series([2, 3])
+ s4 = Series([2, 2], dtype="category")
+ s5 = Series([1, np.nan])
+ s6 = Series([1, 3, 2], dtype="category")
+
+ # mixed dtype, values are all in categories => not-category
+ exp = Series([1, 3, 3, 4, 2, 3, 2, 2, 1, np.nan, 1, 3, 2])
+ res = pd.concat([s1, s2, s3, s4, s5, s6], ignore_index=True)
+ tm.assert_series_equal(res, exp)
+
+ exp = Series([1, 3, 2, 1, np.nan, 2, 2, 2, 3, 3, 4, 1, 3])
+ res = pd.concat([s6, s5, s4, s3, s2, s1], ignore_index=True)
+ tm.assert_series_equal(res, exp)
+
+ def test_concat_categorical_ordered(self):
+ # GH 13524
+
+ s1 = Series(Categorical([1, 2, np.nan], ordered=True))
+ s2 = Series(Categorical([2, 1, 2], ordered=True))
+
+ exp = Series(Categorical([1, 2, np.nan, 2, 1, 2], ordered=True))
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+
+ exp = Series(Categorical([1, 2, np.nan, 2, 1, 2, 1, 2, np.nan], ordered=True))
+ tm.assert_series_equal(pd.concat([s1, s2, s1], ignore_index=True), exp)
+
+ def test_concat_categorical_coercion_nan(self):
+ # GH 13524
+
+ # some edge cases
+ # category + not-category => not category
+ s1 = Series(np.array([np.nan, np.nan], dtype=np.float64), dtype="category")
+ s2 = Series([np.nan, 1])
+
+ exp = Series([np.nan, np.nan, np.nan, 1])
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+
+ s1 = Series([1, np.nan], dtype="category")
+ s2 = Series([np.nan, np.nan])
+
+ exp = Series([1, np.nan, np.nan, np.nan], dtype="float")
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+
+ # mixed dtype, all nan-likes => not-category
+ s1 = Series([np.nan, np.nan], dtype="category")
+ s2 = Series([np.nan, np.nan])
+
+ exp = Series([np.nan, np.nan, np.nan, np.nan])
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+ tm.assert_series_equal(pd.concat([s2, s1], ignore_index=True), exp)
+
+ # all category nan-likes => category
+ s1 = Series([np.nan, np.nan], dtype="category")
+ s2 = Series([np.nan, np.nan], dtype="category")
+
+ exp = Series([np.nan, np.nan, np.nan, np.nan], dtype="category")
+
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+
+ def test_concat_categorical_empty(self):
+ # GH 13524
+
+ s1 = Series([], dtype="category")
+ s2 = Series([1, 2], dtype="category")
+ exp = s2.astype(object)
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+ tm.assert_series_equal(s1._append_internal(s2, ignore_index=True), exp)
+
+ tm.assert_series_equal(pd.concat([s2, s1], ignore_index=True), exp)
+ tm.assert_series_equal(s2._append_internal(s1, ignore_index=True), exp)
+
+ s1 = Series([], dtype="category")
+ s2 = Series([], dtype="category")
+
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), s2)
+ tm.assert_series_equal(s1._append_internal(s2, ignore_index=True), s2)
+
+ s1 = Series([], dtype="category")
+ s2 = Series([], dtype="object")
+
+ # different dtype => not-category
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), s2)
+ tm.assert_series_equal(s1._append_internal(s2, ignore_index=True), s2)
+ tm.assert_series_equal(pd.concat([s2, s1], ignore_index=True), s2)
+ tm.assert_series_equal(s2._append_internal(s1, ignore_index=True), s2)
+
+ s1 = Series([], dtype="category")
+ s2 = Series([np.nan, np.nan])
+
+ exp = Series([np.nan, np.nan], dtype=object)
+ tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
+
+ tm.assert_series_equal(pd.concat([s2, s1], ignore_index=True), exp)
+
+ def test_categorical_concat_append(self):
+ cat = Categorical(["a", "b"], categories=["a", "b"])
+ vals = [1, 2]
+ df = DataFrame({"cats": cat, "vals": vals})
+ cat2 = Categorical(["a", "b", "a", "b"], categories=["a", "b"])
+ vals2 = [1, 2, 1, 2]
+ exp = DataFrame({"cats": cat2, "vals": vals2}, index=Index([0, 1, 0, 1]))
+
+ tm.assert_frame_equal(pd.concat([df, df]), exp)
+
+ # GH 13524 can concat different categories
+ cat3 = Categorical(["a", "b"], categories=["a", "b", "c"])
+ vals3 = [1, 2]
+ df_different_categories = DataFrame({"cats": cat3, "vals": vals3})
+
+ res = pd.concat([df, df_different_categories], ignore_index=True)
+ exp = DataFrame({"cats": list("abab"), "vals": [1, 2, 1, 2]})
+ tm.assert_frame_equal(res, exp)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_categorical.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_categorical.py
new file mode 100644
index 0000000000000000000000000000000000000000..8cd02b49892027e4fef22cc98f57aef169657047
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_categorical.py
@@ -0,0 +1,272 @@
+from datetime import datetime
+
+import numpy as np
+
+from pandas.errors import Pandas4Warning
+
+from pandas.core.dtypes.dtypes import CategoricalDtype
+
+import pandas as pd
+from pandas import (
+ Categorical,
+ DataFrame,
+ Series,
+)
+import pandas._testing as tm
+
+
+class TestCategoricalConcat:
+ def test_categorical_concat(self, sort):
+ # See GH 10177
+ df1 = DataFrame(
+ np.arange(18, dtype="int64").reshape(6, 3), columns=["a", "b", "c"]
+ )
+
+ df2 = DataFrame(np.arange(14, dtype="int64").reshape(7, 2), columns=["a", "c"])
+
+ cat_values = ["one", "one", "two", "one", "two", "two", "one"]
+ df2["h"] = Series(Categorical(cat_values))
+
+ res = pd.concat((df1, df2), axis=0, ignore_index=True, sort=sort)
+ exp = DataFrame(
+ {
+ "a": [0, 3, 6, 9, 12, 15, 0, 2, 4, 6, 8, 10, 12],
+ "b": [
+ 1,
+ 4,
+ 7,
+ 10,
+ 13,
+ 16,
+ np.nan,
+ np.nan,
+ np.nan,
+ np.nan,
+ np.nan,
+ np.nan,
+ np.nan,
+ ],
+ "c": [2, 5, 8, 11, 14, 17, 1, 3, 5, 7, 9, 11, 13],
+ "h": [None] * 6 + cat_values,
+ }
+ )
+ exp["h"] = exp["h"].astype(df2["h"].dtype)
+ tm.assert_frame_equal(res, exp)
+
+ def test_categorical_concat_dtypes(self, using_infer_string):
+ # GH8143
+ index = ["cat", "obj", "num"]
+ cat = Categorical(["a", "b", "c"])
+ obj = Series(["a", "b", "c"])
+ num = Series([1, 2, 3])
+ df = pd.concat([Series(cat), obj, num], axis=1, keys=index)
+
+ result = df.dtypes == (object if not using_infer_string else "str")
+ expected = Series([False, True, False], index=index)
+ tm.assert_series_equal(result, expected)
+
+ result = df.dtypes == "int64"
+ expected = Series([False, False, True], index=index)
+ tm.assert_series_equal(result, expected)
+
+ result = df.dtypes == "category"
+ expected = Series([True, False, False], index=index)
+ tm.assert_series_equal(result, expected)
+
+ def test_concat_categoricalindex(self):
+ # GH 16111, categories that aren't lexsorted
+ categories = [9, 0, 1, 2, 3]
+
+ a = Series(1, index=pd.CategoricalIndex([9, 0], categories=categories))
+ b = Series(2, index=pd.CategoricalIndex([0, 1], categories=categories))
+ c = Series(3, index=pd.CategoricalIndex([1, 2], categories=categories))
+
+ result = pd.concat([a, b, c], axis=1)
+
+ exp_idx = pd.CategoricalIndex([9, 0, 1, 2], categories=categories)
+ exp = DataFrame(
+ {
+ 0: [1, 1, np.nan, np.nan],
+ 1: [np.nan, 2, 2, np.nan],
+ 2: [np.nan, np.nan, 3, 3],
+ },
+ columns=[0, 1, 2],
+ index=exp_idx,
+ )
+ tm.assert_frame_equal(result, exp)
+
+ def test_categorical_concat_preserve(self):
+ # GH 8641 series concat not preserving category dtype
+ # GH 13524 can concat different categories
+ s = Series(list("abc"), dtype="category")
+ s2 = Series(list("abd"), dtype="category")
+
+ exp = Series(list("abcabd"))
+ res = pd.concat([s, s2], ignore_index=True)
+ tm.assert_series_equal(res, exp)
+
+ exp = Series(list("abcabc"), dtype="category")
+ res = pd.concat([s, s], ignore_index=True)
+ tm.assert_series_equal(res, exp)
+
+ exp = Series(list("abcabc"), index=[0, 1, 2, 0, 1, 2], dtype="category")
+ res = pd.concat([s, s])
+ tm.assert_series_equal(res, exp)
+
+ a = Series(np.arange(6, dtype="int64"))
+ b = Series(list("aabbca"))
+
+ df2 = DataFrame({"A": a, "B": b.astype(CategoricalDtype(list("cab")))})
+ res = pd.concat([df2, df2])
+ exp = DataFrame(
+ {
+ "A": pd.concat([a, a]),
+ "B": pd.concat([b, b]).astype(CategoricalDtype(list("cab"))),
+ }
+ )
+ tm.assert_frame_equal(res, exp)
+
+ def test_categorical_index_preserver(self):
+ a = Series(np.arange(6, dtype="int64"))
+ b = Series(list("aabbca"))
+
+ df2 = DataFrame(
+ {"A": a, "B": b.astype(CategoricalDtype(list("cab")))}
+ ).set_index("B")
+ result = pd.concat([df2, df2])
+ expected = DataFrame(
+ {
+ "A": pd.concat([a, a]),
+ "B": pd.concat([b, b]).astype(CategoricalDtype(list("cab"))),
+ }
+ ).set_index("B")
+ tm.assert_frame_equal(result, expected)
+
+ # wrong categories -> uses concat_compat, which casts to object
+ msg = "Constructing a Categorical with a dtype and values containing"
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ df3 = DataFrame(
+ {"A": a, "B": Categorical(b, categories=list("abe"))}
+ ).set_index("B")
+ result = pd.concat([df2, df3])
+ expected = pd.concat(
+ [
+ df2.set_axis(df2.index.astype(object), axis=0),
+ df3.set_axis(df3.index.astype(object), axis=0),
+ ]
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_categorical_tz(self):
+ # GH-23816
+ a = Series(pd.date_range("2017-01-01", periods=2, tz="US/Pacific"))
+ b = Series(["a", "b"], dtype="category")
+ result = pd.concat([a, b], ignore_index=True)
+ expected = Series(
+ [
+ pd.Timestamp("2017-01-01", tz="US/Pacific"),
+ pd.Timestamp("2017-01-02", tz="US/Pacific"),
+ "a",
+ "b",
+ ]
+ )
+ tm.assert_series_equal(result, expected)
+
+ def test_concat_categorical_datetime(self):
+ # GH-39443
+ df1 = DataFrame(
+ {"x": Series(datetime(2021, 1, 1), index=[0], dtype="category")}
+ )
+ df2 = DataFrame(
+ {"x": Series(datetime(2021, 1, 2), index=[1], dtype="category")}
+ )
+
+ result = pd.concat([df1, df2])
+ expected = DataFrame(
+ {"x": Series([datetime(2021, 1, 1), datetime(2021, 1, 2)])}
+ )
+
+ tm.assert_equal(result, expected)
+
+ def test_concat_categorical_unchanged(self):
+ # GH-12007
+ # test fix for when concat on categorical and float
+ # coerces dtype categorical -> float
+ df = DataFrame(Series(["a", "b", "c"], dtype="category", name="A"))
+ ser = Series([0, 1, 2], index=[0, 1, 3], name="B")
+ result = pd.concat([df, ser], axis=1)
+ expected = DataFrame(
+ {
+ "A": Series(["a", "b", "c", np.nan], dtype="category"),
+ "B": Series([0, 1, np.nan, 2], dtype="float"),
+ }
+ )
+ tm.assert_equal(result, expected)
+
+ def test_categorical_concat_gh7864(self):
+ # GH 7864
+ # make sure ordering is preserved
+ df = DataFrame({"id": [1, 2, 3, 4, 5, 6], "raw_grade": list("abbaae")})
+ df["grade"] = Categorical(df["raw_grade"])
+ df["grade"].cat.set_categories(["e", "a", "b"])
+
+ df1 = df[0:3]
+ df2 = df[3:]
+
+ tm.assert_index_equal(df["grade"].cat.categories, df1["grade"].cat.categories)
+ tm.assert_index_equal(df["grade"].cat.categories, df2["grade"].cat.categories)
+
+ dfx = pd.concat([df1, df2])
+ tm.assert_index_equal(df["grade"].cat.categories, dfx["grade"].cat.categories)
+
+ def test_categorical_index_upcast(self):
+ # GH 17629
+ # test upcasting to object when concatenating on categorical indexes
+ # with non-identical categories
+
+ a = DataFrame({"foo": [1, 2]}, index=Categorical(["foo", "bar"]))
+ b = DataFrame({"foo": [4, 3]}, index=Categorical(["baz", "bar"]))
+
+ res = pd.concat([a, b])
+ exp = DataFrame({"foo": [1, 2, 4, 3]}, index=["foo", "bar", "baz", "bar"])
+
+ tm.assert_equal(res, exp)
+
+ a = Series([1, 2], index=Categorical(["foo", "bar"]))
+ b = Series([4, 3], index=Categorical(["baz", "bar"]))
+
+ res = pd.concat([a, b])
+ exp = Series([1, 2, 4, 3], index=["foo", "bar", "baz", "bar"])
+
+ tm.assert_equal(res, exp)
+
+ def test_categorical_missing_from_one_frame(self):
+ # GH 25412
+ df1 = DataFrame({"f1": [1, 2, 3]})
+ df2 = DataFrame({"f1": [2, 3, 1], "f2": Series([4, 4, 4]).astype("category")})
+ result = pd.concat([df1, df2], sort=True)
+ dtype = CategoricalDtype([4])
+ expected = DataFrame(
+ {
+ "f1": [1, 2, 3, 2, 3, 1],
+ "f2": Categorical.from_codes([-1, -1, -1, 0, 0, 0], dtype=dtype),
+ },
+ index=[0, 1, 2, 0, 1, 2],
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_categorical_same_categories_different_order(self):
+ # https://github.com/pandas-dev/pandas/issues/24845
+
+ c1 = pd.CategoricalIndex(["a", "a"], categories=["a", "b"], ordered=False)
+ c2 = pd.CategoricalIndex(["b", "b"], categories=["b", "a"], ordered=False)
+ c3 = pd.CategoricalIndex(
+ ["a", "a", "b", "b"], categories=["a", "b"], ordered=False
+ )
+
+ df1 = DataFrame({"A": [1, 2]}, index=c1)
+ df2 = DataFrame({"A": [3, 4]}, index=c2)
+
+ result = pd.concat((df1, df2))
+ expected = DataFrame({"A": [1, 2, 3, 4]}, index=c3)
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_concat.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_concat.py
new file mode 100644
index 0000000000000000000000000000000000000000..50a290a883e303183055139b0d59d028133d1c7f
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_concat.py
@@ -0,0 +1,1008 @@
+from collections import (
+ abc,
+ deque,
+)
+from collections.abc import Iterator
+from datetime import datetime
+from decimal import Decimal
+import itertools
+
+import numpy as np
+import pytest
+
+from pandas.errors import (
+ InvalidIndexError,
+ Pandas4Warning,
+)
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ MultiIndex,
+ PeriodIndex,
+ RangeIndex,
+ Series,
+ concat,
+ date_range,
+)
+import pandas._testing as tm
+from pandas.core.arrays import SparseArray
+from pandas.tests.extension.decimal import to_decimal
+
+
+class TestConcatenate:
+ def test_append_concat(self):
+ # GH#1815
+ d1 = date_range("12/31/1990", "12/31/1999", freq="YE-DEC")
+ d2 = date_range("12/31/2000", "12/31/2009", freq="YE-DEC")
+
+ s1 = Series(np.random.default_rng(2).standard_normal(10), d1)
+ s2 = Series(np.random.default_rng(2).standard_normal(10), d2)
+
+ s1 = s1.to_period()
+ s2 = s2.to_period()
+
+ # drops index
+ result = concat([s1, s2])
+ assert isinstance(result.index, PeriodIndex)
+ assert result.index[0] == s1.index[0]
+
+ def test_concat_copy(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((4, 3)))
+ df2 = DataFrame(np.random.default_rng(2).integers(0, 10, size=4).reshape(4, 1))
+ df3 = DataFrame({5: "foo"}, index=range(4))
+
+ result = concat([df, df2, df3], axis=1)
+
+ for block in result._mgr.blocks:
+ arr = block.values
+ if arr.dtype.kind == "f":
+ assert arr.base is df._mgr.blocks[0].values.base
+ elif arr.dtype.kind in ["i", "u"]:
+ assert arr.base is df2._mgr.blocks[0].values.base
+ elif arr.dtype == object:
+ assert arr.base is not None
+ elif arr.dtype == "string":
+ tm.shares_memory(arr, df3._mgr.blocks[0].values)
+
+ # Float block was consolidated.
+ df4 = DataFrame(np.random.default_rng(2).standard_normal((4, 1)))
+ result = concat([df, df2, df3, df4], axis=1)
+ for blocks in result._mgr.blocks:
+ arr = blocks.values
+ if arr.dtype.kind == "f":
+ # this is a view on some array in either df or df4
+ assert any(
+ np.shares_memory(arr, block.values)
+ for block in itertools.chain(df._mgr.blocks, df4._mgr.blocks)
+ )
+ elif arr.dtype.kind in ["i", "u"]:
+ assert arr.base is df2._mgr.blocks[0].values.base
+ elif arr.dtype == object:
+ # this is a view on df3
+ assert any(
+ np.shares_memory(arr, block.values) for block in df3._mgr.blocks
+ )
+
+ def test_concat_with_group_keys(self):
+ # axis=0
+ df = DataFrame(np.random.default_rng(2).standard_normal((3, 4)))
+ df2 = DataFrame(np.random.default_rng(2).standard_normal((4, 4)))
+
+ result = concat([df, df2], keys=[0, 1])
+ exp_index = MultiIndex.from_arrays(
+ [[0, 0, 0, 1, 1, 1, 1], [0, 1, 2, 0, 1, 2, 3]]
+ )
+ expected = DataFrame(np.r_[df.values, df2.values], index=exp_index)
+ tm.assert_frame_equal(result, expected)
+
+ result = concat([df, df], keys=[0, 1])
+ exp_index2 = MultiIndex.from_arrays([[0, 0, 0, 1, 1, 1], [0, 1, 2, 0, 1, 2]])
+ expected = DataFrame(np.r_[df.values, df.values], index=exp_index2)
+ tm.assert_frame_equal(result, expected)
+
+ # axis=1
+ df = DataFrame(np.random.default_rng(2).standard_normal((4, 3)))
+ df2 = DataFrame(np.random.default_rng(2).standard_normal((4, 4)))
+
+ result = concat([df, df2], keys=[0, 1], axis=1)
+ expected = DataFrame(np.c_[df.values, df2.values], columns=exp_index)
+ tm.assert_frame_equal(result, expected)
+
+ result = concat([df, df], keys=[0, 1], axis=1)
+ expected = DataFrame(np.c_[df.values, df.values], columns=exp_index2)
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_keys_specific_levels(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 4)))
+ pieces = [df.iloc[:, [0, 1]], df.iloc[:, [2]], df.iloc[:, [3]]]
+ level = ["three", "two", "one", "zero"]
+ result = concat(
+ pieces,
+ axis=1,
+ keys=["one", "two", "three"],
+ levels=[level],
+ names=["group_key"],
+ )
+
+ tm.assert_index_equal(result.columns.levels[0], Index(level, name="group_key"))
+ tm.assert_index_equal(result.columns.levels[1], Index([0, 1, 2, 3]))
+
+ assert result.columns.names == ["group_key", None]
+
+ @pytest.mark.parametrize("mapping", ["mapping", "dict"])
+ def test_concat_mapping(self, mapping, non_dict_mapping_subclass):
+ constructor = dict if mapping == "dict" else non_dict_mapping_subclass
+ frames = constructor(
+ {
+ "foo": DataFrame(np.random.default_rng(2).standard_normal((4, 3))),
+ "bar": DataFrame(np.random.default_rng(2).standard_normal((4, 3))),
+ "baz": DataFrame(np.random.default_rng(2).standard_normal((4, 3))),
+ "qux": DataFrame(np.random.default_rng(2).standard_normal((4, 3))),
+ }
+ )
+
+ sorted_keys = list(frames.keys())
+
+ result = concat(frames)
+ expected = concat([frames[k] for k in sorted_keys], keys=sorted_keys)
+ tm.assert_frame_equal(result, expected)
+
+ result = concat(frames, axis=1)
+ expected = concat([frames[k] for k in sorted_keys], keys=sorted_keys, axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ keys = ["baz", "foo", "bar"]
+ result = concat(frames, keys=keys)
+ expected = concat([frames[k] for k in keys], keys=keys)
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_keys_and_levels(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((1, 3)))
+ df2 = DataFrame(np.random.default_rng(2).standard_normal((1, 4)))
+
+ levels = [["foo", "baz"], ["one", "two"]]
+ names = ["first", "second"]
+ result = concat(
+ [df, df2, df, df2],
+ keys=[("foo", "one"), ("foo", "two"), ("baz", "one"), ("baz", "two")],
+ levels=levels,
+ names=names,
+ )
+ expected = concat([df, df2, df, df2])
+ exp_index = MultiIndex(
+ levels=[*levels, [0]],
+ codes=[[0, 0, 1, 1], [0, 1, 0, 1], [0, 0, 0, 0]],
+ names=[*names, None],
+ )
+ expected.index = exp_index
+
+ tm.assert_frame_equal(result, expected)
+
+ # no names
+ result = concat(
+ [df, df2, df, df2],
+ keys=[("foo", "one"), ("foo", "two"), ("baz", "one"), ("baz", "two")],
+ levels=levels,
+ )
+ assert result.index.names == (None,) * 3
+
+ # no levels
+ result = concat(
+ [df, df2, df, df2],
+ keys=[("foo", "one"), ("foo", "two"), ("baz", "one"), ("baz", "two")],
+ names=["first", "second"],
+ )
+ assert result.index.names == ("first", "second", None)
+ tm.assert_index_equal(
+ result.index.levels[0], Index(["baz", "foo"], name="first")
+ )
+
+ def test_concat_keys_levels_no_overlap(self):
+ # GH #1406
+ df = DataFrame(np.random.default_rng(2).standard_normal((1, 3)), index=["a"])
+ df2 = DataFrame(np.random.default_rng(2).standard_normal((1, 4)), index=["b"])
+
+ msg = "Values not found in passed level"
+ with pytest.raises(ValueError, match=msg):
+ concat([df, df], keys=["one", "two"], levels=[["foo", "bar", "baz"]])
+
+ msg = "Key one not in level"
+ with pytest.raises(ValueError, match=msg):
+ concat([df, df2], keys=["one", "two"], levels=[["foo", "bar", "baz"]])
+
+ def test_crossed_dtypes_weird_corner(self):
+ columns = ["A", "B", "C", "D"]
+ df1 = DataFrame(
+ {
+ "A": np.array([1, 2, 3, 4], dtype="f8"),
+ "B": np.array([1, 2, 3, 4], dtype="i8"),
+ "C": np.array([1, 2, 3, 4], dtype="f8"),
+ "D": np.array([1, 2, 3, 4], dtype="i8"),
+ },
+ columns=columns,
+ )
+
+ df2 = DataFrame(
+ {
+ "A": np.array([1, 2, 3, 4], dtype="i8"),
+ "B": np.array([1, 2, 3, 4], dtype="f8"),
+ "C": np.array([1, 2, 3, 4], dtype="i8"),
+ "D": np.array([1, 2, 3, 4], dtype="f8"),
+ },
+ columns=columns,
+ )
+
+ appended = concat([df1, df2], ignore_index=True)
+ expected = DataFrame(
+ np.concatenate([df1.values, df2.values], axis=0), columns=columns
+ )
+ tm.assert_frame_equal(appended, expected)
+
+ df = DataFrame(np.random.default_rng(2).standard_normal((1, 3)), index=["a"])
+ df2 = DataFrame(np.random.default_rng(2).standard_normal((1, 4)), index=["b"])
+ result = concat([df, df2], keys=["one", "two"], names=["first", "second"])
+ assert result.index.names == ("first", "second")
+
+ def test_with_mixed_tuples(self, sort):
+ # 10697
+ # columns have mixed tuples, so handle properly
+ df1 = DataFrame({"A": "foo", ("B", 1): "bar"}, index=range(2))
+ df2 = DataFrame({"B": "foo", ("B", 1): "bar"}, index=range(2))
+
+ # it works
+ concat([df1, df2], sort=sort)
+
+ def test_concat_mixed_objs_columns(self):
+ # Test column-wise concat for mixed series/frames (axis=1)
+ # G2385
+
+ index = date_range("01-Jan-2013", periods=10, freq="h")
+ arr = np.arange(10, dtype="int64")
+ s1 = Series(arr, index=index)
+ s2 = Series(arr, index=index)
+ df = DataFrame(arr.reshape(-1, 1), index=index)
+
+ expected = DataFrame(
+ np.repeat(arr, 2).reshape(-1, 2), index=index, columns=[0, 0]
+ )
+ result = concat([df, df], axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ expected = DataFrame(
+ np.repeat(arr, 2).reshape(-1, 2), index=index, columns=[0, 1]
+ )
+ result = concat([s1, s2], axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ expected = DataFrame(
+ np.repeat(arr, 3).reshape(-1, 3), index=index, columns=[0, 1, 2]
+ )
+ result = concat([s1, s2, s1], axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ expected = DataFrame(
+ np.repeat(arr, 5).reshape(-1, 5), index=index, columns=[0, 0, 1, 2, 3]
+ )
+ result = concat([s1, df, s2, s2, s1], axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ # with names
+ s1.name = "foo"
+ expected = DataFrame(
+ np.repeat(arr, 3).reshape(-1, 3), index=index, columns=["foo", 0, 0]
+ )
+ result = concat([s1, df, s2], axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ s2.name = "bar"
+ expected = DataFrame(
+ np.repeat(arr, 3).reshape(-1, 3), index=index, columns=["foo", 0, "bar"]
+ )
+ result = concat([s1, df, s2], axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ # ignore index
+ expected = DataFrame(
+ np.repeat(arr, 3).reshape(-1, 3), index=index, columns=[0, 1, 2]
+ )
+ result = concat([s1, df, s2], axis=1, ignore_index=True)
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_mixed_objs_index(self):
+ # Test row-wise concat for mixed series/frames with a common name
+ # GH2385, GH15047
+
+ index = date_range("01-Jan-2013", periods=10, freq="h")
+ arr = np.arange(10, dtype="int64")
+ s1 = Series(arr, index=index)
+ s2 = Series(arr, index=index)
+ df = DataFrame(arr.reshape(-1, 1), index=index)
+
+ expected = DataFrame(
+ np.tile(arr, 3).reshape(-1, 1), index=index.tolist() * 3, columns=[0]
+ )
+ result = concat([s1, df, s2])
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_mixed_objs_index_names(self):
+ # Test row-wise concat for mixed series/frames with distinct names
+ # GH2385, GH15047
+ # GH #60723 & GH #56257 (Updated the test case,
+ # as the above GH PR ones were incorrect)
+
+ index = date_range("01-Jan-2013", periods=10, freq="h")
+ arr = np.arange(10, dtype="int64")
+ s1 = Series(arr, index=index, name="foo")
+ s2 = Series(arr, index=index, name="bar")
+ df = DataFrame(arr.reshape(-1, 1), index=index)
+
+ expected = DataFrame(
+ np.kron(np.where(np.identity(3) == 1, 1, np.nan), arr).T,
+ index=index.tolist() * 3,
+ columns=["foo", 0, "bar"],
+ )
+ result = concat([s1, df, s2])
+ tm.assert_frame_equal(result, expected)
+
+ expected = DataFrame(
+ np.kron(np.where(np.identity(3) == 1, 1, np.nan), arr).T,
+ index=np.arange(30, dtype=np.int64),
+ columns=["foo", 0, "bar"],
+ )
+ result = concat([s1, df, s2], ignore_index=True)
+ tm.assert_frame_equal(result, expected)
+
+ def test_dtype_coercion(self):
+ # 12411
+ df = DataFrame({"date": [pd.Timestamp("20130101").tz_localize("UTC"), pd.NaT]})
+
+ result = concat([df.iloc[[0]], df.iloc[[1]]])
+ tm.assert_series_equal(result.dtypes, df.dtypes)
+
+ # 12045
+ df = DataFrame({"date": [datetime(2012, 1, 1), datetime(1012, 1, 2)]})
+ result = concat([df.iloc[[0]], df.iloc[[1]]])
+ tm.assert_series_equal(result.dtypes, df.dtypes)
+
+ # 11594
+ df = DataFrame({"text": ["some words"] + [None] * 9})
+ result = concat([df.iloc[[0]], df.iloc[[1]]])
+ tm.assert_series_equal(result.dtypes, df.dtypes)
+
+ def test_concat_single_with_key(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 4)))
+
+ result = concat([df], keys=["foo"])
+ expected = concat([df, df], keys=["foo", "bar"])
+ tm.assert_frame_equal(result, expected[:10])
+
+ def test_concat_no_items_raises(self):
+ with pytest.raises(ValueError, match="No objects to concatenate"):
+ concat([])
+
+ def test_concat_exclude_none(self):
+ df = DataFrame(np.random.default_rng(2).standard_normal((10, 4)))
+
+ pieces = [df[:5], None, None, df[5:]]
+ result = concat(pieces)
+ tm.assert_frame_equal(result, df)
+ with pytest.raises(ValueError, match="All objects passed were None"):
+ concat([None, None])
+
+ def test_concat_keys_with_none(self):
+ # #1649
+ df0 = DataFrame([[10, 20, 30], [10, 20, 30], [10, 20, 30]])
+
+ result = concat({"a": None, "b": df0, "c": df0[:2], "d": df0[:1], "e": df0})
+ expected = concat({"b": df0, "c": df0[:2], "d": df0[:1], "e": df0})
+ tm.assert_frame_equal(result, expected)
+
+ result = concat(
+ [None, df0, df0[:2], df0[:1], df0], keys=["a", "b", "c", "d", "e"]
+ )
+ expected = concat([df0, df0[:2], df0[:1], df0], keys=["b", "c", "d", "e"])
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("klass", [range, RangeIndex])
+ @pytest.mark.parametrize("include_none", [True, False])
+ def test_concat_preserves_rangeindex(self, klass, include_none):
+ df = DataFrame([1, 2])
+ df2 = DataFrame([3, 4])
+ data = [df, None, df2, None] if include_none else [df, df2]
+ keys_length = 4 if include_none else 2
+ result = concat(data, keys=klass(keys_length))
+ expected = DataFrame(
+ [1, 2, 3, 4],
+ index=MultiIndex(
+ levels=(
+ RangeIndex(start=0, stop=keys_length, step=keys_length / 2),
+ RangeIndex(start=0, stop=2, step=1),
+ ),
+ codes=(
+ np.array([0, 0, 1, 1], dtype=np.int8),
+ np.array([0, 1, 0, 1], dtype=np.int8),
+ ),
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_bug_1719(self):
+ ts1 = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+ ts2 = ts1[::2]
+
+ # to join with union
+ # these two are of different length!
+ left = concat([ts1, ts2], join="outer", axis=1)
+ msg = "Sorting by default when concatenating all DatetimeIndex is deprecated"
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ right = concat([ts2, ts1], join="outer", axis=1)
+
+ assert len(left) == len(right)
+
+ def test_concat_bug_2972(self):
+ ts0 = Series(np.zeros(5))
+ ts1 = Series(np.ones(5))
+ ts0.name = ts1.name = "same name"
+ result = concat([ts0, ts1], axis=1)
+
+ expected = DataFrame({0: ts0, 1: ts1})
+ expected.columns = ["same name", "same name"]
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_bug_3602(self):
+ # GH 3602, duplicate columns
+ df1 = DataFrame(
+ {
+ "firmNo": [0, 0, 0, 0],
+ "prc": [6, 6, 6, 6],
+ "stringvar": ["rrr", "rrr", "rrr", "rrr"],
+ }
+ )
+ df2 = DataFrame(
+ {"C": [9, 10, 11, 12], "misc": [1, 2, 3, 4], "prc": [6, 6, 6, 6]}
+ )
+ expected = DataFrame(
+ [
+ [0, 6, "rrr", 9, 1, 6],
+ [0, 6, "rrr", 10, 2, 6],
+ [0, 6, "rrr", 11, 3, 6],
+ [0, 6, "rrr", 12, 4, 6],
+ ]
+ )
+ expected.columns = ["firmNo", "prc", "stringvar", "C", "misc", "prc"]
+
+ result = concat([df1, df2], axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_iterables(self):
+ # GH8645 check concat works with tuples, list, generators, and weird
+ # stuff like deque and custom iterables
+ df1 = DataFrame([1, 2, 3])
+ df2 = DataFrame([4, 5, 6])
+ expected = DataFrame([1, 2, 3, 4, 5, 6])
+ tm.assert_frame_equal(concat((df1, df2), ignore_index=True), expected)
+ tm.assert_frame_equal(concat([df1, df2], ignore_index=True), expected)
+ tm.assert_frame_equal(
+ concat((df for df in (df1, df2)), ignore_index=True), expected
+ )
+ tm.assert_frame_equal(concat(deque((df1, df2)), ignore_index=True), expected)
+
+ class CustomIterator1:
+ def __len__(self) -> int:
+ return 2
+
+ def __getitem__(self, index):
+ try:
+ return {0: df1, 1: df2}[index]
+ except KeyError as err:
+ raise IndexError from err
+
+ tm.assert_frame_equal(concat(CustomIterator1(), ignore_index=True), expected)
+
+ class CustomIterator2(abc.Iterable):
+ def __iter__(self) -> Iterator:
+ yield df1
+ yield df2
+
+ tm.assert_frame_equal(concat(CustomIterator2(), ignore_index=True), expected)
+
+ def test_concat_order(self):
+ # GH 17344, GH#47331
+ dfs = [DataFrame(index=range(3), columns=["a", 1, None])]
+ dfs += [DataFrame(index=range(3), columns=[None, 1, "a"]) for _ in range(100)]
+
+ result = concat(dfs, sort=True).columns
+ expected = Index([1, "a", None])
+ tm.assert_index_equal(result, expected)
+
+ def test_concat_different_extension_dtypes_upcasts(self):
+ a = Series(pd.array([1, 2], dtype="Int64"))
+ b = Series(to_decimal([1, 2]))
+
+ result = concat([a, b], ignore_index=True)
+ expected = Series([1, 2, Decimal(1), Decimal(2)], dtype=object)
+ tm.assert_series_equal(result, expected)
+
+ def test_concat_ordered_dict(self):
+ # GH 21510
+ expected = concat(
+ [Series(range(3)), Series(range(4))], keys=["First", "Another"]
+ )
+ result = concat({"First": Series(range(3)), "Another": Series(range(4))})
+ tm.assert_series_equal(result, expected)
+
+ def test_concat_duplicate_indices_raise(self):
+ # GH 45888: test raise for concat DataFrames with duplicate indices
+ # https://github.com/pandas-dev/pandas/issues/36263
+ df1 = DataFrame(
+ np.random.default_rng(2).standard_normal(5),
+ index=[0, 1, 2, 3, 3],
+ columns=["a"],
+ )
+ df2 = DataFrame(
+ np.random.default_rng(2).standard_normal(5),
+ index=[0, 1, 2, 2, 4],
+ columns=["b"],
+ )
+ msg = "Reindexing only valid with uniquely valued Index objects"
+ with pytest.raises(InvalidIndexError, match=msg):
+ concat([df1, df2], axis=1)
+
+
+def test_concat_no_unnecessary_upcast(float_numpy_dtype, frame_or_series):
+ # GH 13247
+ dims = frame_or_series(dtype=object).ndim
+ dt = float_numpy_dtype
+
+ dfs = [
+ frame_or_series(np.array([1], dtype=dt, ndmin=dims)),
+ frame_or_series(np.array([np.nan], dtype=dt, ndmin=dims)),
+ frame_or_series(np.array([5], dtype=dt, ndmin=dims)),
+ ]
+ x = concat(dfs)
+ assert x.values.dtype == dt
+
+
+def test_concat_will_upcast(frame_or_series, any_signed_int_numpy_dtype):
+ dt = any_signed_int_numpy_dtype
+ dims = frame_or_series().ndim
+ dfs = [
+ frame_or_series(np.array([1], dtype=dt, ndmin=dims)),
+ frame_or_series(np.array([np.nan], ndmin=dims)),
+ frame_or_series(np.array([5], dtype=dt, ndmin=dims)),
+ ]
+ x = concat(dfs)
+ assert x.values.dtype == "float64"
+
+
+def test_concat_empty_and_non_empty_frame_regression():
+ # GH 18178 regression test
+ df1 = DataFrame({"foo": [1]})
+ df2 = DataFrame({"foo": []})
+ expected = DataFrame({"foo": [1.0]})
+ result = concat([df1, df2])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_concat_sparse():
+ # GH 23557
+ a = Series(SparseArray([0, 1, 2]))
+ expected = DataFrame(data=[[0, 0], [1, 1], [2, 2]]).astype(
+ pd.SparseDtype(np.int64, 0)
+ )
+ result = concat([a, a], axis=1)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_concat_dense_sparse():
+ # GH 30668
+ dtype = pd.SparseDtype(np.float64, None)
+ a = Series(pd.arrays.SparseArray([1, None]), dtype=dtype)
+ b = Series([1], dtype=float)
+ expected = Series(data=[1, None, 1], index=[0, 1, 0]).astype(dtype)
+ result = concat([a, b], axis=0)
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("keys", [["e", "f", "f"], ["f", "e", "f"]])
+def test_duplicate_keys(keys):
+ # GH 33654
+ df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+ s1 = Series([7, 8, 9], name="c")
+ s2 = Series([10, 11, 12], name="d")
+ result = concat([df, s1, s2], axis=1, keys=keys)
+ expected_values = [[1, 4, 7, 10], [2, 5, 8, 11], [3, 6, 9, 12]]
+ expected_columns = MultiIndex.from_tuples(
+ [(keys[0], "a"), (keys[0], "b"), (keys[1], "c"), (keys[2], "d")]
+ )
+ expected = DataFrame(expected_values, columns=expected_columns)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_duplicate_keys_same_frame():
+ # GH 43595
+ keys = ["e", "e"]
+ df = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+ result = concat([df, df], axis=1, keys=keys)
+ expected_values = [[1, 4, 1, 4], [2, 5, 2, 5], [3, 6, 3, 6]]
+ expected_columns = MultiIndex.from_tuples(
+ [(keys[0], "a"), (keys[0], "b"), (keys[1], "a"), (keys[1], "b")]
+ )
+ expected = DataFrame(expected_values, columns=expected_columns)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager|Passing a SingleBlockManager:DeprecationWarning"
+)
+@pytest.mark.parametrize(
+ "obj",
+ [
+ tm.SubclassedDataFrame({"A": np.arange(0, 10)}),
+ tm.SubclassedSeries(np.arange(0, 10), name="A"),
+ ],
+)
+def test_concat_preserves_subclass(obj):
+ # GH28330 -- preserve subclass
+
+ result = concat([obj, obj])
+ assert isinstance(result, type(obj))
+
+
+def test_concat_frame_axis0_extension_dtypes():
+ # preserve extension dtype (through common_dtype mechanism)
+ df1 = DataFrame({"a": pd.array([1, 2, 3], dtype="Int64")})
+ df2 = DataFrame({"a": np.array([4, 5, 6])})
+
+ result = concat([df1, df2], ignore_index=True)
+ expected = DataFrame({"a": [1, 2, 3, 4, 5, 6]}, dtype="Int64")
+ tm.assert_frame_equal(result, expected)
+
+ result = concat([df2, df1], ignore_index=True)
+ expected = DataFrame({"a": [4, 5, 6, 1, 2, 3]}, dtype="Int64")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_concat_preserves_extension_int64_dtype():
+ # GH 24768
+ df_a = DataFrame({"a": [-1]}, dtype="Int64")
+ df_b = DataFrame({"b": [1]}, dtype="Int64")
+ result = concat([df_a, df_b], ignore_index=True)
+ expected = DataFrame({"a": [-1, None], "b": [None, 1]}, dtype="Int64")
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "dtype1,dtype2,expected_dtype",
+ [
+ ("bool", "bool", "bool"),
+ ("boolean", "bool", "boolean"),
+ ("bool", "boolean", "boolean"),
+ ("boolean", "boolean", "boolean"),
+ ],
+)
+def test_concat_bool_types(dtype1, dtype2, expected_dtype):
+ # GH 42800
+ ser1 = Series([True, False], dtype=dtype1)
+ ser2 = Series([False, True], dtype=dtype2)
+ result = concat([ser1, ser2], ignore_index=True)
+ expected = Series([True, False, False, True], dtype=expected_dtype)
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ ("keys", "integrity"),
+ [
+ (["red"] * 3, True),
+ (["red"] * 3, False),
+ (["red", "blue", "red"], False),
+ (["red", "blue", "red"], True),
+ ],
+)
+def test_concat_repeated_keys(keys, integrity):
+ # GH: 20816
+ series_list = [Series({"a": 1}), Series({"b": 2}), Series({"c": 3})]
+ result = concat(series_list, keys=keys, verify_integrity=integrity)
+ tuples = list(zip(keys, ["a", "b", "c"]))
+ expected = Series([1, 2, 3], index=MultiIndex.from_tuples(tuples))
+ tm.assert_series_equal(result, expected)
+
+
+def test_concat_null_object_with_dti():
+ # GH#40841
+ dti = pd.DatetimeIndex(
+ ["2021-04-08 21:21:14+00:00"], dtype="datetime64[ns, UTC]", name="Time (UTC)"
+ )
+ right = DataFrame(data={"C": [0.5274]}, index=dti)
+
+ idx = Index([None], dtype="object", name="Maybe Time (UTC)")
+ left = DataFrame(data={"A": [None], "B": [np.nan]}, index=idx)
+
+ result = concat([left, right], axis="columns")
+
+ exp_index = Index([None, dti[0]], dtype=object)
+ expected = DataFrame(
+ {
+ "A": np.array([None, np.nan], dtype=object),
+ "B": [np.nan, np.nan],
+ "C": [np.nan, 0.5274],
+ },
+ index=exp_index,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_concat_multiindex_with_empty_rangeindex():
+ # GH#41234
+ mi = MultiIndex.from_tuples([("B", 1), ("C", 1)])
+ df1 = DataFrame([[1, 2]], columns=mi)
+ df2 = DataFrame(index=[1], columns=RangeIndex(0))
+
+ result = concat([df1, df2])
+ expected = DataFrame([[1, 2], [np.nan, np.nan]], columns=mi)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "data",
+ [
+ Series(data=[1, 2]),
+ DataFrame(
+ data={
+ "col1": [1, 2],
+ }
+ ),
+ DataFrame(dtype=float),
+ Series(dtype=float),
+ ],
+)
+def test_concat_drop_attrs(data):
+ # GH#41828
+ df1 = data.copy()
+ df1.attrs = {1: 1}
+ df2 = data.copy()
+ df2.attrs = {1: 2}
+ df = concat([df1, df2])
+ assert len(df.attrs) == 0
+
+
+@pytest.mark.parametrize(
+ "data",
+ [
+ Series(data=[1, 2]),
+ DataFrame(
+ data={
+ "col1": [1, 2],
+ }
+ ),
+ DataFrame(dtype=float),
+ Series(dtype=float),
+ ],
+)
+def test_concat_retain_attrs(data):
+ # GH#41828
+ df1 = data.copy()
+ df1.attrs = {1: 1}
+ df2 = data.copy()
+ df2.attrs = {1: 1}
+ df = concat([df1, df2])
+ assert df.attrs[1] == 1
+
+
+@pytest.mark.parametrize("df_dtype", ["float64", "int64", "datetime64[ns]"])
+@pytest.mark.parametrize("empty_dtype", [None, "float64", "object"])
+def test_concat_ignore_empty_object_float(empty_dtype, df_dtype):
+ # https://github.com/pandas-dev/pandas/issues/45637
+ df = DataFrame({"foo": [1, 2], "bar": [1, 2]}, dtype=df_dtype)
+ empty = DataFrame(columns=["foo", "bar"], dtype=empty_dtype)
+
+ needs_update = False
+ if df_dtype == "datetime64[ns]" or (
+ df_dtype == "float64" and empty_dtype != "float64"
+ ):
+ needs_update = True
+
+ result = concat([empty, df])
+ expected = df
+ if df_dtype == "int64":
+ # TODO what exact behaviour do we want for integer eventually?
+ if empty_dtype == "float64":
+ expected = df.astype("float64")
+ else:
+ expected = df.astype("object")
+
+ if needs_update:
+ # GH#40893 changed the expected here to retain dependence on empty
+ expected = expected.astype(object)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("df_dtype", ["float64", "int64", "datetime64[ns]"])
+@pytest.mark.parametrize("empty_dtype", [None, "float64", "object"])
+def test_concat_ignore_all_na_object_float(empty_dtype, df_dtype):
+ df = DataFrame({"foo": [1, 2], "bar": [1, 2]}, dtype=df_dtype)
+ empty = DataFrame({"foo": [np.nan], "bar": [np.nan]}, dtype=empty_dtype)
+
+ if df_dtype == "int64":
+ # TODO what exact behaviour do we want for integer eventually?
+ if empty_dtype == "object":
+ df_dtype = "object"
+ else:
+ df_dtype = "float64"
+
+ needs_update = False
+ if empty_dtype != df_dtype and empty_dtype is not None:
+ needs_update = True
+ elif df_dtype == "datetime64[ns]":
+ needs_update = True
+
+ result = concat([empty, df], ignore_index=True)
+
+ expected = DataFrame({"foo": [np.nan, 1, 2], "bar": [np.nan, 1, 2]}, dtype=df_dtype)
+ if needs_update:
+ # GH#40893 changed the expected here to retain dependence on empty
+ expected = expected.astype(object)
+ expected.iloc[0] = np.nan
+ tm.assert_frame_equal(result, expected)
+
+
+def test_concat_ignore_empty_from_reindex():
+ # https://github.com/pandas-dev/pandas/pull/43507#issuecomment-920375856
+ df1 = DataFrame({"a": [1], "b": [pd.Timestamp("2012-01-01")]})
+ df2 = DataFrame({"a": [2]})
+
+ aligned = df2.reindex(columns=df1.columns)
+
+ result = concat([df1, aligned], ignore_index=True)
+
+ expected = DataFrame(
+ {
+ "a": [1, 2],
+ "b": pd.array([pd.Timestamp("2012-01-01"), np.nan], dtype=object),
+ },
+ dtype=object,
+ )
+ expected["a"] = expected["a"].astype("int64")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_concat_mismatched_keys_length():
+ # GH#43485
+ ser = Series(range(5))
+ sers = [ser + n for n in range(4)]
+ keys = ["A", "B", "C"]
+
+ msg = r"The length of the keys"
+ with pytest.raises(ValueError, match=msg):
+ concat(sers, keys=keys, axis=1)
+ with pytest.raises(ValueError, match=msg):
+ concat(sers, keys=keys, axis=0)
+ with pytest.raises(ValueError, match=msg):
+ concat((x for x in sers), keys=(y for y in keys), axis=1)
+ with pytest.raises(ValueError, match=msg):
+ concat((x for x in sers), keys=(y for y in keys), axis=0)
+
+
+def test_concat_multiindex_with_category():
+ df1 = DataFrame(
+ {
+ "c1": Series(list("abc"), dtype="category"),
+ "c2": Series(list("eee"), dtype="category"),
+ "i2": Series([1, 2, 3]),
+ }
+ )
+ df1 = df1.set_index(["c1", "c2"])
+ df2 = DataFrame(
+ {
+ "c1": Series(list("abc"), dtype="category"),
+ "c2": Series(list("eee"), dtype="category"),
+ "i2": Series([4, 5, 6]),
+ }
+ )
+ df2 = df2.set_index(["c1", "c2"])
+ result = concat([df1, df2])
+ expected = DataFrame(
+ {
+ "c1": Series(list("abcabc"), dtype="category"),
+ "c2": Series(list("eeeeee"), dtype="category"),
+ "i2": Series([1, 2, 3, 4, 5, 6]),
+ }
+ )
+ expected = expected.set_index(["c1", "c2"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_concat_ea_upcast():
+ # GH#54848
+ df1 = DataFrame(["a"], dtype="string")
+ df2 = DataFrame([1], dtype="Int64")
+ result = concat([df1, df2])
+ expected = DataFrame(["a", 1], index=[0, 0])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_concat_none_with_timezone_timestamp():
+ # GH#52093
+ df1 = DataFrame([{"A": None}])
+ df2 = DataFrame([{"A": pd.Timestamp("1990-12-20 00:00:00+00:00")}])
+ result = concat([df1, df2], ignore_index=True)
+ expected = DataFrame(
+ {"A": [None, pd.Timestamp("1990-12-20 00:00:00+00:00")]}, dtype=object
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_concat_with_series_and_frame_returns_rangeindex_columns():
+ ser = Series([0])
+ df = DataFrame([1, 2])
+ result = concat([ser, df])
+ expected = DataFrame([0, 1, 2], index=[0, 0, 1])
+ tm.assert_frame_equal(result, expected, check_column_type=True)
+
+
+def test_concat_with_moot_ignore_index_and_keys():
+ df1 = DataFrame([[0]])
+ df2 = DataFrame([[42]])
+
+ ignore_index = True
+ keys = ["df1", "df2"]
+ msg = f"Cannot set {ignore_index=} and specify keys. Either should be used."
+ with pytest.raises(ValueError, match=msg):
+ concat([df1, df2], keys=keys, ignore_index=ignore_index)
+
+
+@pytest.mark.parametrize(
+ "inputs, ignore_index, axis, expected",
+ [
+ # Concatenating DataFrame and named Series without ignore_index
+ (
+ [DataFrame({"a": [0, 1], "b": [2, 3]}), Series([4, 5], name="c")],
+ False,
+ 0,
+ DataFrame(
+ {
+ "a": [0, 1, None, None],
+ "b": [2, 3, None, None],
+ "c": [None, None, 4, 5],
+ },
+ index=[0, 1, 0, 1],
+ ),
+ ),
+ # Concatenating DataFrame and named Series with ignore_index
+ (
+ [DataFrame({"a": [0, 1], "b": [2, 3]}), Series([4, 5], name="c")],
+ True,
+ 0,
+ DataFrame(
+ {
+ "a": [0, 1, None, None],
+ "b": [2, 3, None, None],
+ "c": [None, None, 4, 5],
+ },
+ index=[0, 1, 2, 3],
+ ),
+ ),
+ # Concatenating DataFrame and unnamed Series along columns
+ (
+ [DataFrame({"a": [0, 1], "b": [2, 3]}), Series([4, 5]), Series([4, 5])],
+ False,
+ 1,
+ DataFrame({"a": [0, 1], "b": [2, 3], 0: [4, 5], 1: [4, 5]}, index=[0, 1]),
+ ),
+ # Concatenating DataFrame and unnamed Series along columns with ignore_index
+ (
+ [DataFrame({"a": [0, 1], "b": [2, 3]}), Series([4, 5]), Series([4, 5])],
+ True,
+ 1,
+ DataFrame({0: [0, 1], 1: [2, 3], 2: [4, 5], 3: [4, 5]}, index=[0, 1]),
+ ),
+ ],
+)
+def test_concat_of_series_and_frame(inputs, ignore_index, axis, expected):
+ # GH #60723 and #56257
+ result = concat(inputs, ignore_index=ignore_index, axis=axis)
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_dataframe.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_dataframe.py
new file mode 100644
index 0000000000000000000000000000000000000000..b6da5ed76816b527fb23c9f5c1f68de3de8f8440
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_dataframe.py
@@ -0,0 +1,238 @@
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ Series,
+ concat,
+)
+import pandas._testing as tm
+
+
+class TestDataFrameConcat:
+ @pytest.mark.xfail(reason="GH#62888 the `mi[2][1] is 1` check fails")
+ def test_concat_multiindex_level_bool_and_numeric(self):
+ # GH#21108, GH#45101
+ left = DataFrame([123, 456], columns=["data"], index=[True, False])
+ right = DataFrame(
+ [55, 983, 69, 112, 0], columns=["data"], index=[1, 2, 3, 4, 99]
+ )
+ result = concat({"One": left, "Two": right})
+
+ # in particular, the first two entries should not be cast to ints, the
+ # other 1 should not cast to True
+ mi = pd.MultiIndex.from_arrays(
+ [
+ ["One"] * 2 + ["Two"] * 5,
+ np.array([True, False, 1, 2, 3, 4, 99], dtype=object),
+ ],
+ )
+ assert mi[0][1] is True
+ assert type(mi[2][1]) is int
+ expected = DataFrame({"data": [123, 456, 55, 983, 69, 112, 0]}, index=mi)
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_multiple_frames_dtypes(self):
+ # GH#2759
+ df1 = DataFrame(data=np.ones((10, 2)), columns=["foo", "bar"], dtype=np.float64)
+ df2 = DataFrame(data=np.ones((10, 2)), dtype=np.float32)
+ results = concat((df1, df2), axis=1).dtypes
+ expected = Series(
+ [np.dtype("float64")] * 2 + [np.dtype("float32")] * 2,
+ index=["foo", "bar", 0, 1],
+ )
+ tm.assert_series_equal(results, expected)
+
+ def test_concat_tuple_keys(self):
+ # GH#14438
+ df1 = DataFrame(np.ones((2, 2)), columns=list("AB"))
+ df2 = DataFrame(np.ones((3, 2)) * 2, columns=list("AB"))
+ results = concat((df1, df2), keys=[("bee", "bah"), ("bee", "boo")])
+ expected = DataFrame(
+ {
+ "A": {
+ ("bee", "bah", 0): 1.0,
+ ("bee", "bah", 1): 1.0,
+ ("bee", "boo", 0): 2.0,
+ ("bee", "boo", 1): 2.0,
+ ("bee", "boo", 2): 2.0,
+ },
+ "B": {
+ ("bee", "bah", 0): 1.0,
+ ("bee", "bah", 1): 1.0,
+ ("bee", "boo", 0): 2.0,
+ ("bee", "boo", 1): 2.0,
+ ("bee", "boo", 2): 2.0,
+ },
+ }
+ )
+ tm.assert_frame_equal(results, expected)
+
+ def test_concat_named_keys(self):
+ # GH#14252
+ df = DataFrame({"foo": [1, 2], "bar": [0.1, 0.2]})
+ index = Index(["a", "b"], name="baz")
+ concatted_named_from_keys = concat([df, df], keys=index)
+ expected_named = DataFrame(
+ {"foo": [1, 2, 1, 2], "bar": [0.1, 0.2, 0.1, 0.2]},
+ index=pd.MultiIndex.from_product((["a", "b"], [0, 1]), names=["baz", None]),
+ )
+ tm.assert_frame_equal(concatted_named_from_keys, expected_named)
+
+ index_no_name = Index(["a", "b"], name=None)
+ concatted_named_from_names = concat([df, df], keys=index_no_name, names=["baz"])
+ tm.assert_frame_equal(concatted_named_from_names, expected_named)
+
+ concatted_unnamed = concat([df, df], keys=index_no_name)
+ expected_unnamed = DataFrame(
+ {"foo": [1, 2, 1, 2], "bar": [0.1, 0.2, 0.1, 0.2]},
+ index=pd.MultiIndex.from_product((["a", "b"], [0, 1]), names=[None, None]),
+ )
+ tm.assert_frame_equal(concatted_unnamed, expected_unnamed)
+
+ def test_concat_axis_parameter(self):
+ # GH#14369
+ df1 = DataFrame({"A": [0.1, 0.2]}, index=range(2))
+ df2 = DataFrame({"A": [0.3, 0.4]}, index=range(2))
+
+ # Index/row/0 DataFrame
+ expected_index = DataFrame({"A": [0.1, 0.2, 0.3, 0.4]}, index=[0, 1, 0, 1])
+
+ concatted_index = concat([df1, df2], axis="index")
+ tm.assert_frame_equal(concatted_index, expected_index)
+
+ concatted_row = concat([df1, df2], axis="rows")
+ tm.assert_frame_equal(concatted_row, expected_index)
+
+ concatted_0 = concat([df1, df2], axis=0)
+ tm.assert_frame_equal(concatted_0, expected_index)
+
+ # Columns/1 DataFrame
+ expected_columns = DataFrame(
+ [[0.1, 0.3], [0.2, 0.4]], index=[0, 1], columns=["A", "A"]
+ )
+
+ concatted_columns = concat([df1, df2], axis="columns")
+ tm.assert_frame_equal(concatted_columns, expected_columns)
+
+ concatted_1 = concat([df1, df2], axis=1)
+ tm.assert_frame_equal(concatted_1, expected_columns)
+
+ series1 = Series([0.1, 0.2])
+ series2 = Series([0.3, 0.4])
+
+ # Index/row/0 Series
+ expected_index_series = Series([0.1, 0.2, 0.3, 0.4], index=[0, 1, 0, 1])
+
+ concatted_index_series = concat([series1, series2], axis="index")
+ tm.assert_series_equal(concatted_index_series, expected_index_series)
+
+ concatted_row_series = concat([series1, series2], axis="rows")
+ tm.assert_series_equal(concatted_row_series, expected_index_series)
+
+ concatted_0_series = concat([series1, series2], axis=0)
+ tm.assert_series_equal(concatted_0_series, expected_index_series)
+
+ # Columns/1 Series
+ expected_columns_series = DataFrame(
+ [[0.1, 0.3], [0.2, 0.4]], index=[0, 1], columns=[0, 1]
+ )
+
+ concatted_columns_series = concat([series1, series2], axis="columns")
+ tm.assert_frame_equal(concatted_columns_series, expected_columns_series)
+
+ concatted_1_series = concat([series1, series2], axis=1)
+ tm.assert_frame_equal(concatted_1_series, expected_columns_series)
+
+ # Testing ValueError
+ with pytest.raises(ValueError, match="No axis named"):
+ concat([series1, series2], axis="something")
+
+ def test_concat_numerical_names(self):
+ # GH#15262, GH#12223
+ df = DataFrame(
+ {"col": range(9)},
+ dtype="int32",
+ index=(
+ pd.MultiIndex.from_product(
+ [["A0", "A1", "A2"], ["B0", "B1", "B2"]], names=[1, 2]
+ )
+ ),
+ )
+ result = concat((df.iloc[:2, :], df.iloc[-2:, :]))
+ expected = DataFrame(
+ {"col": [0, 1, 7, 8]},
+ dtype="int32",
+ index=pd.MultiIndex.from_tuples(
+ [("A0", "B0"), ("A0", "B1"), ("A2", "B1"), ("A2", "B2")], names=[1, 2]
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_astype_dup_col(self):
+ # GH#23049
+ df = DataFrame([{"a": "b"}])
+ df = concat([df, df], axis=1)
+
+ result = df.astype("category")
+ expected = DataFrame(
+ np.array(["b", "b"]).reshape(1, 2), columns=["a", "a"]
+ ).astype("category")
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_dataframe_keys_bug(self, sort):
+ t1 = DataFrame(
+ {"value": Series([1, 2, 3], index=Index(["a", "b", "c"], name="id"))}
+ )
+ t2 = DataFrame({"value": Series([7, 8], index=Index(["a", "b"], name="id"))})
+
+ # it works
+ result = concat([t1, t2], axis=1, keys=["t1", "t2"], sort=sort)
+ assert list(result.columns) == [("t1", "value"), ("t2", "value")]
+
+ def test_concat_bool_with_int(self):
+ # GH#42092 we may want to change this to return object, but that
+ # would need a deprecation
+ df1 = DataFrame(Series([True, False, True, True], dtype="bool"))
+ df2 = DataFrame(Series([1, 0, 1], dtype="int64"))
+
+ result = concat([df1, df2])
+ expected = concat([df1.astype("int64"), df2])
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_duplicates_in_index_with_keys(self):
+ # GH#42651
+ index = [1, 1, 3]
+ data = [1, 2, 3]
+
+ df = DataFrame(data=data, index=index)
+ result = concat([df], keys=["A"], names=["ID", "date"])
+ mi = pd.MultiIndex.from_product([["A"], index], names=["ID", "date"])
+ expected = DataFrame(data=data, index=mi)
+ tm.assert_frame_equal(result, expected)
+ tm.assert_index_equal(result.index.levels[1], Index([1, 3], name="date"))
+
+ def test_outer_sort_columns(self):
+ # GH#47127
+ df1 = DataFrame({"A": [0], "B": [1], 0: 1})
+ df2 = DataFrame({"A": [100]})
+ result = concat([df1, df2], ignore_index=True, join="outer", sort=True)
+ expected = DataFrame({0: [1.0, np.nan], "A": [0, 100], "B": [1.0, np.nan]})
+ tm.assert_frame_equal(result, expected)
+
+ def test_inner_sort_columns(self):
+ # GH#47127
+ df1 = DataFrame({"A": [0], "B": [1], 0: 1})
+ df2 = DataFrame({"A": [100], 0: 2})
+ result = concat([df1, df2], ignore_index=True, join="inner", sort=True)
+ expected = DataFrame({0: [1, 2], "A": [0, 100]})
+ tm.assert_frame_equal(result, expected)
+
+ def test_sort_columns_one_df(self):
+ # GH#47127
+ df1 = DataFrame({"A": [100], 0: 2})
+ result = concat([df1], ignore_index=True, join="inner", sort=True)
+ expected = DataFrame({0: [2], "A": [100]})
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_datetimes.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_datetimes.py
new file mode 100644
index 0000000000000000000000000000000000000000..8bd737e0d025758953ab841ae64f8eb3e8571438
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_datetimes.py
@@ -0,0 +1,611 @@
+import datetime as dt
+from datetime import datetime
+
+import dateutil
+import numpy as np
+import pytest
+
+from pandas.errors import Pandas4Warning
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ DatetimeIndex,
+ Index,
+ MultiIndex,
+ Series,
+ Timestamp,
+ concat,
+ date_range,
+ to_timedelta,
+)
+import pandas._testing as tm
+
+
+class TestDatetimeConcat:
+ def test_concat_datetime64_block(self):
+ rng = date_range("1/1/2000", periods=10)
+
+ df = DataFrame({"time": rng})
+
+ result = concat([df, df])
+ assert (result.iloc[:10]["time"] == rng).all()
+ assert (result.iloc[10:]["time"] == rng).all()
+
+ def test_concat_datetime_datetime64_frame(self):
+ # GH#2624
+ rows = []
+ rows.append([datetime(2010, 1, 1), 1])
+ rows.append([datetime(2010, 1, 2), "hi"])
+
+ df2_obj = DataFrame.from_records(rows, columns=["date", "test"])
+
+ ind = date_range(start="2000/1/1", freq="D", periods=10)
+ df1 = DataFrame({"date": ind, "test": range(10)})
+
+ # it works!
+ concat([df1, df2_obj])
+
+ def test_concat_datetime_timezone(self):
+ # GH 18523
+ idx1 = date_range(
+ "2011-01-01", periods=3, freq="h", tz="Europe/Paris", unit="ns"
+ )
+ idx2 = date_range(start=idx1[0], end=idx1[-1], freq="h", unit="ns")
+ df1 = DataFrame({"a": [1, 2, 3]}, index=idx1)
+ df2 = DataFrame({"b": [1, 2, 3]}, index=idx2)
+ result = concat([df1, df2], axis=1)
+
+ exp_idx = DatetimeIndex(
+ [
+ "2011-01-01 00:00:00+01:00",
+ "2011-01-01 01:00:00+01:00",
+ "2011-01-01 02:00:00+01:00",
+ ],
+ dtype="M8[ns, Europe/Paris]",
+ freq="h",
+ )
+ expected = DataFrame(
+ [[1, 1], [2, 2], [3, 3]], index=exp_idx, columns=["a", "b"]
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ idx3 = date_range("2011-01-01", periods=3, freq="h", tz="Asia/Tokyo", unit="ns")
+ df3 = DataFrame({"b": [1, 2, 3]}, index=idx3)
+ msg = "Sorting by default when concatenating all DatetimeIndex"
+ with tm.assert_produces_warning(Pandas4Warning, match=msg):
+ result = concat([df1, df3], axis=1)
+
+ exp_idx = DatetimeIndex(
+ [
+ "2010-12-31 15:00:00+00:00",
+ "2010-12-31 16:00:00+00:00",
+ "2010-12-31 17:00:00+00:00",
+ "2010-12-31 23:00:00+00:00",
+ "2011-01-01 00:00:00+00:00",
+ "2011-01-01 01:00:00+00:00",
+ ]
+ ).as_unit("ns")
+
+ expected = DataFrame(
+ [
+ [np.nan, 1],
+ [np.nan, 2],
+ [np.nan, 3],
+ [1, np.nan],
+ [2, np.nan],
+ [3, np.nan],
+ ],
+ index=exp_idx,
+ columns=["a", "b"],
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ # GH 13783: Concat after resample
+ result = concat([df1.resample("h").mean(), df2.resample("h").mean()], sort=True)
+ expected = DataFrame(
+ {"a": [1, 2, 3] + [np.nan] * 3, "b": [np.nan] * 3 + [1, 2, 3]},
+ index=idx1.append(idx1),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_datetimeindex_freq(self):
+ # GH 3232
+ # Monotonic index result
+ dr = date_range("01-Jan-2013", periods=100, freq="50ms", tz="UTC")
+ data = list(range(100))
+ expected = DataFrame(data, index=dr)
+ result = concat([expected[:50], expected[50:]])
+ tm.assert_frame_equal(result, expected)
+
+ # Non-monotonic index result
+ result = concat([expected[50:], expected[:50]])
+ expected = DataFrame(data[50:] + data[:50], index=dr[50:].append(dr[:50]))
+ expected.index._data.freq = None
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_multiindex_datetime_object_index(self):
+ # https://github.com/pandas-dev/pandas/issues/11058
+ idx = Index(
+ [dt.date(2013, 1, 1), dt.date(2014, 1, 1), dt.date(2015, 1, 1)],
+ dtype="object",
+ )
+
+ s = Series(
+ ["a", "b"],
+ index=MultiIndex.from_arrays(
+ [
+ [1, 2],
+ idx[:-1],
+ ],
+ names=["first", "second"],
+ ),
+ )
+ s2 = Series(
+ ["a", "b"],
+ index=MultiIndex.from_arrays(
+ [[1, 2], idx[::2]],
+ names=["first", "second"],
+ ),
+ )
+ mi = MultiIndex.from_arrays(
+ [[1, 2, 2], idx],
+ names=["first", "second"],
+ )
+ assert mi.levels[1].dtype == object
+
+ expected = DataFrame(
+ [["a", "a"], ["b", np.nan], [np.nan, "b"]],
+ index=mi,
+ )
+ result = concat([s, s2], axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_NaT_series(self):
+ # GH 11693
+ # test for merging NaT series with datetime series.
+ x = Series(
+ date_range(
+ "20151124 08:00",
+ "20151124 09:00",
+ freq="1h",
+ tz="US/Eastern",
+ unit="ns",
+ )
+ )
+ y = Series(pd.NaT, index=[0, 1], dtype="datetime64[ns, US/Eastern]")
+ expected = Series([x[0], x[1], pd.NaT, pd.NaT])
+
+ result = concat([x, y], ignore_index=True)
+ tm.assert_series_equal(result, expected)
+
+ # all NaT with tz
+ expected = Series(pd.NaT, index=range(4), dtype="datetime64[ns, US/Eastern]")
+ result = concat([y, y], ignore_index=True)
+ tm.assert_series_equal(result, expected)
+
+ def test_concat_NaT_series2(self):
+ # without tz
+ x = Series(date_range("20151124 08:00", "20151124 09:00", freq="1h", unit="ns"))
+ y = Series(date_range("20151124 10:00", "20151124 11:00", freq="1h", unit="ns"))
+ y[:] = pd.NaT
+ expected = Series([x[0], x[1], pd.NaT, pd.NaT])
+ result = concat([x, y], ignore_index=True)
+ tm.assert_series_equal(result, expected)
+
+ # all NaT without tz
+ x[:] = pd.NaT
+ expected = Series(pd.NaT, index=range(4), dtype="datetime64[ns]")
+ result = concat([x, y], ignore_index=True)
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize("tz", [None, "UTC"])
+ def test_concat_NaT_dataframes(self, tz):
+ # GH 12396
+
+ dti = DatetimeIndex([pd.NaT, pd.NaT], tz=tz)
+ first = DataFrame({0: dti})
+ second = DataFrame(
+ [[Timestamp("2015/01/01", tz=tz)], [Timestamp("2016/01/01", tz=tz)]],
+ index=[2, 3],
+ )
+ expected = DataFrame(
+ [
+ pd.NaT,
+ pd.NaT,
+ Timestamp("2015/01/01", tz=tz),
+ Timestamp("2016/01/01", tz=tz),
+ ]
+ )
+
+ result = concat([first, second], axis=0)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("tz1", [None, "UTC"])
+ @pytest.mark.parametrize("tz2", [None, "UTC"])
+ @pytest.mark.parametrize("item", [pd.NaT, Timestamp("20150101").as_unit("ns")])
+ def test_concat_NaT_dataframes_all_NaT_axis_0(self, tz1, tz2, item):
+ # GH 12396
+
+ # tz-naive
+ first = DataFrame([[pd.NaT], [pd.NaT]]).apply(lambda x: x.dt.tz_localize(tz1))
+ second = DataFrame([item]).apply(lambda x: x.dt.tz_localize(tz2))
+
+ result = concat([first, second], axis=0)
+ expected = DataFrame(Series([pd.NaT, pd.NaT, item], index=[0, 1, 0]))
+ expected = expected.apply(lambda x: x.dt.tz_localize(tz2))
+ if tz1 != tz2:
+ expected = expected.astype(object)
+
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("tz1", [None, "UTC"])
+ @pytest.mark.parametrize("tz2", [None, "UTC"])
+ def test_concat_NaT_dataframes_all_NaT_axis_1(self, tz1, tz2):
+ # GH 12396
+
+ first = DataFrame(Series([pd.NaT, pd.NaT]).dt.tz_localize(tz1))
+ second = DataFrame(Series([pd.NaT]).dt.tz_localize(tz2), columns=[1])
+ expected = DataFrame(
+ {
+ 0: Series([pd.NaT, pd.NaT]).dt.tz_localize(tz1),
+ 1: Series([pd.NaT, pd.NaT]).dt.tz_localize(tz2),
+ }
+ )
+ result = concat([first, second], axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("tz1", [None, "UTC"])
+ @pytest.mark.parametrize("tz2", [None, "UTC"])
+ def test_concat_NaT_series_dataframe_all_NaT(self, tz1, tz2):
+ # GH 12396
+
+ # tz-naive
+ first = Series([pd.NaT, pd.NaT]).dt.tz_localize(tz1)
+ second = DataFrame(
+ [
+ [Timestamp("2015/01/01", tz=tz2)],
+ [Timestamp("2016/01/01", tz=tz2)],
+ ],
+ index=[2, 3],
+ )
+
+ expected = DataFrame(
+ [
+ pd.NaT,
+ pd.NaT,
+ Timestamp("2015/01/01", tz=tz2),
+ Timestamp("2016/01/01", tz=tz2),
+ ]
+ )
+ if tz1 != tz2:
+ expected = expected.astype(object)
+
+ result = concat([first, second])
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_compat_on_non_ns_datetime_EA(self):
+ # GH#33331
+ first = Series(np.array([datetime(2010, 1, 1)], dtype="datetime64[D]"))
+ second = Series(pd.array(["a", "b"], dtype="category"))
+
+ expected = Series([Timestamp("2010-01-01 00:00:00"), "a", "b"])
+
+ result = concat([first, second], ignore_index=True)
+ tm.assert_series_equal(result, expected)
+
+
+class TestTimezoneConcat:
+ def test_concat_tz_series(self):
+ # gh-11755: tz and no tz
+ x = Series(date_range("20151124 08:00", "20151124 09:00", freq="1h", tz="UTC"))
+ y = Series(date_range("2012-01-01", "2012-01-02"))
+ expected = Series([x[0], x[1], y[0], y[1]], dtype="object")
+ result = concat([x, y], ignore_index=True)
+ tm.assert_series_equal(result, expected)
+
+ def test_concat_tz_series2(self):
+ # gh-11887: concat tz and object
+ x = Series(date_range("20151124 08:00", "20151124 09:00", freq="1h", tz="UTC"))
+ y = Series(["a", "b"])
+ expected = Series([x[0], x[1], y[0], y[1]], dtype="object")
+ result = concat([x, y], ignore_index=True)
+ tm.assert_series_equal(result, expected)
+
+ def test_concat_tz_series3(self, unit, unit2):
+ # see gh-12217 and gh-12306
+ # Concatenating two UTC times
+ first = DataFrame([[datetime(2016, 1, 1)]], dtype=f"M8[{unit}]")
+ first[0] = first[0].dt.tz_localize("UTC")
+
+ second = DataFrame([[datetime(2016, 1, 2)]], dtype=f"M8[{unit2}]")
+ second[0] = second[0].dt.tz_localize("UTC")
+
+ result = concat([first, second])
+ exp_unit = tm.get_finest_unit(unit, unit2)
+ assert result[0].dtype == f"datetime64[{exp_unit}, UTC]"
+
+ def test_concat_tz_series4(self, unit, unit2):
+ # Concatenating two London times
+ first = DataFrame([[datetime(2016, 1, 1)]], dtype=f"M8[{unit}]")
+ first[0] = first[0].dt.tz_localize("Europe/London")
+
+ second = DataFrame([[datetime(2016, 1, 2)]], dtype=f"M8[{unit2}]")
+ second[0] = second[0].dt.tz_localize("Europe/London")
+
+ result = concat([first, second])
+ exp_unit = tm.get_finest_unit(unit, unit2)
+ assert result[0].dtype == f"datetime64[{exp_unit}, Europe/London]"
+
+ def test_concat_tz_series5(self, unit, unit2):
+ # Concatenating 2+1 London times
+ first = DataFrame(
+ [[datetime(2016, 1, 1)], [datetime(2016, 1, 2)]], dtype=f"M8[{unit}]"
+ )
+ first[0] = first[0].dt.tz_localize("Europe/London")
+
+ second = DataFrame([[datetime(2016, 1, 3)]], dtype=f"M8[{unit2}]")
+ second[0] = second[0].dt.tz_localize("Europe/London")
+
+ result = concat([first, second])
+ exp_unit = tm.get_finest_unit(unit, unit2)
+ assert result[0].dtype == f"datetime64[{exp_unit}, Europe/London]"
+
+ def test_concat_tz_series6(self, unit, unit2):
+ # Concatenating 1+2 London times
+ first = DataFrame([[datetime(2016, 1, 1)]], dtype=f"M8[{unit}]")
+ first[0] = first[0].dt.tz_localize("Europe/London")
+
+ second = DataFrame(
+ [[datetime(2016, 1, 2)], [datetime(2016, 1, 3)]], dtype=f"M8[{unit2}]"
+ )
+ second[0] = second[0].dt.tz_localize("Europe/London")
+
+ result = concat([first, second])
+ exp_unit = tm.get_finest_unit(unit, unit2)
+ assert result[0].dtype == f"datetime64[{exp_unit}, Europe/London]"
+
+ def test_concat_tz_series_tzlocal(self):
+ # see gh-13583
+ x = [
+ Timestamp("2011-01-01", tz=dateutil.tz.tzlocal()),
+ Timestamp("2011-02-01", tz=dateutil.tz.tzlocal()),
+ ]
+ y = [
+ Timestamp("2012-01-01", tz=dateutil.tz.tzlocal()),
+ Timestamp("2012-02-01", tz=dateutil.tz.tzlocal()),
+ ]
+
+ result = concat([Series(x), Series(y)], ignore_index=True)
+ tm.assert_series_equal(result, Series(x + y))
+ assert result.dtype == "datetime64[us, tzlocal()]"
+
+ def test_concat_tz_series_with_datetimelike(self):
+ # see gh-12620: tz and timedelta
+ x = [
+ Timestamp("2011-01-01", tz="US/Eastern"),
+ Timestamp("2011-02-01", tz="US/Eastern"),
+ ]
+ y = [pd.Timedelta("1 day"), pd.Timedelta("2 day")]
+ result = concat([Series(x), Series(y)], ignore_index=True)
+ tm.assert_series_equal(result, Series(x + y, dtype="object"))
+
+ # tz and period
+ y = [pd.Period("2011-03", freq="M"), pd.Period("2011-04", freq="M")]
+ result = concat([Series(x), Series(y)], ignore_index=True)
+ tm.assert_series_equal(result, Series(x + y, dtype="object"))
+
+ def test_concat_tz_frame(self):
+ df2 = DataFrame(
+ {
+ "A": Timestamp("20130102", tz="US/Eastern"),
+ "B": Timestamp("20130603", tz="CET"),
+ },
+ index=range(5),
+ )
+
+ # concat
+ df3 = concat([df2.A.to_frame(), df2.B.to_frame()], axis=1)
+ tm.assert_frame_equal(df2, df3)
+
+ def test_concat_multiple_tzs(self):
+ # GH#12467
+ # combining datetime tz-aware and naive DataFrames
+ ts1 = Timestamp("2015-01-01", tz=None)
+ ts2 = Timestamp("2015-01-01", tz="UTC")
+ ts3 = Timestamp("2015-01-01", tz="EST")
+
+ df1 = DataFrame({"time": [ts1]})
+ df2 = DataFrame({"time": [ts2]})
+ df3 = DataFrame({"time": [ts3]})
+
+ results = concat([df1, df2]).reset_index(drop=True)
+ expected = DataFrame({"time": [ts1, ts2]}, dtype=object)
+ tm.assert_frame_equal(results, expected)
+
+ results = concat([df1, df3]).reset_index(drop=True)
+ expected = DataFrame({"time": [ts1, ts3]}, dtype=object)
+ tm.assert_frame_equal(results, expected)
+
+ results = concat([df2, df3]).reset_index(drop=True)
+ expected = DataFrame({"time": [ts2, ts3]})
+ tm.assert_frame_equal(results, expected)
+
+ def test_concat_multiindex_with_tz(self):
+ # GH 6606
+ df = DataFrame(
+ {
+ "dt": DatetimeIndex(
+ [
+ datetime(2014, 1, 1),
+ datetime(2014, 1, 2),
+ datetime(2014, 1, 3),
+ ],
+ dtype="M8[ns, US/Pacific]",
+ ),
+ "b": ["A", "B", "C"],
+ "c": [1, 2, 3],
+ "d": [4, 5, 6],
+ }
+ )
+ df = df.set_index(["dt", "b"])
+
+ exp_idx1 = DatetimeIndex(
+ ["2014-01-01", "2014-01-02", "2014-01-03"] * 2,
+ dtype="M8[ns, US/Pacific]",
+ name="dt",
+ )
+ exp_idx2 = Index(["A", "B", "C"] * 2, name="b")
+ exp_idx = MultiIndex.from_arrays([exp_idx1, exp_idx2])
+ expected = DataFrame(
+ {"c": [1, 2, 3] * 2, "d": [4, 5, 6] * 2}, index=exp_idx, columns=["c", "d"]
+ )
+
+ result = concat([df, df])
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_tz_not_aligned(self):
+ # GH#22796
+ ts = pd.to_datetime([1, 2]).tz_localize("UTC")
+ a = DataFrame({"A": ts})
+ b = DataFrame({"A": ts, "B": ts})
+ result = concat([a, b], sort=True, ignore_index=True)
+ expected = DataFrame(
+ {"A": list(ts) + list(ts), "B": [pd.NaT, pd.NaT, *list(ts)]}
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "t1",
+ [
+ "2015-01-01",
+ pytest.param(
+ pd.NaT,
+ marks=pytest.mark.xfail(
+ reason="GH23037 incorrect dtype when concatenating"
+ ),
+ ),
+ ],
+ )
+ def test_concat_tz_NaT(self, t1):
+ # GH#22796
+ # Concatenating tz-aware multicolumn DataFrames
+ ts1 = Timestamp(t1, tz="UTC")
+ ts2 = Timestamp("2015-01-01", tz="UTC")
+ ts3 = Timestamp("2015-01-01", tz="UTC")
+
+ df1 = DataFrame([[ts1, ts2]])
+ df2 = DataFrame([[ts3]])
+
+ result = concat([df1, df2])
+ expected = DataFrame([[ts1, ts2], [ts3, pd.NaT]], index=[0, 0])
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_tz_with_empty(self):
+ # GH 9188
+ result = concat(
+ [DataFrame(date_range("2000", periods=1, tz="UTC")), DataFrame()]
+ )
+ expected = DataFrame(date_range("2000", periods=1, tz="UTC"))
+ tm.assert_frame_equal(result, expected)
+
+
+class TestPeriodConcat:
+ def test_concat_period_series(self):
+ x = Series(pd.PeriodIndex(["2015-11-01", "2015-12-01"], freq="D"))
+ y = Series(pd.PeriodIndex(["2015-10-01", "2016-01-01"], freq="D"))
+ expected = Series([x[0], x[1], y[0], y[1]], dtype="Period[D]")
+ result = concat([x, y], ignore_index=True)
+ tm.assert_series_equal(result, expected)
+
+ def test_concat_period_multiple_freq_series(self):
+ x = Series(pd.PeriodIndex(["2015-11-01", "2015-12-01"], freq="D"))
+ y = Series(pd.PeriodIndex(["2015-10-01", "2016-01-01"], freq="M"))
+ expected = Series([x[0], x[1], y[0], y[1]], dtype="object")
+ result = concat([x, y], ignore_index=True)
+ tm.assert_series_equal(result, expected)
+ assert result.dtype == "object"
+
+ def test_concat_period_other_series(self):
+ x = Series(pd.PeriodIndex(["2015-11-01", "2015-12-01"], freq="D"))
+ y = Series(pd.PeriodIndex(["2015-11-01", "2015-12-01"], freq="M"))
+ expected = Series([x[0], x[1], y[0], y[1]], dtype="object")
+ result = concat([x, y], ignore_index=True)
+ tm.assert_series_equal(result, expected)
+ assert result.dtype == "object"
+
+ def test_concat_period_other_series2(self):
+ # non-period
+ x = Series(pd.PeriodIndex(["2015-11-01", "2015-12-01"], freq="D"))
+ y = Series(DatetimeIndex(["2015-11-01", "2015-12-01"]))
+ expected = Series([x[0], x[1], y[0], y[1]], dtype="object")
+ result = concat([x, y], ignore_index=True)
+ tm.assert_series_equal(result, expected)
+ assert result.dtype == "object"
+
+ def test_concat_period_other_series3(self):
+ x = Series(pd.PeriodIndex(["2015-11-01", "2015-12-01"], freq="D"))
+ y = Series(["A", "B"])
+ expected = Series([x[0], x[1], y[0], y[1]], dtype="object")
+ result = concat([x, y], ignore_index=True)
+ tm.assert_series_equal(result, expected)
+ assert result.dtype == "object"
+
+
+def test_concat_timedelta64_block():
+ rng = to_timedelta(np.arange(10), unit="s")
+
+ df = DataFrame({"time": rng})
+
+ result = concat([df, df])
+ tm.assert_frame_equal(result.iloc[:10], df, check_index_type=False)
+ tm.assert_frame_equal(result.iloc[10:], df, check_index_type=False)
+
+
+def test_concat_multiindex_datetime_nat():
+ # GH#44900
+ left = DataFrame({"a": 1}, index=MultiIndex.from_tuples([(1, pd.NaT)]))
+ right = DataFrame(
+ {"b": 2}, index=MultiIndex.from_tuples([(1, pd.NaT), (2, pd.NaT)])
+ )
+ result = concat([left, right], axis="columns")
+ expected = DataFrame(
+ {"a": [1.0, np.nan], "b": 2}, MultiIndex.from_tuples([(1, pd.NaT), (2, pd.NaT)])
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_concat_float_datetime64():
+ # GH#32934
+ df_time = DataFrame({"A": pd.array(["2000"], dtype="datetime64[ns]")})
+ df_float = DataFrame({"A": pd.array([1.0], dtype="float64")})
+
+ expected = DataFrame(
+ {
+ "A": [
+ pd.array(["2000"], dtype="datetime64[ns]")[0],
+ pd.array([1.0], dtype="float64")[0],
+ ]
+ },
+ index=[0, 0],
+ )
+ result = concat([df_time, df_float])
+ tm.assert_frame_equal(result, expected)
+
+ expected = DataFrame({"A": pd.array([], dtype="object")})
+ result = concat([df_time.iloc[:0], df_float.iloc[:0]])
+ tm.assert_frame_equal(result, expected)
+
+ expected = DataFrame({"A": pd.array([1.0], dtype="object")})
+ result = concat([df_time.iloc[:0], df_float])
+ tm.assert_frame_equal(result, expected)
+
+ expected = DataFrame({"A": pd.array(["2000"], dtype="datetime64[ns]")}).astype(
+ object
+ )
+
+ result = concat([df_time, df_float.iloc[:0]])
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_empty.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_empty.py
new file mode 100644
index 0000000000000000000000000000000000000000..4869cfbf3a5568c7d082ff39a58489a0479b158c
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_empty.py
@@ -0,0 +1,293 @@
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ RangeIndex,
+ Series,
+ concat,
+ date_range,
+)
+import pandas._testing as tm
+
+
+class TestEmptyConcat:
+ def test_handle_empty_objects(self, sort, using_infer_string):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)), columns=list("abcd")
+ )
+
+ dfcopy = df[:5].copy()
+ dfcopy["foo"] = "bar"
+ empty = df[5:5]
+
+ frames = [dfcopy, empty, empty, df[5:]]
+ concatted = concat(frames, axis=0, sort=sort)
+
+ expected = df.reindex(columns=["a", "b", "c", "d", "foo"])
+ expected["foo"] = expected["foo"].astype(
+ object if not using_infer_string else "str"
+ )
+ expected.loc[0:4, "foo"] = "bar"
+
+ tm.assert_frame_equal(concatted, expected)
+
+ # empty as first element with time series
+ # GH3259
+ df = DataFrame(
+ {"A": range(10000)}, index=date_range("20130101", periods=10000, freq="s")
+ )
+ empty = DataFrame()
+ result = concat([df, empty], axis=1)
+ tm.assert_frame_equal(result, df)
+ result = concat([empty, df], axis=1)
+ tm.assert_frame_equal(result, df)
+
+ result = concat([df, empty])
+ tm.assert_frame_equal(result, df)
+ result = concat([empty, df])
+ tm.assert_frame_equal(result, df)
+
+ def test_concat_empty_series(self):
+ # GH 11082
+ s1 = Series([1, 2, 3], name="x")
+ s2 = Series(name="y", dtype="float64")
+ res = concat([s1, s2], axis=1)
+ exp = DataFrame(
+ {"x": [1, 2, 3], "y": [np.nan, np.nan, np.nan]},
+ index=RangeIndex(3),
+ )
+ tm.assert_frame_equal(res, exp)
+
+ s1 = Series([1, 2, 3], name="x")
+ s2 = Series(name="y", dtype="float64")
+ res = concat([s1, s2], axis=0)
+ # name will be reset
+ exp = Series([1, 2, 3], dtype="float64")
+ tm.assert_series_equal(res, exp)
+
+ # empty Series with no name
+ s1 = Series([1, 2, 3], name="x")
+ s2 = Series(name=None, dtype="float64")
+ res = concat([s1, s2], axis=1)
+ exp = DataFrame(
+ {"x": [1, 2, 3], 0: [np.nan, np.nan, np.nan]},
+ columns=["x", 0],
+ index=RangeIndex(3),
+ )
+ tm.assert_frame_equal(res, exp)
+
+ @pytest.mark.parametrize("tz", [None, "UTC"])
+ @pytest.mark.parametrize("values", [[], [1, 2, 3]])
+ def test_concat_empty_series_timelike(self, tz, values):
+ # GH 18447
+
+ first = Series([], dtype="M8[ns]").dt.tz_localize(tz)
+ dtype = None if values else np.float64
+ second = Series(values, dtype=dtype)
+
+ expected = DataFrame(
+ {
+ 0: Series([pd.NaT] * len(values), dtype="M8[ns]").dt.tz_localize(tz),
+ 1: values,
+ }
+ )
+ result = concat([first, second], axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "left,right,expected",
+ [
+ # booleans
+ (np.bool_, np.int32, np.object_), # changed from int32 in 2.0 GH#39817
+ (np.bool_, np.float32, np.object_),
+ # datetime-like
+ ("m8[ns]", np.bool_, np.object_),
+ ("m8[ns]", np.int64, np.object_),
+ ("M8[ns]", np.bool_, np.object_),
+ ("M8[ns]", np.int64, np.object_),
+ # categorical
+ ("category", "category", "category"),
+ ("category", "object", "object"),
+ ],
+ )
+ def test_concat_empty_series_dtypes(self, left, right, expected):
+ # GH#39817, GH#45101
+ result = concat([Series(dtype=left), Series(dtype=right)])
+ assert result.dtype == expected
+
+ @pytest.mark.parametrize(
+ "dtype", ["float64", "int8", "uint8", "bool", "m8[ns]", "M8[ns]"]
+ )
+ def test_concat_empty_series_dtypes_match_roundtrips(self, dtype):
+ dtype = np.dtype(dtype)
+
+ result = concat([Series(dtype=dtype)])
+ assert result.dtype == dtype
+
+ result = concat([Series(dtype=dtype), Series(dtype=dtype)])
+ assert result.dtype == dtype
+
+ @pytest.mark.parametrize("dtype", ["float64", "int8", "uint8", "m8[ns]", "M8[ns]"])
+ @pytest.mark.parametrize(
+ "dtype2",
+ ["float64", "int8", "uint8", "m8[ns]", "M8[ns]"],
+ )
+ def test_concat_empty_series_dtypes_roundtrips(self, dtype, dtype2):
+ # round-tripping with self & like self
+ if dtype == dtype2:
+ pytest.skip("same dtype is not applicable for test")
+
+ def int_result_type(dtype, dtype2):
+ typs = {dtype.kind, dtype2.kind}
+ if not len(typs - {"i", "u", "b"}) and (
+ dtype.kind == "i" or dtype2.kind == "i"
+ ):
+ return "i"
+ elif not len(typs - {"u", "b"}) and (
+ dtype.kind == "u" or dtype2.kind == "u"
+ ):
+ return "u"
+ return None
+
+ def float_result_type(dtype, dtype2):
+ typs = {dtype.kind, dtype2.kind}
+ if not len(typs - {"f", "i", "u"}) and (
+ dtype.kind == "f" or dtype2.kind == "f"
+ ):
+ return "f"
+ return None
+
+ def get_result_type(dtype, dtype2):
+ result = float_result_type(dtype, dtype2)
+ if result is not None:
+ return result
+ result = int_result_type(dtype, dtype2)
+ if result is not None:
+ return result
+ return "O"
+
+ dtype = np.dtype(dtype)
+ dtype2 = np.dtype(dtype2)
+ expected = get_result_type(dtype, dtype2)
+ result = concat([Series(dtype=dtype), Series(dtype=dtype2)]).dtype
+ assert result.kind == expected
+
+ def test_concat_empty_series_dtypes_triple(self):
+ assert (
+ concat(
+ [Series(dtype="M8[ns]"), Series(dtype=np.bool_), Series(dtype=np.int64)]
+ ).dtype
+ == np.object_
+ )
+
+ def test_concat_empty_series_dtype_category_with_array(self):
+ # GH#18515
+ assert (
+ concat(
+ [Series(np.array([]), dtype="category"), Series(dtype="float64")]
+ ).dtype
+ == "float64"
+ )
+
+ def test_concat_empty_series_dtypes_sparse(self):
+ result = concat(
+ [
+ Series(dtype="float64").astype("Sparse"),
+ Series(dtype="float64").astype("Sparse"),
+ ]
+ )
+ assert result.dtype == "Sparse[float64]"
+
+ result = concat(
+ [Series(dtype="float64").astype("Sparse"), Series(dtype="float64")]
+ )
+ expected = pd.SparseDtype(np.float64)
+ assert result.dtype == expected
+
+ result = concat(
+ [Series(dtype="float64").astype("Sparse"), Series(dtype="object")]
+ )
+ expected = pd.SparseDtype("object")
+ assert result.dtype == expected
+
+ def test_concat_empty_df_object_dtype(self):
+ # GH 9149
+ df_1 = DataFrame({"Row": [0, 1, 1], "EmptyCol": np.nan, "NumberCol": [1, 2, 3]})
+ df_2 = DataFrame(columns=df_1.columns)
+ result = concat([df_1, df_2], axis=0)
+ expected = df_1.astype(object)
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_empty_dataframe_dtypes(self):
+ df = DataFrame(columns=list("abc"))
+ df["a"] = df["a"].astype(np.bool_)
+ df["b"] = df["b"].astype(np.int32)
+ df["c"] = df["c"].astype(np.float64)
+
+ result = concat([df, df])
+ assert result["a"].dtype == np.bool_
+ assert result["b"].dtype == np.int32
+ assert result["c"].dtype == np.float64
+
+ result = concat([df, df.astype(np.float64)])
+ assert result["a"].dtype == np.object_
+ assert result["b"].dtype == np.float64
+ assert result["c"].dtype == np.float64
+
+ def test_concat_inner_join_empty(self):
+ # GH 15328
+ df_empty = DataFrame()
+ df_a = DataFrame({"a": [1, 2]}, index=[0, 1], dtype="int64")
+ df_expected = DataFrame({"a": []}, index=RangeIndex(0), dtype="int64")
+
+ result = concat([df_a, df_empty], axis=1, join="inner")
+ tm.assert_frame_equal(result, df_expected)
+
+ result = concat([df_a, df_empty], axis=1, join="outer")
+ tm.assert_frame_equal(result, df_a)
+
+ def test_empty_dtype_coerce(self):
+ # xref to #12411
+ # xref to #12045
+ # xref to #11594
+ # see below
+
+ # 10571
+ df1 = DataFrame(data=[[1, None], [2, None]], columns=["a", "b"])
+ df2 = DataFrame(data=[[3, None], [4, None]], columns=["a", "b"])
+ result = concat([df1, df2])
+ expected = df1.dtypes
+ tm.assert_series_equal(result.dtypes, expected)
+
+ def test_concat_empty_dataframe(self):
+ # 39037
+ df1 = DataFrame(columns=["a", "b"])
+ df2 = DataFrame(columns=["b", "c"])
+ result = concat([df1, df2, df1])
+ expected = DataFrame(columns=["a", "b", "c"])
+ tm.assert_frame_equal(result, expected)
+
+ df3 = DataFrame(columns=["a", "b"])
+ df4 = DataFrame(columns=["b"])
+ result = concat([df3, df4])
+ expected = DataFrame(columns=["a", "b"])
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_empty_dataframe_different_dtypes(self, using_infer_string):
+ # 39037
+ df1 = DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
+ df2 = DataFrame({"a": [1, 2, 3]})
+
+ result = concat([df1[:0], df2[:0]])
+ assert result["a"].dtype == np.int64
+ assert result["b"].dtype == np.object_ if not using_infer_string else "str"
+
+ def test_concat_to_empty_ea(self):
+ """48510 `concat` to an empty EA should maintain type EA dtype."""
+ df_empty = DataFrame({"a": pd.array([], dtype=pd.Int64Dtype())})
+ df_new = DataFrame({"a": pd.array([1, 2, 3], dtype=pd.Int64Dtype())})
+ expected = df_new.copy()
+ result = concat([df_empty, df_new])
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_index.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_index.py
new file mode 100644
index 0000000000000000000000000000000000000000..35649d738ea0ff32dd80979c68f32a1313e349a8
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_index.py
@@ -0,0 +1,456 @@
+from copy import deepcopy
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ MultiIndex,
+ Series,
+ concat,
+)
+import pandas._testing as tm
+
+
+class TestIndexConcat:
+ def test_concat_ignore_index(self, sort):
+ frame1 = DataFrame(
+ {"test1": ["a", "b", "c"], "test2": [1, 2, 3], "test3": [4.5, 3.2, 1.2]}
+ )
+ frame2 = DataFrame({"test3": [5.2, 2.2, 4.3]})
+ frame1.index = Index(["x", "y", "z"])
+ frame2.index = Index(["x", "y", "q"])
+
+ v1 = concat([frame1, frame2], axis=1, ignore_index=True, sort=sort)
+
+ nan = np.nan
+ expected = DataFrame(
+ [
+ [nan, nan, nan, 4.3],
+ ["a", 1, 4.5, 5.2],
+ ["b", 2, 3.2, 2.2],
+ ["c", 3, 1.2, nan],
+ ],
+ index=Index(["q", "x", "y", "z"]),
+ )
+ if not sort:
+ expected = expected.loc[["x", "y", "z", "q"]]
+
+ tm.assert_frame_equal(v1, expected)
+
+ @pytest.mark.parametrize(
+ "name_in1,name_in2,name_in3,name_out",
+ [
+ ("idx", "idx", "idx", "idx"),
+ ("idx", "idx", None, None),
+ ("idx", None, None, None),
+ ("idx1", "idx2", None, None),
+ ("idx1", "idx1", "idx2", None),
+ ("idx1", "idx2", "idx3", None),
+ (None, None, None, None),
+ ],
+ )
+ def test_concat_same_index_names(self, name_in1, name_in2, name_in3, name_out):
+ # GH13475
+ indices = [
+ Index(["a", "b", "c"], name=name_in1),
+ Index(["b", "c", "d"], name=name_in2),
+ Index(["c", "d", "e"], name=name_in3),
+ ]
+ frames = [
+ DataFrame({c: [0, 1, 2]}, index=i) for i, c in zip(indices, ["x", "y", "z"])
+ ]
+ result = concat(frames, axis=1)
+
+ exp_ind = Index(["a", "b", "c", "d", "e"], name=name_out)
+ expected = DataFrame(
+ {
+ "x": [0, 1, 2, np.nan, np.nan],
+ "y": [np.nan, 0, 1, 2, np.nan],
+ "z": [np.nan, np.nan, 0, 1, 2],
+ },
+ index=exp_ind,
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_rename_index(self):
+ a = DataFrame(
+ np.random.default_rng(2).random((3, 3)),
+ columns=list("ABC"),
+ index=Index(list("abc"), name="index_a"),
+ )
+ b = DataFrame(
+ np.random.default_rng(2).random((3, 3)),
+ columns=list("ABC"),
+ index=Index(list("abc"), name="index_b"),
+ )
+
+ result = concat([a, b], keys=["key0", "key1"], names=["lvl0", "lvl1"])
+
+ exp = concat([a, b], keys=["key0", "key1"], names=["lvl0"])
+ names = list(exp.index.names)
+ names[1] = "lvl1"
+ exp.index.set_names(names, inplace=True)
+
+ tm.assert_frame_equal(result, exp)
+ assert result.index.names == exp.index.names
+
+ def test_concat_copy_index_series(self, axis):
+ # GH 29879
+ ser = Series([1, 2])
+ comb = concat([ser, ser], axis=axis)
+ if axis in [0, "index"]:
+ assert comb.index is not ser.index
+ else:
+ assert comb.index is ser.index
+
+ def test_concat_copy_index_frame(self, axis):
+ # GH 29879
+ df = DataFrame([[1, 2], [3, 4]], columns=["a", "b"])
+ comb = concat([df, df], axis=axis)
+ if axis in [0, "index"]:
+ assert not comb.index.is_(df.index)
+ assert comb.columns.is_(df.columns)
+ elif axis in [1, "columns"]:
+ assert comb.index.is_(df.index)
+ assert not comb.columns.is_(df.columns)
+
+ def test_default_index(self):
+ # is_series and ignore_index
+ s1 = Series([1, 2, 3], name="x")
+ s2 = Series([4, 5, 6], name="y")
+ res = concat([s1, s2], axis=1, ignore_index=True)
+ assert isinstance(res.columns, pd.RangeIndex)
+ exp = DataFrame([[1, 4], [2, 5], [3, 6]])
+ # use check_index_type=True to check the result have
+ # RangeIndex (default index)
+ tm.assert_frame_equal(res, exp, check_index_type=True, check_column_type=True)
+
+ # is_series and all inputs have no names
+ s1 = Series([1, 2, 3])
+ s2 = Series([4, 5, 6])
+ res = concat([s1, s2], axis=1, ignore_index=False)
+ assert isinstance(res.columns, pd.RangeIndex)
+ exp = DataFrame([[1, 4], [2, 5], [3, 6]])
+ exp.columns = pd.RangeIndex(2)
+ tm.assert_frame_equal(res, exp, check_index_type=True, check_column_type=True)
+
+ # is_dataframe and ignore_index
+ df1 = DataFrame({"A": [1, 2], "B": [5, 6]})
+ df2 = DataFrame({"A": [3, 4], "B": [7, 8]})
+
+ res = concat([df1, df2], axis=0, ignore_index=True)
+ exp = DataFrame([[1, 5], [2, 6], [3, 7], [4, 8]], columns=["A", "B"])
+ tm.assert_frame_equal(res, exp, check_index_type=True, check_column_type=True)
+
+ res = concat([df1, df2], axis=1, ignore_index=True)
+ exp = DataFrame([[1, 5, 3, 7], [2, 6, 4, 8]])
+ tm.assert_frame_equal(res, exp, check_index_type=True, check_column_type=True)
+
+ def test_dups_index(self):
+ # GH 4771
+
+ # single dtypes
+ df = DataFrame(
+ np.random.default_rng(2).integers(0, 10, size=40).reshape(10, 4),
+ columns=["A", "A", "C", "C"],
+ )
+
+ result = concat([df, df], axis=1)
+ tm.assert_frame_equal(result.iloc[:, :4], df)
+ tm.assert_frame_equal(result.iloc[:, 4:], df)
+
+ result = concat([df, df], axis=0)
+ tm.assert_frame_equal(result.iloc[:10], df)
+ tm.assert_frame_equal(result.iloc[10:], df)
+
+ # multi dtypes
+ df = concat(
+ [
+ DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=["A", "A", "B", "B"],
+ ),
+ DataFrame(
+ np.random.default_rng(2).integers(0, 10, size=20).reshape(10, 2),
+ columns=["A", "C"],
+ ),
+ ],
+ axis=1,
+ )
+
+ result = concat([df, df], axis=1)
+ tm.assert_frame_equal(result.iloc[:, :6], df)
+ tm.assert_frame_equal(result.iloc[:, 6:], df)
+
+ result = concat([df, df], axis=0)
+ tm.assert_frame_equal(result.iloc[:10], df)
+ tm.assert_frame_equal(result.iloc[10:], df)
+
+
+class TestMultiIndexConcat:
+ def test_concat_multiindex_with_keys(self, multiindex_dataframe_random_data):
+ frame = multiindex_dataframe_random_data
+ index = frame.index
+ result = concat([frame, frame], keys=[0, 1], names=["iteration"])
+
+ assert result.index.names == ("iteration", *index.names)
+ tm.assert_frame_equal(result.loc[0], frame)
+ tm.assert_frame_equal(result.loc[1], frame)
+ assert result.index.nlevels == 3
+
+ def test_concat_multiindex_with_none_in_index_names(self):
+ # GH 15787
+ index = MultiIndex.from_product([[1], range(5)], names=["level1", None])
+ df = DataFrame({"col": range(5)}, index=index, dtype=np.int32)
+
+ result = concat([df, df], keys=[1, 2], names=["level2"])
+ index = MultiIndex.from_product(
+ [[1, 2], [1], range(5)], names=["level2", "level1", None]
+ )
+ expected = DataFrame({"col": list(range(5)) * 2}, index=index, dtype=np.int32)
+ tm.assert_frame_equal(result, expected)
+
+ result = concat([df, df[:2]], keys=[1, 2], names=["level2"])
+ level2 = [1] * 5 + [2] * 2
+ level1 = [1] * 7
+ no_name = list(range(5)) + list(range(2))
+ tuples = list(zip(level2, level1, no_name))
+ index = MultiIndex.from_tuples(tuples, names=["level2", "level1", None])
+ expected = DataFrame({"col": no_name}, index=index, dtype=np.int32)
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_multiindex_rangeindex(self):
+ # GH13542
+ # when multi-index levels are RangeIndex objects
+ # there is a bug in concat with objects of len 1
+
+ df = DataFrame(np.random.default_rng(2).standard_normal((9, 2)))
+ df.index = MultiIndex(
+ levels=[pd.RangeIndex(3), pd.RangeIndex(3)],
+ codes=[np.repeat(np.arange(3), 3), np.tile(np.arange(3), 3)],
+ )
+
+ res = concat([df.iloc[[2, 3, 4], :], df.iloc[[5], :]])
+ exp = df.iloc[[2, 3, 4, 5], :]
+ tm.assert_frame_equal(res, exp)
+
+ def test_concat_multiindex_dfs_with_deepcopy(self):
+ # GH 9967
+ example_multiindex1 = MultiIndex.from_product([["a"], ["b"]])
+ example_dataframe1 = DataFrame([0], index=example_multiindex1)
+
+ example_multiindex2 = MultiIndex.from_product([["a"], ["c"]])
+ example_dataframe2 = DataFrame([1], index=example_multiindex2)
+
+ example_dict = {"s1": example_dataframe1, "s2": example_dataframe2}
+ expected_index = MultiIndex(
+ levels=[["s1", "s2"], ["a"], ["b", "c"]],
+ codes=[[0, 1], [0, 0], [0, 1]],
+ names=["testname", None, None],
+ )
+ expected = DataFrame([[0], [1]], index=expected_index)
+ result_copy = concat(deepcopy(example_dict), names=["testname"])
+ tm.assert_frame_equal(result_copy, expected)
+ result_no_copy = concat(example_dict, names=["testname"])
+ tm.assert_frame_equal(result_no_copy, expected)
+
+ @pytest.mark.parametrize(
+ "mi1_list",
+ [
+ [["a"], range(2)],
+ [["b"], np.arange(2.0, 4.0)],
+ [["c"], ["A", "B"]],
+ [["d"], pd.date_range(start="2017", end="2018", periods=2)],
+ ],
+ )
+ @pytest.mark.parametrize(
+ "mi2_list",
+ [
+ [["a"], range(2)],
+ [["b"], np.arange(2.0, 4.0)],
+ [["c"], ["A", "B"]],
+ [["d"], pd.date_range(start="2017", end="2018", periods=2)],
+ ],
+ )
+ def test_concat_with_various_multiindex_dtypes(
+ self, mi1_list: list, mi2_list: list
+ ):
+ # GitHub #23478
+ mi1 = MultiIndex.from_product(mi1_list)
+ mi2 = MultiIndex.from_product(mi2_list)
+
+ df1 = DataFrame(np.zeros((1, len(mi1))), columns=mi1)
+ df2 = DataFrame(np.zeros((1, len(mi2))), columns=mi2)
+
+ if mi1_list[0] == mi2_list[0]:
+ expected_mi = MultiIndex(
+ levels=[mi1_list[0], list(mi1_list[1])],
+ codes=[[0, 0, 0, 0], [0, 1, 0, 1]],
+ )
+ else:
+ expected_mi = MultiIndex(
+ levels=[
+ mi1_list[0] + mi2_list[0],
+ list(mi1_list[1]) + list(mi2_list[1]),
+ ],
+ codes=[[0, 0, 1, 1], [0, 1, 2, 3]],
+ )
+
+ expected_df = DataFrame(np.zeros((1, len(expected_mi))), columns=expected_mi)
+
+ with tm.assert_produces_warning(None):
+ result_df = concat((df1, df2), axis=1)
+
+ tm.assert_frame_equal(expected_df, result_df)
+
+ def test_concat_multiindex_(self):
+ # GitHub #44786
+ df = DataFrame({"col": ["a", "b", "c"]}, index=["1", "2", "2"])
+ df = concat([df], keys=["X"])
+
+ iterables = [["X"], ["1", "2", "2"]]
+ result_index = df.index
+ expected_index = MultiIndex.from_product(iterables)
+
+ tm.assert_index_equal(result_index, expected_index)
+
+ result_df = df
+ expected_df = DataFrame(
+ {"col": ["a", "b", "c"]}, index=MultiIndex.from_product(iterables)
+ )
+ tm.assert_frame_equal(result_df, expected_df)
+
+ def test_concat_with_key_not_unique(self, performance_warning):
+ # GitHub #46519
+ df1 = DataFrame({"name": [1]})
+ df2 = DataFrame({"name": [2]})
+ df3 = DataFrame({"name": [3]})
+ df_a = concat([df1, df2, df3], keys=["x", "y", "x"])
+ # the warning is caused by indexing unsorted multi-index
+ with tm.assert_produces_warning(
+ performance_warning, match="indexing past lexsort depth"
+ ):
+ out_a = df_a.loc[("x", 0), :]
+ df_b = DataFrame(
+ {"name": [1, 2, 3]},
+ index=MultiIndex(
+ levels=[["x", "y"], range(1)], codes=[[0, 1, 0], [0, 0, 0]]
+ ),
+ )
+ with tm.assert_produces_warning(
+ performance_warning, match="indexing past lexsort depth"
+ ):
+ out_b = df_b.loc[("x", 0)]
+
+ tm.assert_frame_equal(out_a, out_b)
+
+ df1 = DataFrame({"name": ["a", "a", "b"]})
+ df2 = DataFrame({"name": ["a", "b"]})
+ df3 = DataFrame({"name": ["c", "d"]})
+ df_a = concat([df1, df2, df3], keys=["x", "y", "x"])
+ with tm.assert_produces_warning(
+ performance_warning, match="indexing past lexsort depth"
+ ):
+ out_a = df_a.loc[("x", 0), :]
+
+ df_b = DataFrame(
+ {
+ "a": ["x", "x", "x", "y", "y", "x", "x"],
+ "b": [0, 1, 2, 0, 1, 0, 1],
+ "name": list("aababcd"),
+ }
+ ).set_index(["a", "b"])
+ df_b.index.names = [None, None]
+ with tm.assert_produces_warning(
+ performance_warning, match="indexing past lexsort depth"
+ ):
+ out_b = df_b.loc[("x", 0), :]
+
+ tm.assert_frame_equal(out_a, out_b)
+
+ def test_concat_with_duplicated_levels(self):
+ # keyword levels should be unique
+ df1 = DataFrame({"A": [1]}, index=["x"])
+ df2 = DataFrame({"A": [1]}, index=["y"])
+ msg = r"Level values not unique: \['x', 'y', 'y'\]"
+ with pytest.raises(ValueError, match=msg):
+ concat([df1, df2], keys=["x", "y"], levels=[["x", "y", "y"]])
+
+ @pytest.mark.parametrize("levels", [[["x", "y"]], [["x", "y", "y"]]])
+ def test_concat_with_levels_with_none_keys(self, levels):
+ df1 = DataFrame({"A": [1]}, index=["x"])
+ df2 = DataFrame({"A": [1]}, index=["y"])
+ msg = "levels supported only when keys is not None"
+ with pytest.raises(ValueError, match=msg):
+ concat([df1, df2], levels=levels)
+
+ def test_concat_range_index_result(self):
+ # GH#47501
+ df1 = DataFrame({"a": [1, 2]})
+ df2 = DataFrame({"b": [1, 2]})
+
+ result = concat([df1, df2], sort=True, axis=1)
+ expected = DataFrame({"a": [1, 2], "b": [1, 2]})
+ tm.assert_frame_equal(result, expected)
+ expected_index = pd.RangeIndex(0, 2)
+ tm.assert_index_equal(result.index, expected_index, exact=True)
+
+ def test_concat_index_keep_dtype(self):
+ # GH#47329
+ df1 = DataFrame([[0, 1, 1]], columns=Index([1, 2, 3], dtype="object"))
+ df2 = DataFrame([[0, 1]], columns=Index([1, 2], dtype="object"))
+ result = concat([df1, df2], ignore_index=True, join="outer", sort=True)
+ expected = DataFrame(
+ [[0, 1, 1.0], [0, 1, np.nan]], columns=Index([1, 2, 3], dtype="object")
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_index_keep_dtype_ea_numeric(self, any_numeric_ea_dtype):
+ # GH#47329
+ df1 = DataFrame(
+ [[0, 1, 1]], columns=Index([1, 2, 3], dtype=any_numeric_ea_dtype)
+ )
+ df2 = DataFrame([[0, 1]], columns=Index([1, 2], dtype=any_numeric_ea_dtype))
+ result = concat([df1, df2], ignore_index=True, join="outer", sort=True)
+ expected = DataFrame(
+ [[0, 1, 1.0], [0, 1, np.nan]],
+ columns=Index([1, 2, 3], dtype=any_numeric_ea_dtype),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("dtype", ["Int8", "Int16", "Int32"])
+ def test_concat_index_find_common(self, dtype):
+ # GH#47329
+ df1 = DataFrame([[0, 1, 1]], columns=Index([1, 2, 3], dtype=dtype))
+ df2 = DataFrame([[0, 1]], columns=Index([1, 2], dtype="Int32"))
+ result = concat([df1, df2], ignore_index=True, join="outer", sort=True)
+ expected = DataFrame(
+ [[0, 1, 1.0], [0, 1, np.nan]], columns=Index([1, 2, 3], dtype="Int32")
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_axis_1_sort_false_rangeindex(self, using_infer_string):
+ # GH 46675
+ s1 = Series(["a", "b", "c"])
+ s2 = Series(["a", "b"])
+ s3 = Series(["a", "b", "c", "d"])
+ s4 = Series([], dtype=object if not using_infer_string else "str")
+ result = concat(
+ [s1, s2, s3, s4], sort=False, join="outer", ignore_index=False, axis=1
+ )
+ expected = DataFrame(
+ [
+ ["a"] * 3 + [np.nan],
+ ["b"] * 3 + [np.nan],
+ ["c", np.nan] * 2,
+ [np.nan] * 2 + ["d"] + [np.nan],
+ ],
+ dtype=object if not using_infer_string else "str",
+ )
+ tm.assert_frame_equal(
+ result, expected, check_index_type=True, check_column_type=True
+ )
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_invalid.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_invalid.py
new file mode 100644
index 0000000000000000000000000000000000000000..3910dfa38a4bce615b5ec47903ceec4a9c9c6714
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_invalid.py
@@ -0,0 +1,54 @@
+from io import StringIO
+
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ concat,
+ read_csv,
+)
+import pandas._testing as tm
+
+
+class TestInvalidConcat:
+ @pytest.mark.parametrize("obj", [1, {}, [1, 2], (1, 2)])
+ def test_concat_invalid(self, obj):
+ # trying to concat an ndframe with a non-ndframe
+ df1 = DataFrame(range(2))
+ msg = (
+ f"cannot concatenate object of type '{type(obj)}'; "
+ "only Series and DataFrame objs are valid"
+ )
+ with pytest.raises(TypeError, match=msg):
+ concat([df1, obj])
+
+ def test_concat_invalid_first_argument(self):
+ df1 = DataFrame(range(2))
+ msg = (
+ "first argument must be an iterable of pandas "
+ 'objects, you passed an object of type "DataFrame"'
+ )
+ with pytest.raises(TypeError, match=msg):
+ concat(df1)
+
+ def test_concat_generator_obj(self):
+ # generator ok though
+ concat(DataFrame(np.random.default_rng(2).random((5, 5))) for _ in range(3))
+
+ def test_concat_textreader_obj(self):
+ # text reader ok
+ # GH6583
+ data = """index,A,B,C,D
+ foo,2,3,4,5
+ bar,7,8,9,10
+ baz,12,13,14,15
+ qux,12,13,14,15
+ foo2,12,13,14,15
+ bar2,12,13,14,15
+ """
+
+ with read_csv(StringIO(data), chunksize=1) as reader:
+ result = concat(reader, ignore_index=True)
+ expected = read_csv(StringIO(data))
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_series.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_series.py
new file mode 100644
index 0000000000000000000000000000000000000000..d49c62c9d7d8099e9bc56774055bef724edb98e3
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_series.py
@@ -0,0 +1,184 @@
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ DatetimeIndex,
+ Index,
+ MultiIndex,
+ Series,
+ concat,
+ date_range,
+)
+import pandas._testing as tm
+
+
+class TestSeriesConcat:
+ @pytest.mark.parametrize("bool_dtype", [bool, "boolean"])
+ @pytest.mark.parametrize("dtype", [np.int64, np.float64, "Int64", "Float64"])
+ def test_concat_bool_and_numeric(self, bool_dtype, dtype):
+ # GH#21108, GH#45101
+ left = Series([True, False], dtype=bool_dtype)
+ right = Series([1, 2], dtype=dtype)
+ result = concat([left, right], ignore_index=True)
+ expected = Series([True, False, 1, 2], dtype=object)
+ assert result.iloc[0] is True
+ assert type(result.iloc[2]) in [int, float] # i.e. not bool
+ tm.assert_series_equal(result, expected)
+
+ def test_concat_series(self):
+ ts = Series(
+ np.arange(20, dtype=np.float64),
+ index=date_range("2020-01-01", periods=20, unit="ns"),
+ name="foo",
+ )
+ ts.name = "foo"
+
+ pieces = [ts[:5], ts[5:15], ts[15:]]
+
+ result = concat(pieces)
+ tm.assert_series_equal(result, ts)
+ assert result.name == ts.name
+
+ result = concat(pieces, keys=[0, 1, 2])
+ expected = ts.copy()
+ exp_codes = [np.repeat([0, 1, 2], [len(x) for x in pieces]), np.arange(len(ts))]
+ exp_index = MultiIndex(
+ levels=[[0, 1, 2], DatetimeIndex(ts.index.to_numpy(dtype="M8[ns]"))],
+ codes=exp_codes,
+ )
+ expected.index = exp_index
+ tm.assert_series_equal(result, expected)
+
+ def test_concat_empty_and_non_empty_series_regression(self):
+ # GH 18187 regression test
+ s1 = Series([1])
+ s2 = Series([], dtype=object)
+
+ expected = s1.astype(object)
+ result = concat([s1, s2])
+ tm.assert_series_equal(result, expected)
+
+ def test_concat_series_axis1(self):
+ ts = Series(
+ np.arange(10, dtype=np.float64), index=date_range("2020-01-01", periods=10)
+ )
+
+ pieces = [ts[:-2], ts[2:], ts[2:-2]]
+
+ result = concat(pieces, axis=1)
+ expected = DataFrame(pieces).T
+ tm.assert_frame_equal(result, expected)
+
+ result = concat(pieces, keys=["A", "B", "C"], axis=1)
+ expected = DataFrame(pieces, index=["A", "B", "C"]).T
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_series_axis1_preserves_series_names(self):
+ # preserve series names, #2489
+ s = Series(np.random.default_rng(2).standard_normal(5), name="A")
+ s2 = Series(np.random.default_rng(2).standard_normal(5), name="B")
+
+ result = concat([s, s2], axis=1)
+ expected = DataFrame({"A": s, "B": s2})
+ tm.assert_frame_equal(result, expected)
+
+ s2.name = None
+ result = concat([s, s2], axis=1)
+ tm.assert_index_equal(result.columns, Index(["A", 0], dtype="object"))
+
+ def test_concat_series_axis1_with_reindex(self, sort):
+ # must reindex, #2603
+ s = Series(
+ np.random.default_rng(2).standard_normal(3), index=["c", "a", "b"], name="A"
+ )
+ s2 = Series(
+ np.random.default_rng(2).standard_normal(4),
+ index=["d", "a", "b", "c"],
+ name="B",
+ )
+ result = concat([s, s2], axis=1, sort=sort)
+ expected = DataFrame({"A": s, "B": s2}, index=["c", "a", "b", "d"])
+ if sort:
+ expected = expected.sort_index()
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_series_axis1_names_applied(self):
+ # ensure names argument is not ignored on axis=1, #23490
+ s = Series([1, 2, 3])
+ s2 = Series([4, 5, 6])
+ result = concat([s, s2], axis=1, keys=["a", "b"], names=["A"])
+ expected = DataFrame(
+ [[1, 4], [2, 5], [3, 6]], columns=Index(["a", "b"], name="A")
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = concat([s, s2], axis=1, keys=[("a", 1), ("b", 2)], names=["A", "B"])
+ expected = DataFrame(
+ [[1, 4], [2, 5], [3, 6]],
+ columns=MultiIndex.from_tuples([("a", 1), ("b", 2)], names=["A", "B"]),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_series_axis1_same_names_ignore_index(self):
+ dates = date_range("01-Jan-2013", "01-Jan-2014", freq="MS")[0:-1]
+ s1 = Series(
+ np.random.default_rng(2).standard_normal(len(dates)),
+ index=dates,
+ name="value",
+ )
+ s2 = Series(
+ np.random.default_rng(2).standard_normal(len(dates)),
+ index=dates,
+ name="value",
+ )
+
+ result = concat([s1, s2], axis=1, ignore_index=True)
+ expected = Index(range(2))
+
+ tm.assert_index_equal(result.columns, expected, exact=True)
+
+ @pytest.mark.parametrize("s1name", [np.int64(190), 190])
+ def test_concat_series_name_npscalar_tuple(self, s1name):
+ # GH21015
+ s2name = (43, 0)
+ s1 = Series({"a": 1, "b": 2}, name=s1name)
+ s2 = Series({"c": 5, "d": 6}, name=s2name)
+ result = concat([s1, s2])
+ expected = Series({"a": 1, "b": 2, "c": 5, "d": 6})
+ tm.assert_series_equal(result, expected)
+
+ def test_concat_series_partial_columns_names(self):
+ # GH10698
+ named_series = Series([1, 2], name="foo")
+ unnamed_series1 = Series([1, 2])
+ unnamed_series2 = Series([4, 5])
+
+ result = concat([named_series, unnamed_series1, unnamed_series2], axis=1)
+ expected = DataFrame(
+ {"foo": [1, 2], 0: [1, 2], 1: [4, 5]}, columns=["foo", 0, 1]
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = concat(
+ [named_series, unnamed_series1, unnamed_series2],
+ axis=1,
+ keys=["red", "blue", "yellow"],
+ )
+ expected = DataFrame(
+ {"red": [1, 2], "blue": [1, 2], "yellow": [4, 5]},
+ columns=["red", "blue", "yellow"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = concat(
+ [named_series, unnamed_series1, unnamed_series2], axis=1, ignore_index=True
+ )
+ expected = DataFrame({0: [1, 2], 1: [1, 2], 2: [4, 5]})
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_series_length_one_reversed(self, frame_or_series):
+ # GH39401
+ obj = frame_or_series([100])
+ result = concat([obj.iloc[::-1]])
+ tm.assert_equal(result, obj)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_sort.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_sort.py
new file mode 100644
index 0000000000000000000000000000000000000000..2724f819588933cc307dd396fdcd024f04c38eaa
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/concat/test_sort.py
@@ -0,0 +1,118 @@
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import DataFrame
+import pandas._testing as tm
+
+
+class TestConcatSort:
+ def test_concat_sorts_columns(self, sort):
+ # GH-4588
+ df1 = DataFrame({"a": [1, 2], "b": [1, 2]}, columns=["b", "a"])
+ df2 = DataFrame({"a": [3, 4], "c": [5, 6]})
+
+ # for sort=True/None
+ expected = DataFrame(
+ {"a": [1, 2, 3, 4], "b": [1, 2, None, None], "c": [None, None, 5, 6]},
+ columns=["a", "b", "c"],
+ )
+
+ if sort is False:
+ expected = expected[["b", "a", "c"]]
+
+ # default
+ with tm.assert_produces_warning(None):
+ result = pd.concat([df1, df2], ignore_index=True, sort=sort)
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_sorts_index(self, sort):
+ df1 = DataFrame({"a": [1, 2, 3]}, index=["c", "a", "b"])
+ df2 = DataFrame({"b": [1, 2]}, index=["a", "b"])
+
+ # For True/None
+ expected = DataFrame(
+ {"a": [2, 3, 1], "b": [1, 2, None]},
+ index=["a", "b", "c"],
+ columns=["a", "b"],
+ )
+ if sort is False:
+ expected = expected.loc[["c", "a", "b"]]
+
+ # Warn and sort by default
+ with tm.assert_produces_warning(None):
+ result = pd.concat([df1, df2], axis=1, sort=sort)
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_inner_sort(self, sort):
+ # https://github.com/pandas-dev/pandas/pull/20613
+ df1 = DataFrame(
+ {"a": [1, 2], "b": [1, 2], "c": [1, 2]}, columns=["b", "a", "c"]
+ )
+ df2 = DataFrame({"a": [1, 2], "b": [3, 4]}, index=[3, 4])
+
+ with tm.assert_produces_warning(None):
+ # unset sort should *not* warn for inner join
+ # since that never sorted
+ result = pd.concat([df1, df2], sort=sort, join="inner", ignore_index=True)
+
+ expected = DataFrame({"b": [1, 2, 3, 4], "a": [1, 2, 1, 2]}, columns=["b", "a"])
+ if sort is True:
+ expected = expected[["a", "b"]]
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_aligned_sort(self):
+ # GH-4588
+ df = DataFrame({"c": [1, 2], "b": [3, 4], "a": [5, 6]}, columns=["c", "b", "a"])
+ result = pd.concat([df, df], sort=True, ignore_index=True)
+ expected = DataFrame(
+ {"a": [5, 6, 5, 6], "b": [3, 4, 3, 4], "c": [1, 2, 1, 2]},
+ columns=["a", "b", "c"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = pd.concat(
+ [df, df[["c", "b"]]], join="inner", sort=True, ignore_index=True
+ )
+ expected = expected[["b", "c"]]
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_aligned_sort_does_not_raise(self):
+ # GH-4588
+ # We catch TypeErrors from sorting internally and do not re-raise.
+ df = DataFrame({1: [1, 2], "a": [3, 4]}, columns=[1, "a"])
+ expected = DataFrame({1: [1, 2, 1, 2], "a": [3, 4, 3, 4]}, columns=[1, "a"])
+ result = pd.concat([df, df], ignore_index=True, sort=True)
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_frame_with_sort_false(self):
+ # GH 43375
+ result = pd.concat(
+ [DataFrame({i: i}, index=[i]) for i in range(2, 0, -1)], sort=False
+ )
+ expected = DataFrame([[2, np.nan], [np.nan, 1]], index=[2, 1], columns=[2, 1])
+
+ tm.assert_frame_equal(result, expected)
+
+ # GH 37937
+ df1 = DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}, index=[1, 2, 3])
+ df2 = DataFrame({"c": [7, 8, 9], "d": [10, 11, 12]}, index=[3, 1, 6])
+ result = pd.concat([df2, df1], axis=1, sort=False)
+ expected = DataFrame(
+ [
+ [7.0, 10.0, 3.0, 6.0],
+ [8.0, 11.0, 1.0, 4.0],
+ [9.0, 12.0, np.nan, np.nan],
+ [np.nan, np.nan, 2.0, 5.0],
+ ],
+ index=[3, 1, 6, 2],
+ columns=["c", "d", "a", "b"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_concat_sort_none_raises(self):
+ # GH#41518
+ df = DataFrame({1: [1, 2], "a": [3, 4]})
+ msg = "The 'sort' keyword only accepts boolean values; None was passed."
+ with pytest.raises(ValueError, match=msg):
+ pd.concat([df, df], sort=None)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_join.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_join.py
new file mode 100644
index 0000000000000000000000000000000000000000..c866b81a5349eea1944209bab572021513f62175
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_join.py
@@ -0,0 +1,1186 @@
+from datetime import datetime
+import re
+
+import numpy as np
+import pytest
+
+import pandas.util._test_decorators as td
+
+import pandas as pd
+from pandas import (
+ Categorical,
+ DataFrame,
+ Index,
+ MultiIndex,
+ Series,
+ Timestamp,
+ bdate_range,
+ concat,
+ merge,
+ option_context,
+)
+import pandas._testing as tm
+
+
+def get_test_data(ngroups=8, n=50):
+ unique_groups = list(range(ngroups))
+ arr = np.asarray(np.tile(unique_groups, n // ngroups))
+
+ if len(arr) < n:
+ arr = np.asarray(list(arr) + unique_groups[: n - len(arr)])
+
+ np.random.default_rng(2).shuffle(arr)
+ return arr
+
+
+class TestJoin:
+ # aggregate multiple columns
+ @pytest.fixture
+ def df(self):
+ df = DataFrame(
+ {
+ "key1": get_test_data(),
+ "key2": get_test_data(),
+ "data1": np.random.default_rng(2).standard_normal(50),
+ "data2": np.random.default_rng(2).standard_normal(50),
+ }
+ )
+
+ # exclude a couple keys for fun
+ df = df[df["key2"] > 1]
+ return df
+
+ @pytest.fixture
+ def df2(self):
+ return DataFrame(
+ {
+ "key1": get_test_data(n=10),
+ "key2": get_test_data(ngroups=4, n=10),
+ "value": np.random.default_rng(2).standard_normal(10),
+ }
+ )
+
+ @pytest.fixture
+ def target_source(self):
+ data = {
+ "A": [0.0, 1.0, 2.0, 3.0, 4.0],
+ "B": [0.0, 1.0, 0.0, 1.0, 0.0],
+ "C": ["foo1", "foo2", "foo3", "foo4", "foo5"],
+ "D": bdate_range("1/1/2009", periods=5),
+ }
+ target = DataFrame(data, index=Index(["a", "b", "c", "d", "e"], dtype=object))
+
+ # Join on string value
+
+ source = DataFrame(
+ {"MergedA": data["A"], "MergedD": data["D"]}, index=data["C"]
+ )
+ return target, source
+
+ def test_left_outer_join(self, df, df2):
+ joined_key2 = merge(df, df2, on="key2")
+ _check_join(df, df2, joined_key2, ["key2"], how="left")
+
+ joined_both = merge(df, df2)
+ _check_join(df, df2, joined_both, ["key1", "key2"], how="left")
+
+ def test_right_outer_join(self, df, df2):
+ joined_key2 = merge(df, df2, on="key2", how="right")
+ _check_join(df, df2, joined_key2, ["key2"], how="right")
+
+ joined_both = merge(df, df2, how="right")
+ _check_join(df, df2, joined_both, ["key1", "key2"], how="right")
+
+ def test_full_outer_join(self, df, df2):
+ joined_key2 = merge(df, df2, on="key2", how="outer")
+ _check_join(df, df2, joined_key2, ["key2"], how="outer")
+
+ joined_both = merge(df, df2, how="outer")
+ _check_join(df, df2, joined_both, ["key1", "key2"], how="outer")
+
+ def test_inner_join(self, df, df2):
+ joined_key2 = merge(df, df2, on="key2", how="inner")
+ _check_join(df, df2, joined_key2, ["key2"], how="inner")
+
+ joined_both = merge(df, df2, how="inner")
+ _check_join(df, df2, joined_both, ["key1", "key2"], how="inner")
+
+ def test_handle_overlap(self, df, df2):
+ joined = merge(df, df2, on="key2", suffixes=(".foo", ".bar"))
+
+ assert "key1.foo" in joined
+ assert "key1.bar" in joined
+
+ def test_handle_overlap_arbitrary_key(self, df, df2):
+ joined = merge(
+ df,
+ df2,
+ left_on="key2",
+ right_on="key1",
+ suffixes=(".foo", ".bar"),
+ )
+ assert "key1.foo" in joined
+ assert "key2.bar" in joined
+
+ @pytest.mark.parametrize(
+ "infer_string", [False, pytest.param(True, marks=td.skip_if_no("pyarrow"))]
+ )
+ def test_join_on(self, target_source, infer_string):
+ target, source = target_source
+
+ merged = target.join(source, on="C")
+ tm.assert_series_equal(merged["MergedA"], target["A"], check_names=False)
+ tm.assert_series_equal(merged["MergedD"], target["D"], check_names=False)
+
+ # join with duplicates (fix regression from DataFrame/Matrix merge)
+ df = DataFrame({"key": ["a", "a", "b", "b", "c"]})
+ df2 = DataFrame({"value": [0, 1, 2]}, index=["a", "b", "c"])
+ joined = df.join(df2, on="key")
+ expected = DataFrame(
+ {"key": ["a", "a", "b", "b", "c"], "value": [0, 0, 1, 1, 2]}
+ )
+ tm.assert_frame_equal(joined, expected)
+
+ # Test when some are missing
+ df_a = DataFrame([[1], [2], [3]], index=["a", "b", "c"], columns=["one"])
+ df_b = DataFrame([["foo"], ["bar"]], index=[1, 2], columns=["two"])
+ df_c = DataFrame([[1], [2]], index=[1, 2], columns=["three"])
+ joined = df_a.join(df_b, on="one")
+ joined = joined.join(df_c, on="one")
+ assert np.isnan(joined["two"]["c"])
+ assert np.isnan(joined["three"]["c"])
+
+ # merge column not p resent
+ with pytest.raises(KeyError, match="^'E'$"):
+ target.join(source, on="E")
+
+ # overlap
+ msg = (
+ "You are trying to merge on float64 and object|str columns for key "
+ "'A'. If you wish to proceed you should use pd.concat"
+ )
+ with pytest.raises(ValueError, match=msg):
+ target.join(source, on="A")
+
+ def test_join_on_fails_with_different_right_index(self):
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).choice(["m", "f"], size=3),
+ "b": np.random.default_rng(2).standard_normal(3),
+ }
+ )
+ df2 = DataFrame(
+ {
+ "a": np.random.default_rng(2).choice(["m", "f"], size=10),
+ "b": np.random.default_rng(2).standard_normal(10),
+ },
+ index=MultiIndex.from_product([range(5), ["A", "B"]]),
+ )
+ msg = r'len\(left_on\) must equal the number of levels in the index of "right"'
+ with pytest.raises(ValueError, match=msg):
+ merge(df, df2, left_on="a", right_index=True)
+
+ def test_join_on_fails_with_different_left_index(self):
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).choice(["m", "f"], size=3),
+ "b": np.random.default_rng(2).standard_normal(3),
+ },
+ index=MultiIndex.from_arrays([range(3), list("abc")]),
+ )
+ df2 = DataFrame(
+ {
+ "a": np.random.default_rng(2).choice(["m", "f"], size=10),
+ "b": np.random.default_rng(2).standard_normal(10),
+ }
+ )
+ msg = r'len\(right_on\) must equal the number of levels in the index of "left"'
+ with pytest.raises(ValueError, match=msg):
+ merge(df, df2, right_on="b", left_index=True)
+
+ def test_join_on_fails_with_different_column_counts(self):
+ df = DataFrame(
+ {
+ "a": np.random.default_rng(2).choice(["m", "f"], size=3),
+ "b": np.random.default_rng(2).standard_normal(3),
+ }
+ )
+ df2 = DataFrame(
+ {
+ "a": np.random.default_rng(2).choice(["m", "f"], size=10),
+ "b": np.random.default_rng(2).standard_normal(10),
+ },
+ index=MultiIndex.from_product([range(5), ["A", "B"]]),
+ )
+ msg = r"len\(right_on\) must equal len\(left_on\)"
+ with pytest.raises(ValueError, match=msg):
+ merge(df, df2, right_on="a", left_on=["a", "b"])
+
+ @pytest.mark.parametrize("wrong_type", [2, "str", None, np.array([0, 1])])
+ def test_join_on_fails_with_wrong_object_type(self, wrong_type):
+ # GH12081 - original issue
+
+ # GH21220 - merging of Series and DataFrame is now allowed
+ # Edited test to remove the Series object from test parameters
+
+ df = DataFrame({"a": [1, 1]})
+ msg = (
+ "Can only merge Series or DataFrame objects, "
+ f"a {type(wrong_type)} was passed"
+ )
+ with pytest.raises(TypeError, match=msg):
+ merge(wrong_type, df, left_on="a", right_on="a")
+ with pytest.raises(TypeError, match=msg):
+ merge(df, wrong_type, left_on="a", right_on="a")
+
+ def test_join_on_pass_vector(self, target_source):
+ target, source = target_source
+ expected = target.join(source, on="C")
+ expected = expected.rename(columns={"C": "key_0"})
+ expected = expected[["key_0", "A", "B", "D", "MergedA", "MergedD"]]
+
+ join_col = target.pop("C")
+ result = target.join(source, on=join_col)
+ tm.assert_frame_equal(result, expected)
+
+ def test_join_with_len0(self, target_source):
+ # nothing to merge
+ target, source = target_source
+ merged = target.join(source.reindex([]), on="C")
+ for col in source:
+ assert col in merged
+ assert merged[col].isna().all()
+
+ merged2 = target.join(source.reindex([]), on="C", how="inner")
+ tm.assert_index_equal(merged2.columns, merged.columns)
+ assert len(merged2) == 0
+
+ def test_join_on_inner(self):
+ df = DataFrame({"key": ["a", "a", "d", "b", "b", "c"]})
+ df2 = DataFrame({"value": [0, 1]}, index=["a", "b"])
+
+ joined = df.join(df2, on="key", how="inner")
+
+ expected = df.join(df2, on="key")
+ expected = expected[expected["value"].notna()]
+ tm.assert_series_equal(joined["key"], expected["key"])
+ tm.assert_series_equal(joined["value"], expected["value"], check_dtype=False)
+ tm.assert_index_equal(joined.index, expected.index)
+
+ def test_join_on_singlekey_list(self):
+ df = DataFrame({"key": ["a", "a", "b", "b", "c"]})
+ df2 = DataFrame({"value": [0, 1, 2]}, index=["a", "b", "c"])
+
+ # corner cases
+ joined = df.join(df2, on=["key"])
+ expected = df.join(df2, on="key")
+
+ tm.assert_frame_equal(joined, expected)
+
+ def test_join_on_series(self, target_source):
+ target, source = target_source
+ result = target.join(source["MergedA"], on="C")
+ expected = target.join(source[["MergedA"]], on="C")
+ tm.assert_frame_equal(result, expected)
+
+ def test_join_on_series_buglet(self):
+ # GH #638
+ df = DataFrame({"a": [1, 1]})
+ ds = Series([2], index=[1], name="b")
+ result = df.join(ds, on="a")
+ expected = DataFrame({"a": [1, 1], "b": [2, 2]}, index=df.index)
+ tm.assert_frame_equal(result, expected)
+
+ def test_join_index_mixed(self, join_type):
+ # no overlapping blocks
+ df1 = DataFrame(index=np.arange(10))
+ df1["bool"] = True
+ df1["string"] = "foo"
+
+ df2 = DataFrame(index=np.arange(5, 15))
+ df2["int"] = 1
+ df2["float"] = 1.0
+
+ joined = df1.join(df2, how=join_type)
+ expected = _join_by_hand(df1, df2, how=join_type)
+ tm.assert_frame_equal(joined, expected)
+
+ joined = df2.join(df1, how=join_type)
+ expected = _join_by_hand(df2, df1, how=join_type)
+ tm.assert_frame_equal(joined, expected)
+
+ def test_join_index_mixed_overlap(self):
+ df1 = DataFrame(
+ {"A": 1.0, "B": 2, "C": "foo", "D": True},
+ index=np.arange(10),
+ columns=["A", "B", "C", "D"],
+ )
+ assert df1["B"].dtype == np.int64
+ assert df1["D"].dtype == np.bool_
+
+ df2 = DataFrame(
+ {"A": 1.0, "B": 2, "C": "foo", "D": True},
+ index=np.arange(0, 10, 2),
+ columns=["A", "B", "C", "D"],
+ )
+
+ # overlap
+ joined = df1.join(df2, lsuffix="_one", rsuffix="_two")
+ expected_columns = [
+ "A_one",
+ "B_one",
+ "C_one",
+ "D_one",
+ "A_two",
+ "B_two",
+ "C_two",
+ "D_two",
+ ]
+ df1.columns = expected_columns[:4]
+ df2.columns = expected_columns[4:]
+ expected = _join_by_hand(df1, df2)
+ tm.assert_frame_equal(joined, expected)
+
+ def test_join_empty_bug(self):
+ # generated an exception in 0.4.3
+ x = DataFrame()
+ x.join(DataFrame([3], index=[0], columns=["A"]), how="outer")
+
+ def test_join_unconsolidated(self):
+ # GH #331
+ a = DataFrame(
+ np.random.default_rng(2).standard_normal((30, 2)), columns=["a", "b"]
+ )
+ c = Series(np.random.default_rng(2).standard_normal(30))
+ a["c"] = c
+ d = DataFrame(np.random.default_rng(2).standard_normal((30, 1)), columns=["q"])
+
+ # it works!
+ a.join(d)
+ d.join(a)
+
+ def test_join_multiindex(self):
+ index1 = MultiIndex.from_arrays(
+ [["a", "a", "a", "b", "b", "b"], [1, 2, 3, 1, 2, 3]],
+ names=["first", "second"],
+ )
+
+ index2 = MultiIndex.from_arrays(
+ [["b", "b", "b", "c", "c", "c"], [1, 2, 3, 1, 2, 3]],
+ names=["first", "second"],
+ )
+
+ df1 = DataFrame(
+ data=np.random.default_rng(2).standard_normal(6),
+ index=index1,
+ columns=["var X"],
+ )
+ df2 = DataFrame(
+ data=np.random.default_rng(2).standard_normal(6),
+ index=index2,
+ columns=["var Y"],
+ )
+
+ df1 = df1.sort_index(level=0)
+ df2 = df2.sort_index(level=0)
+
+ joined = df1.join(df2, how="outer")
+ ex_index = Index(index1.values).union(Index(index2.values))
+ expected = df1.reindex(ex_index).join(df2.reindex(ex_index))
+ expected.index.names = index1.names
+ tm.assert_frame_equal(joined, expected)
+ assert joined.index.names == index1.names
+
+ df1 = df1.sort_index(level=1)
+ df2 = df2.sort_index(level=1)
+
+ joined = df1.join(df2, how="outer").sort_index(level=0)
+ ex_index = Index(index1.values).union(Index(index2.values))
+ expected = df1.reindex(ex_index).join(df2.reindex(ex_index))
+ expected.index.names = index1.names
+
+ tm.assert_frame_equal(joined, expected)
+ assert joined.index.names == index1.names
+
+ def test_join_inner_multiindex(self, lexsorted_two_level_string_multiindex):
+ key1 = ["bar", "bar", "bar", "foo", "foo", "baz", "baz", "qux", "qux", "snap"]
+ key2 = [
+ "two",
+ "one",
+ "three",
+ "one",
+ "two",
+ "one",
+ "two",
+ "two",
+ "three",
+ "one",
+ ]
+
+ data = np.random.default_rng(2).standard_normal(len(key1))
+ data = DataFrame({"key1": key1, "key2": key2, "data": data})
+
+ index = lexsorted_two_level_string_multiindex
+ to_join = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 3)),
+ index=index,
+ columns=["j_one", "j_two", "j_three"],
+ )
+
+ joined = data.join(to_join, on=["key1", "key2"], how="inner")
+ expected = merge(
+ data,
+ to_join.reset_index(),
+ left_on=["key1", "key2"],
+ right_on=["first", "second"],
+ how="inner",
+ sort=False,
+ )
+
+ expected2 = merge(
+ to_join,
+ data,
+ right_on=["key1", "key2"],
+ left_index=True,
+ how="inner",
+ sort=False,
+ )
+ tm.assert_frame_equal(joined, expected2.reindex_like(joined))
+
+ expected2 = merge(
+ to_join,
+ data,
+ right_on=["key1", "key2"],
+ left_index=True,
+ how="inner",
+ sort=False,
+ )
+
+ expected = expected.drop(["first", "second"], axis=1)
+ expected.index = joined.index
+
+ assert joined.index.is_monotonic_increasing
+ tm.assert_frame_equal(joined, expected)
+
+ # _assert_same_contents(expected, expected2.loc[:, expected.columns])
+
+ def test_join_hierarchical_mixed_raises(self):
+ # GH 2024
+ # GH 40993: For raising, enforced in 2.0
+ df = DataFrame([(1, 2, 3), (4, 5, 6)], columns=["a", "b", "c"])
+ new_df = df.groupby(["a"]).agg({"b": ["mean", "sum"]})
+ other_df = DataFrame([(1, 2, 3), (7, 10, 6)], columns=["a", "b", "d"])
+ other_df.set_index("a", inplace=True)
+ # GH 9455, 12219
+ with pytest.raises(
+ pd.errors.MergeError, match="Not allowed to merge between different levels"
+ ):
+ merge(new_df, other_df, left_index=True, right_index=True)
+
+ def test_join_float64_float32(self):
+ a = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 2)),
+ columns=["a", "b"],
+ dtype=np.float64,
+ )
+ b = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 1)),
+ columns=["c"],
+ dtype=np.float32,
+ )
+ joined = a.join(b)
+ assert joined.dtypes["a"] == "float64"
+ assert joined.dtypes["b"] == "float64"
+ assert joined.dtypes["c"] == "float32"
+
+ a = np.random.default_rng(2).integers(0, 5, 100).astype("int64")
+ b = np.random.default_rng(2).random(100).astype("float64")
+ c = np.random.default_rng(2).random(100).astype("float32")
+ df = DataFrame({"a": a, "b": b, "c": c})
+ xpdf = DataFrame({"a": a, "b": b, "c": c})
+ s = DataFrame(
+ np.random.default_rng(2).random(5).astype("float32"), columns=["md"]
+ )
+ rs = df.merge(s, left_on="a", right_index=True)
+ assert rs.dtypes["a"] == "int64"
+ assert rs.dtypes["b"] == "float64"
+ assert rs.dtypes["c"] == "float32"
+ assert rs.dtypes["md"] == "float32"
+
+ xp = xpdf.merge(s, left_on="a", right_index=True)
+ tm.assert_frame_equal(rs, xp)
+
+ def test_join_many_non_unique_index(self):
+ df1 = DataFrame({"a": [1, 1], "b": [1, 1], "c": [10, 20]})
+ df2 = DataFrame({"a": [1, 1], "b": [1, 2], "d": [100, 200]})
+ df3 = DataFrame({"a": [1, 1], "b": [1, 2], "e": [1000, 2000]})
+ idf1 = df1.set_index(["a", "b"])
+ idf2 = df2.set_index(["a", "b"])
+ idf3 = df3.set_index(["a", "b"])
+
+ result = idf1.join([idf2, idf3], how="outer")
+
+ df_partially_merged = merge(df1, df2, on=["a", "b"], how="outer")
+ expected = merge(df_partially_merged, df3, on=["a", "b"], how="outer")
+
+ result = result.reset_index()
+ expected = expected[result.columns]
+ expected["a"] = expected.a.astype("int64")
+ expected["b"] = expected.b.astype("int64")
+ tm.assert_frame_equal(result, expected)
+
+ df1 = DataFrame({"a": [1, 1, 1], "b": [1, 1, 1], "c": [10, 20, 30]})
+ df2 = DataFrame({"a": [1, 1, 1], "b": [1, 1, 2], "d": [100, 200, 300]})
+ df3 = DataFrame({"a": [1, 1, 1], "b": [1, 1, 2], "e": [1000, 2000, 3000]})
+ idf1 = df1.set_index(["a", "b"])
+ idf2 = df2.set_index(["a", "b"])
+ idf3 = df3.set_index(["a", "b"])
+ result = idf1.join([idf2, idf3], how="inner")
+
+ df_partially_merged = merge(df1, df2, on=["a", "b"], how="inner")
+ expected = merge(df_partially_merged, df3, on=["a", "b"], how="inner")
+
+ result = result.reset_index()
+
+ tm.assert_frame_equal(result, expected.loc[:, result.columns])
+
+ # GH 11519
+ df = DataFrame(
+ {
+ "A": ["foo", "bar", "foo", "bar", "foo", "bar", "foo", "foo"],
+ "B": ["one", "one", "two", "three", "two", "two", "one", "three"],
+ "C": np.random.default_rng(2).standard_normal(8),
+ "D": np.random.default_rng(2).standard_normal(8),
+ }
+ )
+ s = Series(
+ np.repeat(np.arange(8), 2), index=np.repeat(np.arange(8), 2), name="TEST"
+ )
+ inner = df.join(s, how="inner")
+ outer = df.join(s, how="outer")
+ left = df.join(s, how="left")
+ right = df.join(s, how="right")
+ tm.assert_frame_equal(inner, outer)
+ tm.assert_frame_equal(inner, left)
+ tm.assert_frame_equal(inner, right)
+
+ @pytest.mark.parametrize(
+ "infer_string", [False, pytest.param(True, marks=td.skip_if_no("pyarrow"))]
+ )
+ def test_join_sort(self, infer_string):
+ with option_context("future.infer_string", infer_string):
+ left = DataFrame(
+ {"key": ["foo", "bar", "baz", "foo"], "value": [1, 2, 3, 4]}
+ )
+ right = DataFrame({"value2": ["a", "b", "c"]}, index=["bar", "baz", "foo"])
+
+ joined = left.join(right, on="key", sort=True)
+ expected = DataFrame(
+ {
+ "key": ["bar", "baz", "foo", "foo"],
+ "value": [2, 3, 1, 4],
+ "value2": ["a", "b", "c", "c"],
+ },
+ index=[1, 2, 0, 3],
+ )
+ tm.assert_frame_equal(joined, expected)
+
+ # smoke test
+ joined = left.join(right, on="key", sort=False)
+ tm.assert_index_equal(joined.index, Index(range(4)), exact=True)
+
+ def test_join_mixed_non_unique_index(self):
+ # GH 12814, unorderable types in py3 with a non-unique index
+ df1 = DataFrame({"a": [1, 2, 3, 4]}, index=[1, 2, 3, "a"])
+ df2 = DataFrame({"b": [5, 6, 7, 8]}, index=[1, 3, 3, 4])
+ result = df1.join(df2)
+ expected = DataFrame(
+ {"a": [1, 2, 3, 3, 4], "b": [5, np.nan, 6, 7, np.nan]},
+ index=[1, 2, 3, 3, "a"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+ df3 = DataFrame({"a": [1, 2, 3, 4]}, index=[1, 2, 2, "a"])
+ df4 = DataFrame({"b": [5, 6, 7, 8]}, index=[1, 2, 3, 4])
+ result = df3.join(df4)
+ expected = DataFrame(
+ {"a": [1, 2, 3, 4], "b": [5, 6, 6, np.nan]}, index=[1, 2, 2, "a"]
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_join_non_unique_period_index(self):
+ # GH #16871
+ index = pd.period_range("2016-01-01", periods=16, freq="M")
+ df = DataFrame(list(range(len(index))), index=index, columns=["pnum"])
+ df2 = concat([df, df])
+ result = df.join(df2, how="inner", rsuffix="_df2")
+ expected = DataFrame(
+ np.tile(np.arange(16, dtype=np.int64).repeat(2).reshape(-1, 1), 2),
+ columns=["pnum", "pnum_df2"],
+ index=df2.sort_index().index,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_mixed_type_join_with_suffix(self, using_infer_string):
+ # GH #916
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((20, 6)),
+ columns=["a", "b", "c", "d", "e", "f"],
+ )
+ df.insert(0, "id", 0)
+ df.insert(5, "dt", "foo")
+
+ grouped = df.groupby("id")
+ msg = re.escape("agg function failed [how->mean,dtype->")
+ if using_infer_string:
+ msg = "dtype 'str' does not support operation 'mean'"
+ with pytest.raises(TypeError, match=msg):
+ grouped.mean()
+ mn = grouped.mean(numeric_only=True)
+ cn = grouped.count()
+
+ # it works!
+ mn.join(cn, rsuffix="_right")
+
+ def test_join_many(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 6)), columns=list("abcdef")
+ )
+ df_list = [df[["a", "b"]], df[["c", "d"]], df[["e", "f"]]]
+
+ joined = df_list[0].join(df_list[1:])
+ tm.assert_frame_equal(joined, df)
+
+ df_list = [df[["a", "b"]][:-2], df[["c", "d"]][2:], df[["e", "f"]][1:9]]
+
+ def _check_diff_index(df_list, result, exp_index):
+ reindexed = [x.reindex(exp_index) for x in df_list]
+ expected = reindexed[0].join(reindexed[1:])
+ tm.assert_frame_equal(result, expected)
+
+ # different join types
+ joined = df_list[0].join(df_list[1:], how="outer")
+ _check_diff_index(df_list, joined, df.index)
+
+ joined = df_list[0].join(df_list[1:])
+ _check_diff_index(df_list, joined, df_list[0].index)
+
+ joined = df_list[0].join(df_list[1:], how="inner")
+ _check_diff_index(df_list, joined, df.index[2:8])
+
+ msg = "Joining multiple DataFrames only supported for joining on index"
+ with pytest.raises(ValueError, match=msg):
+ df_list[0].join(df_list[1:], on="a")
+
+ @pytest.mark.parametrize("how", ["left", "right", "inner", "outer"])
+ def test_join_many_sort_unique(self, how, sort):
+ # https://github.com/pandas-dev/pandas/pull/62954
+ df = DataFrame({"a": [1, 2, 3]}, index=[1, 0, 2])
+ df2 = DataFrame({"b": [4, 5, 6]}, index=[2, 0, 1])
+ if how == "right":
+ expected = DataFrame({"a": [3, 2, 1], "b": [4, 5, 6]}, index=[2, 0, 1])
+ else:
+ expected = DataFrame({"a": [1, 2, 3], "b": [6, 5, 4]}, index=[1, 0, 2])
+ if how == "outer" or sort:
+ # outer always sorts.
+ expected = expected.sort_index()
+ result = df.join([df2], how=how, sort=sort)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("how", ["left", "right", "inner", "outer"])
+ def test_join_many_sort_nonunique(self, how, sort):
+ # https://github.com/pandas-dev/pandas/pull/62954
+ df = DataFrame({"a": [1, 2, 3]}, index=[3, 0, 0])
+ df2 = DataFrame({"b": [4, 5, 6]}, index=[2, 0, 1])
+ if how == "inner":
+ expected = DataFrame({"a": [2, 3], "b": [5, 5]}, index=[0, 0])
+ elif how == "left":
+ expected = DataFrame(
+ {"a": [1, 2, 3], "b": [np.nan, 5.0, 5.0]}, index=[3, 0, 0]
+ )
+ elif how == "right":
+ expected = DataFrame(
+ {"a": [np.nan, 2.0, 3.0, np.nan], "b": [4, 5, 5, 6]}, index=[2, 0, 0, 1]
+ )
+ else:
+ expected = DataFrame(
+ {
+ "a": [2.0, 3.0, np.nan, np.nan, 1.0],
+ "b": [5.0, 5.0, 6.0, 4.0, np.nan],
+ },
+ index=[0, 0, 1, 2, 3],
+ )
+ if sort:
+ expected = expected.sort_index()
+ result = df.join([df2], how=how, sort=sort)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("how", ["inner", "outer", "left", "right"])
+ def test_join_many_datetime_unsorted(self, how):
+ # https://github.com/pandas-dev/pandas/pull/62843
+ index = Index([datetime(2024, 1, 2), datetime(2024, 1, 1)])
+ df = DataFrame({"a": [1, 2]}, index=index)
+ df2 = DataFrame({"b": [1, 2]}, index=index)
+ result = df.join([df2], how=how)
+ if how == "outer":
+ # Outer always sorts the index.
+ expected = DataFrame({"a": [2, 1], "b": [2, 1]}, index=[index[1], index[0]])
+ else:
+ expected = DataFrame({"a": [1, 2], "b": [1, 2]}, index=index)
+ tm.assert_frame_equal(result, expected)
+
+ def test_join_many_mixed(self):
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((8, 4)),
+ columns=["A", "B", "C", "D"],
+ )
+ df["key"] = ["foo", "bar"] * 4
+ df1 = df.loc[:, ["A", "B"]]
+ df2 = df.loc[:, ["C", "D"]]
+ df3 = df.loc[:, ["key"]]
+
+ result = df1.join([df2, df3])
+ tm.assert_frame_equal(result, df)
+
+ def test_join_dups(self):
+ # joining dups
+ df = concat(
+ [
+ DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=["A", "A", "B", "B"],
+ ),
+ DataFrame(
+ np.random.default_rng(2).integers(0, 10, size=20).reshape(10, 2),
+ columns=["A", "C"],
+ ),
+ ],
+ axis=1,
+ )
+
+ expected = concat([df, df], axis=1)
+ result = df.join(df, rsuffix="_2")
+ result.columns = expected.columns
+ tm.assert_frame_equal(result, expected)
+
+ # GH 4975, invalid join on dups
+ w = DataFrame(
+ np.random.default_rng(2).standard_normal((4, 2)), columns=["x", "y"]
+ )
+ x = DataFrame(
+ np.random.default_rng(2).standard_normal((4, 2)), columns=["x", "y"]
+ )
+ y = DataFrame(
+ np.random.default_rng(2).standard_normal((4, 2)), columns=["x", "y"]
+ )
+ z = DataFrame(
+ np.random.default_rng(2).standard_normal((4, 2)), columns=["x", "y"]
+ )
+
+ dta = x.merge(y, left_index=True, right_index=True).merge(
+ z, left_index=True, right_index=True, how="outer"
+ )
+ # GH 40991: As of 2.0 causes duplicate columns
+ with pytest.raises(
+ pd.errors.MergeError,
+ match="Passing 'suffixes' which cause duplicate columns",
+ ):
+ dta.merge(w, left_index=True, right_index=True)
+
+ def test_join_multi_to_multi(self, join_type):
+ # GH 20475
+ leftindex = MultiIndex.from_product(
+ [list("abc"), list("xy"), [1, 2]], names=["abc", "xy", "num"]
+ )
+ left = DataFrame({"v1": range(12)}, index=leftindex)
+
+ rightindex = MultiIndex.from_product(
+ [list("abc"), list("xy")], names=["abc", "xy"]
+ )
+ right = DataFrame({"v2": [100 * i for i in range(1, 7)]}, index=rightindex)
+
+ result = left.join(right, on=["abc", "xy"], how=join_type)
+ expected = (
+ left.reset_index()
+ .merge(right.reset_index(), on=["abc", "xy"], how=join_type)
+ .set_index(["abc", "xy", "num"])
+ )
+ tm.assert_frame_equal(expected, result)
+
+ msg = r'len\(left_on\) must equal the number of levels in the index of "right"'
+ with pytest.raises(ValueError, match=msg):
+ left.join(right, on="xy", how=join_type)
+
+ with pytest.raises(ValueError, match=msg):
+ right.join(left, on=["abc", "xy"], how=join_type)
+
+ def test_join_on_tz_aware_datetimeindex(self):
+ # GH 23931, 26335
+ df1 = DataFrame(
+ {
+ "date": pd.date_range(
+ start="2018-01-01", periods=5, tz="America/Chicago"
+ ),
+ "vals": list("abcde"),
+ }
+ )
+
+ df2 = DataFrame(
+ {
+ "date": pd.date_range(
+ start="2018-01-03", periods=5, tz="America/Chicago"
+ ),
+ "vals_2": list("tuvwx"),
+ }
+ )
+ result = df1.join(df2.set_index("date"), on="date")
+ expected = df1.copy()
+ expected["vals_2"] = Series([np.nan] * 2 + list("tuv"))
+ tm.assert_frame_equal(result, expected)
+
+ def test_join_datetime_string(self):
+ # GH 5647
+ dfa = DataFrame(
+ [
+ ["2012-08-02", "L", 10],
+ ["2012-08-02", "J", 15],
+ ["2013-04-06", "L", 20],
+ ["2013-04-06", "J", 25],
+ ],
+ columns=["x", "y", "a"],
+ )
+ dfa["x"] = pd.to_datetime(dfa["x"]).astype("M8[ns]")
+ dfb = DataFrame(
+ [["2012-08-02", "J", 1], ["2013-04-06", "L", 2]],
+ columns=["x", "y", "z"],
+ index=[2, 4],
+ )
+ dfb["x"] = pd.to_datetime(dfb["x"]).astype("M8[ns]")
+ result = dfb.join(dfa.set_index(["x", "y"]), on=["x", "y"])
+ expected = DataFrame(
+ [
+ [Timestamp("2012-08-02 00:00:00"), "J", 1, 15],
+ [Timestamp("2013-04-06 00:00:00"), "L", 2, 20],
+ ],
+ index=[2, 4],
+ columns=["x", "y", "z", "a"],
+ )
+ expected["x"] = expected["x"].astype("M8[ns]")
+ tm.assert_frame_equal(result, expected)
+
+ def test_join_with_categorical_index(self):
+ # GH47812
+ ix = ["a", "b"]
+ id1 = pd.CategoricalIndex(ix, categories=ix)
+ id2 = pd.CategoricalIndex(reversed(ix), categories=reversed(ix))
+
+ df1 = DataFrame({"c1": ix}, index=id1)
+ df2 = DataFrame({"c2": reversed(ix)}, index=id2)
+ result = df1.join(df2)
+ expected = DataFrame(
+ {"c1": ["a", "b"], "c2": ["a", "b"]},
+ index=pd.CategoricalIndex(["a", "b"], categories=["a", "b"]),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def _check_join(left, right, result, join_col, how="left", lsuffix="_x", rsuffix="_y"):
+ # some smoke tests
+ for c in join_col:
+ assert result[c].notna().all()
+
+ left_grouped = left.groupby(join_col)
+ right_grouped = right.groupby(join_col)
+
+ for group_key, group in result.groupby(join_col):
+ l_joined = _restrict_to_columns(group, left.columns, lsuffix)
+ r_joined = _restrict_to_columns(group, right.columns, rsuffix)
+
+ try:
+ lgroup = left_grouped.get_group(group_key)
+ except KeyError as err:
+ if how in ("left", "inner"):
+ raise AssertionError(
+ f"key {group_key} should not have been in the join"
+ ) from err
+
+ _assert_all_na(l_joined, left.columns, join_col)
+ else:
+ _assert_same_contents(l_joined, lgroup)
+
+ try:
+ rgroup = right_grouped.get_group(group_key)
+ except KeyError as err:
+ if how in ("right", "inner"):
+ raise AssertionError(
+ f"key {group_key} should not have been in the join"
+ ) from err
+
+ _assert_all_na(r_joined, right.columns, join_col)
+ else:
+ _assert_same_contents(r_joined, rgroup)
+
+
+def _restrict_to_columns(group, columns, suffix):
+ found = [
+ c for c in group.columns if c in columns or c.replace(suffix, "") in columns
+ ]
+
+ # filter
+ group = group.loc[:, found]
+
+ # get rid of suffixes, if any
+ group = group.rename(columns=lambda x: x.replace(suffix, ""))
+
+ # put in the right order...
+ group = group.loc[:, columns]
+
+ return group
+
+
+def _assert_same_contents(join_chunk, source):
+ NA_SENTINEL = -1234567 # drop_duplicates not so NA-friendly...
+
+ jvalues = join_chunk.fillna(NA_SENTINEL).drop_duplicates().values
+ svalues = source.fillna(NA_SENTINEL).drop_duplicates().values
+
+ rows = {tuple(row) for row in jvalues}
+ assert len(rows) == len(source)
+ assert all(tuple(row) in rows for row in svalues)
+
+
+def _assert_all_na(join_chunk, source_columns, join_col):
+ for c in source_columns:
+ if c in join_col:
+ continue
+ assert join_chunk[c].isna().all()
+
+
+def _join_by_hand(a, b, how="left"):
+ join_index = a.index.join(b.index, how=how)
+
+ a_re = a.reindex(join_index)
+ b_re = b.reindex(join_index)
+
+ result_columns = a.columns.append(b.columns)
+
+ for col, s in b_re.items():
+ a_re[col] = s
+ return a_re.reindex(columns=result_columns)
+
+
+def test_join_inner_multiindex_deterministic_order():
+ # GH: 36910
+ left = DataFrame(
+ data={"e": 5},
+ index=MultiIndex.from_tuples([(1, 2, 4)], names=("a", "b", "d")),
+ )
+ right = DataFrame(
+ data={"f": 6}, index=MultiIndex.from_tuples([(2, 3)], names=("b", "c"))
+ )
+ result = left.join(right, how="inner")
+ expected = DataFrame(
+ {"e": [5], "f": [6]},
+ index=MultiIndex.from_tuples([(1, 2, 4, 3)], names=("a", "b", "d", "c")),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ ("input_col", "output_cols"), [("b", ["a", "b"]), ("a", ["a_x", "a_y"])]
+)
+def test_join_cross(input_col, output_cols):
+ # GH#5401
+ left = DataFrame({"a": [1, 3]})
+ right = DataFrame({input_col: [3, 4]})
+ result = left.join(right, how="cross", lsuffix="_x", rsuffix="_y")
+ expected = DataFrame({output_cols[0]: [1, 1, 3, 3], output_cols[1]: [3, 4, 3, 4]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_join_multiindex_one_level(join_type):
+ # GH#36909
+ left = DataFrame(
+ data={"c": 3}, index=MultiIndex.from_tuples([(1, 2)], names=("a", "b"))
+ )
+ right = DataFrame(data={"d": 4}, index=MultiIndex.from_tuples([(2,)], names=("b",)))
+ result = left.join(right, how=join_type)
+ if join_type == "right":
+ expected = DataFrame(
+ {"c": [3], "d": [4]},
+ index=MultiIndex.from_tuples([(2, 1)], names=["b", "a"]),
+ )
+ else:
+ expected = DataFrame(
+ {"c": [3], "d": [4]},
+ index=MultiIndex.from_tuples([(1, 2)], names=["a", "b"]),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "categories, values",
+ [
+ (["Y", "X"], ["Y", "X", "X"]),
+ ([2, 1], [2, 1, 1]),
+ ([2.5, 1.5], [2.5, 1.5, 1.5]),
+ (
+ [Timestamp("2020-12-31"), Timestamp("2019-12-31")],
+ [Timestamp("2020-12-31"), Timestamp("2019-12-31"), Timestamp("2019-12-31")],
+ ),
+ ],
+)
+def test_join_multiindex_not_alphabetical_categorical(categories, values):
+ # GH#38502
+ left = DataFrame(
+ {
+ "first": ["A", "A"],
+ "second": Categorical(categories, categories=categories),
+ "value": [1, 2],
+ }
+ ).set_index(["first", "second"])
+ right = DataFrame(
+ {
+ "first": ["A", "A", "B"],
+ "second": Categorical(values, categories=categories),
+ "value": [3, 4, 5],
+ }
+ ).set_index(["first", "second"])
+ result = left.join(right, lsuffix="_left", rsuffix="_right")
+
+ expected = DataFrame(
+ {
+ "first": ["A", "A"],
+ "second": Categorical(categories, categories=categories),
+ "value_left": [1, 2],
+ "value_right": [3, 4],
+ }
+ ).set_index(["first", "second"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "left_empty, how, exp",
+ [
+ (False, "left", "left"),
+ (False, "right", "empty"),
+ (False, "inner", "empty"),
+ (False, "outer", "left"),
+ (False, "cross", "empty"),
+ (True, "left", "empty"),
+ (True, "right", "right"),
+ (True, "inner", "empty"),
+ (True, "outer", "right"),
+ (True, "cross", "empty"),
+ ],
+)
+def test_join_empty(left_empty, how, exp):
+ left = DataFrame({"A": [2, 1], "B": [3, 4]}, dtype="int64").set_index("A")
+ right = DataFrame({"A": [1], "C": [5]}, dtype="int64").set_index("A")
+
+ if left_empty:
+ left = left.head(0)
+ else:
+ right = right.head(0)
+
+ result = left.join(right, how=how)
+
+ if exp == "left":
+ expected = DataFrame({"A": [2, 1], "B": [3, 4], "C": [np.nan, np.nan]})
+ expected = expected.set_index("A")
+ elif exp == "right":
+ expected = DataFrame({"B": [np.nan], "A": [1], "C": [5]})
+ expected = expected.set_index("A")
+ elif exp == "empty":
+ expected = DataFrame(columns=["B", "C"], dtype="int64")
+ if how != "cross":
+ expected = expected.rename_axis("A")
+ if how == "outer":
+ expected = expected.sort_index()
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_join_empty_uncomparable_columns():
+ # GH 57048
+ df1 = DataFrame()
+ df2 = DataFrame(columns=["test"])
+ df3 = DataFrame(columns=["foo", ("bar", "baz")])
+
+ result = df1 + df2
+ expected = DataFrame(columns=["test"])
+ tm.assert_frame_equal(result, expected)
+
+ result = df2 + df3
+ expected = DataFrame(columns=[("bar", "baz"), "foo", "test"])
+ tm.assert_frame_equal(result, expected)
+
+ result = df1 + df3
+ expected = DataFrame(columns=[("bar", "baz"), "foo"])
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "how, values",
+ [
+ ("inner", [0, 1, 2]),
+ ("outer", [0, 1, 2]),
+ ("left", [0, 1, 2]),
+ ("right", [0, 2, 1]),
+ ],
+)
+def test_join_multiindex_categorical_output_index_dtype(how, values):
+ # GH#50906
+ df1 = DataFrame(
+ {
+ "a": Categorical([0, 1, 2]),
+ "b": Categorical([0, 1, 2]),
+ "c": [0, 1, 2],
+ }
+ ).set_index(["a", "b"])
+
+ df2 = DataFrame(
+ {
+ "a": Categorical([0, 2, 1]),
+ "b": Categorical([0, 2, 1]),
+ "d": [0, 2, 1],
+ }
+ ).set_index(["a", "b"])
+
+ expected = DataFrame(
+ {
+ "a": Categorical(values),
+ "b": Categorical(values),
+ "c": values,
+ "d": values,
+ }
+ ).set_index(["a", "b"])
+
+ result = df1.join(df2, how=how)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_join_multiindex_with_none_as_label():
+ # GH 58721
+ df1 = DataFrame(
+ {"A": [1]},
+ index=MultiIndex.from_tuples([(3, 3)], names=["X", None]),
+ )
+ df2 = DataFrame(
+ {"B": [2]},
+ index=MultiIndex.from_tuples([(3, 3)], names=[None, "X"]),
+ )
+
+ result12 = df1.join(df2)
+ expected12 = DataFrame(
+ {"A": [1], "B": [2]},
+ index=MultiIndex.from_tuples([(3, 3)], names=["X", None]),
+ )
+ tm.assert_frame_equal(result12, expected12)
+
+ result21 = df2.join(df1)
+ expected21 = DataFrame(
+ {"B": [2], "A": [1]},
+ index=MultiIndex.from_tuples([(3, 3)], names=[None, "X"]),
+ )
+ tm.assert_frame_equal(result21, expected21)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge.py
new file mode 100644
index 0000000000000000000000000000000000000000..717645a3bcb8cbd57e5cf5ed0c3c5a55d802ecd9
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge.py
@@ -0,0 +1,3229 @@
+from datetime import (
+ date,
+ datetime,
+ timedelta,
+)
+import re
+
+import numpy as np
+import pytest
+
+from pandas.compat import PY314
+
+from pandas.core.dtypes.common import (
+ is_object_dtype,
+ is_string_dtype,
+)
+from pandas.core.dtypes.dtypes import CategoricalDtype
+
+import pandas as pd
+from pandas import (
+ Categorical,
+ CategoricalIndex,
+ DataFrame,
+ DatetimeIndex,
+ Index,
+ IntervalIndex,
+ MultiIndex,
+ PeriodIndex,
+ RangeIndex,
+ Series,
+ TimedeltaIndex,
+)
+import pandas._testing as tm
+from pandas.core.reshape.concat import concat
+from pandas.core.reshape.merge import (
+ MergeError,
+ merge,
+)
+
+
+def get_test_data(ngroups=8, n=50):
+ unique_groups = list(range(ngroups))
+ arr = np.asarray(np.tile(unique_groups, n // ngroups))
+
+ if len(arr) < n:
+ arr = np.asarray(list(arr) + unique_groups[: n - len(arr)])
+
+ np.random.default_rng(2).shuffle(arr)
+ return arr
+
+
+@pytest.fixture
+def dfs_for_indicator():
+ df1 = DataFrame({"col1": [0, 1], "col_conflict": [1, 2], "col_left": ["a", "b"]})
+ df2 = DataFrame(
+ {
+ "col1": [1, 2, 3, 4, 5],
+ "col_conflict": [1, 2, 3, 4, 5],
+ "col_right": [2, 2, 2, 2, 2],
+ }
+ )
+ return df1, df2
+
+
+class TestMerge:
+ @pytest.fixture
+ def df(self):
+ df = DataFrame(
+ {
+ "key1": get_test_data(),
+ "key2": get_test_data(),
+ "data1": np.random.default_rng(2).standard_normal(50),
+ "data2": np.random.default_rng(2).standard_normal(50),
+ }
+ )
+
+ # exclude a couple keys for fun
+ df = df[df["key2"] > 1]
+ return df
+
+ @pytest.fixture
+ def df2(self):
+ return DataFrame(
+ {
+ "key1": get_test_data(n=10),
+ "key2": get_test_data(ngroups=4, n=10),
+ "value": np.random.default_rng(2).standard_normal(10),
+ }
+ )
+
+ @pytest.fixture
+ def left(self):
+ return DataFrame(
+ {
+ "key": ["a", "b", "c", "d", "e", "e", "a"],
+ "v1": np.random.default_rng(2).standard_normal(7),
+ }
+ )
+
+ def test_merge_inner_join_empty(self):
+ # GH 15328
+ df_empty = DataFrame()
+ df_a = DataFrame({"a": [1, 2]}, index=[0, 1], dtype="int64")
+ result = merge(df_empty, df_a, left_index=True, right_index=True)
+ expected = DataFrame({"a": []}, dtype="int64")
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_common(self, df, df2):
+ joined = merge(df, df2)
+ exp = merge(df, df2, on=["key1", "key2"])
+ tm.assert_frame_equal(joined, exp)
+
+ def test_merge_non_string_columns(self):
+ # https://github.com/pandas-dev/pandas/issues/17962
+ # Checks that method runs for non string column names
+ left = DataFrame(
+ {0: [1, 0, 1, 0], 1: [0, 1, 0, 0], 2: [0, 0, 2, 0], 3: [1, 0, 0, 3]}
+ )
+
+ right = left.astype(float)
+ expected = left
+ result = merge(left, right)
+ tm.assert_frame_equal(expected, result)
+
+ def test_merge_index_as_on_arg(self, df, df2):
+ # GH14355
+
+ left = df.set_index("key1")
+ right = df2.set_index("key1")
+ result = merge(left, right, on="key1")
+ expected = merge(df, df2, on="key1").set_index("key1")
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_index_singlekey_right_vs_left(self):
+ left = DataFrame(
+ {
+ "key": ["a", "b", "c", "d", "e", "e", "a"],
+ "v1": np.random.default_rng(2).standard_normal(7),
+ }
+ )
+ right = DataFrame(
+ {"v2": np.random.default_rng(2).standard_normal(4)},
+ index=["d", "b", "c", "a"],
+ )
+
+ merged1 = merge(
+ left, right, left_on="key", right_index=True, how="left", sort=False
+ )
+ merged2 = merge(
+ right, left, right_on="key", left_index=True, how="right", sort=False
+ )
+ tm.assert_frame_equal(merged1, merged2.loc[:, merged1.columns])
+
+ merged1 = merge(
+ left, right, left_on="key", right_index=True, how="left", sort=True
+ )
+ merged2 = merge(
+ right, left, right_on="key", left_index=True, how="right", sort=True
+ )
+ tm.assert_frame_equal(merged1, merged2.loc[:, merged1.columns])
+
+ def test_merge_index_singlekey_inner(self):
+ left = DataFrame(
+ {
+ "key": ["a", "b", "c", "d", "e", "e", "a"],
+ "v1": np.random.default_rng(2).standard_normal(7),
+ }
+ )
+ right = DataFrame(
+ {"v2": np.random.default_rng(2).standard_normal(4)},
+ index=["d", "b", "c", "a"],
+ )
+
+ # inner join
+ result = merge(left, right, left_on="key", right_index=True, how="inner")
+ expected = left.join(right, on="key").loc[result.index]
+ tm.assert_frame_equal(result, expected)
+
+ result = merge(right, left, right_on="key", left_index=True, how="inner")
+ expected = left.join(right, on="key").loc[result.index]
+ tm.assert_frame_equal(result, expected.loc[:, result.columns])
+
+ def test_merge_misspecified(self, df, df2, left):
+ right = DataFrame(
+ {"v2": np.random.default_rng(2).standard_normal(4)},
+ index=["d", "b", "c", "a"],
+ )
+ msg = "Must pass right_on or right_index=True"
+ with pytest.raises(pd.errors.MergeError, match=msg):
+ merge(left, right, left_index=True)
+ msg = "Must pass left_on or left_index=True"
+ with pytest.raises(pd.errors.MergeError, match=msg):
+ merge(left, right, right_index=True)
+
+ msg = (
+ 'Can only pass argument "on" OR "left_on" and "right_on", not '
+ "a combination of both"
+ )
+ with pytest.raises(pd.errors.MergeError, match=msg):
+ merge(left, left, left_on="key", on="key")
+
+ msg = r"len\(right_on\) must equal len\(left_on\)"
+ with pytest.raises(ValueError, match=msg):
+ merge(df, df2, left_on=["key1"], right_on=["key1", "key2"])
+
+ def test_index_and_on_parameters_confusion(self, df, df2):
+ msg = "right_index parameter must be of type bool, not "
+ with pytest.raises(ValueError, match=msg):
+ merge(
+ df,
+ df2,
+ how="left",
+ left_index=False,
+ right_index=["key1", "key2"],
+ )
+ msg = "left_index parameter must be of type bool, not "
+ with pytest.raises(ValueError, match=msg):
+ merge(
+ df,
+ df2,
+ how="left",
+ left_index=["key1", "key2"],
+ right_index=False,
+ )
+ with pytest.raises(ValueError, match=msg):
+ merge(
+ df,
+ df2,
+ how="left",
+ left_index=["key1", "key2"],
+ right_index=["key1", "key2"],
+ )
+
+ def test_merge_overlap(self, left):
+ merged = merge(left, left, on="key")
+ exp_len = (left["key"].value_counts() ** 2).sum()
+ assert len(merged) == exp_len
+ assert "v1_x" in merged
+ assert "v1_y" in merged
+
+ def test_merge_different_column_key_names(self):
+ left = DataFrame({"lkey": ["foo", "bar", "baz", "foo"], "value": [1, 2, 3, 4]})
+ right = DataFrame({"rkey": ["foo", "bar", "qux", "foo"], "value": [5, 6, 7, 8]})
+
+ merged = left.merge(
+ right, left_on="lkey", right_on="rkey", how="outer", sort=True
+ )
+
+ exp = Series(["bar", "baz", "foo", "foo", "foo", "foo", np.nan], name="lkey")
+ tm.assert_series_equal(merged["lkey"], exp)
+
+ exp = Series(["bar", np.nan, "foo", "foo", "foo", "foo", "qux"], name="rkey")
+ tm.assert_series_equal(merged["rkey"], exp)
+
+ exp = Series([2, 3, 1, 1, 4, 4, np.nan], name="value_x")
+ tm.assert_series_equal(merged["value_x"], exp)
+
+ exp = Series([6, np.nan, 5, 8, 5, 8, 7], name="value_y")
+ tm.assert_series_equal(merged["value_y"], exp)
+
+ def test_merge_copy(self):
+ left = DataFrame({"a": 0, "b": 1}, index=range(10))
+ right = DataFrame({"c": "foo", "d": "bar"}, index=range(10))
+
+ merged = merge(left, right, left_index=True, right_index=True)
+
+ merged["a"] = 6
+ assert (left["a"] == 0).all()
+
+ merged["d"] = "peekaboo"
+ assert (right["d"] == "bar").all()
+
+ def test_merge_nocopy(self, using_infer_string):
+ left = DataFrame({"a": 0, "b": 1}, index=range(10))
+ right = DataFrame({"c": "foo", "d": "bar"}, index=range(10))
+
+ merged = merge(left, right, left_index=True, right_index=True)
+
+ assert np.shares_memory(merged["a"]._values, left["a"]._values)
+ if not using_infer_string:
+ assert np.shares_memory(merged["d"]._values, right["d"]._values)
+
+ def test_intelligently_handle_join_key(self):
+ # #733, be a bit more 1337 about not returning unconsolidated DataFrame
+
+ left = DataFrame(
+ {"key": [1, 1, 2, 2, 3], "value": list(range(5))}, columns=["value", "key"]
+ )
+ right = DataFrame({"key": [1, 1, 2, 3, 4, 5], "rvalue": list(range(6))})
+
+ joined = merge(left, right, on="key", how="outer")
+ expected = DataFrame(
+ {
+ "key": [1, 1, 1, 1, 2, 2, 3, 4, 5],
+ "value": np.array([0, 0, 1, 1, 2, 3, 4, np.nan, np.nan]),
+ "rvalue": [0, 1, 0, 1, 2, 2, 3, 4, 5],
+ },
+ columns=["value", "key", "rvalue"],
+ )
+ tm.assert_frame_equal(joined, expected)
+
+ def test_merge_join_key_dtype_cast(self):
+ # #8596
+
+ df1 = DataFrame({"key": [1], "v1": [10]})
+ df2 = DataFrame({"key": [2], "v1": [20]})
+ df = merge(df1, df2, how="outer")
+ assert df["key"].dtype == "int64"
+
+ df1 = DataFrame({"key": [True], "v1": [1]})
+ df2 = DataFrame({"key": [False], "v1": [0]})
+ df = merge(df1, df2, how="outer")
+
+ # GH13169
+ # GH#40073
+ assert df["key"].dtype == "bool"
+
+ df1 = DataFrame({"val": [1]})
+ df2 = DataFrame({"val": [2]})
+ lkey = np.array([1])
+ rkey = np.array([2])
+ df = merge(df1, df2, left_on=lkey, right_on=rkey, how="outer")
+ assert df["key_0"].dtype == np.dtype(int)
+
+ def test_handle_join_key_pass_array(self):
+ left = DataFrame(
+ {"key": [1, 1, 2, 2, 3], "value": np.arange(5)},
+ columns=["value", "key"],
+ dtype="int64",
+ )
+ right = DataFrame({"rvalue": np.arange(6)}, dtype="int64")
+ key = np.array([1, 1, 2, 3, 4, 5], dtype="int64")
+
+ merged = merge(left, right, left_on="key", right_on=key, how="outer")
+ merged2 = merge(right, left, left_on=key, right_on="key", how="outer")
+
+ tm.assert_series_equal(merged["key"], merged2["key"])
+ assert merged["key"].notna().all()
+ assert merged2["key"].notna().all()
+
+ left = DataFrame({"value": np.arange(5)}, columns=["value"])
+ right = DataFrame({"rvalue": np.arange(6)})
+ lkey = np.array([1, 1, 2, 2, 3])
+ rkey = np.array([1, 1, 2, 3, 4, 5])
+
+ merged = merge(left, right, left_on=lkey, right_on=rkey, how="outer")
+ expected = Series([1, 1, 1, 1, 2, 2, 3, 4, 5], dtype=int, name="key_0")
+ tm.assert_series_equal(merged["key_0"], expected)
+
+ left = DataFrame({"value": np.arange(3)})
+ right = DataFrame({"rvalue": np.arange(6)})
+
+ key = np.array([0, 1, 1, 2, 2, 3], dtype=np.int64)
+ merged = merge(left, right, left_index=True, right_on=key, how="outer")
+ tm.assert_series_equal(merged["key_0"], Series(key, name="key_0"))
+
+ def test_no_overlap_more_informative_error(self):
+ dt = datetime.now()
+ df1 = DataFrame({"x": ["a"]}, index=[dt])
+
+ df2 = DataFrame({"y": ["b", "c"]}, index=[dt, dt])
+
+ msg = (
+ "No common columns to perform merge on. "
+ f"Merge options: left_on={None}, right_on={None}, "
+ f"left_index={False}, right_index={False}"
+ )
+
+ with pytest.raises(MergeError, match=msg):
+ merge(df1, df2)
+
+ def test_merge_non_unique_indexes(self):
+ dt = datetime(2012, 5, 1)
+ dt2 = datetime(2012, 5, 2)
+ dt3 = datetime(2012, 5, 3)
+ dt4 = datetime(2012, 5, 4)
+
+ df1 = DataFrame({"x": ["a"]}, index=[dt])
+ df2 = DataFrame({"y": ["b", "c"]}, index=[dt, dt])
+ _check_merge(df1, df2)
+
+ # Not monotonic
+ df1 = DataFrame({"x": ["a", "b", "q"]}, index=[dt2, dt, dt4])
+ df2 = DataFrame(
+ {"y": ["c", "d", "e", "f", "g", "h"]}, index=[dt3, dt3, dt2, dt2, dt, dt]
+ )
+ _check_merge(df1, df2)
+
+ df1 = DataFrame({"x": ["a", "b"]}, index=[dt, dt])
+ df2 = DataFrame({"y": ["c", "d"]}, index=[dt, dt])
+ _check_merge(df1, df2)
+
+ def test_merge_non_unique_index_many_to_many(self):
+ dt = datetime(2012, 5, 1)
+ dt2 = datetime(2012, 5, 2)
+ dt3 = datetime(2012, 5, 3)
+ df1 = DataFrame({"x": ["a", "b", "c", "d"]}, index=[dt2, dt2, dt, dt])
+ df2 = DataFrame(
+ {"y": ["e", "f", "g", " h", "i"]}, index=[dt2, dt2, dt3, dt, dt]
+ )
+ _check_merge(df1, df2)
+
+ def test_left_merge_empty_dataframe(self):
+ left = DataFrame({"key": [1], "value": [2]})
+ right = DataFrame({"key": []})
+
+ result = merge(left, right, on="key", how="left")
+ tm.assert_frame_equal(result, left)
+
+ result = merge(right, left, on="key", how="right")
+ tm.assert_frame_equal(result, left)
+
+ def test_merge_empty_dataframe(self, index, join_type):
+ # GH52777
+ left = DataFrame([], index=index[:0])
+ right = left.copy()
+
+ result = left.join(right, how=join_type)
+ tm.assert_frame_equal(result, left)
+
+ @pytest.mark.parametrize(
+ "kwarg",
+ [
+ {"left_index": True, "right_index": True},
+ {"left_index": True, "right_on": "x"},
+ {"left_on": "a", "right_index": True},
+ {"left_on": "a", "right_on": "x"},
+ ],
+ )
+ def test_merge_left_empty_right_empty(self, join_type, kwarg):
+ # GH 10824
+ left = DataFrame(columns=["a", "b", "c"])
+ right = DataFrame(columns=["x", "y", "z"])
+
+ exp_in = DataFrame(columns=["a", "b", "c", "x", "y", "z"], dtype=object)
+
+ result = merge(left, right, how=join_type, **kwarg)
+ tm.assert_frame_equal(result, exp_in)
+
+ def test_merge_left_empty_right_notempty(self):
+ # GH 10824
+ left = DataFrame(columns=["a", "b", "c"])
+ right = DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]], columns=["x", "y", "z"])
+
+ exp_out = DataFrame(
+ {
+ "a": np.array([np.nan] * 3, dtype=object),
+ "b": np.array([np.nan] * 3, dtype=object),
+ "c": np.array([np.nan] * 3, dtype=object),
+ "x": [1, 4, 7],
+ "y": [2, 5, 8],
+ "z": [3, 6, 9],
+ },
+ columns=["a", "b", "c", "x", "y", "z"],
+ )
+ exp_in = exp_out[0:0] # make empty DataFrame keeping dtype
+
+ def check1(exp, kwarg):
+ result = merge(left, right, how="inner", **kwarg)
+ tm.assert_frame_equal(result, exp)
+ result = merge(left, right, how="left", **kwarg)
+ tm.assert_frame_equal(result, exp)
+
+ def check2(exp, kwarg):
+ result = merge(left, right, how="right", **kwarg)
+ tm.assert_frame_equal(result, exp)
+ result = merge(left, right, how="outer", **kwarg)
+ tm.assert_frame_equal(result, exp)
+
+ for kwarg in [
+ {"left_index": True, "right_index": True},
+ {"left_index": True, "right_on": "x"},
+ ]:
+ check1(exp_in, kwarg)
+ check2(exp_out, kwarg)
+
+ kwarg = {"left_on": "a", "right_index": True}
+ check1(exp_in, kwarg)
+ exp_out["a"] = [0, 1, 2]
+ check2(exp_out, kwarg)
+
+ kwarg = {"left_on": "a", "right_on": "x"}
+ check1(exp_in, kwarg)
+ exp_out["a"] = np.array([np.nan] * 3, dtype=object)
+ check2(exp_out, kwarg)
+
+ def test_merge_left_notempty_right_empty(self):
+ # GH 10824
+ left = DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]], columns=["a", "b", "c"])
+ right = DataFrame(columns=["x", "y", "z"])
+
+ exp_out = DataFrame(
+ {
+ "a": [1, 4, 7],
+ "b": [2, 5, 8],
+ "c": [3, 6, 9],
+ "x": np.array([np.nan] * 3, dtype=object),
+ "y": np.array([np.nan] * 3, dtype=object),
+ "z": np.array([np.nan] * 3, dtype=object),
+ },
+ columns=["a", "b", "c", "x", "y", "z"],
+ )
+ exp_in = exp_out[0:0] # make empty DataFrame keeping dtype
+ # result will have object dtype
+ exp_in.index = exp_in.index.astype(object)
+
+ def check1(exp, kwarg):
+ result = merge(left, right, how="inner", **kwarg)
+ tm.assert_frame_equal(result, exp)
+ result = merge(left, right, how="right", **kwarg)
+ tm.assert_frame_equal(result, exp)
+
+ def check2(exp, kwarg):
+ result = merge(left, right, how="left", **kwarg)
+ tm.assert_frame_equal(result, exp)
+ result = merge(left, right, how="outer", **kwarg)
+ tm.assert_frame_equal(result, exp)
+
+ # TODO: should the next loop be un-indented? doing so breaks this test
+ for kwarg in [
+ {"left_index": True, "right_index": True},
+ {"left_index": True, "right_on": "x"},
+ {"left_on": "a", "right_index": True},
+ {"left_on": "a", "right_on": "x"},
+ ]:
+ check1(exp_in, kwarg)
+ check2(exp_out, kwarg)
+
+ @pytest.mark.parametrize(
+ "series_of_dtype",
+ [
+ Series([1], dtype="int64"),
+ Series([1], dtype="Int64"),
+ Series([1.23]),
+ Series(["foo"]),
+ Series([True]),
+ Series([pd.Timestamp("2018-01-01")]),
+ Series([pd.Timestamp("2018-01-01", tz="US/Eastern")]),
+ ],
+ )
+ @pytest.mark.parametrize(
+ "series_of_dtype2",
+ [
+ Series([1], dtype="int64"),
+ Series([1], dtype="Int64"),
+ Series([1.23]),
+ Series(["foo"]),
+ Series([True]),
+ Series([pd.Timestamp("2018-01-01")]),
+ Series([pd.Timestamp("2018-01-01", tz="US/Eastern")]),
+ ],
+ )
+ def test_merge_empty_frame(self, series_of_dtype, series_of_dtype2):
+ # GH 25183
+ df = DataFrame(
+ {"key": series_of_dtype, "value": series_of_dtype2},
+ columns=["key", "value"],
+ )
+ df_empty = df[:0]
+ expected = DataFrame(
+ {
+ "key": Series(dtype=df.dtypes["key"]),
+ "value_x": Series(dtype=df.dtypes["value"]),
+ "value_y": Series(dtype=df.dtypes["value"]),
+ },
+ columns=["key", "value_x", "value_y"],
+ )
+ actual = df_empty.merge(df, on="key")
+ tm.assert_frame_equal(actual, expected)
+
+ @pytest.mark.parametrize(
+ "series_of_dtype",
+ [
+ Series([1], dtype="int64"),
+ Series([1], dtype="Int64"),
+ Series([1.23]),
+ Series(["foo"]),
+ Series([True]),
+ Series([pd.Timestamp("2018-01-01")]),
+ Series([pd.Timestamp("2018-01-01", tz="US/Eastern")]),
+ ],
+ )
+ @pytest.mark.parametrize(
+ "series_of_dtype_all_na",
+ [
+ Series([pd.NA], dtype="Int64"),
+ Series([np.nan], dtype="float"),
+ Series([np.nan], dtype="object"),
+ Series([pd.NaT]),
+ ],
+ )
+ def test_merge_all_na_column(self, series_of_dtype, series_of_dtype_all_na):
+ # GH 25183
+ df_left = DataFrame(
+ {"key": series_of_dtype, "value": series_of_dtype_all_na},
+ columns=["key", "value"],
+ )
+ df_right = DataFrame(
+ {"key": series_of_dtype, "value": series_of_dtype_all_na},
+ columns=["key", "value"],
+ )
+ expected = DataFrame(
+ {
+ "key": series_of_dtype,
+ "value_x": series_of_dtype_all_na,
+ "value_y": series_of_dtype_all_na,
+ },
+ columns=["key", "value_x", "value_y"],
+ )
+ actual = df_left.merge(df_right, on="key")
+ tm.assert_frame_equal(actual, expected)
+
+ def test_merge_nosort(self):
+ # GH#2098
+
+ d = {
+ "var1": np.random.default_rng(2).integers(0, 10, size=10),
+ "var2": np.random.default_rng(2).integers(0, 10, size=10),
+ "var3": [
+ datetime(2012, 1, 12),
+ datetime(2011, 2, 4),
+ datetime(2010, 2, 3),
+ datetime(2012, 1, 12),
+ datetime(2011, 2, 4),
+ datetime(2012, 4, 3),
+ datetime(2012, 3, 4),
+ datetime(2008, 5, 1),
+ datetime(2010, 2, 3),
+ datetime(2012, 2, 3),
+ ],
+ }
+ df = DataFrame.from_dict(d)
+ var3 = df.var3.unique()
+ var3 = np.sort(var3)
+ new = DataFrame.from_dict(
+ {"var3": var3, "var8": np.random.default_rng(2).random(7)}
+ )
+
+ result = df.merge(new, on="var3", sort=False)
+ exp = merge(df, new, on="var3", sort=False)
+ tm.assert_frame_equal(result, exp)
+
+ assert (df.var3.unique() == result.var3.unique()).all()
+
+ @pytest.mark.parametrize(
+ ("sort", "values"), [(False, [1, 1, 0, 1, 1]), (True, [0, 1, 1, 1, 1])]
+ )
+ @pytest.mark.parametrize("how", ["left", "right"])
+ def test_merge_same_order_left_right(self, sort, values, how):
+ # GH#35382
+ df = DataFrame({"a": [1, 0, 1]})
+
+ result = df.merge(df, on="a", how=how, sort=sort)
+ expected = DataFrame(values, columns=["a"])
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_nan_right(self):
+ df1 = DataFrame({"i1": [0, 1], "i2": [0, 1]})
+ df2 = DataFrame({"i1": [0], "i3": [0]})
+ result = df1.join(df2, on="i1", rsuffix="_")
+ expected = (
+ DataFrame(
+ {
+ "i1": {0: 0.0, 1: 1},
+ "i2": {0: 0, 1: 1},
+ "i1_": {0: 0, 1: np.nan},
+ "i3": {0: 0.0, 1: np.nan},
+ None: {0: 0, 1: 0},
+ },
+ columns=Index(["i1", "i2", "i1_", "i3", None], dtype=object),
+ )
+ .set_index(None)
+ .reset_index()[["i1", "i2", "i1_", "i3"]]
+ )
+ result.columns = result.columns.astype("object")
+ tm.assert_frame_equal(result, expected, check_dtype=False)
+
+ def test_merge_nan_right2(self):
+ df1 = DataFrame({"i1": [0, 1], "i2": [0.5, 1.5]})
+ df2 = DataFrame({"i1": [0], "i3": [0.7]})
+ result = df1.join(df2, rsuffix="_", on="i1")
+ expected = DataFrame(
+ {
+ "i1": {0: 0, 1: 1},
+ "i1_": {0: 0.0, 1: np.nan},
+ "i2": {0: 0.5, 1: 1.5},
+ "i3": {0: 0.69999999999999996, 1: np.nan},
+ }
+ )[["i1", "i2", "i1_", "i3"]]
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager|Passing a SingleBlockManager:DeprecationWarning"
+ )
+ def test_merge_type(self, df, df2):
+ class NotADataFrame(DataFrame):
+ @property
+ def _constructor(self):
+ return NotADataFrame
+
+ nad = NotADataFrame(df)
+ result = nad.merge(df2, on="key1")
+
+ assert isinstance(result, NotADataFrame)
+
+ def test_join_append_timedeltas(self):
+ # timedelta64 issues with join/merge
+ # GH 5695
+
+ d = DataFrame.from_dict(
+ {"d": [datetime(2013, 11, 5, 5, 56)], "t": [timedelta(0, 22500)]}
+ )
+ df = DataFrame(columns=list("dt"))
+ df = concat([df, d], ignore_index=True)
+ result = concat([df, d], ignore_index=True)
+ expected = DataFrame(
+ {
+ "d": [datetime(2013, 11, 5, 5, 56), datetime(2013, 11, 5, 5, 56)],
+ "t": [timedelta(0, 22500), timedelta(0, 22500)],
+ },
+ dtype=object,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_join_append_timedeltas2(self):
+ # timedelta64 issues with join/merge
+ # GH 5695
+ td = np.timedelta64(300000000, "ns")
+ lhs = DataFrame(Series([td, td], index=["A", "B"]))
+ rhs = DataFrame(Series([td], index=["A"]))
+
+ result = lhs.join(rhs, rsuffix="r", how="left")
+ expected = DataFrame(
+ {
+ "0": Series([td, td], index=list("AB")),
+ "0r": Series([td, pd.NaT], index=list("AB")),
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("unit", ["D", "h", "m", "s", "ms", "us", "ns"])
+ def test_other_datetime_unit(self, unit):
+ # GH 13389
+ df1 = DataFrame({"entity_id": [101, 102]})
+ ser = Series([None, None], index=[101, 102], name="days")
+
+ dtype = f"datetime64[{unit}]"
+
+ if unit in ["D", "h", "m"]:
+ # not supported so we cast to the nearest supported unit, seconds
+ exp_dtype = "datetime64[s]"
+ else:
+ exp_dtype = dtype
+ df2 = ser.astype(exp_dtype).to_frame("days")
+ assert df2["days"].dtype == exp_dtype
+
+ result = df1.merge(df2, left_on="entity_id", right_index=True)
+
+ days = np.array(["nat", "nat"], dtype=exp_dtype)
+ days = pd.core.arrays.DatetimeArray._simple_new(days, dtype=days.dtype)
+ exp = DataFrame(
+ {
+ "entity_id": [101, 102],
+ "days": days,
+ },
+ columns=["entity_id", "days"],
+ )
+ assert exp["days"].dtype == exp_dtype
+ tm.assert_frame_equal(result, exp)
+
+ @pytest.mark.parametrize("unit", ["D", "h", "m", "s", "ms", "us", "ns"])
+ def test_other_timedelta_unit(self, unit):
+ # GH 13389
+ df1 = DataFrame({"entity_id": [101, 102]})
+ ser = Series([None, None], index=[101, 102], name="days")
+
+ dtype = f"m8[{unit}]"
+ if unit in ["D", "h", "m"]:
+ # We cannot astype, instead do nearest supported unit, i.e. "s"
+ msg = "Supported resolutions are 's', 'ms', 'us', 'ns'"
+ with pytest.raises(ValueError, match=msg):
+ ser.astype(dtype)
+
+ df2 = ser.astype("m8[s]").to_frame("days")
+ else:
+ df2 = ser.astype(dtype).to_frame("days")
+ assert df2["days"].dtype == dtype
+
+ result = df1.merge(df2, left_on="entity_id", right_index=True)
+
+ exp = DataFrame(
+ {"entity_id": [101, 102], "days": np.array(["nat", "nat"], dtype=dtype)},
+ columns=["entity_id", "days"],
+ )
+ tm.assert_frame_equal(result, exp)
+
+ def test_overlapping_columns_error_message(self):
+ df = DataFrame({"key": [1, 2, 3], "v1": [4, 5, 6], "v2": [7, 8, 9]})
+ df2 = DataFrame({"key": [1, 2, 3], "v1": [4, 5, 6], "v2": [7, 8, 9]})
+
+ df.columns = ["key", "foo", "foo"]
+ df2.columns = ["key", "bar", "bar"]
+ expected = DataFrame(
+ {
+ "key": [1, 2, 3],
+ "v1": [4, 5, 6],
+ "v2": [7, 8, 9],
+ "v3": [4, 5, 6],
+ "v4": [7, 8, 9],
+ }
+ )
+ expected.columns = ["key", "foo", "foo", "bar", "bar"]
+ tm.assert_frame_equal(merge(df, df2), expected)
+
+ # #2649, #10639
+ df2.columns = ["key1", "foo", "foo"]
+ msg = r"Data columns not unique: Index\(\['foo'\], dtype='object|str'\)"
+ with pytest.raises(MergeError, match=msg):
+ merge(df, df2)
+
+ def test_merge_on_datetime64tz(self):
+ # GH11405
+ left = DataFrame(
+ {
+ "key": pd.date_range("20151010", periods=2, tz="US/Eastern"),
+ "value": [1, 2],
+ }
+ )
+ right = DataFrame(
+ {
+ "key": pd.date_range("20151011", periods=3, tz="US/Eastern"),
+ "value": [1, 2, 3],
+ }
+ )
+
+ expected = DataFrame(
+ {
+ "key": pd.date_range("20151010", periods=4, tz="US/Eastern"),
+ "value_x": [1, 2, np.nan, np.nan],
+ "value_y": [np.nan, 1, 2, 3],
+ }
+ )
+ result = merge(left, right, on="key", how="outer")
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_datetime64tz_values(self):
+ left = DataFrame(
+ {
+ "key": [1, 2],
+ "value": pd.date_range(
+ "20151010", periods=2, tz="US/Eastern", unit="ns"
+ ),
+ }
+ )
+ right = DataFrame(
+ {
+ "key": [2, 3],
+ "value": pd.date_range(
+ "20151011", periods=2, tz="US/Eastern", unit="ns"
+ ),
+ }
+ )
+ expected = DataFrame(
+ {
+ "key": [1, 2, 3],
+ "value_x": [
+ *list(
+ pd.date_range("20151010", periods=2, tz="US/Eastern", unit="ns")
+ ),
+ pd.NaT,
+ ],
+ "value_y": [
+ pd.NaT,
+ *list(
+ pd.date_range("20151011", periods=2, tz="US/Eastern", unit="ns")
+ ),
+ ],
+ }
+ )
+ result = merge(left, right, on="key", how="outer")
+ tm.assert_frame_equal(result, expected)
+ assert result["value_x"].dtype == "datetime64[ns, US/Eastern]"
+ assert result["value_y"].dtype == "datetime64[ns, US/Eastern]"
+
+ def test_merge_on_datetime64tz_empty(self):
+ # https://github.com/pandas-dev/pandas/issues/25014
+ dtz = pd.DatetimeTZDtype(tz="UTC")
+ right = DataFrame(
+ {
+ "date": DatetimeIndex(["2018"], dtype=dtz),
+ "value": [4.0],
+ "date2": DatetimeIndex(["2019"], dtype=dtz),
+ },
+ columns=["date", "value", "date2"],
+ )
+ left = right[:0]
+ result = left.merge(right, on="date")
+ expected = DataFrame(
+ {
+ "date": Series(dtype=dtz),
+ "value_x": Series(dtype=float),
+ "date2_x": Series(dtype=dtz),
+ "value_y": Series(dtype=float),
+ "date2_y": Series(dtype=dtz),
+ },
+ columns=["date", "value_x", "date2_x", "value_y", "date2_y"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_datetime64tz_with_dst_transition(self):
+ # GH 18885
+ df1 = DataFrame(
+ pd.date_range("2017-10-29 01:00", periods=4, freq="h", tz="Europe/Madrid"),
+ columns=["date"],
+ )
+ df1["value"] = 1
+ df2 = DataFrame(
+ {
+ "date": pd.to_datetime(
+ [
+ "2017-10-29 03:00:00",
+ "2017-10-29 04:00:00",
+ "2017-10-29 05:00:00",
+ ]
+ ),
+ "value": 2,
+ }
+ )
+ df2["date"] = df2["date"].dt.tz_localize("UTC").dt.tz_convert("Europe/Madrid")
+ result = merge(df1, df2, how="outer", on="date")
+ expected = DataFrame(
+ {
+ "date": pd.date_range(
+ "2017-10-29 01:00", periods=7, freq="h", tz="Europe/Madrid"
+ ),
+ "value_x": [1] * 4 + [np.nan] * 3,
+ "value_y": [np.nan] * 4 + [2] * 3,
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_non_unique_period_index(self):
+ # GH #16871
+ index = pd.period_range("2016-01-01", periods=16, freq="M")
+ df = DataFrame(list(range(len(index))), index=index, columns=["pnum"])
+ df2 = concat([df, df])
+ result = df.merge(df2, left_index=True, right_index=True, how="inner")
+ expected = DataFrame(
+ np.tile(np.arange(16, dtype=np.int64).repeat(2).reshape(-1, 1), 2),
+ columns=["pnum_x", "pnum_y"],
+ index=df2.sort_index().index,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_on_periods(self):
+ left = DataFrame(
+ {"key": pd.period_range("20151010", periods=2, freq="D"), "value": [1, 2]}
+ )
+ right = DataFrame(
+ {
+ "key": pd.period_range("20151011", periods=3, freq="D"),
+ "value": [1, 2, 3],
+ }
+ )
+
+ expected = DataFrame(
+ {
+ "key": pd.period_range("20151010", periods=4, freq="D"),
+ "value_x": [1, 2, np.nan, np.nan],
+ "value_y": [np.nan, 1, 2, 3],
+ }
+ )
+ result = merge(left, right, on="key", how="outer")
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_period_values(self):
+ left = DataFrame(
+ {"key": [1, 2], "value": pd.period_range("20151010", periods=2, freq="D")}
+ )
+ right = DataFrame(
+ {"key": [2, 3], "value": pd.period_range("20151011", periods=2, freq="D")}
+ )
+
+ exp_x = pd.period_range("20151010", periods=2, freq="D")
+ exp_y = pd.period_range("20151011", periods=2, freq="D")
+ expected = DataFrame(
+ {
+ "key": [1, 2, 3],
+ "value_x": [*list(exp_x), pd.NaT],
+ "value_y": [pd.NaT, *list(exp_y)],
+ }
+ )
+ result = merge(left, right, on="key", how="outer")
+ tm.assert_frame_equal(result, expected)
+ assert result["value_x"].dtype == "Period[D]"
+ assert result["value_y"].dtype == "Period[D]"
+
+ def test_indicator(self, dfs_for_indicator):
+ # PR #10054. xref #7412 and closes #8790.
+ df1, df2 = dfs_for_indicator
+ df1_copy = df1.copy()
+
+ df2_copy = df2.copy()
+
+ df_result = DataFrame(
+ {
+ "col1": [0, 1, 2, 3, 4, 5],
+ "col_conflict_x": [1, 2, np.nan, np.nan, np.nan, np.nan],
+ "col_left": ["a", "b", np.nan, np.nan, np.nan, np.nan],
+ "col_conflict_y": [np.nan, 1, 2, 3, 4, 5],
+ "col_right": [np.nan, 2, 2, 2, 2, 2],
+ }
+ )
+ df_result["_merge"] = Categorical(
+ [
+ "left_only",
+ "both",
+ "right_only",
+ "right_only",
+ "right_only",
+ "right_only",
+ ],
+ categories=["left_only", "right_only", "both"],
+ )
+
+ df_result = df_result[
+ [
+ "col1",
+ "col_conflict_x",
+ "col_left",
+ "col_conflict_y",
+ "col_right",
+ "_merge",
+ ]
+ ]
+
+ test = merge(df1, df2, on="col1", how="outer", indicator=True)
+ tm.assert_frame_equal(test, df_result)
+ test = df1.merge(df2, on="col1", how="outer", indicator=True)
+ tm.assert_frame_equal(test, df_result)
+
+ # No side effects
+ tm.assert_frame_equal(df1, df1_copy)
+ tm.assert_frame_equal(df2, df2_copy)
+
+ # Check with custom name
+ df_result_custom_name = df_result
+ df_result_custom_name = df_result_custom_name.rename(
+ columns={"_merge": "custom_name"}
+ )
+
+ test_custom_name = merge(
+ df1, df2, on="col1", how="outer", indicator="custom_name"
+ )
+ tm.assert_frame_equal(test_custom_name, df_result_custom_name)
+ test_custom_name = df1.merge(
+ df2, on="col1", how="outer", indicator="custom_name"
+ )
+ tm.assert_frame_equal(test_custom_name, df_result_custom_name)
+
+ def test_merge_indicator_arg_validation(self, dfs_for_indicator):
+ # Check only accepts strings and booleans
+ df1, df2 = dfs_for_indicator
+
+ msg = "indicator option can only accept boolean or string arguments"
+ with pytest.raises(ValueError, match=msg):
+ merge(df1, df2, on="col1", how="outer", indicator=5)
+ with pytest.raises(ValueError, match=msg):
+ df1.merge(df2, on="col1", how="outer", indicator=5)
+
+ def test_merge_indicator_result_integrity(self, dfs_for_indicator):
+ # Check result integrity
+ df1, df2 = dfs_for_indicator
+
+ test2 = merge(df1, df2, on="col1", how="left", indicator=True)
+ assert (test2._merge != "right_only").all()
+ test2 = df1.merge(df2, on="col1", how="left", indicator=True)
+ assert (test2._merge != "right_only").all()
+
+ test3 = merge(df1, df2, on="col1", how="right", indicator=True)
+ assert (test3._merge != "left_only").all()
+ test3 = df1.merge(df2, on="col1", how="right", indicator=True)
+ assert (test3._merge != "left_only").all()
+
+ test4 = merge(df1, df2, on="col1", how="inner", indicator=True)
+ assert (test4._merge == "both").all()
+ test4 = df1.merge(df2, on="col1", how="inner", indicator=True)
+ assert (test4._merge == "both").all()
+
+ def test_merge_indicator_invalid(self, dfs_for_indicator):
+ # Check if working name in df
+ df1, _ = dfs_for_indicator
+
+ for i in ["_right_indicator", "_left_indicator", "_merge"]:
+ df_badcolumn = DataFrame({"col1": [1, 2], i: [2, 2]})
+
+ msg = (
+ "Cannot use `indicator=True` option when data contains a "
+ f"column named {i}|"
+ "Cannot use name of an existing column for indicator column"
+ )
+ with pytest.raises(ValueError, match=msg):
+ merge(df1, df_badcolumn, on="col1", how="outer", indicator=True)
+ with pytest.raises(ValueError, match=msg):
+ df1.merge(df_badcolumn, on="col1", how="outer", indicator=True)
+
+ # Check for name conflict with custom name
+ df_badcolumn = DataFrame({"col1": [1, 2], "custom_column_name": [2, 2]})
+
+ msg = "Cannot use name of an existing column for indicator column"
+ with pytest.raises(ValueError, match=msg):
+ merge(
+ df1,
+ df_badcolumn,
+ on="col1",
+ how="outer",
+ indicator="custom_column_name",
+ )
+ with pytest.raises(ValueError, match=msg):
+ df1.merge(
+ df_badcolumn, on="col1", how="outer", indicator="custom_column_name"
+ )
+
+ def test_merge_indicator_multiple_columns(self):
+ # Merge on multiple columns
+ df3 = DataFrame({"col1": [0, 1], "col2": ["a", "b"]})
+
+ df4 = DataFrame({"col1": [1, 1, 3], "col2": ["b", "x", "y"]})
+
+ hand_coded_result = DataFrame(
+ {"col1": [0, 1, 1, 3], "col2": ["a", "b", "x", "y"]}
+ )
+ hand_coded_result["_merge"] = Categorical(
+ ["left_only", "both", "right_only", "right_only"],
+ categories=["left_only", "right_only", "both"],
+ )
+
+ test5 = merge(df3, df4, on=["col1", "col2"], how="outer", indicator=True)
+ tm.assert_frame_equal(test5, hand_coded_result)
+ test5 = df3.merge(df4, on=["col1", "col2"], how="outer", indicator=True)
+ tm.assert_frame_equal(test5, hand_coded_result)
+
+ def test_validation(self):
+ left = DataFrame(
+ {"a": ["a", "b", "c", "d"], "b": ["cat", "dog", "weasel", "horse"]},
+ index=range(4),
+ )
+
+ right = DataFrame(
+ {
+ "a": ["a", "b", "c", "d", "e"],
+ "c": ["meow", "bark", "um... weasel noise?", "nay", "chirp"],
+ },
+ index=range(5),
+ )
+
+ # Make sure no side effects.
+ left_copy = left.copy()
+ right_copy = right.copy()
+
+ result = merge(left, right, left_index=True, right_index=True, validate="1:1")
+ tm.assert_frame_equal(left, left_copy)
+ tm.assert_frame_equal(right, right_copy)
+
+ # make sure merge still correct
+ expected = DataFrame(
+ {
+ "a_x": ["a", "b", "c", "d"],
+ "b": ["cat", "dog", "weasel", "horse"],
+ "a_y": ["a", "b", "c", "d"],
+ "c": ["meow", "bark", "um... weasel noise?", "nay"],
+ },
+ index=range(4),
+ columns=["a_x", "b", "a_y", "c"],
+ )
+
+ result = merge(
+ left, right, left_index=True, right_index=True, validate="one_to_one"
+ )
+ tm.assert_frame_equal(result, expected)
+
+ expected_2 = DataFrame(
+ {
+ "a": ["a", "b", "c", "d"],
+ "b": ["cat", "dog", "weasel", "horse"],
+ "c": ["meow", "bark", "um... weasel noise?", "nay"],
+ },
+ index=range(4),
+ )
+
+ result = merge(left, right, on="a", validate="1:1")
+ tm.assert_frame_equal(left, left_copy)
+ tm.assert_frame_equal(right, right_copy)
+ tm.assert_frame_equal(result, expected_2)
+
+ result = merge(left, right, on="a", validate="one_to_one")
+ tm.assert_frame_equal(result, expected_2)
+
+ # One index, one column
+ expected_3 = DataFrame(
+ {
+ "b": ["cat", "dog", "weasel", "horse"],
+ "a": ["a", "b", "c", "d"],
+ "c": ["meow", "bark", "um... weasel noise?", "nay"],
+ },
+ columns=["b", "a", "c"],
+ index=range(4),
+ )
+
+ left_index_reset = left.set_index("a")
+ result = merge(
+ left_index_reset,
+ right,
+ left_index=True,
+ right_on="a",
+ validate="one_to_one",
+ )
+ tm.assert_frame_equal(result, expected_3)
+
+ # Dups on right
+ right_w_dups = concat([right, DataFrame({"a": ["e"], "c": ["moo"]}, index=[4])])
+ merge(
+ left,
+ right_w_dups,
+ left_index=True,
+ right_index=True,
+ validate="one_to_many",
+ )
+
+ msg = "Merge keys are not unique in right dataset; not a one-to-one merge"
+ with pytest.raises(MergeError, match=msg):
+ merge(
+ left,
+ right_w_dups,
+ left_index=True,
+ right_index=True,
+ validate="one_to_one",
+ )
+
+ with pytest.raises(MergeError, match=msg):
+ merge(left, right_w_dups, on="a", validate="one_to_one")
+
+ # Dups on left
+ left_w_dups = concat(
+ [left, DataFrame({"a": ["a"], "c": ["cow"]}, index=[3])], sort=True
+ )
+ merge(
+ left_w_dups,
+ right,
+ left_index=True,
+ right_index=True,
+ validate="many_to_one",
+ )
+
+ msg = "Merge keys are not unique in left dataset; not a one-to-one merge"
+ with pytest.raises(MergeError, match=msg):
+ merge(
+ left_w_dups,
+ right,
+ left_index=True,
+ right_index=True,
+ validate="one_to_one",
+ )
+
+ with pytest.raises(MergeError, match=msg):
+ merge(left_w_dups, right, on="a", validate="one_to_one")
+
+ # Dups on both
+ merge(left_w_dups, right_w_dups, on="a", validate="many_to_many")
+
+ msg = "Merge keys are not unique in right dataset; not a many-to-one merge"
+ with pytest.raises(MergeError, match=msg):
+ merge(
+ left_w_dups,
+ right_w_dups,
+ left_index=True,
+ right_index=True,
+ validate="many_to_one",
+ )
+
+ msg = "Merge keys are not unique in left dataset; not a one-to-many merge"
+ with pytest.raises(MergeError, match=msg):
+ merge(left_w_dups, right_w_dups, on="a", validate="one_to_many")
+
+ # Check invalid arguments
+ msg = (
+ '"jibberish" is not a valid argument. '
+ "Valid arguments are:\n"
+ '- "1:1"\n'
+ '- "1:m"\n'
+ '- "m:1"\n'
+ '- "m:m"\n'
+ '- "one_to_one"\n'
+ '- "one_to_many"\n'
+ '- "many_to_one"\n'
+ '- "many_to_many"'
+ )
+ with pytest.raises(ValueError, match=msg):
+ merge(left, right, on="a", validate="jibberish")
+
+ # Two column merge, dups in both, but jointly no dups.
+ left = DataFrame(
+ {
+ "a": ["a", "a", "b", "b"],
+ "b": [0, 1, 0, 1],
+ "c": ["cat", "dog", "weasel", "horse"],
+ },
+ index=range(4),
+ )
+
+ right = DataFrame(
+ {
+ "a": ["a", "a", "b"],
+ "b": [0, 1, 0],
+ "d": ["meow", "bark", "um... weasel noise?"],
+ },
+ index=range(3),
+ )
+
+ expected_multi = DataFrame(
+ {
+ "a": ["a", "a", "b"],
+ "b": [0, 1, 0],
+ "c": ["cat", "dog", "weasel"],
+ "d": ["meow", "bark", "um... weasel noise?"],
+ },
+ index=range(3),
+ )
+
+ msg = (
+ "Merge keys are not unique in either left or right dataset; "
+ "not a one-to-one merge"
+ )
+ with pytest.raises(MergeError, match=msg):
+ merge(left, right, on="a", validate="1:1")
+
+ result = merge(left, right, on=["a", "b"], validate="1:1")
+ tm.assert_frame_equal(result, expected_multi)
+
+ def test_merge_validate_error_message(self):
+ # GH#62742
+ left = DataFrame({"key": [1, 1, 2]})
+ right = DataFrame({"key": [1, 2, 2]})
+
+ with pytest.raises(MergeError, match="Duplicates in left:\n key\n 1 ...\n"):
+ merge(left, right, validate="1:1")
+ with pytest.raises(MergeError, match="Duplicates in left:\n key\n 1 ..."):
+ merge(left, right, validate="1:m")
+ with pytest.raises(MergeError, match="Duplicates in right:\n key\n 2 ..."):
+ merge(left, right, validate="1:1")
+ with pytest.raises(MergeError, match="Duplicates in right:\n key\n 2 ..."):
+ merge(left, right, validate="m:1")
+
+ right = DataFrame({"key": [1, 2, 3]})
+ with pytest.raises(MergeError, match="Duplicates in left:\n key\n 1 ..."):
+ merge(left, right, validate="1:1")
+ with pytest.raises(MergeError, match="Duplicates in right:\n key\n 1 ..."):
+ merge(right, left, validate="1:1")
+
+ def test_merge_two_empty_df_no_division_error(self):
+ # GH17776, PR #17846
+ a = DataFrame({"a": [], "b": [], "c": []})
+ with np.errstate(divide="raise"):
+ merge(a, a, on=("a", "b"))
+
+ @pytest.mark.parametrize("how", ["right", "outer"])
+ @pytest.mark.parametrize(
+ "index,expected_index",
+ [
+ (
+ CategoricalIndex([1, 2, 4]),
+ CategoricalIndex([1, 2, 4, None, None, None]),
+ ),
+ (
+ DatetimeIndex(
+ ["2001-01-01", "2002-02-02", "2003-03-03"], dtype="M8[ns]"
+ ),
+ DatetimeIndex(
+ ["2001-01-01", "2002-02-02", "2003-03-03", pd.NaT, pd.NaT, pd.NaT],
+ dtype="M8[ns]",
+ ),
+ ),
+ *[
+ (
+ Index([1, 2, 3], dtype=dtyp),
+ Index([1, 2, 3, None, None, None], dtype=np.float64),
+ )
+ for dtyp in tm.ALL_REAL_NUMPY_DTYPES
+ ],
+ (
+ IntervalIndex.from_tuples([(1, 2), (2, 3), (3, 4)]),
+ IntervalIndex.from_tuples(
+ [(1, 2), (2, 3), (3, 4), np.nan, np.nan, np.nan]
+ ),
+ ),
+ (
+ PeriodIndex(["2001-01-01", "2001-01-02", "2001-01-03"], freq="D"),
+ PeriodIndex(
+ ["2001-01-01", "2001-01-02", "2001-01-03", pd.NaT, pd.NaT, pd.NaT],
+ freq="D",
+ ),
+ ),
+ (
+ TimedeltaIndex(["1D", "2D", "3D"]),
+ TimedeltaIndex(["1D", "2D", "3D", pd.NaT, pd.NaT, pd.NaT]),
+ ),
+ ],
+ )
+ def test_merge_on_index_with_more_values(self, how, index, expected_index):
+ # GH 24212
+ # pd.merge gets [0, 1, 2, -1, -1, -1] as left_indexer, ensure that
+ # -1 is interpreted as a missing value instead of the last element
+ if index.dtype == "float32" and expected_index.dtype == "float64":
+ # GH#41626
+ expected_index = expected_index.astype("float32")
+ df1 = DataFrame({"a": [0, 1, 2], "key": [0, 1, 2]}, index=index)
+ df2 = DataFrame({"b": [0, 1, 2, 3, 4, 5]})
+ result = df1.merge(df2, left_on="key", right_index=True, how=how)
+ expected = DataFrame(
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ [2, 2, 2],
+ [np.nan, 3, 3],
+ [np.nan, 4, 4],
+ [np.nan, 5, 5],
+ ],
+ columns=["a", "key", "b"],
+ )
+ expected.set_index(expected_index, inplace=True)
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_right_index_right(self):
+ # Note: the expected output here is probably incorrect.
+ # See https://github.com/pandas-dev/pandas/issues/17257 for more.
+ # We include this as a regression test for GH-24897.
+ left = DataFrame({"a": [1, 2, 3], "key": [0, 1, 1]})
+ right = DataFrame({"b": [1, 2, 3]})
+
+ expected = DataFrame(
+ {"a": [1, 2, 3, None], "key": [0, 1, 1, 2], "b": [1, 2, 2, 3]},
+ columns=["a", "key", "b"],
+ index=[0, 1, 2, np.nan],
+ )
+ result = left.merge(right, left_on="key", right_index=True, how="right")
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("how", ["left", "right"])
+ def test_merge_preserves_row_order(self, how):
+ # GH 27453
+ left_df = DataFrame({"animal": ["dog", "pig"], "max_speed": [40, 11]})
+ right_df = DataFrame({"animal": ["quetzal", "pig"], "max_speed": [80, 11]})
+ result = left_df.merge(right_df, on=["animal", "max_speed"], how=how)
+ if how == "right":
+ expected = DataFrame({"animal": ["quetzal", "pig"], "max_speed": [80, 11]})
+ else:
+ expected = DataFrame({"animal": ["dog", "pig"], "max_speed": [40, 11]})
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_take_missing_values_from_index_of_other_dtype(self):
+ # GH 24212
+ left = DataFrame(
+ {
+ "a": [1, 2, 3],
+ "key": Categorical(["a", "a", "b"], categories=list("abc")),
+ }
+ )
+ right = DataFrame({"b": [1, 2, 3]}, index=CategoricalIndex(["a", "b", "c"]))
+ result = left.merge(right, left_on="key", right_index=True, how="right")
+ expected = DataFrame(
+ {
+ "a": [1, 2, 3, None],
+ "key": Categorical(["a", "a", "b", "c"]),
+ "b": [1, 1, 2, 3],
+ },
+ index=[0, 1, 2, np.nan],
+ )
+ expected = expected.reindex(columns=["a", "key", "b"])
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_readonly(self):
+ # https://github.com/pandas-dev/pandas/issues/27943
+ data1 = DataFrame(
+ np.arange(20).reshape((4, 5)) + 1, columns=["a", "b", "c", "d", "e"]
+ )
+ data2 = DataFrame(
+ np.arange(20).reshape((5, 4)) + 1, columns=["a", "b", "x", "y"]
+ )
+
+ # make each underlying block array / column array read-only
+ for block in data1._mgr.blocks:
+ block.values.flags.writeable = False
+
+ data1.merge(data2) # no error
+
+ def test_merge_how_validation(self):
+ # https://github.com/pandas-dev/pandas/issues/59422
+ data1 = DataFrame(
+ np.arange(20).reshape((4, 5)) + 1, columns=["a", "b", "c", "d", "e"]
+ )
+ data2 = DataFrame(
+ np.arange(20).reshape((5, 4)) + 1, columns=["a", "b", "x", "y"]
+ )
+ msg = (
+ "'full' is not a valid Merge type: left, right, inner, outer, "
+ "left_anti, right_anti, cross, asof"
+ )
+ with pytest.raises(ValueError, match=re.escape(msg)):
+ data1.merge(data2, how="full")
+
+
+def _check_merge(x, y):
+ for how in ["inner", "left", "outer"]:
+ for sort in [True, False]:
+ result = x.join(y, how=how, sort=sort)
+
+ expected = merge(x.reset_index(), y.reset_index(), how=how, sort=sort)
+ expected = expected.set_index("index")
+
+ # TODO check_names on merge?
+ tm.assert_frame_equal(result, expected, check_names=False)
+
+
+class TestMergeDtypes:
+ @pytest.mark.parametrize("dtype", [object, "category"])
+ def test_different(self, dtype):
+ left = DataFrame(
+ {
+ "A": ["foo", "bar"],
+ "B": Series(["foo", "bar"]).astype("category"),
+ "C": [1, 2],
+ "D": [1.0, 2.0],
+ "E": Series([1, 2], dtype="uint64"),
+ "F": Series([1, 2], dtype="int32"),
+ }
+ )
+ right_vals = Series(["foo", "bar"], dtype=dtype)
+ right = DataFrame({"A": right_vals})
+
+ # GH 9780
+ # We allow merging on object and categorical cols and cast
+ # categorical cols to object
+ result = merge(left, right, on="A")
+ assert is_object_dtype(result.A.dtype) or is_string_dtype(result.A.dtype)
+
+ @pytest.mark.parametrize("d2", [np.int64, np.float64, np.float32, np.float16])
+ def test_join_multi_dtypes(self, any_int_numpy_dtype, d2):
+ dtype1 = np.dtype(any_int_numpy_dtype)
+ dtype2 = np.dtype(d2)
+
+ left = DataFrame(
+ {
+ "k1": np.array([0, 1, 2] * 8, dtype=dtype1),
+ "k2": ["foo", "bar"] * 12,
+ "v": np.array(np.arange(24), dtype=np.int64),
+ }
+ )
+
+ index = MultiIndex.from_tuples([(2, "bar"), (1, "foo")])
+ right = DataFrame({"v2": np.array([5, 7], dtype=dtype2)}, index=index)
+
+ result = left.join(right, on=["k1", "k2"])
+
+ expected = left.copy()
+
+ if dtype2.kind == "i":
+ dtype2 = np.dtype("float64")
+ expected["v2"] = np.array(np.nan, dtype=dtype2)
+ expected.loc[(expected.k1 == 2) & (expected.k2 == "bar"), "v2"] = 5
+ expected.loc[(expected.k1 == 1) & (expected.k2 == "foo"), "v2"] = 7
+
+ tm.assert_frame_equal(result, expected)
+
+ result = left.join(right, on=["k1", "k2"], sort=True)
+ expected.sort_values(["k1", "k2"], kind="mergesort", inplace=True)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "int_vals, float_vals, exp_vals",
+ [
+ ([1, 2, 3], [1.0, 2.0, 3.0], {"X": [1, 2, 3], "Y": [1.0, 2.0, 3.0]}),
+ ([1, 2, 3], [1.0, 3.0], {"X": [1, 3], "Y": [1.0, 3.0]}),
+ ([1, 2], [1.0, 2.0, 3.0], {"X": [1, 2], "Y": [1.0, 2.0]}),
+ ],
+ )
+ def test_merge_on_ints_floats(self, int_vals, float_vals, exp_vals):
+ # GH 16572
+ # Check that float column is not cast to object if
+ # merging on float and int columns
+ A = DataFrame({"X": int_vals})
+ B = DataFrame({"Y": float_vals})
+ expected = DataFrame(exp_vals)
+
+ result = A.merge(B, left_on="X", right_on="Y")
+ tm.assert_frame_equal(result, expected)
+
+ result = B.merge(A, left_on="Y", right_on="X")
+ tm.assert_frame_equal(result, expected[["Y", "X"]])
+
+ def test_merge_key_dtype_cast(self):
+ # GH 17044
+ df1 = DataFrame({"key": [1.0, 2.0], "v1": [10, 20]}, columns=["key", "v1"])
+ df2 = DataFrame({"key": [2], "v2": [200]}, columns=["key", "v2"])
+ result = df1.merge(df2, on="key", how="left")
+ expected = DataFrame(
+ {"key": [1.0, 2.0], "v1": [10, 20], "v2": [np.nan, 200.0]},
+ columns=["key", "v1", "v2"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_on_ints_floats_warning(self):
+ # GH 16572
+ # merge will produce a warning when merging on int and
+ # float columns where the float values are not exactly
+ # equal to their int representation
+ A = DataFrame({"X": [1, 2, 3]})
+ B = DataFrame({"Y": [1.1, 2.5, 3.0]})
+ expected = DataFrame({"X": [3], "Y": [3.0]})
+
+ msg = "the float values are not equal to their int representation"
+ with tm.assert_produces_warning(UserWarning, match=msg):
+ result = A.merge(B, left_on="X", right_on="Y")
+ tm.assert_frame_equal(result, expected)
+
+ with tm.assert_produces_warning(UserWarning, match=msg):
+ result = B.merge(A, left_on="Y", right_on="X")
+ tm.assert_frame_equal(result, expected[["Y", "X"]])
+
+ # test no warning if float has NaNs
+ B = DataFrame({"Y": [np.nan, np.nan, 3.0]})
+
+ with tm.assert_produces_warning(None):
+ result = B.merge(A, left_on="Y", right_on="X")
+ tm.assert_frame_equal(result, expected[["Y", "X"]])
+
+ def test_merge_incompat_infer_boolean_object(self):
+ # GH21119: bool + object bool merge OK
+ df1 = DataFrame({"key": Series([True, False], dtype=object)})
+ df2 = DataFrame({"key": [True, False]})
+
+ expected = DataFrame({"key": [True, False]}, dtype=object)
+ result = merge(df1, df2, on="key")
+ tm.assert_frame_equal(result, expected)
+ result = merge(df2, df1, on="key")
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_incompat_infer_boolean_object_with_missing(self):
+ # GH21119: bool + object bool merge OK
+ # with missing value
+ df1 = DataFrame({"key": Series([True, False, np.nan], dtype=object)})
+ df2 = DataFrame({"key": [True, False]})
+
+ expected = DataFrame({"key": [True, False]}, dtype=object)
+ result = merge(df1, df2, on="key")
+ tm.assert_frame_equal(result, expected)
+ result = merge(df2, df1, on="key")
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "df1_vals, df2_vals",
+ [
+ # merge on category coerces to object
+ ([0, 1, 2], Series(["a", "b", "a"]).astype("category")),
+ ([0.0, 1.0, 2.0], Series(["a", "b", "a"]).astype("category")),
+ # no not infer
+ ([0, 1], Series([False, True], dtype=object)),
+ ([0, 1], Series([False, True], dtype=bool)),
+ ],
+ )
+ def test_merge_incompat_dtypes_are_ok(self, df1_vals, df2_vals):
+ # these are explicitly allowed incompat merges, that pass thru
+ # the result type is dependent on if the values on the rhs are
+ # inferred, otherwise these will be coerced to object
+
+ df1 = DataFrame({"A": df1_vals})
+ df2 = DataFrame({"A": df2_vals})
+
+ result = merge(df1, df2, on=["A"])
+ assert is_object_dtype(result.A.dtype)
+ result = merge(df2, df1, on=["A"])
+ assert is_object_dtype(result.A.dtype) or is_string_dtype(result.A.dtype)
+
+ @pytest.mark.parametrize(
+ "df1_vals, df2_vals",
+ [
+ # do not infer to numeric
+ (Series([1, 2], dtype="uint64"), ["a", "b", "c"]),
+ (Series([1, 2], dtype="int32"), ["a", "b", "c"]),
+ ([0, 1, 2], ["0", "1", "2"]),
+ ([0.0, 1.0, 2.0], ["0", "1", "2"]),
+ (
+ pd.date_range("1/1/2011", periods=2, freq="D"),
+ ["2011-01-01", "2011-01-02"],
+ ),
+ (pd.date_range("1/1/2011", periods=2, freq="D"), [0, 1]),
+ (pd.date_range("1/1/2011", periods=2, freq="D"), [0.0, 1.0]),
+ (
+ pd.date_range("20130101", periods=3),
+ pd.date_range("20130101", periods=3, tz="US/Eastern"),
+ ),
+ ],
+ )
+ def test_merge_incompat_dtypes_error(self, df1_vals, df2_vals):
+ # GH 9780, GH 15800
+ # Raise a ValueError when a user tries to merge on
+ # dtypes that are incompatible (e.g., obj and int/float)
+
+ df1 = DataFrame({"A": df1_vals})
+ df2 = DataFrame({"A": df2_vals})
+
+ msg = (
+ f"You are trying to merge on {df1['A'].dtype} and {df2['A'].dtype} "
+ "columns for key 'A'. If you wish to proceed you should use pd.concat"
+ )
+ msg = re.escape(msg)
+ with pytest.raises(ValueError, match=msg):
+ merge(df1, df2, on=["A"])
+
+ # Check that error still raised when swapping order of dataframes
+ msg = (
+ f"You are trying to merge on {df2['A'].dtype} and {df1['A'].dtype} "
+ "columns for key 'A'. If you wish to proceed you should use pd.concat"
+ )
+ msg = re.escape(msg)
+ with pytest.raises(ValueError, match=msg):
+ merge(df2, df1, on=["A"])
+
+ # Check that error still raised when merging on multiple columns
+ # The error message should mention the first incompatible column
+ if len(df1_vals) == len(df2_vals):
+ # Column A in df1 and df2 is of compatible (the same) dtype
+ # Columns B and C in df1 and df2 are of incompatible dtypes
+ df3 = DataFrame({"A": df2_vals, "B": df1_vals, "C": df1_vals})
+ df4 = DataFrame({"A": df2_vals, "B": df2_vals, "C": df2_vals})
+
+ # Check that error raised correctly when merging all columns A, B, and C
+ # The error message should mention key 'B'
+ msg = (
+ f"You are trying to merge on {df3['B'].dtype} and {df4['B'].dtype} "
+ "columns for key 'B'. If you wish to proceed you should use pd.concat"
+ )
+ msg = re.escape(msg)
+ with pytest.raises(ValueError, match=msg):
+ merge(df3, df4)
+
+ # Check that error raised correctly when merging columns A and C
+ # The error message should mention key 'C'
+ msg = (
+ f"You are trying to merge on {df3['C'].dtype} and {df4['C'].dtype} "
+ "columns for key 'C'. If you wish to proceed you should use pd.concat"
+ )
+ msg = re.escape(msg)
+ with pytest.raises(ValueError, match=msg):
+ merge(df3, df4, on=["A", "C"])
+
+ @pytest.mark.parametrize(
+ "expected_data, how",
+ [
+ ([1, 2], "outer"),
+ ([], "inner"),
+ ([2], "right"),
+ ([1], "left"),
+ ],
+ )
+ def test_merge_EA_dtype(self, any_numeric_ea_dtype, how, expected_data):
+ # GH#40073
+ d1 = DataFrame([(1,)], columns=["id"], dtype=any_numeric_ea_dtype)
+ d2 = DataFrame([(2,)], columns=["id"], dtype=any_numeric_ea_dtype)
+ result = merge(d1, d2, how=how)
+ exp_index = RangeIndex(len(expected_data))
+ expected = DataFrame(
+ expected_data, index=exp_index, columns=["id"], dtype=any_numeric_ea_dtype
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "expected_data, how",
+ [
+ (["a", "b"], "outer"),
+ ([], "inner"),
+ (["b"], "right"),
+ (["a"], "left"),
+ ],
+ )
+ def test_merge_string_dtype(self, how, expected_data, any_string_dtype):
+ # GH#40073
+ d1 = DataFrame([("a",)], columns=["id"], dtype=any_string_dtype)
+ d2 = DataFrame([("b",)], columns=["id"], dtype=any_string_dtype)
+ result = merge(d1, d2, how=how)
+ exp_idx = RangeIndex(len(expected_data))
+ expected = DataFrame(
+ expected_data, index=exp_idx, columns=["id"], dtype=any_string_dtype
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "how, expected_data",
+ [
+ ("inner", [[True, 1, 4], [False, 5, 3]]),
+ ("outer", [[False, 5, 3], [True, 1, 4]]),
+ ("left", [[True, 1, 4], [False, 5, 3]]),
+ ("right", [[False, 5, 3], [True, 1, 4]]),
+ ],
+ )
+ def test_merge_bool_dtype(self, how, expected_data):
+ # GH#40073
+ df1 = DataFrame({"A": [True, False], "B": [1, 5]})
+ df2 = DataFrame({"A": [False, True], "C": [3, 4]})
+ result = merge(df1, df2, how=how)
+ expected = DataFrame(expected_data, columns=["A", "B", "C"])
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_ea_with_string(self, join_type, string_dtype):
+ # GH 43734 Avoid the use of `assign` with multi-index
+ df1 = DataFrame(
+ data={
+ ("lvl0", "lvl1-a"): ["1", "2", "3", "4", None],
+ ("lvl0", "lvl1-b"): ["4", "5", "6", "7", "8"],
+ },
+ dtype=pd.StringDtype(),
+ )
+ df1_copy = df1.copy()
+ df2 = DataFrame(
+ data={
+ ("lvl0", "lvl1-a"): ["1", "2", "3", pd.NA, "5"],
+ ("lvl0", "lvl1-c"): ["7", "8", "9", pd.NA, "11"],
+ },
+ dtype=string_dtype,
+ )
+ df2_copy = df2.copy()
+ merged = merge(left=df1, right=df2, on=[("lvl0", "lvl1-a")], how=join_type)
+
+ # No change in df1 and df2
+ tm.assert_frame_equal(df1, df1_copy)
+ tm.assert_frame_equal(df2, df2_copy)
+
+ # Check the expected types for the merged data frame
+ expected = Series(
+ [np.dtype("O"), pd.StringDtype(), np.dtype("O")],
+ index=MultiIndex.from_tuples(
+ [("lvl0", "lvl1-a"), ("lvl0", "lvl1-b"), ("lvl0", "lvl1-c")]
+ ),
+ )
+ tm.assert_series_equal(merged.dtypes, expected)
+
+ @pytest.mark.parametrize(
+ "left_empty, how, exp",
+ [
+ (False, "left", "left"),
+ (False, "right", "empty"),
+ (False, "inner", "empty"),
+ (False, "outer", "left"),
+ (False, "cross", "empty_cross"),
+ (True, "left", "empty"),
+ (True, "right", "right"),
+ (True, "inner", "empty"),
+ (True, "outer", "right"),
+ (True, "cross", "empty_cross"),
+ ],
+ )
+ def test_merge_empty(self, left_empty, how, exp):
+ left = DataFrame({"A": [2, 1], "B": [3, 4]})
+ right = DataFrame({"A": [1], "C": [5]}, dtype="int64")
+
+ if left_empty:
+ left = left.head(0)
+ else:
+ right = right.head(0)
+
+ result = left.merge(right, how=how)
+
+ if exp == "left":
+ expected = DataFrame({"A": [2, 1], "B": [3, 4], "C": [np.nan, np.nan]})
+ elif exp == "right":
+ expected = DataFrame({"A": [1], "B": [np.nan], "C": [5]})
+ elif exp == "empty":
+ expected = DataFrame(columns=["A", "B", "C"], dtype="int64")
+ elif exp == "empty_cross":
+ expected = DataFrame(columns=["A_x", "B", "A_y", "C"], dtype="int64")
+
+ if how == "outer":
+ expected = expected.sort_values("A", ignore_index=True)
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_with_uintc_columns(self):
+ df1 = DataFrame({"a": ["foo", "bar"], "b": np.array([1, 2], dtype=np.uintc)})
+ df2 = DataFrame({"a": ["foo", "baz"], "b": np.array([3, 4], dtype=np.uintc)})
+ result = df1.merge(df2, how="outer")
+ expected = DataFrame(
+ {
+ "a": ["bar", "baz", "foo", "foo"],
+ "b": np.array([2, 4, 1, 3], dtype=np.uintc),
+ }
+ )
+ tm.assert_frame_equal(result.reset_index(drop=True), expected)
+
+ def test_merge_with_intc_columns(self):
+ df1 = DataFrame({"a": ["foo", "bar"], "b": np.array([1, 2], dtype=np.intc)})
+ df2 = DataFrame({"a": ["foo", "baz"], "b": np.array([3, 4], dtype=np.intc)})
+ result = df1.merge(df2, how="outer")
+ expected = DataFrame(
+ {
+ "a": ["bar", "baz", "foo", "foo"],
+ "b": np.array([2, 4, 1, 3], dtype=np.intc),
+ }
+ )
+ tm.assert_frame_equal(result.reset_index(drop=True), expected)
+
+ def test_merge_intc_non_monotonic(self):
+ df = DataFrame({"join_key": Series([0, 2, 1], dtype=np.intc)})
+ df_details = DataFrame(
+ {"join_key": Series([0, 1, 2], dtype=np.intc), "value": ["a", "b", "c"]}
+ )
+ merged = df.merge(df_details, on="join_key", how="left")
+ expected = DataFrame(
+ {"join_key": np.array([0, 2, 1], dtype=np.intc), "value": ["a", "c", "b"]}
+ )
+ tm.assert_frame_equal(merged.reset_index(drop=True), expected)
+
+
+@pytest.fixture
+def left():
+ return DataFrame(
+ {
+ "X": Series(
+ np.random.default_rng(2).choice(["foo", "bar"], size=(10,))
+ ).astype(CategoricalDtype(["foo", "bar"])),
+ "Y": np.random.default_rng(2).choice(["one", "two", "three"], size=(10,)),
+ }
+ )
+
+
+@pytest.fixture
+def right():
+ return DataFrame(
+ {
+ "X": Series(["foo", "bar"]).astype(CategoricalDtype(["foo", "bar"])),
+ "Z": [1, 2],
+ }
+ )
+
+
+class TestMergeCategorical:
+ def test_identical(self, left, using_infer_string):
+ # merging on the same, should preserve dtypes
+ merged = merge(left, left, on="X")
+ result = merged.dtypes.sort_index()
+ dtype = np.dtype("O") if not using_infer_string else "str"
+ expected = Series(
+ [CategoricalDtype(categories=["foo", "bar"]), dtype, dtype],
+ index=["X", "Y_x", "Y_y"],
+ )
+ tm.assert_series_equal(result, expected)
+
+ def test_basic(self, left, right, using_infer_string):
+ # we have matching Categorical dtypes in X
+ # so should preserve the merged column
+ merged = merge(left, right, on="X")
+ result = merged.dtypes.sort_index()
+ dtype = np.dtype("O") if not using_infer_string else "str"
+ expected = Series(
+ [
+ CategoricalDtype(categories=["foo", "bar"]),
+ dtype,
+ np.dtype("int64"),
+ ],
+ index=["X", "Y", "Z"],
+ )
+ tm.assert_series_equal(result, expected)
+
+ def test_merge_categorical(self):
+ # GH 9426
+
+ right = DataFrame(
+ {
+ "c": {0: "a", 1: "b", 2: "c", 3: "d", 4: "e"},
+ "d": {0: "null", 1: "null", 2: "null", 3: "null", 4: "null"},
+ }
+ )
+ left = DataFrame(
+ {
+ "a": {0: "f", 1: "f", 2: "f", 3: "f", 4: "f"},
+ "b": {0: "g", 1: "g", 2: "g", 3: "g", 4: "g"},
+ }
+ )
+ df = merge(left, right, how="left", left_on="b", right_on="c")
+
+ # object-object
+ expected = df.copy()
+
+ # object-cat
+ # note that we propagate the category
+ # because we don't have any matching rows
+ cright = right.copy()
+ cright["d"] = cright["d"].astype("category")
+ result = merge(left, cright, how="left", left_on="b", right_on="c")
+ expected["d"] = expected["d"].astype(CategoricalDtype(["null"]))
+ tm.assert_frame_equal(result, expected)
+
+ # cat-object
+ cleft = left.copy()
+ cleft["b"] = cleft["b"].astype("category")
+ result = merge(cleft, cright, how="left", left_on="b", right_on="c")
+ tm.assert_frame_equal(result, expected)
+
+ # cat-cat
+ cright = right.copy()
+ cright["d"] = cright["d"].astype("category")
+ cleft = left.copy()
+ cleft["b"] = cleft["b"].astype("category")
+ result = merge(cleft, cright, how="left", left_on="b", right_on="c")
+ tm.assert_frame_equal(result, expected)
+
+ def tests_merge_categorical_unordered_equal(self):
+ # GH-19551
+ df1 = DataFrame(
+ {
+ "Foo": Categorical(["A", "B", "C"], categories=["A", "B", "C"]),
+ "Left": ["A0", "B0", "C0"],
+ }
+ )
+
+ df2 = DataFrame(
+ {
+ "Foo": Categorical(["C", "B", "A"], categories=["C", "B", "A"]),
+ "Right": ["C1", "B1", "A1"],
+ }
+ )
+ result = merge(df1, df2, on=["Foo"])
+ expected = DataFrame(
+ {
+ "Foo": Categorical(["A", "B", "C"]),
+ "Left": ["A0", "B0", "C0"],
+ "Right": ["A1", "B1", "C1"],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("ordered", [True, False])
+ def test_multiindex_merge_with_unordered_categoricalindex(self, ordered):
+ # GH 36973
+ pcat = CategoricalDtype(categories=["P2", "P1"], ordered=ordered)
+ df1 = DataFrame(
+ {
+ "id": ["C", "C", "D"],
+ "p": Categorical(["P2", "P1", "P2"], dtype=pcat),
+ "a": [0, 1, 2],
+ }
+ ).set_index(["id", "p"])
+ df2 = DataFrame(
+ {
+ "id": ["A", "C", "C"],
+ "p": Categorical(["P2", "P2", "P1"], dtype=pcat),
+ "d1": [10, 11, 12],
+ }
+ ).set_index(["id", "p"])
+ result = merge(df1, df2, how="left", left_index=True, right_index=True)
+ expected = DataFrame(
+ {
+ "id": ["C", "C", "D"],
+ "p": Categorical(["P2", "P1", "P2"], dtype=pcat),
+ "a": [0, 1, 2],
+ "d1": [11.0, 12.0, np.nan],
+ }
+ ).set_index(["id", "p"])
+ tm.assert_frame_equal(result, expected)
+
+ def test_other_columns(self, left, right, using_infer_string):
+ # non-merge columns should preserve if possible
+ right = right.assign(Z=right.Z.astype("category"))
+
+ merged = merge(left, right, on="X")
+ result = merged.dtypes.sort_index()
+ dtype = np.dtype("O") if not using_infer_string else "str"
+ expected = Series(
+ [
+ CategoricalDtype(categories=["foo", "bar"]),
+ dtype,
+ CategoricalDtype(categories=[1, 2]),
+ ],
+ index=["X", "Y", "Z"],
+ )
+ tm.assert_series_equal(result, expected)
+
+ # categories are preserved
+ assert left.X.values._categories_match_up_to_permutation(merged.X.values)
+ assert right.Z.values._categories_match_up_to_permutation(merged.Z.values)
+
+ @pytest.mark.parametrize(
+ "change",
+ [
+ lambda x: x,
+ lambda x: x.astype(CategoricalDtype(["foo", "bar", "bah"])),
+ lambda x: x.astype(CategoricalDtype(ordered=True)),
+ ],
+ )
+ def test_dtype_on_merged_different(
+ self, change, join_type, left, right, using_infer_string
+ ):
+ # our merging columns, X now has 2 different dtypes
+ # so we must be object as a result
+
+ X = change(right.X.astype("object"))
+ right = right.assign(X=X)
+ assert isinstance(left.X.values.dtype, CategoricalDtype)
+ # assert not left.X.values._categories_match_up_to_permutation(right.X.values)
+
+ merged = merge(left, right, on="X", how=join_type)
+
+ result = merged.dtypes.sort_index()
+ dtype = np.dtype("O") if not using_infer_string else "str"
+ expected = Series([dtype, dtype, np.dtype("int64")], index=["X", "Y", "Z"])
+ tm.assert_series_equal(result, expected)
+
+ def test_self_join_multiple_categories(self):
+ # GH 16767
+ # non-duplicates should work with multiple categories
+ m = 5
+ df = DataFrame(
+ {
+ "a": ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"] * m,
+ "b": ["t", "w", "x", "y", "z"] * 2 * m,
+ "c": [
+ letter
+ for each in ["m", "n", "u", "p", "o"]
+ for letter in [each] * 2 * m
+ ],
+ "d": [
+ letter
+ for each in [
+ "aa",
+ "bb",
+ "cc",
+ "dd",
+ "ee",
+ "ff",
+ "gg",
+ "hh",
+ "ii",
+ "jj",
+ ]
+ for letter in [each] * m
+ ],
+ }
+ )
+
+ # change them all to categorical variables
+ df = df.apply(lambda x: x.astype("category"))
+
+ # self-join should equal ourselves
+ result = merge(df, df, on=list(df.columns))
+
+ tm.assert_frame_equal(result, df)
+
+ def test_dtype_on_categorical_dates(self):
+ # GH 16900
+ # dates should not be coerced to ints
+
+ df = DataFrame(
+ [[date(2001, 1, 1), 1.1], [date(2001, 1, 2), 1.3]], columns=["date", "num2"]
+ )
+ df["date"] = df["date"].astype("category")
+
+ df2 = DataFrame(
+ [[date(2001, 1, 1), 1.3], [date(2001, 1, 3), 1.4]], columns=["date", "num4"]
+ )
+ df2["date"] = df2["date"].astype("category")
+
+ expected_outer = DataFrame(
+ [
+ [pd.Timestamp("2001-01-01").date(), 1.1, 1.3],
+ [pd.Timestamp("2001-01-02").date(), 1.3, np.nan],
+ [pd.Timestamp("2001-01-03").date(), np.nan, 1.4],
+ ],
+ columns=["date", "num2", "num4"],
+ )
+ result_outer = merge(df, df2, how="outer", on=["date"])
+ tm.assert_frame_equal(result_outer, expected_outer)
+
+ expected_inner = DataFrame(
+ [[pd.Timestamp("2001-01-01").date(), 1.1, 1.3]],
+ columns=["date", "num2", "num4"],
+ )
+ result_inner = merge(df, df2, how="inner", on=["date"])
+ tm.assert_frame_equal(result_inner, expected_inner)
+
+ @pytest.mark.parametrize("ordered", [True, False])
+ @pytest.mark.parametrize(
+ "category_column,categories,expected_categories",
+ [
+ ([False, True, True, False], [True, False], [True, False]),
+ ([2, 1, 1, 2], [1, 2], [1, 2]),
+ (["False", "True", "True", "False"], ["True", "False"], ["True", "False"]),
+ ],
+ )
+ def test_merging_with_bool_or_int_cateorical_column(
+ self, category_column, categories, expected_categories, ordered
+ ):
+ # GH 17187
+ # merging with a boolean/int categorical column
+ df1 = DataFrame({"id": [1, 2, 3, 4], "cat": category_column})
+ df1["cat"] = df1["cat"].astype(CategoricalDtype(categories, ordered=ordered))
+ df2 = DataFrame({"id": [2, 4], "num": [1, 9]})
+ result = df1.merge(df2)
+ expected = DataFrame({"id": [2, 4], "cat": expected_categories, "num": [1, 9]})
+ expected["cat"] = expected["cat"].astype(
+ CategoricalDtype(categories, ordered=ordered)
+ )
+ tm.assert_frame_equal(expected, result)
+
+ def test_merge_on_int_array(self):
+ # GH 23020
+ df = DataFrame({"A": Series([1, 2, pd.NA], dtype="Int64"), "B": 1})
+ result = merge(df, df, on="A")
+ expected = DataFrame(
+ {"A": Series([1, 2, pd.NA], dtype="Int64"), "B_x": 1, "B_y": 1}
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_category_index_levels_stay_category(self):
+ # GH#37480
+ df1 = DataFrame(
+ {
+ "x": list(range(1, 10)),
+ "y": list(range(1, 10)),
+ "z": list(range(1, 10)),
+ "d": list(range(1, 10)),
+ }
+ )
+
+ df2 = df1.astype({"x": "category", "y": "category", "z": "category"})
+
+ df3 = df2.iloc[:4, :].groupby(["z", "x"], observed=True).agg({"d": "sum"})
+ df4 = df2.iloc[7:, :].groupby(["z", "x", "y"], observed=True).agg({"d": "sum"})
+ result = merge(df3, df4, left_index=True, right_index=True, how="outer")
+
+ cats = list(range(1, 10))
+ z = Categorical(list(range(1, 5)) + list(range(8, 10)), categories=cats)
+ x = Categorical(list(range(1, 5)) + list(range(8, 10)), categories=cats)
+ y = Categorical([np.nan] * 4 + list(range(8, 10)), categories=cats)
+ idx = MultiIndex.from_arrays([z, x, y], names=["z", "x", "y"])
+ expected = DataFrame(
+ {
+ "d_x": list(map(float, range(1, 5))) + [np.nan] * 2,
+ "d_y": [np.nan] * 4 + list(map(float, range(8, 10))),
+ },
+ index=idx,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+class TestMergeOnIndexes:
+ @pytest.mark.parametrize(
+ "how, sort, expected",
+ [
+ ("inner", False, DataFrame({"a": [20, 10], "b": [200, 100]}, index=[2, 1])),
+ ("inner", True, DataFrame({"a": [10, 20], "b": [100, 200]}, index=[1, 2])),
+ (
+ "left",
+ False,
+ DataFrame({"a": [20, 10, 0], "b": [200, 100, np.nan]}, index=[2, 1, 0]),
+ ),
+ (
+ "left",
+ True,
+ DataFrame({"a": [0, 10, 20], "b": [np.nan, 100, 200]}, index=[0, 1, 2]),
+ ),
+ (
+ "right",
+ False,
+ DataFrame(
+ {"a": [np.nan, 10, 20], "b": [300, 100, 200]}, index=[3, 1, 2]
+ ),
+ ),
+ (
+ "right",
+ True,
+ DataFrame(
+ {"a": [10, 20, np.nan], "b": [100, 200, 300]}, index=[1, 2, 3]
+ ),
+ ),
+ (
+ "outer",
+ False,
+ DataFrame(
+ {"a": [0, 10, 20, np.nan], "b": [np.nan, 100, 200, 300]},
+ index=[0, 1, 2, 3],
+ ),
+ ),
+ (
+ "outer",
+ True,
+ DataFrame(
+ {"a": [0, 10, 20, np.nan], "b": [np.nan, 100, 200, 300]},
+ index=[0, 1, 2, 3],
+ ),
+ ),
+ ],
+ )
+ def test_merge_on_indexes(self, how, sort, expected):
+ left_df = DataFrame({"a": [20, 10, 0]}, index=[2, 1, 0])
+ right_df = DataFrame({"b": [300, 100, 200]}, index=[3, 1, 2])
+ result = merge(
+ left_df, right_df, left_index=True, right_index=True, how=how, sort=sort
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "index",
+ [
+ Index([1, 2, 4], dtype=dtyp, name="index_col")
+ for dtyp in tm.ALL_REAL_NUMPY_DTYPES
+ ]
+ + [
+ CategoricalIndex(["A", "B", "C"], categories=["A", "B", "C"], name="index_col"),
+ RangeIndex(start=0, stop=3, name="index_col"),
+ DatetimeIndex(["2018-01-01", "2018-01-02", "2018-01-03"], name="index_col"),
+ ],
+ ids=lambda x: f"{type(x).__name__}[{x.dtype}]",
+)
+def test_merge_index_types(index):
+ # gh-20777
+ # assert key access is consistent across index types
+ left = DataFrame({"left_data": [1, 2, 3]}, index=index)
+ right = DataFrame({"right_data": [1.0, 2.0, 3.0]}, index=index)
+
+ result = left.merge(right, on=["index_col"])
+
+ expected = DataFrame(
+ {"left_data": [1, 2, 3], "right_data": [1.0, 2.0, 3.0]}, index=index
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "on,left_on,right_on,left_index,right_index,nm",
+ [
+ (["outer", "inner"], None, None, False, False, "B"),
+ (None, None, None, True, True, "B"),
+ (None, ["outer", "inner"], None, False, True, "B"),
+ (None, None, ["outer", "inner"], True, False, "B"),
+ (["outer", "inner"], None, None, False, False, None),
+ (None, None, None, True, True, None),
+ (None, ["outer", "inner"], None, False, True, None),
+ (None, None, ["outer", "inner"], True, False, None),
+ ],
+)
+def test_merge_series(on, left_on, right_on, left_index, right_index, nm):
+ # GH 21220
+ a = DataFrame(
+ {"A": [1, 2, 3, 4]},
+ index=MultiIndex.from_product([["a", "b"], [0, 1]], names=["outer", "inner"]),
+ )
+ b = Series(
+ [1, 2, 3, 4],
+ index=MultiIndex.from_product([["a", "b"], [1, 2]], names=["outer", "inner"]),
+ name=nm,
+ )
+ expected = DataFrame(
+ {"A": [2, 4], "B": [1, 3]},
+ index=MultiIndex.from_product([["a", "b"], [1]], names=["outer", "inner"]),
+ )
+ if nm is not None:
+ result = merge(
+ a,
+ b,
+ on=on,
+ left_on=left_on,
+ right_on=right_on,
+ left_index=left_index,
+ right_index=right_index,
+ )
+ tm.assert_frame_equal(result, expected)
+ else:
+ msg = "Cannot merge a Series without a name"
+ with pytest.raises(ValueError, match=msg):
+ result = merge(
+ a,
+ b,
+ on=on,
+ left_on=left_on,
+ right_on=right_on,
+ left_index=left_index,
+ right_index=right_index,
+ )
+
+
+def test_merge_series_multilevel():
+ # GH#47946
+ # GH 40993: For raising, enforced in 2.0
+ a = DataFrame(
+ {"A": [1, 2, 3, 4]},
+ index=MultiIndex.from_product([["a", "b"], [0, 1]], names=["outer", "inner"]),
+ )
+ b = Series(
+ [1, 2, 3, 4],
+ index=MultiIndex.from_product([["a", "b"], [1, 2]], names=["outer", "inner"]),
+ name=("B", "C"),
+ )
+ with pytest.raises(
+ MergeError, match="Not allowed to merge between different levels"
+ ):
+ merge(a, b, on=["outer", "inner"])
+
+
+@pytest.mark.parametrize(
+ "col1, col2, kwargs, expected_cols",
+ [
+ (0, 0, {"suffixes": ("", "_dup")}, ["0", "0_dup"]),
+ (0, 0, {"suffixes": (None, "_dup")}, [0, "0_dup"]),
+ (0, 0, {"suffixes": ("_x", "_y")}, ["0_x", "0_y"]),
+ (0, 0, {"suffixes": ["_x", "_y"]}, ["0_x", "0_y"]),
+ ("a", 0, {"suffixes": (None, "_y")}, ["a", 0]),
+ (0.0, 0.0, {"suffixes": ("_x", None)}, ["0.0_x", 0.0]),
+ ("b", "b", {"suffixes": (None, "_y")}, ["b", "b_y"]),
+ ("a", "a", {"suffixes": ("_x", None)}, ["a_x", "a"]),
+ ("a", "b", {"suffixes": ("_x", None)}, ["a", "b"]),
+ ("a", "a", {"suffixes": (None, "_x")}, ["a", "a_x"]),
+ (0, 0, {"suffixes": ("_a", None)}, ["0_a", 0]),
+ ("a", "a", {}, ["a_x", "a_y"]),
+ (0, 0, {}, ["0_x", "0_y"]),
+ ],
+)
+def test_merge_suffix(col1, col2, kwargs, expected_cols):
+ # issue: 24782
+ a = DataFrame({col1: [1, 2, 3]})
+ b = DataFrame({col2: [4, 5, 6]})
+
+ expected = DataFrame([[1, 4], [2, 5], [3, 6]], columns=expected_cols)
+
+ result = a.merge(b, left_index=True, right_index=True, **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+ result = merge(a, b, left_index=True, right_index=True, **kwargs)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "how,expected",
+ [
+ (
+ "right",
+ {"A": [100, 200, 300], "B1": [60, 70, np.nan], "B2": [600, 700, 800]},
+ ),
+ (
+ "outer",
+ {
+ "A": [1, 100, 200, 300],
+ "B1": [80, 60, 70, np.nan],
+ "B2": [np.nan, 600, 700, 800],
+ },
+ ),
+ ],
+)
+def test_merge_duplicate_suffix(how, expected):
+ left_df = DataFrame({"A": [100, 200, 1], "B": [60, 70, 80]})
+ right_df = DataFrame({"A": [100, 200, 300], "B": [600, 700, 800]})
+ result = merge(left_df, right_df, on="A", how=how, suffixes=("_x", "_x"))
+ expected = DataFrame(expected)
+ expected.columns = ["A", "B_x", "B_x"]
+
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "col1, col2, suffixes",
+ [("a", "a", (None, None)), ("a", "a", ("", None)), (0, 0, (None, ""))],
+)
+def test_merge_suffix_error(col1, col2, suffixes):
+ # issue: 24782
+ a = DataFrame({col1: [1, 2, 3]})
+ b = DataFrame({col2: [3, 4, 5]})
+
+ # TODO: might reconsider current raise behaviour, see issue 24782
+ msg = "columns overlap but no suffix specified"
+ with pytest.raises(ValueError, match=msg):
+ merge(a, b, left_index=True, right_index=True, suffixes=suffixes)
+
+
+@pytest.mark.parametrize("suffixes", [{"left", "right"}, {"left": 0, "right": 0}])
+def test_merge_suffix_raises(suffixes):
+ a = DataFrame({"a": [1, 2, 3]})
+ b = DataFrame({"b": [3, 4, 5]})
+
+ with pytest.raises(TypeError, match="Passing 'suffixes' as a"):
+ merge(a, b, left_index=True, right_index=True, suffixes=suffixes)
+
+
+TWO_GOT_THREE = "2, got 3" if PY314 else "2"
+
+
+@pytest.mark.parametrize(
+ "col1, col2, suffixes, msg",
+ [
+ (
+ "a",
+ "a",
+ ("a", "b", "c"),
+ (rf"too many values to unpack \(expected {TWO_GOT_THREE}\)"),
+ ),
+ ("a", "a", tuple("a"), r"not enough values to unpack \(expected 2, got 1\)"),
+ ],
+)
+def test_merge_suffix_length_error(col1, col2, suffixes, msg):
+ a = DataFrame({col1: [1, 2, 3]})
+ b = DataFrame({col2: [3, 4, 5]})
+
+ with pytest.raises(ValueError, match=msg):
+ merge(a, b, left_index=True, right_index=True, suffixes=suffixes)
+
+
+@pytest.mark.parametrize("cat_dtype", ["one", "two"])
+@pytest.mark.parametrize("reverse", [True, False])
+def test_merge_equal_cat_dtypes(cat_dtype, reverse):
+ # see gh-22501
+ cat_dtypes = {
+ "one": CategoricalDtype(categories=["a", "b", "c"], ordered=False),
+ "two": CategoricalDtype(categories=["a", "b", "c"], ordered=False),
+ }
+
+ df1 = DataFrame(
+ {"foo": Series(["a", "b", "c"]).astype(cat_dtypes["one"]), "left": [1, 2, 3]}
+ ).set_index("foo")
+
+ data_foo = ["a", "b", "c"]
+ data_right = [1, 2, 3]
+
+ if reverse:
+ data_foo.reverse()
+ data_right.reverse()
+
+ df2 = DataFrame(
+ {"foo": Series(data_foo).astype(cat_dtypes[cat_dtype]), "right": data_right}
+ ).set_index("foo")
+
+ result = df1.merge(df2, left_index=True, right_index=True)
+
+ expected = DataFrame(
+ {
+ "left": [1, 2, 3],
+ "right": [1, 2, 3],
+ "foo": Series(["a", "b", "c"]).astype(cat_dtypes["one"]),
+ }
+ ).set_index("foo")
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_equal_cat_dtypes2():
+ # see gh-22501
+ cat_dtype = CategoricalDtype(categories=["a", "b", "c"], ordered=False)
+
+ # Test Data
+ df1 = DataFrame(
+ {"foo": Series(["a", "b"]).astype(cat_dtype), "left": [1, 2]}
+ ).set_index("foo")
+
+ df2 = DataFrame(
+ {"foo": Series(["a", "b", "c"]).astype(cat_dtype), "right": [3, 2, 1]}
+ ).set_index("foo")
+
+ result = df1.merge(df2, left_index=True, right_index=True)
+
+ expected = DataFrame(
+ {"left": [1, 2], "right": [3, 2], "foo": Series(["a", "b"]).astype(cat_dtype)}
+ ).set_index("foo")
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_on_cat_and_ext_array():
+ # GH 28668
+ right = DataFrame(
+ {"a": Series([pd.Interval(0, 1), pd.Interval(1, 2)], dtype="interval")}
+ )
+ left = right.copy()
+ left["a"] = left["a"].astype("category")
+
+ result = merge(left, right, how="inner", on="a")
+ expected = right.copy()
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_multiindex_columns():
+ # Issue #28518
+ # Verify that merging two dataframes give the expected labels
+ # The original cause of this issue come from a bug lexsort_depth and is tested in
+ # test_lexsort_depth
+
+ letters = ["a", "b", "c", "d"]
+ numbers = ["1", "2", "3"]
+ index = MultiIndex.from_product((letters, numbers), names=["outer", "inner"])
+
+ frame_x = DataFrame(columns=index)
+ frame_x["id"] = ""
+ frame_y = DataFrame(columns=index)
+ frame_y["id"] = ""
+
+ l_suf = "_x"
+ r_suf = "_y"
+ result = frame_x.merge(frame_y, on="id", suffixes=((l_suf, r_suf)))
+
+ # Constructing the expected results
+ tuples = [(letter + l_suf, num) for letter in letters for num in numbers]
+ tuples += [("id", "")]
+ tuples += [(letter + r_suf, num) for letter in letters for num in numbers]
+
+ expected_index = MultiIndex.from_tuples(tuples, names=["outer", "inner"])
+ expected = DataFrame(columns=expected_index)
+
+ tm.assert_frame_equal(result, expected, check_dtype=False)
+
+
+def test_merge_datetime_upcast_dtype():
+ # https://github.com/pandas-dev/pandas/issues/31208
+ df1 = DataFrame({"x": ["a", "b", "c"], "y": ["1", "2", "4"]})
+ df2 = DataFrame(
+ {"y": ["1", "2", "3"], "z": pd.to_datetime(["2000", "2001", "2002"])}
+ )
+ result = merge(df1, df2, how="left", on="y")
+ expected = DataFrame(
+ {
+ "x": ["a", "b", "c"],
+ "y": ["1", "2", "4"],
+ "z": pd.to_datetime(["2000", "2001", "NaT"]),
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("n_categories", [5, 128])
+def test_categorical_non_unique_monotonic(n_categories):
+ # GH 28189
+ # With n_categories as 5, we test the int8 case is hit in libjoin,
+ # with n_categories as 128 we test the int16 case.
+ left_index = CategoricalIndex([0, *list(range(n_categories))])
+ df1 = DataFrame(range(n_categories + 1), columns=["value"], index=left_index)
+ df2 = DataFrame(
+ [[6]],
+ columns=["value"],
+ index=CategoricalIndex([0], categories=list(range(n_categories))),
+ )
+
+ result = merge(df1, df2, how="left", left_index=True, right_index=True)
+ expected = DataFrame(
+ [[i, 6.0] if i < 2 else [i, np.nan] for i in range(n_categories + 1)],
+ columns=["value_x", "value_y"],
+ index=left_index,
+ )
+ tm.assert_frame_equal(expected, result)
+
+
+def test_merge_join_categorical_multiindex():
+ # From issue 16627
+ a = {
+ "Cat1": Categorical(["a", "b", "a", "c", "a", "b"], ["a", "b", "c"]),
+ "Int1": [0, 1, 0, 1, 0, 0],
+ }
+ a = DataFrame(a)
+
+ b = {
+ "Cat": Categorical(["a", "b", "c", "a", "b", "c"], ["a", "b", "c"]),
+ "Int": [0, 0, 0, 1, 1, 1],
+ "Factor": [1.1, 1.2, 1.3, 1.4, 1.5, 1.6],
+ }
+ b = DataFrame(b).set_index(["Cat", "Int"])["Factor"]
+
+ expected = merge(
+ a,
+ b.reset_index(),
+ left_on=["Cat1", "Int1"],
+ right_on=["Cat", "Int"],
+ how="left",
+ )
+ expected = expected.drop(["Cat", "Int"], axis=1)
+ result = a.join(b, on=["Cat1", "Int1"])
+ tm.assert_frame_equal(expected, result)
+
+ # Same test, but with ordered categorical
+ a = {
+ "Cat1": Categorical(
+ ["a", "b", "a", "c", "a", "b"], ["b", "a", "c"], ordered=True
+ ),
+ "Int1": [0, 1, 0, 1, 0, 0],
+ }
+ a = DataFrame(a)
+
+ b = {
+ "Cat": Categorical(
+ ["a", "b", "c", "a", "b", "c"], ["b", "a", "c"], ordered=True
+ ),
+ "Int": [0, 0, 0, 1, 1, 1],
+ "Factor": [1.1, 1.2, 1.3, 1.4, 1.5, 1.6],
+ }
+ b = DataFrame(b).set_index(["Cat", "Int"])["Factor"]
+
+ expected = merge(
+ a,
+ b.reset_index(),
+ left_on=["Cat1", "Int1"],
+ right_on=["Cat", "Int"],
+ how="left",
+ )
+ expected = expected.drop(["Cat", "Int"], axis=1)
+ result = a.join(b, on=["Cat1", "Int1"])
+ tm.assert_frame_equal(expected, result)
+
+
+@pytest.mark.parametrize("func", ["merge", "merge_asof"])
+@pytest.mark.parametrize(
+ ("kwargs", "err_msg"),
+ [
+ ({"left_on": "a", "left_index": True}, ["left_on", "left_index"]),
+ ({"right_on": "a", "right_index": True}, ["right_on", "right_index"]),
+ ],
+)
+def test_merge_join_cols_error_reporting_duplicates(func, kwargs, err_msg):
+ # GH: 16228
+ left = DataFrame({"a": [1, 2], "b": [3, 4]})
+ right = DataFrame({"a": [1, 1], "c": [5, 6]})
+ msg = rf'Can only pass argument "{err_msg[0]}" OR "{err_msg[1]}" not both\.'
+ with pytest.raises(MergeError, match=msg):
+ getattr(pd, func)(left, right, **kwargs)
+
+
+@pytest.mark.parametrize("func", ["merge", "merge_asof"])
+@pytest.mark.parametrize(
+ ("kwargs", "err_msg"),
+ [
+ ({"left_on": "a"}, ["right_on", "right_index"]),
+ ({"right_on": "a"}, ["left_on", "left_index"]),
+ ],
+)
+def test_merge_join_cols_error_reporting_missing(func, kwargs, err_msg):
+ # GH: 16228
+ left = DataFrame({"a": [1, 2], "b": [3, 4]})
+ right = DataFrame({"a": [1, 1], "c": [5, 6]})
+ msg = rf'Must pass "{err_msg[0]}" OR "{err_msg[1]}"\.'
+ with pytest.raises(MergeError, match=msg):
+ getattr(pd, func)(left, right, **kwargs)
+
+
+@pytest.mark.parametrize("func", ["merge", "merge_asof"])
+@pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"right_index": True},
+ {"left_index": True},
+ ],
+)
+def test_merge_join_cols_error_reporting_on_and_index(func, kwargs):
+ # GH: 16228
+ left = DataFrame({"a": [1, 2], "b": [3, 4]})
+ right = DataFrame({"a": [1, 1], "c": [5, 6]})
+ msg = (
+ r'Can only pass argument "on" OR "left_index" '
+ r'and "right_index", not a combination of both\.'
+ )
+ with pytest.raises(MergeError, match=msg):
+ getattr(pd, func)(left, right, on="a", **kwargs)
+
+
+def test_merge_right_left_index():
+ # GH#38616
+ left = DataFrame({"x": [1, 1], "z": ["foo", "foo"]})
+ right = DataFrame({"x": [1, 1], "z": ["foo", "foo"]})
+ result = merge(left, right, how="right", left_index=True, right_on="x")
+ expected = DataFrame(
+ {
+ "x": [1, 1],
+ "x_x": [1, 1],
+ "z_x": ["foo", "foo"],
+ "x_y": [1, 1],
+ "z_y": ["foo", "foo"],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_result_empty_index_and_on():
+ # GH#33814
+ df1 = DataFrame({"a": [1], "b": [2]}).set_index(["a", "b"])
+ df2 = DataFrame({"b": [1]}).set_index(["b"])
+ expected = DataFrame({"a": [], "b": []}, dtype=np.int64).set_index(["a", "b"])
+ result = merge(df1, df2, left_on=["b"], right_index=True)
+ tm.assert_frame_equal(result, expected)
+
+ result = merge(df2, df1, left_index=True, right_on=["b"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_suffixes_produce_dup_columns_raises():
+ # GH#22818; Enforced in 2.0
+ left = DataFrame({"a": [1, 2, 3], "b": 1, "b_x": 2})
+ right = DataFrame({"a": [1, 2, 3], "b": 2})
+
+ with pytest.raises(MergeError, match="Passing 'suffixes' which cause duplicate"):
+ merge(left, right, on="a")
+
+ with pytest.raises(MergeError, match="Passing 'suffixes' which cause duplicate"):
+ merge(right, left, on="a", suffixes=("_y", "_x"))
+
+
+def test_merge_duplicate_columns_with_suffix_no_warning():
+ # GH#22818
+ # Do not raise warning when duplicates are caused by duplicates in origin
+ left = DataFrame([[1, 1, 1], [2, 2, 2]], columns=["a", "b", "b"])
+ right = DataFrame({"a": [1, 3], "b": 2})
+ result = merge(left, right, on="a")
+ expected = DataFrame([[1, 1, 1, 2]], columns=["a", "b_x", "b_x", "b_y"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_duplicate_columns_with_suffix_causing_another_duplicate_raises():
+ # GH#22818, Enforced in 2.0
+ # This should raise warning because suffixes cause another collision
+ left = DataFrame([[1, 1, 1, 1], [2, 2, 2, 2]], columns=["a", "b", "b", "b_x"])
+ right = DataFrame({"a": [1, 3], "b": 2})
+ with pytest.raises(MergeError, match="Passing 'suffixes' which cause duplicate"):
+ merge(left, right, on="a")
+
+
+def test_merge_string_float_column_result():
+ # GH 13353
+ df1 = DataFrame([[1, 2], [3, 4]], columns=Index(["a", 114.0]))
+ df2 = DataFrame([[9, 10], [11, 12]], columns=["x", "y"])
+ result = merge(df2, df1, how="inner", left_index=True, right_index=True)
+ expected = DataFrame(
+ [[9, 10, 1, 2], [11, 12, 3, 4]], columns=Index(["x", "y", "a", 114.0])
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_mergeerror_on_left_index_mismatched_dtypes():
+ # GH 22449
+ df_1 = DataFrame(data=["X"], columns=["C"], index=[22])
+ df_2 = DataFrame(data=["X"], columns=["C"], index=[999])
+ with pytest.raises(MergeError, match="Can only pass argument"):
+ merge(df_1, df_2, on=["C"], left_index=True)
+
+
+def test_merge_on_left_categoricalindex():
+ # GH#48464 don't raise when left_on is a CategoricalIndex
+ ci = CategoricalIndex(range(3))
+
+ right = DataFrame({"A": ci, "B": range(3)})
+ left = DataFrame({"C": range(3, 6)})
+
+ res = merge(left, right, left_on=ci, right_on="A")
+ expected = merge(left, right, left_on=ci._data, right_on="A")
+ tm.assert_frame_equal(res, expected)
+
+
+@pytest.mark.parametrize("dtype", [None, "Int64"])
+def test_merge_outer_with_NaN(dtype):
+ # GH#43550
+ item = np.nan if dtype is None else pd.NA
+ left = DataFrame({"key": [1, 2], "col1": [1, 2]}, dtype=dtype)
+ right = DataFrame({"key": [item, item], "col2": [3, 4]}, dtype=dtype)
+ result = merge(left, right, on="key", how="outer")
+ expected = DataFrame(
+ {
+ "key": [1, 2, item, item],
+ "col1": [1, 2, item, item],
+ "col2": [item, item, 3, 4],
+ },
+ dtype=dtype,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ # switch left and right
+ result = merge(right, left, on="key", how="outer")
+ expected = DataFrame(
+ {
+ "key": [1, 2, item, item],
+ "col2": [item, item, 3, 4],
+ "col1": [1, 2, item, item],
+ },
+ dtype=dtype,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_different_index_names():
+ # GH#45094
+ left = DataFrame({"a": [1]}, index=Index([1], name="c"))
+ right = DataFrame({"a": [1]}, index=Index([1], name="d"))
+ result = merge(left, right, left_on="c", right_on="d")
+ expected = DataFrame({"a_x": [1], "a_y": 1})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_ea(any_numeric_ea_dtype, join_type):
+ # GH#44240
+ left = DataFrame({"a": [1, 2, 3], "b": 1}, dtype=any_numeric_ea_dtype)
+ right = DataFrame({"a": [1, 2, 3], "c": 2}, dtype=any_numeric_ea_dtype)
+ result = left.merge(right, how=join_type)
+ expected = DataFrame({"a": [1, 2, 3], "b": 1, "c": 2}, dtype=any_numeric_ea_dtype)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_ea_and_non_ea(any_numeric_ea_dtype, join_type):
+ # GH#44240
+ left = DataFrame({"a": [1, 2, 3], "b": 1}, dtype=any_numeric_ea_dtype)
+ right = DataFrame({"a": [1, 2, 3], "c": 2}, dtype=any_numeric_ea_dtype.lower())
+ result = left.merge(right, how=join_type)
+ expected = DataFrame(
+ {
+ "a": Series([1, 2, 3], dtype=any_numeric_ea_dtype),
+ "b": Series([1, 1, 1], dtype=any_numeric_ea_dtype),
+ "c": Series([2, 2, 2], dtype=any_numeric_ea_dtype.lower()),
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("dtype", ["int64", "int64[pyarrow]"])
+def test_merge_arrow_and_numpy_dtypes(dtype):
+ # GH#52406
+ pytest.importorskip("pyarrow")
+ df = DataFrame({"a": [1, 2]}, dtype=dtype)
+ df2 = DataFrame({"a": [1, 2]}, dtype="int64[pyarrow]")
+ result = df.merge(df2)
+ expected = df.copy()
+ tm.assert_frame_equal(result, expected)
+
+ result = df2.merge(df)
+ expected = df2.copy()
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("tz", [None, "America/Chicago"])
+def test_merge_datetime_different_resolution(tz, join_type):
+ # https://github.com/pandas-dev/pandas/issues/53200
+ vals = [
+ pd.Timestamp(2023, 5, 12, tz=tz),
+ pd.Timestamp(2023, 5, 13, tz=tz),
+ pd.Timestamp(2023, 5, 14, tz=tz),
+ ]
+ df1 = DataFrame({"t": vals[:2], "a": [1.0, 2.0]})
+ df1["t"] = df1["t"].dt.as_unit("ns")
+ df2 = DataFrame({"t": vals[1:], "b": [1.0, 2.0]})
+ df2["t"] = df2["t"].dt.as_unit("s")
+
+ expected = DataFrame({"t": vals, "a": [1.0, 2.0, np.nan], "b": [np.nan, 1.0, 2.0]})
+ expected["t"] = expected["t"].dt.as_unit("ns")
+ if join_type == "inner":
+ expected = expected.iloc[[1]].reset_index(drop=True)
+ elif join_type == "left":
+ expected = expected.iloc[[0, 1]]
+ elif join_type == "right":
+ expected = expected.iloc[[1, 2]].reset_index(drop=True)
+
+ result = df1.merge(df2, on="t", how=join_type)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_multiindex_single_level():
+ # GH52331
+ df = DataFrame({"col": ["A", "B"]})
+ df2 = DataFrame(
+ data={"b": [100]},
+ index=MultiIndex.from_tuples([("A",), ("C",)], names=["col"]),
+ )
+ expected = DataFrame({"col": ["A", "B"], "b": [100, np.nan]})
+
+ result = df.merge(df2, left_on=["col"], right_index=True, how="left")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_multiindex_reset_index_mixed():
+ # GH#62150
+ df = DataFrame(
+ {("column_1", ""): [1, 1], ("column_2", ""): [2, 2]},
+ index=MultiIndex.from_arrays(
+ [[1, 1], ["metadata_1", "metadata_2"]], names=["index", "metadata"]
+ ),
+ )
+
+ df2 = DataFrame(
+ data=[1, 1],
+ index=Index([1, 1], name="index"),
+ columns=MultiIndex.from_product([["new_data"], [""]]),
+ )
+
+ with tm.assert_produces_warning(pd.errors.PerformanceWarning):
+ result = df.reset_index().merge(df2.reset_index(), on="index")
+
+ expected = DataFrame(
+ {
+ ("index", ""): [1, 1, 1, 1],
+ ("metadata", ""): ["metadata_1", "metadata_1", "metadata_2", "metadata_2"],
+ ("column_1", ""): [1, 1, 1, 1],
+ ("column_2", ""): [2, 2, 2, 2],
+ ("new_data", ""): [1, 1, 1, 1],
+ }
+ )
+ expected.columns = MultiIndex.from_tuples(expected.columns)
+
+ tm.assert_frame_equal(result, expected)
+
+ result2 = df.reset_index().merge(df2.reset_index(), on=[("index", "")])
+ tm.assert_frame_equal(result2, expected)
+
+
+@pytest.mark.parametrize("on_index", [True, False])
+@pytest.mark.parametrize("left_unique", [True, False])
+@pytest.mark.parametrize("left_monotonic", [True, False])
+@pytest.mark.parametrize("right_unique", [True, False])
+@pytest.mark.parametrize("right_monotonic", [True, False])
+def test_merge_combinations(
+ join_type,
+ sort,
+ on_index,
+ left_unique,
+ left_monotonic,
+ right_unique,
+ right_monotonic,
+):
+ how = join_type
+ # GH 54611
+ left = [2, 3]
+ if left_unique:
+ left.append(4 if left_monotonic else 1)
+ else:
+ left.append(3 if left_monotonic else 2)
+
+ right = [2, 3]
+ if right_unique:
+ right.append(4 if right_monotonic else 1)
+ else:
+ right.append(3 if right_monotonic else 2)
+
+ left = DataFrame({"key": left})
+ right = DataFrame({"key": right})
+
+ if on_index:
+ left = left.set_index("key")
+ right = right.set_index("key")
+ on_kwargs = {"left_index": True, "right_index": True}
+ else:
+ on_kwargs = {"on": "key"}
+
+ result = merge(left, right, how=how, sort=sort, **on_kwargs)
+
+ if on_index:
+ left = left.reset_index()
+ right = right.reset_index()
+
+ if how in ["left", "right", "inner"]:
+ if how in ["left", "inner"]:
+ expected, other, other_unique = left, right, right_unique
+ else:
+ expected, other, other_unique = right, left, left_unique
+ if how == "inner":
+ keep_values = set(left["key"].values).intersection(right["key"].values)
+ keep_mask = expected["key"].isin(keep_values)
+ expected = expected[keep_mask]
+ if sort:
+ expected = expected.sort_values("key")
+ if not other_unique:
+ other_value_counts = other["key"].value_counts()
+ repeats = other_value_counts.reindex(expected["key"].values, fill_value=1)
+ repeats = repeats.astype(np.intp)
+ expected = expected["key"].repeat(repeats.values)
+ expected = expected.to_frame()
+ elif how == "outer":
+ left_counts = left["key"].value_counts()
+ right_counts = right["key"].value_counts()
+ expected_counts = left_counts.mul(right_counts, fill_value=1)
+ expected_counts = expected_counts.astype(np.intp)
+ expected = expected_counts.index.values.repeat(expected_counts.values)
+ expected = DataFrame({"key": expected})
+ expected = expected.sort_values("key")
+
+ if on_index:
+ expected = expected.set_index("key")
+ else:
+ expected = expected.reset_index(drop=True)
+
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_ea_int_and_float_numpy():
+ # GH#46178
+ df1 = DataFrame([1.0, pd.NA], dtype=pd.Int64Dtype())
+ df2 = DataFrame([1.5])
+ expected = DataFrame(columns=[0], dtype="Int64")
+
+ with tm.assert_produces_warning(UserWarning, match="You are merging"):
+ result = df1.merge(df2)
+ tm.assert_frame_equal(result, expected)
+
+ with tm.assert_produces_warning(UserWarning, match="You are merging"):
+ result = df2.merge(df1)
+ tm.assert_frame_equal(result, expected.astype("float64"))
+
+ df2 = DataFrame([1.0])
+ expected = DataFrame([1], columns=[0], dtype="Int64")
+ result = df1.merge(df2)
+ tm.assert_frame_equal(result, expected)
+
+ result = df2.merge(df1)
+ tm.assert_frame_equal(result, expected.astype("float64"))
+
+
+def test_merge_arrow_string_index(any_string_dtype):
+ # GH#54894
+ pytest.importorskip("pyarrow")
+ left = DataFrame({"a": ["a", "b"]}, dtype=any_string_dtype)
+ right = DataFrame({"b": 1}, index=Index(["a", "c"], dtype=any_string_dtype))
+ result = left.merge(right, left_on="a", right_index=True, how="left")
+ expected = DataFrame(
+ {"a": Series(["a", "b"], dtype=any_string_dtype), "b": [1, np.nan]}
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("left_empty", [True, False])
+@pytest.mark.parametrize("right_empty", [True, False])
+def test_merge_empty_frames_column_order(left_empty, right_empty):
+ # GH 51929
+ df1 = DataFrame(1, index=[0], columns=["A", "B"])
+ df2 = DataFrame(1, index=[0], columns=["A", "C", "D"])
+
+ if left_empty:
+ df1 = df1.iloc[:0]
+ if right_empty:
+ df2 = df2.iloc[:0]
+
+ result = merge(df1, df2, on=["A"], how="outer")
+ expected = DataFrame(1, index=range(1), columns=["A", "B", "C", "D"])
+ if left_empty and right_empty:
+ expected = expected.iloc[:0]
+ elif left_empty:
+ expected["B"] = np.nan
+ elif right_empty:
+ expected[["C", "D"]] = np.nan
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("how", ["left", "right", "inner", "outer"])
+def test_merge_datetime_and_timedelta(how):
+ left = DataFrame({"key": Series([1, None], dtype="datetime64[ns]")})
+ right = DataFrame({"key": Series([1], dtype="timedelta64[ns]")})
+
+ msg = (
+ f"You are trying to merge on {left['key'].dtype} and {right['key'].dtype} "
+ "columns for key 'key'. If you wish to proceed you should use pd.concat"
+ )
+ with pytest.raises(ValueError, match=re.escape(msg)):
+ left.merge(right, on="key", how=how)
+
+ msg = (
+ f"You are trying to merge on {right['key'].dtype} and {left['key'].dtype} "
+ "columns for key 'key'. If you wish to proceed you should use pd.concat"
+ )
+ with pytest.raises(ValueError, match=re.escape(msg)):
+ right.merge(left, on="key", how=how)
+
+
+def test_merge_on_all_nan_column():
+ # GH#59421
+ left = DataFrame({"x": [1, 2, 3], "y": [np.nan, np.nan, np.nan], "z": [4, 5, 6]})
+ right = DataFrame({"x": [1, 2, 3], "y": [np.nan, np.nan, np.nan], "zz": [4, 5, 6]})
+ result = left.merge(right, on=["x", "y"], how="outer")
+ # Should not trigger array bounds eerror with bounds checking or asan enabled.
+ expected = DataFrame(
+ {"x": [1, 2, 3], "y": [np.nan, np.nan, np.nan], "z": [4, 5, 6], "zz": [4, 5, 6]}
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("suffixes", [("_dup", ""), ("", "_dup")])
+def test_merge_for_suffix_collisions(suffixes):
+ # GH#61402
+ df1 = DataFrame({"col1": [1], "col2": [2]})
+ df2 = DataFrame({"col1": [1], "col2": [2], "col2_dup": [3]})
+ with pytest.raises(MergeError, match="duplicate columns"):
+ merge(df1, df2, on="col1", suffixes=suffixes)
+
+
+def test_merge_categorical_key_recursion():
+ # GH#56376
+ lt = CategoricalDtype(categories=np.asarray([1, 2, 3], dtype="int64"))
+ rt = CategoricalDtype(categories=np.asarray([1, 2, 3], dtype="float64"))
+ left = DataFrame({"key": Series([1, 2], dtype=lt)})
+ right = DataFrame({"key": Series([1, 3], dtype=rt)})
+
+ result = left.merge(right, on="key", how="outer")
+ expected = left.astype("int64").merge(
+ right.astype("float64"), on="key", how="outer"
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_pyarrow_datetime_duplicates():
+ # GH#61926
+ pytest.importorskip("pyarrow")
+
+ t = pd.date_range("2025-07-06", periods=3, freq="h")
+ df1 = DataFrame({"time": t, "val1": [1, 2, 3]})
+ df1 = df1.convert_dtypes(dtype_backend="pyarrow")
+
+ df2 = DataFrame({"time": t.repeat(2), "val2": [10, 20, 30, 40, 50, 60]})
+ df2 = df2.convert_dtypes(dtype_backend="pyarrow")
+
+ result = merge(df1, df2, on="time", how="left")
+
+ expected = DataFrame(
+ {
+ "time": t.repeat(2),
+ "val1": [1, 1, 2, 2, 3, 3],
+ "val2": [10, 20, 30, 40, 50, 60],
+ }
+ )
+ expected = expected.convert_dtypes(dtype_backend="pyarrow")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_right_on_and_right_index():
+ df1 = DataFrame({"col": [1, 2, 3]})
+ df2 = DataFrame({"col": [2, 3, 4]})
+
+ with pytest.raises(pd.errors.MergeError):
+ df1.merge(df2, left_on="col", right_on="col", right_index=True)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_antijoin.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_antijoin.py
new file mode 100644
index 0000000000000000000000000000000000000000..006622c6e5e94cdddc9dc50f96114bde28a73525
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_antijoin.py
@@ -0,0 +1,280 @@
+import numpy as np
+import pytest
+
+import pandas.util._test_decorators as td
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ MultiIndex,
+)
+import pandas._testing as tm
+from pandas.core.reshape.merge import merge
+
+
+def test_merge_antijoin():
+ # GH#42916
+ left = DataFrame({"A": [1, 2, 3]}, index=["a", "b", "c"])
+ right = DataFrame({"B": [1, 2, 4]}, index=["a", "b", "d"])
+
+ result = merge(left, right, how="left_anti", left_index=True, right_index=True)
+ expected = DataFrame({"A": [3], "B": [np.nan]}, index=["c"])
+ tm.assert_frame_equal(result, expected)
+
+ result = merge(left, right, how="right_anti", left_index=True, right_index=True)
+ expected = DataFrame({"A": [np.nan], "B": [4]}, index=["d"])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_antijoin_on_different_columns():
+ left = DataFrame({"A": [1.0, 2.0, 3.0], "B": ["a", "b", "c"]}).astype({"B": object})
+ right = DataFrame({"C": [1.0, 2.0, 4.0], "D": ["a", "d", "b"]}).astype(
+ {"D": object}
+ )
+
+ result = merge(left, right, how="left_anti", left_on="B", right_on="D")
+ expected = DataFrame(
+ {
+ "A": [3.0],
+ "B": ["c"],
+ "C": [np.nan],
+ "D": [np.nan],
+ },
+ index=[2],
+ ).astype({"B": object, "D": object})
+ tm.assert_frame_equal(result, expected)
+
+ result = merge(left, right, how="right_anti", left_on="B", right_on="D")
+ expected = DataFrame(
+ {
+ "A": [np.nan],
+ "B": [np.nan],
+ "C": [2.0],
+ "D": ["d"],
+ },
+ index=[1],
+ ).astype({"B": object, "D": object})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_antijoin_nonunique_keys():
+ left = DataFrame({"A": [1.0, 2.0, 3.0], "B": ["a", "b", "b"]}).astype({"B": object})
+ right = DataFrame({"C": [1.0, 2.0, 4.0], "D": ["b", "d", "d"]}).astype(
+ {"D": object}
+ )
+
+ result = merge(left, right, how="left_anti", left_on="B", right_on="D")
+ expected = DataFrame(
+ {
+ "A": [1.0],
+ "B": ["a"],
+ "C": [np.nan],
+ "D": [np.nan],
+ },
+ index=[0],
+ ).astype({"B": object, "D": object})
+ tm.assert_frame_equal(result, expected)
+
+ result = merge(left, right, how="right_anti", left_on="B", right_on="D")
+ expected = DataFrame(
+ {
+ "A": [np.nan, np.nan],
+ "B": [np.nan, np.nan],
+ "C": [2.0, 4.0],
+ "D": ["d", "d"],
+ },
+ index=[2, 3],
+ ).astype({"B": object, "D": object})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_antijoin_same_df():
+ left = DataFrame({"A": [1, 2, 3]}, index=["a", "b", "c"], dtype=np.int64)
+ result = merge(left, left, how="left_anti", left_index=True, right_index=True)
+ expected = DataFrame([], columns=["A_x", "A_y"], dtype=np.int64)
+ tm.assert_frame_equal(result, expected, check_index_type=False)
+
+
+def test_merge_antijoin_nans():
+ left = DataFrame({"A": [1.0, 2.0, np.nan], "C": ["a", "b", "c"]}).astype(
+ {"C": object}
+ )
+ right = DataFrame({"A": [3.0, 2.0, np.nan], "D": ["d", "e", "f"]}).astype(
+ {"D": object}
+ )
+ result = merge(left, right, how="left_anti", on="A")
+ expected = DataFrame({"A": [1.0], "C": ["a"], "D": [np.nan]}).astype(
+ {"C": object, "D": object}
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_antijoin_on_datetime64tz():
+ # GH11405
+ left = DataFrame(
+ {
+ "key": pd.date_range("20151010", periods=2, tz="US/Eastern"),
+ "value": [1.0, 2.0],
+ }
+ )
+ right = DataFrame(
+ {
+ "key": pd.date_range("20151011", periods=3, tz="US/Eastern"),
+ "value": [1.0, 2.0, 3.0],
+ }
+ )
+
+ expected = DataFrame(
+ {
+ "key": pd.date_range("20151010", periods=1, tz="US/Eastern"),
+ "value_x": [1.0],
+ "value_y": [np.nan],
+ },
+ index=[0],
+ )
+ result = merge(left, right, on="key", how="left_anti")
+ tm.assert_frame_equal(result, expected)
+
+ expected = DataFrame(
+ {
+ "key": pd.date_range("20151012", periods=2, tz="US/Eastern"),
+ "value_x": [np.nan, np.nan],
+ "value_y": [2.0, 3.0],
+ },
+ index=[1, 2],
+ )
+ result = merge(left, right, on="key", how="right_anti")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_antijoin_multiindex():
+ left = DataFrame(
+ {
+ "A": [1, 2, 3],
+ "B": [4, 5, 6],
+ },
+ index=MultiIndex.from_tuples(
+ [("a", "x"), ("b", "y"), ("c", "z")], names=["first", "second"]
+ ),
+ )
+ right = DataFrame(
+ {
+ "C": [7, 8, 9],
+ "D": [10, 11, 12],
+ },
+ index=MultiIndex.from_tuples(
+ [("a", "x"), ("b", "y"), ("c", "w")], names=["first", "second"]
+ ),
+ )
+
+ result = merge(left, right, how="left_anti", left_index=True, right_index=True)
+ expected = DataFrame(
+ {
+ "A": [3],
+ "B": [6],
+ "C": [np.nan],
+ "D": [np.nan],
+ },
+ index=MultiIndex.from_tuples([("c", "z")], names=["first", "second"]),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = merge(left, right, how="right_anti", left_index=True, right_index=True)
+ expected = DataFrame(
+ {
+ "A": [np.nan],
+ "B": [np.nan],
+ "C": [9],
+ "D": [12],
+ },
+ index=MultiIndex.from_tuples([("c", "w")], names=["first", "second"]),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "dtype",
+ [
+ "Int64",
+ pytest.param("int64[pyarrow]", marks=td.skip_if_no("pyarrow")),
+ pytest.param("timestamp[s][pyarrow]", marks=td.skip_if_no("pyarrow")),
+ pytest.param("string[pyarrow]", marks=td.skip_if_no("pyarrow")),
+ ],
+)
+def test_merge_antijoin_extension_dtype(dtype):
+ left = DataFrame(
+ {
+ "join_col": [1, 3, 5],
+ "left_val": [1, 2, 3],
+ }
+ )
+ right = DataFrame(
+ {
+ "join_col": [2, 3, 4],
+ "right_val": [1, 2, 3],
+ }
+ )
+ left = left.astype({"join_col": dtype})
+ right = right.astype({"join_col": dtype})
+ result = merge(left, right, how="left_anti", on="join_col")
+ expected = DataFrame(
+ {
+ "join_col": [1, 5],
+ "left_val": [1, 3],
+ "right_val": [np.nan, np.nan],
+ },
+ index=[0, 2],
+ )
+ expected = expected.astype({"join_col": dtype})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_antijoin_empty_dataframe():
+ left = DataFrame({"A": [], "B": []})
+ right = DataFrame({"C": [], "D": []})
+
+ result = merge(left, right, how="left_anti", left_on="A", right_on="C")
+ expected = DataFrame({"A": [], "B": [], "C": [], "D": []})
+ tm.assert_frame_equal(result, expected)
+
+ result = merge(left, right, how="right_anti", left_on="A", right_on="C")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_antijoin_no_common_elements():
+ left = DataFrame({"A": [1, 2, 3]})
+ right = DataFrame({"B": [4, 5, 6]})
+
+ result = merge(left, right, how="left_anti", left_on="A", right_on="B")
+ expected = DataFrame({"A": [1, 2, 3], "B": [np.nan, np.nan, np.nan]})
+ tm.assert_frame_equal(result, expected)
+
+ result = merge(left, right, how="right_anti", left_on="A", right_on="B")
+ expected = DataFrame({"A": [np.nan, np.nan, np.nan], "B": [4, 5, 6]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_antijoin_with_null_values():
+ left = DataFrame({"A": [1.0, 2.0, None, 4.0]})
+ right = DataFrame({"B": [2.0, None, 5.0]})
+
+ result = merge(left, right, how="left_anti", left_on="A", right_on="B")
+ expected = DataFrame({"A": [1.0, 4.0], "B": [np.nan, np.nan]}, index=[0, 3])
+ tm.assert_frame_equal(result, expected)
+
+ result = merge(left, right, how="right_anti", left_on="A", right_on="B")
+ expected = DataFrame({"A": [np.nan], "B": [5.0]}, index=[2])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_antijoin_with_mixed_dtypes():
+ left = DataFrame({"A": [1, "2", 3.0]})
+ right = DataFrame({"B": ["2", 3.0, 4]})
+
+ result = merge(left, right, how="left_anti", left_on="A", right_on="B")
+ expected = DataFrame({"A": [1], "B": [np.nan]}, dtype=object)
+ tm.assert_frame_equal(result, expected)
+
+ result = merge(left, right, how="right_anti", left_on="A", right_on="B")
+ expected = DataFrame({"A": [np.nan], "B": [4]}, dtype=object, index=[2])
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_asof.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_asof.py
new file mode 100644
index 0000000000000000000000000000000000000000..4aff226c77593bd5fc3a0af376f2cd4c80584aab
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_asof.py
@@ -0,0 +1,3634 @@
+import datetime
+
+import numpy as np
+import pytest
+
+import pandas.util._test_decorators as td
+
+import pandas as pd
+from pandas import (
+ Index,
+ Timedelta,
+ merge_asof,
+ option_context,
+ to_datetime,
+)
+import pandas._testing as tm
+from pandas.core.reshape.merge import MergeError
+
+
+class TestAsOfMerge:
+ def prep_data(self, df, dedupe=False):
+ if dedupe:
+ df = df.drop_duplicates(["time", "ticker"], keep="last").reset_index(
+ drop=True
+ )
+ df.time = to_datetime(df.time)
+ return df
+
+ @pytest.fixture
+ def trades(self):
+ df = pd.DataFrame(
+ [
+ ["20160525 13:30:00.023", "MSFT", "51.9500", "75", "NASDAQ"],
+ ["20160525 13:30:00.038", "MSFT", "51.9500", "155", "NASDAQ"],
+ ["20160525 13:30:00.048", "GOOG", "720.7700", "100", "NASDAQ"],
+ ["20160525 13:30:00.048", "GOOG", "720.9200", "100", "NASDAQ"],
+ ["20160525 13:30:00.048", "GOOG", "720.9300", "200", "NASDAQ"],
+ ["20160525 13:30:00.048", "GOOG", "720.9300", "300", "NASDAQ"],
+ ["20160525 13:30:00.048", "GOOG", "720.9300", "600", "NASDAQ"],
+ ["20160525 13:30:00.048", "GOOG", "720.9300", "44", "NASDAQ"],
+ ["20160525 13:30:00.074", "AAPL", "98.6700", "478343", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.6700", "478343", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.6600", "6", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.6500", "30", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.6500", "75", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.6500", "20", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.6500", "35", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.6500", "10", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.5500", "6", "ARCA"],
+ ["20160525 13:30:00.075", "AAPL", "98.5500", "6", "ARCA"],
+ ["20160525 13:30:00.076", "AAPL", "98.5600", "1000", "ARCA"],
+ ["20160525 13:30:00.076", "AAPL", "98.5600", "200", "ARCA"],
+ ["20160525 13:30:00.076", "AAPL", "98.5600", "300", "ARCA"],
+ ["20160525 13:30:00.076", "AAPL", "98.5600", "400", "ARCA"],
+ ["20160525 13:30:00.076", "AAPL", "98.5600", "600", "ARCA"],
+ ["20160525 13:30:00.076", "AAPL", "98.5600", "200", "ARCA"],
+ ["20160525 13:30:00.078", "MSFT", "51.9500", "783", "NASDAQ"],
+ ["20160525 13:30:00.078", "MSFT", "51.9500", "100", "NASDAQ"],
+ ["20160525 13:30:00.078", "MSFT", "51.9500", "100", "NASDAQ"],
+ ],
+ columns="time,ticker,price,quantity,marketCenter".split(","),
+ )
+ df["price"] = df["price"].astype("float64")
+ df["quantity"] = df["quantity"].astype("int64")
+ return self.prep_data(df)
+
+ @pytest.fixture
+ def quotes(self):
+ df = pd.DataFrame(
+ [
+ ["20160525 13:30:00.023", "GOOG", "720.50", "720.93"],
+ ["20160525 13:30:00.023", "MSFT", "51.95", "51.95"],
+ ["20160525 13:30:00.041", "MSFT", "51.95", "51.95"],
+ ["20160525 13:30:00.048", "GOOG", "720.50", "720.93"],
+ ["20160525 13:30:00.048", "GOOG", "720.50", "720.93"],
+ ["20160525 13:30:00.048", "GOOG", "720.50", "720.93"],
+ ["20160525 13:30:00.048", "GOOG", "720.50", "720.93"],
+ ["20160525 13:30:00.072", "GOOG", "720.50", "720.88"],
+ ["20160525 13:30:00.075", "AAPL", "98.55", "98.56"],
+ ["20160525 13:30:00.076", "AAPL", "98.55", "98.56"],
+ ["20160525 13:30:00.076", "AAPL", "98.55", "98.56"],
+ ["20160525 13:30:00.076", "AAPL", "98.55", "98.56"],
+ ["20160525 13:30:00.078", "MSFT", "51.95", "51.95"],
+ ["20160525 13:30:00.078", "MSFT", "51.95", "51.95"],
+ ["20160525 13:30:00.078", "MSFT", "51.95", "51.95"],
+ ["20160525 13:30:00.078", "MSFT", "51.92", "51.95"],
+ ],
+ columns="time,ticker,bid,ask".split(","),
+ )
+ df["bid"] = df["bid"].astype("float64")
+ df["ask"] = df["ask"].astype("float64")
+ return self.prep_data(df, dedupe=True)
+
+ @pytest.fixture
+ def asof(self):
+ df = pd.DataFrame(
+ [
+ [
+ "20160525 13:30:00.023",
+ "MSFT",
+ "51.95",
+ "75",
+ "NASDAQ",
+ "51.95",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.038",
+ "MSFT",
+ "51.95",
+ "155",
+ "NASDAQ",
+ "51.95",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.77",
+ "100",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.92",
+ "100",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "200",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "300",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "600",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "44",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.074",
+ "AAPL",
+ "98.67",
+ "478343",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.67",
+ "478343",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.66",
+ "6",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "30",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "75",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "20",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "35",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "10",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.55",
+ "6",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.55",
+ "6",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "1000",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "200",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "300",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "400",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "600",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "200",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "783",
+ "NASDAQ",
+ "51.92",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "100",
+ "NASDAQ",
+ "51.92",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "100",
+ "NASDAQ",
+ "51.92",
+ "51.95",
+ ],
+ ],
+ columns="time,ticker,price,quantity,marketCenter,bid,ask".split(","),
+ )
+ df["price"] = df["price"].astype("float64")
+ df["quantity"] = df["quantity"].astype("int64")
+ df["bid"] = df["bid"].astype("float64")
+ df["ask"] = df["ask"].astype("float64")
+ return self.prep_data(df)
+
+ @pytest.fixture
+ def tolerance(self):
+ df = pd.DataFrame(
+ [
+ [
+ "20160525 13:30:00.023",
+ "MSFT",
+ "51.95",
+ "75",
+ "NASDAQ",
+ "51.95",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.038",
+ "MSFT",
+ "51.95",
+ "155",
+ "NASDAQ",
+ "51.95",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.77",
+ "100",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.92",
+ "100",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "200",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "300",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "600",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "44",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.074",
+ "AAPL",
+ "98.67",
+ "478343",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.67",
+ "478343",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.66",
+ "6",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "30",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "75",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "20",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "35",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "10",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.55",
+ "6",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.55",
+ "6",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "1000",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "200",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "300",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "400",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "600",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "200",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "783",
+ "NASDAQ",
+ "51.92",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "100",
+ "NASDAQ",
+ "51.92",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "100",
+ "NASDAQ",
+ "51.92",
+ "51.95",
+ ],
+ ],
+ columns="time,ticker,price,quantity,marketCenter,bid,ask".split(","),
+ )
+ df["price"] = df["price"].astype("float64")
+ df["quantity"] = df["quantity"].astype("int64")
+ df["bid"] = df["bid"].astype("float64")
+ df["ask"] = df["ask"].astype("float64")
+ return self.prep_data(df)
+
+ def test_examples1(self):
+ """doc-string examples"""
+ left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
+ right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "right_val": [1, 2, 3, 6, 7]})
+
+ expected = pd.DataFrame(
+ {"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [1, 3, 7]}
+ )
+
+ result = merge_asof(left, right, on="a")
+ tm.assert_frame_equal(result, expected)
+
+ def test_examples2(self, unit):
+ """doc-string examples"""
+ if unit == "s":
+ pytest.skip(
+ "This test is invalid for unit='s' because that would "
+ "round the trades['time']]"
+ )
+ trades = pd.DataFrame(
+ {
+ "time": to_datetime(
+ [
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.038",
+ "20160525 13:30:00.048",
+ "20160525 13:30:00.048",
+ "20160525 13:30:00.048",
+ ]
+ ).astype(f"M8[{unit}]"),
+ "ticker": ["MSFT", "MSFT", "GOOG", "GOOG", "AAPL"],
+ "price": [51.95, 51.95, 720.77, 720.92, 98.00],
+ "quantity": [75, 155, 100, 100, 100],
+ },
+ columns=["time", "ticker", "price", "quantity"],
+ )
+
+ quotes = pd.DataFrame(
+ {
+ "time": to_datetime(
+ [
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.030",
+ "20160525 13:30:00.041",
+ "20160525 13:30:00.048",
+ "20160525 13:30:00.049",
+ "20160525 13:30:00.072",
+ "20160525 13:30:00.075",
+ ]
+ ).astype(f"M8[{unit}]"),
+ "ticker": [
+ "GOOG",
+ "MSFT",
+ "MSFT",
+ "MSFT",
+ "GOOG",
+ "AAPL",
+ "GOOG",
+ "MSFT",
+ ],
+ "bid": [720.50, 51.95, 51.97, 51.99, 720.50, 97.99, 720.50, 52.01],
+ "ask": [720.93, 51.96, 51.98, 52.00, 720.93, 98.01, 720.88, 52.03],
+ },
+ columns=["time", "ticker", "bid", "ask"],
+ )
+
+ merge_asof(trades, quotes, on="time", by="ticker")
+
+ merge_asof(trades, quotes, on="time", by="ticker", tolerance=Timedelta("2ms"))
+
+ expected = pd.DataFrame(
+ {
+ "time": to_datetime(
+ [
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.038",
+ "20160525 13:30:00.048",
+ "20160525 13:30:00.048",
+ "20160525 13:30:00.048",
+ ]
+ ).astype(f"M8[{unit}]"),
+ "ticker": ["MSFT", "MSFT", "GOOG", "GOOG", "AAPL"],
+ "price": [51.95, 51.95, 720.77, 720.92, 98.00],
+ "quantity": [75, 155, 100, 100, 100],
+ "bid": [np.nan, 51.97, np.nan, np.nan, np.nan],
+ "ask": [np.nan, 51.98, np.nan, np.nan, np.nan],
+ },
+ columns=["time", "ticker", "price", "quantity", "bid", "ask"],
+ )
+
+ result = merge_asof(
+ trades,
+ quotes,
+ on="time",
+ by="ticker",
+ tolerance=Timedelta("10ms"),
+ allow_exact_matches=False,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_examples3(self):
+ """doc-string examples"""
+ # GH14887
+
+ left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
+ right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "right_val": [1, 2, 3, 6, 7]})
+
+ expected = pd.DataFrame(
+ {"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [1, 6, np.nan]}
+ )
+
+ result = merge_asof(left, right, on="a", direction="forward")
+ tm.assert_frame_equal(result, expected)
+
+ def test_examples4(self):
+ """doc-string examples"""
+ # GH14887
+
+ left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
+ right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "right_val": [1, 2, 3, 6, 7]})
+
+ expected = pd.DataFrame(
+ {"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [1, 6, 7]}
+ )
+
+ result = merge_asof(left, right, on="a", direction="nearest")
+ tm.assert_frame_equal(result, expected)
+
+ def test_basic(self, trades, asof, quotes):
+ expected = asof
+
+ result = merge_asof(trades, quotes, on="time", by="ticker")
+ tm.assert_frame_equal(result, expected)
+
+ def test_basic_categorical(self, trades, asof, quotes):
+ expected = asof
+ trades.ticker = trades.ticker.astype("category")
+ quotes.ticker = quotes.ticker.astype("category")
+ expected.ticker = expected.ticker.astype("category")
+
+ result = merge_asof(trades, quotes, on="time", by="ticker")
+ tm.assert_frame_equal(result, expected)
+
+ def test_basic_left_index(self, trades, asof, quotes):
+ # GH14253
+ expected = asof
+ trades = trades.set_index("time")
+
+ result = merge_asof(
+ trades, quotes, left_index=True, right_on="time", by="ticker"
+ )
+ # left-only index uses right"s index, oddly
+ expected.index = result.index
+ # time column appears after left"s columns
+ expected = expected[result.columns]
+ tm.assert_frame_equal(result, expected)
+
+ def test_basic_right_index(self, trades, asof, quotes):
+ expected = asof
+ quotes = quotes.set_index("time")
+
+ result = merge_asof(
+ trades, quotes, left_on="time", right_index=True, by="ticker"
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_basic_left_index_right_index(self, trades, asof, quotes):
+ expected = asof.set_index("time")
+ trades = trades.set_index("time")
+ quotes = quotes.set_index("time")
+
+ result = merge_asof(
+ trades, quotes, left_index=True, right_index=True, by="ticker"
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_multi_index_left(self, trades, quotes):
+ # MultiIndex is prohibited
+ trades = trades.set_index(["time", "price"])
+ quotes = quotes.set_index("time")
+ with pytest.raises(MergeError, match="left can only have one index"):
+ merge_asof(trades, quotes, left_index=True, right_index=True)
+
+ def test_multi_index_right(self, trades, quotes):
+ # MultiIndex is prohibited
+ trades = trades.set_index("time")
+ quotes = quotes.set_index(["time", "bid"])
+ with pytest.raises(MergeError, match="right can only have one index"):
+ merge_asof(trades, quotes, left_index=True, right_index=True)
+
+ def test_on_and_index_left_on(self, trades, quotes):
+ # "on" parameter and index together is prohibited
+ trades = trades.set_index("time")
+ quotes = quotes.set_index("time")
+ msg = 'Can only pass argument "left_on" OR "left_index" not both.'
+ with pytest.raises(MergeError, match=msg):
+ merge_asof(
+ trades, quotes, left_on="price", left_index=True, right_index=True
+ )
+
+ def test_on_and_index_right_on(self, trades, quotes):
+ trades = trades.set_index("time")
+ quotes = quotes.set_index("time")
+ msg = 'Can only pass argument "right_on" OR "right_index" not both.'
+ with pytest.raises(MergeError, match=msg):
+ merge_asof(
+ trades, quotes, right_on="bid", left_index=True, right_index=True
+ )
+
+ def test_basic_left_by_right_by(self, trades, asof, quotes):
+ # GH14253
+ expected = asof
+
+ result = merge_asof(
+ trades, quotes, on="time", left_by="ticker", right_by="ticker"
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_missing_right_by(self, trades, asof, quotes):
+ expected = asof
+
+ q = quotes[quotes.ticker != "MSFT"]
+ result = merge_asof(trades, q, on="time", by="ticker")
+ expected.loc[expected.ticker == "MSFT", ["bid", "ask"]] = np.nan
+ tm.assert_frame_equal(result, expected)
+
+ def test_multiby(self):
+ # GH13936
+ trades = pd.DataFrame(
+ {
+ "time": to_datetime(
+ [
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.046",
+ "20160525 13:30:00.048",
+ "20160525 13:30:00.050",
+ ]
+ ),
+ "ticker": ["MSFT", "MSFT", "GOOG", "GOOG", "AAPL"],
+ "exch": ["ARCA", "NSDQ", "NSDQ", "BATS", "NSDQ"],
+ "price": [51.95, 51.95, 720.77, 720.92, 98.00],
+ "quantity": [75, 155, 100, 100, 100],
+ },
+ columns=["time", "ticker", "exch", "price", "quantity"],
+ )
+
+ quotes = pd.DataFrame(
+ {
+ "time": to_datetime(
+ [
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.030",
+ "20160525 13:30:00.041",
+ "20160525 13:30:00.045",
+ "20160525 13:30:00.049",
+ ]
+ ),
+ "ticker": ["GOOG", "MSFT", "MSFT", "MSFT", "GOOG", "AAPL"],
+ "exch": ["BATS", "NSDQ", "ARCA", "ARCA", "NSDQ", "ARCA"],
+ "bid": [720.51, 51.95, 51.97, 51.99, 720.50, 97.99],
+ "ask": [720.92, 51.96, 51.98, 52.00, 720.93, 98.01],
+ },
+ columns=["time", "ticker", "exch", "bid", "ask"],
+ )
+
+ expected = pd.DataFrame(
+ {
+ "time": to_datetime(
+ [
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.046",
+ "20160525 13:30:00.048",
+ "20160525 13:30:00.050",
+ ]
+ ),
+ "ticker": ["MSFT", "MSFT", "GOOG", "GOOG", "AAPL"],
+ "exch": ["ARCA", "NSDQ", "NSDQ", "BATS", "NSDQ"],
+ "price": [51.95, 51.95, 720.77, 720.92, 98.00],
+ "quantity": [75, 155, 100, 100, 100],
+ "bid": [np.nan, 51.95, 720.50, 720.51, np.nan],
+ "ask": [np.nan, 51.96, 720.93, 720.92, np.nan],
+ },
+ columns=["time", "ticker", "exch", "price", "quantity", "bid", "ask"],
+ )
+
+ result = merge_asof(trades, quotes, on="time", by=["ticker", "exch"])
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("dtype", ["object", "string"])
+ def test_multiby_heterogeneous_types(self, dtype):
+ # GH13936
+ trades = pd.DataFrame(
+ {
+ "time": to_datetime(
+ [
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.046",
+ "20160525 13:30:00.048",
+ "20160525 13:30:00.050",
+ ]
+ ),
+ "ticker": [0, 0, 1, 1, 2],
+ "exch": ["ARCA", "NSDQ", "NSDQ", "BATS", "NSDQ"],
+ "price": [51.95, 51.95, 720.77, 720.92, 98.00],
+ "quantity": [75, 155, 100, 100, 100],
+ },
+ columns=["time", "ticker", "exch", "price", "quantity"],
+ )
+ trades = trades.astype({"ticker": dtype, "exch": dtype})
+
+ quotes = pd.DataFrame(
+ {
+ "time": to_datetime(
+ [
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.030",
+ "20160525 13:30:00.041",
+ "20160525 13:30:00.045",
+ "20160525 13:30:00.049",
+ ]
+ ),
+ "ticker": [1, 0, 0, 0, 1, 2],
+ "exch": ["BATS", "NSDQ", "ARCA", "ARCA", "NSDQ", "ARCA"],
+ "bid": [720.51, 51.95, 51.97, 51.99, 720.50, 97.99],
+ "ask": [720.92, 51.96, 51.98, 52.00, 720.93, 98.01],
+ },
+ columns=["time", "ticker", "exch", "bid", "ask"],
+ )
+ quotes = quotes.astype({"ticker": dtype, "exch": dtype})
+
+ expected = pd.DataFrame(
+ {
+ "time": to_datetime(
+ [
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.023",
+ "20160525 13:30:00.046",
+ "20160525 13:30:00.048",
+ "20160525 13:30:00.050",
+ ]
+ ),
+ "ticker": [0, 0, 1, 1, 2],
+ "exch": ["ARCA", "NSDQ", "NSDQ", "BATS", "NSDQ"],
+ "price": [51.95, 51.95, 720.77, 720.92, 98.00],
+ "quantity": [75, 155, 100, 100, 100],
+ "bid": [np.nan, 51.95, 720.50, 720.51, np.nan],
+ "ask": [np.nan, 51.96, 720.93, 720.92, np.nan],
+ },
+ columns=["time", "ticker", "exch", "price", "quantity", "bid", "ask"],
+ )
+ expected = expected.astype({"ticker": dtype, "exch": dtype})
+
+ result = merge_asof(trades, quotes, on="time", by=["ticker", "exch"])
+ tm.assert_frame_equal(result, expected)
+
+ def test_mismatched_index_dtype(self):
+ # similar to test_multiby_indexed, but we change the dtype on left.index
+ left = pd.DataFrame(
+ [
+ [to_datetime("20160602"), 1, "a"],
+ [to_datetime("20160602"), 2, "a"],
+ [to_datetime("20160603"), 1, "b"],
+ [to_datetime("20160603"), 2, "b"],
+ ],
+ columns=["time", "k1", "k2"],
+ ).set_index("time")
+ # different dtype for the index
+ left.index = left.index - pd.Timestamp(0)
+
+ right = pd.DataFrame(
+ [
+ [to_datetime("20160502"), 1, "a", 1.0],
+ [to_datetime("20160502"), 2, "a", 2.0],
+ [to_datetime("20160503"), 1, "b", 3.0],
+ [to_datetime("20160503"), 2, "b", 4.0],
+ ],
+ columns=["time", "k1", "k2", "value"],
+ ).set_index("time")
+
+ msg = "incompatible merge keys"
+ with pytest.raises(MergeError, match=msg):
+ merge_asof(left, right, left_index=True, right_index=True, by=["k1", "k2"])
+
+ def test_multiby_indexed(self):
+ # GH15676
+ left = pd.DataFrame(
+ [
+ [to_datetime("20160602"), 1, "a"],
+ [to_datetime("20160602"), 2, "a"],
+ [to_datetime("20160603"), 1, "b"],
+ [to_datetime("20160603"), 2, "b"],
+ ],
+ columns=["time", "k1", "k2"],
+ ).set_index("time")
+
+ right = pd.DataFrame(
+ [
+ [to_datetime("20160502"), 1, "a", 1.0],
+ [to_datetime("20160502"), 2, "a", 2.0],
+ [to_datetime("20160503"), 1, "b", 3.0],
+ [to_datetime("20160503"), 2, "b", 4.0],
+ ],
+ columns=["time", "k1", "k2", "value"],
+ ).set_index("time")
+
+ expected = pd.DataFrame(
+ [
+ [to_datetime("20160602"), 1, "a", 1.0],
+ [to_datetime("20160602"), 2, "a", 2.0],
+ [to_datetime("20160603"), 1, "b", 3.0],
+ [to_datetime("20160603"), 2, "b", 4.0],
+ ],
+ columns=["time", "k1", "k2", "value"],
+ ).set_index("time")
+
+ result = merge_asof(
+ left, right, left_index=True, right_index=True, by=["k1", "k2"]
+ )
+
+ tm.assert_frame_equal(expected, result)
+
+ with pytest.raises(
+ MergeError, match="left_by and right_by must be the same length"
+ ):
+ merge_asof(
+ left,
+ right,
+ left_index=True,
+ right_index=True,
+ left_by=["k1", "k2"],
+ right_by=["k1"],
+ )
+
+ def test_basic2(self, datapath):
+ expected = pd.DataFrame(
+ [
+ [
+ "20160525 13:30:00.023",
+ "MSFT",
+ "51.95",
+ "75",
+ "NASDAQ",
+ "51.95",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.038",
+ "MSFT",
+ "51.95",
+ "155",
+ "NASDAQ",
+ "51.95",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.77",
+ "100",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.92",
+ "100",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "200",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "300",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "600",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "44",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.074",
+ "AAPL",
+ "98.67",
+ "478343",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.67",
+ "478343",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.66",
+ "6",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "30",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "75",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "20",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "35",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "10",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.55",
+ "6",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.55",
+ "6",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "1000",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "200",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "300",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "400",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "600",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "200",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "783",
+ "NASDAQ",
+ "51.92",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "100",
+ "NASDAQ",
+ "51.92",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "100",
+ "NASDAQ",
+ "51.92",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.084",
+ "AAPL",
+ "98.64",
+ "40",
+ "NASDAQ",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.084",
+ "AAPL",
+ "98.55",
+ "149",
+ "EDGX",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.086",
+ "AAPL",
+ "98.56",
+ "500",
+ "ARCA",
+ "98.55",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.104",
+ "AAPL",
+ "98.63",
+ "647",
+ "EDGX",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.104",
+ "AAPL",
+ "98.63",
+ "300",
+ "EDGX",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.104",
+ "AAPL",
+ "98.63",
+ "50",
+ "NASDAQ",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.104",
+ "AAPL",
+ "98.63",
+ "50",
+ "NASDAQ",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.104",
+ "AAPL",
+ "98.63",
+ "70",
+ "NASDAQ",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.104",
+ "AAPL",
+ "98.63",
+ "70",
+ "NASDAQ",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.104",
+ "AAPL",
+ "98.63",
+ "1",
+ "NASDAQ",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.104",
+ "AAPL",
+ "98.63",
+ "62",
+ "NASDAQ",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.104",
+ "AAPL",
+ "98.63",
+ "10",
+ "NASDAQ",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.104",
+ "AAPL",
+ "98.63",
+ "100",
+ "ARCA",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.105",
+ "AAPL",
+ "98.63",
+ "100",
+ "ARCA",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.105",
+ "AAPL",
+ "98.63",
+ "700",
+ "ARCA",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.106",
+ "AAPL",
+ "98.63",
+ "61",
+ "EDGX",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.107",
+ "AAPL",
+ "98.63",
+ "100",
+ "ARCA",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.107",
+ "AAPL",
+ "98.63",
+ "53",
+ "ARCA",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.108",
+ "AAPL",
+ "98.63",
+ "100",
+ "ARCA",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.108",
+ "AAPL",
+ "98.63",
+ "839",
+ "ARCA",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.115",
+ "AAPL",
+ "98.63",
+ "5",
+ "EDGX",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.118",
+ "AAPL",
+ "98.63",
+ "295",
+ "EDGX",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.118",
+ "AAPL",
+ "98.63",
+ "5",
+ "EDGX",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.128",
+ "AAPL",
+ "98.63",
+ "100",
+ "NASDAQ",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.128",
+ "AAPL",
+ "98.63",
+ "100",
+ "NASDAQ",
+ "98.62",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.128",
+ "MSFT",
+ "51.92",
+ "100",
+ "ARCA",
+ "51.92",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.129",
+ "AAPL",
+ "98.62",
+ "100",
+ "NASDAQ",
+ "98.61",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.129",
+ "AAPL",
+ "98.62",
+ "10",
+ "NASDAQ",
+ "98.61",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.129",
+ "AAPL",
+ "98.62",
+ "59",
+ "NASDAQ",
+ "98.61",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.129",
+ "AAPL",
+ "98.62",
+ "31",
+ "NASDAQ",
+ "98.61",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.129",
+ "AAPL",
+ "98.62",
+ "69",
+ "NASDAQ",
+ "98.61",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.129",
+ "AAPL",
+ "98.62",
+ "12",
+ "NASDAQ",
+ "98.61",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.129",
+ "AAPL",
+ "98.62",
+ "12",
+ "EDGX",
+ "98.61",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.129",
+ "AAPL",
+ "98.62",
+ "100",
+ "ARCA",
+ "98.61",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.129",
+ "AAPL",
+ "98.62",
+ "100",
+ "ARCA",
+ "98.61",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.130",
+ "MSFT",
+ "51.95",
+ "317",
+ "ARCA",
+ "51.93",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.130",
+ "MSFT",
+ "51.95",
+ "283",
+ "ARCA",
+ "51.93",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.135",
+ "MSFT",
+ "51.93",
+ "100",
+ "EDGX",
+ "51.92",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.135",
+ "AAPL",
+ "98.62",
+ "100",
+ "ARCA",
+ "98.61",
+ "98.62",
+ ],
+ [
+ "20160525 13:30:00.144",
+ "AAPL",
+ "98.62",
+ "12",
+ "NASDAQ",
+ "98.61",
+ "98.62",
+ ],
+ [
+ "20160525 13:30:00.144",
+ "AAPL",
+ "98.62",
+ "88",
+ "NASDAQ",
+ "98.61",
+ "98.62",
+ ],
+ [
+ "20160525 13:30:00.144",
+ "AAPL",
+ "98.62",
+ "162",
+ "NASDAQ",
+ "98.61",
+ "98.62",
+ ],
+ [
+ "20160525 13:30:00.144",
+ "AAPL",
+ "98.61",
+ "100",
+ "BATS",
+ "98.61",
+ "98.62",
+ ],
+ [
+ "20160525 13:30:00.144",
+ "AAPL",
+ "98.62",
+ "61",
+ "ARCA",
+ "98.61",
+ "98.62",
+ ],
+ [
+ "20160525 13:30:00.144",
+ "AAPL",
+ "98.62",
+ "25",
+ "ARCA",
+ "98.61",
+ "98.62",
+ ],
+ [
+ "20160525 13:30:00.144",
+ "AAPL",
+ "98.62",
+ "14",
+ "ARCA",
+ "98.61",
+ "98.62",
+ ],
+ [
+ "20160525 13:30:00.145",
+ "AAPL",
+ "98.62",
+ "12",
+ "ARCA",
+ "98.6",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.145",
+ "AAPL",
+ "98.62",
+ "100",
+ "ARCA",
+ "98.6",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.145",
+ "AAPL",
+ "98.63",
+ "100",
+ "NASDAQ",
+ "98.6",
+ "98.63",
+ ],
+ [
+ "20160525 13:30:00.145",
+ "AAPL",
+ "98.63",
+ "100",
+ "NASDAQ",
+ "98.6",
+ "98.63",
+ ],
+ ],
+ columns="time,ticker,price,quantity,marketCenter,bid,ask".split(","),
+ )
+ expected["price"] = expected["price"].astype("float64")
+ expected["quantity"] = expected["quantity"].astype("int64")
+ expected["bid"] = expected["bid"].astype("float64")
+ expected["ask"] = expected["ask"].astype("float64")
+ expected = self.prep_data(expected)
+
+ trades = pd.DataFrame(
+ [
+ ["20160525 13:30:00.023", "MSFT", "51.9500", "75", "NASDAQ"],
+ ["20160525 13:30:00.038", "MSFT", "51.9500", "155", "NASDAQ"],
+ ["20160525 13:30:00.048", "GOOG", "720.7700", "100", "NASDAQ"],
+ ["20160525 13:30:00.048", "GOOG", "720.9200", "100", "NASDAQ"],
+ ["20160525 13:30:00.048", "GOOG", "720.9300", "200", "NASDAQ"],
+ ["20160525 13:30:00.048", "GOOG", "720.9300", "300", "NASDAQ"],
+ ["20160525 13:30:00.048", "GOOG", "720.9300", "600", "NASDAQ"],
+ ["20160525 13:30:00.048", "GOOG", "720.9300", "44", "NASDAQ"],
+ ["20160525 13:30:00.074", "AAPL", "98.6700", "478343", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.6700", "478343", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.6600", "6", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.6500", "30", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.6500", "75", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.6500", "20", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.6500", "35", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.6500", "10", "NASDAQ"],
+ ["20160525 13:30:00.075", "AAPL", "98.5500", "6", "ARCA"],
+ ["20160525 13:30:00.075", "AAPL", "98.5500", "6", "ARCA"],
+ ["20160525 13:30:00.076", "AAPL", "98.5600", "1000", "ARCA"],
+ ["20160525 13:30:00.076", "AAPL", "98.5600", "200", "ARCA"],
+ ["20160525 13:30:00.076", "AAPL", "98.5600", "300", "ARCA"],
+ ["20160525 13:30:00.076", "AAPL", "98.5600", "400", "ARCA"],
+ ["20160525 13:30:00.076", "AAPL", "98.5600", "600", "ARCA"],
+ ["20160525 13:30:00.076", "AAPL", "98.5600", "200", "ARCA"],
+ ["20160525 13:30:00.078", "MSFT", "51.9500", "783", "NASDAQ"],
+ ["20160525 13:30:00.078", "MSFT", "51.9500", "100", "NASDAQ"],
+ ["20160525 13:30:00.078", "MSFT", "51.9500", "100", "NASDAQ"],
+ ["20160525 13:30:00.084", "AAPL", "98.6400", "40", "NASDAQ"],
+ ["20160525 13:30:00.084", "AAPL", "98.5500", "149", "EDGX"],
+ ["20160525 13:30:00.086", "AAPL", "98.5600", "500", "ARCA"],
+ ["20160525 13:30:00.104", "AAPL", "98.6300", "647", "EDGX"],
+ ["20160525 13:30:00.104", "AAPL", "98.6300", "300", "EDGX"],
+ ["20160525 13:30:00.104", "AAPL", "98.6300", "50", "NASDAQ"],
+ ["20160525 13:30:00.104", "AAPL", "98.6300", "50", "NASDAQ"],
+ ["20160525 13:30:00.104", "AAPL", "98.6300", "70", "NASDAQ"],
+ ["20160525 13:30:00.104", "AAPL", "98.6300", "70", "NASDAQ"],
+ ["20160525 13:30:00.104", "AAPL", "98.6300", "1", "NASDAQ"],
+ ["20160525 13:30:00.104", "AAPL", "98.6300", "62", "NASDAQ"],
+ ["20160525 13:30:00.104", "AAPL", "98.6300", "10", "NASDAQ"],
+ ["20160525 13:30:00.104", "AAPL", "98.6300", "100", "ARCA"],
+ ["20160525 13:30:00.105", "AAPL", "98.6300", "100", "ARCA"],
+ ["20160525 13:30:00.105", "AAPL", "98.6300", "700", "ARCA"],
+ ["20160525 13:30:00.106", "AAPL", "98.6300", "61", "EDGX"],
+ ["20160525 13:30:00.107", "AAPL", "98.6300", "100", "ARCA"],
+ ["20160525 13:30:00.107", "AAPL", "98.6300", "53", "ARCA"],
+ ["20160525 13:30:00.108", "AAPL", "98.6300", "100", "ARCA"],
+ ["20160525 13:30:00.108", "AAPL", "98.6300", "839", "ARCA"],
+ ["20160525 13:30:00.115", "AAPL", "98.6300", "5", "EDGX"],
+ ["20160525 13:30:00.118", "AAPL", "98.6300", "295", "EDGX"],
+ ["20160525 13:30:00.118", "AAPL", "98.6300", "5", "EDGX"],
+ ["20160525 13:30:00.128", "AAPL", "98.6300", "100", "NASDAQ"],
+ ["20160525 13:30:00.128", "AAPL", "98.6300", "100", "NASDAQ"],
+ ["20160525 13:30:00.128", "MSFT", "51.9200", "100", "ARCA"],
+ ["20160525 13:30:00.129", "AAPL", "98.6200", "100", "NASDAQ"],
+ ["20160525 13:30:00.129", "AAPL", "98.6200", "10", "NASDAQ"],
+ ["20160525 13:30:00.129", "AAPL", "98.6200", "59", "NASDAQ"],
+ ["20160525 13:30:00.129", "AAPL", "98.6200", "31", "NASDAQ"],
+ ["20160525 13:30:00.129", "AAPL", "98.6200", "69", "NASDAQ"],
+ ["20160525 13:30:00.129", "AAPL", "98.6200", "12", "NASDAQ"],
+ ["20160525 13:30:00.129", "AAPL", "98.6200", "12", "EDGX"],
+ ["20160525 13:30:00.129", "AAPL", "98.6200", "100", "ARCA"],
+ ["20160525 13:30:00.129", "AAPL", "98.6200", "100", "ARCA"],
+ ["20160525 13:30:00.130", "MSFT", "51.9500", "317", "ARCA"],
+ ["20160525 13:30:00.130", "MSFT", "51.9500", "283", "ARCA"],
+ ["20160525 13:30:00.135", "MSFT", "51.9300", "100", "EDGX"],
+ ["20160525 13:30:00.135", "AAPL", "98.6200", "100", "ARCA"],
+ ["20160525 13:30:00.144", "AAPL", "98.6200", "12", "NASDAQ"],
+ ["20160525 13:30:00.144", "AAPL", "98.6200", "88", "NASDAQ"],
+ ["20160525 13:30:00.144", "AAPL", "98.6200", "162", "NASDAQ"],
+ ["20160525 13:30:00.144", "AAPL", "98.6100", "100", "BATS"],
+ ["20160525 13:30:00.144", "AAPL", "98.6200", "61", "ARCA"],
+ ["20160525 13:30:00.144", "AAPL", "98.6200", "25", "ARCA"],
+ ["20160525 13:30:00.144", "AAPL", "98.6200", "14", "ARCA"],
+ ["20160525 13:30:00.145", "AAPL", "98.6200", "12", "ARCA"],
+ ["20160525 13:30:00.145", "AAPL", "98.6200", "100", "ARCA"],
+ ["20160525 13:30:00.145", "AAPL", "98.6300", "100", "NASDAQ"],
+ ["20160525 13:30:00.145", "AAPL", "98.6300", "100", "NASDAQ"],
+ ],
+ columns="time,ticker,price,quantity,marketCenter".split(","),
+ )
+ trades["price"] = trades["price"].astype("float64")
+ trades["quantity"] = trades["quantity"].astype("int64")
+ trades = self.prep_data(trades)
+
+ quotes = pd.DataFrame(
+ [
+ ["20160525 13:30:00.023", "GOOG", "720.50", "720.93"],
+ ["20160525 13:30:00.023", "MSFT", "51.95", "51.95"],
+ ["20160525 13:30:00.041", "MSFT", "51.95", "51.95"],
+ ["20160525 13:30:00.048", "GOOG", "720.50", "720.93"],
+ ["20160525 13:30:00.048", "GOOG", "720.50", "720.93"],
+ ["20160525 13:30:00.048", "GOOG", "720.50", "720.93"],
+ ["20160525 13:30:00.048", "GOOG", "720.50", "720.93"],
+ ["20160525 13:30:00.072", "GOOG", "720.50", "720.88"],
+ ["20160525 13:30:00.075", "AAPL", "98.55", "98.56"],
+ ["20160525 13:30:00.076", "AAPL", "98.55", "98.56"],
+ ["20160525 13:30:00.076", "AAPL", "98.55", "98.56"],
+ ["20160525 13:30:00.076", "AAPL", "98.55", "98.56"],
+ ["20160525 13:30:00.078", "MSFT", "51.95", "51.95"],
+ ["20160525 13:30:00.078", "MSFT", "51.95", "51.95"],
+ ["20160525 13:30:00.078", "MSFT", "51.95", "51.95"],
+ ["20160525 13:30:00.078", "MSFT", "51.92", "51.95"],
+ ["20160525 13:30:00.079", "MSFT", "51.92", "51.95"],
+ ["20160525 13:30:00.080", "AAPL", "98.55", "98.56"],
+ ["20160525 13:30:00.084", "AAPL", "98.55", "98.56"],
+ ["20160525 13:30:00.086", "AAPL", "98.55", "98.63"],
+ ["20160525 13:30:00.088", "AAPL", "98.65", "98.63"],
+ ["20160525 13:30:00.089", "AAPL", "98.63", "98.63"],
+ ["20160525 13:30:00.104", "AAPL", "98.63", "98.63"],
+ ["20160525 13:30:00.104", "AAPL", "98.63", "98.63"],
+ ["20160525 13:30:00.104", "AAPL", "98.63", "98.63"],
+ ["20160525 13:30:00.104", "AAPL", "98.63", "98.63"],
+ ["20160525 13:30:00.104", "AAPL", "98.62", "98.63"],
+ ["20160525 13:30:00.105", "AAPL", "98.62", "98.63"],
+ ["20160525 13:30:00.107", "AAPL", "98.62", "98.63"],
+ ["20160525 13:30:00.115", "AAPL", "98.62", "98.63"],
+ ["20160525 13:30:00.115", "AAPL", "98.62", "98.63"],
+ ["20160525 13:30:00.118", "AAPL", "98.62", "98.63"],
+ ["20160525 13:30:00.128", "AAPL", "98.62", "98.63"],
+ ["20160525 13:30:00.128", "AAPL", "98.62", "98.63"],
+ ["20160525 13:30:00.129", "AAPL", "98.62", "98.63"],
+ ["20160525 13:30:00.129", "AAPL", "98.61", "98.63"],
+ ["20160525 13:30:00.129", "AAPL", "98.62", "98.63"],
+ ["20160525 13:30:00.129", "AAPL", "98.62", "98.63"],
+ ["20160525 13:30:00.129", "AAPL", "98.61", "98.63"],
+ ["20160525 13:30:00.130", "MSFT", "51.93", "51.95"],
+ ["20160525 13:30:00.130", "MSFT", "51.93", "51.95"],
+ ["20160525 13:30:00.130", "AAPL", "98.61", "98.63"],
+ ["20160525 13:30:00.131", "AAPL", "98.61", "98.62"],
+ ["20160525 13:30:00.131", "AAPL", "98.61", "98.62"],
+ ["20160525 13:30:00.135", "MSFT", "51.92", "51.95"],
+ ["20160525 13:30:00.135", "AAPL", "98.61", "98.62"],
+ ["20160525 13:30:00.136", "AAPL", "98.61", "98.62"],
+ ["20160525 13:30:00.136", "AAPL", "98.61", "98.62"],
+ ["20160525 13:30:00.144", "AAPL", "98.61", "98.62"],
+ ["20160525 13:30:00.144", "AAPL", "98.61", "98.62"],
+ ["20160525 13:30:00.145", "AAPL", "98.61", "98.62"],
+ ["20160525 13:30:00.145", "AAPL", "98.61", "98.63"],
+ ["20160525 13:30:00.145", "AAPL", "98.61", "98.63"],
+ ["20160525 13:30:00.145", "AAPL", "98.60", "98.63"],
+ ["20160525 13:30:00.145", "AAPL", "98.61", "98.63"],
+ ["20160525 13:30:00.145", "AAPL", "98.60", "98.63"],
+ ],
+ columns="time,ticker,bid,ask".split(","),
+ )
+ quotes["bid"] = quotes["bid"].astype("float64")
+ quotes["ask"] = quotes["ask"].astype("float64")
+ quotes = self.prep_data(quotes, dedupe=True)
+
+ result = merge_asof(trades, quotes, on="time", by="ticker")
+ tm.assert_frame_equal(result, expected)
+
+ def test_basic_no_by(self, trades, asof, quotes):
+ f = lambda x: (
+ x[x.ticker == "MSFT"].drop("ticker", axis=1).reset_index(drop=True)
+ )
+
+ # just use a single ticker
+ expected = f(asof)
+ trades = f(trades)
+ quotes = f(quotes)
+
+ result = merge_asof(trades, quotes, on="time")
+ tm.assert_frame_equal(result, expected)
+
+ def test_valid_join_keys(self, trades, quotes):
+ msg = r"incompatible merge keys \[1\] .* must be the same type"
+
+ with pytest.raises(MergeError, match=msg):
+ merge_asof(trades, quotes, left_on="time", right_on="bid", by="ticker")
+
+ with pytest.raises(MergeError, match="can only asof on a key for left"):
+ merge_asof(trades, quotes, on=["time", "ticker"], by="ticker")
+
+ with pytest.raises(MergeError, match="can only asof on a key for left"):
+ merge_asof(trades, quotes, by="ticker")
+
+ def test_with_duplicates(self, datapath, trades, quotes, asof):
+ q = (
+ pd.concat([quotes, quotes])
+ .sort_values(["time", "ticker"])
+ .reset_index(drop=True)
+ )
+ result = merge_asof(trades, q, on="time", by="ticker")
+ expected = self.prep_data(asof)
+ tm.assert_frame_equal(result, expected)
+
+ def test_with_duplicates_no_on(self):
+ df1 = pd.DataFrame({"key": [1, 1, 3], "left_val": [1, 2, 3]})
+ df2 = pd.DataFrame({"key": [1, 2, 2], "right_val": [1, 2, 3]})
+ result = merge_asof(df1, df2, on="key")
+ expected = pd.DataFrame(
+ {"key": [1, 1, 3], "left_val": [1, 2, 3], "right_val": [1, 1, 3]}
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_valid_allow_exact_matches(self, trades, quotes):
+ msg = "allow_exact_matches must be boolean, passed foo"
+
+ with pytest.raises(MergeError, match=msg):
+ merge_asof(
+ trades, quotes, on="time", by="ticker", allow_exact_matches="foo"
+ )
+
+ def test_valid_tolerance(self, trades, quotes):
+ # dti
+ merge_asof(trades, quotes, on="time", by="ticker", tolerance=Timedelta("1s"))
+
+ # integer
+ merge_asof(
+ trades.reset_index(),
+ quotes.reset_index(),
+ on="index",
+ by="ticker",
+ tolerance=1,
+ )
+
+ msg = r"incompatible tolerance .*, must be compat with type .*"
+
+ # incompat
+ with pytest.raises(MergeError, match=msg):
+ merge_asof(trades, quotes, on="time", by="ticker", tolerance=1)
+
+ # invalid
+ with pytest.raises(MergeError, match=msg):
+ merge_asof(
+ trades.reset_index(),
+ quotes.reset_index(),
+ on="index",
+ by="ticker",
+ tolerance=1.0,
+ )
+
+ msg = "tolerance must be positive"
+
+ # invalid negative
+ with pytest.raises(MergeError, match=msg):
+ merge_asof(
+ trades, quotes, on="time", by="ticker", tolerance=-Timedelta("1s")
+ )
+
+ with pytest.raises(MergeError, match=msg):
+ merge_asof(
+ trades.reset_index(),
+ quotes.reset_index(),
+ on="index",
+ by="ticker",
+ tolerance=-1,
+ )
+
+ def test_non_sorted(self, trades, quotes):
+ trades = trades.sort_values("time", ascending=False)
+ quotes = quotes.sort_values("time", ascending=False)
+
+ # we require that we are already sorted on time & quotes
+ assert not trades.time.is_monotonic_increasing
+ assert not quotes.time.is_monotonic_increasing
+ with pytest.raises(ValueError, match="left keys must be sorted"):
+ merge_asof(trades, quotes, on="time", by="ticker")
+
+ trades = trades.sort_values("time")
+ assert trades.time.is_monotonic_increasing
+ assert not quotes.time.is_monotonic_increasing
+ with pytest.raises(ValueError, match="right keys must be sorted"):
+ merge_asof(trades, quotes, on="time", by="ticker")
+
+ quotes = quotes.sort_values("time")
+ assert trades.time.is_monotonic_increasing
+ assert quotes.time.is_monotonic_increasing
+
+ # ok, though has dupes
+ merge_asof(trades, quotes, on="time", by="ticker")
+
+ @pytest.mark.parametrize(
+ "tolerance_ts",
+ [Timedelta("1day"), datetime.timedelta(days=1)],
+ ids=["Timedelta", "datetime.timedelta"],
+ )
+ def test_tolerance(self, tolerance_ts, trades, quotes, tolerance):
+ result = merge_asof(
+ trades, quotes, on="time", by="ticker", tolerance=tolerance_ts
+ )
+ expected = tolerance
+ tm.assert_frame_equal(result, expected)
+
+ def test_tolerance_forward(self):
+ # GH14887
+
+ left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
+ right = pd.DataFrame({"a": [1, 2, 3, 7, 11], "right_val": [1, 2, 3, 7, 11]})
+
+ expected = pd.DataFrame(
+ {"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [1, np.nan, 11]}
+ )
+
+ result = merge_asof(left, right, on="a", direction="forward", tolerance=1)
+ tm.assert_frame_equal(result, expected)
+
+ def test_tolerance_nearest(self):
+ # GH14887
+
+ left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
+ right = pd.DataFrame({"a": [1, 2, 3, 7, 11], "right_val": [1, 2, 3, 7, 11]})
+
+ expected = pd.DataFrame(
+ {"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [1, np.nan, 11]}
+ )
+
+ result = merge_asof(left, right, on="a", direction="nearest", tolerance=1)
+ tm.assert_frame_equal(result, expected)
+
+ def test_tolerance_tz(self, unit):
+ # GH 14844
+ left = pd.DataFrame(
+ {
+ "date": pd.date_range(
+ start=to_datetime("2016-01-02"),
+ freq="D",
+ periods=5,
+ tz=datetime.UTC,
+ unit=unit,
+ ),
+ "value1": np.arange(5),
+ }
+ )
+ right = pd.DataFrame(
+ {
+ "date": pd.date_range(
+ start=to_datetime("2016-01-01"),
+ freq="D",
+ periods=5,
+ tz=datetime.UTC,
+ unit=unit,
+ ),
+ "value2": list("ABCDE"),
+ }
+ )
+ result = merge_asof(left, right, on="date", tolerance=Timedelta("1 day"))
+
+ expected = pd.DataFrame(
+ {
+ "date": pd.date_range(
+ start=to_datetime("2016-01-02"),
+ freq="D",
+ periods=5,
+ tz=datetime.UTC,
+ unit=unit,
+ ),
+ "value1": np.arange(5),
+ "value2": list("BCDEE"),
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_tolerance_float(self):
+ # GH22981
+ left = pd.DataFrame({"a": [1.1, 3.5, 10.9], "left_val": ["a", "b", "c"]})
+ right = pd.DataFrame(
+ {"a": [1.0, 2.5, 3.3, 7.5, 11.5], "right_val": [1.0, 2.5, 3.3, 7.5, 11.5]}
+ )
+
+ expected = pd.DataFrame(
+ {
+ "a": [1.1, 3.5, 10.9],
+ "left_val": ["a", "b", "c"],
+ "right_val": [1, 3.3, np.nan],
+ }
+ )
+
+ result = merge_asof(left, right, on="a", direction="nearest", tolerance=0.5)
+ tm.assert_frame_equal(result, expected)
+
+ def test_index_tolerance(self, trades, quotes, tolerance):
+ # GH 15135
+ expected = tolerance.set_index("time")
+ trades = trades.set_index("time")
+ quotes = quotes.set_index("time")
+
+ result = merge_asof(
+ trades,
+ quotes,
+ left_index=True,
+ right_index=True,
+ by="ticker",
+ tolerance=Timedelta("1day"),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_allow_exact_matches(self, trades, quotes):
+ result = merge_asof(
+ trades, quotes, on="time", by="ticker", allow_exact_matches=False
+ )
+ df = pd.DataFrame(
+ [
+ [
+ "20160525 13:30:00.023",
+ "MSFT",
+ "51.95",
+ "75",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.038",
+ "MSFT",
+ "51.95",
+ "155",
+ "NASDAQ",
+ "51.95",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.77",
+ "100",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.92",
+ "100",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "200",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "300",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "600",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "44",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.074",
+ "AAPL",
+ "98.67",
+ "478343",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.67",
+ "478343",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.66",
+ "6",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "30",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "75",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "20",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "35",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "10",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ ["20160525 13:30:00.075", "AAPL", "98.55", "6", "ARCA", np.nan, np.nan],
+ ["20160525 13:30:00.075", "AAPL", "98.55", "6", "ARCA", np.nan, np.nan],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "1000",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "200",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "300",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "400",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "600",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "200",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "783",
+ "NASDAQ",
+ "51.95",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "100",
+ "NASDAQ",
+ "51.95",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "100",
+ "NASDAQ",
+ "51.95",
+ "51.95",
+ ],
+ ],
+ columns="time,ticker,price,quantity,marketCenter,bid,ask".split(","),
+ )
+ df["price"] = df["price"].astype("float64")
+ df["quantity"] = df["quantity"].astype("int64")
+ df["bid"] = df["bid"].astype("float64")
+ df["ask"] = df["ask"].astype("float64")
+ expected = self.prep_data(df)
+ tm.assert_frame_equal(result, expected)
+
+ def test_allow_exact_matches_forward(self):
+ # GH14887
+
+ left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
+ right = pd.DataFrame({"a": [1, 2, 3, 7, 11], "right_val": [1, 2, 3, 7, 11]})
+
+ expected = pd.DataFrame(
+ {"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [2, 7, 11]}
+ )
+
+ result = merge_asof(
+ left, right, on="a", direction="forward", allow_exact_matches=False
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_allow_exact_matches_nearest(self):
+ # GH14887
+
+ left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
+ right = pd.DataFrame({"a": [1, 2, 3, 7, 11], "right_val": [1, 2, 3, 7, 11]})
+
+ expected = pd.DataFrame(
+ {"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [2, 3, 11]}
+ )
+
+ result = merge_asof(
+ left, right, on="a", direction="nearest", allow_exact_matches=False
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_allow_exact_matches_and_tolerance(self, trades, quotes):
+ result = merge_asof(
+ trades,
+ quotes,
+ on="time",
+ by="ticker",
+ tolerance=Timedelta("100ms"),
+ allow_exact_matches=False,
+ )
+ df = pd.DataFrame(
+ [
+ [
+ "20160525 13:30:00.023",
+ "MSFT",
+ "51.95",
+ "75",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.038",
+ "MSFT",
+ "51.95",
+ "155",
+ "NASDAQ",
+ "51.95",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.77",
+ "100",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.92",
+ "100",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "200",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "300",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "600",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.048",
+ "GOOG",
+ "720.93",
+ "44",
+ "NASDAQ",
+ "720.5",
+ "720.93",
+ ],
+ [
+ "20160525 13:30:00.074",
+ "AAPL",
+ "98.67",
+ "478343",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.67",
+ "478343",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.66",
+ "6",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "30",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "75",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "20",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "35",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ [
+ "20160525 13:30:00.075",
+ "AAPL",
+ "98.65",
+ "10",
+ "NASDAQ",
+ np.nan,
+ np.nan,
+ ],
+ ["20160525 13:30:00.075", "AAPL", "98.55", "6", "ARCA", np.nan, np.nan],
+ ["20160525 13:30:00.075", "AAPL", "98.55", "6", "ARCA", np.nan, np.nan],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "1000",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "200",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "300",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "400",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "600",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.076",
+ "AAPL",
+ "98.56",
+ "200",
+ "ARCA",
+ "98.55",
+ "98.56",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "783",
+ "NASDAQ",
+ "51.95",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "100",
+ "NASDAQ",
+ "51.95",
+ "51.95",
+ ],
+ [
+ "20160525 13:30:00.078",
+ "MSFT",
+ "51.95",
+ "100",
+ "NASDAQ",
+ "51.95",
+ "51.95",
+ ],
+ ],
+ columns="time,ticker,price,quantity,marketCenter,bid,ask".split(","),
+ )
+ df["price"] = df["price"].astype("float64")
+ df["quantity"] = df["quantity"].astype("int64")
+ df["bid"] = df["bid"].astype("float64")
+ df["ask"] = df["ask"].astype("float64")
+ expected = self.prep_data(df)
+ tm.assert_frame_equal(result, expected)
+
+ def test_allow_exact_matches_and_tolerance2(self):
+ # GH 13695
+ df1 = pd.DataFrame(
+ {"time": to_datetime(["2016-07-15 13:30:00.030"]), "username": ["bob"]}
+ )
+ df2 = pd.DataFrame(
+ {
+ "time": to_datetime(
+ ["2016-07-15 13:30:00.000", "2016-07-15 13:30:00.030"]
+ ),
+ "version": [1, 2],
+ }
+ )
+
+ result = merge_asof(df1, df2, on="time")
+ expected = pd.DataFrame(
+ {
+ "time": to_datetime(["2016-07-15 13:30:00.030"]),
+ "username": ["bob"],
+ "version": [2],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = merge_asof(df1, df2, on="time", allow_exact_matches=False)
+ expected = pd.DataFrame(
+ {
+ "time": to_datetime(["2016-07-15 13:30:00.030"]),
+ "username": ["bob"],
+ "version": [1],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = merge_asof(
+ df1,
+ df2,
+ on="time",
+ allow_exact_matches=False,
+ tolerance=Timedelta("10ms"),
+ )
+ expected = pd.DataFrame(
+ {
+ "time": to_datetime(["2016-07-15 13:30:00.030"]),
+ "username": ["bob"],
+ "version": [np.nan],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_allow_exact_matches_and_tolerance3(self):
+ # GH 13709
+ df1 = pd.DataFrame(
+ {
+ "time": to_datetime(
+ ["2016-07-15 13:30:00.030", "2016-07-15 13:30:00.030"]
+ ),
+ "username": ["bob", "charlie"],
+ }
+ )
+ df2 = pd.DataFrame(
+ {
+ "time": to_datetime(
+ ["2016-07-15 13:30:00.000", "2016-07-15 13:30:00.030"]
+ ),
+ "version": [1, 2],
+ }
+ )
+
+ result = merge_asof(
+ df1,
+ df2,
+ on="time",
+ allow_exact_matches=False,
+ tolerance=Timedelta("10ms"),
+ )
+ expected = pd.DataFrame(
+ {
+ "time": to_datetime(
+ ["2016-07-15 13:30:00.030", "2016-07-15 13:30:00.030"]
+ ),
+ "username": ["bob", "charlie"],
+ "version": [np.nan, np.nan],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_allow_exact_matches_and_tolerance_forward(self):
+ # GH14887
+
+ left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
+ right = pd.DataFrame({"a": [1, 3, 4, 6, 11], "right_val": [1, 3, 4, 6, 11]})
+
+ expected = pd.DataFrame(
+ {"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [np.nan, 6, 11]}
+ )
+
+ result = merge_asof(
+ left,
+ right,
+ on="a",
+ direction="forward",
+ allow_exact_matches=False,
+ tolerance=1,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_allow_exact_matches_and_tolerance_nearest(self):
+ # GH14887
+
+ left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]})
+ right = pd.DataFrame({"a": [1, 3, 4, 6, 11], "right_val": [1, 3, 4, 7, 11]})
+
+ expected = pd.DataFrame(
+ {"a": [1, 5, 10], "left_val": ["a", "b", "c"], "right_val": [np.nan, 4, 11]}
+ )
+
+ result = merge_asof(
+ left,
+ right,
+ on="a",
+ direction="nearest",
+ allow_exact_matches=False,
+ tolerance=1,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_forward_by(self):
+ # GH14887
+
+ left = pd.DataFrame(
+ {
+ "a": [1, 5, 10, 12, 15],
+ "b": ["X", "X", "Y", "Z", "Y"],
+ "left_val": ["a", "b", "c", "d", "e"],
+ }
+ )
+ right = pd.DataFrame(
+ {
+ "a": [1, 6, 11, 15, 16],
+ "b": ["X", "Z", "Y", "Z", "Y"],
+ "right_val": [1, 6, 11, 15, 16],
+ }
+ )
+
+ expected = pd.DataFrame(
+ {
+ "a": [1, 5, 10, 12, 15],
+ "b": ["X", "X", "Y", "Z", "Y"],
+ "left_val": ["a", "b", "c", "d", "e"],
+ "right_val": [1, np.nan, 11, 15, 16],
+ }
+ )
+
+ result = merge_asof(left, right, on="a", by="b", direction="forward")
+ tm.assert_frame_equal(result, expected)
+
+ def test_nearest_by(self):
+ # GH14887
+
+ left = pd.DataFrame(
+ {
+ "a": [1, 5, 10, 12, 15],
+ "b": ["X", "X", "Z", "Z", "Y"],
+ "left_val": ["a", "b", "c", "d", "e"],
+ }
+ )
+ right = pd.DataFrame(
+ {
+ "a": [1, 6, 11, 15, 16],
+ "b": ["X", "Z", "Z", "Z", "Y"],
+ "right_val": [1, 6, 11, 15, 16],
+ }
+ )
+
+ expected = pd.DataFrame(
+ {
+ "a": [1, 5, 10, 12, 15],
+ "b": ["X", "X", "Z", "Z", "Y"],
+ "left_val": ["a", "b", "c", "d", "e"],
+ "right_val": [1, 1, 11, 11, 16],
+ }
+ )
+
+ result = merge_asof(left, right, on="a", by="b", direction="nearest")
+ tm.assert_frame_equal(result, expected)
+
+ def test_by_int(self):
+ # we specialize by type, so test that this is correct
+ df1 = pd.DataFrame(
+ {
+ "time": to_datetime(
+ [
+ "20160525 13:30:00.020",
+ "20160525 13:30:00.030",
+ "20160525 13:30:00.040",
+ "20160525 13:30:00.050",
+ "20160525 13:30:00.060",
+ ]
+ ),
+ "key": [1, 2, 1, 3, 2],
+ "value1": [1.1, 1.2, 1.3, 1.4, 1.5],
+ },
+ columns=["time", "key", "value1"],
+ )
+
+ df2 = pd.DataFrame(
+ {
+ "time": to_datetime(
+ [
+ "20160525 13:30:00.015",
+ "20160525 13:30:00.020",
+ "20160525 13:30:00.025",
+ "20160525 13:30:00.035",
+ "20160525 13:30:00.040",
+ "20160525 13:30:00.055",
+ "20160525 13:30:00.060",
+ "20160525 13:30:00.065",
+ ]
+ ),
+ "key": [2, 1, 1, 3, 2, 1, 2, 3],
+ "value2": [2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8],
+ },
+ columns=["time", "key", "value2"],
+ )
+
+ result = merge_asof(df1, df2, on="time", by="key")
+
+ expected = pd.DataFrame(
+ {
+ "time": to_datetime(
+ [
+ "20160525 13:30:00.020",
+ "20160525 13:30:00.030",
+ "20160525 13:30:00.040",
+ "20160525 13:30:00.050",
+ "20160525 13:30:00.060",
+ ]
+ ),
+ "key": [1, 2, 1, 3, 2],
+ "value1": [1.1, 1.2, 1.3, 1.4, 1.5],
+ "value2": [2.2, 2.1, 2.3, 2.4, 2.7],
+ },
+ columns=["time", "key", "value1", "value2"],
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_on_float(self):
+ # mimics how to determine the minimum-price variation
+ df1 = pd.DataFrame(
+ {
+ "price": [5.01, 0.0023, 25.13, 340.05, 30.78, 1040.90, 0.0078],
+ "symbol": list("ABCDEFG"),
+ },
+ columns=["symbol", "price"],
+ )
+
+ df2 = pd.DataFrame(
+ {"price": [0.0, 1.0, 100.0], "mpv": [0.0001, 0.01, 0.05]},
+ columns=["price", "mpv"],
+ )
+
+ df1 = df1.sort_values("price").reset_index(drop=True)
+
+ result = merge_asof(df1, df2, on="price")
+
+ expected = pd.DataFrame(
+ {
+ "symbol": list("BGACEDF"),
+ "price": [0.0023, 0.0078, 5.01, 25.13, 30.78, 340.05, 1040.90],
+ "mpv": [0.0001, 0.0001, 0.01, 0.01, 0.01, 0.05, 0.05],
+ },
+ columns=["symbol", "price", "mpv"],
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_on_specialized_type(self, any_real_numpy_dtype):
+ # see gh-13936
+ dtype = np.dtype(any_real_numpy_dtype).type
+
+ df1 = pd.DataFrame(
+ {"value": [5, 2, 25, 100, 78, 120, 79], "symbol": list("ABCDEFG")},
+ columns=["symbol", "value"],
+ )
+ df1.value = dtype(df1.value)
+
+ df2 = pd.DataFrame(
+ {"value": [0, 80, 120, 125], "result": list("xyzw")},
+ columns=["value", "result"],
+ )
+ df2.value = dtype(df2.value)
+
+ df1 = df1.sort_values("value").reset_index(drop=True)
+ result = merge_asof(df1, df2, on="value")
+
+ expected = pd.DataFrame(
+ {
+ "symbol": list("BACEGDF"),
+ "value": [2, 5, 25, 78, 79, 100, 120],
+ "result": list("xxxxxyz"),
+ },
+ columns=["symbol", "value", "result"],
+ )
+ expected.value = dtype(expected.value)
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_on_specialized_type_by_int(self, any_real_numpy_dtype):
+ # see gh-13936
+ dtype = np.dtype(any_real_numpy_dtype).type
+
+ df1 = pd.DataFrame(
+ {
+ "value": [5, 2, 25, 100, 78, 120, 79],
+ "key": [1, 2, 3, 2, 3, 1, 2],
+ "symbol": list("ABCDEFG"),
+ },
+ columns=["symbol", "key", "value"],
+ )
+ df1.value = dtype(df1.value)
+
+ df2 = pd.DataFrame(
+ {"value": [0, 80, 120, 125], "key": [1, 2, 2, 3], "result": list("xyzw")},
+ columns=["value", "key", "result"],
+ )
+ df2.value = dtype(df2.value)
+
+ df1 = df1.sort_values("value").reset_index(drop=True)
+ result = merge_asof(df1, df2, on="value", by="key")
+
+ expected = pd.DataFrame(
+ {
+ "symbol": list("BACEGDF"),
+ "key": [2, 1, 3, 3, 2, 2, 1],
+ "value": [2, 5, 25, 78, 79, 100, 120],
+ "result": [np.nan, "x", np.nan, np.nan, np.nan, "y", "x"],
+ },
+ columns=["symbol", "key", "value", "result"],
+ )
+ expected.value = dtype(expected.value)
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_on_float_by_int(self):
+ # type specialize both "by" and "on" parameters
+ df1 = pd.DataFrame(
+ {
+ "symbol": list("AAABBBCCC"),
+ "exch": [1, 2, 3, 1, 2, 3, 1, 2, 3],
+ "price": [
+ 3.26,
+ 3.2599,
+ 3.2598,
+ 12.58,
+ 12.59,
+ 12.5,
+ 378.15,
+ 378.2,
+ 378.25,
+ ],
+ },
+ columns=["symbol", "exch", "price"],
+ )
+
+ df2 = pd.DataFrame(
+ {
+ "exch": [1, 1, 1, 2, 2, 2, 3, 3, 3],
+ "price": [0.0, 1.0, 100.0, 0.0, 5.0, 100.0, 0.0, 5.0, 1000.0],
+ "mpv": [0.0001, 0.01, 0.05, 0.0001, 0.01, 0.1, 0.0001, 0.25, 1.0],
+ },
+ columns=["exch", "price", "mpv"],
+ )
+
+ df1 = df1.sort_values("price").reset_index(drop=True)
+ df2 = df2.sort_values("price").reset_index(drop=True)
+
+ result = merge_asof(df1, df2, on="price", by="exch")
+
+ expected = pd.DataFrame(
+ {
+ "symbol": list("AAABBBCCC"),
+ "exch": [3, 2, 1, 3, 1, 2, 1, 2, 3],
+ "price": [
+ 3.2598,
+ 3.2599,
+ 3.26,
+ 12.5,
+ 12.58,
+ 12.59,
+ 378.15,
+ 378.2,
+ 378.25,
+ ],
+ "mpv": [0.0001, 0.0001, 0.01, 0.25, 0.01, 0.01, 0.05, 0.1, 0.25],
+ },
+ columns=["symbol", "exch", "price", "mpv"],
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_datatype_error_raises(self):
+ msg = r"Incompatible merge dtype, .*, both sides must have numeric dtype"
+
+ left = pd.DataFrame({"left_val": [1, 5, 10], "a": ["a", "b", "c"]})
+ right = pd.DataFrame({"right_val": [1, 2, 3, 6, 7], "a": [1, 2, 3, 6, 7]})
+
+ with pytest.raises(MergeError, match=msg):
+ merge_asof(left, right, on="a")
+
+ def test_merge_datatype_categorical_error_raises(self):
+ msg = (
+ r"incompatible merge keys \[0\] .* both sides category, "
+ "but not equal ones"
+ )
+
+ left = pd.DataFrame(
+ {"left_val": [1, 5, 10], "a": pd.Categorical(["a", "b", "c"])}
+ )
+ right = pd.DataFrame(
+ {
+ "right_val": [1, 2, 3, 6, 7],
+ "a": pd.Categorical(["a", "X", "c", "X", "b"]),
+ }
+ )
+
+ with pytest.raises(MergeError, match=msg):
+ merge_asof(left, right, on="a")
+
+ def test_merge_groupby_multiple_column_with_categorical_column(self):
+ # GH 16454
+ df = pd.DataFrame({"x": [0], "y": [0], "z": pd.Categorical([0])})
+ result = merge_asof(df, df, on="x", by=["y", "z"])
+ expected = pd.DataFrame({"x": [0], "y": [0], "z": pd.Categorical([0])})
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "func", [lambda x: x, to_datetime], ids=["numeric", "datetime"]
+ )
+ @pytest.mark.parametrize("side", ["left", "right"])
+ def test_merge_on_nans(self, func, side):
+ # GH 23189
+ msg = f"Merge keys contain null values on {side} side"
+ nulls = func([1.0, 5.0, np.nan])
+ non_nulls = func([1.0, 5.0, 10.0])
+ df_null = pd.DataFrame({"a": nulls, "left_val": ["a", "b", "c"]})
+ df = pd.DataFrame({"a": non_nulls, "right_val": [1, 6, 11]})
+
+ with pytest.raises(ValueError, match=msg):
+ if side == "left":
+ merge_asof(df_null, df, on="a")
+ else:
+ merge_asof(df, df_null, on="a")
+
+ def test_by_nullable(self, any_numeric_ea_dtype, using_infer_string):
+ # Note: this test passes if instead of using pd.array we use
+ # np.array([np.nan, 1]). Other than that, I (@jbrockmendel)
+ # have NO IDEA what the expected behavior is.
+ # TODO(GH#32306): may be relevant to the expected behavior here.
+
+ arr = pd.array([pd.NA, 0, 1], dtype=any_numeric_ea_dtype)
+ if arr.dtype.kind in ["i", "u"]:
+ max_val = np.iinfo(arr.dtype.numpy_dtype).max
+ else:
+ max_val = np.finfo(arr.dtype.numpy_dtype).max
+ # set value s.t. (at least for integer dtypes) arr._values_for_argsort
+ # is not an injection
+ arr[2] = max_val
+
+ left = pd.DataFrame(
+ {
+ "by_col1": arr,
+ "by_col2": ["HELLO", "To", "You"],
+ "on_col": [2, 4, 6],
+ "value": ["a", "c", "e"],
+ }
+ )
+ right = pd.DataFrame(
+ {
+ "by_col1": arr,
+ "by_col2": ["WORLD", "Wide", "Web"],
+ "on_col": [1, 2, 6],
+ "value": ["b", "d", "f"],
+ }
+ )
+
+ result = merge_asof(left, right, by=["by_col1", "by_col2"], on="on_col")
+ expected = pd.DataFrame(
+ {
+ "by_col1": arr,
+ "by_col2": ["HELLO", "To", "You"],
+ "on_col": [2, 4, 6],
+ "value_x": ["a", "c", "e"],
+ }
+ )
+ expected["value_y"] = np.array([np.nan, np.nan, np.nan], dtype=object)
+ if using_infer_string:
+ expected["value_y"] = expected["value_y"].astype("str")
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_by_col_tz_aware(self):
+ # GH 21184
+ left = pd.DataFrame(
+ {
+ "by_col": pd.DatetimeIndex(["2018-01-01"]).tz_localize("UTC"),
+ "on_col": [2],
+ "values": ["a"],
+ }
+ )
+ right = pd.DataFrame(
+ {
+ "by_col": pd.DatetimeIndex(["2018-01-01"]).tz_localize("UTC"),
+ "on_col": [1],
+ "values": ["b"],
+ }
+ )
+ result = merge_asof(left, right, by="by_col", on="on_col")
+ expected = pd.DataFrame(
+ [[pd.Timestamp("2018-01-01", tz="UTC"), 2, "a", "b"]],
+ columns=["by_col", "on_col", "values_x", "values_y"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_by_mixed_tz_aware(self, using_infer_string):
+ # GH 26649
+ left = pd.DataFrame(
+ {
+ "by_col1": pd.DatetimeIndex(["2018-01-01"]).tz_localize("UTC"),
+ "by_col2": ["HELLO"],
+ "on_col": [2],
+ "value": ["a"],
+ }
+ )
+ right = pd.DataFrame(
+ {
+ "by_col1": pd.DatetimeIndex(["2018-01-01"]).tz_localize("UTC"),
+ "by_col2": ["WORLD"],
+ "on_col": [1],
+ "value": ["b"],
+ }
+ )
+ result = merge_asof(left, right, by=["by_col1", "by_col2"], on="on_col")
+ expected = pd.DataFrame(
+ [[pd.Timestamp("2018-01-01", tz="UTC"), "HELLO", 2, "a"]],
+ columns=["by_col1", "by_col2", "on_col", "value_x"],
+ )
+ expected["value_y"] = np.array([np.nan], dtype=object)
+ if using_infer_string:
+ expected["value_y"] = expected["value_y"].astype("str")
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("dtype", ["float64", "int16", "m8[ns]", "M8[us]"])
+ def test_by_dtype(self, dtype):
+ # GH 55453, GH 22794
+ left = pd.DataFrame(
+ {
+ "by_col": np.array([1], dtype=dtype),
+ "on_col": [2],
+ "value": ["a"],
+ }
+ )
+ right = pd.DataFrame(
+ {
+ "by_col": np.array([1], dtype=dtype),
+ "on_col": [1],
+ "value": ["b"],
+ }
+ )
+ result = merge_asof(left, right, by="by_col", on="on_col")
+ expected = pd.DataFrame(
+ {
+ "by_col": np.array([1], dtype=dtype),
+ "on_col": [2],
+ "value_x": ["a"],
+ "value_y": ["b"],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_timedelta_tolerance_nearest(self, unit):
+ # GH 27642
+ if unit == "s":
+ pytest.skip(
+ "This test is invalid with unit='s' because that would "
+ "round left['time']"
+ )
+
+ left = pd.DataFrame(
+ list(zip([0, 5, 10, 15, 20, 25], [0, 1, 2, 3, 4, 5], strict=True)),
+ columns=["time", "left"],
+ )
+
+ left["time"] = pd.to_timedelta(left["time"], "ms").astype(f"m8[{unit}]")
+
+ right = pd.DataFrame(
+ list(zip([0, 3, 9, 12, 15, 18], [0, 1, 2, 3, 4, 5], strict=True)),
+ columns=["time", "right"],
+ )
+
+ right["time"] = pd.to_timedelta(right["time"], "ms").astype(f"m8[{unit}]")
+
+ expected = pd.DataFrame(
+ list(
+ zip(
+ [0, 5, 10, 15, 20, 25],
+ [0, 1, 2, 3, 4, 5],
+ [0, np.nan, 2, 4, np.nan, np.nan],
+ strict=True,
+ )
+ ),
+ columns=["time", "left", "right"],
+ )
+
+ expected["time"] = pd.to_timedelta(expected["time"], "ms").astype(f"m8[{unit}]")
+
+ result = merge_asof(
+ left, right, on="time", tolerance=Timedelta("1ms"), direction="nearest"
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_int_type_tolerance(self, any_int_dtype):
+ # GH #28870
+
+ left = pd.DataFrame({"a": [0, 10, 20], "left_val": [1, 2, 3]})
+ right = pd.DataFrame({"a": [5, 15, 25], "right_val": [1, 2, 3]})
+ left["a"] = left["a"].astype(any_int_dtype)
+ right["a"] = right["a"].astype(any_int_dtype)
+
+ expected = pd.DataFrame(
+ {"a": [0, 10, 20], "left_val": [1, 2, 3], "right_val": [np.nan, 1.0, 2.0]}
+ )
+ expected["a"] = expected["a"].astype(any_int_dtype)
+
+ result = merge_asof(left, right, on="a", tolerance=10)
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_index_column_tz(self):
+ # GH 29864
+ index = pd.date_range("2019-10-01", freq="30min", periods=5, tz="UTC")
+ left = pd.DataFrame([0.9, 0.8, 0.7, 0.6], columns=["xyz"], index=index[1:])
+ right = pd.DataFrame({"from_date": index, "abc": [2.46] * 4 + [2.19]})
+ result = merge_asof(
+ left=left, right=right, left_index=True, right_on=["from_date"]
+ )
+ expected = pd.DataFrame(
+ {
+ "xyz": [0.9, 0.8, 0.7, 0.6],
+ "from_date": index[1:],
+ "abc": [2.46] * 3 + [2.19],
+ },
+ index=pd.date_range(
+ "2019-10-01 00:30:00", freq="30min", periods=4, tz="UTC"
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = merge_asof(
+ left=right, right=left, right_index=True, left_on=["from_date"]
+ )
+ expected = pd.DataFrame(
+ {
+ "from_date": index,
+ "abc": [2.46] * 4 + [2.19],
+ "xyz": [np.nan, 0.9, 0.8, 0.7, 0.6],
+ },
+ index=Index([0, 1, 2, 3, 4]),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_left_index_right_index_tolerance(self, unit):
+ # https://github.com/pandas-dev/pandas/issues/35558
+ if unit == "s":
+ pytest.skip(
+ "This test is invalid with unit='s' because that would round dr1"
+ )
+
+ dr1 = pd.date_range(
+ start="1/1/2020", end="1/20/2020", freq="2D", unit=unit
+ ) + Timedelta(seconds=0.4).as_unit(unit)
+ dr2 = pd.date_range(start="1/1/2020", end="2/1/2020", unit=unit)
+
+ df1 = pd.DataFrame({"val1": "foo"}, index=pd.DatetimeIndex(dr1))
+ df2 = pd.DataFrame({"val2": "bar"}, index=pd.DatetimeIndex(dr2))
+
+ expected = pd.DataFrame(
+ {"val1": "foo", "val2": "bar"}, index=pd.DatetimeIndex(dr1)
+ )
+ result = merge_asof(
+ df1,
+ df2,
+ left_index=True,
+ right_index=True,
+ tolerance=Timedelta(seconds=0.5),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "infer_string", [False, pytest.param(True, marks=td.skip_if_no("pyarrow"))]
+)
+@pytest.mark.parametrize(
+ "kwargs", [{"on": "x"}, {"left_index": True, "right_index": True}]
+)
+@pytest.mark.parametrize(
+ "data",
+ [["2019-06-01 00:09:12", "2019-06-01 00:10:29"], [1.0, "2019-06-01 00:10:29"]],
+)
+def test_merge_asof_non_numerical_dtype(kwargs, data, infer_string):
+ # GH#29130
+ with option_context("future.infer_string", infer_string):
+ left = pd.DataFrame({"x": data}, index=data)
+ right = pd.DataFrame({"x": data}, index=data)
+ with pytest.raises(
+ MergeError,
+ match=r"Incompatible merge dtype, .*, both sides must have numeric dtype",
+ ):
+ merge_asof(left, right, **kwargs)
+
+
+def test_merge_asof_non_numerical_dtype_object():
+ # GH#29130
+ left = pd.DataFrame({"a": ["12", "13", "15"], "left_val1": ["a", "b", "c"]})
+ right = pd.DataFrame({"a": ["a", "b", "c"], "left_val": ["d", "e", "f"]})
+ with pytest.raises(
+ MergeError,
+ match=r"Incompatible merge dtype, .*, both sides must have numeric dtype",
+ ):
+ merge_asof(
+ left,
+ right,
+ left_on="left_val1",
+ right_on="a",
+ left_by="a",
+ right_by="left_val",
+ )
+
+
+@pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"right_index": True, "left_index": True},
+ {"left_on": "left_time", "right_index": True},
+ {"left_index": True, "right_on": "right"},
+ ],
+)
+def test_merge_asof_index_behavior(kwargs):
+ # GH 33463
+ index = Index([1, 5, 10], name="test")
+ left = pd.DataFrame({"left": ["a", "b", "c"], "left_time": [1, 4, 10]}, index=index)
+ right = pd.DataFrame({"right": [1, 2, 3, 6, 7]}, index=[1, 2, 3, 6, 7])
+ result = merge_asof(left, right, **kwargs)
+
+ expected = pd.DataFrame(
+ {"left": ["a", "b", "c"], "left_time": [1, 4, 10], "right": [1, 3, 7]},
+ index=index,
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_asof_numeric_column_in_index():
+ # GH#34488
+ left = pd.DataFrame({"b": [10, 11, 12]}, index=Index([1, 2, 3], name="a"))
+ right = pd.DataFrame({"c": [20, 21, 22]}, index=Index([0, 2, 3], name="a"))
+
+ result = merge_asof(left, right, left_on="a", right_on="a")
+ expected = pd.DataFrame({"a": [1, 2, 3], "b": [10, 11, 12], "c": [20, 21, 22]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_asof_numeric_column_in_multiindex():
+ # GH#34488
+ left = pd.DataFrame(
+ {"b": [10, 11, 12]},
+ index=pd.MultiIndex.from_arrays([[1, 2, 3], ["a", "b", "c"]], names=["a", "z"]),
+ )
+ right = pd.DataFrame(
+ {"c": [20, 21, 22]},
+ index=pd.MultiIndex.from_arrays([[1, 2, 3], ["x", "y", "z"]], names=["a", "y"]),
+ )
+
+ result = merge_asof(left, right, left_on="a", right_on="a")
+ expected = pd.DataFrame({"a": [1, 2, 3], "b": [10, 11, 12], "c": [20, 21, 22]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_asof_numeri_column_in_index_object_dtype():
+ # GH#34488
+ left = pd.DataFrame({"b": [10, 11, 12]}, index=Index(["1", "2", "3"], name="a"))
+ right = pd.DataFrame({"c": [20, 21, 22]}, index=Index(["m", "n", "o"], name="a"))
+
+ with pytest.raises(
+ MergeError,
+ match=r"Incompatible merge dtype, .*, both sides must have numeric dtype",
+ ):
+ merge_asof(left, right, left_on="a", right_on="a")
+
+ left = left.reset_index().set_index(["a", "b"])
+ right = right.reset_index().set_index(["a", "c"])
+
+ with pytest.raises(
+ MergeError,
+ match=r"Incompatible merge dtype, .*, both sides must have numeric dtype",
+ ):
+ merge_asof(left, right, left_on="a", right_on="a")
+
+
+def test_merge_asof_array_as_on(unit):
+ # GH#42844
+ dti = pd.DatetimeIndex(
+ ["2021/01/01 00:37", "2021/01/01 01:40"], dtype=f"M8[{unit}]"
+ )
+ right = pd.DataFrame(
+ {
+ "a": [2, 6],
+ "ts": dti,
+ }
+ )
+ ts_merge = pd.date_range(
+ start=pd.Timestamp("2021/01/01 00:00"), periods=3, freq="1h", unit=unit
+ )
+ left = pd.DataFrame({"b": [4, 8, 7]})
+ result = merge_asof(
+ left,
+ right,
+ left_on=ts_merge,
+ right_on="ts",
+ allow_exact_matches=False,
+ direction="backward",
+ )
+ expected = pd.DataFrame({"b": [4, 8, 7], "a": [np.nan, 2, 6], "ts": ts_merge})
+ tm.assert_frame_equal(result, expected)
+
+ result = merge_asof(
+ right,
+ left,
+ left_on="ts",
+ right_on=ts_merge,
+ allow_exact_matches=False,
+ direction="backward",
+ )
+ expected = pd.DataFrame(
+ {
+ "a": [2, 6],
+ "ts": dti,
+ "b": [4, 8],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_asof_raise_for_duplicate_columns():
+ # GH#50102
+ left = pd.DataFrame([[1, 2, "a"]], columns=["a", "a", "left_val"])
+ right = pd.DataFrame([[1, 1, 1]], columns=["a", "a", "right_val"])
+
+ with pytest.raises(ValueError, match="column label 'a'"):
+ merge_asof(left, right, on="a")
+
+ with pytest.raises(ValueError, match="column label 'a'"):
+ merge_asof(left, right, left_on="a", right_on="right_val")
+
+ with pytest.raises(ValueError, match="column label 'a'"):
+ merge_asof(left, right, left_on="left_val", right_on="a")
+
+
+@pytest.mark.parametrize(
+ "dtype",
+ [
+ "Int64",
+ pytest.param("int64[pyarrow]", marks=td.skip_if_no("pyarrow")),
+ pytest.param("timestamp[s][pyarrow]", marks=td.skip_if_no("pyarrow")),
+ ],
+)
+def test_merge_asof_extension_dtype(dtype):
+ # GH 52904
+ left = pd.DataFrame(
+ {
+ "join_col": [1, 3, 5],
+ "left_val": [1, 2, 3],
+ }
+ )
+ right = pd.DataFrame(
+ {
+ "join_col": [2, 3, 4],
+ "right_val": [1, 2, 3],
+ }
+ )
+ left = left.astype({"join_col": dtype})
+ right = right.astype({"join_col": dtype})
+ result = merge_asof(left, right, on="join_col")
+ expected = pd.DataFrame(
+ {
+ "join_col": [1, 3, 5],
+ "left_val": [1, 2, 3],
+ "right_val": [np.nan, 2.0, 3.0],
+ }
+ )
+ expected = expected.astype({"join_col": dtype})
+ tm.assert_frame_equal(result, expected)
+
+
+@td.skip_if_no("pyarrow")
+def test_merge_asof_pyarrow_td_tolerance():
+ # GH 56486
+ ser = pd.Series(
+ [datetime.datetime(2023, 1, 1)], dtype="timestamp[us, UTC][pyarrow]"
+ )
+ df = pd.DataFrame(
+ {
+ "timestamp": ser,
+ "value": [1],
+ }
+ )
+ result = merge_asof(df, df, on="timestamp", tolerance=Timedelta("1s"))
+ expected = pd.DataFrame(
+ {
+ "timestamp": ser,
+ "value_x": [1],
+ "value_y": [1],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_asof_read_only_ndarray():
+ # GH 53513
+ left = pd.Series([2], index=[2], name="left")
+ right = pd.Series([1], index=[1], name="right")
+ # set to read-only
+ left.index.values.flags.writeable = False
+ right.index.values.flags.writeable = False
+ result = merge_asof(left, right, left_index=True, right_index=True)
+ expected = pd.DataFrame({"left": [2], "right": [1]}, index=[2])
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_asof_multiby_with_categorical():
+ # GH 43541
+ left = pd.DataFrame(
+ {
+ "c1": pd.Categorical(["a", "a", "b", "b"], categories=["a", "b"]),
+ "c2": ["x"] * 4,
+ "t": [1] * 4,
+ "v": range(4),
+ }
+ )
+ right = pd.DataFrame(
+ {
+ "c1": pd.Categorical(["b", "b"], categories=["b", "a"]),
+ "c2": ["x"] * 2,
+ "t": [1, 2],
+ "v": range(2),
+ }
+ )
+ result = merge_asof(
+ left,
+ right,
+ by=["c1", "c2"],
+ on="t",
+ direction="forward",
+ suffixes=["_left", "_right"],
+ )
+ expected = pd.DataFrame(
+ {
+ "c1": pd.Categorical(["a", "a", "b", "b"], categories=["a", "b"]),
+ "c2": ["x"] * 4,
+ "t": [1] * 4,
+ "v_left": range(4),
+ "v_right": [np.nan, np.nan, 0.0, 0.0],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_cross.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_cross.py
new file mode 100644
index 0000000000000000000000000000000000000000..6ab80cf0e0823e2f953c6bcdf0aaa56ed5db028e
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_cross.py
@@ -0,0 +1,109 @@
+import pytest
+
+from pandas import (
+ DataFrame,
+ Series,
+)
+import pandas._testing as tm
+from pandas.core.reshape.merge import (
+ MergeError,
+ merge,
+)
+
+
+@pytest.mark.parametrize(
+ ("input_col", "output_cols"), [("b", ["a", "b"]), ("a", ["a_x", "a_y"])]
+)
+def test_merge_cross(input_col, output_cols):
+ # GH#5401
+ left = DataFrame({"a": [1, 3]})
+ right = DataFrame({input_col: [3, 4]})
+ left_copy = left.copy()
+ right_copy = right.copy()
+ result = merge(left, right, how="cross")
+ expected = DataFrame({output_cols[0]: [1, 1, 3, 3], output_cols[1]: [3, 4, 3, 4]})
+ tm.assert_frame_equal(result, expected)
+ tm.assert_frame_equal(left, left_copy)
+ tm.assert_frame_equal(right, right_copy)
+
+
+@pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"left_index": True},
+ {"right_index": True},
+ {"on": "a"},
+ {"left_on": "a"},
+ {"right_on": "b"},
+ ],
+)
+def test_merge_cross_error_reporting(kwargs):
+ # GH#5401
+ left = DataFrame({"a": [1, 3]})
+ right = DataFrame({"b": [3, 4]})
+ msg = (
+ "Can not pass on, right_on, left_on or set right_index=True or left_index=True"
+ )
+ with pytest.raises(MergeError, match=msg):
+ merge(left, right, how="cross", **kwargs)
+
+
+def test_merge_cross_mixed_dtypes():
+ # GH#5401
+ left = DataFrame(["a", "b", "c"], columns=["A"])
+ right = DataFrame(range(2), columns=["B"])
+ result = merge(left, right, how="cross")
+ expected = DataFrame({"A": ["a", "a", "b", "b", "c", "c"], "B": [0, 1, 0, 1, 0, 1]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_cross_more_than_one_column():
+ # GH#5401
+ left = DataFrame({"A": list("ab"), "B": [2, 1]})
+ right = DataFrame({"C": range(2), "D": range(4, 6)})
+ result = merge(left, right, how="cross")
+ expected = DataFrame(
+ {
+ "A": ["a", "a", "b", "b"],
+ "B": [2, 2, 1, 1],
+ "C": [0, 1, 0, 1],
+ "D": [4, 5, 4, 5],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_merge_cross_null_values(nulls_fixture):
+ # GH#5401
+ left = DataFrame({"a": [1, nulls_fixture]})
+ right = DataFrame({"b": ["a", "b"], "c": [1.0, 2.0]})
+ result = merge(left, right, how="cross")
+ expected = DataFrame(
+ {
+ "a": [1, 1, nulls_fixture, nulls_fixture],
+ "b": ["a", "b", "a", "b"],
+ "c": [1.0, 2.0, 1.0, 2.0],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_join_cross_error_reporting():
+ # GH#5401
+ left = DataFrame({"a": [1, 3]})
+ right = DataFrame({"a": [3, 4]})
+ msg = (
+ "Can not pass on, right_on, left_on or set right_index=True or left_index=True"
+ )
+ with pytest.raises(MergeError, match=msg):
+ left.join(right, how="cross", on="a")
+
+
+def test_merge_cross_series():
+ # GH#54055
+ ls = Series([1, 2, 3, 4], index=[1, 2, 3, 4], name="left")
+ rs = Series([3, 4, 5, 6], index=[3, 4, 5, 6], name="right")
+ res = merge(ls, rs, how="cross")
+
+ expected = merge(ls.to_frame(), rs.to_frame(), how="cross")
+ tm.assert_frame_equal(res, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_index_as_string.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_index_as_string.py
new file mode 100644
index 0000000000000000000000000000000000000000..b4271d4face4fb4e90477196cec7919845f37813
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_index_as_string.py
@@ -0,0 +1,186 @@
+import numpy as np
+import pytest
+
+from pandas import DataFrame
+import pandas._testing as tm
+
+
+@pytest.fixture
+def df1():
+ return DataFrame(
+ {
+ "outer": [1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4],
+ "inner": [1, 2, 3, 1, 2, 3, 4, 1, 2, 1, 2],
+ "v1": np.linspace(0, 1, 11),
+ }
+ )
+
+
+@pytest.fixture
+def df2():
+ return DataFrame(
+ {
+ "outer": [1, 1, 1, 1, 1, 1, 2, 2, 3, 3, 3, 3],
+ "inner": [1, 2, 2, 3, 3, 4, 2, 3, 1, 1, 2, 3],
+ "v2": np.linspace(10, 11, 12),
+ }
+ )
+
+
+@pytest.fixture(params=[[], ["outer"], ["outer", "inner"]])
+def left_df(request, df1):
+ """Construct left test DataFrame with specified levels
+ (any of 'outer', 'inner', and 'v1')
+ """
+ levels = request.param
+ if levels:
+ df1 = df1.set_index(levels)
+
+ return df1
+
+
+@pytest.fixture(params=[[], ["outer"], ["outer", "inner"]])
+def right_df(request, df2):
+ """Construct right test DataFrame with specified levels
+ (any of 'outer', 'inner', and 'v2')
+ """
+ levels = request.param
+
+ if levels:
+ df2 = df2.set_index(levels)
+
+ return df2
+
+
+def compute_expected(df_left, df_right, on=None, left_on=None, right_on=None, how=None):
+ """
+ Compute the expected merge result for the test case.
+
+ This method computes the expected result of merging two DataFrames on
+ a combination of their columns and index levels. It does so by
+ explicitly dropping/resetting their named index levels, performing a
+ merge on their columns, and then finally restoring the appropriate
+ index in the result.
+
+ Parameters
+ ----------
+ df_left : DataFrame
+ The left DataFrame (may have zero or more named index levels)
+ df_right : DataFrame
+ The right DataFrame (may have zero or more named index levels)
+ on : list of str
+ The on parameter to the merge operation
+ left_on : list of str
+ The left_on parameter to the merge operation
+ right_on : list of str
+ The right_on parameter to the merge operation
+ how : str
+ The how parameter to the merge operation
+
+ Returns
+ -------
+ DataFrame
+ The expected merge result
+ """
+ # Handle on param if specified
+ if on is not None:
+ left_on, right_on = on, on
+
+ # Compute input named index levels
+ left_levels = [n for n in df_left.index.names if n is not None]
+ right_levels = [n for n in df_right.index.names if n is not None]
+
+ # Compute output named index levels
+ output_levels = [i for i in left_on if i in right_levels and i in left_levels]
+
+ # Drop index levels that aren't involved in the merge
+ drop_left = [n for n in left_levels if n not in left_on]
+ if drop_left:
+ df_left = df_left.reset_index(drop_left, drop=True)
+
+ drop_right = [n for n in right_levels if n not in right_on]
+ if drop_right:
+ df_right = df_right.reset_index(drop_right, drop=True)
+
+ # Convert remaining index levels to columns
+ reset_left = [n for n in left_levels if n in left_on]
+ if reset_left:
+ df_left = df_left.reset_index(level=reset_left)
+
+ reset_right = [n for n in right_levels if n in right_on]
+ if reset_right:
+ df_right = df_right.reset_index(level=reset_right)
+
+ # Perform merge
+ expected = df_left.merge(df_right, left_on=left_on, right_on=right_on, how=how)
+
+ # Restore index levels
+ if output_levels:
+ expected = expected.set_index(output_levels)
+
+ return expected
+
+
+@pytest.mark.parametrize(
+ "on,how",
+ [
+ (["outer"], "inner"),
+ (["inner"], "left"),
+ (["outer", "inner"], "right"),
+ (["inner", "outer"], "outer"),
+ ],
+)
+def test_merge_indexes_and_columns_on(left_df, right_df, on, how):
+ # Construct expected result
+ expected = compute_expected(left_df, right_df, on=on, how=how)
+
+ # Perform merge
+ result = left_df.merge(right_df, on=on, how=how)
+ tm.assert_frame_equal(result, expected, check_like=True)
+
+
+@pytest.mark.parametrize(
+ "left_on,right_on,how",
+ [
+ (["outer"], ["outer"], "inner"),
+ (["inner"], ["inner"], "right"),
+ (["outer", "inner"], ["outer", "inner"], "left"),
+ (["inner", "outer"], ["inner", "outer"], "outer"),
+ ],
+)
+def test_merge_indexes_and_columns_lefton_righton(
+ left_df, right_df, left_on, right_on, how
+):
+ # Construct expected result
+ expected = compute_expected(
+ left_df, right_df, left_on=left_on, right_on=right_on, how=how
+ )
+
+ # Perform merge
+ result = left_df.merge(right_df, left_on=left_on, right_on=right_on, how=how)
+ tm.assert_frame_equal(result, expected, check_like=True)
+
+
+@pytest.mark.parametrize("left_index", ["inner", ["inner", "outer"]])
+def test_join_indexes_and_columns_on(df1, df2, left_index, join_type):
+ # Construct left_df
+ left_df = df1.set_index(left_index)
+
+ # Construct right_df
+ right_df = df2.set_index(["outer", "inner"])
+
+ # Result
+ expected = (
+ left_df.reset_index()
+ .join(
+ right_df, on=["outer", "inner"], how=join_type, lsuffix="_x", rsuffix="_y"
+ )
+ .set_index(left_index)
+ )
+
+ # Perform join
+ result = left_df.join(
+ right_df, on=["outer", "inner"], how=join_type, lsuffix="_x", rsuffix="_y"
+ )
+
+ tm.assert_frame_equal(result, expected, check_like=True)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_ordered.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_ordered.py
new file mode 100644
index 0000000000000000000000000000000000000000..7deecf6bbf4a41a36d7fd3db11cb32ab509926b5
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_merge_ordered.py
@@ -0,0 +1,241 @@
+import re
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ merge_ordered,
+)
+import pandas._testing as tm
+
+
+@pytest.fixture
+def left():
+ return DataFrame({"key": ["a", "c", "e"], "lvalue": [1, 2.0, 3]})
+
+
+@pytest.fixture
+def right():
+ return DataFrame({"key": ["b", "c", "d", "f"], "rvalue": [1, 2, 3.0, 4]})
+
+
+class TestMergeOrdered:
+ def test_basic(self, left, right):
+ result = merge_ordered(left, right, on="key")
+ expected = DataFrame(
+ {
+ "key": ["a", "b", "c", "d", "e", "f"],
+ "lvalue": [1, np.nan, 2, np.nan, 3, np.nan],
+ "rvalue": [np.nan, 1, 2, 3, np.nan, 4],
+ }
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_ffill(self, left, right):
+ result = merge_ordered(left, right, on="key", fill_method="ffill")
+ expected = DataFrame(
+ {
+ "key": ["a", "b", "c", "d", "e", "f"],
+ "lvalue": [1.0, 1, 2, 2, 3, 3.0],
+ "rvalue": [np.nan, 1, 2, 3, 3, 4],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_multigroup(self, left, right):
+ left = pd.concat([left, left], ignore_index=True)
+
+ left["group"] = ["a"] * 3 + ["b"] * 3
+
+ result = merge_ordered(
+ left, right, on="key", left_by="group", fill_method="ffill"
+ )
+ expected = DataFrame(
+ {
+ "key": ["a", "b", "c", "d", "e", "f"] * 2,
+ "lvalue": [1.0, 1, 2, 2, 3, 3.0] * 2,
+ "rvalue": [np.nan, 1, 2, 3, 3, 4] * 2,
+ }
+ )
+ expected["group"] = ["a"] * 6 + ["b"] * 6
+
+ tm.assert_frame_equal(result, expected.loc[:, result.columns])
+
+ result2 = merge_ordered(
+ right, left, on="key", right_by="group", fill_method="ffill"
+ )
+ tm.assert_frame_equal(result, result2.loc[:, result.columns])
+
+ result = merge_ordered(left, right, on="key", left_by="group")
+ assert result["group"].notna().all()
+
+ @pytest.mark.filterwarnings(
+ "ignore:Passing a BlockManager|Passing a SingleBlockManager:DeprecationWarning"
+ )
+ def test_merge_type(self, left, right):
+ class NotADataFrame(DataFrame):
+ @property
+ def _constructor(self):
+ return NotADataFrame
+
+ nad = NotADataFrame(left)
+ result = nad.merge(right, on="key")
+
+ assert isinstance(result, NotADataFrame)
+
+ @pytest.mark.parametrize(
+ "df_seq, pattern",
+ [
+ ((), "[Nn]o objects"),
+ ([], "[Nn]o objects"),
+ ({}, "[Nn]o objects"),
+ ([None], "objects.*None"),
+ ([None, None], "objects.*None"),
+ ],
+ )
+ def test_empty_sequence_concat(self, df_seq, pattern):
+ # GH 9157
+ with pytest.raises(ValueError, match=pattern):
+ pd.concat(df_seq)
+
+ @pytest.mark.parametrize(
+ "arg", [[DataFrame()], [None, DataFrame()], [DataFrame(), None]]
+ )
+ def test_empty_sequence_concat_ok(self, arg):
+ pd.concat(arg)
+
+ def test_doc_example(self):
+ left = DataFrame(
+ {
+ "group": list("aaabbb"),
+ "key": ["a", "c", "e", "a", "c", "e"],
+ "lvalue": [1, 2, 3] * 2,
+ }
+ )
+
+ right = DataFrame({"key": ["b", "c", "d"], "rvalue": [1, 2, 3]})
+
+ result = merge_ordered(left, right, fill_method="ffill", left_by="group")
+
+ expected = DataFrame(
+ {
+ "group": list("aaaaabbbbb"),
+ "key": ["a", "b", "c", "d", "e"] * 2,
+ "lvalue": [1, 1, 2, 2, 3] * 2,
+ "rvalue": [np.nan, 1, 2, 3, 3] * 2,
+ }
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "left, right, on, left_by, right_by, expected",
+ [
+ (
+ {"G": ["g", "g"], "H": ["h", "h"], "T": [1, 3]},
+ {"T": [2], "E": [1]},
+ ["T"],
+ ["G", "H"],
+ None,
+ {
+ "G": ["g"] * 3,
+ "H": ["h"] * 3,
+ "T": [1, 2, 3],
+ "E": [np.nan, 1.0, np.nan],
+ },
+ ),
+ (
+ {"G": ["g", "g"], "H": ["h", "h"], "T": [1, 3]},
+ {"T": [2], "E": [1]},
+ "T",
+ ["G", "H"],
+ None,
+ {
+ "G": ["g"] * 3,
+ "H": ["h"] * 3,
+ "T": [1, 2, 3],
+ "E": [np.nan, 1.0, np.nan],
+ },
+ ),
+ (
+ {"T": [2], "E": [1]},
+ {"G": ["g", "g"], "H": ["h", "h"], "T": [1, 3]},
+ ["T"],
+ None,
+ ["G", "H"],
+ {
+ "T": [1, 2, 3],
+ "E": [np.nan, 1.0, np.nan],
+ "G": ["g"] * 3,
+ "H": ["h"] * 3,
+ },
+ ),
+ ],
+ )
+ def test_list_type_by(self, left, right, on, left_by, right_by, expected):
+ # GH 35269
+ left = DataFrame(left)
+ right = DataFrame(right)
+ result = merge_ordered(
+ left=left,
+ right=right,
+ on=on,
+ left_by=left_by,
+ right_by=right_by,
+ )
+ expected = DataFrame(expected)
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_left_by_length_equals_to_right_shape0(self):
+ # GH 38166
+ left = DataFrame([["g", "h", 1], ["g", "h", 3]], columns=list("GHE"))
+ right = DataFrame([[2, 1]], columns=list("ET"))
+ result = merge_ordered(left, right, on="E", left_by=["G", "H"])
+ expected = DataFrame(
+ {"G": ["g"] * 3, "H": ["h"] * 3, "E": [1, 2, 3], "T": [np.nan, 1.0, np.nan]}
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_elements_not_in_by_but_in_df(self):
+ # GH 38167
+ left = DataFrame([["g", "h", 1], ["g", "h", 3]], columns=list("GHE"))
+ right = DataFrame([[2, 1]], columns=list("ET"))
+ msg = r"\{'h'\} not found in left columns"
+ with pytest.raises(KeyError, match=msg):
+ merge_ordered(left, right, on="E", left_by=["G", "h"])
+
+ @pytest.mark.parametrize("invalid_method", ["linear", "carrot"])
+ def test_ffill_validate_fill_method(self, left, right, invalid_method):
+ # GH 55884
+ with pytest.raises(
+ ValueError, match=re.escape("fill_method must be 'ffill' or None")
+ ):
+ merge_ordered(left, right, on="key", fill_method=invalid_method)
+
+ def test_ffill_left_merge(self):
+ # GH 57010
+ df1 = DataFrame(
+ {
+ "key": ["a", "c", "e", "a", "c", "e"],
+ "lvalue": [1, 2, 3, 1, 2, 3],
+ "group": ["a", "a", "a", "b", "b", "b"],
+ }
+ )
+ df2 = DataFrame({"key": ["b", "c", "d"], "rvalue": [1, 2, 3]})
+ result = merge_ordered(
+ df1, df2, fill_method="ffill", left_by="group", how="left"
+ )
+ expected = DataFrame(
+ {
+ "key": ["a", "c", "e", "a", "c", "e"],
+ "lvalue": [1, 2, 3, 1, 2, 3],
+ "group": ["a", "a", "a", "b", "b", "b"],
+ "rvalue": [np.nan, 2.0, 2.0, np.nan, 2.0, 2.0],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_multi.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_multi.py
new file mode 100644
index 0000000000000000000000000000000000000000..7ae2fffa042050836473eba8c621817c799a9bb5
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/merge/test_multi.py
@@ -0,0 +1,930 @@
+import numpy as np
+import pytest
+
+import pandas.util._test_decorators as td
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ MultiIndex,
+ RangeIndex,
+ Series,
+ Timestamp,
+ option_context,
+)
+import pandas._testing as tm
+from pandas.core.reshape.concat import concat
+from pandas.core.reshape.merge import merge
+
+
+@pytest.fixture
+def left():
+ """left dataframe (not multi-indexed) for multi-index join tests"""
+ # a little relevant example with NAs
+ key1 = ["bar", "bar", "bar", "foo", "foo", "baz", "baz", "qux", "qux", "snap"]
+ key2 = ["two", "one", "three", "one", "two", "one", "two", "two", "three", "one"]
+
+ data = np.random.default_rng(2).standard_normal(len(key1))
+ return DataFrame({"key1": key1, "key2": key2, "data": data})
+
+
+@pytest.fixture
+def right(multiindex_dataframe_random_data):
+ """right dataframe (multi-indexed) for multi-index join tests"""
+ df = multiindex_dataframe_random_data
+ df.index.names = ["key1", "key2"]
+
+ df.columns = ["j_one", "j_two", "j_three"]
+ return df
+
+
+@pytest.fixture
+def left_multi():
+ return DataFrame(
+ {
+ "Origin": ["A", "A", "B", "B", "C"],
+ "Destination": ["A", "B", "A", "C", "A"],
+ "Period": ["AM", "AM", "IP", "AM", "OP"],
+ "TripPurp": ["hbw", "nhb", "hbo", "nhb", "hbw"],
+ "Trips": [1987, 3647, 2470, 4296, 4444],
+ },
+ columns=["Origin", "Destination", "Period", "TripPurp", "Trips"],
+ ).set_index(["Origin", "Destination", "Period", "TripPurp"])
+
+
+@pytest.fixture
+def right_multi():
+ return DataFrame(
+ {
+ "Origin": ["A", "A", "B", "B", "C", "C", "E"],
+ "Destination": ["A", "B", "A", "B", "A", "B", "F"],
+ "Period": ["AM", "AM", "IP", "AM", "OP", "IP", "AM"],
+ "LinkType": ["a", "b", "c", "b", "a", "b", "a"],
+ "Distance": [100, 80, 90, 80, 75, 35, 55],
+ },
+ columns=["Origin", "Destination", "Period", "LinkType", "Distance"],
+ ).set_index(["Origin", "Destination", "Period", "LinkType"])
+
+
+@pytest.fixture
+def on_cols_multi():
+ return ["Origin", "Destination", "Period"]
+
+
+class TestMergeMulti:
+ def test_merge_on_multikey(self, left, right, join_type):
+ on_cols = ["key1", "key2"]
+ result = left.join(right, on=on_cols, how=join_type).reset_index(drop=True)
+
+ expected = merge(left, right.reset_index(), on=on_cols, how=join_type)
+
+ tm.assert_frame_equal(result, expected)
+
+ result = left.join(right, on=on_cols, how=join_type, sort=True).reset_index(
+ drop=True
+ )
+
+ expected = merge(
+ left, right.reset_index(), on=on_cols, how=join_type, sort=True
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "infer_string", [False, pytest.param(True, marks=td.skip_if_no("pyarrow"))]
+ )
+ def test_left_join_multi_index(self, sort, infer_string):
+ with option_context("future.infer_string", infer_string):
+ icols = ["1st", "2nd", "3rd"]
+
+ def bind_cols(df):
+ iord = lambda a: 0 if a != a else ord(a)
+ f = lambda ts: ts.map(iord) - ord("a")
+ return f(df["1st"]) + f(df["3rd"]) * 1e2 + df["2nd"].fillna(0) * 10
+
+ def run_asserts(left, right, sort):
+ res = left.join(right, on=icols, how="left", sort=sort)
+
+ assert len(left) < len(res) + 1
+ assert not res["4th"].isna().any()
+ assert not res["5th"].isna().any()
+
+ tm.assert_series_equal(res["4th"], -res["5th"], check_names=False)
+ result = bind_cols(res.iloc[:, :-2])
+ tm.assert_series_equal(res["4th"], result, check_names=False)
+ assert result.name is None
+
+ if sort:
+ tm.assert_frame_equal(res, res.sort_values(icols, kind="mergesort"))
+
+ out = merge(left, right.reset_index(), on=icols, sort=sort, how="left")
+
+ res.index = RangeIndex(len(res))
+ tm.assert_frame_equal(out, res)
+
+ lc = list(map(chr, np.arange(ord("a"), ord("z") + 1)))
+ left = DataFrame(
+ np.random.default_rng(2).choice(lc, (50, 2)), columns=["1st", "3rd"]
+ )
+ # Explicit cast to float to avoid implicit cast when setting nan
+ left.insert(
+ 1,
+ "2nd",
+ np.random.default_rng(2).integers(0, 10, len(left)).astype("float"),
+ )
+ right = left.sample(frac=1, random_state=np.random.default_rng(2))
+
+ left["4th"] = bind_cols(left)
+ right["5th"] = -bind_cols(right)
+ right.set_index(icols, inplace=True)
+
+ run_asserts(left, right, sort)
+
+ # inject some nulls
+ left.loc[1::4, "1st"] = np.nan
+ left.loc[2::5, "2nd"] = np.nan
+ left.loc[3::6, "3rd"] = np.nan
+ left["4th"] = bind_cols(left)
+
+ i = np.random.default_rng(2).permutation(len(left))
+ right = left.iloc[i, :-1]
+ right["5th"] = -bind_cols(right)
+ right.set_index(icols, inplace=True)
+
+ run_asserts(left, right, sort)
+
+ def test_merge_right_vs_left(self, left, right, sort):
+ # compare left vs right merge with multikey
+ on_cols = ["key1", "key2"]
+ merged_left_right = left.merge(
+ right, left_on=on_cols, right_index=True, how="left", sort=sort
+ )
+
+ merge_right_left = right.merge(
+ left, right_on=on_cols, left_index=True, how="right", sort=sort
+ )
+
+ # Reorder columns
+ merge_right_left = merge_right_left[merged_left_right.columns]
+
+ tm.assert_frame_equal(merged_left_right, merge_right_left)
+
+ def test_merge_multiple_cols_with_mixed_cols_index(self):
+ # GH29522
+ s = Series(
+ range(6),
+ MultiIndex.from_product([["A", "B"], [1, 2, 3]], names=["lev1", "lev2"]),
+ name="Amount",
+ )
+ df = DataFrame({"lev1": list("AAABBB"), "lev2": [1, 2, 3, 1, 2, 3], "col": 0})
+ result = merge(df, s.reset_index(), on=["lev1", "lev2"])
+ expected = DataFrame(
+ {
+ "lev1": list("AAABBB"),
+ "lev2": [1, 2, 3, 1, 2, 3],
+ "col": [0] * 6,
+ "Amount": range(6),
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_compress_group_combinations(self):
+ # ~ 40000000 possible unique groups
+ key1 = [str(i) for i in range(10000)]
+ key1 = np.tile(key1, 2)
+ key2 = key1[::-1]
+
+ df = DataFrame(
+ {
+ "key1": key1,
+ "key2": key2,
+ "value1": np.random.default_rng(2).standard_normal(20000),
+ }
+ )
+
+ df2 = DataFrame(
+ {
+ "key1": key1[::2],
+ "key2": key2[::2],
+ "value2": np.random.default_rng(2).standard_normal(10000),
+ }
+ )
+
+ # just to hit the label compression code path
+ merge(df, df2, how="outer")
+
+ def test_left_join_index_preserve_order(self):
+ on_cols = ["k1", "k2"]
+ left = DataFrame(
+ {
+ "k1": [0, 1, 2] * 8,
+ "k2": ["foo", "bar"] * 12,
+ "v": np.array(np.arange(24), dtype=np.int64),
+ }
+ )
+
+ index = MultiIndex.from_tuples([(2, "bar"), (1, "foo")])
+ right = DataFrame({"v2": [5, 7]}, index=index)
+
+ result = left.join(right, on=on_cols)
+
+ expected = left.copy()
+ expected["v2"] = np.nan
+ expected.loc[(expected.k1 == 2) & (expected.k2 == "bar"), "v2"] = 5
+ expected.loc[(expected.k1 == 1) & (expected.k2 == "foo"), "v2"] = 7
+
+ tm.assert_frame_equal(result, expected)
+
+ result.sort_values(on_cols, kind="mergesort", inplace=True)
+ expected = left.join(right, on=on_cols, sort=True)
+
+ tm.assert_frame_equal(result, expected)
+
+ # test join with multi dtypes blocks
+ left = DataFrame(
+ {
+ "k1": [0, 1, 2] * 8,
+ "k2": ["foo", "bar"] * 12,
+ "k3": np.array([0, 1, 2] * 8, dtype=np.float32),
+ "v": np.array(np.arange(24), dtype=np.int32),
+ }
+ )
+
+ index = MultiIndex.from_tuples([(2, "bar"), (1, "foo")])
+ right = DataFrame({"v2": [5, 7]}, index=index)
+
+ result = left.join(right, on=on_cols)
+
+ expected = left.copy()
+ expected["v2"] = np.nan
+ expected.loc[(expected.k1 == 2) & (expected.k2 == "bar"), "v2"] = 5
+ expected.loc[(expected.k1 == 1) & (expected.k2 == "foo"), "v2"] = 7
+
+ tm.assert_frame_equal(result, expected)
+
+ result = result.sort_values(on_cols, kind="mergesort")
+ expected = left.join(right, on=on_cols, sort=True)
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_left_join_index_multi_match_multiindex(self):
+ left = DataFrame(
+ [
+ ["X", "Y", "C", "a"],
+ ["W", "Y", "C", "e"],
+ ["V", "Q", "A", "h"],
+ ["V", "R", "D", "i"],
+ ["X", "Y", "D", "b"],
+ ["X", "Y", "A", "c"],
+ ["W", "Q", "B", "f"],
+ ["W", "R", "C", "g"],
+ ["V", "Y", "C", "j"],
+ ["X", "Y", "B", "d"],
+ ],
+ columns=["cola", "colb", "colc", "tag"],
+ index=[3, 2, 0, 1, 7, 6, 4, 5, 9, 8],
+ )
+
+ right = DataFrame(
+ [
+ ["W", "R", "C", 0],
+ ["W", "Q", "B", 3],
+ ["W", "Q", "B", 8],
+ ["X", "Y", "A", 1],
+ ["X", "Y", "A", 4],
+ ["X", "Y", "B", 5],
+ ["X", "Y", "C", 6],
+ ["X", "Y", "C", 9],
+ ["X", "Q", "C", -6],
+ ["X", "R", "C", -9],
+ ["V", "Y", "C", 7],
+ ["V", "R", "D", 2],
+ ["V", "R", "D", -1],
+ ["V", "Q", "A", -3],
+ ],
+ columns=["col1", "col2", "col3", "val"],
+ ).set_index(["col1", "col2", "col3"])
+
+ result = left.join(right, on=["cola", "colb", "colc"], how="left")
+
+ expected = DataFrame(
+ [
+ ["X", "Y", "C", "a", 6],
+ ["X", "Y", "C", "a", 9],
+ ["W", "Y", "C", "e", np.nan],
+ ["V", "Q", "A", "h", -3],
+ ["V", "R", "D", "i", 2],
+ ["V", "R", "D", "i", -1],
+ ["X", "Y", "D", "b", np.nan],
+ ["X", "Y", "A", "c", 1],
+ ["X", "Y", "A", "c", 4],
+ ["W", "Q", "B", "f", 3],
+ ["W", "Q", "B", "f", 8],
+ ["W", "R", "C", "g", 0],
+ ["V", "Y", "C", "j", 7],
+ ["X", "Y", "B", "d", 5],
+ ],
+ columns=["cola", "colb", "colc", "tag", "val"],
+ index=[3, 3, 2, 0, 1, 1, 7, 6, 6, 4, 4, 5, 9, 8],
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ result = left.join(right, on=["cola", "colb", "colc"], how="left", sort=True)
+
+ expected = expected.sort_values(["cola", "colb", "colc"], kind="mergesort")
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_left_join_index_multi_match(self):
+ left = DataFrame(
+ [["c", 0], ["b", 1], ["a", 2], ["b", 3]],
+ columns=["tag", "val"],
+ index=[2, 0, 1, 3],
+ )
+
+ right = DataFrame(
+ [
+ ["a", "v"],
+ ["c", "w"],
+ ["c", "x"],
+ ["d", "y"],
+ ["a", "z"],
+ ["c", "r"],
+ ["e", "q"],
+ ["c", "s"],
+ ],
+ columns=["tag", "char"],
+ ).set_index("tag")
+
+ result = left.join(right, on="tag", how="left")
+
+ expected = DataFrame(
+ [
+ ["c", 0, "w"],
+ ["c", 0, "x"],
+ ["c", 0, "r"],
+ ["c", 0, "s"],
+ ["b", 1, np.nan],
+ ["a", 2, "v"],
+ ["a", 2, "z"],
+ ["b", 3, np.nan],
+ ],
+ columns=["tag", "val", "char"],
+ index=[2, 2, 2, 2, 0, 1, 1, 3],
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ result = left.join(right, on="tag", how="left", sort=True)
+ expected2 = expected.sort_values("tag", kind="mergesort")
+
+ tm.assert_frame_equal(result, expected2)
+
+ # GH7331 - maintain left frame order in left merge
+ result = merge(left, right.reset_index(), how="left", on="tag")
+ expected.index = RangeIndex(len(expected))
+ tm.assert_frame_equal(result, expected)
+
+ def test_left_merge_na_buglet(self):
+ left = DataFrame(
+ {
+ "id": list("abcde"),
+ "v1": np.random.default_rng(2).standard_normal(5),
+ "v2": np.random.default_rng(2).standard_normal(5),
+ "dummy": list("abcde"),
+ "v3": np.random.default_rng(2).standard_normal(5),
+ },
+ columns=["id", "v1", "v2", "dummy", "v3"],
+ )
+ right = DataFrame(
+ {
+ "id": ["a", "b", np.nan, np.nan, np.nan],
+ "sv3": [1.234, 5.678, np.nan, np.nan, np.nan],
+ }
+ )
+
+ result = merge(left, right, on="id", how="left")
+
+ rdf = right.drop(["id"], axis=1)
+ expected = left.join(rdf)
+ tm.assert_frame_equal(result, expected)
+
+ def test_merge_na_keys(self):
+ data = [
+ [1950, "A", 1.5],
+ [1950, "B", 1.5],
+ [1955, "B", 1.5],
+ [1960, "B", np.nan],
+ [1970, "B", 4.0],
+ [1950, "C", 4.0],
+ [1960, "C", np.nan],
+ [1965, "C", 3.0],
+ [1970, "C", 4.0],
+ ]
+
+ frame = DataFrame(data, columns=["year", "panel", "data"])
+
+ other_data = [
+ [1960, "A", np.nan],
+ [1970, "A", np.nan],
+ [1955, "A", np.nan],
+ [1965, "A", np.nan],
+ [1965, "B", np.nan],
+ [1955, "C", np.nan],
+ ]
+ other = DataFrame(other_data, columns=["year", "panel", "data"])
+
+ result = frame.merge(other, how="outer")
+
+ expected = frame.fillna(-999).merge(other.fillna(-999), how="outer")
+ expected = expected.replace(-999, np.nan)
+
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("klass", [None, np.asarray, Series, Index])
+ def test_merge_datetime_index(self, klass):
+ # see gh-19038
+ df = DataFrame(
+ [1, 2, 3], ["2016-01-01", "2017-01-01", "2018-01-01"], columns=["a"]
+ )
+ df.index = pd.to_datetime(df.index)
+ on_vector = df.index.year
+
+ if klass is not None:
+ on_vector = klass(on_vector)
+
+ exp_years = np.array([2016, 2017, 2018], dtype=np.int32)
+ expected = DataFrame({"a": [1, 2, 3], "key_1": exp_years})
+
+ result = df.merge(df, on=["a", on_vector], how="inner")
+ tm.assert_frame_equal(result, expected)
+
+ expected = DataFrame({"key_0": exp_years, "a_x": [1, 2, 3], "a_y": [1, 2, 3]})
+
+ result = df.merge(df, on=[df.index.year], how="inner")
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("merge_type", ["left", "right"])
+ def test_merge_datetime_multi_index_empty_df(self, merge_type):
+ # see gh-36895
+
+ left = DataFrame(
+ data={
+ "data": [1.5, 1.5],
+ },
+ index=MultiIndex.from_tuples(
+ [[Timestamp("1950-01-01"), "A"], [Timestamp("1950-01-02"), "B"]],
+ names=["date", "panel"],
+ ),
+ )
+
+ right = DataFrame(
+ index=MultiIndex.from_tuples([], names=["date", "panel"]), columns=["state"]
+ )
+
+ expected_index = MultiIndex.from_tuples(
+ [[Timestamp("1950-01-01"), "A"], [Timestamp("1950-01-02"), "B"]],
+ names=["date", "panel"],
+ )
+
+ if merge_type == "left":
+ expected = DataFrame(
+ data={
+ "data": [1.5, 1.5],
+ "state": np.array([np.nan, np.nan], dtype=object),
+ },
+ index=expected_index,
+ )
+ results_merge = left.merge(right, how="left", on=["date", "panel"])
+ results_join = left.join(right, how="left")
+ else:
+ expected = DataFrame(
+ data={
+ "state": np.array([np.nan, np.nan], dtype=object),
+ "data": [1.5, 1.5],
+ },
+ index=expected_index,
+ )
+ results_merge = right.merge(left, how="right", on=["date", "panel"])
+ results_join = right.join(left, how="right")
+
+ tm.assert_frame_equal(results_merge, expected)
+ tm.assert_frame_equal(results_join, expected)
+
+ @pytest.fixture
+ def household(self):
+ household = DataFrame(
+ {
+ "household_id": [1, 2, 3],
+ "male": [0, 1, 0],
+ "wealth": [196087.3, 316478.7, 294750],
+ },
+ columns=["household_id", "male", "wealth"],
+ ).set_index("household_id")
+ return household
+
+ @pytest.fixture
+ def portfolio(self):
+ portfolio = DataFrame(
+ {
+ "household_id": [1, 2, 2, 3, 3, 3, 4],
+ "asset_id": [
+ "nl0000301109",
+ "nl0000289783",
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "lu0197800237",
+ "nl0000289965",
+ np.nan,
+ ],
+ "name": [
+ "ABN Amro",
+ "Robeco",
+ "Royal Dutch Shell",
+ "Royal Dutch Shell",
+ "AAB Eastern Europe Equity Fund",
+ "Postbank BioTech Fonds",
+ np.nan,
+ ],
+ "share": [1.0, 0.4, 0.6, 0.15, 0.6, 0.25, 1.0],
+ },
+ columns=["household_id", "asset_id", "name", "share"],
+ ).set_index(["household_id", "asset_id"])
+ return portfolio
+
+ @pytest.fixture
+ def expected(self):
+ expected = (
+ DataFrame(
+ {
+ "male": [0, 1, 1, 0, 0, 0],
+ "wealth": [
+ 196087.3,
+ 316478.7,
+ 316478.7,
+ 294750.0,
+ 294750.0,
+ 294750.0,
+ ],
+ "name": [
+ "ABN Amro",
+ "Robeco",
+ "Royal Dutch Shell",
+ "Royal Dutch Shell",
+ "AAB Eastern Europe Equity Fund",
+ "Postbank BioTech Fonds",
+ ],
+ "share": [1.00, 0.40, 0.60, 0.15, 0.60, 0.25],
+ "household_id": [1, 2, 2, 3, 3, 3],
+ "asset_id": [
+ "nl0000301109",
+ "nl0000289783",
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "lu0197800237",
+ "nl0000289965",
+ ],
+ }
+ )
+ .set_index(["household_id", "asset_id"])
+ .reindex(columns=["male", "wealth", "name", "share"])
+ )
+ return expected
+
+ def test_join_multi_levels(self, portfolio, household, expected):
+ portfolio = portfolio.copy()
+ household = household.copy()
+
+ # GH 3662
+ # merge multi-levels
+ result = household.join(portfolio, how="inner")
+ tm.assert_frame_equal(result, expected)
+
+ def test_join_multi_levels_merge_equivalence(self, portfolio, household, expected):
+ portfolio = portfolio.copy()
+ household = household.copy()
+
+ # equivalency
+ result = merge(
+ household.reset_index(),
+ portfolio.reset_index(),
+ on=["household_id"],
+ how="inner",
+ ).set_index(["household_id", "asset_id"])
+ tm.assert_frame_equal(result, expected)
+
+ def test_join_multi_levels_outer(self, portfolio, household, expected):
+ portfolio = portfolio.copy()
+ household = household.copy()
+
+ result = household.join(portfolio, how="outer")
+ expected = concat(
+ [
+ expected,
+ (
+ DataFrame(
+ {"share": [1.00]},
+ index=MultiIndex.from_tuples(
+ [(4, np.nan)], names=["household_id", "asset_id"]
+ ),
+ )
+ ),
+ ],
+ axis=0,
+ sort=True,
+ ).reindex(columns=expected.columns)
+ tm.assert_frame_equal(result, expected, check_index_type=False)
+
+ def test_join_multi_levels_invalid(self, portfolio, household):
+ portfolio = portfolio.copy()
+ household = household.copy()
+
+ # invalid cases
+ household.index.name = "foo"
+
+ with pytest.raises(
+ ValueError, match="cannot join with no overlapping index names"
+ ):
+ household.join(portfolio, how="inner")
+
+ portfolio2 = portfolio.copy()
+ portfolio2.index.set_names(["household_id", "foo"])
+
+ with pytest.raises(ValueError, match="columns overlap but no suffix specified"):
+ portfolio2.join(portfolio, how="inner")
+
+ def test_join_multi_levels2(self):
+ # some more advanced merges
+ # GH6360
+ household = DataFrame(
+ {
+ "household_id": [1, 2, 2, 3, 3, 3, 4],
+ "asset_id": [
+ "nl0000301109",
+ "nl0000301109",
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "lu0197800237",
+ "nl0000289965",
+ np.nan,
+ ],
+ "share": [1.0, 0.4, 0.6, 0.15, 0.6, 0.25, 1.0],
+ },
+ columns=["household_id", "asset_id", "share"],
+ ).set_index(["household_id", "asset_id"])
+
+ log_return = DataFrame(
+ {
+ "asset_id": [
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "lu0197800237",
+ "lu0197800237",
+ ],
+ "t": [233, 234, 235, 180, 181],
+ "log_return": [
+ 0.09604978,
+ -0.06524096,
+ 0.03532373,
+ 0.03025441,
+ 0.036997,
+ ],
+ }
+ ).set_index(["asset_id", "t"])
+
+ expected = (
+ DataFrame(
+ {
+ "household_id": [2, 2, 2, 3, 3, 3, 3, 3],
+ "asset_id": [
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "lu0197800237",
+ "lu0197800237",
+ ],
+ "t": [233, 234, 235, 233, 234, 235, 180, 181],
+ "share": [0.6, 0.6, 0.6, 0.15, 0.15, 0.15, 0.6, 0.6],
+ "log_return": [
+ 0.09604978,
+ -0.06524096,
+ 0.03532373,
+ 0.09604978,
+ -0.06524096,
+ 0.03532373,
+ 0.03025441,
+ 0.036997,
+ ],
+ }
+ )
+ .set_index(["household_id", "asset_id", "t"])
+ .reindex(columns=["share", "log_return"])
+ )
+
+ # this is the equivalency
+ result = merge(
+ household.reset_index(),
+ log_return.reset_index(),
+ on=["asset_id"],
+ how="inner",
+ ).set_index(["household_id", "asset_id", "t"])
+ tm.assert_frame_equal(result, expected)
+
+ expected = (
+ DataFrame(
+ {
+ "household_id": [2, 2, 2, 3, 3, 3, 3, 3, 3, 1, 2, 4],
+ "asset_id": [
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "gb00b03mlx29",
+ "lu0197800237",
+ "lu0197800237",
+ "nl0000289965",
+ "nl0000301109",
+ "nl0000301109",
+ None,
+ ],
+ "t": [
+ 233,
+ 234,
+ 235,
+ 233,
+ 234,
+ 235,
+ 180,
+ 181,
+ None,
+ None,
+ None,
+ None,
+ ],
+ "share": [
+ 0.6,
+ 0.6,
+ 0.6,
+ 0.15,
+ 0.15,
+ 0.15,
+ 0.6,
+ 0.6,
+ 0.25,
+ 1.0,
+ 0.4,
+ 1.0,
+ ],
+ "log_return": [
+ 0.09604978,
+ -0.06524096,
+ 0.03532373,
+ 0.09604978,
+ -0.06524096,
+ 0.03532373,
+ 0.03025441,
+ 0.036997,
+ None,
+ None,
+ None,
+ None,
+ ],
+ }
+ )
+ .set_index(["household_id", "asset_id", "t"])
+ .reindex(columns=["share", "log_return"])
+ )
+
+ result = merge(
+ household.reset_index(),
+ log_return.reset_index(),
+ on=["asset_id"],
+ how="outer",
+ ).set_index(["household_id", "asset_id", "t"])
+
+ tm.assert_frame_equal(result, expected)
+
+
+class TestJoinMultiMulti:
+ def test_join_multi_multi(self, left_multi, right_multi, join_type, on_cols_multi):
+ left_names = left_multi.index.names
+ right_names = right_multi.index.names
+ if join_type == "right":
+ level_order = right_names + left_names.difference(right_names)
+ else:
+ level_order = left_names + right_names.difference(left_names)
+ # Multi-index join tests
+ expected = (
+ merge(
+ left_multi.reset_index(),
+ right_multi.reset_index(),
+ how=join_type,
+ on=on_cols_multi,
+ )
+ .set_index(level_order)
+ .sort_index()
+ )
+
+ result = left_multi.join(right_multi, how=join_type).sort_index()
+ tm.assert_frame_equal(result, expected)
+
+ def test_join_multi_empty_frames(
+ self, left_multi, right_multi, join_type, on_cols_multi
+ ):
+ left_multi = left_multi.drop(columns=left_multi.columns)
+ right_multi = right_multi.drop(columns=right_multi.columns)
+
+ left_names = left_multi.index.names
+ right_names = right_multi.index.names
+ if join_type == "right":
+ level_order = right_names + left_names.difference(right_names)
+ else:
+ level_order = left_names + right_names.difference(left_names)
+
+ expected = (
+ merge(
+ left_multi.reset_index(),
+ right_multi.reset_index(),
+ how=join_type,
+ on=on_cols_multi,
+ )
+ .set_index(level_order)
+ .sort_index()
+ )
+
+ result = left_multi.join(right_multi, how=join_type).sort_index()
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("box", [None, np.asarray, Series, Index])
+ def test_merge_datetime_index(self, box):
+ # see gh-19038
+ df = DataFrame(
+ [1, 2, 3], ["2016-01-01", "2017-01-01", "2018-01-01"], columns=["a"]
+ )
+ df.index = pd.to_datetime(df.index)
+ on_vector = df.index.year
+
+ if box is not None:
+ on_vector = box(on_vector)
+
+ exp_years = np.array([2016, 2017, 2018], dtype=np.int32)
+ expected = DataFrame({"a": [1, 2, 3], "key_1": exp_years})
+
+ result = df.merge(df, on=["a", on_vector], how="inner")
+ tm.assert_frame_equal(result, expected)
+
+ expected = DataFrame({"key_0": exp_years, "a_x": [1, 2, 3], "a_y": [1, 2, 3]})
+
+ result = df.merge(df, on=[df.index.year], how="inner")
+ tm.assert_frame_equal(result, expected)
+
+ def test_single_common_level(self):
+ index_left = MultiIndex.from_tuples(
+ [("K0", "X0"), ("K0", "X1"), ("K1", "X2")], names=["key", "X"]
+ )
+
+ left = DataFrame(
+ {"A": ["A0", "A1", "A2"], "B": ["B0", "B1", "B2"]}, index=index_left
+ )
+
+ index_right = MultiIndex.from_tuples(
+ [("K0", "Y0"), ("K1", "Y1"), ("K2", "Y2"), ("K2", "Y3")], names=["key", "Y"]
+ )
+
+ right = DataFrame(
+ {"C": ["C0", "C1", "C2", "C3"], "D": ["D0", "D1", "D2", "D3"]},
+ index=index_right,
+ )
+
+ result = left.join(right)
+ expected = merge(
+ left.reset_index(), right.reset_index(), on=["key"], how="inner"
+ ).set_index(["key", "X", "Y"])
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_join_multi_wrong_order(self):
+ # GH 25760
+ # GH 28956
+
+ midx1 = MultiIndex.from_product([[1, 2], [3, 4]], names=["a", "b"])
+ midx3 = MultiIndex.from_tuples([(4, 1), (3, 2), (3, 1)], names=["b", "a"])
+
+ left = DataFrame(index=midx1, data={"x": [10, 20, 30, 40]})
+ right = DataFrame(index=midx3, data={"y": ["foo", "bar", "fing"]})
+
+ result = left.join(right)
+
+ expected = DataFrame(
+ index=midx1,
+ data={"x": [10, 20, 30, 40], "y": ["fing", "foo", "bar", np.nan]},
+ )
+
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_crosstab.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_crosstab.py
new file mode 100644
index 0000000000000000000000000000000000000000..1482da8a074eb41b64d276683ffc7258b4e9d0bb
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_crosstab.py
@@ -0,0 +1,879 @@
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ CategoricalDtype,
+ CategoricalIndex,
+ DataFrame,
+ Index,
+ MultiIndex,
+ Series,
+ crosstab,
+)
+import pandas._testing as tm
+
+
+@pytest.fixture
+def df():
+ df = DataFrame(
+ {
+ "A": [
+ "foo",
+ "foo",
+ "foo",
+ "foo",
+ "bar",
+ "bar",
+ "bar",
+ "bar",
+ "foo",
+ "foo",
+ "foo",
+ ],
+ "B": [
+ "one",
+ "one",
+ "one",
+ "two",
+ "one",
+ "one",
+ "one",
+ "two",
+ "two",
+ "two",
+ "one",
+ ],
+ "C": [
+ "dull",
+ "dull",
+ "shiny",
+ "dull",
+ "dull",
+ "shiny",
+ "shiny",
+ "dull",
+ "shiny",
+ "shiny",
+ "shiny",
+ ],
+ "D": np.random.default_rng(2).standard_normal(11),
+ "E": np.random.default_rng(2).standard_normal(11),
+ "F": np.random.default_rng(2).standard_normal(11),
+ }
+ )
+
+ return pd.concat([df, df], ignore_index=True)
+
+
+class TestCrosstab:
+ def test_crosstab_single(self, df):
+ result = crosstab(df["A"], df["C"])
+ expected = df.groupby(["A", "C"]).size().unstack()
+ tm.assert_frame_equal(result, expected.fillna(0).astype(np.int64))
+
+ def test_crosstab_multiple(self, df):
+ result = crosstab(df["A"], [df["B"], df["C"]])
+ expected = df.groupby(["A", "B", "C"]).size()
+ expected = expected.unstack("B").unstack("C").fillna(0).astype(np.int64)
+ tm.assert_frame_equal(result, expected)
+
+ result = crosstab([df["B"], df["C"]], df["A"])
+ expected = df.groupby(["B", "C", "A"]).size()
+ expected = expected.unstack("A").fillna(0).astype(np.int64)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("box", [np.array, list, tuple])
+ def test_crosstab_ndarray(self, box):
+ # GH 44076
+ a = box(np.random.default_rng(2).integers(0, 5, size=100))
+ b = box(np.random.default_rng(2).integers(0, 3, size=100))
+ c = box(np.random.default_rng(2).integers(0, 10, size=100))
+
+ df = DataFrame({"a": a, "b": b, "c": c})
+
+ result = crosstab(a, [b, c], rownames=["a"], colnames=("b", "c"))
+ expected = crosstab(df["a"], [df["b"], df["c"]])
+ tm.assert_frame_equal(result, expected)
+
+ result = crosstab([b, c], a, colnames=["a"], rownames=("b", "c"))
+ expected = crosstab([df["b"], df["c"]], df["a"])
+ tm.assert_frame_equal(result, expected)
+
+ # assign arbitrary names
+ result = crosstab(a, c)
+ expected = crosstab(df["a"], df["c"])
+ expected.index.names = ["row_0"]
+ expected.columns.names = ["col_0"]
+ tm.assert_frame_equal(result, expected)
+
+ def test_crosstab_non_aligned(self):
+ # GH 17005
+ a = Series([0, 1, 1], index=["a", "b", "c"])
+ b = Series([3, 4, 3, 4, 3], index=["a", "b", "c", "d", "f"])
+ c = np.array([3, 4, 3], dtype=np.int64)
+
+ expected = DataFrame(
+ [[1, 0], [1, 1]],
+ index=Index([0, 1], name="row_0"),
+ columns=Index([3, 4], name="col_0"),
+ )
+
+ result = crosstab(a, b)
+ tm.assert_frame_equal(result, expected)
+
+ result = crosstab(a, c)
+ tm.assert_frame_equal(result, expected)
+
+ def test_crosstab_margins(self):
+ a = np.random.default_rng(2).integers(0, 7, size=100)
+ b = np.random.default_rng(2).integers(0, 3, size=100)
+ c = np.random.default_rng(2).integers(0, 5, size=100)
+
+ df = DataFrame({"a": a, "b": b, "c": c})
+
+ result = crosstab(a, [b, c], rownames=["a"], colnames=("b", "c"), margins=True)
+
+ assert result.index.names == ("a",)
+ assert result.columns.names == ["b", "c"]
+
+ all_cols = result["All", ""]
+ exp_cols = df.groupby(["a"]).size().astype("i8")
+ # to keep index.name
+ exp_margin = Series([len(df)], index=Index(["All"], name="a"))
+ exp_cols = pd.concat([exp_cols, exp_margin])
+ exp_cols.name = ("All", "")
+
+ tm.assert_series_equal(all_cols, exp_cols)
+
+ all_rows = result.loc["All"]
+ exp_rows = df.groupby(["b", "c"]).size().astype("i8")
+ exp_rows = pd.concat([exp_rows, Series([len(df)], index=[("All", "")])])
+ exp_rows.name = "All"
+
+ exp_rows = exp_rows.reindex(all_rows.index)
+ exp_rows = exp_rows.fillna(0).astype(np.int64)
+ tm.assert_series_equal(all_rows, exp_rows)
+
+ def test_crosstab_margins_set_margin_name(self):
+ # GH 15972
+ a = np.random.default_rng(2).integers(0, 7, size=100)
+ b = np.random.default_rng(2).integers(0, 3, size=100)
+ c = np.random.default_rng(2).integers(0, 5, size=100)
+
+ df = DataFrame({"a": a, "b": b, "c": c})
+
+ result = crosstab(
+ a,
+ [b, c],
+ rownames=["a"],
+ colnames=("b", "c"),
+ margins=True,
+ margins_name="TOTAL",
+ )
+
+ assert result.index.names == ("a",)
+ assert result.columns.names == ["b", "c"]
+
+ all_cols = result["TOTAL", ""]
+ exp_cols = df.groupby(["a"]).size().astype("i8")
+ # to keep index.name
+ exp_margin = Series([len(df)], index=Index(["TOTAL"], name="a"))
+ exp_cols = pd.concat([exp_cols, exp_margin])
+ exp_cols.name = ("TOTAL", "")
+
+ tm.assert_series_equal(all_cols, exp_cols)
+
+ all_rows = result.loc["TOTAL"]
+ exp_rows = df.groupby(["b", "c"]).size().astype("i8")
+ exp_rows = pd.concat([exp_rows, Series([len(df)], index=[("TOTAL", "")])])
+ exp_rows.name = "TOTAL"
+
+ exp_rows = exp_rows.reindex(all_rows.index)
+ exp_rows = exp_rows.fillna(0).astype(np.int64)
+ tm.assert_series_equal(all_rows, exp_rows)
+
+ msg = "margins_name argument must be a string"
+ for margins_name in [666, None, ["a", "b"]]:
+ with pytest.raises(ValueError, match=msg):
+ crosstab(
+ a,
+ [b, c],
+ rownames=["a"],
+ colnames=("b", "c"),
+ margins=True,
+ margins_name=margins_name,
+ )
+
+ def test_crosstab_pass_values(self):
+ a = np.random.default_rng(2).integers(0, 7, size=100)
+ b = np.random.default_rng(2).integers(0, 3, size=100)
+ c = np.random.default_rng(2).integers(0, 5, size=100)
+ values = np.random.default_rng(2).standard_normal(100)
+
+ table = crosstab(
+ [a, b], c, values, aggfunc="sum", rownames=["foo", "bar"], colnames=["baz"]
+ )
+
+ df = DataFrame({"foo": a, "bar": b, "baz": c, "values": values})
+
+ expected = df.pivot_table(
+ "values", index=["foo", "bar"], columns="baz", aggfunc="sum"
+ )
+ tm.assert_frame_equal(table, expected)
+
+ def test_crosstab_dropna(self):
+ # GH 3820
+ a = np.array(["foo", "foo", "foo", "bar", "bar", "foo", "foo"], dtype=object)
+ b = np.array(["one", "one", "two", "one", "two", "two", "two"], dtype=object)
+ c = np.array(
+ ["dull", "dull", "dull", "dull", "dull", "shiny", "shiny"], dtype=object
+ )
+ res = crosstab(a, [b, c], rownames=["a"], colnames=["b", "c"], dropna=False)
+ m = MultiIndex.from_tuples(
+ [("one", "dull"), ("one", "shiny"), ("two", "dull"), ("two", "shiny")],
+ names=["b", "c"],
+ )
+ tm.assert_index_equal(res.columns, m)
+
+ def test_crosstab_no_overlap(self):
+ # GS 10291
+
+ s1 = Series([1, 2, 3], index=[1, 2, 3])
+ s2 = Series([4, 5, 6], index=[4, 5, 6])
+
+ actual = crosstab(s1, s2)
+ expected = DataFrame(
+ index=Index([], dtype="int64", name="row_0"),
+ columns=Index([], dtype="int64", name="col_0"),
+ )
+
+ tm.assert_frame_equal(actual, expected)
+
+ def test_margin_dropna(self):
+ # GH 12577
+ # pivot_table counts null into margin ('All')
+ # when margins=true and dropna=true
+
+ df = DataFrame({"a": [1, 2, 2, 2, 2, np.nan], "b": [3, 3, 4, 4, 4, 4]})
+ actual = crosstab(df.a, df.b, margins=True, dropna=True)
+ expected = DataFrame([[1, 0, 1], [1, 3, 4], [2, 3, 5]])
+ expected.index = Index([1.0, 2.0, "All"], name="a")
+ expected.columns = Index([3, 4, "All"], name="b")
+ tm.assert_frame_equal(actual, expected)
+
+ def test_margin_dropna2(self):
+ df = DataFrame(
+ {"a": [1, np.nan, np.nan, np.nan, 2, np.nan], "b": [3, np.nan, 4, 4, 4, 4]}
+ )
+ actual = crosstab(df.a, df.b, margins=True, dropna=True)
+ expected = DataFrame([[1, 0, 1], [0, 1, 1], [1, 1, 2]])
+ expected.index = Index([1.0, 2.0, "All"], name="a")
+ expected.columns = Index([3.0, 4.0, "All"], name="b")
+ tm.assert_frame_equal(actual, expected)
+
+ def test_margin_dropna3(self):
+ df = DataFrame(
+ {"a": [1, np.nan, np.nan, np.nan, np.nan, 2], "b": [3, 3, 4, 4, 4, 4]}
+ )
+ actual = crosstab(df.a, df.b, margins=True, dropna=True)
+ expected = DataFrame([[1, 0, 1], [0, 1, 1], [1, 1, 2]])
+ expected.index = Index([1.0, 2.0, "All"], name="a")
+ expected.columns = Index([3, 4, "All"], name="b")
+ tm.assert_frame_equal(actual, expected)
+
+ def test_margin_dropna4(self):
+ # GH 12642
+ # _add_margins raises KeyError: Level None not found
+ # when margins=True and dropna=False
+ # GH: 10772: Keep np.nan in result with dropna=False
+ df = DataFrame({"a": [1, 2, 2, 2, 2, np.nan], "b": [3, 3, 4, 4, 4, 4]})
+ actual = crosstab(df.a, df.b, margins=True, dropna=False)
+ expected = DataFrame([[1, 0, 1], [1, 3, 4], [0, 1, 1], [2, 4, 6]])
+ expected.index = Index([1.0, 2.0, np.nan, "All"], name="a")
+ expected.columns = Index([3, 4, "All"], name="b")
+ tm.assert_frame_equal(actual, expected)
+
+ def test_margin_dropna5(self):
+ # GH: 10772: Keep np.nan in result with dropna=False
+ df = DataFrame(
+ {"a": [1, np.nan, np.nan, np.nan, 2, np.nan], "b": [3, np.nan, 4, 4, 4, 4]}
+ )
+ actual = crosstab(df.a, df.b, margins=True, dropna=False)
+ expected = DataFrame(
+ [[1, 0, 0, 1.0], [0, 1, 0, 1.0], [0, 3, 1, 4.0], [1, 4, 1, 6.0]]
+ )
+ expected.index = Index([1.0, 2.0, np.nan, "All"], name="a")
+ expected.columns = Index([3.0, 4.0, np.nan, "All"], name="b")
+ tm.assert_frame_equal(actual, expected, check_dtype=False)
+
+ def test_margin_dropna6(self):
+ # GH: 10772: Keep np.nan in result with dropna=False
+ a = np.array(["foo", "foo", "foo", "bar", "bar", "foo", "foo"], dtype=object)
+ b = np.array(["one", "one", "two", "one", "two", np.nan, "two"], dtype=object)
+ c = np.array(
+ ["dull", "dull", "dull", "dull", "dull", "shiny", "shiny"], dtype=object
+ )
+
+ actual = crosstab(
+ a, [b, c], rownames=["a"], colnames=["b", "c"], margins=True, dropna=False
+ )
+ m = MultiIndex.from_arrays(
+ [
+ ["one", "one", "two", "two", np.nan, np.nan, "All"],
+ ["dull", "shiny", "dull", "shiny", "dull", "shiny", ""],
+ ],
+ names=["b", "c"],
+ )
+ expected = DataFrame(
+ [[1, 0, 1, 0, 0, 0, 2], [2, 0, 1, 1, 0, 1, 5], [3, 0, 2, 1, 0, 1, 7]],
+ columns=m,
+ )
+ expected.index = Index(["bar", "foo", "All"], name="a")
+ tm.assert_frame_equal(actual, expected)
+
+ actual = crosstab(
+ [a, b], c, rownames=["a", "b"], colnames=["c"], margins=True, dropna=False
+ )
+ m = MultiIndex.from_arrays(
+ [
+ ["bar", "bar", "bar", "foo", "foo", "foo", "All"],
+ ["one", "two", np.nan, "one", "two", np.nan, ""],
+ ],
+ names=["a", "b"],
+ )
+ expected = DataFrame(
+ [
+ [1, 0, 1.0],
+ [1, 0, 1.0],
+ [0, 0, np.nan],
+ [2, 0, 2.0],
+ [1, 1, 2.0],
+ [0, 1, 1.0],
+ [5, 2, 7.0],
+ ],
+ index=m,
+ )
+ expected.columns = Index(["dull", "shiny", "All"], name="c")
+ tm.assert_frame_equal(actual, expected)
+
+ actual = crosstab(
+ [a, b], c, rownames=["a", "b"], colnames=["c"], margins=True, dropna=True
+ )
+ m = MultiIndex.from_arrays(
+ [["bar", "bar", "foo", "foo", "All"], ["one", "two", "one", "two", ""]],
+ names=["a", "b"],
+ )
+ expected = DataFrame(
+ [[1, 0, 1], [1, 0, 1], [2, 0, 2], [1, 1, 2], [5, 1, 6]], index=m
+ )
+ expected.columns = Index(["dull", "shiny", "All"], name="c")
+ tm.assert_frame_equal(actual, expected)
+
+ def test_crosstab_normalize(self):
+ # Issue 12578
+ df = DataFrame(
+ {"a": [1, 2, 2, 2, 2], "b": [3, 3, 4, 4, 4], "c": [1, 1, np.nan, 1, 1]}
+ )
+
+ rindex = Index([1, 2], name="a")
+ cindex = Index([3, 4], name="b")
+ full_normal = DataFrame([[0.2, 0], [0.2, 0.6]], index=rindex, columns=cindex)
+ row_normal = DataFrame([[1.0, 0], [0.25, 0.75]], index=rindex, columns=cindex)
+ col_normal = DataFrame([[0.5, 0], [0.5, 1.0]], index=rindex, columns=cindex)
+
+ # Check all normalize args
+ tm.assert_frame_equal(crosstab(df.a, df.b, normalize="all"), full_normal)
+ tm.assert_frame_equal(crosstab(df.a, df.b, normalize=True), full_normal)
+ tm.assert_frame_equal(crosstab(df.a, df.b, normalize="index"), row_normal)
+ tm.assert_frame_equal(crosstab(df.a, df.b, normalize="columns"), col_normal)
+ tm.assert_frame_equal(
+ crosstab(df.a, df.b, normalize=1),
+ crosstab(df.a, df.b, normalize="columns"),
+ )
+ tm.assert_frame_equal(
+ crosstab(df.a, df.b, normalize=0), crosstab(df.a, df.b, normalize="index")
+ )
+
+ row_normal_margins = DataFrame(
+ [[1.0, 0], [0.25, 0.75], [0.4, 0.6]],
+ index=Index([1, 2, "All"], name="a", dtype="object"),
+ columns=Index([3, 4], name="b", dtype="object"),
+ )
+ col_normal_margins = DataFrame(
+ [[0.5, 0, 0.2], [0.5, 1.0, 0.8]],
+ index=Index([1, 2], name="a", dtype="object"),
+ columns=Index([3, 4, "All"], name="b", dtype="object"),
+ )
+
+ all_normal_margins = DataFrame(
+ [[0.2, 0, 0.2], [0.2, 0.6, 0.8], [0.4, 0.6, 1]],
+ index=Index([1, 2, "All"], name="a", dtype="object"),
+ columns=Index([3, 4, "All"], name="b", dtype="object"),
+ )
+ tm.assert_frame_equal(
+ crosstab(df.a, df.b, normalize="index", margins=True), row_normal_margins
+ )
+ tm.assert_frame_equal(
+ crosstab(df.a, df.b, normalize="columns", margins=True), col_normal_margins
+ )
+ tm.assert_frame_equal(
+ crosstab(df.a, df.b, normalize=True, margins=True), all_normal_margins
+ )
+
+ def test_crosstab_normalize_arrays(self):
+ # GH#12578
+ df = DataFrame(
+ {"a": [1, 2, 2, 2, 2], "b": [3, 3, 4, 4, 4], "c": [1, 1, np.nan, 1, 1]}
+ )
+
+ # Test arrays
+ crosstab(
+ [np.array([1, 1, 2, 2]), np.array([1, 2, 1, 2])], np.array([1, 2, 1, 2])
+ )
+
+ # Test with aggfunc
+ norm_counts = DataFrame(
+ [[0.25, 0, 0.25], [0.25, 0.5, 0.75], [0.5, 0.5, 1]],
+ index=Index([1, 2, "All"], name="a", dtype="object"),
+ columns=Index([3, 4, "All"], name="b"),
+ )
+ test_case = crosstab(
+ df.a, df.b, df.c, aggfunc="count", normalize="all", margins=True
+ )
+ tm.assert_frame_equal(test_case, norm_counts)
+
+ df = DataFrame(
+ {"a": [1, 2, 2, 2, 2], "b": [3, 3, 4, 4, 4], "c": [0, 4, np.nan, 3, 3]}
+ )
+
+ norm_sum = DataFrame(
+ [[0, 0, 0.0], [0.4, 0.6, 1], [0.4, 0.6, 1]],
+ index=Index([1, 2, "All"], name="a", dtype="object"),
+ columns=Index([3, 4, "All"], name="b", dtype="object"),
+ )
+ test_case = crosstab(
+ df.a, df.b, df.c, aggfunc=np.sum, normalize="all", margins=True
+ )
+ tm.assert_frame_equal(test_case, norm_sum)
+
+ def test_crosstab_with_empties(self):
+ # Check handling of empties
+ df = DataFrame(
+ {
+ "a": [1, 2, 2, 2, 2],
+ "b": [3, 3, 4, 4, 4],
+ "c": [np.nan, np.nan, np.nan, np.nan, np.nan],
+ }
+ )
+
+ empty = DataFrame(
+ [[0.0, 0.0], [0.0, 0.0]],
+ index=Index([1, 2], name="a", dtype="int64"),
+ columns=Index([3, 4], name="b"),
+ )
+
+ for i in [True, "index", "columns"]:
+ calculated = crosstab(df.a, df.b, values=df.c, aggfunc="count", normalize=i)
+ tm.assert_frame_equal(empty, calculated)
+
+ nans = DataFrame(
+ [[0.0, np.nan], [0.0, 0.0]],
+ index=Index([1, 2], name="a", dtype="int64"),
+ columns=Index([3, 4], name="b"),
+ )
+
+ calculated = crosstab(df.a, df.b, values=df.c, aggfunc="count", normalize=False)
+ tm.assert_frame_equal(nans, calculated)
+
+ def test_crosstab_errors(self):
+ # Issue 12578
+
+ df = DataFrame(
+ {"a": [1, 2, 2, 2, 2], "b": [3, 3, 4, 4, 4], "c": [1, 1, np.nan, 1, 1]}
+ )
+
+ error = "values cannot be used without an aggfunc."
+ with pytest.raises(ValueError, match=error):
+ crosstab(df.a, df.b, values=df.c)
+
+ error = "aggfunc cannot be used without values"
+ with pytest.raises(ValueError, match=error):
+ crosstab(df.a, df.b, aggfunc=np.mean)
+
+ error = "Not a valid normalize argument"
+ with pytest.raises(ValueError, match=error):
+ crosstab(df.a, df.b, normalize="42")
+
+ with pytest.raises(ValueError, match=error):
+ crosstab(df.a, df.b, normalize=42)
+
+ error = "Not a valid margins argument"
+ with pytest.raises(ValueError, match=error):
+ crosstab(df.a, df.b, normalize="all", margins=42)
+
+ def test_crosstab_with_categorial_columns(self):
+ # GH 8860
+ df = DataFrame(
+ {
+ "MAKE": ["Honda", "Acura", "Tesla", "Honda", "Honda", "Acura"],
+ "MODEL": ["Sedan", "Sedan", "Electric", "Pickup", "Sedan", "Sedan"],
+ }
+ )
+ categories = ["Sedan", "Electric", "Pickup"]
+ df["MODEL"] = df["MODEL"].astype("category").cat.set_categories(categories)
+ result = crosstab(df["MAKE"], df["MODEL"])
+
+ expected_index = Index(["Acura", "Honda", "Tesla"], name="MAKE")
+ expected_columns = CategoricalIndex(
+ categories, categories=categories, ordered=False, name="MODEL"
+ )
+ expected_data = [[2, 0, 0], [2, 0, 1], [0, 1, 0]]
+ expected = DataFrame(
+ expected_data, index=expected_index, columns=expected_columns
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_crosstab_with_numpy_size(self):
+ # GH 4003
+ df = DataFrame(
+ {
+ "A": ["one", "one", "two", "three"] * 6,
+ "B": ["A", "B", "C"] * 8,
+ "C": ["foo", "foo", "foo", "bar", "bar", "bar"] * 4,
+ "D": np.random.default_rng(2).standard_normal(24),
+ "E": np.random.default_rng(2).standard_normal(24),
+ }
+ )
+ result = crosstab(
+ index=[df["A"], df["B"]],
+ columns=[df["C"]],
+ margins=True,
+ aggfunc=np.size,
+ values=df["D"],
+ )
+ expected_index = MultiIndex(
+ levels=[["All", "one", "three", "two"], ["", "A", "B", "C"]],
+ codes=[[1, 1, 1, 2, 2, 2, 3, 3, 3, 0], [1, 2, 3, 1, 2, 3, 1, 2, 3, 0]],
+ names=["A", "B"],
+ )
+ expected_column = Index(["bar", "foo", "All"], name="C")
+ expected_data = np.array(
+ [
+ [2.0, 2.0, 4.0],
+ [2.0, 2.0, 4.0],
+ [2.0, 2.0, 4.0],
+ [2.0, np.nan, 2.0],
+ [np.nan, 2.0, 2.0],
+ [2.0, np.nan, 2.0],
+ [np.nan, 2.0, 2.0],
+ [2.0, np.nan, 2.0],
+ [np.nan, 2.0, 2.0],
+ [12.0, 12.0, 24.0],
+ ]
+ )
+ expected = DataFrame(
+ expected_data, index=expected_index, columns=expected_column
+ )
+ # aggfunc is np.size, resulting in integers
+ expected["All"] = expected["All"].astype("int64")
+ tm.assert_frame_equal(result, expected)
+
+ def test_crosstab_duplicate_names(self):
+ # GH 13279 / 22529
+
+ s1 = Series(range(3), name="foo")
+ s2_foo = Series(range(1, 4), name="foo")
+ s2_bar = Series(range(1, 4), name="bar")
+ s3 = Series(range(3), name="waldo")
+
+ # check result computed with duplicate labels against
+ # result computed with unique labels, then relabelled
+ mapper = {"bar": "foo"}
+
+ # duplicate row, column labels
+ result = crosstab(s1, s2_foo)
+ expected = crosstab(s1, s2_bar).rename_axis(columns=mapper, axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ # duplicate row, unique column labels
+ result = crosstab([s1, s2_foo], s3)
+ expected = crosstab([s1, s2_bar], s3).rename_axis(index=mapper, axis=0)
+ tm.assert_frame_equal(result, expected)
+
+ # unique row, duplicate column labels
+ result = crosstab(s3, [s1, s2_foo])
+ expected = crosstab(s3, [s1, s2_bar]).rename_axis(columns=mapper, axis=1)
+
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("names", [["a", ("b", "c")], [("a", "b"), "c"]])
+ def test_crosstab_tuple_name(self, names):
+ s1 = Series(range(3), name=names[0])
+ s2 = Series(range(1, 4), name=names[1])
+
+ mi = MultiIndex.from_arrays([range(3), range(1, 4)], names=names)
+ expected = Series(1, index=mi).unstack(1, fill_value=0)
+
+ result = crosstab(s1, s2)
+ tm.assert_frame_equal(result, expected)
+
+ def test_crosstab_both_tuple_names(self):
+ # GH 18321
+ s1 = Series(range(3), name=("a", "b"))
+ s2 = Series(range(3), name=("c", "d"))
+
+ expected = DataFrame(
+ np.eye(3, dtype="int64"),
+ index=Index(range(3), name=("a", "b")),
+ columns=Index(range(3), name=("c", "d")),
+ )
+ result = crosstab(s1, s2)
+ tm.assert_frame_equal(result, expected)
+
+ def test_crosstab_unsorted_order(self):
+ df = DataFrame({"b": [3, 1, 2], "a": [5, 4, 6]}, index=["C", "A", "B"])
+ result = crosstab(df.index, [df.b, df.a])
+ e_idx = Index(["A", "B", "C"], name="row_0")
+ e_columns = MultiIndex.from_tuples([(1, 4), (2, 6), (3, 5)], names=["b", "a"])
+ expected = DataFrame(
+ [[1, 0, 0], [0, 1, 0], [0, 0, 1]], index=e_idx, columns=e_columns
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_crosstab_normalize_multiple_columns(self):
+ # GH 15150
+ df = DataFrame(
+ {
+ "A": ["one", "one", "two", "three"] * 6,
+ "B": ["A", "B", "C"] * 8,
+ "C": ["foo", "foo", "foo", "bar", "bar", "bar"] * 4,
+ "D": [0] * 24,
+ "E": [0] * 24,
+ }
+ )
+
+ result = crosstab(
+ [df.A, df.B],
+ df.C,
+ values=df.D,
+ aggfunc=np.sum,
+ normalize=True,
+ margins=True,
+ )
+ expected = DataFrame(
+ np.array([0] * 29 + [1], dtype=float).reshape(10, 3),
+ columns=Index(["bar", "foo", "All"], name="C"),
+ index=MultiIndex.from_tuples(
+ [
+ ("one", "A"),
+ ("one", "B"),
+ ("one", "C"),
+ ("three", "A"),
+ ("three", "B"),
+ ("three", "C"),
+ ("two", "A"),
+ ("two", "B"),
+ ("two", "C"),
+ ("All", ""),
+ ],
+ names=["A", "B"],
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_margin_normalize(self):
+ # GH 27500
+ df = DataFrame(
+ {
+ "A": ["foo", "foo", "foo", "foo", "foo", "bar", "bar", "bar", "bar"],
+ "B": ["one", "one", "one", "two", "two", "one", "one", "two", "two"],
+ "C": [
+ "small",
+ "large",
+ "large",
+ "small",
+ "small",
+ "large",
+ "small",
+ "small",
+ "large",
+ ],
+ "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
+ "E": [2, 4, 5, 5, 6, 6, 8, 9, 9],
+ }
+ )
+ # normalize on index
+ result = crosstab(
+ [df.A, df.B], df.C, margins=True, margins_name="Sub-Total", normalize=0
+ )
+ expected = DataFrame(
+ [[0.5, 0.5], [0.5, 0.5], [0.666667, 0.333333], [0, 1], [0.444444, 0.555556]]
+ )
+ expected.index = MultiIndex(
+ levels=[["Sub-Total", "bar", "foo"], ["", "one", "two"]],
+ codes=[[1, 1, 2, 2, 0], [1, 2, 1, 2, 0]],
+ names=["A", "B"],
+ )
+ expected.columns = Index(["large", "small"], name="C")
+ tm.assert_frame_equal(result, expected)
+
+ # normalize on columns
+ result = crosstab(
+ [df.A, df.B], df.C, margins=True, margins_name="Sub-Total", normalize=1
+ )
+ expected = DataFrame(
+ [
+ [0.25, 0.2, 0.222222],
+ [0.25, 0.2, 0.222222],
+ [0.5, 0.2, 0.333333],
+ [0, 0.4, 0.222222],
+ ]
+ )
+ expected.columns = Index(["large", "small", "Sub-Total"], name="C")
+ expected.index = MultiIndex(
+ levels=[["bar", "foo"], ["one", "two"]],
+ codes=[[0, 0, 1, 1], [0, 1, 0, 1]],
+ names=["A", "B"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+ # normalize on both index and column
+ result = crosstab(
+ [df.A, df.B], df.C, margins=True, margins_name="Sub-Total", normalize=True
+ )
+ expected = DataFrame(
+ [
+ [0.111111, 0.111111, 0.222222],
+ [0.111111, 0.111111, 0.222222],
+ [0.222222, 0.111111, 0.333333],
+ [0.000000, 0.222222, 0.222222],
+ [0.444444, 0.555555, 1],
+ ]
+ )
+ expected.columns = Index(["large", "small", "Sub-Total"], name="C")
+ expected.index = MultiIndex(
+ levels=[["Sub-Total", "bar", "foo"], ["", "one", "two"]],
+ codes=[[1, 1, 2, 2, 0], [1, 2, 1, 2, 0]],
+ names=["A", "B"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_margin_normalize_multiple_columns(self):
+ # GH 35144
+ # use multiple columns with margins and normalization
+ df = DataFrame(
+ {
+ "A": ["foo", "foo", "foo", "foo", "foo", "bar", "bar", "bar", "bar"],
+ "B": ["one", "one", "one", "two", "two", "one", "one", "two", "two"],
+ "C": [
+ "small",
+ "large",
+ "large",
+ "small",
+ "small",
+ "large",
+ "small",
+ "small",
+ "large",
+ ],
+ "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
+ "E": [2, 4, 5, 5, 6, 6, 8, 9, 9],
+ }
+ )
+ result = crosstab(
+ index=df.C,
+ columns=[df.A, df.B],
+ margins=True,
+ margins_name="margin",
+ normalize=True,
+ )
+ expected = DataFrame(
+ [
+ [0.111111, 0.111111, 0.222222, 0.000000, 0.444444],
+ [0.111111, 0.111111, 0.111111, 0.222222, 0.555556],
+ [0.222222, 0.222222, 0.333333, 0.222222, 1.0],
+ ],
+ index=["large", "small", "margin"],
+ )
+ expected.columns = MultiIndex(
+ levels=[["bar", "foo", "margin"], ["", "one", "two"]],
+ codes=[[0, 0, 1, 1, 2], [1, 2, 1, 2, 0]],
+ names=["A", "B"],
+ )
+ expected.index.name = "C"
+ tm.assert_frame_equal(result, expected)
+
+ def test_margin_support_Float(self):
+ # GH 50313
+ # use Float64 formats and function aggfunc with margins
+ df = DataFrame(
+ {"A": [1, 2, 2, 1], "B": [3, 3, 4, 5], "C": [-1.0, 10.0, 1.0, 10.0]},
+ dtype="Float64",
+ )
+ result = crosstab(
+ df["A"],
+ df["B"],
+ values=df["C"],
+ aggfunc="sum",
+ margins=True,
+ )
+ expected = DataFrame(
+ [
+ [-1.0, pd.NA, 10.0, 9.0],
+ [10.0, 1.0, pd.NA, 11.0],
+ [9.0, 1.0, 10.0, 20.0],
+ ],
+ index=Index([1.0, 2.0, "All"], dtype="object", name="A"),
+ columns=Index([3.0, 4.0, 5.0, "All"], dtype="object", name="B"),
+ dtype="Float64",
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_margin_with_ordered_categorical_column(self):
+ # GH 25278
+ df = DataFrame(
+ {
+ "First": ["B", "B", "C", "A", "B", "C"],
+ "Second": ["C", "B", "B", "B", "C", "A"],
+ }
+ )
+ df["First"] = df["First"].astype(CategoricalDtype(ordered=True))
+ customized_categories_order = ["C", "A", "B"]
+ df["First"] = df["First"].cat.reorder_categories(customized_categories_order)
+ result = crosstab(df["First"], df["Second"], margins=True)
+
+ expected_index = Index(["C", "A", "B", "All"], name="First")
+ expected_columns = Index(["A", "B", "C", "All"], name="Second")
+ expected_data = [[1, 1, 0, 2], [0, 1, 0, 1], [0, 1, 2, 3], [1, 3, 2, 6]]
+ expected = DataFrame(
+ expected_data, index=expected_index, columns=expected_columns
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("a_dtype", ["category", "int64"])
+@pytest.mark.parametrize("b_dtype", ["category", "int64"])
+def test_categoricals(a_dtype, b_dtype):
+ # https://github.com/pandas-dev/pandas/issues/37465
+ g = np.random.default_rng(2)
+ a = Series(g.integers(0, 3, size=100)).astype(a_dtype)
+ b = Series(g.integers(0, 2, size=100)).astype(b_dtype)
+ result = crosstab(a, b, margins=True, dropna=False)
+ columns = Index([0, 1, "All"], dtype="object", name="col_0")
+ index = Index([0, 1, 2, "All"], dtype="object", name="row_0")
+ values = [[10, 18, 28], [23, 16, 39], [17, 16, 33], [50, 50, 100]]
+ expected = DataFrame(values, index, columns)
+ tm.assert_frame_equal(result, expected)
+
+ # Verify when categorical does not have all values present
+ a.loc[a == 1] = 2
+ a_is_cat = isinstance(a.dtype, CategoricalDtype)
+ assert not a_is_cat or a.value_counts().loc[1] == 0
+ result = crosstab(a, b, margins=True, dropna=False)
+ values = [[10, 18, 28], [0, 0, 0], [40, 32, 72], [50, 50, 100]]
+ expected = DataFrame(values, index, columns)
+ if not a_is_cat:
+ expected = expected.loc[[0, 2, "All"]]
+ expected["All"] = expected["All"].astype("int64")
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_cut.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_cut.py
new file mode 100644
index 0000000000000000000000000000000000000000..909c10d3f73b20bf55ceba1663e89fd0c692cb30
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_cut.py
@@ -0,0 +1,828 @@
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ Categorical,
+ DataFrame,
+ DatetimeIndex,
+ Index,
+ Interval,
+ IntervalIndex,
+ Series,
+ TimedeltaIndex,
+ Timestamp,
+ cut,
+ date_range,
+ interval_range,
+ isna,
+ qcut,
+ timedelta_range,
+ to_datetime,
+)
+import pandas._testing as tm
+from pandas.api.types import CategoricalDtype
+import pandas.core.reshape.tile as tmod
+
+
+def test_simple():
+ data = np.ones(5, dtype="int64")
+ result = cut(data, 4, labels=False)
+
+ expected = np.array([1, 1, 1, 1, 1])
+ tm.assert_numpy_array_equal(result, expected, check_dtype=False)
+
+
+@pytest.mark.parametrize("func", [list, np.array])
+def test_bins(func):
+ data = func([0.2, 1.4, 2.5, 6.2, 9.7, 2.1])
+ result, bins = cut(data, 3, retbins=True)
+
+ intervals = IntervalIndex.from_breaks(bins.round(3))
+ intervals = intervals.take([0, 0, 0, 1, 2, 0])
+ expected = Categorical(intervals, ordered=True)
+
+ tm.assert_categorical_equal(result, expected)
+ tm.assert_almost_equal(bins, np.array([0.1905, 3.36666667, 6.53333333, 9.7]))
+
+
+def test_right():
+ data = np.array([0.2, 1.4, 2.5, 6.2, 9.7, 2.1, 2.575])
+ result, bins = cut(data, 4, right=True, retbins=True)
+
+ intervals = IntervalIndex.from_breaks(bins.round(3))
+ expected = Categorical(intervals, ordered=True)
+ expected = expected.take([0, 0, 0, 2, 3, 0, 0])
+
+ tm.assert_categorical_equal(result, expected)
+ tm.assert_almost_equal(bins, np.array([0.1905, 2.575, 4.95, 7.325, 9.7]))
+
+
+def test_no_right():
+ data = np.array([0.2, 1.4, 2.5, 6.2, 9.7, 2.1, 2.575])
+ result, bins = cut(data, 4, right=False, retbins=True)
+
+ intervals = IntervalIndex.from_breaks(bins.round(3), closed="left")
+ intervals = intervals.take([0, 0, 0, 2, 3, 0, 1])
+ expected = Categorical(intervals, ordered=True)
+
+ tm.assert_categorical_equal(result, expected)
+ tm.assert_almost_equal(bins, np.array([0.2, 2.575, 4.95, 7.325, 9.7095]))
+
+
+def test_bins_from_interval_index():
+ c = cut(range(5), 3)
+ expected = c
+ result = cut(range(5), bins=expected.categories)
+ tm.assert_categorical_equal(result, expected)
+
+ expected = Categorical.from_codes(
+ np.append(c.codes, -1), categories=c.categories, ordered=True
+ )
+ result = cut(range(6), bins=expected.categories)
+ tm.assert_categorical_equal(result, expected)
+
+
+def test_bins_from_interval_index_doc_example():
+ # Make sure we preserve the bins.
+ ages = np.array([10, 15, 13, 12, 23, 25, 28, 59, 60])
+ c = cut(ages, bins=[0, 18, 35, 70])
+ expected = IntervalIndex.from_tuples([(0, 18), (18, 35), (35, 70)])
+ tm.assert_index_equal(c.categories, expected)
+
+ result = cut([25, 20, 50], bins=c.categories)
+ tm.assert_index_equal(result.categories, expected)
+ tm.assert_numpy_array_equal(result.codes, np.array([1, 1, 2], dtype="int8"))
+
+
+def test_bins_not_overlapping_from_interval_index():
+ # see gh-23980
+ msg = "Overlapping IntervalIndex is not accepted"
+ ii = IntervalIndex.from_tuples([(0, 10), (2, 12), (4, 14)])
+
+ with pytest.raises(ValueError, match=msg):
+ cut([5, 6], bins=ii)
+
+
+def test_bins_not_monotonic():
+ msg = "bins must increase monotonically"
+ data = [0.2, 1.4, 2.5, 6.2, 9.7, 2.1]
+
+ with pytest.raises(ValueError, match=msg):
+ cut(data, [0.1, 1.5, 1, 10])
+
+
+@pytest.mark.parametrize(
+ "x, bins, expected",
+ [
+ (
+ date_range("2017-12-31", periods=3),
+ [Timestamp.min, Timestamp("2018-01-01"), Timestamp.max],
+ IntervalIndex.from_tuples(
+ [
+ (Timestamp.min, Timestamp("2018-01-01")),
+ (Timestamp("2018-01-01"), Timestamp.max),
+ ]
+ ),
+ ),
+ (
+ [-1, 0, 1],
+ np.array(
+ [np.iinfo(np.int64).min, 0, np.iinfo(np.int64).max], dtype="int64"
+ ),
+ IntervalIndex.from_tuples(
+ [(np.iinfo(np.int64).min, 0), (0, np.iinfo(np.int64).max)]
+ ),
+ ),
+ (
+ [
+ np.timedelta64(-1, "ns"),
+ np.timedelta64(0, "ns"),
+ np.timedelta64(1, "ns"),
+ ],
+ np.array(
+ [
+ np.timedelta64(-np.iinfo(np.int64).max, "ns"),
+ np.timedelta64(0, "ns"),
+ np.timedelta64(np.iinfo(np.int64).max, "ns"),
+ ]
+ ),
+ IntervalIndex.from_tuples(
+ [
+ (
+ np.timedelta64(-np.iinfo(np.int64).max, "ns"),
+ np.timedelta64(0, "ns"),
+ ),
+ (
+ np.timedelta64(0, "ns"),
+ np.timedelta64(np.iinfo(np.int64).max, "ns"),
+ ),
+ ]
+ ),
+ ),
+ ],
+)
+def test_bins_monotonic_not_overflowing(x, bins, expected):
+ # GH 26045
+ result = cut(x, bins)
+ tm.assert_index_equal(result.categories, expected)
+
+
+def test_wrong_num_labels():
+ msg = "Bin labels must be one fewer than the number of bin edges"
+ data = [0.2, 1.4, 2.5, 6.2, 9.7, 2.1]
+
+ with pytest.raises(ValueError, match=msg):
+ cut(data, [0, 1, 10], labels=["foo", "bar", "baz"])
+
+
+@pytest.mark.parametrize(
+ "x,bins,msg",
+ [
+ ([], 2, "Cannot cut empty array"),
+ ([1, 2, 3], 0.5, "`bins` should be a positive integer"),
+ ],
+)
+def test_cut_corner(x, bins, msg):
+ with pytest.raises(ValueError, match=msg):
+ cut(x, bins)
+
+
+@pytest.mark.parametrize("arg", [2, np.eye(2), DataFrame(np.eye(2))])
+@pytest.mark.parametrize("cut_func", [cut, qcut])
+def test_cut_not_1d_arg(arg, cut_func):
+ msg = "Input array must be 1 dimensional"
+ with pytest.raises(ValueError, match=msg):
+ cut_func(arg, 2)
+
+
+@pytest.mark.parametrize(
+ "data",
+ [
+ [0, 1, 2, 3, 4, np.inf],
+ [-np.inf, 0, 1, 2, 3, 4],
+ [-np.inf, 0, 1, 2, 3, 4, np.inf],
+ ],
+)
+def test_int_bins_with_inf(data):
+ # GH 24314
+ msg = "cannot specify integer `bins` when input data contains infinity"
+ with pytest.raises(ValueError, match=msg):
+ cut(data, bins=3)
+
+
+def test_cut_out_of_range_more():
+ # see gh-1511
+ name = "x"
+
+ ser = Series([0, -1, 0, 1, -3], name=name)
+ ind = cut(ser, [0, 1], labels=False)
+
+ exp = Series([np.nan, np.nan, np.nan, 0, np.nan], name=name)
+ tm.assert_series_equal(ind, exp)
+
+
+@pytest.mark.parametrize(
+ "right,breaks,closed",
+ [
+ (True, [-1e-3, 0.25, 0.5, 0.75, 1], "right"),
+ (False, [0, 0.25, 0.5, 0.75, 1 + 1e-3], "left"),
+ ],
+)
+def test_labels(right, breaks, closed):
+ arr = np.tile(np.arange(0, 1.01, 0.1), 4)
+
+ result, bins = cut(arr, 4, retbins=True, right=right)
+ ex_levels = IntervalIndex.from_breaks(breaks, closed=closed)
+ tm.assert_index_equal(result.categories, ex_levels)
+
+
+def test_cut_pass_series_name_to_factor():
+ name = "foo"
+ ser = Series(np.random.default_rng(2).standard_normal(100), name=name)
+
+ factor = cut(ser, 4)
+ assert factor.name == name
+
+
+def test_label_precision():
+ arr = np.arange(0, 0.73, 0.01)
+ result = cut(arr, 4, precision=2)
+
+ ex_levels = IntervalIndex.from_breaks([-0.00072, 0.18, 0.36, 0.54, 0.72])
+ tm.assert_index_equal(result.categories, ex_levels)
+
+
+@pytest.mark.parametrize("labels", [None, False])
+def test_na_handling(labels):
+ arr = np.arange(0, 0.75, 0.01)
+ arr[::3] = np.nan
+
+ result = cut(arr, 4, labels=labels)
+ result = np.asarray(result)
+
+ expected = np.where(isna(arr), np.nan, result)
+ tm.assert_almost_equal(result, expected)
+
+
+def test_inf_handling():
+ data = np.arange(6)
+ data_ser = Series(data, dtype="int64")
+
+ bins = [-np.inf, 2, 4, np.inf]
+ result = cut(data, bins)
+ result_ser = cut(data_ser, bins)
+
+ ex_uniques = IntervalIndex.from_breaks(bins)
+ tm.assert_index_equal(result.categories, ex_uniques)
+
+ assert result[5] == Interval(4, np.inf)
+ assert result[0] == Interval(-np.inf, 2)
+ assert result_ser[5] == Interval(4, np.inf)
+ assert result_ser[0] == Interval(-np.inf, 2)
+
+
+def test_cut_out_of_bounds():
+ arr = np.random.default_rng(2).standard_normal(100)
+ result = cut(arr, [-1, 0, 1])
+
+ mask = isna(result)
+ ex_mask = (arr < -1) | (arr > 1)
+ tm.assert_numpy_array_equal(mask, ex_mask)
+
+
+@pytest.mark.parametrize(
+ "get_labels,get_expected",
+ [
+ (
+ lambda labels: labels,
+ lambda labels: Categorical(
+ ["Medium"] + 4 * ["Small"] + ["Medium", "Large"],
+ categories=labels,
+ ordered=True,
+ ),
+ ),
+ (
+ lambda labels: Categorical.from_codes([0, 1, 2], labels),
+ lambda labels: Categorical.from_codes([1] + 4 * [0] + [1, 2], labels),
+ ),
+ ],
+)
+def test_cut_pass_labels(get_labels, get_expected):
+ bins = [0, 25, 50, 100]
+ arr = [50, 5, 10, 15, 20, 30, 70]
+ labels = ["Small", "Medium", "Large"]
+
+ result = cut(arr, bins, labels=get_labels(labels))
+ tm.assert_categorical_equal(result, get_expected(labels))
+
+
+def test_cut_pass_labels_compat():
+ # see gh-16459
+ arr = [50, 5, 10, 15, 20, 30, 70]
+ labels = ["Good", "Medium", "Bad"]
+
+ result = cut(arr, 3, labels=labels)
+ exp = cut(arr, 3, labels=Categorical(labels, categories=labels, ordered=True))
+ tm.assert_categorical_equal(result, exp)
+
+
+@pytest.mark.parametrize("x", [np.arange(11.0), np.arange(11.0) / 1e10])
+def test_round_frac_just_works(x):
+ # It works.
+ cut(x, 2)
+
+
+@pytest.mark.parametrize(
+ "val,precision,expected",
+ [
+ (-117.9998, 3, -118),
+ (117.9998, 3, 118),
+ (117.9998, 2, 118),
+ (0.000123456, 2, 0.00012),
+ ],
+)
+def test_round_frac(val, precision, expected):
+ # see gh-1979
+ result = tmod._round_frac(val, precision=precision)
+ assert result == expected
+
+
+def test_cut_return_intervals():
+ ser = Series([0, 1, 2, 3, 4, 5, 6, 7, 8])
+ result = cut(ser, 3)
+
+ exp_bins = np.linspace(0, 8, num=4).round(3)
+ exp_bins[0] -= 0.008
+
+ expected = Series(
+ IntervalIndex.from_breaks(exp_bins, closed="right").take(
+ [0, 0, 0, 1, 1, 1, 2, 2, 2]
+ )
+ ).astype(CategoricalDtype(ordered=True))
+ tm.assert_series_equal(result, expected)
+
+
+def test_series_ret_bins():
+ # see gh-8589
+ ser = Series(np.arange(4))
+ result, bins = cut(ser, 2, retbins=True)
+
+ expected = Series(
+ IntervalIndex.from_breaks([-0.003, 1.5, 3], closed="right").repeat(2)
+ ).astype(CategoricalDtype(ordered=True))
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "kwargs,msg",
+ [
+ ({"duplicates": "drop"}, None),
+ ({}, "Bin edges must be unique"),
+ ({"duplicates": "raise"}, "Bin edges must be unique"),
+ ({"duplicates": "foo"}, "invalid value for 'duplicates' parameter"),
+ ],
+)
+def test_cut_duplicates_bin(kwargs, msg):
+ # see gh-20947
+ bins = [0, 2, 4, 6, 10, 10]
+ values = Series(np.array([1, 3, 5, 7, 9]), index=["a", "b", "c", "d", "e"])
+
+ if msg is not None:
+ with pytest.raises(ValueError, match=msg):
+ cut(values, bins, **kwargs)
+ else:
+ result = cut(values, bins, **kwargs)
+ expected = cut(values, pd.unique(np.asarray(bins)))
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("data", [9.0, -9.0, 0.0])
+@pytest.mark.parametrize("length", [1, 2])
+def test_single_bin(data, length):
+ # see gh-14652, gh-15428
+ ser = Series([data] * length)
+ result = cut(ser, 1, labels=False)
+
+ expected = Series([0] * length, dtype=np.intp)
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "values,threshold",
+ [
+ ([0.1, 0.1, 0.1], 0.001), # small positive values
+ ([-0.1, -0.1, -0.1], 0.001), # negative values
+ ([0.01, 0.01, 0.01], 0.0001), # very small values
+ ],
+)
+def test_single_bin_edge_adjustment(values, threshold):
+ # gh-58517 - edge adjustment mutation when all values are same
+ result, bins = cut(values, 3, retbins=True)
+
+ bin_range = bins[-1] - bins[0]
+ assert bin_range < threshold
+
+
+@pytest.mark.parametrize(
+ "array_1_writeable,array_2_writeable", [(True, True), (True, False), (False, False)]
+)
+def test_cut_read_only(array_1_writeable, array_2_writeable):
+ # issue 18773
+ array_1 = np.arange(0, 100, 10)
+ array_1.flags.writeable = array_1_writeable
+
+ array_2 = np.arange(0, 100, 10)
+ array_2.flags.writeable = array_2_writeable
+
+ hundred_elements = np.arange(100)
+ tm.assert_categorical_equal(
+ cut(hundred_elements, array_1), cut(hundred_elements, array_2)
+ )
+
+
+@pytest.mark.parametrize(
+ "conv",
+ [
+ lambda v: Timestamp(v),
+ lambda v: to_datetime(v),
+ lambda v: np.datetime64(v),
+ lambda v: Timestamp(v).to_pydatetime(),
+ ],
+)
+def test_datetime_bin(conv):
+ data = [np.datetime64("2012-12-13"), np.datetime64("2012-12-15")]
+ bin_data = ["2012-12-12", "2012-12-14", "2012-12-16"]
+
+ expected = Series(
+ IntervalIndex(
+ [
+ Interval(Timestamp(bin_data[0]), Timestamp(bin_data[1])),
+ Interval(Timestamp(bin_data[1]), Timestamp(bin_data[2])),
+ ]
+ )
+ )
+
+ bins = [conv(v) for v in bin_data]
+ result = Series(cut(data, bins=bins))
+
+ if type(bins[0]) is np.datetime64:
+ # The bins have microsecond dtype -> so does result
+ expected = expected.astype("interval[datetime64[s]]")
+
+ expected = expected.astype(CategoricalDtype(ordered=True))
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("box", [Series, Index, np.array, list])
+def test_datetime_cut(unit, box):
+ # see gh-14714
+ #
+ # Testing time data when it comes in various collection types.
+ data = to_datetime(["2013-01-01", "2013-01-02", "2013-01-03"]).astype(f"M8[{unit}]")
+ data = box(data)
+ result, _ = cut(data, 3, retbins=True)
+
+ if unit == "s":
+ # See https://github.com/pandas-dev/pandas/pull/56101#discussion_r1405325425
+ # for why we round to 8 seconds instead of 7
+ left = DatetimeIndex(
+ ["2012-12-31 23:57:08", "2013-01-01 16:00:00", "2013-01-02 08:00:00"],
+ dtype=f"M8[{unit}]",
+ )
+ else:
+ left = DatetimeIndex(
+ [
+ "2012-12-31 23:57:07.200000",
+ "2013-01-01 16:00:00",
+ "2013-01-02 08:00:00",
+ ],
+ dtype=f"M8[{unit}]",
+ )
+ right = DatetimeIndex(
+ ["2013-01-01 16:00:00", "2013-01-02 08:00:00", "2013-01-03 00:00:00"],
+ dtype=f"M8[{unit}]",
+ )
+
+ exp_intervals = IntervalIndex.from_arrays(left, right)
+ expected = Series(exp_intervals).astype(CategoricalDtype(ordered=True))
+ tm.assert_series_equal(Series(result), expected)
+
+
+@pytest.mark.parametrize("box", [list, np.array, Index, Series])
+def test_datetime_tz_cut_mismatched_tzawareness(box):
+ # GH#54964
+ bins = box(
+ [
+ Timestamp("2013-01-01 04:57:07.200000"),
+ Timestamp("2013-01-01 21:00:00"),
+ Timestamp("2013-01-02 13:00:00"),
+ Timestamp("2013-01-03 05:00:00"),
+ ]
+ )
+ ser = Series(date_range("20130101", periods=3, tz="US/Eastern"))
+
+ msg = "Cannot use timezone-naive bins with timezone-aware values"
+ with pytest.raises(ValueError, match=msg):
+ cut(ser, bins)
+
+
+@pytest.mark.parametrize(
+ "bins",
+ [
+ 3,
+ [
+ Timestamp("2013-01-01 04:57:07.200000", tz="UTC").tz_convert("US/Eastern"),
+ Timestamp("2013-01-01 21:00:00", tz="UTC").tz_convert("US/Eastern"),
+ Timestamp("2013-01-02 13:00:00", tz="UTC").tz_convert("US/Eastern"),
+ Timestamp("2013-01-03 05:00:00", tz="UTC").tz_convert("US/Eastern"),
+ ],
+ ],
+)
+@pytest.mark.parametrize("box", [list, np.array, Index, Series])
+def test_datetime_tz_cut(bins, box):
+ # see gh-19872
+ tz = "US/Eastern"
+ ser = Series(date_range("20130101", periods=3, tz=tz, unit="ns"))
+
+ if not isinstance(bins, int):
+ bins = box(bins)
+
+ result = cut(ser, bins)
+ ii = IntervalIndex(
+ [
+ Interval(
+ Timestamp("2012-12-31 23:57:07.200000", tz=tz),
+ Timestamp("2013-01-01 16:00:00", tz=tz),
+ ),
+ Interval(
+ Timestamp("2013-01-01 16:00:00", tz=tz),
+ Timestamp("2013-01-02 08:00:00", tz=tz),
+ ),
+ Interval(
+ Timestamp("2013-01-02 08:00:00", tz=tz),
+ Timestamp("2013-01-03 00:00:00", tz=tz),
+ ),
+ ]
+ )
+ if isinstance(bins, int):
+ # the dtype is inferred from ser, which has nanosecond unit
+ ii = ii.astype("interval[datetime64[ns, US/Eastern]]")
+ expected = Series(ii).astype(CategoricalDtype(ordered=True))
+ tm.assert_series_equal(result, expected)
+
+
+def test_datetime_nan_error():
+ msg = "bins must be of datetime64 dtype"
+
+ with pytest.raises(ValueError, match=msg):
+ cut(date_range("20130101", periods=3), bins=[0, 2, 4])
+
+
+def test_datetime_nan_mask():
+ result = cut(
+ date_range("20130102", periods=5), bins=date_range("20130101", periods=2)
+ )
+
+ mask = result.categories.isna()
+ tm.assert_numpy_array_equal(mask, np.array([False]))
+
+ mask = result.isna()
+ tm.assert_numpy_array_equal(mask, np.array([False, True, True, True, True]))
+
+
+@pytest.mark.parametrize("tz", [None, "UTC", "US/Pacific"])
+def test_datetime_cut_roundtrip(tz, unit):
+ # see gh-19891
+ ser = Series(date_range("20180101", periods=3, tz=tz, unit=unit))
+ result, result_bins = cut(ser, 2, retbins=True)
+
+ expected = cut(ser, result_bins)
+ tm.assert_series_equal(result, expected)
+
+ if unit == "s":
+ # TODO: constructing DatetimeIndex with dtype="M8[s]" without truncating
+ # the first entry here raises in array_to_datetime. Should truncate
+ # instead of raising?
+ # See https://github.com/pandas-dev/pandas/pull/56101#discussion_r1405325425
+ # for why we round to 8 seconds instead of 7
+ expected_bins = DatetimeIndex(
+ ["2017-12-31 23:57:08", "2018-01-02 00:00:00", "2018-01-03 00:00:00"],
+ dtype=f"M8[{unit}]",
+ )
+ else:
+ expected_bins = DatetimeIndex(
+ [
+ "2017-12-31 23:57:07.200000",
+ "2018-01-02 00:00:00",
+ "2018-01-03 00:00:00",
+ ],
+ dtype=f"M8[{unit}]",
+ )
+ expected_bins = expected_bins.tz_localize(tz)
+ tm.assert_index_equal(result_bins, expected_bins)
+
+
+def test_timedelta_cut_roundtrip():
+ # see gh-19891
+ ser = Series(timedelta_range("1day", periods=3))
+ result, result_bins = cut(ser, 2, retbins=True)
+
+ expected = cut(ser, result_bins)
+ tm.assert_series_equal(result, expected)
+
+ expected_bins = TimedeltaIndex(
+ ["0 days 23:57:07.200000", "2 days 00:00:00", "3 days 00:00:00"]
+ )
+ tm.assert_index_equal(result_bins, expected_bins)
+
+
+@pytest.mark.parametrize("bins", [6, 7])
+@pytest.mark.parametrize(
+ "box, compare",
+ [
+ (Series, tm.assert_series_equal),
+ (np.array, tm.assert_categorical_equal),
+ (list, tm.assert_equal),
+ ],
+)
+def test_cut_bool_coercion_to_int(bins, box, compare):
+ # issue 20303
+ data_expected = box([0, 1, 1, 0, 1] * 10)
+ data_result = box([False, True, True, False, True] * 10)
+ expected = cut(data_expected, bins, duplicates="drop")
+ result = cut(data_result, bins, duplicates="drop")
+ compare(result, expected)
+
+
+@pytest.mark.parametrize("labels", ["foo", 1, True])
+def test_cut_incorrect_labels(labels):
+ # GH 13318
+ values = range(5)
+ msg = "Bin labels must either be False, None or passed in as a list-like argument"
+ with pytest.raises(ValueError, match=msg):
+ cut(values, 4, labels=labels)
+
+
+@pytest.mark.parametrize("bins", [3, [0, 5, 15]])
+@pytest.mark.parametrize("right", [True, False])
+@pytest.mark.parametrize("include_lowest", [True, False])
+def test_cut_nullable_integer(bins, right, include_lowest):
+ a = np.random.default_rng(2).integers(0, 10, size=50).astype(float)
+ a[::2] = np.nan
+ b = a.astype(object)
+ b[::2] = pd.NA
+ result = cut(
+ pd.array(b, dtype="Int64"), bins, right=right, include_lowest=include_lowest
+ )
+ expected = cut(a, bins, right=right, include_lowest=include_lowest)
+ tm.assert_categorical_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "data, bins, labels, expected_codes, expected_labels",
+ [
+ ([15, 17, 19], [14, 16, 18, 20], ["A", "B", "A"], [0, 1, 0], ["A", "B"]),
+ ([1, 3, 5], [0, 2, 4, 6, 8], [2, 0, 1, 2], [2, 0, 1], [0, 1, 2]),
+ ],
+)
+def test_cut_non_unique_labels(data, bins, labels, expected_codes, expected_labels):
+ # GH 33141
+ result = cut(data, bins=bins, labels=labels, ordered=False)
+ expected = Categorical.from_codes(
+ expected_codes, categories=expected_labels, ordered=False
+ )
+ tm.assert_categorical_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "data, bins, labels, expected_codes, expected_labels",
+ [
+ ([15, 17, 19], [14, 16, 18, 20], ["C", "B", "A"], [0, 1, 2], ["C", "B", "A"]),
+ ([1, 3, 5], [0, 2, 4, 6, 8], [3, 0, 1, 2], [0, 1, 2], [3, 0, 1, 2]),
+ ],
+)
+def test_cut_unordered_labels(data, bins, labels, expected_codes, expected_labels):
+ # GH 33141
+ result = cut(data, bins=bins, labels=labels, ordered=False)
+ expected = Categorical.from_codes(
+ expected_codes, categories=expected_labels, ordered=False
+ )
+ tm.assert_categorical_equal(result, expected)
+
+
+def test_cut_unordered_with_missing_labels_raises_error():
+ # GH 33141
+ msg = "'labels' must be provided if 'ordered = False'"
+ with pytest.raises(ValueError, match=msg):
+ cut([0.5, 3], bins=[0, 1, 2], ordered=False)
+
+
+def test_cut_unordered_with_series_labels():
+ # https://github.com/pandas-dev/pandas/issues/36603
+ ser = Series([1, 2, 3, 4, 5])
+ bins = Series([0, 2, 4, 6])
+ labels = Series(["a", "b", "c"])
+ result = cut(ser, bins=bins, labels=labels, ordered=False)
+ expected = Series(["a", "a", "b", "b", "c"], dtype="category")
+ tm.assert_series_equal(result, expected)
+
+
+def test_cut_no_warnings():
+ df = DataFrame({"value": np.random.default_rng(2).integers(0, 100, 20)})
+ labels = [f"{i} - {i + 9}" for i in range(0, 100, 10)]
+ with tm.assert_produces_warning(False):
+ df["group"] = cut(df.value, range(0, 105, 10), right=False, labels=labels)
+
+
+def test_cut_with_duplicated_index_lowest_included():
+ # GH 42185
+ expected = Series(
+ [Interval(-0.001, 2, closed="right")] * 3
+ + [Interval(2, 4, closed="right"), Interval(-0.001, 2, closed="right")],
+ index=[0, 1, 2, 3, 0],
+ dtype="category",
+ ).cat.as_ordered()
+
+ ser = Series([0, 1, 2, 3, 0], index=[0, 1, 2, 3, 0])
+ result = cut(ser, bins=[0, 2, 4], include_lowest=True)
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.filterwarnings("ignore:invalid value encountered in cast:RuntimeWarning")
+def test_cut_with_nonexact_categorical_indices():
+ # GH 42424
+
+ ser = Series(range(100))
+ ser1 = cut(ser, 10).value_counts().head(5)
+ ser2 = cut(ser, 10).value_counts().tail(5)
+ result = DataFrame({"1": ser1, "2": ser2})
+
+ index = pd.CategoricalIndex(
+ [
+ Interval(-0.099, 9.9, closed="right"),
+ Interval(9.9, 19.8, closed="right"),
+ Interval(19.8, 29.7, closed="right"),
+ Interval(29.7, 39.6, closed="right"),
+ Interval(39.6, 49.5, closed="right"),
+ Interval(49.5, 59.4, closed="right"),
+ Interval(59.4, 69.3, closed="right"),
+ Interval(69.3, 79.2, closed="right"),
+ Interval(79.2, 89.1, closed="right"),
+ Interval(89.1, 99, closed="right"),
+ ],
+ ordered=True,
+ )
+
+ expected = DataFrame(
+ {"1": [10] * 5 + [np.nan] * 5, "2": [np.nan] * 5 + [10] * 5}, index=index
+ )
+
+ tm.assert_frame_equal(expected, result)
+
+
+def test_cut_with_timestamp_tuple_labels():
+ # GH 40661
+ labels = [(Timestamp(10),), (Timestamp(20),), (Timestamp(30),)]
+ result = cut([2, 4, 6], bins=[1, 3, 5, 7], labels=labels)
+
+ expected = Categorical.from_codes([0, 1, 2], labels, ordered=True)
+ tm.assert_categorical_equal(result, expected)
+
+
+def test_cut_bins_datetime_intervalindex():
+ # https://github.com/pandas-dev/pandas/issues/46218
+ bins = interval_range(Timestamp("2022-02-25"), Timestamp("2022-02-27"), freq="1D")
+ # passing Series instead of list is important to trigger bug
+ result = cut(Series([Timestamp("2022-02-26")]), bins=bins)
+ expected = Categorical.from_codes([0], bins, ordered=True)
+ tm.assert_categorical_equal(result.array, expected)
+
+
+def test_cut_with_nullable_int64():
+ # GH 30787
+ series = Series([0, 1, 2, 3, 4, pd.NA, 6, 7], dtype="Int64")
+ bins = [0, 2, 4, 6, 8]
+ intervals = IntervalIndex.from_breaks(bins)
+
+ expected = Series(
+ Categorical.from_codes([-1, 0, 0, 1, 1, -1, 2, 3], intervals, ordered=True)
+ )
+
+ result = cut(series, bins=bins)
+
+ tm.assert_series_equal(result, expected)
+
+
+def test_cut_datetime_array_no_attributeerror():
+ # GH 55431
+ ser = Series(to_datetime(["2023-10-06 12:00:00+0000", "2023-10-07 12:00:00+0000"]))
+
+ result = cut(ser.array, bins=2)
+
+ categories = result.categories
+ expected = Categorical.from_codes([0, 1], categories=categories, ordered=True)
+
+ tm.assert_categorical_equal(
+ result, expected, check_dtype=True, check_category_order=True
+ )
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_from_dummies.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_from_dummies.py
new file mode 100644
index 0000000000000000000000000000000000000000..0997baf7c3f74bf6b73e62e02283cea9eb1bd696
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_from_dummies.py
@@ -0,0 +1,477 @@
+import numpy as np
+import pytest
+
+from pandas import (
+ DataFrame,
+ Series,
+ from_dummies,
+ get_dummies,
+)
+import pandas._testing as tm
+
+
+@pytest.fixture
+def dummies_basic():
+ return DataFrame(
+ {
+ "col1_a": [1, 0, 1],
+ "col1_b": [0, 1, 0],
+ "col2_a": [0, 1, 0],
+ "col2_b": [1, 0, 0],
+ "col2_c": [0, 0, 1],
+ },
+ )
+
+
+@pytest.fixture
+def dummies_with_unassigned():
+ return DataFrame(
+ {
+ "col1_a": [1, 0, 0],
+ "col1_b": [0, 1, 0],
+ "col2_a": [0, 1, 0],
+ "col2_b": [0, 0, 0],
+ "col2_c": [0, 0, 1],
+ },
+ )
+
+
+def test_error_wrong_data_type():
+ dummies = [0, 1, 0]
+ with pytest.raises(
+ TypeError,
+ match=r"Expected 'data' to be a 'DataFrame'; Received 'data' of type: list",
+ ):
+ from_dummies(dummies)
+
+
+def test_error_no_prefix_contains_unassigned():
+ dummies = DataFrame({"a": [1, 0, 0], "b": [0, 1, 0]})
+ with pytest.raises(
+ ValueError,
+ match=(
+ r"Dummy DataFrame contains unassigned value\(s\); "
+ r"First instance in row: 2"
+ ),
+ ):
+ from_dummies(dummies)
+
+
+def test_error_no_prefix_wrong_default_category_type():
+ dummies = DataFrame({"a": [1, 0, 1], "b": [0, 1, 1]})
+ with pytest.raises(
+ TypeError,
+ match=(
+ r"Expected 'default_category' to be of type 'None', 'Hashable', or 'dict'; "
+ r"Received 'default_category' of type: list"
+ ),
+ ):
+ from_dummies(dummies, default_category=["c", "d"])
+
+
+def test_error_no_prefix_multi_assignment():
+ dummies = DataFrame({"a": [1, 0, 1], "b": [0, 1, 1]})
+ with pytest.raises(
+ ValueError,
+ match=(
+ r"Dummy DataFrame contains multi-assignment\(s\); "
+ r"First instance in row: 2"
+ ),
+ ):
+ from_dummies(dummies)
+
+
+def test_error_no_prefix_contains_nan():
+ dummies = DataFrame({"a": [1, 0, 0], "b": [0, 1, np.nan]})
+ with pytest.raises(
+ ValueError, match=r"Dummy DataFrame contains NA value in column: 'b'"
+ ):
+ from_dummies(dummies)
+
+
+def test_error_contains_non_dummies():
+ dummies = DataFrame(
+ {"a": [1, 6, 3, 1], "b": [0, 1, 0, 2], "c": ["c1", "c2", "c3", "c4"]}
+ )
+ with pytest.raises(
+ TypeError,
+ match=r"Passed DataFrame contains non-dummy data",
+ ):
+ from_dummies(dummies)
+
+
+def test_error_with_prefix_multiple_separators():
+ dummies = DataFrame(
+ {
+ "col1_a": [1, 0, 1],
+ "col1_b": [0, 1, 0],
+ "col2-a": [0, 1, 0],
+ "col2-b": [1, 0, 1],
+ },
+ )
+ with pytest.raises(
+ ValueError,
+ match=(r"Separator not specified for column: col2-a"),
+ ):
+ from_dummies(dummies, sep="_")
+
+
+def test_error_with_prefix_sep_wrong_type(dummies_basic):
+ with pytest.raises(
+ TypeError,
+ match=(
+ r"Expected 'sep' to be of type 'str' or 'None'; "
+ r"Received 'sep' of type: list"
+ ),
+ ):
+ from_dummies(dummies_basic, sep=["_"])
+
+
+def test_error_with_prefix_contains_unassigned(dummies_with_unassigned):
+ with pytest.raises(
+ ValueError,
+ match=(
+ r"Dummy DataFrame contains unassigned value\(s\); "
+ r"First instance in row: 2"
+ ),
+ ):
+ from_dummies(dummies_with_unassigned, sep="_")
+
+
+def test_error_with_prefix_default_category_wrong_type(dummies_with_unassigned):
+ with pytest.raises(
+ TypeError,
+ match=(
+ r"Expected 'default_category' to be of type 'None', 'Hashable', or 'dict'; "
+ r"Received 'default_category' of type: list"
+ ),
+ ):
+ from_dummies(dummies_with_unassigned, sep="_", default_category=["x", "y"])
+
+
+def test_error_with_prefix_default_category_dict_not_complete(
+ dummies_with_unassigned,
+):
+ with pytest.raises(
+ ValueError,
+ match=(
+ r"Length of 'default_category' \(1\) did not match "
+ r"the length of the columns being encoded \(2\)"
+ ),
+ ):
+ from_dummies(dummies_with_unassigned, sep="_", default_category={"col1": "x"})
+
+
+def test_error_with_prefix_contains_nan(dummies_basic):
+ # Set float64 dtype to avoid upcast when setting np.nan
+ dummies_basic["col2_c"] = dummies_basic["col2_c"].astype("float64")
+ dummies_basic.loc[2, "col2_c"] = np.nan
+ with pytest.raises(
+ ValueError, match=r"Dummy DataFrame contains NA value in column: 'col2_c'"
+ ):
+ from_dummies(dummies_basic, sep="_")
+
+
+def test_error_with_prefix_contains_non_dummies(dummies_basic):
+ # Set object dtype to avoid upcast when setting "str"
+ dummies_basic["col2_c"] = dummies_basic["col2_c"].astype(object)
+ dummies_basic.loc[2, "col2_c"] = "str"
+ with pytest.raises(TypeError, match=r"Passed DataFrame contains non-dummy data"):
+ from_dummies(dummies_basic, sep="_")
+
+
+def test_error_with_prefix_double_assignment():
+ dummies = DataFrame(
+ {
+ "col1_a": [1, 0, 1],
+ "col1_b": [1, 1, 0],
+ "col2_a": [0, 1, 0],
+ "col2_b": [1, 0, 0],
+ "col2_c": [0, 0, 1],
+ },
+ )
+ with pytest.raises(
+ ValueError,
+ match=(
+ r"Dummy DataFrame contains multi-assignment\(s\); "
+ r"First instance in row: 0"
+ ),
+ ):
+ from_dummies(dummies, sep="_")
+
+
+def test_roundtrip_series_to_dataframe():
+ categories = Series(["a", "b", "c", "a"])
+ dummies = get_dummies(categories)
+ result = from_dummies(dummies)
+ expected = DataFrame({"": ["a", "b", "c", "a"]})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_roundtrip_single_column_dataframe():
+ categories = DataFrame({"": ["a", "b", "c", "a"]})
+ dummies = get_dummies(categories)
+ result = from_dummies(dummies, sep="_")
+ expected = categories
+ tm.assert_frame_equal(result, expected)
+
+
+def test_roundtrip_with_prefixes():
+ categories = DataFrame({"col1": ["a", "b", "a"], "col2": ["b", "a", "c"]})
+ dummies = get_dummies(categories)
+ result = from_dummies(dummies, sep="_")
+ expected = categories
+ tm.assert_frame_equal(result, expected)
+
+
+def test_no_prefix_string_cats_basic():
+ dummies = DataFrame({"a": [1, 0, 0, 1], "b": [0, 1, 0, 0], "c": [0, 0, 1, 0]})
+ expected = DataFrame({"": ["a", "b", "c", "a"]})
+ result = from_dummies(dummies)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_no_prefix_string_cats_basic_bool_values():
+ dummies = DataFrame(
+ {
+ "a": [True, False, False, True],
+ "b": [False, True, False, False],
+ "c": [False, False, True, False],
+ }
+ )
+ expected = DataFrame({"": ["a", "b", "c", "a"]})
+ result = from_dummies(dummies)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_no_prefix_string_cats_basic_mixed_bool_values():
+ dummies = DataFrame(
+ {"a": [1, 0, 0, 1], "b": [False, True, False, False], "c": [0, 0, 1, 0]}
+ )
+ expected = DataFrame({"": ["a", "b", "c", "a"]})
+ result = from_dummies(dummies)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_no_prefix_int_cats_basic():
+ dummies = DataFrame(
+ {1: [1, 0, 0, 0], 25: [0, 1, 0, 0], 2: [0, 0, 1, 0], 5: [0, 0, 0, 1]}
+ )
+ expected = DataFrame({"": [1, 25, 2, 5]})
+ result = from_dummies(dummies)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_no_prefix_float_cats_basic():
+ dummies = DataFrame(
+ {1.0: [1, 0, 0, 0], 25.0: [0, 1, 0, 0], 2.5: [0, 0, 1, 0], 5.84: [0, 0, 0, 1]}
+ )
+ expected = DataFrame({"": [1.0, 25.0, 2.5, 5.84]})
+ result = from_dummies(dummies)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_no_prefix_mixed_cats_basic():
+ dummies = DataFrame(
+ {
+ 1.23: [1, 0, 0, 0, 0],
+ "c": [0, 1, 0, 0, 0],
+ 2: [0, 0, 1, 0, 0],
+ False: [0, 0, 0, 1, 0],
+ None: [0, 0, 0, 0, 1],
+ }
+ )
+ expected = DataFrame({"": [1.23, "c", 2, False, None]}, dtype="object")
+ result = from_dummies(dummies)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_no_prefix_string_cats_contains_get_dummies_NaN_column():
+ dummies = DataFrame({"a": [1, 0, 0], "b": [0, 1, 0], "NaN": [0, 0, 1]})
+ expected = DataFrame({"": ["a", "b", "NaN"]})
+ result = from_dummies(dummies)
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "default_category, expected",
+ [
+ pytest.param(
+ "c",
+ {"": ["a", "b", "c"]},
+ id="default_category is a str",
+ ),
+ pytest.param(
+ 1,
+ {"": ["a", "b", 1]},
+ id="default_category is an int",
+ ),
+ pytest.param(
+ 1.25,
+ {"": ["a", "b", 1.25]},
+ id="default_category is a float",
+ ),
+ pytest.param(
+ 0,
+ {"": ["a", "b", 0]},
+ id="default_category is a 0",
+ ),
+ pytest.param(
+ False,
+ {"": ["a", "b", False]},
+ id="default_category is a bool",
+ ),
+ pytest.param(
+ (1, 2),
+ {"": ["a", "b", (1, 2)]},
+ id="default_category is a tuple",
+ ),
+ ],
+)
+def test_no_prefix_string_cats_default_category(
+ default_category, expected, using_infer_string
+):
+ dummies = DataFrame({"a": [1, 0, 0], "b": [0, 1, 0]})
+ result = from_dummies(dummies, default_category=default_category)
+ expected = DataFrame(expected, dtype=dummies.columns.dtype)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_with_prefix_basic(dummies_basic):
+ expected = DataFrame({"col1": ["a", "b", "a"], "col2": ["b", "a", "c"]})
+ result = from_dummies(dummies_basic, sep="_")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_with_prefix_contains_get_dummies_NaN_column():
+ dummies = DataFrame(
+ {
+ "col1_a": [1, 0, 0],
+ "col1_b": [0, 1, 0],
+ "col1_NaN": [0, 0, 1],
+ "col2_a": [0, 1, 0],
+ "col2_b": [0, 0, 0],
+ "col2_c": [0, 0, 1],
+ "col2_NaN": [1, 0, 0],
+ },
+ )
+ expected = DataFrame({"col1": ["a", "b", "NaN"], "col2": ["NaN", "a", "c"]})
+ result = from_dummies(dummies, sep="_")
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "default_category, expected",
+ [
+ pytest.param(
+ "x",
+ {"col1": ["a", "b", "x"], "col2": ["x", "a", "c"]},
+ id="default_category is a str",
+ ),
+ pytest.param(
+ 0,
+ {"col1": ["a", "b", 0], "col2": [0, "a", "c"]},
+ id="default_category is a 0",
+ ),
+ pytest.param(
+ False,
+ {"col1": ["a", "b", False], "col2": [False, "a", "c"]},
+ id="default_category is a False",
+ ),
+ pytest.param(
+ {"col2": 1, "col1": 2.5},
+ {"col1": ["a", "b", 2.5], "col2": [1, "a", "c"]},
+ id="default_category is a dict with int and float values",
+ ),
+ pytest.param(
+ {"col2": None, "col1": False},
+ {"col1": ["a", "b", False], "col2": [None, "a", "c"]},
+ id="default_category is a dict with bool and None values",
+ ),
+ pytest.param(
+ {"col2": (1, 2), "col1": [1.25, False]},
+ {"col1": ["a", "b", [1.25, False]], "col2": [(1, 2), "a", "c"]},
+ id="default_category is a dict with list and tuple values",
+ ),
+ ],
+)
+def test_with_prefix_default_category(
+ dummies_with_unassigned, default_category, expected, using_infer_string
+):
+ result = from_dummies(
+ dummies_with_unassigned, sep="_", default_category=default_category
+ )
+ expected = DataFrame(expected)
+ if using_infer_string:
+ expected = expected.astype("str")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_ea_categories():
+ # GH 54300
+ df = DataFrame({"a": [1, 0, 0, 1], "b": [0, 1, 0, 0], "c": [0, 0, 1, 0]})
+ df.columns = df.columns.astype("string[python]")
+ result = from_dummies(df)
+ expected = DataFrame({"": Series(list("abca"), dtype="string[python]")})
+ tm.assert_frame_equal(result, expected)
+
+
+def test_ea_categories_with_sep():
+ # GH 54300
+ df = DataFrame(
+ {
+ "col1_a": [1, 0, 1],
+ "col1_b": [0, 1, 0],
+ "col2_a": [0, 1, 0],
+ "col2_b": [1, 0, 0],
+ "col2_c": [0, 0, 1],
+ }
+ )
+ df.columns = df.columns.astype("string[python]")
+ result = from_dummies(df, sep="_")
+ expected = DataFrame(
+ {
+ "col1": Series(list("aba"), dtype="string[python]"),
+ "col2": Series(list("bac"), dtype="string[python]"),
+ }
+ )
+ expected.columns = expected.columns.astype("string[python]")
+ tm.assert_frame_equal(result, expected)
+
+
+def test_maintain_original_index():
+ # GH 54300
+ df = DataFrame(
+ {"a": [1, 0, 0, 1], "b": [0, 1, 0, 0], "c": [0, 0, 1, 0]}, index=list("abcd")
+ )
+ result = from_dummies(df)
+ expected = DataFrame({"": list("abca")}, index=list("abcd"))
+ tm.assert_frame_equal(result, expected)
+
+
+def test_int_columns_with_float_default():
+ # https://github.com/pandas-dev/pandas/pull/60694
+ df = DataFrame(
+ {
+ 3: [1, 0, 0],
+ 4: [0, 1, 0],
+ },
+ )
+ with pytest.raises(ValueError, match="Trying to coerce float values to integers"):
+ from_dummies(df, default_category=0.5)
+
+
+def test_object_dtype_preserved():
+ # https://github.com/pandas-dev/pandas/pull/60694
+ # When the input has object dtype, the result should as
+ # well even when infer_string is True.
+ df = DataFrame(
+ {
+ "x": [1, 0, 0],
+ "y": [0, 1, 0],
+ },
+ )
+ df.columns = df.columns.astype("object")
+ result = from_dummies(df, default_category="z")
+ expected = DataFrame({"": ["x", "y", "z"]}, dtype="object")
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_get_dummies.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_get_dummies.py
new file mode 100644
index 0000000000000000000000000000000000000000..c776e7b2e3d9a4f6ab99042cf95a6996d1ba5b60
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_get_dummies.py
@@ -0,0 +1,741 @@
+import re
+import unicodedata
+
+import numpy as np
+import pytest
+
+import pandas.util._test_decorators as td
+
+from pandas.core.dtypes.common import is_integer_dtype
+
+import pandas as pd
+from pandas import (
+ ArrowDtype,
+ Categorical,
+ CategoricalDtype,
+ CategoricalIndex,
+ DataFrame,
+ Index,
+ RangeIndex,
+ Series,
+ SparseDtype,
+ get_dummies,
+)
+import pandas._testing as tm
+from pandas.core.arrays.sparse import SparseArray
+
+try:
+ import pyarrow as pa
+except ImportError:
+ pa = None
+
+
+class TestGetDummies:
+ @pytest.fixture
+ def df(self):
+ return DataFrame({"A": ["a", "b", "a"], "B": ["b", "b", "c"], "C": [1, 2, 3]})
+
+ @pytest.fixture(params=["uint8", "i8", np.float64, bool, None])
+ def dtype(self, request):
+ return np.dtype(request.param)
+
+ @pytest.fixture(params=["dense", "sparse"])
+ def sparse(self, request):
+ # params are strings to simplify reading test results,
+ # e.g. TestGetDummies::test_basic[uint8-sparse] instead of [uint8-True]
+ return request.param == "sparse"
+
+ def effective_dtype(self, dtype):
+ if dtype is None:
+ return np.uint8
+ return dtype
+
+ def test_get_dummies_raises_on_dtype_object(self, df):
+ msg = "dtype=object is not a valid dtype for get_dummies"
+ with pytest.raises(ValueError, match=msg):
+ get_dummies(df, dtype="object")
+
+ def test_get_dummies_basic(self, sparse, dtype):
+ s_list = list("abc")
+ s_series = Series(s_list)
+ s_series_index = Series(s_list, list("ABC"))
+
+ expected = DataFrame(
+ {"a": [1, 0, 0], "b": [0, 1, 0], "c": [0, 0, 1]},
+ dtype=self.effective_dtype(dtype),
+ )
+ if sparse:
+ if dtype.kind == "b":
+ expected = expected.apply(SparseArray, fill_value=False)
+ else:
+ expected = expected.apply(SparseArray, fill_value=0.0)
+ result = get_dummies(s_list, sparse=sparse, dtype=dtype)
+ tm.assert_frame_equal(result, expected)
+
+ result = get_dummies(s_series, sparse=sparse, dtype=dtype)
+ tm.assert_frame_equal(result, expected)
+
+ expected.index = list("ABC")
+ result = get_dummies(s_series_index, sparse=sparse, dtype=dtype)
+ tm.assert_frame_equal(result, expected)
+
+ def test_get_dummies_basic_types(self, sparse, dtype, using_infer_string):
+ # GH 10531
+ s_list = list("abc")
+ s_series = Series(s_list)
+ s_df = DataFrame(
+ {"a": [0, 1, 0, 1, 2], "b": ["A", "A", "B", "C", "C"], "c": [2, 3, 3, 3, 2]}
+ )
+
+ expected = DataFrame(
+ {"a": [1, 0, 0], "b": [0, 1, 0], "c": [0, 0, 1]},
+ dtype=self.effective_dtype(dtype),
+ columns=list("abc"),
+ )
+ if sparse:
+ if is_integer_dtype(dtype):
+ fill_value = 0
+ elif dtype == bool:
+ fill_value = False
+ else:
+ fill_value = 0.0
+
+ expected = expected.apply(SparseArray, fill_value=fill_value)
+ result = get_dummies(s_list, sparse=sparse, dtype=dtype)
+ tm.assert_frame_equal(result, expected)
+
+ result = get_dummies(s_series, sparse=sparse, dtype=dtype)
+ tm.assert_frame_equal(result, expected)
+
+ result = get_dummies(s_df, columns=s_df.columns, sparse=sparse, dtype=dtype)
+ if sparse:
+ dtype_name = f"Sparse[{self.effective_dtype(dtype).name}, {fill_value}]"
+ else:
+ dtype_name = self.effective_dtype(dtype).name
+
+ expected = Series({dtype_name: 8}, name="count")
+ result = result.dtypes.value_counts()
+ result.index = [str(i) for i in result.index]
+ tm.assert_series_equal(result, expected)
+
+ result = get_dummies(s_df, columns=["a"], sparse=sparse, dtype=dtype)
+
+ key = "str" if using_infer_string else "object"
+ expected_counts = {"int64": 1, key: 1}
+ expected_counts[dtype_name] = 3 + expected_counts.get(dtype_name, 0)
+
+ expected = Series(expected_counts, name="count").sort_index()
+ result = result.dtypes.value_counts()
+ result.index = [str(i) for i in result.index]
+ result = result.sort_index()
+ tm.assert_series_equal(result, expected)
+
+ def test_get_dummies_just_na(self, sparse):
+ just_na_list = [np.nan]
+ just_na_series = Series(just_na_list)
+ just_na_series_index = Series(just_na_list, index=["A"])
+
+ res_list = get_dummies(just_na_list, sparse=sparse)
+ res_series = get_dummies(just_na_series, sparse=sparse)
+ res_series_index = get_dummies(just_na_series_index, sparse=sparse)
+
+ assert res_list.empty
+ assert res_series.empty
+ assert res_series_index.empty
+
+ assert res_list.index.tolist() == [0]
+ assert res_series.index.tolist() == [0]
+ assert res_series_index.index.tolist() == ["A"]
+
+ def test_get_dummies_include_na(self, sparse, dtype):
+ s = ["a", "b", np.nan]
+ res = get_dummies(s, sparse=sparse, dtype=dtype)
+ exp = DataFrame(
+ {"a": [1, 0, 0], "b": [0, 1, 0]}, dtype=self.effective_dtype(dtype)
+ )
+ if sparse:
+ if dtype.kind == "b":
+ exp = exp.apply(SparseArray, fill_value=False)
+ else:
+ exp = exp.apply(SparseArray, fill_value=0.0)
+ tm.assert_frame_equal(res, exp)
+
+ # Sparse dataframes do not allow nan labelled columns, see #GH8822
+ res_na = get_dummies(s, dummy_na=True, sparse=sparse, dtype=dtype)
+ exp_na = DataFrame(
+ {np.nan: [0, 0, 1], "a": [1, 0, 0], "b": [0, 1, 0]},
+ dtype=self.effective_dtype(dtype),
+ )
+ exp_na = exp_na.reindex(["a", "b", np.nan], axis=1)
+ # hack (NaN handling in assert_index_equal)
+ exp_na.columns = res_na.columns
+ if sparse:
+ if dtype.kind == "b":
+ exp_na = exp_na.apply(SparseArray, fill_value=False)
+ else:
+ exp_na = exp_na.apply(SparseArray, fill_value=0.0)
+ tm.assert_frame_equal(res_na, exp_na)
+
+ res_just_na = get_dummies([np.nan], dummy_na=True, sparse=sparse, dtype=dtype)
+ exp_just_na = DataFrame(
+ Series(1, index=[0]), columns=[np.nan], dtype=self.effective_dtype(dtype)
+ )
+ tm.assert_numpy_array_equal(res_just_na.values, exp_just_na.values)
+
+ def test_get_dummies_unicode(self, sparse):
+ # See GH 6885 - get_dummies chokes on unicode values
+ e = "e"
+ eacute = unicodedata.lookup("LATIN SMALL LETTER E WITH ACUTE")
+ s = [e, eacute, eacute]
+ res = get_dummies(s, prefix="letter", sparse=sparse)
+ exp = DataFrame(
+ {"letter_e": [True, False, False], f"letter_{eacute}": [False, True, True]}
+ )
+ if sparse:
+ exp = exp.apply(SparseArray, fill_value=False)
+ tm.assert_frame_equal(res, exp)
+
+ def test_dataframe_dummies_all_obj(self, df, sparse):
+ df = df[["A", "B"]]
+ result = get_dummies(df, sparse=sparse)
+ expected = DataFrame(
+ {"A_a": [1, 0, 1], "A_b": [0, 1, 0], "B_b": [1, 1, 0], "B_c": [0, 0, 1]},
+ dtype=bool,
+ )
+ if sparse:
+ expected = DataFrame(
+ {
+ "A_a": SparseArray([1, 0, 1], dtype="bool"),
+ "A_b": SparseArray([0, 1, 0], dtype="bool"),
+ "B_b": SparseArray([1, 1, 0], dtype="bool"),
+ "B_c": SparseArray([0, 0, 1], dtype="bool"),
+ }
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_dataframe_dummies_string_dtype(self, df, any_string_dtype):
+ # GH44965
+ df = df[["A", "B"]]
+ df = df.astype({"A": "str", "B": any_string_dtype})
+ result = get_dummies(df)
+ expected = DataFrame(
+ {
+ "A_a": [1, 0, 1],
+ "A_b": [0, 1, 0],
+ "B_b": [1, 1, 0],
+ "B_c": [0, 0, 1],
+ },
+ dtype=bool,
+ )
+ if any_string_dtype == "string" and any_string_dtype.na_value is pd.NA:
+ expected[["B_b", "B_c"]] = expected[["B_b", "B_c"]].astype("boolean")
+ tm.assert_frame_equal(result, expected)
+
+ def test_dataframe_dummies_mix_default(self, df, sparse, dtype):
+ result = get_dummies(df, sparse=sparse, dtype=dtype)
+ if sparse:
+ arr = SparseArray
+ if dtype.kind == "b":
+ typ = SparseDtype(dtype, False)
+ else:
+ typ = SparseDtype(dtype, 0)
+ else:
+ arr = np.array
+ typ = dtype
+ expected = DataFrame(
+ {
+ "C": [1, 2, 3],
+ "A_a": arr([1, 0, 1], dtype=typ),
+ "A_b": arr([0, 1, 0], dtype=typ),
+ "B_b": arr([1, 1, 0], dtype=typ),
+ "B_c": arr([0, 0, 1], dtype=typ),
+ }
+ )
+ expected = expected[["C", "A_a", "A_b", "B_b", "B_c"]]
+ tm.assert_frame_equal(result, expected)
+
+ def test_dataframe_dummies_prefix_list(self, df, sparse):
+ prefixes = ["from_A", "from_B"]
+ result = get_dummies(df, prefix=prefixes, sparse=sparse)
+ expected = DataFrame(
+ {
+ "C": [1, 2, 3],
+ "from_A_a": [True, False, True],
+ "from_A_b": [False, True, False],
+ "from_B_b": [True, True, False],
+ "from_B_c": [False, False, True],
+ },
+ )
+ expected[["C"]] = df[["C"]]
+ cols = ["from_A_a", "from_A_b", "from_B_b", "from_B_c"]
+ expected = expected[["C", *cols]]
+
+ typ = SparseArray if sparse else Series
+ expected[cols] = expected[cols].apply(lambda x: typ(x))
+ tm.assert_frame_equal(result, expected)
+
+ def test_dataframe_dummies_prefix_str(self, df, sparse):
+ # not that you should do this...
+ result = get_dummies(df, prefix="bad", sparse=sparse)
+ bad_columns = ["bad_a", "bad_b", "bad_b", "bad_c"]
+ expected = DataFrame(
+ [
+ [1, True, False, True, False],
+ [2, False, True, True, False],
+ [3, True, False, False, True],
+ ],
+ columns=["C", *bad_columns],
+ )
+ expected = expected.astype({"C": np.int64})
+ if sparse:
+ # work around astyping & assigning with duplicate columns
+ # https://github.com/pandas-dev/pandas/issues/14427
+ expected = pd.concat(
+ [
+ Series([1, 2, 3], name="C"),
+ Series([True, False, True], name="bad_a", dtype="Sparse[bool]"),
+ Series([False, True, False], name="bad_b", dtype="Sparse[bool]"),
+ Series([True, True, False], name="bad_b", dtype="Sparse[bool]"),
+ Series([False, False, True], name="bad_c", dtype="Sparse[bool]"),
+ ],
+ axis=1,
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_dataframe_dummies_subset(self, df, sparse):
+ result = get_dummies(df, prefix=["from_A"], columns=["A"], sparse=sparse)
+ expected = DataFrame(
+ {
+ "B": ["b", "b", "c"],
+ "C": [1, 2, 3],
+ "from_A_a": [1, 0, 1],
+ "from_A_b": [0, 1, 0],
+ },
+ )
+ cols = expected.columns
+ expected[cols[1:]] = expected[cols[1:]].astype(bool)
+ expected[["C"]] = df[["C"]]
+ if sparse:
+ cols = ["from_A_a", "from_A_b"]
+ expected[cols] = expected[cols].astype(SparseDtype("bool", False))
+ tm.assert_frame_equal(result, expected)
+
+ def test_dataframe_dummies_prefix_sep(self, df, sparse):
+ result = get_dummies(df, prefix_sep="..", sparse=sparse)
+ expected = DataFrame(
+ {
+ "C": [1, 2, 3],
+ "A..a": [True, False, True],
+ "A..b": [False, True, False],
+ "B..b": [True, True, False],
+ "B..c": [False, False, True],
+ },
+ )
+ expected[["C"]] = df[["C"]]
+ expected = expected[["C", "A..a", "A..b", "B..b", "B..c"]]
+ if sparse:
+ cols = ["A..a", "A..b", "B..b", "B..c"]
+ expected[cols] = expected[cols].astype(SparseDtype("bool", False))
+
+ tm.assert_frame_equal(result, expected)
+
+ result = get_dummies(df, prefix_sep=["..", "__"], sparse=sparse)
+ expected = expected.rename(columns={"B..b": "B__b", "B..c": "B__c"})
+ tm.assert_frame_equal(result, expected)
+
+ result = get_dummies(df, prefix_sep={"A": "..", "B": "__"}, sparse=sparse)
+ tm.assert_frame_equal(result, expected)
+
+ def test_dataframe_dummies_prefix_bad_length(self, df, sparse):
+ msg = re.escape(
+ "Length of 'prefix' (1) did not match the length of the columns being "
+ "encoded (2)"
+ )
+ with pytest.raises(ValueError, match=msg):
+ get_dummies(df, prefix=["too few"], sparse=sparse)
+
+ def test_dataframe_dummies_prefix_sep_bad_length(self, df, sparse):
+ msg = re.escape(
+ "Length of 'prefix_sep' (1) did not match the length of the columns being "
+ "encoded (2)"
+ )
+ with pytest.raises(ValueError, match=msg):
+ get_dummies(df, prefix_sep=["bad"], sparse=sparse)
+
+ def test_dataframe_dummies_prefix_dict(self, sparse):
+ prefixes = {"A": "from_A", "B": "from_B"}
+ df = DataFrame({"C": [1, 2, 3], "A": ["a", "b", "a"], "B": ["b", "b", "c"]})
+ result = get_dummies(df, prefix=prefixes, sparse=sparse)
+
+ expected = DataFrame(
+ {
+ "C": [1, 2, 3],
+ "from_A_a": [1, 0, 1],
+ "from_A_b": [0, 1, 0],
+ "from_B_b": [1, 1, 0],
+ "from_B_c": [0, 0, 1],
+ }
+ )
+
+ columns = ["from_A_a", "from_A_b", "from_B_b", "from_B_c"]
+ expected[columns] = expected[columns].astype(bool)
+ if sparse:
+ expected[columns] = expected[columns].astype(SparseDtype("bool", False))
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_dataframe_dummies_with_na(self, df, sparse, dtype):
+ df.loc[3, :] = [np.nan, np.nan, np.nan]
+ result = get_dummies(df, dummy_na=True, sparse=sparse, dtype=dtype).sort_index(
+ axis=1
+ )
+
+ if sparse:
+ arr = SparseArray
+ if dtype.kind == "b":
+ typ = SparseDtype(dtype, False)
+ else:
+ typ = SparseDtype(dtype, 0)
+ else:
+ arr = np.array
+ typ = dtype
+
+ expected = DataFrame(
+ {
+ "C": [1, 2, 3, np.nan],
+ "A_a": arr([1, 0, 1, 0], dtype=typ),
+ "A_b": arr([0, 1, 0, 0], dtype=typ),
+ "A_nan": arr([0, 0, 0, 1], dtype=typ),
+ "B_b": arr([1, 1, 0, 0], dtype=typ),
+ "B_c": arr([0, 0, 1, 0], dtype=typ),
+ "B_nan": arr([0, 0, 0, 1], dtype=typ),
+ }
+ ).sort_index(axis=1)
+
+ tm.assert_frame_equal(result, expected)
+
+ result = get_dummies(df, dummy_na=False, sparse=sparse, dtype=dtype)
+ expected = expected[["C", "A_a", "A_b", "B_b", "B_c"]]
+ tm.assert_frame_equal(result, expected)
+
+ def test_dataframe_dummies_with_categorical(self, df, sparse, dtype):
+ df["cat"] = Categorical(["x", "y", "y"])
+ result = get_dummies(df, sparse=sparse, dtype=dtype).sort_index(axis=1)
+ if sparse:
+ arr = SparseArray
+ if dtype.kind == "b":
+ typ = SparseDtype(dtype, False)
+ else:
+ typ = SparseDtype(dtype, 0)
+ else:
+ arr = np.array
+ typ = dtype
+
+ expected = DataFrame(
+ {
+ "C": [1, 2, 3],
+ "A_a": arr([1, 0, 1], dtype=typ),
+ "A_b": arr([0, 1, 0], dtype=typ),
+ "B_b": arr([1, 1, 0], dtype=typ),
+ "B_c": arr([0, 0, 1], dtype=typ),
+ "cat_x": arr([1, 0, 0], dtype=typ),
+ "cat_y": arr([0, 1, 1], dtype=typ),
+ }
+ ).sort_index(axis=1)
+
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "get_dummies_kwargs,expected",
+ [
+ (
+ {"data": DataFrame({"ä": ["a"]})},
+ "ä_a",
+ ),
+ (
+ {"data": DataFrame({"x": ["ä"]})},
+ "x_ä",
+ ),
+ (
+ {"data": DataFrame({"x": ["a"]}), "prefix": "ä"},
+ "ä_a",
+ ),
+ (
+ {"data": DataFrame({"x": ["a"]}), "prefix_sep": "ä"},
+ "xäa",
+ ),
+ ],
+ )
+ def test_dataframe_dummies_unicode(self, get_dummies_kwargs, expected):
+ # GH22084 get_dummies incorrectly encodes unicode characters
+ # in dataframe column names
+ result = get_dummies(**get_dummies_kwargs)
+ expected = DataFrame({expected: [True]})
+ tm.assert_frame_equal(result, expected)
+
+ def test_get_dummies_basic_drop_first(self, sparse):
+ # GH12402 Add a new parameter `drop_first` to avoid collinearity
+ # Basic case
+ s_list = list("abc")
+ s_series = Series(s_list)
+ s_series_index = Series(s_list, list("ABC"))
+
+ expected = DataFrame({"b": [0, 1, 0], "c": [0, 0, 1]}, dtype=bool)
+
+ result = get_dummies(s_list, drop_first=True, sparse=sparse)
+ if sparse:
+ expected = expected.apply(SparseArray, fill_value=False)
+ tm.assert_frame_equal(result, expected)
+
+ result = get_dummies(s_series, drop_first=True, sparse=sparse)
+ tm.assert_frame_equal(result, expected)
+
+ expected.index = list("ABC")
+ result = get_dummies(s_series_index, drop_first=True, sparse=sparse)
+ tm.assert_frame_equal(result, expected)
+
+ def test_get_dummies_basic_drop_first_one_level(self, sparse):
+ # Test the case that categorical variable only has one level.
+ s_list = list("aaa")
+ s_series = Series(s_list)
+ s_series_index = Series(s_list, list("ABC"))
+
+ expected = DataFrame(index=RangeIndex(3))
+
+ result = get_dummies(s_list, drop_first=True, sparse=sparse)
+ tm.assert_frame_equal(result, expected)
+
+ result = get_dummies(s_series, drop_first=True, sparse=sparse)
+ tm.assert_frame_equal(result, expected)
+
+ expected = DataFrame(index=list("ABC"))
+ result = get_dummies(s_series_index, drop_first=True, sparse=sparse)
+ tm.assert_frame_equal(result, expected)
+
+ def test_get_dummies_basic_drop_first_NA(self, sparse):
+ # Test NA handling together with drop_first
+ s_NA = ["a", "b", np.nan]
+ res = get_dummies(s_NA, drop_first=True, sparse=sparse)
+ exp = DataFrame({"b": [0, 1, 0]}, dtype=bool)
+ if sparse:
+ exp = exp.apply(SparseArray, fill_value=False)
+
+ tm.assert_frame_equal(res, exp)
+
+ res_na = get_dummies(s_NA, dummy_na=True, drop_first=True, sparse=sparse)
+ exp_na = DataFrame({"b": [0, 1, 0], np.nan: [0, 0, 1]}, dtype=bool).reindex(
+ ["b", np.nan], axis=1
+ )
+ if sparse:
+ exp_na = exp_na.apply(SparseArray, fill_value=False)
+ tm.assert_frame_equal(res_na, exp_na)
+
+ res_just_na = get_dummies(
+ [np.nan], dummy_na=True, drop_first=True, sparse=sparse
+ )
+ exp_just_na = DataFrame(index=RangeIndex(1))
+ tm.assert_frame_equal(res_just_na, exp_just_na)
+
+ def test_dataframe_dummies_drop_first(self, df, sparse):
+ df = df[["A", "B"]]
+ result = get_dummies(df, drop_first=True, sparse=sparse)
+ expected = DataFrame({"A_b": [0, 1, 0], "B_c": [0, 0, 1]}, dtype=bool)
+ if sparse:
+ expected = expected.apply(SparseArray, fill_value=False)
+ tm.assert_frame_equal(result, expected)
+
+ def test_dataframe_dummies_drop_first_with_categorical(self, df, sparse, dtype):
+ df["cat"] = Categorical(["x", "y", "y"])
+ result = get_dummies(df, drop_first=True, sparse=sparse)
+ expected = DataFrame(
+ {"C": [1, 2, 3], "A_b": [0, 1, 0], "B_c": [0, 0, 1], "cat_y": [0, 1, 1]}
+ )
+ cols = ["A_b", "B_c", "cat_y"]
+ expected[cols] = expected[cols].astype(bool)
+ expected = expected[["C", "A_b", "B_c", "cat_y"]]
+ if sparse:
+ for col in cols:
+ expected[col] = SparseArray(expected[col])
+ tm.assert_frame_equal(result, expected)
+
+ def test_dataframe_dummies_drop_first_with_na(self, df, sparse):
+ df.loc[3, :] = [np.nan, np.nan, np.nan]
+ result = get_dummies(
+ df, dummy_na=True, drop_first=True, sparse=sparse
+ ).sort_index(axis=1)
+ expected = DataFrame(
+ {
+ "C": [1, 2, 3, np.nan],
+ "A_b": [0, 1, 0, 0],
+ "A_nan": [0, 0, 0, 1],
+ "B_c": [0, 0, 1, 0],
+ "B_nan": [0, 0, 0, 1],
+ }
+ )
+ cols = ["A_b", "A_nan", "B_c", "B_nan"]
+ expected[cols] = expected[cols].astype(bool)
+ expected = expected.sort_index(axis=1)
+ if sparse:
+ for col in cols:
+ expected[col] = SparseArray(expected[col])
+
+ tm.assert_frame_equal(result, expected)
+
+ result = get_dummies(df, dummy_na=False, drop_first=True, sparse=sparse)
+ expected = expected[["C", "A_b", "B_c"]]
+ tm.assert_frame_equal(result, expected)
+
+ def test_get_dummies_int_int(self):
+ data = Series([1, 2, 1])
+ result = get_dummies(data)
+ expected = DataFrame([[1, 0], [0, 1], [1, 0]], columns=[1, 2], dtype=bool)
+ tm.assert_frame_equal(result, expected)
+
+ data = Series(Categorical(["a", "b", "a"]))
+ result = get_dummies(data)
+ expected = DataFrame(
+ [[1, 0], [0, 1], [1, 0]], columns=Categorical(["a", "b"]), dtype=bool
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_get_dummies_int_df(self, dtype):
+ data = DataFrame(
+ {
+ "A": [1, 2, 1],
+ "B": Categorical(["a", "b", "a"]),
+ "C": [1, 2, 1],
+ "D": [1.0, 2.0, 1.0],
+ }
+ )
+ columns = ["C", "D", "A_1", "A_2", "B_a", "B_b"]
+ expected = DataFrame(
+ [[1, 1.0, 1, 0, 1, 0], [2, 2.0, 0, 1, 0, 1], [1, 1.0, 1, 0, 1, 0]],
+ columns=columns,
+ )
+ expected[columns[2:]] = expected[columns[2:]].astype(dtype)
+ result = get_dummies(data, columns=["A", "B"], dtype=dtype)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("ordered", [True, False])
+ def test_dataframe_dummies_preserve_categorical_dtype(self, dtype, ordered):
+ # GH13854
+ cat = Categorical(list("xy"), categories=list("xyz"), ordered=ordered)
+ result = get_dummies(cat, dtype=dtype)
+
+ data = np.array([[1, 0, 0], [0, 1, 0]], dtype=self.effective_dtype(dtype))
+ cols = CategoricalIndex(
+ cat.categories, categories=cat.categories, ordered=ordered
+ )
+ expected = DataFrame(data, columns=cols, dtype=self.effective_dtype(dtype))
+
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("sparse", [True, False])
+ def test_get_dummies_dont_sparsify_all_columns(self, sparse):
+ # GH18914
+ df = DataFrame.from_dict({"GDP": [1, 2], "Nation": ["AB", "CD"]})
+ df = get_dummies(df, columns=["Nation"], sparse=sparse)
+ df2 = df.reindex(columns=["GDP"])
+
+ tm.assert_frame_equal(df[["GDP"]], df2)
+
+ def test_get_dummies_duplicate_columns(self, df):
+ # GH20839
+ df.columns = ["A", "A", "A"]
+ result = get_dummies(df).sort_index(axis=1)
+
+ expected = DataFrame(
+ [
+ [1, True, False, True, False],
+ [2, False, True, True, False],
+ [3, True, False, False, True],
+ ],
+ columns=["A", "A_a", "A_b", "A_b", "A_c"],
+ ).sort_index(axis=1)
+
+ expected = expected.astype({"A": np.int64})
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_get_dummies_all_sparse(self):
+ df = DataFrame({"A": [1, 2]})
+ result = get_dummies(df, columns=["A"], sparse=True)
+ dtype = SparseDtype("bool", False)
+ expected = DataFrame(
+ {
+ "A_1": SparseArray([1, 0], dtype=dtype),
+ "A_2": SparseArray([0, 1], dtype=dtype),
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("values", ["baz"])
+ def test_get_dummies_with_string_values(self, values):
+ # issue #28383
+ df = DataFrame(
+ {
+ "bar": [1, 2, 3, 4, 5, 6],
+ "foo": ["one", "one", "one", "two", "two", "two"],
+ "baz": ["A", "B", "C", "A", "B", "C"],
+ "zoo": ["x", "y", "z", "q", "w", "t"],
+ }
+ )
+
+ msg = "Input must be a list-like for parameter `columns`"
+
+ with pytest.raises(TypeError, match=msg):
+ get_dummies(df, columns=values)
+
+ def test_get_dummies_ea_dtype_series(self, any_numeric_ea_and_arrow_dtype):
+ # GH#32430
+ ser = Series(list("abca"))
+ result = get_dummies(ser, dtype=any_numeric_ea_and_arrow_dtype)
+ expected = DataFrame(
+ {"a": [1, 0, 0, 1], "b": [0, 1, 0, 0], "c": [0, 0, 1, 0]},
+ dtype=any_numeric_ea_and_arrow_dtype,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_get_dummies_ea_dtype_dataframe(self, any_numeric_ea_and_arrow_dtype):
+ # GH#32430
+ df = DataFrame({"x": list("abca")})
+ result = get_dummies(df, dtype=any_numeric_ea_and_arrow_dtype)
+ expected = DataFrame(
+ {"x_a": [1, 0, 0, 1], "x_b": [0, 1, 0, 0], "x_c": [0, 0, 1, 0]},
+ dtype=any_numeric_ea_and_arrow_dtype,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("dtype_type", ["string", "category"])
+ def test_get_dummies_ea_dtype(self, dtype_type, string_dtype_no_object):
+ # GH#56273
+ dtype = string_dtype_no_object
+ exp_dtype = "boolean" if dtype.na_value is pd.NA else "bool"
+ if dtype_type == "category":
+ dtype = CategoricalDtype(Index(["a"], dtype))
+ df = DataFrame({"name": Series(["a"], dtype=dtype), "x": 1})
+ result = get_dummies(df)
+ expected = DataFrame({"x": 1, "name_a": Series([True], dtype=exp_dtype)})
+ tm.assert_frame_equal(result, expected)
+
+ @td.skip_if_no("pyarrow")
+ def test_get_dummies_arrow_dtype(self):
+ # GH#56273
+ df = DataFrame({"name": Series(["a"], dtype=ArrowDtype(pa.string())), "x": 1})
+ result = get_dummies(df)
+ expected = DataFrame({"x": 1, "name_a": Series([True], dtype="bool[pyarrow]")})
+ tm.assert_frame_equal(result, expected)
+
+ df = DataFrame(
+ {
+ "name": Series(
+ ["a"],
+ dtype=CategoricalDtype(Index(["a"], dtype=ArrowDtype(pa.string()))),
+ ),
+ "x": 1,
+ }
+ )
+ result = get_dummies(df)
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_melt.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_melt.py
new file mode 100644
index 0000000000000000000000000000000000000000..fba9c28282e9491ce5f237e4d92068a4320953ca
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_melt.py
@@ -0,0 +1,1280 @@
+import re
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ DataFrame,
+ Index,
+ date_range,
+ lreshape,
+ melt,
+ wide_to_long,
+)
+import pandas._testing as tm
+
+
+@pytest.fixture
+def df():
+ res = DataFrame(
+ np.random.default_rng(2).standard_normal((10, 4)),
+ columns=Index(list("ABCD")),
+ index=date_range("2000-01-01", periods=10, freq="B"),
+ )
+ res["id1"] = (res["A"] > 0).astype(np.int64)
+ res["id2"] = (res["B"] > 0).astype(np.int64)
+ return res
+
+
+@pytest.fixture
+def df1():
+ res = DataFrame(
+ [
+ [1.067683, -1.110463, 0.20867],
+ [-1.321405, 0.368915, -1.055342],
+ [-0.807333, 0.08298, -0.873361],
+ ]
+ )
+ res.columns = [list("ABC"), list("abc")]
+ res.columns.names = ["CAP", "low"]
+ return res
+
+
+@pytest.fixture
+def var_name():
+ return "var"
+
+
+@pytest.fixture
+def value_name():
+ return "val"
+
+
+class TestMelt:
+ def test_top_level_method(self, df):
+ result = melt(df)
+ assert result.columns.tolist() == ["variable", "value"]
+
+ def test_method_signatures(self, df, df1, var_name, value_name):
+ tm.assert_frame_equal(df.melt(), melt(df))
+
+ tm.assert_frame_equal(
+ df.melt(id_vars=["id1", "id2"], value_vars=["A", "B"]),
+ melt(df, id_vars=["id1", "id2"], value_vars=["A", "B"]),
+ )
+
+ tm.assert_frame_equal(
+ df.melt(var_name=var_name, value_name=value_name),
+ melt(df, var_name=var_name, value_name=value_name),
+ )
+
+ tm.assert_frame_equal(df1.melt(col_level=0), melt(df1, col_level=0))
+
+ def test_default_col_names(self, df):
+ result = df.melt()
+ assert result.columns.tolist() == ["variable", "value"]
+
+ result1 = df.melt(id_vars=["id1"])
+ assert result1.columns.tolist() == ["id1", "variable", "value"]
+
+ result2 = df.melt(id_vars=["id1", "id2"])
+ assert result2.columns.tolist() == ["id1", "id2", "variable", "value"]
+
+ def test_value_vars(self, df):
+ result3 = df.melt(id_vars=["id1", "id2"], value_vars="A")
+ assert len(result3) == 10
+
+ result4 = df.melt(id_vars=["id1", "id2"], value_vars=["A", "B"])
+ expected4 = DataFrame(
+ {
+ "id1": df["id1"].tolist() * 2,
+ "id2": df["id2"].tolist() * 2,
+ "variable": ["A"] * 10 + ["B"] * 10,
+ "value": (df["A"].tolist() + df["B"].tolist()),
+ },
+ columns=["id1", "id2", "variable", "value"],
+ )
+ tm.assert_frame_equal(result4, expected4)
+
+ @pytest.mark.parametrize("type_", (tuple, list, np.array))
+ def test_value_vars_types(self, type_, df):
+ # GH 15348
+ expected = DataFrame(
+ {
+ "id1": df["id1"].tolist() * 2,
+ "id2": df["id2"].tolist() * 2,
+ "variable": ["A"] * 10 + ["B"] * 10,
+ "value": (df["A"].tolist() + df["B"].tolist()),
+ },
+ columns=["id1", "id2", "variable", "value"],
+ )
+ result = df.melt(id_vars=["id1", "id2"], value_vars=type_(("A", "B")))
+ tm.assert_frame_equal(result, expected)
+
+ def test_vars_work_with_multiindex(self, df1):
+ expected = DataFrame(
+ {
+ ("A", "a"): df1[("A", "a")],
+ "CAP": ["B"] * len(df1),
+ "low": ["b"] * len(df1),
+ "value": df1[("B", "b")],
+ },
+ columns=[("A", "a"), "CAP", "low", "value"],
+ )
+
+ result = df1.melt(id_vars=[("A", "a")], value_vars=[("B", "b")])
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "id_vars, value_vars, col_level, expected",
+ [
+ (
+ ["A"],
+ ["B"],
+ 0,
+ {
+ "A": {0: 1.067683, 1: -1.321405, 2: -0.807333},
+ "CAP": {0: "B", 1: "B", 2: "B"},
+ "value": {0: -1.110463, 1: 0.368915, 2: 0.08298},
+ },
+ ),
+ (
+ ["a"],
+ ["b"],
+ 1,
+ {
+ "a": {0: 1.067683, 1: -1.321405, 2: -0.807333},
+ "low": {0: "b", 1: "b", 2: "b"},
+ "value": {0: -1.110463, 1: 0.368915, 2: 0.08298},
+ },
+ ),
+ ],
+ )
+ def test_single_vars_work_with_multiindex(
+ self, id_vars, value_vars, col_level, expected, df1
+ ):
+ result = df1.melt(id_vars, value_vars, col_level=col_level)
+ expected = DataFrame(expected)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "id_vars, value_vars",
+ [
+ [("A", "a"), [("B", "b")]],
+ [[("A", "a")], ("B", "b")],
+ [("A", "a"), ("B", "b")],
+ ],
+ )
+ def test_tuple_vars_fail_with_multiindex(self, id_vars, value_vars, df1):
+ # melt should fail with an informative error message if
+ # the columns have a MultiIndex and a tuple is passed
+ # for id_vars or value_vars.
+ msg = r"(id|value)_vars must be a list of tuples when columns are a MultiIndex"
+ with pytest.raises(ValueError, match=msg):
+ df1.melt(id_vars=id_vars, value_vars=value_vars)
+
+ def test_custom_var_name(self, df, var_name):
+ result5 = df.melt(var_name=var_name)
+ assert result5.columns.tolist() == ["var", "value"]
+
+ result6 = df.melt(id_vars=["id1"], var_name=var_name)
+ assert result6.columns.tolist() == ["id1", "var", "value"]
+
+ result7 = df.melt(id_vars=["id1", "id2"], var_name=var_name)
+ assert result7.columns.tolist() == ["id1", "id2", "var", "value"]
+
+ result8 = df.melt(id_vars=["id1", "id2"], value_vars="A", var_name=var_name)
+ assert result8.columns.tolist() == ["id1", "id2", "var", "value"]
+
+ result9 = df.melt(
+ id_vars=["id1", "id2"], value_vars=["A", "B"], var_name=var_name
+ )
+ expected9 = DataFrame(
+ {
+ "id1": df["id1"].tolist() * 2,
+ "id2": df["id2"].tolist() * 2,
+ var_name: ["A"] * 10 + ["B"] * 10,
+ "value": (df["A"].tolist() + df["B"].tolist()),
+ },
+ columns=["id1", "id2", var_name, "value"],
+ )
+ tm.assert_frame_equal(result9, expected9)
+
+ def test_custom_value_name(self, df, value_name):
+ result10 = df.melt(value_name=value_name)
+ assert result10.columns.tolist() == ["variable", "val"]
+
+ result11 = df.melt(id_vars=["id1"], value_name=value_name)
+ assert result11.columns.tolist() == ["id1", "variable", "val"]
+
+ result12 = df.melt(id_vars=["id1", "id2"], value_name=value_name)
+ assert result12.columns.tolist() == ["id1", "id2", "variable", "val"]
+
+ result13 = df.melt(
+ id_vars=["id1", "id2"], value_vars="A", value_name=value_name
+ )
+ assert result13.columns.tolist() == ["id1", "id2", "variable", "val"]
+
+ result14 = df.melt(
+ id_vars=["id1", "id2"], value_vars=["A", "B"], value_name=value_name
+ )
+ expected14 = DataFrame(
+ {
+ "id1": df["id1"].tolist() * 2,
+ "id2": df["id2"].tolist() * 2,
+ "variable": ["A"] * 10 + ["B"] * 10,
+ value_name: (df["A"].tolist() + df["B"].tolist()),
+ },
+ columns=["id1", "id2", "variable", value_name],
+ )
+ tm.assert_frame_equal(result14, expected14)
+
+ def test_custom_var_and_value_name(self, df, value_name, var_name):
+ result15 = df.melt(var_name=var_name, value_name=value_name)
+ assert result15.columns.tolist() == ["var", "val"]
+
+ result16 = df.melt(id_vars=["id1"], var_name=var_name, value_name=value_name)
+ assert result16.columns.tolist() == ["id1", "var", "val"]
+
+ result17 = df.melt(
+ id_vars=["id1", "id2"], var_name=var_name, value_name=value_name
+ )
+ assert result17.columns.tolist() == ["id1", "id2", "var", "val"]
+
+ result18 = df.melt(
+ id_vars=["id1", "id2"],
+ value_vars="A",
+ var_name=var_name,
+ value_name=value_name,
+ )
+ assert result18.columns.tolist() == ["id1", "id2", "var", "val"]
+
+ result19 = df.melt(
+ id_vars=["id1", "id2"],
+ value_vars=["A", "B"],
+ var_name=var_name,
+ value_name=value_name,
+ )
+ expected19 = DataFrame(
+ {
+ "id1": df["id1"].tolist() * 2,
+ "id2": df["id2"].tolist() * 2,
+ var_name: ["A"] * 10 + ["B"] * 10,
+ value_name: (df["A"].tolist() + df["B"].tolist()),
+ },
+ columns=["id1", "id2", var_name, value_name],
+ )
+ tm.assert_frame_equal(result19, expected19)
+
+ df20 = df.copy()
+ df20.columns.name = "foo"
+ result20 = df20.melt()
+ assert result20.columns.tolist() == ["foo", "value"]
+
+ @pytest.mark.parametrize("col_level", [0, "CAP"])
+ def test_col_level(self, col_level, df1):
+ res = df1.melt(col_level=col_level)
+ assert res.columns.tolist() == ["CAP", "value"]
+
+ def test_multiindex(self, df1):
+ res = df1.melt()
+ assert res.columns.tolist() == ["CAP", "low", "value"]
+
+ @pytest.mark.parametrize(
+ "col",
+ [
+ date_range("2010", periods=5, tz="US/Pacific"),
+ pd.Categorical(["a", "b", "c", "a", "d"]),
+ [0, 1, 0, 0, 0],
+ ],
+ )
+ def test_pandas_dtypes(self, col):
+ # GH 15785
+ col = pd.Series(col)
+ df = DataFrame(
+ {"klass": range(5), "col": col, "attr1": [1, 0, 0, 0, 0], "attr2": col}
+ )
+ expected_value = pd.concat([pd.Series([1, 0, 0, 0, 0]), col], ignore_index=True)
+ result = melt(
+ df, id_vars=["klass", "col"], var_name="attribute", value_name="value"
+ )
+ expected = DataFrame(
+ {
+ 0: list(range(5)) * 2,
+ 1: pd.concat([col] * 2, ignore_index=True),
+ 2: ["attr1"] * 5 + ["attr2"] * 5,
+ 3: expected_value,
+ }
+ )
+ expected.columns = ["klass", "col", "attribute", "value"]
+ tm.assert_frame_equal(result, expected)
+
+ def test_preserve_category(self):
+ # GH 15853
+ data = DataFrame({"A": [1, 2], "B": pd.Categorical(["X", "Y"])})
+ result = melt(data, ["B"], ["A"])
+ expected = DataFrame(
+ {"B": pd.Categorical(["X", "Y"]), "variable": ["A", "A"], "value": [1, 2]}
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_melt_missing_columns_raises(self):
+ # GH-23575
+ # This test is to ensure that pandas raises an error if melting is
+ # attempted with column names absent from the dataframe
+
+ # Generate data
+ df = DataFrame(
+ np.random.default_rng(2).standard_normal((5, 4)), columns=list("abcd")
+ )
+
+ # Try to melt with missing `value_vars` column name
+ msg = "The following id_vars or value_vars are not present in the DataFrame:"
+ with pytest.raises(KeyError, match=msg):
+ df.melt(["a", "b"], ["C", "d"])
+
+ # Try to melt with missing `id_vars` column name
+ with pytest.raises(KeyError, match=msg):
+ df.melt(["A", "b"], ["c", "d"])
+
+ # Multiple missing
+ with pytest.raises(
+ KeyError,
+ match=msg,
+ ):
+ df.melt(["a", "b", "not_here", "or_there"], ["c", "d"])
+
+ # Multiindex melt fails if column is missing from multilevel melt
+ df.columns = [list("ABCD"), list("abcd")]
+ with pytest.raises(KeyError, match=msg):
+ df.melt([("E", "a")], [("B", "b")])
+ # Multiindex fails if column is missing from single level melt
+ with pytest.raises(KeyError, match=msg):
+ df.melt(["A"], ["F"], col_level=0)
+
+ def test_melt_mixed_int_str_id_vars(self):
+ # GH 29718
+ df = DataFrame({0: ["foo"], "a": ["bar"], "b": [1], "d": [2]})
+ result = melt(df, id_vars=[0, "a"], value_vars=["b", "d"])
+ expected = DataFrame(
+ {0: ["foo"] * 2, "a": ["bar"] * 2, "variable": list("bd"), "value": [1, 2]}
+ )
+ # the df's columns are mixed type and thus object -> preserves object dtype
+ expected["variable"] = expected["variable"].astype(object)
+ tm.assert_frame_equal(result, expected)
+
+ def test_melt_mixed_int_str_value_vars(self):
+ # GH 29718
+ df = DataFrame({0: ["foo"], "a": ["bar"]})
+ result = melt(df, value_vars=[0, "a"])
+ expected = DataFrame({"variable": [0, "a"], "value": ["foo", "bar"]})
+ tm.assert_frame_equal(result, expected)
+
+ def test_ignore_index(self):
+ # GH 17440
+ df = DataFrame({"foo": [0], "bar": [1]}, index=["first"])
+ result = melt(df, ignore_index=False)
+ expected = DataFrame(
+ {"variable": ["foo", "bar"], "value": [0, 1]}, index=["first", "first"]
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_ignore_multiindex(self):
+ # GH 17440
+ index = pd.MultiIndex.from_tuples(
+ [("first", "second"), ("first", "third")], names=["baz", "foobar"]
+ )
+ df = DataFrame({"foo": [0, 1], "bar": [2, 3]}, index=index)
+ result = melt(df, ignore_index=False)
+
+ expected_index = pd.MultiIndex.from_tuples(
+ [("first", "second"), ("first", "third")] * 2, names=["baz", "foobar"]
+ )
+ expected = DataFrame(
+ {"variable": ["foo"] * 2 + ["bar"] * 2, "value": [0, 1, 2, 3]},
+ index=expected_index,
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_ignore_index_name_and_type(self):
+ # GH 17440
+ index = Index(["foo", "bar"], dtype="category", name="baz")
+ df = DataFrame({"x": [0, 1], "y": [2, 3]}, index=index)
+ result = melt(df, ignore_index=False)
+
+ expected_index = Index(["foo", "bar"] * 2, dtype="category", name="baz")
+ expected = DataFrame(
+ {"variable": ["x", "x", "y", "y"], "value": [0, 1, 2, 3]},
+ index=expected_index,
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_melt_with_duplicate_columns(self):
+ # GH#41951
+ df = DataFrame([["id", 2, 3]], columns=["a", "b", "b"])
+ result = df.melt(id_vars=["a"], value_vars=["b"])
+ expected = DataFrame(
+ [["id", "b", 2], ["id", "b", 3]], columns=["a", "variable", "value"]
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("dtype", ["Int8", "Int64"])
+ def test_melt_ea_dtype(self, dtype):
+ # GH#41570
+ df = DataFrame(
+ {
+ "a": pd.Series([1, 2], dtype="Int8"),
+ "b": pd.Series([3, 4], dtype=dtype),
+ }
+ )
+ result = df.melt()
+ expected = DataFrame(
+ {
+ "variable": ["a", "a", "b", "b"],
+ "value": pd.Series([1, 2, 3, 4], dtype=dtype),
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_melt_ea_columns(self):
+ # GH 54297
+ df = DataFrame(
+ {
+ "A": {0: "a", 1: "b", 2: "c"},
+ "B": {0: 1, 1: 3, 2: 5},
+ "C": {0: 2, 1: 4, 2: 6},
+ }
+ )
+ df.columns = df.columns.astype("string[python]")
+ result = df.melt(id_vars=["A"], value_vars=["B"])
+ expected = DataFrame(
+ {
+ "A": list("abc"),
+ "variable": pd.Series(["B"] * 3, dtype="string[python]"),
+ "value": [1, 3, 5],
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_melt_preserves_datetime(self):
+ df = DataFrame(
+ data=[
+ {
+ "type": "A0",
+ "start_date": pd.Timestamp("2023/03/01", tz="Asia/Tokyo"),
+ "end_date": pd.Timestamp("2023/03/10", tz="Asia/Tokyo"),
+ },
+ {
+ "type": "A1",
+ "start_date": pd.Timestamp("2023/03/01", tz="Asia/Tokyo"),
+ "end_date": pd.Timestamp("2023/03/11", tz="Asia/Tokyo"),
+ },
+ ],
+ index=["aaaa", "bbbb"],
+ )
+ result = df.melt(
+ id_vars=["type"],
+ value_vars=["start_date", "end_date"],
+ var_name="start/end",
+ value_name="date",
+ )
+ expected = DataFrame(
+ {
+ "type": {0: "A0", 1: "A1", 2: "A0", 3: "A1"},
+ "start/end": {
+ 0: "start_date",
+ 1: "start_date",
+ 2: "end_date",
+ 3: "end_date",
+ },
+ "date": {
+ 0: pd.Timestamp("2023-03-01 00:00:00+0900", tz="Asia/Tokyo"),
+ 1: pd.Timestamp("2023-03-01 00:00:00+0900", tz="Asia/Tokyo"),
+ 2: pd.Timestamp("2023-03-10 00:00:00+0900", tz="Asia/Tokyo"),
+ 3: pd.Timestamp("2023-03-11 00:00:00+0900", tz="Asia/Tokyo"),
+ },
+ }
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_melt_allows_non_scalar_id_vars(self):
+ df = DataFrame(
+ data={"a": [1, 2, 3], "b": [4, 5, 6]},
+ index=["11", "22", "33"],
+ )
+ result = df.melt(
+ id_vars="a",
+ var_name=0,
+ value_name=1,
+ )
+ expected = DataFrame({"a": [1, 2, 3], 0: ["b"] * 3, 1: [4, 5, 6]})
+ tm.assert_frame_equal(result, expected)
+
+ def test_melt_allows_non_string_var_name(self):
+ df = DataFrame(
+ data={"a": [1, 2, 3], "b": [4, 5, 6]},
+ index=["11", "22", "33"],
+ )
+ result = df.melt(
+ id_vars=["a"],
+ var_name=0,
+ value_name=1,
+ )
+ expected = DataFrame({"a": [1, 2, 3], 0: ["b"] * 3, 1: [4, 5, 6]})
+ tm.assert_frame_equal(result, expected)
+
+ def test_melt_non_scalar_var_name_raises(self):
+ df = DataFrame(
+ data={"a": [1, 2, 3], "b": [4, 5, 6]},
+ index=["11", "22", "33"],
+ )
+ with pytest.raises(ValueError, match=r".* must be a scalar."):
+ df.melt(id_vars=["a"], var_name=[1, 2])
+
+ def test_melt_multiindex_columns_var_name(self):
+ # GH 58033
+ df = DataFrame({("A", "a"): [1], ("A", "b"): [2]})
+
+ expected = DataFrame(
+ [("A", "a", 1), ("A", "b", 2)], columns=["first", "second", "value"]
+ )
+
+ tm.assert_frame_equal(df.melt(var_name=["first", "second"]), expected)
+ tm.assert_frame_equal(df.melt(var_name=["first"]), expected[["first", "value"]])
+
+ def test_melt_multiindex_columns_var_name_too_many(self):
+ # GH 58033
+ df = DataFrame({("A", "a"): [1], ("A", "b"): [2]})
+
+ with pytest.raises(
+ ValueError, match="but the dataframe columns only have 2 levels"
+ ):
+ df.melt(var_name=["first", "second", "third"])
+
+ def test_melt_duplicate_column_header_raises(self):
+ # GH61475
+ df = DataFrame([[1, 2, 3], [3, 4, 5]], columns=["A", "A", "B"])
+ msg = "id_vars cannot contain duplicate columns."
+
+ with pytest.raises(ValueError, match=msg):
+ df.melt(id_vars=["A"], value_vars=["B"])
+
+
+class TestLreshape:
+ def test_pairs(self):
+ data = {
+ "birthdt": [
+ "08jan2009",
+ "20dec2008",
+ "30dec2008",
+ "21dec2008",
+ "11jan2009",
+ ],
+ "birthwt": [1766, 3301, 1454, 3139, 4133],
+ "id": [101, 102, 103, 104, 105],
+ "sex": ["Male", "Female", "Female", "Female", "Female"],
+ "visitdt1": [
+ "11jan2009",
+ "22dec2008",
+ "04jan2009",
+ "29dec2008",
+ "20jan2009",
+ ],
+ "visitdt2": ["21jan2009", np.nan, "22jan2009", "31dec2008", "03feb2009"],
+ "visitdt3": ["05feb2009", np.nan, np.nan, "02jan2009", "15feb2009"],
+ "wt1": [1823, 3338, 1549, 3298, 4306],
+ "wt2": [2011.0, np.nan, 1892.0, 3338.0, 4575.0],
+ "wt3": [2293.0, np.nan, np.nan, 3377.0, 4805.0],
+ }
+
+ df = DataFrame(data)
+
+ spec = {
+ "visitdt": [f"visitdt{i:d}" for i in range(1, 4)],
+ "wt": [f"wt{i:d}" for i in range(1, 4)],
+ }
+ result = lreshape(df, spec)
+
+ exp_data = {
+ "birthdt": [
+ "08jan2009",
+ "20dec2008",
+ "30dec2008",
+ "21dec2008",
+ "11jan2009",
+ "08jan2009",
+ "30dec2008",
+ "21dec2008",
+ "11jan2009",
+ "08jan2009",
+ "21dec2008",
+ "11jan2009",
+ ],
+ "birthwt": [
+ 1766,
+ 3301,
+ 1454,
+ 3139,
+ 4133,
+ 1766,
+ 1454,
+ 3139,
+ 4133,
+ 1766,
+ 3139,
+ 4133,
+ ],
+ "id": [101, 102, 103, 104, 105, 101, 103, 104, 105, 101, 104, 105],
+ "sex": [
+ "Male",
+ "Female",
+ "Female",
+ "Female",
+ "Female",
+ "Male",
+ "Female",
+ "Female",
+ "Female",
+ "Male",
+ "Female",
+ "Female",
+ ],
+ "visitdt": [
+ "11jan2009",
+ "22dec2008",
+ "04jan2009",
+ "29dec2008",
+ "20jan2009",
+ "21jan2009",
+ "22jan2009",
+ "31dec2008",
+ "03feb2009",
+ "05feb2009",
+ "02jan2009",
+ "15feb2009",
+ ],
+ "wt": [
+ 1823.0,
+ 3338.0,
+ 1549.0,
+ 3298.0,
+ 4306.0,
+ 2011.0,
+ 1892.0,
+ 3338.0,
+ 4575.0,
+ 2293.0,
+ 3377.0,
+ 4805.0,
+ ],
+ }
+ exp = DataFrame(exp_data, columns=result.columns)
+ tm.assert_frame_equal(result, exp)
+
+ result = lreshape(df, spec, dropna=False)
+ exp_data = {
+ "birthdt": [
+ "08jan2009",
+ "20dec2008",
+ "30dec2008",
+ "21dec2008",
+ "11jan2009",
+ "08jan2009",
+ "20dec2008",
+ "30dec2008",
+ "21dec2008",
+ "11jan2009",
+ "08jan2009",
+ "20dec2008",
+ "30dec2008",
+ "21dec2008",
+ "11jan2009",
+ ],
+ "birthwt": [
+ 1766,
+ 3301,
+ 1454,
+ 3139,
+ 4133,
+ 1766,
+ 3301,
+ 1454,
+ 3139,
+ 4133,
+ 1766,
+ 3301,
+ 1454,
+ 3139,
+ 4133,
+ ],
+ "id": [
+ 101,
+ 102,
+ 103,
+ 104,
+ 105,
+ 101,
+ 102,
+ 103,
+ 104,
+ 105,
+ 101,
+ 102,
+ 103,
+ 104,
+ 105,
+ ],
+ "sex": [
+ "Male",
+ "Female",
+ "Female",
+ "Female",
+ "Female",
+ "Male",
+ "Female",
+ "Female",
+ "Female",
+ "Female",
+ "Male",
+ "Female",
+ "Female",
+ "Female",
+ "Female",
+ ],
+ "visitdt": [
+ "11jan2009",
+ "22dec2008",
+ "04jan2009",
+ "29dec2008",
+ "20jan2009",
+ "21jan2009",
+ np.nan,
+ "22jan2009",
+ "31dec2008",
+ "03feb2009",
+ "05feb2009",
+ np.nan,
+ np.nan,
+ "02jan2009",
+ "15feb2009",
+ ],
+ "wt": [
+ 1823.0,
+ 3338.0,
+ 1549.0,
+ 3298.0,
+ 4306.0,
+ 2011.0,
+ np.nan,
+ 1892.0,
+ 3338.0,
+ 4575.0,
+ 2293.0,
+ np.nan,
+ np.nan,
+ 3377.0,
+ 4805.0,
+ ],
+ }
+ exp = DataFrame(exp_data, columns=result.columns)
+ tm.assert_frame_equal(result, exp)
+
+ spec = {
+ "visitdt": [f"visitdt{i:d}" for i in range(1, 3)],
+ "wt": [f"wt{i:d}" for i in range(1, 4)],
+ }
+ msg = "All column lists must be same length"
+ with pytest.raises(ValueError, match=msg):
+ lreshape(df, spec)
+
+
+class TestWideToLong:
+ def test_simple(self):
+ x = np.random.default_rng(2).standard_normal(3)
+ df = DataFrame(
+ {
+ "A1970": {0: "a", 1: "b", 2: "c"},
+ "A1980": {0: "d", 1: "e", 2: "f"},
+ "B1970": {0: 2.5, 1: 1.2, 2: 0.7},
+ "B1980": {0: 3.2, 1: 1.3, 2: 0.1},
+ "X": dict(zip(range(3), x)),
+ }
+ )
+ df["id"] = df.index
+ exp_data = {
+ "X": x.tolist() + x.tolist(),
+ "A": ["a", "b", "c", "d", "e", "f"],
+ "B": [2.5, 1.2, 0.7, 3.2, 1.3, 0.1],
+ "year": [1970, 1970, 1970, 1980, 1980, 1980],
+ "id": [0, 1, 2, 0, 1, 2],
+ }
+ expected = DataFrame(exp_data)
+ expected = expected.set_index(["id", "year"])[["X", "A", "B"]]
+ result = wide_to_long(df, ["A", "B"], i="id", j="year")
+ tm.assert_frame_equal(result, expected)
+
+ def test_stubs(self):
+ # GH9204 wide_to_long call should not modify 'stubs' list
+ df = DataFrame([[0, 1, 2, 3, 8], [4, 5, 6, 7, 9]])
+ df.columns = ["id", "inc1", "inc2", "edu1", "edu2"]
+ stubs = ["inc", "edu"]
+
+ wide_to_long(df, stubs, i="id", j="age")
+
+ assert stubs == ["inc", "edu"]
+
+ def test_separating_character(self):
+ # GH14779
+
+ x = np.random.default_rng(2).standard_normal(3)
+ df = DataFrame(
+ {
+ "A.1970": {0: "a", 1: "b", 2: "c"},
+ "A.1980": {0: "d", 1: "e", 2: "f"},
+ "B.1970": {0: 2.5, 1: 1.2, 2: 0.7},
+ "B.1980": {0: 3.2, 1: 1.3, 2: 0.1},
+ "X": dict(zip(range(3), x)),
+ }
+ )
+ df["id"] = df.index
+ exp_data = {
+ "X": x.tolist() + x.tolist(),
+ "A": ["a", "b", "c", "d", "e", "f"],
+ "B": [2.5, 1.2, 0.7, 3.2, 1.3, 0.1],
+ "year": [1970, 1970, 1970, 1980, 1980, 1980],
+ "id": [0, 1, 2, 0, 1, 2],
+ }
+ expected = DataFrame(exp_data)
+ expected = expected.set_index(["id", "year"])[["X", "A", "B"]]
+ result = wide_to_long(df, ["A", "B"], i="id", j="year", sep=".")
+ tm.assert_frame_equal(result, expected)
+
+ def test_escapable_characters(self):
+ x = np.random.default_rng(2).standard_normal(3)
+ df = DataFrame(
+ {
+ "A(quarterly)1970": {0: "a", 1: "b", 2: "c"},
+ "A(quarterly)1980": {0: "d", 1: "e", 2: "f"},
+ "B(quarterly)1970": {0: 2.5, 1: 1.2, 2: 0.7},
+ "B(quarterly)1980": {0: 3.2, 1: 1.3, 2: 0.1},
+ "X": dict(zip(range(3), x)),
+ }
+ )
+ df["id"] = df.index
+ exp_data = {
+ "X": x.tolist() + x.tolist(),
+ "A(quarterly)": ["a", "b", "c", "d", "e", "f"],
+ "B(quarterly)": [2.5, 1.2, 0.7, 3.2, 1.3, 0.1],
+ "year": [1970, 1970, 1970, 1980, 1980, 1980],
+ "id": [0, 1, 2, 0, 1, 2],
+ }
+ expected = DataFrame(exp_data)
+ expected = expected.set_index(["id", "year"])[
+ ["X", "A(quarterly)", "B(quarterly)"]
+ ]
+ result = wide_to_long(df, ["A(quarterly)", "B(quarterly)"], i="id", j="year")
+ tm.assert_frame_equal(result, expected)
+
+ def test_unbalanced(self):
+ # test that we can have a varying amount of time variables
+ df = DataFrame(
+ {
+ "A2010": [1.0, 2.0],
+ "A2011": [3.0, 4.0],
+ "B2010": [5.0, 6.0],
+ "X": ["X1", "X2"],
+ }
+ )
+ df["id"] = df.index
+ exp_data = {
+ "X": ["X1", "X2", "X1", "X2"],
+ "A": [1.0, 2.0, 3.0, 4.0],
+ "B": [5.0, 6.0, np.nan, np.nan],
+ "id": [0, 1, 0, 1],
+ "year": [2010, 2010, 2011, 2011],
+ }
+ expected = DataFrame(exp_data)
+ expected = expected.set_index(["id", "year"])[["X", "A", "B"]]
+ result = wide_to_long(df, ["A", "B"], i="id", j="year")
+ tm.assert_frame_equal(result, expected)
+
+ def test_character_overlap(self):
+ # Test we handle overlapping characters in both id_vars and value_vars
+ df = DataFrame(
+ {
+ "A11": ["a11", "a22", "a33"],
+ "A12": ["a21", "a22", "a23"],
+ "B11": ["b11", "b12", "b13"],
+ "B12": ["b21", "b22", "b23"],
+ "BB11": [1, 2, 3],
+ "BB12": [4, 5, 6],
+ "BBBX": [91, 92, 93],
+ "BBBZ": [91, 92, 93],
+ }
+ )
+ df["id"] = df.index
+ expected = DataFrame(
+ {
+ "BBBX": [91, 92, 93, 91, 92, 93],
+ "BBBZ": [91, 92, 93, 91, 92, 93],
+ "A": ["a11", "a22", "a33", "a21", "a22", "a23"],
+ "B": ["b11", "b12", "b13", "b21", "b22", "b23"],
+ "BB": [1, 2, 3, 4, 5, 6],
+ "id": [0, 1, 2, 0, 1, 2],
+ "year": [11, 11, 11, 12, 12, 12],
+ }
+ )
+ expected = expected.set_index(["id", "year"])[["BBBX", "BBBZ", "A", "B", "BB"]]
+ result = wide_to_long(df, ["A", "B", "BB"], i="id", j="year")
+ tm.assert_frame_equal(result.sort_index(axis=1), expected.sort_index(axis=1))
+
+ def test_invalid_separator(self):
+ # if an invalid separator is supplied an empty data frame is returned
+ sep = "nope!"
+ df = DataFrame(
+ {
+ "A2010": [1.0, 2.0],
+ "A2011": [3.0, 4.0],
+ "B2010": [5.0, 6.0],
+ "X": ["X1", "X2"],
+ }
+ )
+ df["id"] = df.index
+ exp_data = {
+ "X": "",
+ "A2010": [],
+ "A2011": [],
+ "B2010": [],
+ "id": [],
+ "year": [],
+ "A": [],
+ "B": [],
+ }
+ expected = DataFrame(exp_data).astype({"year": np.int64})
+ expected = expected.set_index(["id", "year"])[
+ ["X", "A2010", "A2011", "B2010", "A", "B"]
+ ]
+ expected.index = expected.index.set_levels([0, 1], level=0)
+ result = wide_to_long(df, ["A", "B"], i="id", j="year", sep=sep)
+ tm.assert_frame_equal(result.sort_index(axis=1), expected.sort_index(axis=1))
+
+ def test_num_string_disambiguation(self):
+ # Test that we can disambiguate number value_vars from
+ # string value_vars
+ df = DataFrame(
+ {
+ "A11": ["a11", "a22", "a33"],
+ "A12": ["a21", "a22", "a23"],
+ "B11": ["b11", "b12", "b13"],
+ "B12": ["b21", "b22", "b23"],
+ "BB11": [1, 2, 3],
+ "BB12": [4, 5, 6],
+ "Arating": [91, 92, 93],
+ "Arating_old": [91, 92, 93],
+ }
+ )
+ df["id"] = df.index
+ expected = DataFrame(
+ {
+ "Arating": [91, 92, 93, 91, 92, 93],
+ "Arating_old": [91, 92, 93, 91, 92, 93],
+ "A": ["a11", "a22", "a33", "a21", "a22", "a23"],
+ "B": ["b11", "b12", "b13", "b21", "b22", "b23"],
+ "BB": [1, 2, 3, 4, 5, 6],
+ "id": [0, 1, 2, 0, 1, 2],
+ "year": [11, 11, 11, 12, 12, 12],
+ }
+ )
+ expected = expected.set_index(["id", "year"])[
+ ["Arating", "Arating_old", "A", "B", "BB"]
+ ]
+ result = wide_to_long(df, ["A", "B", "BB"], i="id", j="year")
+ tm.assert_frame_equal(result.sort_index(axis=1), expected.sort_index(axis=1))
+
+ def test_invalid_suffixtype(self):
+ # If all stubs names end with a string, but a numeric suffix is
+ # assumed, an empty data frame is returned
+ df = DataFrame(
+ {
+ "Aone": [1.0, 2.0],
+ "Atwo": [3.0, 4.0],
+ "Bone": [5.0, 6.0],
+ "X": ["X1", "X2"],
+ }
+ )
+ df["id"] = df.index
+ exp_data = {
+ "X": "",
+ "Aone": [],
+ "Atwo": [],
+ "Bone": [],
+ "id": [],
+ "year": [],
+ "A": [],
+ "B": [],
+ }
+ expected = DataFrame(exp_data).astype({"year": np.int64})
+
+ expected = expected.set_index(["id", "year"])
+ expected.index = expected.index.set_levels([0, 1], level=0)
+ result = wide_to_long(df, ["A", "B"], i="id", j="year")
+ tm.assert_frame_equal(result.sort_index(axis=1), expected.sort_index(axis=1))
+
+ def test_multiple_id_columns(self):
+ # Taken from http://www.ats.ucla.edu/stat/stata/modules/reshapel.htm
+ df = DataFrame(
+ {
+ "famid": [1, 1, 1, 2, 2, 2, 3, 3, 3],
+ "birth": [1, 2, 3, 1, 2, 3, 1, 2, 3],
+ "ht1": [2.8, 2.9, 2.2, 2, 1.8, 1.9, 2.2, 2.3, 2.1],
+ "ht2": [3.4, 3.8, 2.9, 3.2, 2.8, 2.4, 3.3, 3.4, 2.9],
+ }
+ )
+ expected = DataFrame(
+ {
+ "ht": [
+ 2.8,
+ 3.4,
+ 2.9,
+ 3.8,
+ 2.2,
+ 2.9,
+ 2.0,
+ 3.2,
+ 1.8,
+ 2.8,
+ 1.9,
+ 2.4,
+ 2.2,
+ 3.3,
+ 2.3,
+ 3.4,
+ 2.1,
+ 2.9,
+ ],
+ "famid": [1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3],
+ "birth": [1, 1, 2, 2, 3, 3, 1, 1, 2, 2, 3, 3, 1, 1, 2, 2, 3, 3],
+ "age": [1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2],
+ }
+ )
+ expected = expected.set_index(["famid", "birth", "age"])[["ht"]]
+ result = wide_to_long(df, "ht", i=["famid", "birth"], j="age")
+ tm.assert_frame_equal(result, expected)
+
+ def test_non_unique_idvars(self):
+ # GH16382
+ # Raise an error message if non unique id vars (i) are passed
+ df = DataFrame(
+ {"A_A1": [1, 2, 3, 4, 5], "B_B1": [1, 2, 3, 4, 5], "x": [1, 1, 1, 1, 1]}
+ )
+ msg = "the id variables need to uniquely identify each row"
+ with pytest.raises(ValueError, match=msg):
+ wide_to_long(df, ["A_A", "B_B"], i="x", j="colname")
+
+ def test_cast_j_int(self):
+ df = DataFrame(
+ {
+ "actor_1": ["CCH Pounder", "Johnny Depp", "Christoph Waltz"],
+ "actor_2": ["Joel David Moore", "Orlando Bloom", "Rory Kinnear"],
+ "actor_fb_likes_1": [1000.0, 40000.0, 11000.0],
+ "actor_fb_likes_2": [936.0, 5000.0, 393.0],
+ "title": ["Avatar", "Pirates of the Caribbean", "Spectre"],
+ }
+ )
+
+ expected = DataFrame(
+ {
+ "actor": [
+ "CCH Pounder",
+ "Johnny Depp",
+ "Christoph Waltz",
+ "Joel David Moore",
+ "Orlando Bloom",
+ "Rory Kinnear",
+ ],
+ "actor_fb_likes": [1000.0, 40000.0, 11000.0, 936.0, 5000.0, 393.0],
+ "num": [1, 1, 1, 2, 2, 2],
+ "title": [
+ "Avatar",
+ "Pirates of the Caribbean",
+ "Spectre",
+ "Avatar",
+ "Pirates of the Caribbean",
+ "Spectre",
+ ],
+ }
+ ).set_index(["title", "num"])
+ result = wide_to_long(
+ df, ["actor", "actor_fb_likes"], i="title", j="num", sep="_"
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_identical_stubnames(self):
+ df = DataFrame(
+ {
+ "A2010": [1.0, 2.0],
+ "A2011": [3.0, 4.0],
+ "B2010": [5.0, 6.0],
+ "A": ["X1", "X2"],
+ }
+ )
+ msg = "stubname can't be identical to a column name"
+ with pytest.raises(ValueError, match=msg):
+ wide_to_long(df, ["A", "B"], i="A", j="colname")
+
+ def test_nonnumeric_suffix(self):
+ df = DataFrame(
+ {
+ "treatment_placebo": [1.0, 2.0],
+ "treatment_test": [3.0, 4.0],
+ "result_placebo": [5.0, 6.0],
+ "A": ["X1", "X2"],
+ }
+ )
+ expected = DataFrame(
+ {
+ "A": ["X1", "X2", "X1", "X2"],
+ "colname": ["placebo", "placebo", "test", "test"],
+ "result": [5.0, 6.0, np.nan, np.nan],
+ "treatment": [1.0, 2.0, 3.0, 4.0],
+ }
+ )
+ expected = expected.set_index(["A", "colname"])
+ result = wide_to_long(
+ df, ["result", "treatment"], i="A", j="colname", suffix="[a-z]+", sep="_"
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_mixed_type_suffix(self):
+ df = DataFrame(
+ {
+ "A": ["X1", "X2"],
+ "result_1": [0, 9],
+ "result_foo": [5.0, 6.0],
+ "treatment_1": [1.0, 2.0],
+ "treatment_foo": [3.0, 4.0],
+ }
+ )
+ expected = DataFrame(
+ {
+ "A": ["X1", "X2", "X1", "X2"],
+ "colname": ["1", "1", "foo", "foo"],
+ "result": [0.0, 9.0, 5.0, 6.0],
+ "treatment": [1.0, 2.0, 3.0, 4.0],
+ }
+ ).set_index(["A", "colname"])
+ result = wide_to_long(
+ df, ["result", "treatment"], i="A", j="colname", suffix=".+", sep="_"
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_float_suffix(self):
+ df = DataFrame(
+ {
+ "treatment_1.1": [1.0, 2.0],
+ "treatment_2.1": [3.0, 4.0],
+ "result_1.2": [5.0, 6.0],
+ "result_1": [0, 9],
+ "A": ["X1", "X2"],
+ }
+ )
+ expected = DataFrame(
+ {
+ "A": ["X1", "X2", "X1", "X2", "X1", "X2", "X1", "X2"],
+ "colname": [1.2, 1.2, 1.0, 1.0, 1.1, 1.1, 2.1, 2.1],
+ "result": [5.0, 6.0, 0.0, 9.0, np.nan, np.nan, np.nan, np.nan],
+ "treatment": [np.nan, np.nan, np.nan, np.nan, 1.0, 2.0, 3.0, 4.0],
+ }
+ )
+ expected = expected.set_index(["A", "colname"])
+ result = wide_to_long(
+ df, ["result", "treatment"], i="A", j="colname", suffix="[0-9.]+", sep="_"
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_col_substring_of_stubname(self):
+ # GH22468
+ # Don't raise ValueError when a column name is a substring
+ # of a stubname that's been passed as a string
+ wide_data = {
+ "node_id": {0: 0, 1: 1, 2: 2, 3: 3, 4: 4},
+ "A": {0: 0.80, 1: 0.0, 2: 0.25, 3: 1.0, 4: 0.81},
+ "PA0": {0: 0.74, 1: 0.56, 2: 0.56, 3: 0.98, 4: 0.6},
+ "PA1": {0: 0.77, 1: 0.64, 2: 0.52, 3: 0.98, 4: 0.67},
+ "PA3": {0: 0.34, 1: 0.70, 2: 0.52, 3: 0.98, 4: 0.67},
+ }
+ wide_df = DataFrame.from_dict(wide_data)
+ expected = wide_to_long(wide_df, stubnames=["PA"], i=["node_id", "A"], j="time")
+ result = wide_to_long(wide_df, stubnames="PA", i=["node_id", "A"], j="time")
+ tm.assert_frame_equal(result, expected)
+
+ def test_raise_of_column_name_value(self):
+ # GH34731, enforced in 2.0
+ # raise a ValueError if the resultant value column name matches
+ # a name in the dataframe already (default name is "value")
+ df = DataFrame({"col": list("ABC"), "value": range(10, 16, 2)})
+
+ with pytest.raises(
+ ValueError, match=re.escape("value_name (value) cannot match")
+ ):
+ df.melt(id_vars="value", value_name="value")
+
+ def test_missing_stubname(self, any_string_dtype):
+ # GH46044
+ df = DataFrame({"id": ["1", "2"], "a-1": [100, 200], "a-2": [300, 400]})
+ df = df.astype({"id": any_string_dtype})
+ result = wide_to_long(
+ df,
+ stubnames=["a", "b"],
+ i="id",
+ j="num",
+ sep="-",
+ )
+ index = Index(
+ [("1", 1), ("2", 1), ("1", 2), ("2", 2)],
+ name=("id", "num"),
+ )
+ expected = DataFrame(
+ {"a": [100, 200, 300, 400], "b": [np.nan] * 4},
+ index=index,
+ )
+ new_level = expected.index.levels[0].astype(any_string_dtype)
+ if any_string_dtype == "object":
+ new_level = expected.index.levels[0].astype("str")
+ expected.index = expected.index.set_levels(new_level, level=0)
+ tm.assert_frame_equal(result, expected)
+
+
+def test_wide_to_long_string_columns(string_storage):
+ # GH 57066
+ string_dtype = pd.StringDtype(string_storage, na_value=np.nan)
+ df = DataFrame(
+ {
+ "ID": {0: 1},
+ "R_test1": {0: 1},
+ "R_test2": {0: 1},
+ "R_test3": {0: 2},
+ "D": {0: 1},
+ }
+ )
+ df.columns = df.columns.astype(string_dtype)
+ result = wide_to_long(
+ df, stubnames="R", i="ID", j="UNPIVOTED", sep="_", suffix=".*"
+ )
+ expected = DataFrame(
+ [[1, 1], [1, 1], [1, 2]],
+ columns=Index(["D", "R"]),
+ index=pd.MultiIndex.from_arrays(
+ [
+ [1, 1, 1],
+ Index(["test1", "test2", "test3"], dtype=string_dtype),
+ ],
+ names=["ID", "UNPIVOTED"],
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_pivot.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_pivot.py
new file mode 100644
index 0000000000000000000000000000000000000000..6745ba0bac765a17b583724cc57dc207bb2e81a6
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_pivot.py
@@ -0,0 +1,2961 @@
+from datetime import (
+ date,
+ datetime,
+ timedelta,
+)
+from itertools import product
+import re
+
+import numpy as np
+import pytest
+
+from pandas._config import using_string_dtype
+
+import pandas as pd
+from pandas import (
+ ArrowDtype,
+ Categorical,
+ DataFrame,
+ Grouper,
+ Index,
+ MultiIndex,
+ Series,
+ concat,
+ date_range,
+)
+import pandas._testing as tm
+from pandas.api.types import CategoricalDtype
+from pandas.core.reshape import reshape as reshape_lib
+from pandas.core.reshape.pivot import pivot_table
+
+
+class TestPivotTable:
+ @pytest.fixture
+ def data(self):
+ return DataFrame(
+ {
+ "A": [
+ "foo",
+ "foo",
+ "foo",
+ "foo",
+ "bar",
+ "bar",
+ "bar",
+ "bar",
+ "foo",
+ "foo",
+ "foo",
+ ],
+ "B": [
+ "one",
+ "one",
+ "one",
+ "two",
+ "one",
+ "one",
+ "one",
+ "two",
+ "two",
+ "two",
+ "one",
+ ],
+ "C": [
+ "dull",
+ "dull",
+ "shiny",
+ "dull",
+ "dull",
+ "shiny",
+ "shiny",
+ "dull",
+ "shiny",
+ "shiny",
+ "shiny",
+ ],
+ "D": np.random.default_rng(2).standard_normal(11),
+ "E": np.random.default_rng(2).standard_normal(11),
+ "F": np.random.default_rng(2).standard_normal(11),
+ }
+ )
+
+ def test_pivot_table(self, observed, data):
+ index = ["A", "B"]
+ columns = "C"
+ table = pivot_table(
+ data, values="D", index=index, columns=columns, observed=observed
+ )
+
+ table2 = data.pivot_table(
+ values="D", index=index, columns=columns, observed=observed
+ )
+ tm.assert_frame_equal(table, table2)
+
+ # this works
+ pivot_table(data, values="D", index=index, observed=observed)
+
+ if len(index) > 1:
+ assert table.index.names == tuple(index)
+ else:
+ assert table.index.name == index[0]
+
+ if len(columns) > 1:
+ assert table.columns.names == columns
+ else:
+ assert table.columns.name == columns[0]
+
+ expected = data.groupby([*index, columns])["D"].agg("mean").unstack()
+ tm.assert_frame_equal(table, expected)
+
+ def test_pivot_table_categorical_observed_equal(self, observed):
+ # issue #24923
+ df = DataFrame(
+ {"col1": list("abcde"), "col2": list("fghij"), "col3": [1, 2, 3, 4, 5]}
+ )
+
+ expected = df.pivot_table(
+ index="col1", values="col3", columns="col2", aggfunc="sum", fill_value=0
+ )
+
+ expected.index = expected.index.astype("category")
+ expected.columns = expected.columns.astype("category")
+
+ df.col1 = df.col1.astype("category")
+ df.col2 = df.col2.astype("category")
+
+ result = df.pivot_table(
+ index="col1",
+ values="col3",
+ columns="col2",
+ aggfunc="sum",
+ fill_value=0,
+ observed=observed,
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_nocols(self):
+ df = DataFrame(
+ {"rows": ["a", "b", "c"], "cols": ["x", "y", "z"], "values": [1, 2, 3]}
+ )
+ rs = df.pivot_table(columns="cols", aggfunc="sum")
+ xp = df.pivot_table(index="cols", aggfunc="sum").T
+ tm.assert_frame_equal(rs, xp)
+
+ rs = df.pivot_table(columns="cols", aggfunc={"values": "mean"})
+ xp = df.pivot_table(index="cols", aggfunc={"values": "mean"}).T
+ tm.assert_frame_equal(rs, xp)
+
+ def test_pivot_table_dropna(self):
+ df = DataFrame(
+ {
+ "amount": {0: 60000, 1: 100000, 2: 50000, 3: 30000},
+ "customer": {0: "A", 1: "A", 2: "B", 3: "C"},
+ "month": {0: 201307, 1: 201309, 2: 201308, 3: 201310},
+ "product": {0: "a", 1: "b", 2: "c", 3: "d"},
+ "quantity": {0: 2000000, 1: 500000, 2: 1000000, 3: 1000000},
+ }
+ )
+ pv_col = df.pivot_table(
+ "quantity", "month", ["customer", "product"], dropna=False
+ )
+ pv_ind = df.pivot_table(
+ "quantity", ["customer", "product"], "month", dropna=False
+ )
+
+ m = MultiIndex.from_tuples(
+ [
+ ("A", "a"),
+ ("A", "b"),
+ ("A", "c"),
+ ("A", "d"),
+ ("B", "a"),
+ ("B", "b"),
+ ("B", "c"),
+ ("B", "d"),
+ ("C", "a"),
+ ("C", "b"),
+ ("C", "c"),
+ ("C", "d"),
+ ],
+ names=["customer", "product"],
+ )
+ tm.assert_index_equal(pv_col.columns, m)
+ tm.assert_index_equal(pv_ind.index, m)
+
+ def test_pivot_table_categorical(self):
+ cat1 = Categorical(
+ ["a", "a", "b", "b"], categories=["a", "b", "z"], ordered=True
+ )
+ cat2 = Categorical(
+ ["c", "d", "c", "d"], categories=["c", "d", "y"], ordered=True
+ )
+ df = DataFrame({"A": cat1, "B": cat2, "values": [1, 2, 3, 4]})
+ result = pivot_table(
+ df, values="values", index=["A", "B"], dropna=True, observed=False
+ )
+
+ exp_index = MultiIndex.from_arrays([cat1, cat2], names=["A", "B"])
+ expected = DataFrame({"values": [1.0, 2.0, 3.0, 4.0]}, index=exp_index)
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_dropna_categoricals(self, dropna):
+ # GH 15193
+ categories = ["a", "b", "c", "d"]
+
+ df = DataFrame(
+ {
+ "A": ["a", "a", "a", "b", "b", "b", "c", "c", "c"],
+ "B": [1, 2, 3, 1, 2, 3, 1, 2, 3],
+ "C": range(9),
+ }
+ )
+
+ df["A"] = df["A"].astype(CategoricalDtype(categories, ordered=False))
+ result = df.pivot_table(
+ index="B", columns="A", values="C", dropna=dropna, observed=False
+ )
+ expected_columns = Series(["a", "b", "c"], name="A")
+ expected_columns = expected_columns.astype(
+ CategoricalDtype(categories, ordered=False)
+ )
+ expected_index = Series([1, 2, 3], name="B")
+ expected = DataFrame(
+ [[0.0, 3.0, 6.0], [1.0, 4.0, 7.0], [2.0, 5.0, 8.0]],
+ index=expected_index,
+ columns=expected_columns,
+ )
+ if not dropna:
+ # add back the non observed to compare
+ expected = expected.reindex(columns=Categorical(categories)).astype("float")
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_with_non_observable_dropna(self, dropna):
+ # gh-21133
+ df = DataFrame(
+ {
+ "A": Categorical(
+ [np.nan, "low", "high", "low", "high"],
+ categories=["low", "high"],
+ ordered=True,
+ ),
+ "B": [0.0, 1.0, 2.0, 3.0, 4.0],
+ }
+ )
+
+ result = df.pivot_table(index="A", values="B", dropna=dropna, observed=False)
+ if dropna:
+ values = [2.0, 3.0]
+ codes = [0, 1]
+ else:
+ # GH: 10772
+ values = [2.0, 3.0, 0.0]
+ codes = [0, 1, -1]
+ expected = DataFrame(
+ {"B": values},
+ index=Index(
+ Categorical.from_codes(codes, categories=["low", "high"], ordered=True),
+ name="A",
+ ),
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_with_non_observable_dropna_multi_cat(self, dropna):
+ # gh-21378
+ df = DataFrame(
+ {
+ "A": Categorical(
+ ["left", "low", "high", "low", "high"],
+ categories=["low", "high", "left"],
+ ordered=True,
+ ),
+ "B": range(5),
+ }
+ )
+
+ result = df.pivot_table(index="A", values="B", dropna=dropna, observed=False)
+ expected = DataFrame(
+ {"B": [2.0, 3.0, 0.0]},
+ index=Index(
+ Categorical.from_codes(
+ [0, 1, 2], categories=["low", "high", "left"], ordered=True
+ ),
+ name="A",
+ ),
+ )
+ if not dropna:
+ expected["B"] = expected["B"].astype(float)
+
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "left_right", [([0] * 4, [1] * 4), (range(3), range(1, 4))]
+ )
+ def test_pivot_with_interval_index(self, left_right, dropna, closed):
+ # GH 25814
+ left, right = left_right
+ interval_values = Categorical(pd.IntervalIndex.from_arrays(left, right, closed))
+ df = DataFrame({"A": interval_values, "B": 1})
+
+ result = df.pivot_table(index="A", values="B", dropna=dropna, observed=False)
+ expected = DataFrame(
+ {"B": 1.0}, index=Index(interval_values.unique(), name="A")
+ )
+ if not dropna:
+ expected = expected.astype(float)
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_with_interval_index_margins(self):
+ # GH 25815
+ ordered_cat = pd.IntervalIndex.from_arrays([0, 0, 1, 1], [1, 1, 2, 2])
+ df = DataFrame(
+ {
+ "A": np.arange(4, 0, -1, dtype=np.intp),
+ "B": ["a", "b", "a", "b"],
+ "C": Categorical(ordered_cat, ordered=True).sort_values(
+ ascending=False
+ ),
+ }
+ )
+
+ pivot_tab = pivot_table(
+ df,
+ index="C",
+ columns="B",
+ values="A",
+ aggfunc="sum",
+ margins=True,
+ observed=False,
+ )
+
+ result = pivot_tab["All"]
+ expected = Series(
+ [3, 7, 10],
+ index=Index([pd.Interval(0, 1), pd.Interval(1, 2), "All"], name="C"),
+ name="All",
+ dtype=np.intp,
+ )
+ tm.assert_series_equal(result, expected)
+
+ def test_pass_array(self, data):
+ result = data.pivot_table("D", index=data.A, columns=data.C)
+ expected = data.pivot_table("D", index="A", columns="C")
+ tm.assert_frame_equal(result, expected)
+
+ def test_pass_function(self, data):
+ result = data.pivot_table("D", index=lambda x: x // 5, columns=data.C)
+ expected = data.pivot_table("D", index=data.index // 5, columns="C")
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_multiple(self, data):
+ index = ["A", "B"]
+ columns = "C"
+ table = pivot_table(data, index=index, columns=columns)
+ expected = data.groupby([*index, columns]).agg("mean").unstack()
+ tm.assert_frame_equal(table, expected)
+
+ def test_pivot_dtypes(self):
+ # can convert dtypes
+ f = DataFrame(
+ {
+ "a": ["cat", "bat", "cat", "bat"],
+ "v": [1, 2, 3, 4],
+ "i": ["a", "b", "a", "b"],
+ }
+ )
+ assert f.dtypes["v"] == "int64"
+
+ z = pivot_table(
+ f, values="v", index=["a"], columns=["i"], fill_value=0, aggfunc="sum"
+ )
+ result = z.dtypes
+ expected = Series([np.dtype("int64")] * 2, index=Index(list("ab"), name="i"))
+ tm.assert_series_equal(result, expected)
+
+ # cannot convert dtypes
+ f = DataFrame(
+ {
+ "a": ["cat", "bat", "cat", "bat"],
+ "v": [1.5, 2.5, 3.5, 4.5],
+ "i": ["a", "b", "a", "b"],
+ }
+ )
+ assert f.dtypes["v"] == "float64"
+
+ z = pivot_table(
+ f, values="v", index=["a"], columns=["i"], fill_value=0, aggfunc="mean"
+ )
+ result = z.dtypes
+ expected = Series([np.dtype("float64")] * 2, index=Index(list("ab"), name="i"))
+ tm.assert_series_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "columns,values",
+ [
+ ("bool1", ["float1", "float2"]),
+ ("bool1", ["float1", "float2", "bool1"]),
+ ("bool2", ["float1", "float2", "bool1"]),
+ ],
+ )
+ def test_pivot_preserve_dtypes(self, columns, values):
+ # GH 7142 regression test
+ v = np.arange(5, dtype=np.float64)
+ df = DataFrame(
+ {"float1": v, "float2": v + 2.0, "bool1": v <= 2, "bool2": v <= 3}
+ )
+
+ df_res = df.reset_index().pivot_table(
+ index="index", columns=columns, values=values
+ )
+
+ result = dict(df_res.dtypes)
+ expected = {col: np.dtype("float64") for col in df_res}
+ assert result == expected
+
+ def test_pivot_no_values(self):
+ # GH 14380
+ idx = pd.DatetimeIndex(
+ ["2011-01-01", "2011-02-01", "2011-01-02", "2011-01-01", "2011-01-02"]
+ )
+ df = DataFrame({"A": [1, 2, 3, 4, 5]}, index=idx)
+ res = df.pivot_table(index=df.index.month, columns=df.index.day)
+
+ exp_columns = MultiIndex.from_tuples([("A", 1), ("A", 2)])
+ exp_columns = exp_columns.set_levels(
+ exp_columns.levels[1].astype(np.int32), level=1
+ )
+ exp = DataFrame(
+ [[2.5, 4.0], [2.0, np.nan]],
+ index=Index([1, 2], dtype=np.int32),
+ columns=exp_columns,
+ )
+ tm.assert_frame_equal(res, exp)
+
+ df = DataFrame(
+ {
+ "A": [1, 2, 3, 4, 5],
+ "dt": date_range("2011-01-01", freq="D", periods=5, unit="ns"),
+ },
+ index=idx,
+ )
+ res = df.pivot_table(index=df.index.month, columns=Grouper(key="dt", freq="ME"))
+ exp_columns = MultiIndex.from_arrays(
+ [["A"], pd.DatetimeIndex(["2011-01-31"], dtype="M8[ns]")],
+ names=[None, "dt"],
+ )
+ exp = DataFrame(
+ [3.25, 2.0], index=Index([1, 2], dtype=np.int32), columns=exp_columns
+ )
+ tm.assert_frame_equal(res, exp)
+
+ res = df.pivot_table(
+ index=Grouper(freq="YE"), columns=Grouper(key="dt", freq="ME")
+ )
+ exp = DataFrame(
+ [3.0],
+ index=pd.DatetimeIndex(["2011-12-31"], freq="YE"),
+ columns=exp_columns,
+ )
+ tm.assert_frame_equal(res, exp)
+
+ def test_pivot_multi_values(self, data):
+ result = pivot_table(
+ data, values=["D", "E"], index="A", columns=["B", "C"], fill_value=0
+ )
+ expected = pivot_table(
+ data.drop(["F"], axis=1), index="A", columns=["B", "C"], fill_value=0
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_multi_functions(self, data):
+ f = lambda func: pivot_table(
+ data, values=["D", "E"], index=["A", "B"], columns="C", aggfunc=func
+ )
+ result = f(["mean", "std"])
+ means = f("mean")
+ stds = f("std")
+ expected = concat([means, stds], keys=["mean", "std"], axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ # margins not supported??
+ f = lambda func: pivot_table(
+ data,
+ values=["D", "E"],
+ index=["A", "B"],
+ columns="C",
+ aggfunc=func,
+ margins=True,
+ )
+ result = f(["mean", "std"])
+ means = f("mean")
+ stds = f("std")
+ expected = concat([means, stds], keys=["mean", "std"], axis=1)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("method", [True, False])
+ def test_pivot_index_with_nan(self, method):
+ # GH 3588
+ nan = np.nan
+ df = DataFrame(
+ {
+ "a": ["R1", "R2", nan, "R4"],
+ "b": ["C1", "C2", "C3", "C4"],
+ "c": [10, 15, 17, 20],
+ }
+ )
+ if method:
+ result = df.pivot(index="a", columns="b", values="c")
+ else:
+ result = pd.pivot(df, index="a", columns="b", values="c")
+ expected = DataFrame(
+ [
+ [nan, nan, 17, nan],
+ [10, nan, nan, nan],
+ [nan, 15, nan, nan],
+ [nan, nan, nan, 20],
+ ],
+ index=Index([nan, "R1", "R2", "R4"], name="a"),
+ columns=Index(["C1", "C2", "C3", "C4"], name="b"),
+ )
+ tm.assert_frame_equal(result, expected)
+ tm.assert_frame_equal(df.pivot(index="b", columns="a", values="c"), expected.T)
+
+ @pytest.mark.parametrize("method", [True, False])
+ def test_pivot_index_with_nan_dates(self, method):
+ # GH9491
+ df = DataFrame(
+ {
+ "a": date_range("2014-02-01", periods=6, freq="D"),
+ "c": 100 + np.arange(6),
+ }
+ )
+ df["b"] = df["a"] - pd.Timestamp("2014-02-02")
+ df.loc[1, "a"] = df.loc[3, "a"] = np.nan
+ df.loc[1, "b"] = df.loc[4, "b"] = np.nan
+
+ if method:
+ pv = df.pivot(index="a", columns="b", values="c")
+ else:
+ pv = pd.pivot(df, index="a", columns="b", values="c")
+ assert pv.notna().values.sum() == len(df)
+
+ for _, row in df.iterrows():
+ assert pv.loc[row["a"], row["b"]] == row["c"]
+
+ if method:
+ result = df.pivot(index="b", columns="a", values="c")
+ else:
+ result = pd.pivot(df, index="b", columns="a", values="c")
+ tm.assert_frame_equal(result, pv.T)
+
+ @pytest.mark.parametrize("method", [True, False])
+ def test_pivot_with_tz(self, method, unit):
+ # GH 5878
+ df = DataFrame(
+ {
+ "dt1": pd.DatetimeIndex(
+ [
+ datetime(2013, 1, 1, 9, 0),
+ datetime(2013, 1, 2, 9, 0),
+ datetime(2013, 1, 1, 9, 0),
+ datetime(2013, 1, 2, 9, 0),
+ ],
+ dtype=f"M8[{unit}, US/Pacific]",
+ ),
+ "dt2": pd.DatetimeIndex(
+ [
+ datetime(2014, 1, 1, 9, 0),
+ datetime(2014, 1, 1, 9, 0),
+ datetime(2014, 1, 2, 9, 0),
+ datetime(2014, 1, 2, 9, 0),
+ ],
+ dtype=f"M8[{unit}, Asia/Tokyo]",
+ ),
+ "data1": np.arange(4, dtype="int64"),
+ "data2": np.arange(4, dtype="int64"),
+ }
+ )
+
+ exp_col1 = Index(["data1", "data1", "data2", "data2"])
+ exp_col2 = pd.DatetimeIndex(
+ ["2014/01/01 09:00", "2014/01/02 09:00"] * 2,
+ name="dt2",
+ dtype=f"M8[{unit}, Asia/Tokyo]",
+ )
+ exp_col = MultiIndex.from_arrays([exp_col1, exp_col2])
+ exp_idx = pd.DatetimeIndex(
+ ["2013/01/01 09:00", "2013/01/02 09:00"],
+ name="dt1",
+ dtype=f"M8[{unit}, US/Pacific]",
+ )
+ expected = DataFrame(
+ [[0, 2, 0, 2], [1, 3, 1, 3]],
+ index=exp_idx,
+ columns=exp_col,
+ )
+
+ if method:
+ pv = df.pivot(index="dt1", columns="dt2")
+ else:
+ pv = pd.pivot(df, index="dt1", columns="dt2")
+ tm.assert_frame_equal(pv, expected)
+
+ expected = DataFrame(
+ [[0, 2], [1, 3]],
+ index=exp_idx,
+ columns=exp_col2[:2],
+ )
+
+ if method:
+ pv = df.pivot(index="dt1", columns="dt2", values="data1")
+ else:
+ pv = pd.pivot(df, index="dt1", columns="dt2", values="data1")
+ tm.assert_frame_equal(pv, expected)
+
+ def test_pivot_tz_in_values(self):
+ # GH 14948
+ df = DataFrame(
+ [
+ {
+ "uid": "aa",
+ "ts": pd.Timestamp("2016-08-12 13:00:00-0700", tz="US/Pacific"),
+ },
+ {
+ "uid": "aa",
+ "ts": pd.Timestamp("2016-08-12 08:00:00-0700", tz="US/Pacific"),
+ },
+ {
+ "uid": "aa",
+ "ts": pd.Timestamp("2016-08-12 14:00:00-0700", tz="US/Pacific"),
+ },
+ {
+ "uid": "aa",
+ "ts": pd.Timestamp("2016-08-25 11:00:00-0700", tz="US/Pacific"),
+ },
+ {
+ "uid": "aa",
+ "ts": pd.Timestamp("2016-08-25 13:00:00-0700", tz="US/Pacific"),
+ },
+ ]
+ )
+
+ df = df.set_index("ts").reset_index()
+ mins = df.ts.map(lambda x: x.replace(hour=0, minute=0, second=0))
+
+ result = pivot_table(
+ df.set_index("ts").reset_index(),
+ values="ts",
+ index=["uid"],
+ columns=[mins],
+ aggfunc="min",
+ )
+ expected = DataFrame(
+ [
+ [
+ pd.Timestamp("2016-08-12 08:00:00-0700", tz="US/Pacific"),
+ pd.Timestamp("2016-08-25 11:00:00-0700", tz="US/Pacific"),
+ ]
+ ],
+ index=Index(["aa"], name="uid"),
+ columns=pd.DatetimeIndex(
+ [
+ pd.Timestamp("2016-08-12 00:00:00", tz="US/Pacific"),
+ pd.Timestamp("2016-08-25 00:00:00", tz="US/Pacific"),
+ ],
+ name="ts",
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("method", [True, False])
+ def test_pivot_periods(self, method):
+ df = DataFrame(
+ {
+ "p1": [
+ pd.Period("2013-01-01", "D"),
+ pd.Period("2013-01-02", "D"),
+ pd.Period("2013-01-01", "D"),
+ pd.Period("2013-01-02", "D"),
+ ],
+ "p2": [
+ pd.Period("2013-01", "M"),
+ pd.Period("2013-01", "M"),
+ pd.Period("2013-02", "M"),
+ pd.Period("2013-02", "M"),
+ ],
+ "data1": np.arange(4, dtype="int64"),
+ "data2": np.arange(4, dtype="int64"),
+ }
+ )
+
+ exp_col1 = Index(["data1", "data1", "data2", "data2"])
+ exp_col2 = pd.PeriodIndex(["2013-01", "2013-02"] * 2, name="p2", freq="M")
+ exp_col = MultiIndex.from_arrays([exp_col1, exp_col2])
+ expected = DataFrame(
+ [[0, 2, 0, 2], [1, 3, 1, 3]],
+ index=pd.PeriodIndex(["2013-01-01", "2013-01-02"], name="p1", freq="D"),
+ columns=exp_col,
+ )
+ if method:
+ pv = df.pivot(index="p1", columns="p2")
+ else:
+ pv = pd.pivot(df, index="p1", columns="p2")
+ tm.assert_frame_equal(pv, expected)
+
+ expected = DataFrame(
+ [[0, 2], [1, 3]],
+ index=pd.PeriodIndex(["2013-01-01", "2013-01-02"], name="p1", freq="D"),
+ columns=pd.PeriodIndex(["2013-01", "2013-02"], name="p2", freq="M"),
+ )
+ if method:
+ pv = df.pivot(index="p1", columns="p2", values="data1")
+ else:
+ pv = pd.pivot(df, index="p1", columns="p2", values="data1")
+ tm.assert_frame_equal(pv, expected)
+
+ def test_pivot_periods_with_margins(self):
+ # GH 28323
+ df = DataFrame(
+ {
+ "a": [1, 1, 2, 2],
+ "b": [
+ pd.Period("2019Q1"),
+ pd.Period("2019Q2"),
+ pd.Period("2019Q1"),
+ pd.Period("2019Q2"),
+ ],
+ "x": 1.0,
+ }
+ )
+
+ expected = DataFrame(
+ data=1.0,
+ index=Index([1, 2, "All"], name="a"),
+ columns=Index([pd.Period("2019Q1"), pd.Period("2019Q2"), "All"], name="b"),
+ )
+
+ result = df.pivot_table(index="a", columns="b", values="x", margins=True)
+ tm.assert_frame_equal(expected, result)
+
+ @pytest.mark.parametrize("box", [list, np.array, Series, Index])
+ @pytest.mark.parametrize("method", [True, False])
+ def test_pivot_with_list_like_values(self, box, method):
+ # issue #17160
+ values = box(["baz", "zoo"])
+ df = DataFrame(
+ {
+ "foo": ["one", "one", "one", "two", "two", "two"],
+ "bar": ["A", "B", "C", "A", "B", "C"],
+ "baz": [1, 2, 3, 4, 5, 6],
+ "zoo": ["x", "y", "z", "q", "w", "t"],
+ }
+ )
+
+ if method:
+ result = df.pivot(index="foo", columns="bar", values=values)
+ else:
+ result = pd.pivot(df, index="foo", columns="bar", values=values)
+
+ data = [[1, 2, 3, "x", "y", "z"], [4, 5, 6, "q", "w", "t"]]
+ index = Index(data=["one", "two"], name="foo")
+ columns = MultiIndex(
+ levels=[["baz", "zoo"], ["A", "B", "C"]],
+ codes=[[0, 0, 0, 1, 1, 1], [0, 1, 2, 0, 1, 2]],
+ names=[None, "bar"],
+ )
+ expected = DataFrame(data=data, index=index, columns=columns)
+ expected["baz"] = expected["baz"].astype(object)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "values",
+ [
+ ["bar", "baz"],
+ np.array(["bar", "baz"]),
+ Series(["bar", "baz"]),
+ Index(["bar", "baz"]),
+ ],
+ )
+ @pytest.mark.parametrize("method", [True, False])
+ def test_pivot_with_list_like_values_nans(self, values, method):
+ # issue #17160
+ df = DataFrame(
+ {
+ "foo": ["one", "one", "one", "two", "two", "two"],
+ "bar": ["A", "B", "C", "A", "B", "C"],
+ "baz": [1, 2, 3, 4, 5, 6],
+ "zoo": ["x", "y", "z", "q", "w", "t"],
+ }
+ )
+
+ if method:
+ result = df.pivot(index="zoo", columns="foo", values=values)
+ else:
+ result = pd.pivot(df, index="zoo", columns="foo", values=values)
+
+ data = [
+ [np.nan, "A", np.nan, 4],
+ [np.nan, "C", np.nan, 6],
+ [np.nan, "B", np.nan, 5],
+ ["A", np.nan, 1, np.nan],
+ ["B", np.nan, 2, np.nan],
+ ["C", np.nan, 3, np.nan],
+ ]
+ index = Index(data=["q", "t", "w", "x", "y", "z"], name="zoo")
+ columns = MultiIndex(
+ levels=[["bar", "baz"], ["one", "two"]],
+ codes=[[0, 0, 1, 1], [0, 1, 0, 1]],
+ names=[None, "foo"],
+ )
+ expected = DataFrame(data=data, index=index, columns=columns)
+ expected["baz"] = expected["baz"].astype(object)
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_columns_none_raise_error(self):
+ # GH 30924
+ df = DataFrame({"col1": ["a", "b", "c"], "col2": [1, 2, 3], "col3": [1, 2, 3]})
+ msg = r"pivot\(\) missing 1 required keyword-only argument: 'columns'"
+ with pytest.raises(TypeError, match=msg):
+ df.pivot(index="col1", values="col3")
+
+ @pytest.mark.xfail(
+ reason="MultiIndexed unstack with tuple names fails with KeyError GH#19966"
+ )
+ @pytest.mark.parametrize("method", [True, False])
+ def test_pivot_with_multiindex(self, method):
+ # issue #17160
+ index = Index(data=[0, 1, 2, 3, 4, 5])
+ data = [
+ ["one", "A", 1, "x"],
+ ["one", "B", 2, "y"],
+ ["one", "C", 3, "z"],
+ ["two", "A", 4, "q"],
+ ["two", "B", 5, "w"],
+ ["two", "C", 6, "t"],
+ ]
+ columns = MultiIndex(
+ levels=[["bar", "baz"], ["first", "second"]],
+ codes=[[0, 0, 1, 1], [0, 1, 0, 1]],
+ )
+ df = DataFrame(data=data, index=index, columns=columns, dtype="object")
+ if method:
+ result = df.pivot(
+ index=("bar", "first"),
+ columns=("bar", "second"),
+ values=("baz", "first"),
+ )
+ else:
+ result = pd.pivot(
+ df,
+ index=("bar", "first"),
+ columns=("bar", "second"),
+ values=("baz", "first"),
+ )
+
+ data = {
+ "A": Series([1, 4], index=["one", "two"]),
+ "B": Series([2, 5], index=["one", "two"]),
+ "C": Series([3, 6], index=["one", "two"]),
+ }
+ expected = DataFrame(data)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("method", [True, False])
+ def test_pivot_with_tuple_of_values(self, method):
+ # issue #17160
+ df = DataFrame(
+ {
+ "foo": ["one", "one", "one", "two", "two", "two"],
+ "bar": ["A", "B", "C", "A", "B", "C"],
+ "baz": [1, 2, 3, 4, 5, 6],
+ "zoo": ["x", "y", "z", "q", "w", "t"],
+ }
+ )
+ with pytest.raises(KeyError, match=r"^\('bar', 'baz'\)$"):
+ # tuple is seen as a single column name
+ if method:
+ df.pivot(index="zoo", columns="foo", values=("bar", "baz"))
+ else:
+ pd.pivot(df, index="zoo", columns="foo", values=("bar", "baz"))
+
+ def _check_output(
+ self,
+ result,
+ values_col,
+ data,
+ index=None,
+ columns=None,
+ margins_col="All",
+ ):
+ if index is None:
+ index = ["A", "B"]
+ if columns is None:
+ columns = ["C"]
+ col_margins = result.loc[result.index[:-1], margins_col]
+ expected_col_margins = data.groupby(index)[values_col].mean()
+ tm.assert_series_equal(col_margins, expected_col_margins, check_names=False)
+ assert col_margins.name == margins_col
+
+ result = result.sort_index()
+ index_margins = result.loc[(margins_col, "")].iloc[:-1]
+
+ expected_ix_margins = data.groupby(columns)[values_col].mean()
+ tm.assert_series_equal(index_margins, expected_ix_margins, check_names=False)
+ assert index_margins.name == (margins_col, "")
+
+ grand_total_margins = result.loc[(margins_col, ""), margins_col]
+ expected_total_margins = data[values_col].mean()
+ assert grand_total_margins == expected_total_margins
+
+ def test_margins(self, data):
+ # column specified
+ result = data.pivot_table(
+ values="D", index=["A", "B"], columns="C", margins=True, aggfunc="mean"
+ )
+ self._check_output(result, "D", data)
+
+ # Set a different margins_name (not 'All')
+ result = data.pivot_table(
+ values="D",
+ index=["A", "B"],
+ columns="C",
+ margins=True,
+ aggfunc="mean",
+ margins_name="Totals",
+ )
+ self._check_output(result, "D", data, margins_col="Totals")
+
+ # no column specified
+ table = data.pivot_table(
+ index=["A", "B"], columns="C", margins=True, aggfunc="mean"
+ )
+ for value_col in table.columns.levels[0]:
+ self._check_output(table[value_col], value_col, data)
+
+ def test_no_col(self, data, using_infer_string):
+ # no col
+
+ # to help with a buglet
+ data.columns = [k * 2 for k in data.columns]
+ msg = re.escape("agg function failed [how->mean,dtype->")
+ if using_infer_string:
+ msg = "dtype 'str' does not support operation 'mean'"
+ with pytest.raises(TypeError, match=msg):
+ data.pivot_table(index=["AA", "BB"], margins=True, aggfunc="mean")
+ table = data.drop(columns="CC").pivot_table(
+ index=["AA", "BB"], margins=True, aggfunc="mean"
+ )
+ for value_col in table.columns:
+ totals = table.loc[("All", ""), value_col]
+ assert totals == data[value_col].mean()
+
+ with pytest.raises(TypeError, match=msg):
+ data.pivot_table(index=["AA", "BB"], margins=True, aggfunc="mean")
+ table = data.drop(columns="CC").pivot_table(
+ index=["AA", "BB"], margins=True, aggfunc="mean"
+ )
+ for item in ["DD", "EE", "FF"]:
+ totals = table.loc[("All", ""), item]
+ assert totals == data[item].mean()
+
+ @pytest.mark.parametrize(
+ "columns, aggfunc, values, expected_columns",
+ [
+ (
+ "A",
+ "mean",
+ [[5.5, 5.5, 2.2, 2.2], [8.0, 8.0, 4.4, 4.4]],
+ Index(["bar", "All", "foo", "All"], name="A"),
+ ),
+ (
+ ["A", "B"],
+ "sum",
+ [
+ [9, 13, 22, 5, 6, 11],
+ [14, 18, 32, 11, 11, 22],
+ ],
+ MultiIndex.from_tuples(
+ [
+ ("bar", "one"),
+ ("bar", "two"),
+ ("bar", "All"),
+ ("foo", "one"),
+ ("foo", "two"),
+ ("foo", "All"),
+ ],
+ names=["A", "B"],
+ ),
+ ),
+ ],
+ )
+ def test_margin_with_only_columns_defined(
+ self, columns, aggfunc, values, expected_columns, using_infer_string
+ ):
+ # GH 31016
+ df = DataFrame(
+ {
+ "A": ["foo", "foo", "foo", "foo", "foo", "bar", "bar", "bar", "bar"],
+ "B": ["one", "one", "one", "two", "two", "one", "one", "two", "two"],
+ "C": [
+ "small",
+ "large",
+ "large",
+ "small",
+ "small",
+ "large",
+ "small",
+ "small",
+ "large",
+ ],
+ "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
+ "E": [2, 4, 5, 5, 6, 6, 8, 9, 9],
+ }
+ )
+ if aggfunc != "sum":
+ msg = re.escape("agg function failed [how->mean,dtype->")
+ if using_infer_string:
+ msg = "dtype 'str' does not support operation 'mean'"
+ with pytest.raises(TypeError, match=msg):
+ df.pivot_table(columns=columns, margins=True, aggfunc=aggfunc)
+ if "B" not in columns:
+ df = df.drop(columns="B")
+ result = df.drop(columns="C").pivot_table(
+ columns=columns, margins=True, aggfunc=aggfunc
+ )
+ expected = DataFrame(values, index=Index(["D", "E"]), columns=expected_columns)
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_margins_dtype(self, data):
+ # GH 17013
+
+ df = data.copy()
+ df[["D", "E", "F"]] = np.arange(len(df) * 3).reshape(len(df), 3).astype("i8")
+
+ mi_val = [*list(product(["bar", "foo"], ["one", "two"])), ("All", "")]
+ mi = MultiIndex.from_tuples(mi_val, names=("A", "B"))
+ expected = DataFrame(
+ {"dull": [12, 21, 3, 9, 45], "shiny": [33, 0, 36, 51, 120]}, index=mi
+ ).rename_axis("C", axis=1)
+ expected["All"] = expected["dull"] + expected["shiny"]
+
+ result = df.pivot_table(
+ values="D",
+ index=["A", "B"],
+ columns="C",
+ margins=True,
+ aggfunc="sum",
+ fill_value=0,
+ )
+
+ tm.assert_frame_equal(expected, result)
+
+ def test_margins_dtype_len(self, data):
+ mi_val = [*list(product(["bar", "foo"], ["one", "two"])), ("All", "")]
+ mi = MultiIndex.from_tuples(mi_val, names=("A", "B"))
+ expected = DataFrame(
+ {"dull": [1, 1, 2, 1, 5], "shiny": [2, 0, 2, 2, 6]}, index=mi
+ ).rename_axis("C", axis=1)
+ expected["All"] = expected["dull"] + expected["shiny"]
+
+ result = data.pivot_table(
+ values="D",
+ index=["A", "B"],
+ columns="C",
+ margins=True,
+ aggfunc=len,
+ fill_value=0,
+ )
+
+ tm.assert_frame_equal(expected, result)
+
+ @pytest.mark.parametrize("cols", [(1, 2), ("a", "b"), (1, "b"), ("a", 1)])
+ def test_pivot_table_multiindex_only(self, cols):
+ # GH 17038
+ df2 = DataFrame({cols[0]: [1, 2, 3], cols[1]: [1, 2, 3], "v": [4, 5, 6]})
+
+ result = df2.pivot_table(values="v", columns=cols)
+ expected = DataFrame(
+ [[4.0, 5.0, 6.0]],
+ columns=MultiIndex.from_tuples([(1, 1), (2, 2), (3, 3)], names=cols),
+ index=Index(["v"], dtype="str" if cols == ("a", "b") else "object"),
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_retains_tz(self):
+ dti = date_range("2016-01-01", periods=3, tz="Europe/Amsterdam")
+ df = DataFrame(
+ {
+ "A": np.random.default_rng(2).standard_normal(3),
+ "B": np.random.default_rng(2).standard_normal(3),
+ "C": dti,
+ }
+ )
+ result = df.pivot_table(index=["B", "C"], dropna=False)
+
+ # check tz retention
+ assert result.index.levels[1].equals(dti)
+
+ def test_pivot_integer_columns(self):
+ # caused by upstream bug in unstack
+
+ d = date.min
+ data = list(
+ product(
+ ["foo", "bar"],
+ ["A", "B", "C"],
+ ["x1", "x2"],
+ [d + timedelta(i) for i in range(20)],
+ [1.0],
+ )
+ )
+ df = DataFrame(data)
+ table = df.pivot_table(values=4, index=[0, 1, 3], columns=[2])
+
+ df2 = df.rename(columns=str)
+ table2 = df2.pivot_table(values="4", index=["0", "1", "3"], columns=["2"])
+
+ tm.assert_frame_equal(table, table2, check_names=False)
+
+ def test_pivot_no_level_overlap(self):
+ # GH #1181
+
+ data = DataFrame(
+ {
+ "a": ["a", "a", "a", "a", "b", "b", "b", "b"] * 2,
+ "b": [0, 0, 0, 0, 1, 1, 1, 1] * 2,
+ "c": (["foo"] * 4 + ["bar"] * 4) * 2,
+ "value": np.random.default_rng(2).standard_normal(16),
+ }
+ )
+
+ table = data.pivot_table("value", index="a", columns=["b", "c"])
+
+ grouped = data.groupby(["a", "b", "c"])["value"].mean()
+ expected = grouped.unstack("b").unstack("c").dropna(axis=1, how="all")
+ tm.assert_frame_equal(table, expected)
+
+ def test_pivot_columns_lexsorted(self):
+ n = 10000
+
+ dtype = np.dtype(
+ [
+ ("Index", object),
+ ("Symbol", object),
+ ("Year", int),
+ ("Month", int),
+ ("Day", int),
+ ("Quantity", int),
+ ("Price", float),
+ ]
+ )
+
+ products = np.array(
+ [
+ ("SP500", "ADBE"),
+ ("SP500", "NVDA"),
+ ("SP500", "ORCL"),
+ ("NDQ100", "AAPL"),
+ ("NDQ100", "MSFT"),
+ ("NDQ100", "GOOG"),
+ ("FTSE", "DGE.L"),
+ ("FTSE", "TSCO.L"),
+ ("FTSE", "GSK.L"),
+ ],
+ dtype=[("Index", object), ("Symbol", object)],
+ )
+ items = np.empty(n, dtype=dtype)
+ iproduct = np.random.default_rng(2).integers(0, len(products), n)
+ items["Index"] = products["Index"][iproduct]
+ items["Symbol"] = products["Symbol"][iproduct]
+ dr = date_range(date(2000, 1, 1), date(2010, 12, 31))
+ dates = dr[np.random.default_rng(2).integers(0, len(dr), n)]
+ items["Year"] = dates.year
+ items["Month"] = dates.month
+ items["Day"] = dates.day
+ items["Price"] = np.random.default_rng(2).lognormal(4.0, 2.0, n)
+
+ df = DataFrame(items)
+
+ pivoted = df.pivot_table(
+ "Price",
+ index=["Month", "Day"],
+ columns=["Index", "Symbol", "Year"],
+ aggfunc="mean",
+ )
+
+ assert pivoted.columns.is_monotonic_increasing
+
+ def test_pivot_complex_aggfunc(self, data):
+ f = {"D": ["std"], "E": ["sum"]}
+ expected = data.groupby(["A", "B"]).agg(f).unstack("B")
+ result = data.pivot_table(index="A", columns="B", aggfunc=f)
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_margins_no_values_no_cols(self, data):
+ # Regression test on pivot table: no values or cols passed.
+ result = data[["A", "B"]].pivot_table(
+ index=["A", "B"], aggfunc=len, margins=True
+ )
+ result_list = result.tolist()
+ assert sum(result_list[:-1]) == result_list[-1]
+
+ def test_margins_no_values_two_rows(self, data):
+ # Regression test on pivot table: no values passed but rows are a
+ # multi-index
+ result = data[["A", "B", "C"]].pivot_table(
+ index=["A", "B"], columns="C", aggfunc=len, margins=True
+ )
+ assert result.All.tolist() == [3.0, 1.0, 4.0, 3.0, 11.0]
+
+ def test_margins_no_values_one_row_one_col(self, data):
+ # Regression test on pivot table: no values passed but row and col
+ # defined
+ result = data[["A", "B"]].pivot_table(
+ index="A", columns="B", aggfunc=len, margins=True
+ )
+ assert result.All.tolist() == [4.0, 7.0, 11.0]
+
+ def test_margins_no_values_two_row_two_cols(self, data):
+ # Regression test on pivot table: no values passed but rows and cols
+ # are multi-indexed
+ data["D"] = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"]
+ result = data[["A", "B", "C", "D"]].pivot_table(
+ index=["A", "B"], columns=["C", "D"], aggfunc=len, margins=True
+ )
+ assert result.All.tolist() == [3.0, 1.0, 4.0, 3.0, 11.0]
+
+ @pytest.mark.parametrize("margin_name", ["foo", "one", 666, None, ["a", "b"]])
+ def test_pivot_table_with_margins_set_margin_name(self, margin_name, data):
+ # see gh-3335
+ msg = (
+ f'Conflicting name "{margin_name}" in margins|'
+ "margins_name argument must be a string"
+ )
+ with pytest.raises(ValueError, match=msg):
+ # multi-index index
+ pivot_table(
+ data,
+ values="D",
+ index=["A", "B"],
+ columns=["C"],
+ margins=True,
+ margins_name=margin_name,
+ )
+ with pytest.raises(ValueError, match=msg):
+ # multi-index column
+ pivot_table(
+ data,
+ values="D",
+ index=["C"],
+ columns=["A", "B"],
+ margins=True,
+ margins_name=margin_name,
+ )
+ with pytest.raises(ValueError, match=msg):
+ # non-multi-index index/column
+ pivot_table(
+ data,
+ values="D",
+ index=["A"],
+ columns=["B"],
+ margins=True,
+ margins_name=margin_name,
+ )
+
+ def test_pivot_timegrouper(self):
+ df = DataFrame(
+ {
+ "Branch": "A A A A A A A B".split(),
+ "Buyer": "Carl Mark Carl Carl Joe Joe Joe Carl".split(),
+ "Quantity": [1, 3, 5, 1, 8, 1, 9, 3],
+ "Date": [
+ datetime(2013, 1, 1),
+ datetime(2013, 1, 1),
+ datetime(2013, 10, 1),
+ datetime(2013, 10, 2),
+ datetime(2013, 10, 1),
+ datetime(2013, 10, 2),
+ datetime(2013, 12, 2),
+ datetime(2013, 12, 2),
+ ],
+ }
+ ).set_index("Date")
+
+ expected = DataFrame(
+ np.array([10, 18, 3], dtype="int64").reshape(1, 3),
+ index=pd.DatetimeIndex([datetime(2013, 12, 31)], freq="YE"),
+ columns="Carl Joe Mark".split(),
+ )
+ expected.index.name = "Date"
+ expected.columns.name = "Buyer"
+
+ result = pivot_table(
+ df,
+ index=Grouper(freq="YE"),
+ columns="Buyer",
+ values="Quantity",
+ aggfunc="sum",
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = pivot_table(
+ df,
+ index="Buyer",
+ columns=Grouper(freq="YE"),
+ values="Quantity",
+ aggfunc="sum",
+ )
+ tm.assert_frame_equal(result, expected.T)
+
+ expected = DataFrame(
+ np.array([1, np.nan, 3, 9, 18, np.nan]).reshape(2, 3),
+ index=pd.DatetimeIndex(
+ [datetime(2013, 1, 1), datetime(2013, 7, 1)], freq="6MS"
+ ),
+ columns="Carl Joe Mark".split(),
+ )
+ expected.index.name = "Date"
+ expected.columns.name = "Buyer"
+
+ result = pivot_table(
+ df,
+ index=Grouper(freq="6MS"),
+ columns="Buyer",
+ values="Quantity",
+ aggfunc="sum",
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = pivot_table(
+ df,
+ index="Buyer",
+ columns=Grouper(freq="6MS"),
+ values="Quantity",
+ aggfunc="sum",
+ )
+ tm.assert_frame_equal(result, expected.T)
+
+ # passing the name
+ df = df.reset_index()
+ result = pivot_table(
+ df,
+ index=Grouper(freq="6MS", key="Date"),
+ columns="Buyer",
+ values="Quantity",
+ aggfunc="sum",
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = pivot_table(
+ df,
+ index="Buyer",
+ columns=Grouper(freq="6MS", key="Date"),
+ values="Quantity",
+ aggfunc="sum",
+ )
+ tm.assert_frame_equal(result, expected.T)
+
+ msg = "'The grouper name foo is not found'"
+ with pytest.raises(KeyError, match=msg):
+ pivot_table(
+ df,
+ index=Grouper(freq="6MS", key="foo"),
+ columns="Buyer",
+ values="Quantity",
+ aggfunc="sum",
+ )
+ with pytest.raises(KeyError, match=msg):
+ pivot_table(
+ df,
+ index="Buyer",
+ columns=Grouper(freq="6MS", key="foo"),
+ values="Quantity",
+ aggfunc="sum",
+ )
+
+ # passing the level
+ df = df.set_index("Date")
+ result = pivot_table(
+ df,
+ index=Grouper(freq="6MS", level="Date"),
+ columns="Buyer",
+ values="Quantity",
+ aggfunc="sum",
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = pivot_table(
+ df,
+ index="Buyer",
+ columns=Grouper(freq="6MS", level="Date"),
+ values="Quantity",
+ aggfunc="sum",
+ )
+ tm.assert_frame_equal(result, expected.T)
+
+ msg = "The level foo is not valid"
+ with pytest.raises(ValueError, match=msg):
+ pivot_table(
+ df,
+ index=Grouper(freq="6MS", level="foo"),
+ columns="Buyer",
+ values="Quantity",
+ aggfunc="sum",
+ )
+ with pytest.raises(ValueError, match=msg):
+ pivot_table(
+ df,
+ index="Buyer",
+ columns=Grouper(freq="6MS", level="foo"),
+ values="Quantity",
+ aggfunc="sum",
+ )
+
+ def test_pivot_timegrouper_double(self):
+ # double grouper
+ df = DataFrame(
+ {
+ "Branch": "A A A A A A A B".split(),
+ "Buyer": "Carl Mark Carl Carl Joe Joe Joe Carl".split(),
+ "Quantity": [1, 3, 5, 1, 8, 1, 9, 3],
+ "Date": [
+ datetime(2013, 11, 1, 13, 0),
+ datetime(2013, 9, 1, 13, 5),
+ datetime(2013, 10, 1, 20, 0),
+ datetime(2013, 10, 2, 10, 0),
+ datetime(2013, 11, 1, 20, 0),
+ datetime(2013, 10, 2, 10, 0),
+ datetime(2013, 10, 2, 12, 0),
+ datetime(2013, 12, 5, 14, 0),
+ ],
+ "PayDay": [
+ datetime(2013, 10, 4, 0, 0),
+ datetime(2013, 10, 15, 13, 5),
+ datetime(2013, 9, 5, 20, 0),
+ datetime(2013, 11, 2, 10, 0),
+ datetime(2013, 10, 7, 20, 0),
+ datetime(2013, 9, 5, 10, 0),
+ datetime(2013, 12, 30, 12, 0),
+ datetime(2013, 11, 20, 14, 0),
+ ],
+ }
+ )
+
+ result = pivot_table(
+ df,
+ index=Grouper(freq="ME", key="Date"),
+ columns=Grouper(freq="ME", key="PayDay"),
+ values="Quantity",
+ aggfunc="sum",
+ )
+ expected = DataFrame(
+ np.array(
+ [
+ np.nan,
+ 3,
+ np.nan,
+ np.nan,
+ 6,
+ np.nan,
+ 1,
+ 9,
+ np.nan,
+ 9,
+ np.nan,
+ np.nan,
+ np.nan,
+ np.nan,
+ 3,
+ np.nan,
+ ]
+ ).reshape(4, 4),
+ index=pd.DatetimeIndex(
+ [
+ datetime(2013, 9, 30),
+ datetime(2013, 10, 31),
+ datetime(2013, 11, 30),
+ datetime(2013, 12, 31),
+ ],
+ freq="ME",
+ ),
+ columns=pd.DatetimeIndex(
+ [
+ datetime(2013, 9, 30),
+ datetime(2013, 10, 31),
+ datetime(2013, 11, 30),
+ datetime(2013, 12, 31),
+ ],
+ freq="ME",
+ ),
+ )
+ expected.index.name = "Date"
+ expected.columns.name = "PayDay"
+
+ tm.assert_frame_equal(result, expected)
+
+ result = pivot_table(
+ df,
+ index=Grouper(freq="ME", key="PayDay"),
+ columns=Grouper(freq="ME", key="Date"),
+ values="Quantity",
+ aggfunc="sum",
+ )
+ tm.assert_frame_equal(result, expected.T)
+
+ tuples = [
+ (datetime(2013, 9, 30), datetime(2013, 10, 31)),
+ (datetime(2013, 10, 31), datetime(2013, 9, 30)),
+ (datetime(2013, 10, 31), datetime(2013, 11, 30)),
+ (datetime(2013, 10, 31), datetime(2013, 12, 31)),
+ (datetime(2013, 11, 30), datetime(2013, 10, 31)),
+ (datetime(2013, 12, 31), datetime(2013, 11, 30)),
+ ]
+ idx = MultiIndex.from_tuples(tuples, names=["Date", "PayDay"])
+ expected = DataFrame(
+ np.array(
+ [3, np.nan, 6, np.nan, 1, np.nan, 9, np.nan, 9, np.nan, np.nan, 3]
+ ).reshape(6, 2),
+ index=idx,
+ columns=["A", "B"],
+ )
+ expected.columns.name = "Branch"
+
+ result = pivot_table(
+ df,
+ index=[Grouper(freq="ME", key="Date"), Grouper(freq="ME", key="PayDay")],
+ columns=["Branch"],
+ values="Quantity",
+ aggfunc="sum",
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = pivot_table(
+ df,
+ index=["Branch"],
+ columns=[Grouper(freq="ME", key="Date"), Grouper(freq="ME", key="PayDay")],
+ values="Quantity",
+ aggfunc="sum",
+ )
+ tm.assert_frame_equal(result, expected.T)
+
+ def test_pivot_datetime_tz(self):
+ dates1 = pd.DatetimeIndex(
+ [
+ "2011-07-19 07:00:00",
+ "2011-07-19 08:00:00",
+ "2011-07-19 09:00:00",
+ "2011-07-19 07:00:00",
+ "2011-07-19 08:00:00",
+ "2011-07-19 09:00:00",
+ ],
+ dtype="M8[ns, US/Pacific]",
+ name="dt1",
+ )
+ dates2 = pd.DatetimeIndex(
+ [
+ "2013-01-01 15:00:00",
+ "2013-01-01 15:00:00",
+ "2013-01-01 15:00:00",
+ "2013-02-01 15:00:00",
+ "2013-02-01 15:00:00",
+ "2013-02-01 15:00:00",
+ ],
+ dtype="M8[ns, Asia/Tokyo]",
+ )
+ df = DataFrame(
+ {
+ "label": ["a", "a", "a", "b", "b", "b"],
+ "dt1": dates1,
+ "dt2": dates2,
+ "value1": np.arange(6, dtype="int64"),
+ "value2": [1, 2] * 3,
+ }
+ )
+
+ exp_idx = dates1[:3]
+ exp_col1 = Index(["value1", "value1"])
+ exp_col2 = Index(["a", "b"], name="label")
+ exp_col = MultiIndex.from_arrays([exp_col1, exp_col2])
+ expected = DataFrame(
+ [[0.0, 3.0], [1.0, 4.0], [2.0, 5.0]], index=exp_idx, columns=exp_col
+ )
+ result = pivot_table(df, index=["dt1"], columns=["label"], values=["value1"])
+ tm.assert_frame_equal(result, expected)
+
+ exp_col1 = Index(["sum", "sum", "sum", "sum", "mean", "mean", "mean", "mean"])
+ exp_col2 = Index(["value1", "value1", "value2", "value2"] * 2)
+ exp_col3 = pd.DatetimeIndex(
+ ["2013-01-01 15:00:00", "2013-02-01 15:00:00"] * 4,
+ dtype="M8[ns, Asia/Tokyo]",
+ name="dt2",
+ )
+ exp_col = MultiIndex.from_arrays([exp_col1, exp_col2, exp_col3])
+ expected1 = DataFrame(
+ np.array(
+ [
+ [
+ 0,
+ 3,
+ 1,
+ 2,
+ ],
+ [1, 4, 2, 1],
+ [2, 5, 1, 2],
+ ],
+ dtype="int64",
+ ),
+ index=exp_idx,
+ columns=exp_col[:4],
+ )
+ expected2 = DataFrame(
+ np.array(
+ [
+ [0.0, 3.0, 1.0, 2.0],
+ [1.0, 4.0, 2.0, 1.0],
+ [2.0, 5.0, 1.0, 2.0],
+ ],
+ ),
+ index=exp_idx,
+ columns=exp_col[4:],
+ )
+ expected = concat([expected1, expected2], axis=1)
+
+ result = pivot_table(
+ df,
+ index=["dt1"],
+ columns=["dt2"],
+ values=["value1", "value2"],
+ aggfunc=["sum", "mean"],
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_dtaccessor(self):
+ # GH 8103
+ dates1 = pd.DatetimeIndex(
+ [
+ "2011-07-19 07:00:00",
+ "2011-07-19 08:00:00",
+ "2011-07-19 09:00:00",
+ "2011-07-19 07:00:00",
+ "2011-07-19 08:00:00",
+ "2011-07-19 09:00:00",
+ ]
+ )
+ dates2 = pd.DatetimeIndex(
+ [
+ "2013-01-01 15:00:00",
+ "2013-01-01 15:00:00",
+ "2013-01-01 15:00:00",
+ "2013-02-01 15:00:00",
+ "2013-02-01 15:00:00",
+ "2013-02-01 15:00:00",
+ ]
+ )
+ df = DataFrame(
+ {
+ "label": ["a", "a", "a", "b", "b", "b"],
+ "dt1": dates1,
+ "dt2": dates2,
+ "value1": np.arange(6, dtype="int64"),
+ "value2": [1, 2] * 3,
+ }
+ )
+
+ result = pivot_table(
+ df, index="label", columns=df["dt1"].dt.hour, values="value1"
+ )
+
+ exp_idx = Index(["a", "b"], name="label")
+ expected = DataFrame(
+ {7: [0.0, 3.0], 8: [1.0, 4.0], 9: [2.0, 5.0]},
+ index=exp_idx,
+ columns=Index([7, 8, 9], dtype=np.int32, name="dt1"),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = pivot_table(
+ df, index=df["dt2"].dt.month, columns=df["dt1"].dt.hour, values="value1"
+ )
+
+ expected = DataFrame(
+ {7: [0.0, 3.0], 8: [1.0, 4.0], 9: [2.0, 5.0]},
+ index=Index([1, 2], dtype=np.int32, name="dt2"),
+ columns=Index([7, 8, 9], dtype=np.int32, name="dt1"),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = pivot_table(
+ df,
+ index=df["dt2"].dt.year.values,
+ columns=[df["dt1"].dt.hour, df["dt2"].dt.month],
+ values="value1",
+ )
+
+ exp_col = MultiIndex.from_arrays(
+ [
+ np.array([7, 7, 8, 8, 9, 9], dtype=np.int32),
+ np.array([1, 2] * 3, dtype=np.int32),
+ ],
+ names=["dt1", "dt2"],
+ )
+ expected = DataFrame(
+ np.array([[0.0, 3.0, 1.0, 4.0, 2.0, 5.0]]),
+ index=Index([2013], dtype=np.int32),
+ columns=exp_col,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = pivot_table(
+ df,
+ index=np.array(["X", "X", "X", "X", "Y", "Y"]),
+ columns=[df["dt1"].dt.hour, df["dt2"].dt.month],
+ values="value1",
+ )
+ expected = DataFrame(
+ np.array(
+ [[0, 3, 1, np.nan, 2, np.nan], [np.nan, np.nan, np.nan, 4, np.nan, 5]]
+ ),
+ index=["X", "Y"],
+ columns=exp_col,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_daily(self):
+ rng = date_range("1/1/2000", "12/31/2004", freq="D")
+ ts = Series(np.arange(len(rng)), index=rng)
+
+ result = pivot_table(
+ DataFrame(ts), index=ts.index.year, columns=ts.index.dayofyear
+ )
+ result.columns = result.columns.droplevel(0)
+
+ doy = np.asarray(ts.index.dayofyear)
+
+ expected = {}
+ for y in ts.index.year.unique().values:
+ mask = ts.index.year == y
+ expected[y] = Series(ts.values[mask], index=doy[mask])
+ expected = DataFrame(expected, dtype=float).T
+ expected.index = expected.index.astype(np.int32)
+ tm.assert_frame_equal(result, expected)
+
+ def test_monthly(self):
+ rng = date_range("1/1/2000", "12/31/2004", freq="ME")
+ ts = Series(np.arange(len(rng)), index=rng)
+
+ result = pivot_table(DataFrame(ts), index=ts.index.year, columns=ts.index.month)
+ result.columns = result.columns.droplevel(0)
+
+ month = np.asarray(ts.index.month)
+ expected = {}
+ for y in ts.index.year.unique().values:
+ mask = ts.index.year == y
+ expected[y] = Series(ts.values[mask], index=month[mask])
+ expected = DataFrame(expected, dtype=float).T
+ expected.index = expected.index.astype(np.int32)
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_with_iterator_values(self, data):
+ # GH 12017
+ aggs = {"D": "sum", "E": "mean"}
+
+ pivot_values_list = pivot_table(
+ data, index=["A"], values=list(aggs.keys()), aggfunc=aggs
+ )
+
+ pivot_values_keys = pivot_table(
+ data, index=["A"], values=aggs.keys(), aggfunc=aggs
+ )
+ tm.assert_frame_equal(pivot_values_keys, pivot_values_list)
+
+ agg_values_gen = (value for value in aggs)
+ pivot_values_gen = pivot_table(
+ data, index=["A"], values=agg_values_gen, aggfunc=aggs
+ )
+ tm.assert_frame_equal(pivot_values_gen, pivot_values_list)
+
+ def test_pivot_table_margins_name_with_aggfunc_list(self):
+ # GH 13354
+ margins_name = "Weekly"
+ costs = DataFrame(
+ {
+ "item": ["bacon", "cheese", "bacon", "cheese"],
+ "cost": [2.5, 4.5, 3.2, 3.3],
+ "day": ["ME", "ME", "T", "T"],
+ }
+ )
+ table = costs.pivot_table(
+ index="item",
+ columns="day",
+ margins=True,
+ margins_name=margins_name,
+ aggfunc=["mean", "max"],
+ )
+ ix = Index(["bacon", "cheese", margins_name], name="item")
+ tups = [
+ ("mean", "cost", "ME"),
+ ("mean", "cost", "T"),
+ ("mean", "cost", margins_name),
+ ("max", "cost", "ME"),
+ ("max", "cost", "T"),
+ ("max", "cost", margins_name),
+ ]
+ cols = MultiIndex.from_tuples(tups, names=[None, None, "day"])
+ expected = DataFrame(table.values, index=ix, columns=cols)
+ tm.assert_frame_equal(table, expected)
+
+ def test_categorical_margins(self, observed):
+ # GH 10989
+ df = DataFrame(
+ {"x": np.arange(8), "y": np.arange(8) // 4, "z": np.arange(8) % 2}
+ )
+
+ expected = DataFrame([[1.0, 2.0, 1.5], [5, 6, 5.5], [3, 4, 3.5]])
+ expected.index = Index([0, 1, "All"], name="y")
+ expected.columns = Index([0, 1, "All"], name="z")
+
+ table = df.pivot_table("x", "y", "z", dropna=observed, margins=True)
+ tm.assert_frame_equal(table, expected)
+
+ def test_categorical_margins_category(self, observed):
+ df = DataFrame(
+ {"x": np.arange(8), "y": np.arange(8) // 4, "z": np.arange(8) % 2}
+ )
+
+ expected = DataFrame([[1.0, 2.0, 1.5], [5, 6, 5.5], [3, 4, 3.5]])
+ expected.index = Index([0, 1, "All"], name="y")
+ expected.columns = Index([0, 1, "All"], name="z")
+
+ df.y = df.y.astype("category")
+ df.z = df.z.astype("category")
+ table = df.pivot_table(
+ "x", "y", "z", dropna=observed, margins=True, observed=False
+ )
+ tm.assert_frame_equal(table, expected)
+
+ def test_margins_casted_to_float(self):
+ # GH 24893
+ df = DataFrame(
+ {
+ "A": [2, 4, 6, 8],
+ "B": [1, 4, 5, 8],
+ "C": [1, 3, 4, 6],
+ "D": ["X", "X", "Y", "Y"],
+ }
+ )
+
+ result = pivot_table(df, index="D", margins=True)
+ expected = DataFrame(
+ {"A": [3.0, 7.0, 5], "B": [2.5, 6.5, 4.5], "C": [2.0, 5.0, 3.5]},
+ index=Index(["X", "Y", "All"], name="D"),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_with_categorical(self, observed, ordered):
+ # gh-21370
+ idx = [np.nan, "low", "high", "low", np.nan]
+ col = [np.nan, "A", "B", np.nan, "A"]
+ df = DataFrame(
+ {
+ "In": Categorical(idx, categories=["low", "high"], ordered=ordered),
+ "Col": Categorical(col, categories=["A", "B"], ordered=ordered),
+ "Val": range(1, 6),
+ }
+ )
+ # case with index/columns/value
+ result = df.pivot_table(
+ index="In", columns="Col", values="Val", observed=observed
+ )
+
+ expected_cols = pd.CategoricalIndex(["A", "B"], ordered=ordered, name="Col")
+
+ expected = DataFrame(data=[[2.0, np.nan], [np.nan, 3.0]], columns=expected_cols)
+ expected.index = Index(
+ Categorical(["low", "high"], categories=["low", "high"], ordered=ordered),
+ name="In",
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ # case with columns/value
+ result = df.pivot_table(columns="Col", values="Val", observed=observed)
+
+ expected = DataFrame(
+ data=[[3.5, 3.0]], columns=expected_cols, index=Index(["Val"])
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_categorical_aggfunc(self, observed):
+ # GH 9534
+ df = DataFrame(
+ {"C1": ["A", "B", "C", "C"], "C2": ["a", "a", "b", "b"], "V": [1, 2, 3, 4]}
+ )
+ df["C1"] = df["C1"].astype("category")
+ result = df.pivot_table(
+ "V",
+ index="C1",
+ columns="C2",
+ dropna=observed,
+ aggfunc="count",
+ observed=False,
+ )
+
+ expected_index = pd.CategoricalIndex(
+ ["A", "B", "C"], categories=["A", "B", "C"], ordered=False, name="C1"
+ )
+ expected_columns = Index(["a", "b"], name="C2")
+ expected_data = np.array([[1, 0], [1, 0], [0, 2]], dtype=np.int64)
+ expected = DataFrame(
+ expected_data, index=expected_index, columns=expected_columns
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_categorical_pivot_index_ordering(self, observed):
+ # GH 8731
+ df = DataFrame(
+ {
+ "Sales": [100, 120, 220],
+ "Month": ["January", "January", "January"],
+ "Year": [2013, 2014, 2013],
+ }
+ )
+ months = [
+ "January",
+ "February",
+ "March",
+ "April",
+ "May",
+ "June",
+ "July",
+ "August",
+ "September",
+ "October",
+ "November",
+ "December",
+ ]
+ df["Month"] = df["Month"].astype("category").cat.set_categories(months)
+ result = df.pivot_table(
+ values="Sales",
+ index="Month",
+ columns="Year",
+ observed=observed,
+ aggfunc="sum",
+ )
+ expected_columns = Index([2013, 2014], name="Year", dtype="int64")
+ expected_index = pd.CategoricalIndex(
+ months, categories=months, ordered=False, name="Month"
+ )
+ expected_data = [[320, 120]] + [[0, 0]] * 11
+ expected = DataFrame(
+ expected_data, index=expected_index, columns=expected_columns
+ )
+ if observed:
+ expected = expected.loc[["January"]]
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_not_series(self):
+ # GH 4386
+ # pivot_table always returns a DataFrame
+ # when values is not list like and columns is None
+ # and aggfunc is not instance of list
+ df = DataFrame({"col1": [3, 4, 5], "col2": ["C", "D", "E"], "col3": [1, 3, 9]})
+
+ result = df.pivot_table("col1", index=["col3", "col2"], aggfunc="sum")
+ m = MultiIndex.from_arrays([[1, 3, 9], ["C", "D", "E"]], names=["col3", "col2"])
+ expected = DataFrame([3, 4, 5], index=m, columns=["col1"])
+
+ tm.assert_frame_equal(result, expected)
+
+ result = df.pivot_table("col1", index="col3", columns="col2", aggfunc="sum")
+ expected = DataFrame(
+ [[3, np.nan, np.nan], [np.nan, 4, np.nan], [np.nan, np.nan, 5]],
+ index=Index([1, 3, 9], name="col3"),
+ columns=Index(["C", "D", "E"], name="col2"),
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ result = df.pivot_table("col1", index="col3", aggfunc=["sum"])
+ m = MultiIndex.from_arrays([["sum"], ["col1"]])
+ expected = DataFrame([3, 4, 5], index=Index([1, 3, 9], name="col3"), columns=m)
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_margins_name_unicode(self):
+ # issue #13292
+ greek = "\u0394\u03bf\u03ba\u03b9\u03bc\u03ae"
+ frame = DataFrame({"foo": [1, 2, 3]}, columns=Index(["foo"], dtype=object))
+ table = pivot_table(
+ frame, index=["foo"], aggfunc=len, margins=True, margins_name=greek
+ )
+ index = Index([1, 2, 3, greek], dtype="object", name="foo")
+ expected = DataFrame(index=index, columns=[])
+ tm.assert_frame_equal(table, expected)
+
+ def test_pivot_string_as_func(self):
+ # GH #18713
+ # for correctness purposes
+ data = DataFrame(
+ {
+ "A": [
+ "foo",
+ "foo",
+ "foo",
+ "foo",
+ "bar",
+ "bar",
+ "bar",
+ "bar",
+ "foo",
+ "foo",
+ "foo",
+ ],
+ "B": [
+ "one",
+ "one",
+ "one",
+ "two",
+ "one",
+ "one",
+ "one",
+ "two",
+ "two",
+ "two",
+ "one",
+ ],
+ "C": range(11),
+ }
+ )
+
+ result = pivot_table(data, index="A", columns="B", aggfunc="sum")
+ mi = MultiIndex(
+ levels=[["C"], ["one", "two"]], codes=[[0, 0], [0, 1]], names=[None, "B"]
+ )
+ expected = DataFrame(
+ {("C", "one"): {"bar": 15, "foo": 13}, ("C", "two"): {"bar": 7, "foo": 20}},
+ columns=mi,
+ ).rename_axis("A")
+ tm.assert_frame_equal(result, expected)
+
+ result = pivot_table(data, index="A", columns="B", aggfunc=["sum", "mean"])
+ mi = MultiIndex(
+ levels=[["sum", "mean"], ["C"], ["one", "two"]],
+ codes=[[0, 0, 1, 1], [0, 0, 0, 0], [0, 1, 0, 1]],
+ names=[None, None, "B"],
+ )
+ expected = DataFrame(
+ {
+ ("mean", "C", "one"): {"bar": 5.0, "foo": 3.25},
+ ("mean", "C", "two"): {"bar": 7.0, "foo": 6.666666666666667},
+ ("sum", "C", "one"): {"bar": 15, "foo": 13},
+ ("sum", "C", "two"): {"bar": 7, "foo": 20},
+ },
+ columns=mi,
+ ).rename_axis("A")
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("kwargs", [{"a": 2}, {"a": 2, "b": 3}, {"b": 3, "a": 2}])
+ def test_pivot_table_kwargs(self, kwargs):
+ # GH#57884
+ def f(x, a, b=3):
+ return x.sum() * a + b
+
+ def g(x):
+ return f(x, **kwargs)
+
+ df = DataFrame(
+ {
+ "A": ["good", "bad", "good", "bad", "good"],
+ "B": ["one", "two", "one", "three", "two"],
+ "X": [2, 5, 4, 20, 10],
+ }
+ )
+ result = pivot_table(
+ df, index="A", columns="B", values="X", aggfunc=f, **kwargs
+ )
+ expected = pivot_table(df, index="A", columns="B", values="X", aggfunc=g)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "kwargs", [{}, {"b": 10}, {"a": 3}, {"a": 3, "b": 10}, {"b": 10, "a": 3}]
+ )
+ def test_pivot_table_kwargs_margin(self, data, kwargs):
+ # GH#57884
+ def f(x, a=5, b=7):
+ return (x.sum() + b) * a
+
+ def g(x):
+ return f(x, **kwargs)
+
+ result = data.pivot_table(
+ values="D",
+ index=["A", "B"],
+ columns="C",
+ aggfunc=f,
+ margins=True,
+ fill_value=0,
+ **kwargs,
+ )
+
+ expected = data.pivot_table(
+ values="D",
+ index=["A", "B"],
+ columns="C",
+ aggfunc=g,
+ margins=True,
+ fill_value=0,
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "f, f_numpy",
+ [
+ ("sum", np.sum),
+ ("mean", np.mean),
+ ("min", np.min),
+ (["sum", "mean"], [np.sum, np.mean]),
+ (["sum", "min"], [np.sum, np.min]),
+ (["max", "mean"], [np.max, np.mean]),
+ ],
+ )
+ def test_pivot_string_func_vs_func(self, f, f_numpy, data):
+ # GH #18713
+ # for consistency purposes
+ data = data.drop(columns="C")
+ result = pivot_table(data, index="A", columns="B", aggfunc=f)
+ expected = pivot_table(data, index="A", columns="B", aggfunc=f_numpy)
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.slow
+ def test_pivot_number_of_levels_larger_than_int32_warns(
+ self, performance_warning, monkeypatch
+ ):
+ # GH 20601
+ # GH 26314: Change ValueError to PerformanceWarning
+ class MockUnstacker(reshape_lib._Unstacker):
+ def __init__(self, *args, **kwargs) -> None:
+ # __init__ will raise the warning
+ super().__init__(*args, **kwargs)
+ raise Exception("Don't compute final result.")
+
+ def _make_selectors(self) -> None:
+ pass
+
+ with monkeypatch.context() as m:
+ m.setattr(reshape_lib, "_Unstacker", MockUnstacker)
+ df = DataFrame(
+ {"ind1": np.arange(2**16), "ind2": np.arange(2**16), "count": 0}
+ )
+
+ msg = "The following operation may generate"
+ with tm.assert_produces_warning(performance_warning, match=msg):
+ with pytest.raises(Exception, match="Don't compute final result."):
+ df.pivot_table(
+ index="ind1", columns="ind2", values="count", aggfunc="count"
+ )
+
+ def test_pivot_table_aggfunc_dropna(self, dropna):
+ # GH 22159
+ df = DataFrame(
+ {
+ "fruit": ["apple", "peach", "apple"],
+ "size": [1, 1, 2],
+ "taste": [7, 6, 6],
+ }
+ )
+
+ def ret_one(x):
+ return 1
+
+ def ret_sum(x):
+ return sum(x)
+
+ def ret_none(x):
+ return np.nan
+
+ result = pivot_table(
+ df, columns="fruit", aggfunc=[ret_sum, ret_none, ret_one], dropna=dropna
+ )
+
+ data = [[3, 1, np.nan, np.nan, 1, 1], [13, 6, np.nan, np.nan, 1, 1]]
+ col = MultiIndex.from_product(
+ [["ret_sum", "ret_none", "ret_one"], ["apple", "peach"]],
+ names=[None, "fruit"],
+ )
+ expected = DataFrame(data, index=["size", "taste"], columns=col)
+
+ if dropna:
+ expected = expected.dropna(axis="columns")
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_aggfunc_scalar_dropna(self, dropna):
+ # GH 22159
+ df = DataFrame(
+ {"A": ["one", "two", "one"], "x": [3, np.nan, 2], "y": [1, np.nan, np.nan]}
+ )
+
+ result = pivot_table(df, columns="A", aggfunc="mean", dropna=dropna)
+
+ data = [[2.5, np.nan], [1, np.nan]]
+ col = Index(["one", "two"], name="A")
+ expected = DataFrame(data, index=["x", "y"], columns=col)
+
+ if dropna:
+ expected = expected.dropna(axis="columns")
+
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("margins", [True, False])
+ def test_pivot_table_empty_aggfunc(self, margins):
+ # GH 9186 & GH 13483 & GH 49240
+ df = DataFrame(
+ {
+ "A": [2, 2, 3, 3, 2],
+ "id": [5, 6, 7, 8, 9],
+ "C": ["p", "q", "q", "p", "q"],
+ "D": [None, None, None, None, None],
+ }
+ )
+ result = df.pivot_table(
+ index="A", columns="D", values="id", aggfunc=np.size, margins=margins
+ )
+ exp_cols = Index([], name="D")
+ expected = DataFrame(index=Index([], dtype="int64", name="A"), columns=exp_cols)
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_no_column_raises(self):
+ # GH 10326
+ def agg(arr):
+ return np.mean(arr)
+
+ df = DataFrame({"X": [0, 0, 1, 1], "Y": [0, 1, 0, 1], "Z": [10, 20, 30, 40]})
+ with pytest.raises(KeyError, match="notpresent"):
+ df.pivot_table("notpresent", "X", "Y", aggfunc=agg)
+
+ def test_pivot_table_multiindex_columns_doctest_case(self):
+ # The relevant characteristic is that the call
+ # to maybe_downcast_to_dtype(agged[v], data[v].dtype) in
+ # __internal_pivot_table has `agged[v]` a DataFrame instead of Series,
+ # In this case this is because agged.columns is a MultiIndex and 'v'
+ # is only indexing on its first level.
+ df = DataFrame(
+ {
+ "A": ["foo", "foo", "foo", "foo", "foo", "bar", "bar", "bar", "bar"],
+ "B": ["one", "one", "one", "two", "two", "one", "one", "two", "two"],
+ "C": [
+ "small",
+ "large",
+ "large",
+ "small",
+ "small",
+ "large",
+ "small",
+ "small",
+ "large",
+ ],
+ "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
+ "E": [2, 4, 5, 5, 6, 6, 8, 9, 9],
+ }
+ )
+
+ table = pivot_table(
+ df,
+ values=["D", "E"],
+ index=["A", "C"],
+ aggfunc={"D": "mean", "E": ["min", "max", "mean"]},
+ )
+ cols = MultiIndex.from_tuples(
+ [("D", "mean"), ("E", "max"), ("E", "mean"), ("E", "min")]
+ )
+ index = MultiIndex.from_tuples(
+ [("bar", "large"), ("bar", "small"), ("foo", "large"), ("foo", "small")],
+ names=["A", "C"],
+ )
+ vals = np.array(
+ [
+ [5.5, 9.0, 7.5, 6.0],
+ [5.5, 9.0, 8.5, 8.0],
+ [2.0, 5.0, 4.5, 4.0],
+ [2.33333333, 6.0, 4.33333333, 2.0],
+ ]
+ )
+ expected = DataFrame(vals, columns=cols, index=index)
+ expected[("E", "min")] = expected[("E", "min")].astype(np.int64)
+ expected[("E", "max")] = expected[("E", "max")].astype(np.int64)
+ tm.assert_frame_equal(table, expected)
+
+ def test_pivot_table_sort_false(self):
+ # GH#39143
+ df = DataFrame(
+ {
+ "a": ["d1", "d4", "d3"],
+ "col": ["a", "b", "c"],
+ "num": [23, 21, 34],
+ "year": ["2018", "2018", "2019"],
+ }
+ )
+ result = df.pivot_table(
+ index=["a", "col"], columns="year", values="num", aggfunc="sum", sort=False
+ )
+ expected = DataFrame(
+ [[23, np.nan], [21, np.nan], [np.nan, 34]],
+ columns=Index(["2018", "2019"], name="year"),
+ index=MultiIndex.from_arrays(
+ [["d1", "d4", "d3"], ["a", "b", "c"]], names=["a", "col"]
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_nullable_margins(self):
+ # GH#48681
+ df = DataFrame(
+ {"a": "A", "b": [1, 2], "sales": Series([10, 11], dtype="Int64")}
+ )
+
+ result = df.pivot_table(index="b", columns="a", margins=True, aggfunc="sum")
+ expected = DataFrame(
+ [[10, 10], [11, 11], [21, 21]],
+ index=Index([1, 2, "All"], name="b"),
+ columns=MultiIndex.from_tuples(
+ [("sales", "A"), ("sales", "All")], names=[None, "a"]
+ ),
+ dtype="Int64",
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_sort_false_with_multiple_values(self):
+ df = DataFrame(
+ {
+ "firstname": ["John", "Michael"],
+ "lastname": ["Foo", "Bar"],
+ "height": [173, 182],
+ "age": [47, 33],
+ }
+ )
+ result = df.pivot_table(
+ index=["lastname", "firstname"], values=["height", "age"], sort=False
+ )
+ expected = DataFrame(
+ [[173.0, 47.0], [182.0, 33.0]],
+ columns=["height", "age"],
+ index=MultiIndex.from_tuples(
+ [("Foo", "John"), ("Bar", "Michael")],
+ names=["lastname", "firstname"],
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_with_margins_and_numeric_columns(self):
+ # GH 26568
+ df = DataFrame([["a", "x", 1], ["a", "y", 2], ["b", "y", 3], ["b", "z", 4]])
+ df.columns = [10, 20, 30]
+
+ result = df.pivot_table(
+ index=10, columns=20, values=30, aggfunc="sum", fill_value=0, margins=True
+ )
+
+ expected = DataFrame([[1, 2, 0, 3], [0, 3, 4, 7], [1, 5, 4, 10]])
+ expected.columns = ["x", "y", "z", "All"]
+ expected.index = ["a", "b", "All"]
+ expected.columns.name = 20
+ expected.index.name = 10
+
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize(
+ "dtype,expected_dtype", [("Int64", "Float64"), ("int64", "float64")]
+ )
+ def test_pivot_ea_dtype_dropna(self, dropna, dtype, expected_dtype):
+ # GH#47477
+ # GH#47971
+ df = DataFrame({"x": "a", "y": "b", "age": Series([20, 40], dtype=dtype)})
+ result = df.pivot_table(
+ index="x", columns="y", values="age", aggfunc="mean", dropna=dropna
+ )
+ expected = DataFrame(
+ [[30]],
+ index=Index(["a"], name="x"),
+ columns=Index(["b"], name="y"),
+ dtype=expected_dtype,
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_datetime_warning(self):
+ # GH#48683
+ df = DataFrame(
+ {
+ "a": "A",
+ "b": [1, 2],
+ "date": pd.Timestamp("2019-12-31"),
+ "sales": [10.0, 11],
+ }
+ )
+ with tm.assert_produces_warning(None):
+ result = df.pivot_table(
+ index=["b", "date"], columns="a", margins=True, aggfunc="sum"
+ )
+ expected = DataFrame(
+ [[10.0, 10.0], [11.0, 11.0], [21.0, 21.0]],
+ index=MultiIndex.from_arrays(
+ [
+ Index([1, 2, "All"], name="b"),
+ Index(
+ [pd.Timestamp("2019-12-31"), pd.Timestamp("2019-12-31"), ""],
+ dtype=object,
+ name="date",
+ ),
+ ]
+ ),
+ columns=MultiIndex.from_tuples(
+ [("sales", "A"), ("sales", "All")], names=[None, "a"]
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_with_mixed_nested_tuples(self):
+ # GH 50342
+ df = DataFrame(
+ {
+ "A": ["foo", "foo", "foo", "foo", "foo", "bar", "bar", "bar", "bar"],
+ "B": ["one", "one", "one", "two", "two", "one", "one", "two", "two"],
+ "C": [
+ "small",
+ "large",
+ "large",
+ "small",
+ "small",
+ "large",
+ "small",
+ "small",
+ "large",
+ ],
+ "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
+ "E": [2, 4, 5, 5, 6, 6, 8, 9, 9],
+ ("col5",): [
+ "foo",
+ "foo",
+ "foo",
+ "foo",
+ "foo",
+ "bar",
+ "bar",
+ "bar",
+ "bar",
+ ],
+ ("col6", 6): [
+ "one",
+ "one",
+ "one",
+ "two",
+ "two",
+ "one",
+ "one",
+ "two",
+ "two",
+ ],
+ (7, "seven"): [
+ "small",
+ "large",
+ "large",
+ "small",
+ "small",
+ "large",
+ "small",
+ "small",
+ "large",
+ ],
+ }
+ )
+ result = pivot_table(
+ df, values="D", index=["A", "B"], columns=[(7, "seven")], aggfunc="sum"
+ )
+ expected = DataFrame(
+ [[4.0, 5.0], [7.0, 6.0], [4.0, 1.0], [np.nan, 6.0]],
+ columns=Index(["large", "small"], name=(7, "seven")),
+ index=MultiIndex.from_arrays(
+ [["bar", "bar", "foo", "foo"], ["one", "two"] * 2], names=["A", "B"]
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_aggfunc_nunique_with_different_values(self):
+ test = DataFrame(
+ {
+ "a": range(10),
+ "b": range(10),
+ "c": range(10),
+ "d": range(10),
+ }
+ )
+
+ columnval = MultiIndex.from_arrays(
+ [
+ ["nunique" for i in range(10)],
+ ["c" for i in range(10)],
+ range(10),
+ ],
+ names=(None, None, "b"),
+ )
+ nparr = np.full((10, 10), np.nan)
+ np.fill_diagonal(nparr, 1.0)
+
+ expected = DataFrame(nparr, index=Index(range(10), name="a"), columns=columnval)
+ result = test.pivot_table(
+ index=[
+ "a",
+ ],
+ columns=[
+ "b",
+ ],
+ values=[
+ "c",
+ ],
+ aggfunc=["nunique"],
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_index_and_column_keys_with_nan(self, dropna):
+ # GH#61113
+ data = {"row": [None, *range(4)], "col": [*range(4), None], "val": range(5)}
+ df = DataFrame(data)
+ result = df.pivot_table(values="val", index="row", columns="col", dropna=dropna)
+ e_axis = [*range(4), None]
+ nan = np.nan
+ e_data = [
+ [nan, 1.0, nan, nan, nan],
+ [nan, nan, 2.0, nan, nan],
+ [nan, nan, nan, 3.0, nan],
+ [nan, nan, nan, nan, 4.0],
+ [0.0, nan, nan, nan, nan],
+ ]
+ expected = DataFrame(
+ data=e_data,
+ index=Index(data=e_axis, name="row"),
+ columns=Index(data=e_axis, name="col"),
+ )
+ if dropna:
+ expected = expected.loc[[0, 1, 2], [1, 2, 3]]
+
+ tm.assert_frame_equal(left=result, right=expected)
+
+ @pytest.mark.parametrize(
+ "index, columns, e_data, e_index, e_cols",
+ [
+ (
+ "Category",
+ "Value",
+ [
+ [1.0, np.nan, 1.0, np.nan],
+ [np.nan, 1.0, np.nan, 1.0],
+ ],
+ Index(data=["A", "B"], name="Category"),
+ Index(data=[10, 20, 40, 50], name="Value"),
+ ),
+ (
+ "Value",
+ "Category",
+ [
+ [1.0, np.nan],
+ [np.nan, 1.0],
+ [1.0, np.nan],
+ [np.nan, 1.0],
+ ],
+ Index(data=[10, 20, 40, 50], name="Value"),
+ Index(data=["A", "B"], name="Category"),
+ ),
+ ],
+ ids=["values-and-columns", "values-and-index"],
+ )
+ def test_pivot_table_values_as_two_params(
+ self, index, columns, e_data, e_index, e_cols
+ ):
+ # GH#57876
+ data = {"Category": ["A", "B", "A", "B"], "Value": [10, 20, 40, 50]}
+ df = DataFrame(data)
+ result = df.pivot_table(
+ index=index, columns=columns, values="Value", aggfunc="count"
+ )
+ expected = DataFrame(data=e_data, index=e_index, columns=e_cols)
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_table_margins_include_nan_groups(self):
+ # GH#61509
+ df = DataFrame(
+ {
+ "i": [1, 2, 3],
+ "g1": ["a", "b", "b"],
+ "g2": ["x", None, None],
+ }
+ )
+
+ result = df.pivot_table(
+ index="g1",
+ columns="g2",
+ values="i",
+ aggfunc="count",
+ dropna=False,
+ margins=True,
+ )
+
+ expected = DataFrame(
+ {
+ "x": {"a": 1.0, "b": np.nan, "All": 1.0},
+ np.nan: {"a": np.nan, "b": 2.0, "All": 2.0},
+ "All": {"a": 1.0, "b": 2.0, "All": 3.0},
+ }
+ )
+ expected.index.name = "g1"
+ expected.columns.name = "g2"
+ tm.assert_frame_equal(result, expected, check_dtype=False)
+
+
+class TestPivot:
+ def test_pivot(self):
+ data = {
+ "index": ["A", "B", "C", "C", "B", "A"],
+ "columns": ["One", "One", "One", "Two", "Two", "Two"],
+ "values": [1.0, 2.0, 3.0, 3.0, 2.0, 1.0],
+ }
+
+ frame = DataFrame(data)
+ pivoted = frame.pivot(index="index", columns="columns", values="values")
+
+ expected = DataFrame(
+ {
+ "One": {"A": 1.0, "B": 2.0, "C": 3.0},
+ "Two": {"A": 1.0, "B": 2.0, "C": 3.0},
+ }
+ )
+
+ expected.index.name, expected.columns.name = "index", "columns"
+ tm.assert_frame_equal(pivoted, expected)
+
+ # name tracking
+ assert pivoted.index.name == "index"
+ assert pivoted.columns.name == "columns"
+
+ # don't specify values
+ pivoted = frame.pivot(index="index", columns="columns")
+ assert pivoted.index.name == "index"
+ assert pivoted.columns.names == (None, "columns")
+
+ def test_pivot_duplicates(self):
+ data = DataFrame(
+ {
+ "a": ["bar", "bar", "foo", "foo", "foo"],
+ "b": ["one", "two", "one", "one", "two"],
+ "c": [1.0, 2.0, 3.0, 3.0, 4.0],
+ }
+ )
+ with pytest.raises(ValueError, match="duplicate entries"):
+ data.pivot(index="a", columns="b", values="c")
+
+ def test_pivot_empty(self):
+ df = DataFrame(columns=["a", "b", "c"])
+ result = df.pivot(index="a", columns="b", values="c")
+ expected = DataFrame(index=[], columns=[])
+ tm.assert_frame_equal(result, expected, check_names=False)
+
+ def test_pivot_integer_bug(self, any_string_dtype):
+ df = DataFrame(
+ data=[("A", "1", "A1"), ("B", "2", "B2")], dtype=any_string_dtype
+ )
+
+ result = df.pivot(index=1, columns=0, values=2)
+ expected_columns = Index(["A", "B"], name=0, dtype=any_string_dtype)
+ tm.assert_index_equal(result.columns, expected_columns)
+
+ def test_pivot_index_none(self):
+ # GH#3962
+ data = {
+ "index": ["A", "B", "C", "C", "B", "A"],
+ "columns": ["One", "One", "One", "Two", "Two", "Two"],
+ "values": [1.0, 2.0, 3.0, 3.0, 2.0, 1.0],
+ }
+
+ frame = DataFrame(data).set_index("index")
+ result = frame.pivot(columns="columns", values="values")
+ expected = DataFrame(
+ {
+ "One": {"A": 1.0, "B": 2.0, "C": 3.0},
+ "Two": {"A": 1.0, "B": 2.0, "C": 3.0},
+ }
+ )
+
+ expected.index.name, expected.columns.name = "index", "columns"
+ tm.assert_frame_equal(result, expected)
+
+ # omit values
+ result = frame.pivot(columns="columns")
+
+ expected.columns = MultiIndex.from_tuples(
+ [("values", "One"), ("values", "Two")], names=[None, "columns"]
+ )
+ expected.index.name = "index"
+ tm.assert_frame_equal(result, expected, check_names=False)
+ assert result.index.name == "index"
+ assert result.columns.names == (None, "columns")
+ expected.columns = expected.columns.droplevel(0)
+ result = frame.pivot(columns="columns", values="values")
+
+ expected.columns.name = "columns"
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_index_list_values_none_immutable_args(self):
+ # GH37635
+ df = DataFrame(
+ {
+ "lev1": [1, 1, 1, 2, 2, 2],
+ "lev2": [1, 1, 2, 1, 1, 2],
+ "lev3": [1, 2, 1, 2, 1, 2],
+ "lev4": [1, 2, 3, 4, 5, 6],
+ "values": [0, 1, 2, 3, 4, 5],
+ }
+ )
+ index = ["lev1", "lev2"]
+ columns = ["lev3"]
+ result = df.pivot(index=index, columns=columns)
+
+ expected = DataFrame(
+ np.array(
+ [
+ [1.0, 2.0, 0.0, 1.0],
+ [3.0, np.nan, 2.0, np.nan],
+ [5.0, 4.0, 4.0, 3.0],
+ [np.nan, 6.0, np.nan, 5.0],
+ ]
+ ),
+ index=MultiIndex.from_arrays(
+ [(1, 1, 2, 2), (1, 2, 1, 2)], names=["lev1", "lev2"]
+ ),
+ columns=MultiIndex.from_arrays(
+ [("lev4", "lev4", "values", "values"), (1, 2, 1, 2)],
+ names=[None, "lev3"],
+ ),
+ )
+
+ tm.assert_frame_equal(result, expected)
+
+ assert index == ["lev1", "lev2"]
+ assert columns == ["lev3"]
+
+ def test_pivot_columns_not_given(self):
+ # GH#48293
+ df = DataFrame({"a": [1], "b": 1})
+ with pytest.raises(TypeError, match="missing 1 required keyword-only argument"):
+ df.pivot()
+
+ # this still fails because columns=None gets passed down to unstack as level=None
+ # while at that point None was converted to NaN
+ @pytest.mark.xfail(
+ using_string_dtype(), reason="TODO(infer_string) None is cast to NaN"
+ )
+ def test_pivot_columns_is_none(self):
+ # GH#48293
+ df = DataFrame({None: [1], "b": 2, "c": 3})
+ result = df.pivot(columns=None)
+ expected = DataFrame({("b", 1): [2], ("c", 1): 3})
+ tm.assert_frame_equal(result, expected)
+
+ result = df.pivot(columns=None, index="b")
+ expected = DataFrame({("c", 1): 3}, index=Index([2], name="b"))
+ tm.assert_frame_equal(result, expected)
+
+ result = df.pivot(columns=None, index="b", values="c")
+ expected = DataFrame({1: 3}, index=Index([2], name="b"))
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_index_is_none(self, using_infer_string):
+ # GH#48293
+ df = DataFrame({None: [1], "b": 2, "c": 3})
+
+ result = df.pivot(columns="b", index=None)
+ expected = DataFrame({("c", 2): 3}, index=[1])
+ expected.columns.names = [None, "b"]
+ tm.assert_frame_equal(result, expected)
+
+ result = df.pivot(columns="b", index=None, values="c")
+ expected = DataFrame(3, index=[1], columns=Index([2], name="b"))
+ if using_infer_string:
+ expected.index.name = np.nan
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_values_is_none(self):
+ # GH#48293
+ df = DataFrame({None: [1], "b": 2, "c": 3})
+
+ result = df.pivot(columns="b", index="c", values=None)
+ expected = DataFrame(
+ 1, index=Index([3], name="c"), columns=Index([2], name="b")
+ )
+ tm.assert_frame_equal(result, expected)
+
+ result = df.pivot(columns="b", values=None)
+ expected = DataFrame(1, index=[0], columns=Index([2], name="b"))
+ tm.assert_frame_equal(result, expected)
+
+ def test_pivot_not_changing_index_name(self):
+ # GH#52692
+ df = DataFrame({"one": ["a"], "two": 0, "three": 1})
+ expected = df.copy(deep=True)
+ df.pivot(index="one", columns="two", values="three")
+ tm.assert_frame_equal(df, expected)
+
+ def test_pivot_table_empty_dataframe_correct_index(self):
+ # GH 21932
+ df = DataFrame([], columns=["a", "b", "value"])
+ pivot = df.pivot_table(index="a", columns="b", values="value", aggfunc="count")
+
+ expected = Index([], dtype="object", name="b")
+ tm.assert_index_equal(pivot.columns, expected)
+
+ def test_pivot_table_handles_explicit_datetime_types(self):
+ # GH#43574
+ df = DataFrame(
+ [
+ {"a": "x", "date_str": "2023-01-01", "amount": 1},
+ {"a": "y", "date_str": "2023-01-02", "amount": 2},
+ {"a": "z", "date_str": "2023-01-03", "amount": 3},
+ ]
+ )
+ df["date"] = pd.to_datetime(df["date_str"])
+
+ with tm.assert_produces_warning(False):
+ pivot = df.pivot_table(
+ index=["a", "date"], values=["amount"], aggfunc="sum", margins=True
+ )
+
+ expected = MultiIndex.from_tuples(
+ [
+ ("x", datetime.strptime("2023-01-01 00:00:00", "%Y-%m-%d %H:%M:%S")),
+ ("y", datetime.strptime("2023-01-02 00:00:00", "%Y-%m-%d %H:%M:%S")),
+ ("z", datetime.strptime("2023-01-03 00:00:00", "%Y-%m-%d %H:%M:%S")),
+ ("All", ""),
+ ],
+ names=["a", "date"],
+ )
+ tm.assert_index_equal(pivot.index, expected)
+
+ def test_pivot_table_with_margins_and_numeric_column_names(self):
+ # GH#26568
+ df = DataFrame([["a", "x", 1], ["a", "y", 2], ["b", "y", 3], ["b", "z", 4]])
+
+ result = df.pivot_table(
+ index=0, columns=1, values=2, aggfunc="sum", fill_value=0, margins=True
+ )
+
+ expected = DataFrame(
+ [[1, 2, 0, 3], [0, 3, 4, 7], [1, 5, 4, 10]],
+ columns=Index(["x", "y", "z", "All"], name=1),
+ index=Index(["a", "b", "All"], name=0),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.parametrize("m", [1, 10])
+ def test_unstack_copy(self, m):
+ # GH#56633
+ levels = np.arange(m)
+ index = MultiIndex.from_product([levels] * 2)
+ values = np.arange(m * m * 100).reshape(m * m, 100)
+ df = DataFrame(values, index, np.arange(100))
+ df_orig = df.copy()
+ result = df.unstack(sort=False)
+ result.iloc[0, 0] = -1
+ tm.assert_frame_equal(df, df_orig)
+
+ def test_pivot_empty_with_datetime(self):
+ # GH#59126
+ df = DataFrame(
+ {
+ "timestamp": Series([], dtype=pd.DatetimeTZDtype(tz="UTC")),
+ "category": Series([], dtype=str),
+ "value": Series([], dtype=str),
+ }
+ )
+ df_pivoted = df.pivot_table(
+ index="category", columns="value", values="timestamp"
+ )
+ assert df_pivoted.empty
+
+ def test_pivot_margins_with_none_index(self):
+ # GH#58722
+ df = DataFrame(
+ {
+ "x": [1, 1, 2],
+ "y": [3, 3, 4],
+ "z": [5, 5, 6],
+ "w": [7, 8, 9],
+ }
+ )
+ result = df.pivot_table(
+ index=None,
+ columns=["y", "z"],
+ values="w",
+ margins=True,
+ aggfunc="count",
+ )
+ expected = DataFrame(
+ [[2, 2, 1, 1]],
+ index=["w"],
+ columns=MultiIndex(
+ levels=[[3, 4], [5, 6, "All"]],
+ codes=[[0, 0, 1, 1], [0, 2, 1, 2]],
+ names=["y", "z"],
+ ),
+ )
+ tm.assert_frame_equal(result, expected)
+
+ @pytest.mark.filterwarnings("ignore:Passing a BlockManager:DeprecationWarning")
+ def test_pivot_with_pyarrow_categorical(self):
+ # GH#53051
+ pa = pytest.importorskip("pyarrow")
+
+ df = DataFrame(
+ {"string_column": ["A", "B", "C"], "number_column": [1, 2, 3]}
+ ).astype(
+ {
+ "string_column": ArrowDtype(pa.dictionary(pa.int32(), pa.string())),
+ "number_column": "float[pyarrow]",
+ }
+ )
+
+ df = df.pivot(columns=["string_column"], values=["number_column"])
+
+ multi_index = MultiIndex.from_arrays(
+ [["number_column", "number_column", "number_column"], ["A", "B", "C"]],
+ names=(None, "string_column"),
+ )
+ df_expected = DataFrame(
+ [[1.0, np.nan, np.nan], [np.nan, 2.0, np.nan], [np.nan, np.nan, 3.0]],
+ columns=multi_index,
+ )
+ tm.assert_frame_equal(
+ df, df_expected, check_dtype=False, check_column_type=False
+ )
+
+ @pytest.mark.parametrize("freq", ["D", "M", "Q", "Y"])
+ def test_pivot_empty_dataframe_period_dtype(self, freq):
+ # GH#62705
+
+ dtype = pd.PeriodDtype(freq=freq)
+ df = DataFrame({"index": [], "columns": [], "values": []})
+ df = df.astype({"values": dtype})
+ result = df.pivot(index="index", columns="columns", values="values")
+
+ expected_index = Index([], name="index", dtype="float64")
+ expected_columns = Index([], name="columns", dtype="float64")
+ expected = DataFrame(
+ index=expected_index, columns=expected_columns, dtype=dtype
+ )
+
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_pivot_multilevel.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_pivot_multilevel.py
new file mode 100644
index 0000000000000000000000000000000000000000..af70210b37f3c0396b1dd972f64822001c04dc50
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_pivot_multilevel.py
@@ -0,0 +1,301 @@
+import numpy as np
+import pytest
+
+from pandas._libs import lib
+
+import pandas as pd
+from pandas import (
+ Index,
+ MultiIndex,
+)
+import pandas._testing as tm
+
+
+@pytest.mark.parametrize(
+ "input_index, input_columns, input_values, "
+ "expected_values, expected_columns, expected_index",
+ [
+ (
+ ["lev4"],
+ "lev3",
+ "values",
+ [
+ [0.0, np.nan],
+ [np.nan, 1.0],
+ [2.0, np.nan],
+ [np.nan, 3.0],
+ [4.0, np.nan],
+ [np.nan, 5.0],
+ [6.0, np.nan],
+ [np.nan, 7.0],
+ ],
+ Index([1, 2], name="lev3"),
+ Index([1, 2, 3, 4, 5, 6, 7, 8], name="lev4"),
+ ),
+ (
+ ["lev4"],
+ "lev3",
+ lib.no_default,
+ [
+ [1.0, np.nan, 1.0, np.nan, 0.0, np.nan],
+ [np.nan, 1.0, np.nan, 1.0, np.nan, 1.0],
+ [1.0, np.nan, 2.0, np.nan, 2.0, np.nan],
+ [np.nan, 1.0, np.nan, 2.0, np.nan, 3.0],
+ [2.0, np.nan, 1.0, np.nan, 4.0, np.nan],
+ [np.nan, 2.0, np.nan, 1.0, np.nan, 5.0],
+ [2.0, np.nan, 2.0, np.nan, 6.0, np.nan],
+ [np.nan, 2.0, np.nan, 2.0, np.nan, 7.0],
+ ],
+ MultiIndex.from_tuples(
+ [
+ ("lev1", 1),
+ ("lev1", 2),
+ ("lev2", 1),
+ ("lev2", 2),
+ ("values", 1),
+ ("values", 2),
+ ],
+ names=[None, "lev3"],
+ ),
+ Index([1, 2, 3, 4, 5, 6, 7, 8], name="lev4"),
+ ),
+ (
+ ["lev1", "lev2"],
+ "lev3",
+ "values",
+ [[0, 1], [2, 3], [4, 5], [6, 7]],
+ Index([1, 2], name="lev3"),
+ MultiIndex.from_tuples(
+ [(1, 1), (1, 2), (2, 1), (2, 2)], names=["lev1", "lev2"]
+ ),
+ ),
+ (
+ ["lev1", "lev2"],
+ "lev3",
+ lib.no_default,
+ [[1, 2, 0, 1], [3, 4, 2, 3], [5, 6, 4, 5], [7, 8, 6, 7]],
+ MultiIndex.from_tuples(
+ [("lev4", 1), ("lev4", 2), ("values", 1), ("values", 2)],
+ names=[None, "lev3"],
+ ),
+ MultiIndex.from_tuples(
+ [(1, 1), (1, 2), (2, 1), (2, 2)], names=["lev1", "lev2"]
+ ),
+ ),
+ ],
+)
+def test_pivot_list_like_index(
+ input_index,
+ input_columns,
+ input_values,
+ expected_values,
+ expected_columns,
+ expected_index,
+):
+ # GH 21425, test when index is given a list
+ df = pd.DataFrame(
+ {
+ "lev1": [1, 1, 1, 1, 2, 2, 2, 2],
+ "lev2": [1, 1, 2, 2, 1, 1, 2, 2],
+ "lev3": [1, 2, 1, 2, 1, 2, 1, 2],
+ "lev4": [1, 2, 3, 4, 5, 6, 7, 8],
+ "values": [0, 1, 2, 3, 4, 5, 6, 7],
+ }
+ )
+
+ result = df.pivot(index=input_index, columns=input_columns, values=input_values)
+ expected = pd.DataFrame(
+ expected_values, columns=expected_columns, index=expected_index
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "input_index, input_columns, input_values, "
+ "expected_values, expected_columns, expected_index",
+ [
+ (
+ "lev4",
+ ["lev3"],
+ "values",
+ [
+ [0.0, np.nan],
+ [np.nan, 1.0],
+ [2.0, np.nan],
+ [np.nan, 3.0],
+ [4.0, np.nan],
+ [np.nan, 5.0],
+ [6.0, np.nan],
+ [np.nan, 7.0],
+ ],
+ Index([1, 2], name="lev3"),
+ Index([1, 2, 3, 4, 5, 6, 7, 8], name="lev4"),
+ ),
+ (
+ ["lev1", "lev2"],
+ ["lev3"],
+ "values",
+ [[0, 1], [2, 3], [4, 5], [6, 7]],
+ Index([1, 2], name="lev3"),
+ MultiIndex.from_tuples(
+ [(1, 1), (1, 2), (2, 1), (2, 2)], names=["lev1", "lev2"]
+ ),
+ ),
+ (
+ ["lev1"],
+ ["lev2", "lev3"],
+ "values",
+ [[0, 1, 2, 3], [4, 5, 6, 7]],
+ MultiIndex.from_tuples(
+ [(1, 1), (1, 2), (2, 1), (2, 2)], names=["lev2", "lev3"]
+ ),
+ Index([1, 2], name="lev1"),
+ ),
+ (
+ ["lev1", "lev2"],
+ ["lev3", "lev4"],
+ "values",
+ [
+ [0.0, 1.0, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan],
+ [np.nan, np.nan, 2.0, 3.0, np.nan, np.nan, np.nan, np.nan],
+ [np.nan, np.nan, np.nan, np.nan, 4.0, 5.0, np.nan, np.nan],
+ [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, 6.0, 7.0],
+ ],
+ MultiIndex.from_tuples(
+ [(1, 1), (2, 2), (1, 3), (2, 4), (1, 5), (2, 6), (1, 7), (2, 8)],
+ names=["lev3", "lev4"],
+ ),
+ MultiIndex.from_tuples(
+ [(1, 1), (1, 2), (2, 1), (2, 2)], names=["lev1", "lev2"]
+ ),
+ ),
+ ],
+)
+def test_pivot_list_like_columns(
+ input_index,
+ input_columns,
+ input_values,
+ expected_values,
+ expected_columns,
+ expected_index,
+):
+ # GH 21425, test when columns is given a list
+ df = pd.DataFrame(
+ {
+ "lev1": [1, 1, 1, 1, 2, 2, 2, 2],
+ "lev2": [1, 1, 2, 2, 1, 1, 2, 2],
+ "lev3": [1, 2, 1, 2, 1, 2, 1, 2],
+ "lev4": [1, 2, 3, 4, 5, 6, 7, 8],
+ "values": [0, 1, 2, 3, 4, 5, 6, 7],
+ }
+ )
+
+ result = df.pivot(index=input_index, columns=input_columns, values=input_values)
+ expected = pd.DataFrame(
+ expected_values, columns=expected_columns, index=expected_index
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+def test_pivot_multiindexed_rows_and_cols():
+ # GH 36360
+
+ df = pd.DataFrame(
+ data=np.arange(12).reshape(4, 3),
+ columns=MultiIndex.from_tuples(
+ [(0, 0), (0, 1), (0, 2)], names=["col_L0", "col_L1"]
+ ),
+ index=MultiIndex.from_tuples(
+ [(0, 0, 0), (0, 0, 1), (1, 1, 1), (1, 0, 0)],
+ names=["idx_L0", "idx_L1", "idx_L2"],
+ ),
+ )
+
+ res = df.pivot_table(
+ index=["idx_L0"],
+ columns=["idx_L1"],
+ values=[(0, 1)],
+ aggfunc=lambda col: col.values.sum(),
+ )
+
+ expected = pd.DataFrame(
+ data=[[5, np.nan], [10, 7.0]],
+ columns=MultiIndex.from_tuples(
+ [(0, 1, 0), (0, 1, 1)], names=["col_L0", "col_L1", "idx_L1"]
+ ),
+ index=Index([0, 1], dtype="int64", name="idx_L0"),
+ )
+ expected = expected.astype("float64")
+
+ tm.assert_frame_equal(res, expected)
+
+
+def test_pivot_df_multiindex_index_none():
+ # GH 23955
+ df = pd.DataFrame(
+ [
+ ["A", "A1", "label1", 1],
+ ["A", "A2", "label2", 2],
+ ["B", "A1", "label1", 3],
+ ["B", "A2", "label2", 4],
+ ],
+ columns=["index_1", "index_2", "label", "value"],
+ )
+ df = df.set_index(["index_1", "index_2"])
+
+ result = df.pivot(columns="label", values="value")
+ expected = pd.DataFrame(
+ [[1.0, np.nan], [np.nan, 2.0], [3.0, np.nan], [np.nan, 4.0]],
+ index=df.index,
+ columns=Index(["label1", "label2"], name="label"),
+ )
+ tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "index, columns, e_data, e_index, e_cols",
+ [
+ (
+ "index",
+ ["col", "value"],
+ [
+ [50.0, np.nan, 100.0, np.nan],
+ [np.nan, 100.0, np.nan, 200.0],
+ ],
+ Index(data=["A", "B"], name="index"),
+ MultiIndex.from_arrays(
+ arrays=[[1, 1, 2, 2], [50, 100, 100, 200]], names=["col", "value"]
+ ),
+ ),
+ (
+ ["index", "value"],
+ "col",
+ [
+ [50.0, np.nan],
+ [np.nan, 100.0],
+ [100.0, np.nan],
+ [np.nan, 200.0],
+ ],
+ MultiIndex.from_arrays(
+ arrays=[["A", "A", "B", "B"], [50, 100, 100, 200]],
+ names=["index", "value"],
+ ),
+ Index(data=[1, 2], name="col"),
+ ),
+ ],
+ ids=["values-and-columns", "values-and-index"],
+)
+def test_pivot_table_multiindex_values_as_two_params(
+ index, columns, e_data, e_index, e_cols
+):
+ # GH#61292
+ data = [
+ ["A", 1, 50, -1],
+ ["B", 1, 100, -2],
+ ["A", 2, 100, -2],
+ ["B", 2, 200, -4],
+ ]
+ df = pd.DataFrame(data=data, columns=["index", "col", "value", "extra"])
+ result = df.pivot_table(values="value", index=index, columns=columns)
+ expected = pd.DataFrame(data=e_data, index=e_index, columns=e_cols)
+ tm.assert_frame_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_qcut.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_qcut.py
new file mode 100644
index 0000000000000000000000000000000000000000..51617bc3536807fca79f06c8476dba0694f5d445
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_qcut.py
@@ -0,0 +1,308 @@
+import os
+
+import numpy as np
+import pytest
+
+import pandas as pd
+from pandas import (
+ Categorical,
+ DatetimeIndex,
+ Interval,
+ IntervalIndex,
+ NaT,
+ Series,
+ Timedelta,
+ TimedeltaIndex,
+ Timestamp,
+ cut,
+ date_range,
+ isna,
+ qcut,
+ timedelta_range,
+)
+import pandas._testing as tm
+from pandas.api.types import CategoricalDtype
+
+from pandas.tseries.offsets import Day
+
+
+def test_qcut():
+ arr = np.random.default_rng(2).standard_normal(1000)
+
+ # We store the bins as Index that have been
+ # rounded to comparisons are a bit tricky.
+ labels, _ = qcut(arr, 4, retbins=True)
+ ex_bins = np.quantile(arr, [0, 0.25, 0.5, 0.75, 1.0])
+
+ result = labels.categories.left.values
+ assert np.allclose(result, ex_bins[:-1], atol=1e-2)
+
+ result = labels.categories.right.values
+ assert np.allclose(result, ex_bins[1:], atol=1e-2)
+
+ ex_levels = cut(arr, ex_bins, include_lowest=True)
+ tm.assert_categorical_equal(labels, ex_levels)
+
+
+def test_qcut_bounds():
+ arr = np.random.default_rng(2).standard_normal(1000)
+
+ factor = qcut(arr, 10, labels=False)
+ assert len(np.unique(factor)) == 10
+
+
+def test_qcut_specify_quantiles():
+ arr = np.random.default_rng(2).standard_normal(100)
+ factor = qcut(arr, [0, 0.25, 0.5, 0.75, 1.0])
+
+ expected = qcut(arr, 4)
+ tm.assert_categorical_equal(factor, expected)
+
+
+def test_qcut_all_bins_same():
+ with pytest.raises(ValueError, match="edges.*unique"):
+ qcut([0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 3)
+
+
+def test_qcut_include_lowest():
+ values = np.arange(10)
+ ii = qcut(values, 4)
+
+ ex_levels = IntervalIndex(
+ [
+ Interval(-0.001, 2.25),
+ Interval(2.25, 4.5),
+ Interval(4.5, 6.75),
+ Interval(6.75, 9),
+ ]
+ )
+ tm.assert_index_equal(ii.categories, ex_levels)
+
+
+def test_qcut_nas():
+ arr = np.random.default_rng(2).standard_normal(100)
+ arr[:20] = np.nan
+
+ result = qcut(arr, 4)
+ assert isna(result[:20]).all()
+
+
+def test_qcut_index():
+ result = qcut([0, 2], 2)
+ intervals = [Interval(-0.001, 1), Interval(1, 2)]
+
+ expected = Categorical(intervals, ordered=True)
+ tm.assert_categorical_equal(result, expected)
+
+
+def test_qcut_binning_issues(datapath):
+ # see gh-1978, gh-1979
+ cut_file = datapath(os.path.join("reshape", "data", "cut_data.csv"))
+ arr = np.loadtxt(cut_file)
+ result = qcut(arr, 20)
+
+ starts = result.categories.left
+ ends = result.categories.right
+ assert (starts < ends).all()
+ assert (starts[1:] <= ends[:-1]).all()
+
+
+def test_qcut_return_intervals():
+ ser = Series([0, 1, 2, 3, 4, 5, 6, 7, 8])
+ res = qcut(ser, [0, 0.333, 0.666, 1])
+
+ exp_levels = np.array(
+ [Interval(-0.001, 2.664), Interval(2.664, 5.328), Interval(5.328, 8)]
+ )
+ exp = Series(exp_levels.take([0, 0, 0, 1, 1, 1, 2, 2, 2])).astype(
+ CategoricalDtype(ordered=True)
+ )
+ tm.assert_series_equal(res, exp)
+
+
+@pytest.mark.parametrize("labels", ["foo", 1, True])
+def test_qcut_incorrect_labels(labels):
+ # GH 13318
+ values = range(5)
+ msg = "Bin labels must either be False, None or passed in as a list-like argument"
+ with pytest.raises(ValueError, match=msg):
+ qcut(values, 4, labels=labels)
+
+
+@pytest.mark.parametrize("labels", [["a", "b", "c"], list(range(3))])
+def test_qcut_wrong_length_labels(labels):
+ # GH 13318
+ values = range(10)
+ msg = "Bin labels must be one fewer than the number of bin edges"
+ with pytest.raises(ValueError, match=msg):
+ qcut(values, 4, labels=labels)
+
+
+@pytest.mark.parametrize(
+ "labels, expected",
+ [
+ (["a", "b", "c"], ["a", "b", "c"]),
+ (list(range(3)), [0, 1, 2]),
+ ],
+)
+def test_qcut_list_like_labels(labels, expected):
+ # GH 13318
+ values = range(3)
+ result = qcut(values, 3, labels=labels)
+ expected = Categorical(expected, ordered=True)
+ tm.assert_categorical_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "kwargs,msg",
+ [
+ ({"duplicates": "drop"}, None),
+ ({}, "Bin edges must be unique"),
+ ({"duplicates": "raise"}, "Bin edges must be unique"),
+ ({"duplicates": "foo"}, "invalid value for 'duplicates' parameter"),
+ ],
+)
+def test_qcut_duplicates_bin(kwargs, msg):
+ # see gh-7751
+ values = [0, 0, 0, 0, 1, 2, 3]
+
+ if msg is not None:
+ with pytest.raises(ValueError, match=msg):
+ qcut(values, 3, **kwargs)
+ else:
+ result = qcut(values, 3, **kwargs)
+ expected = IntervalIndex([Interval(-0.001, 1), Interval(1, 3)])
+ tm.assert_index_equal(result.categories, expected)
+
+
+@pytest.mark.parametrize(
+ "data,start,end", [(9.0, 8.999, 9.0), (0.0, -0.001, 0.0), (-9.0, -9.001, -9.0)]
+)
+@pytest.mark.parametrize("length", [1, 2])
+@pytest.mark.parametrize("labels", [None, False])
+def test_single_quantile(data, start, end, length, labels):
+ # see gh-15431
+ ser = Series([data] * length)
+ result = qcut(ser, 1, labels=labels)
+
+ if labels is None:
+ intervals = IntervalIndex([Interval(start, end)] * length, closed="right")
+ expected = Series(intervals).astype(CategoricalDtype(ordered=True))
+ else:
+ expected = Series([0] * length, dtype=np.intp)
+
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "ser",
+ [
+ DatetimeIndex(["20180101", NaT, "20180103"]),
+ TimedeltaIndex(["0 days", NaT, "2 days"]),
+ ],
+ ids=lambda x: str(x.dtype),
+)
+def test_qcut_nat(ser, unit):
+ # see gh-19768
+ ser = Series(ser)
+ ser = ser.dt.as_unit(unit)
+ td = Timedelta(1, unit=unit).as_unit(unit)
+
+ left = Series([ser[0] - td, np.nan, ser[2] - Day()], dtype=ser.dtype)
+ right = Series([ser[2] - Day(), np.nan, ser[2]], dtype=ser.dtype)
+ intervals = IntervalIndex.from_arrays(left, right)
+ expected = Series(Categorical(intervals, ordered=True))
+
+ result = qcut(ser, 2)
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize("bins", [3, np.linspace(0, 1, 4)])
+def test_datetime_tz_qcut(bins):
+ # see gh-19872
+ tz = "US/Eastern"
+ ser = Series(date_range("20130101", periods=3, tz=tz, unit="ns"))
+
+ result = qcut(ser, bins)
+ expected = Series(
+ IntervalIndex(
+ [
+ Interval(
+ Timestamp("2012-12-31 23:59:59.999999999", tz=tz),
+ Timestamp("2013-01-01 16:00:00", tz=tz),
+ ),
+ Interval(
+ Timestamp("2013-01-01 16:00:00", tz=tz),
+ Timestamp("2013-01-02 08:00:00", tz=tz),
+ ),
+ Interval(
+ Timestamp("2013-01-02 08:00:00", tz=tz),
+ Timestamp("2013-01-03 00:00:00", tz=tz),
+ ),
+ ]
+ )
+ ).astype(CategoricalDtype(ordered=True))
+ tm.assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+ "arg,expected_bins",
+ [
+ [
+ timedelta_range("1day", periods=3),
+ TimedeltaIndex(["1 days", "2 days", "3 days"]),
+ ],
+ [
+ date_range("20180101", periods=3),
+ DatetimeIndex(["2018-01-01", "2018-01-02", "2018-01-03"]),
+ ],
+ ],
+)
+def test_date_like_qcut_bins(arg, expected_bins, unit):
+ # see gh-19891
+ arg = arg.as_unit(unit)
+ expected_bins = expected_bins.as_unit(unit)
+ ser = Series(arg)
+ result, result_bins = qcut(ser, 2, retbins=True)
+ tm.assert_index_equal(result_bins, expected_bins)
+
+
+@pytest.mark.parametrize("bins", [6, 7])
+@pytest.mark.parametrize(
+ "box, compare",
+ [
+ (Series, tm.assert_series_equal),
+ (np.array, tm.assert_categorical_equal),
+ (list, tm.assert_equal),
+ ],
+)
+def test_qcut_bool_coercion_to_int(bins, box, compare):
+ # issue 20303
+ data_expected = box([0, 1, 1, 0, 1] * 10)
+ data_result = box([False, True, True, False, True] * 10)
+ expected = qcut(data_expected, bins, duplicates="drop")
+ result = qcut(data_result, bins, duplicates="drop")
+ compare(result, expected)
+
+
+@pytest.mark.parametrize("q", [2, 5, 10])
+def test_qcut_nullable_integer(q, any_numeric_ea_dtype):
+ arr = pd.array(np.arange(100), dtype=any_numeric_ea_dtype)
+ arr[::2] = pd.NA
+
+ result = qcut(arr, q)
+ expected = qcut(arr.astype(float), q)
+
+ tm.assert_categorical_equal(result, expected)
+
+
+@pytest.mark.parametrize("scale", [1.0, 1 / 3, 17.0])
+@pytest.mark.parametrize("q", [3, 7, 9])
+@pytest.mark.parametrize("precision", [1, 3, 16])
+def test_qcut_contains(scale, q, precision):
+ # GH-59355
+ arr = (scale * np.arange(q + 1)).round(precision)
+ result = qcut(arr, q, precision=precision)
+
+ for value, bucket in zip(arr, result):
+ assert value in bucket
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_union_categoricals.py b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_union_categoricals.py
new file mode 100644
index 0000000000000000000000000000000000000000..081feae6fc43fef590d546ad040d4dc0f412028d
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/reshape/test_union_categoricals.py
@@ -0,0 +1,369 @@
+import numpy as np
+import pytest
+
+from pandas.core.dtypes.concat import union_categoricals
+
+import pandas as pd
+from pandas import (
+ Categorical,
+ CategoricalIndex,
+ Series,
+)
+import pandas._testing as tm
+
+
+class TestUnionCategoricals:
+ @pytest.mark.parametrize(
+ "a, b, combined",
+ [
+ (list("abc"), list("abd"), list("abcabd")),
+ ([0, 1, 2], [2, 3, 4], [0, 1, 2, 2, 3, 4]),
+ ([0, 1.2, 2], [2, 3.4, 4], [0, 1.2, 2, 2, 3.4, 4]),
+ (
+ ["b", "b", np.nan, "a"],
+ ["a", np.nan, "c"],
+ ["b", "b", np.nan, "a", "a", np.nan, "c"],
+ ),
+ (
+ pd.date_range("2014-01-01", "2014-01-05"),
+ pd.date_range("2014-01-06", "2014-01-07"),
+ pd.date_range("2014-01-01", "2014-01-07"),
+ ),
+ (
+ pd.date_range("2014-01-01", "2014-01-05", tz="US/Central"),
+ pd.date_range("2014-01-06", "2014-01-07", tz="US/Central"),
+ pd.date_range("2014-01-01", "2014-01-07", tz="US/Central"),
+ ),
+ (
+ pd.period_range("2014-01-01", "2014-01-05"),
+ pd.period_range("2014-01-06", "2014-01-07"),
+ pd.period_range("2014-01-01", "2014-01-07"),
+ ),
+ ],
+ )
+ @pytest.mark.parametrize("box", [Categorical, CategoricalIndex, Series])
+ def test_union_categorical(self, a, b, combined, box):
+ # GH 13361
+ result = union_categoricals([box(Categorical(a)), box(Categorical(b))])
+ expected = Categorical(combined)
+ tm.assert_categorical_equal(result, expected)
+
+ def test_union_categorical_ordered_appearance(self):
+ # new categories ordered by appearance
+ s = Categorical(["x", "y", "z"])
+ s2 = Categorical(["a", "b", "c"])
+ result = union_categoricals([s, s2])
+ expected = Categorical(
+ ["x", "y", "z", "a", "b", "c"], categories=["x", "y", "z", "a", "b", "c"]
+ )
+ tm.assert_categorical_equal(result, expected)
+
+ def test_union_categorical_ordered_true(self):
+ s = Categorical([0, 1.2, 2], ordered=True)
+ s2 = Categorical([0, 1.2, 2], ordered=True)
+ result = union_categoricals([s, s2])
+ expected = Categorical([0, 1.2, 2, 0, 1.2, 2], ordered=True)
+ tm.assert_categorical_equal(result, expected)
+
+ def test_union_categorical_match_types(self):
+ # must exactly match types
+ s = Categorical([0, 1.2, 2])
+ s2 = Categorical([2, 3, 4])
+ msg = "dtype of categories must be the same"
+ with pytest.raises(TypeError, match=msg):
+ union_categoricals([s, s2])
+
+ def test_union_categorical_empty(self):
+ msg = "No Categoricals to union"
+ with pytest.raises(ValueError, match=msg):
+ union_categoricals([])
+
+ def test_union_categoricals_nan(self):
+ # GH 13759
+ res = union_categoricals(
+ [Categorical([1, 2, np.nan]), Categorical([3, 2, np.nan])]
+ )
+ exp = Categorical([1, 2, np.nan, 3, 2, np.nan])
+ tm.assert_categorical_equal(res, exp)
+
+ res = union_categoricals(
+ [Categorical(["A", "B"]), Categorical(["B", "B", np.nan])]
+ )
+ exp = Categorical(["A", "B", "B", "B", np.nan])
+ tm.assert_categorical_equal(res, exp)
+
+ val1 = [pd.Timestamp("2011-01-01"), pd.Timestamp("2011-03-01"), pd.NaT]
+ val2 = [pd.NaT, pd.Timestamp("2011-01-01"), pd.Timestamp("2011-02-01")]
+
+ res = union_categoricals([Categorical(val1), Categorical(val2)])
+ exp = Categorical(
+ val1 + val2,
+ categories=[
+ pd.Timestamp("2011-01-01"),
+ pd.Timestamp("2011-03-01"),
+ pd.Timestamp("2011-02-01"),
+ ],
+ )
+ tm.assert_categorical_equal(res, exp)
+
+ # all NaN
+ res = union_categoricals(
+ [
+ Categorical(np.array([np.nan, np.nan], dtype=object)),
+ Categorical(["X"], categories=pd.Index(["X"], dtype=object)),
+ ]
+ )
+ exp = Categorical([np.nan, np.nan, "X"])
+ tm.assert_categorical_equal(res, exp)
+
+ res = union_categoricals(
+ [Categorical([np.nan, np.nan]), Categorical([np.nan, np.nan])]
+ )
+ exp = Categorical([np.nan, np.nan, np.nan, np.nan])
+ tm.assert_categorical_equal(res, exp)
+
+ @pytest.mark.parametrize("val", [[], ["1"]])
+ def test_union_categoricals_empty(self, val, request, using_infer_string):
+ # GH 13759
+ if using_infer_string and val == ["1"]:
+ request.applymarker(
+ pytest.mark.xfail(
+ reason="TDOD(infer_string) object and strings dont match"
+ )
+ )
+ res = union_categoricals([Categorical([]), Categorical(val)])
+ exp = Categorical(val)
+ tm.assert_categorical_equal(res, exp)
+
+ def test_union_categorical_same_category(self):
+ # check fastpath
+ c1 = Categorical([1, 2, 3, 4], categories=[1, 2, 3, 4])
+ c2 = Categorical([3, 2, 1, np.nan], categories=[1, 2, 3, 4])
+ res = union_categoricals([c1, c2])
+ exp = Categorical([1, 2, 3, 4, 3, 2, 1, np.nan], categories=[1, 2, 3, 4])
+ tm.assert_categorical_equal(res, exp)
+
+ def test_union_categorical_same_category_str(self):
+ c1 = Categorical(["z", "z", "z"], categories=["x", "y", "z"])
+ c2 = Categorical(["x", "x", "x"], categories=["x", "y", "z"])
+ res = union_categoricals([c1, c2])
+ exp = Categorical(["z", "z", "z", "x", "x", "x"], categories=["x", "y", "z"])
+ tm.assert_categorical_equal(res, exp)
+
+ def test_union_categorical_same_categories_different_order(self):
+ # https://github.com/pandas-dev/pandas/issues/19096
+ c1 = Categorical(["a", "b", "c"], categories=["a", "b", "c"])
+ c2 = Categorical(["a", "b", "c"], categories=["b", "a", "c"])
+ result = union_categoricals([c1, c2])
+ expected = Categorical(
+ ["a", "b", "c", "a", "b", "c"], categories=["a", "b", "c"]
+ )
+ tm.assert_categorical_equal(result, expected)
+
+ def test_union_categoricals_ordered(self):
+ c1 = Categorical([1, 2, 3], ordered=True)
+ c2 = Categorical([1, 2, 3], ordered=False)
+
+ msg = "Categorical.ordered must be the same"
+ with pytest.raises(TypeError, match=msg):
+ union_categoricals([c1, c2])
+
+ res = union_categoricals([c1, c1])
+ exp = Categorical([1, 2, 3, 1, 2, 3], ordered=True)
+ tm.assert_categorical_equal(res, exp)
+
+ c1 = Categorical([1, 2, 3, np.nan], ordered=True)
+ c2 = Categorical([3, 2], categories=[1, 2, 3], ordered=True)
+
+ res = union_categoricals([c1, c2])
+ exp = Categorical([1, 2, 3, np.nan, 3, 2], ordered=True)
+ tm.assert_categorical_equal(res, exp)
+
+ c1 = Categorical([1, 2, 3], ordered=True)
+ c2 = Categorical([1, 2, 3], categories=[3, 2, 1], ordered=True)
+
+ msg = "to union ordered Categoricals, all categories must be the same"
+ with pytest.raises(TypeError, match=msg):
+ union_categoricals([c1, c2])
+
+ def test_union_categoricals_ignore_order(self):
+ # GH 15219
+ c1 = Categorical([1, 2, 3], ordered=True)
+ c2 = Categorical([1, 2, 3], ordered=False)
+
+ res = union_categoricals([c1, c2], ignore_order=True)
+ exp = Categorical([1, 2, 3, 1, 2, 3])
+ tm.assert_categorical_equal(res, exp)
+
+ msg = "Categorical.ordered must be the same"
+ with pytest.raises(TypeError, match=msg):
+ union_categoricals([c1, c2], ignore_order=False)
+
+ res = union_categoricals([c1, c1], ignore_order=True)
+ exp = Categorical([1, 2, 3, 1, 2, 3])
+ tm.assert_categorical_equal(res, exp)
+
+ res = union_categoricals([c1, c1], ignore_order=False)
+ exp = Categorical([1, 2, 3, 1, 2, 3], categories=[1, 2, 3], ordered=True)
+ tm.assert_categorical_equal(res, exp)
+
+ c1 = Categorical([1, 2, 3, np.nan], ordered=True)
+ c2 = Categorical([3, 2], categories=[1, 2, 3], ordered=True)
+
+ res = union_categoricals([c1, c2], ignore_order=True)
+ exp = Categorical([1, 2, 3, np.nan, 3, 2])
+ tm.assert_categorical_equal(res, exp)
+
+ c1 = Categorical([1, 2, 3], ordered=True)
+ c2 = Categorical([1, 2, 3], categories=[3, 2, 1], ordered=True)
+
+ res = union_categoricals([c1, c2], ignore_order=True)
+ exp = Categorical([1, 2, 3, 1, 2, 3])
+ tm.assert_categorical_equal(res, exp)
+
+ res = union_categoricals([c2, c1], ignore_order=True, sort_categories=True)
+ exp = Categorical([1, 2, 3, 1, 2, 3], categories=[1, 2, 3])
+ tm.assert_categorical_equal(res, exp)
+
+ c1 = Categorical([1, 2, 3], ordered=True)
+ c2 = Categorical([4, 5, 6], ordered=True)
+ result = union_categoricals([c1, c2], ignore_order=True)
+ expected = Categorical([1, 2, 3, 4, 5, 6])
+ tm.assert_categorical_equal(result, expected)
+
+ msg = "to union ordered Categoricals, all categories must be the same"
+ with pytest.raises(TypeError, match=msg):
+ union_categoricals([c1, c2], ignore_order=False)
+
+ with pytest.raises(TypeError, match=msg):
+ union_categoricals([c1, c2])
+
+ def test_union_categoricals_sort(self):
+ # GH 13846
+ c1 = Categorical(["x", "y", "z"])
+ c2 = Categorical(["a", "b", "c"])
+ result = union_categoricals([c1, c2], sort_categories=True)
+ expected = Categorical(
+ ["x", "y", "z", "a", "b", "c"], categories=["a", "b", "c", "x", "y", "z"]
+ )
+ tm.assert_categorical_equal(result, expected)
+
+ # fastpath
+ c1 = Categorical(["a", "b"], categories=["b", "a", "c"])
+ c2 = Categorical(["b", "c"], categories=["b", "a", "c"])
+ result = union_categoricals([c1, c2], sort_categories=True)
+ expected = Categorical(["a", "b", "b", "c"], categories=["a", "b", "c"])
+ tm.assert_categorical_equal(result, expected)
+
+ c1 = Categorical(["a", "b"], categories=["c", "a", "b"])
+ c2 = Categorical(["b", "c"], categories=["c", "a", "b"])
+ result = union_categoricals([c1, c2], sort_categories=True)
+ expected = Categorical(["a", "b", "b", "c"], categories=["a", "b", "c"])
+ tm.assert_categorical_equal(result, expected)
+
+ # fastpath - skip resort
+ c1 = Categorical(["a", "b"], categories=["a", "b", "c"])
+ c2 = Categorical(["b", "c"], categories=["a", "b", "c"])
+ result = union_categoricals([c1, c2], sort_categories=True)
+ expected = Categorical(["a", "b", "b", "c"], categories=["a", "b", "c"])
+ tm.assert_categorical_equal(result, expected)
+
+ c1 = Categorical(["x", np.nan])
+ c2 = Categorical([np.nan, "b"])
+ result = union_categoricals([c1, c2], sort_categories=True)
+ expected = Categorical(["x", np.nan, np.nan, "b"], categories=["b", "x"])
+ tm.assert_categorical_equal(result, expected)
+
+ c1 = Categorical([np.nan])
+ c2 = Categorical([np.nan])
+ result = union_categoricals([c1, c2], sort_categories=True)
+ expected = Categorical([np.nan, np.nan])
+ tm.assert_categorical_equal(result, expected)
+
+ c1 = Categorical([])
+ c2 = Categorical([])
+ result = union_categoricals([c1, c2], sort_categories=True)
+ expected = Categorical([])
+ tm.assert_categorical_equal(result, expected)
+
+ c1 = Categorical(["b", "a"], categories=["b", "a", "c"], ordered=True)
+ c2 = Categorical(["a", "c"], categories=["b", "a", "c"], ordered=True)
+ msg = "Cannot use sort_categories=True with ordered Categoricals"
+ with pytest.raises(TypeError, match=msg):
+ union_categoricals([c1, c2], sort_categories=True)
+
+ def test_union_categoricals_sort_false(self):
+ # GH 13846
+ c1 = Categorical(["x", "y", "z"])
+ c2 = Categorical(["a", "b", "c"])
+ result = union_categoricals([c1, c2], sort_categories=False)
+ expected = Categorical(
+ ["x", "y", "z", "a", "b", "c"], categories=["x", "y", "z", "a", "b", "c"]
+ )
+ tm.assert_categorical_equal(result, expected)
+
+ def test_union_categoricals_sort_false_fastpath(self):
+ # fastpath
+ c1 = Categorical(["a", "b"], categories=["b", "a", "c"])
+ c2 = Categorical(["b", "c"], categories=["b", "a", "c"])
+ result = union_categoricals([c1, c2], sort_categories=False)
+ expected = Categorical(["a", "b", "b", "c"], categories=["b", "a", "c"])
+ tm.assert_categorical_equal(result, expected)
+
+ def test_union_categoricals_sort_false_skipresort(self):
+ # fastpath - skip resort
+ c1 = Categorical(["a", "b"], categories=["a", "b", "c"])
+ c2 = Categorical(["b", "c"], categories=["a", "b", "c"])
+ result = union_categoricals([c1, c2], sort_categories=False)
+ expected = Categorical(["a", "b", "b", "c"], categories=["a", "b", "c"])
+ tm.assert_categorical_equal(result, expected)
+
+ def test_union_categoricals_sort_false_one_nan(self):
+ c1 = Categorical(["x", np.nan])
+ c2 = Categorical([np.nan, "b"])
+ result = union_categoricals([c1, c2], sort_categories=False)
+ expected = Categorical(["x", np.nan, np.nan, "b"], categories=["x", "b"])
+ tm.assert_categorical_equal(result, expected)
+
+ def test_union_categoricals_sort_false_only_nan(self):
+ c1 = Categorical([np.nan])
+ c2 = Categorical([np.nan])
+ result = union_categoricals([c1, c2], sort_categories=False)
+ expected = Categorical([np.nan, np.nan])
+ tm.assert_categorical_equal(result, expected)
+
+ def test_union_categoricals_sort_false_empty(self):
+ c1 = Categorical([])
+ c2 = Categorical([])
+ result = union_categoricals([c1, c2], sort_categories=False)
+ expected = Categorical([])
+ tm.assert_categorical_equal(result, expected)
+
+ def test_union_categoricals_sort_false_ordered_true(self):
+ c1 = Categorical(["b", "a"], categories=["b", "a", "c"], ordered=True)
+ c2 = Categorical(["a", "c"], categories=["b", "a", "c"], ordered=True)
+ result = union_categoricals([c1, c2], sort_categories=False)
+ expected = Categorical(
+ ["b", "a", "a", "c"], categories=["b", "a", "c"], ordered=True
+ )
+ tm.assert_categorical_equal(result, expected)
+
+ def test_union_categorical_unwrap(self):
+ # GH 14173
+ c1 = Categorical(["a", "b"])
+ c2 = Series(["b", "c"], dtype="category")
+ result = union_categoricals([c1, c2])
+ expected = Categorical(["a", "b", "b", "c"])
+ tm.assert_categorical_equal(result, expected)
+
+ c2 = CategoricalIndex(c2)
+ result = union_categoricals([c1, c2])
+ tm.assert_categorical_equal(result, expected)
+
+ c1 = Series(c1)
+ result = union_categoricals([c1, c2])
+ tm.assert_categorical_equal(result, expected)
+
+ msg = "all components to combine must be Categorical"
+ with pytest.raises(TypeError, match=msg):
+ union_categoricals([c1, ["a", "b", "c"]])
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/scalar/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_arithmetic.py b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_arithmetic.py
new file mode 100644
index 0000000000000000000000000000000000000000..7e1d53e2068553566ea5ce91c41c31d38ccd09bd
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_arithmetic.py
@@ -0,0 +1,197 @@
+from datetime import timedelta
+
+import numpy as np
+import pytest
+
+from pandas import (
+ Interval,
+ Timedelta,
+ Timestamp,
+)
+import pandas._testing as tm
+
+
+class TestIntervalArithmetic:
+ def test_interval_add(self, closed):
+ interval = Interval(0, 1, closed=closed)
+ expected = Interval(1, 2, closed=closed)
+
+ result = interval + 1
+ assert result == expected
+
+ result = 1 + interval
+ assert result == expected
+
+ result = interval
+ result += 1
+ assert result == expected
+
+ msg = r"unsupported operand type\(s\) for \+"
+ with pytest.raises(TypeError, match=msg):
+ interval + interval
+
+ with pytest.raises(TypeError, match=msg):
+ interval + "foo"
+
+ def test_interval_sub(self, closed):
+ interval = Interval(0, 1, closed=closed)
+ expected = Interval(-1, 0, closed=closed)
+
+ result = interval - 1
+ assert result == expected
+
+ result = interval
+ result -= 1
+ assert result == expected
+
+ msg = r"unsupported operand type\(s\) for -"
+ with pytest.raises(TypeError, match=msg):
+ interval - interval
+
+ with pytest.raises(TypeError, match=msg):
+ interval - "foo"
+
+ def test_interval_mult(self, closed):
+ interval = Interval(0, 1, closed=closed)
+ expected = Interval(0, 2, closed=closed)
+
+ result = interval * 2
+ assert result == expected
+
+ result = 2 * interval
+ assert result == expected
+
+ result = interval
+ result *= 2
+ assert result == expected
+
+ msg = r"unsupported operand type\(s\) for \*"
+ with pytest.raises(TypeError, match=msg):
+ interval * interval
+
+ msg = r"can\'t multiply sequence by non-int"
+ with pytest.raises(TypeError, match=msg):
+ interval * "foo"
+
+ def test_interval_div(self, closed):
+ interval = Interval(0, 1, closed=closed)
+ expected = Interval(0, 0.5, closed=closed)
+
+ result = interval / 2.0
+ assert result == expected
+
+ result = interval
+ result /= 2.0
+ assert result == expected
+
+ msg = r"unsupported operand type\(s\) for /"
+ with pytest.raises(TypeError, match=msg):
+ interval / interval
+
+ with pytest.raises(TypeError, match=msg):
+ interval / "foo"
+
+ def test_interval_floordiv(self, closed):
+ interval = Interval(1, 2, closed=closed)
+ expected = Interval(0, 1, closed=closed)
+
+ result = interval // 2
+ assert result == expected
+
+ result = interval
+ result //= 2
+ assert result == expected
+
+ msg = r"unsupported operand type\(s\) for //"
+ with pytest.raises(TypeError, match=msg):
+ interval // interval
+
+ with pytest.raises(TypeError, match=msg):
+ interval // "foo"
+
+ @pytest.mark.parametrize("method", ["__add__", "__sub__"])
+ @pytest.mark.parametrize(
+ "interval",
+ [
+ Interval(
+ Timestamp("2017-01-01 00:00:00"), Timestamp("2018-01-01 00:00:00")
+ ),
+ Interval(Timedelta(days=7), Timedelta(days=14)),
+ ],
+ )
+ @pytest.mark.parametrize(
+ "delta", [Timedelta(days=7), timedelta(7), np.timedelta64(7, "D")]
+ )
+ def test_time_interval_add_subtract_timedelta(self, interval, delta, method):
+ # https://github.com/pandas-dev/pandas/issues/32023
+ result = getattr(interval, method)(delta)
+ left = getattr(interval.left, method)(delta)
+ right = getattr(interval.right, method)(delta)
+ expected = Interval(left, right)
+
+ assert result == expected
+
+ @pytest.mark.parametrize("interval", [Interval(1, 2), Interval(1.0, 2.0)])
+ @pytest.mark.parametrize(
+ "delta", [Timedelta(days=7), timedelta(7), np.timedelta64(7, "D")]
+ )
+ @pytest.mark.filterwarnings(
+ "ignore:.*'generic' unit for NumPy timedelta:DeprecationWarning"
+ )
+ def test_numeric_interval_add_timedelta_raises(self, interval, delta):
+ # https://github.com/pandas-dev/pandas/issues/32023
+ msg = "|".join(
+ [
+ "unsupported operand",
+ "cannot use operands",
+ "Only numeric, Timestamp and Timedelta endpoints are allowed",
+ ]
+ )
+ with pytest.raises((TypeError, ValueError), match=msg):
+ interval + delta
+
+ with pytest.raises((TypeError, ValueError), match=msg):
+ delta + interval
+
+ @pytest.mark.parametrize(
+ "delta",
+ [timedelta(0), np.timedelta64(0, "ns"), Timedelta(0)],
+ )
+ def test_timedelta_add_timestamp_interval(self, delta):
+ expected = Interval(Timestamp("2020-01-01"), Timestamp("2020-02-01"))
+
+ result = delta + expected
+ assert result == expected
+
+ result = expected + delta
+ assert result == expected
+
+
+class TestIntervalComparisons:
+ def test_interval_equal(self):
+ assert Interval(0, 1) == Interval(0, 1, closed="right")
+ assert Interval(0, 1) != Interval(0, 1, closed="left")
+ assert Interval(0, 1) != 0
+
+ def test_interval_comparison(self):
+ msg = (
+ "'<' not supported between instances of "
+ "'pandas._libs.interval.Interval' and 'int'"
+ )
+ with pytest.raises(TypeError, match=msg):
+ Interval(0, 1) < 2
+
+ assert Interval(0, 1) < Interval(1, 2)
+ assert Interval(0, 1) < Interval(0, 2)
+ assert Interval(0, 1) < Interval(0.5, 1.5)
+ assert Interval(0, 1) <= Interval(0, 1)
+ assert Interval(0, 1) > Interval(-1, 2)
+ assert Interval(0, 1) >= Interval(0, 1)
+
+ def test_equality_comparison_broadcasts_over_array(self):
+ # https://github.com/pandas-dev/pandas/issues/35931
+ interval = Interval(0, 1)
+ arr = np.array([interval, interval])
+ result = interval == arr
+ expected = np.array([True, True])
+ tm.assert_numpy_array_equal(result, expected)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_constructors.py b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_constructors.py
new file mode 100644
index 0000000000000000000000000000000000000000..a4bc00b923434f8d62c8332b3104696051fd287e
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_constructors.py
@@ -0,0 +1,51 @@
+import pytest
+
+from pandas import (
+ Interval,
+ Period,
+ Timestamp,
+)
+
+
+class TestIntervalConstructors:
+ @pytest.mark.parametrize(
+ "left, right",
+ [
+ ("a", "z"),
+ (("a", "b"), ("c", "d")),
+ (list("AB"), list("ab")),
+ (Interval(0, 1), Interval(1, 2)),
+ (Period("2018Q1", freq="Q"), Period("2018Q1", freq="Q")),
+ ],
+ )
+ def test_construct_errors(self, left, right):
+ # GH#23013
+ msg = "Only numeric, Timestamp and Timedelta endpoints are allowed"
+ with pytest.raises(ValueError, match=msg):
+ Interval(left, right)
+
+ def test_constructor_errors(self):
+ msg = "invalid option for 'closed': foo"
+ with pytest.raises(ValueError, match=msg):
+ Interval(0, 1, closed="foo")
+
+ msg = "left side of interval must be <= right side"
+ with pytest.raises(ValueError, match=msg):
+ Interval(1, 0)
+
+ @pytest.mark.parametrize(
+ "tz_left, tz_right", [(None, "UTC"), ("UTC", None), ("UTC", "US/Eastern")]
+ )
+ def test_constructor_errors_tz(self, tz_left, tz_right):
+ # GH#18538
+ left = Timestamp("2017-01-01", tz=tz_left)
+ right = Timestamp("2017-01-02", tz=tz_right)
+
+ if tz_left is None or tz_right is None:
+ error = TypeError
+ msg = "Cannot compare tz-naive and tz-aware timestamps"
+ else:
+ error = ValueError
+ msg = "left and right must have the same time zone"
+ with pytest.raises(error, match=msg):
+ Interval(left, right)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_contains.py b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_contains.py
new file mode 100644
index 0000000000000000000000000000000000000000..8dfca117a658b2a163ef35699c903ad14a032062
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_contains.py
@@ -0,0 +1,73 @@
+import pytest
+
+from pandas import (
+ Interval,
+ Timedelta,
+ Timestamp,
+)
+
+
+class TestContains:
+ def test_contains(self):
+ interval = Interval(0, 1)
+ assert 0.5 in interval
+ assert 1 in interval
+ assert 0 not in interval
+
+ interval_both = Interval(0, 1, "both")
+ assert 0 in interval_both
+ assert 1 in interval_both
+
+ interval_neither = Interval(0, 1, closed="neither")
+ assert 0 not in interval_neither
+ assert 0.5 in interval_neither
+ assert 1 not in interval_neither
+
+ def test_contains_interval(self, inclusive_endpoints_fixture):
+ interval1 = Interval(0, 1, "both")
+ interval2 = Interval(0, 1, inclusive_endpoints_fixture)
+ assert interval1 in interval1
+ assert interval2 in interval2
+ assert interval2 in interval1
+ assert interval1 not in interval2 or inclusive_endpoints_fixture == "both"
+
+ def test_contains_infinite_length(self):
+ interval1 = Interval(0, 1, "both")
+ interval2 = Interval(float("-inf"), float("inf"), "neither")
+ assert interval1 in interval2
+ assert interval2 not in interval1
+
+ def test_contains_zero_length(self):
+ interval1 = Interval(0, 1, "both")
+ interval2 = Interval(-1, -1, "both")
+ interval3 = Interval(0.5, 0.5, "both")
+ assert interval2 not in interval1
+ assert interval3 in interval1
+ assert interval2 not in interval3 and interval3 not in interval2
+ assert interval1 not in interval2 and interval1 not in interval3
+
+ @pytest.mark.parametrize(
+ "type1",
+ [
+ (0, 1),
+ (Timestamp(2000, 1, 1, 0), Timestamp(2000, 1, 1, 1)),
+ (Timedelta("0h"), Timedelta("1h")),
+ ],
+ )
+ @pytest.mark.parametrize(
+ "type2",
+ [
+ (0, 1),
+ (Timestamp(2000, 1, 1, 0), Timestamp(2000, 1, 1, 1)),
+ (Timedelta("0h"), Timedelta("1h")),
+ ],
+ )
+ def test_contains_mixed_types(self, type1, type2):
+ interval1 = Interval(*type1)
+ interval2 = Interval(*type2)
+ if type1 == type2:
+ assert interval1 in interval2
+ else:
+ msg = "^'<=' not supported between instances of"
+ with pytest.raises(TypeError, match=msg):
+ interval1 in interval2
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_formats.py b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_formats.py
new file mode 100644
index 0000000000000000000000000000000000000000..6bf7aa91df3cebc41712c611aaa3781b638009d1
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_formats.py
@@ -0,0 +1,11 @@
+from pandas import Interval
+
+
+def test_interval_repr():
+ interval = Interval(0, 1)
+ assert repr(interval) == "Interval(0, 1, closed='right')"
+ assert str(interval) == "(0, 1]"
+
+ interval_left = Interval(0, 1, closed="left")
+ assert repr(interval_left) == "Interval(0, 1, closed='left')"
+ assert str(interval_left) == "[0, 1)"
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_interval.py b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_interval.py
new file mode 100644
index 0000000000000000000000000000000000000000..91b31e82f9c524f87e2849360cfd44b2f77b0c9c
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_interval.py
@@ -0,0 +1,87 @@
+import numpy as np
+import pytest
+
+from pandas import (
+ Interval,
+ Timedelta,
+ Timestamp,
+)
+
+
+@pytest.fixture
+def interval():
+ return Interval(0, 1)
+
+
+class TestInterval:
+ def test_properties(self, interval):
+ assert interval.closed == "right"
+ assert interval.left == 0
+ assert interval.right == 1
+ assert interval.mid == 0.5
+
+ def test_hash(self, interval):
+ # should not raise
+ hash(interval)
+
+ @pytest.mark.parametrize(
+ "left, right, expected",
+ [
+ (0, 5, 5),
+ (-2, 5.5, 7.5),
+ (10, 10, 0),
+ (10, np.inf, np.inf),
+ (-np.inf, -5, np.inf),
+ (-np.inf, np.inf, np.inf),
+ (Timedelta("0 days"), Timedelta("5 days"), Timedelta("5 days")),
+ (Timedelta("10 days"), Timedelta("10 days"), Timedelta("0 days")),
+ (Timedelta("1h10min"), Timedelta("5h5min"), Timedelta("3h55min")),
+ (Timedelta("5s"), Timedelta("1h"), Timedelta("59min55s")),
+ ],
+ )
+ def test_length(self, left, right, expected):
+ # GH 18789
+ iv = Interval(left, right)
+ result = iv.length
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "left, right, expected",
+ [
+ ("2017-01-01", "2017-01-06", "5 days"),
+ ("2017-01-01", "2017-01-01 12:00:00", "12 hours"),
+ ("2017-01-01 12:00", "2017-01-01 12:00:00", "0 days"),
+ ("2017-01-01 12:01", "2017-01-05 17:31:00", "4 days 5 hours 30 min"),
+ ],
+ )
+ @pytest.mark.parametrize("tz", (None, "UTC", "CET", "US/Eastern"))
+ def test_length_timestamp(self, tz, left, right, expected):
+ # GH 18789
+ iv = Interval(Timestamp(left, tz=tz), Timestamp(right, tz=tz))
+ result = iv.length
+ expected = Timedelta(expected)
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "left, right",
+ [
+ (0, 1),
+ (Timedelta("0 days"), Timedelta("1 day")),
+ (Timestamp("2018-01-01"), Timestamp("2018-01-02")),
+ (
+ Timestamp("2018-01-01", tz="US/Eastern"),
+ Timestamp("2018-01-02", tz="US/Eastern"),
+ ),
+ ],
+ )
+ def test_is_empty(self, left, right, closed):
+ # GH27219
+ # non-empty always return False
+ iv = Interval(left, right, closed)
+ assert iv.is_empty is False
+
+ # same endpoint is empty except when closed='both' (contains one point)
+ iv = Interval(left, left, closed)
+ result = iv.is_empty
+ expected = closed != "both"
+ assert result is expected
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_overlaps.py b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_overlaps.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fcf59d7bb4afc0077884de68dc335aff25c2cc5
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/interval/test_overlaps.py
@@ -0,0 +1,67 @@
+import pytest
+
+from pandas import (
+ Interval,
+ Timedelta,
+ Timestamp,
+)
+
+
+@pytest.fixture(
+ params=[
+ (Timedelta("0 days"), Timedelta("1 day")),
+ (Timestamp("2018-01-01"), Timedelta("1 day")),
+ (0, 1),
+ ],
+ ids=lambda x: type(x[0]).__name__,
+)
+def start_shift(request):
+ """
+ Fixture for generating intervals of types from a start value and a shift
+ value that can be added to start to generate an endpoint
+ """
+ return request.param
+
+
+class TestOverlaps:
+ def test_overlaps_self(self, start_shift, closed):
+ start, shift = start_shift
+ interval = Interval(start, start + shift, closed)
+ assert interval.overlaps(interval)
+
+ def test_overlaps_nested(self, start_shift, closed, other_closed):
+ start, shift = start_shift
+ interval1 = Interval(start, start + 3 * shift, other_closed)
+ interval2 = Interval(start + shift, start + 2 * shift, closed)
+
+ # nested intervals should always overlap
+ assert interval1.overlaps(interval2)
+
+ def test_overlaps_disjoint(self, start_shift, closed, other_closed):
+ start, shift = start_shift
+ interval1 = Interval(start, start + shift, other_closed)
+ interval2 = Interval(start + 2 * shift, start + 3 * shift, closed)
+
+ # disjoint intervals should never overlap
+ assert not interval1.overlaps(interval2)
+
+ def test_overlaps_endpoint(self, start_shift, closed, other_closed):
+ start, shift = start_shift
+ interval1 = Interval(start, start + shift, other_closed)
+ interval2 = Interval(start + shift, start + 2 * shift, closed)
+
+ # overlap if shared endpoint is closed for both (overlap at a point)
+ result = interval1.overlaps(interval2)
+ expected = interval1.closed_right and interval2.closed_left
+ assert result == expected
+
+ @pytest.mark.parametrize(
+ "other",
+ [10, True, "foo", Timedelta("1 day"), Timestamp("2018-01-01")],
+ ids=lambda x: type(x).__name__,
+ )
+ def test_overlaps_invalid_type(self, other):
+ interval = Interval(0, 1)
+ msg = f"`other` must be an Interval, got {type(other).__name__}"
+ with pytest.raises(TypeError, match=msg):
+ interval.overlaps(other)
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/scalar/period/__init__.py b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/period/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/.venv/lib/python3.13/site-packages/pandas/tests/scalar/period/test_arithmetic.py b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/period/test_arithmetic.py
new file mode 100644
index 0000000000000000000000000000000000000000..97e486f9af06025b1ec53515fed64af3adf67a8d
--- /dev/null
+++ b/.venv/lib/python3.13/site-packages/pandas/tests/scalar/period/test_arithmetic.py
@@ -0,0 +1,487 @@
+from datetime import timedelta
+
+import numpy as np
+import pytest
+
+from pandas._libs.tslibs.period import IncompatibleFrequency
+
+from pandas import (
+ NaT,
+ Period,
+ Timedelta,
+ Timestamp,
+ offsets,
+)
+
+
+class TestPeriodArithmetic:
+ def test_add_overflow_raises(self):
+ # GH#55503
+ per = Timestamp.max.to_period("ns")
+
+ msg = "|".join(
+ [
+ "Python int too large to convert to C long",
+ # windows, 32bit linux builds
+ "int too big to convert",
+ ]
+ )
+ with pytest.raises(OverflowError, match=msg):
+ per + 1
+
+ msg = "value too large"
+ with pytest.raises(OverflowError, match=msg):
+ per + Timedelta(1)
+ with pytest.raises(OverflowError, match=msg):
+ per + offsets.Nano(1)
+
+ def test_period_add_integer(self):
+ per1 = Period(freq="D", year=2008, month=1, day=1)
+ per2 = Period(freq="D", year=2008, month=1, day=2)
+ assert per1 + 1 == per2
+ assert 1 + per1 == per2
+
+ def test_period_add_invalid(self):
+ # GH#4731
+ per1 = Period(freq="D", year=2008, month=1, day=1)
+ per2 = Period(freq="D", year=2008, month=1, day=2)
+
+ msg = "|".join(
+ [
+ r"unsupported operand type\(s\)",
+ "can only concatenate str",
+ "must be str, not Period",
+ ]
+ )
+ with pytest.raises(TypeError, match=msg):
+ per1 + "str"
+ with pytest.raises(TypeError, match=msg):
+ "str" + per1
+ with pytest.raises(TypeError, match=msg):
+ per1 + per2
+
+ def test_period_sub_period_annual(self):
+ left, right = Period("2011", freq="Y"), Period("2007", freq="Y")
+ result = left - right
+ assert result == 4 * right.freq
+
+ msg = r"Input has different freq=M from Period\(freq=Y-DEC\)"
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ left - Period("2007-01", freq="M")
+
+ def test_period_sub_period(self):
+ per1 = Period("2011-01-01", freq="D")
+ per2 = Period("2011-01-15", freq="D")
+
+ off = per1.freq
+ assert per1 - per2 == -14 * off
+ assert per2 - per1 == 14 * off
+
+ msg = r"Input has different freq=M from Period\(freq=D\)"
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ per1 - Period("2011-02", freq="M")
+
+ @pytest.mark.parametrize("n", [1, 2, 3, 4])
+ def test_sub_n_gt_1_ticks(self, tick_classes, n):
+ # GH#23878
+ p1 = Period("19910905", freq=tick_classes(n))
+ p2 = Period("19920406", freq=tick_classes(n))
+
+ expected = Period(str(p2), freq=p2.freq.base) - Period(
+ str(p1), freq=p1.freq.base
+ )
+
+ assert (p2 - p1) == expected
+
+ @pytest.mark.parametrize("normalize", [True, False])
+ @pytest.mark.parametrize("n", [1, 2, 3, 4])
+ @pytest.mark.parametrize(
+ "offset, kwd_name",
+ [
+ (offsets.YearEnd, "month"),
+ (offsets.QuarterEnd, "startingMonth"),
+ (offsets.MonthEnd, None),
+ (offsets.Week, "weekday"),
+ ],
+ )
+ def test_sub_n_gt_1_offsets(self, offset, kwd_name, n, normalize):
+ # GH#23878
+ kwds = {kwd_name: 3} if kwd_name is not None else {}
+ p1_d = "19910905"
+ p2_d = "19920406"
+ p1 = Period(p1_d, freq=offset(n, normalize, **kwds))
+ p2 = Period(p2_d, freq=offset(n, normalize, **kwds))
+
+ expected = Period(p2_d, freq=p2.freq.base) - Period(p1_d, freq=p1.freq.base)
+
+ assert (p2 - p1) == expected
+
+ def test_period_add_offset(self):
+ # freq is DateOffset
+ for freq in ["Y", "2Y", "3Y"]:
+ per = Period("2011", freq=freq)
+ exp = Period("2013", freq=freq)
+ assert per + offsets.YearEnd(2) == exp
+ assert offsets.YearEnd(2) + per == exp
+
+ for off in [
+ offsets.YearBegin(2),
+ offsets.MonthBegin(1),
+ offsets.Minute(),
+ np.timedelta64(365, "D"),
+ timedelta(365),
+ ]:
+ msg = "Input has different freq|Input cannot be converted to Period"
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ per + off
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ off + per
+
+ for freq in ["M", "2M", "3M"]:
+ per = Period("2011-03", freq=freq)
+ exp = Period("2011-05", freq=freq)
+ assert per + offsets.MonthEnd(2) == exp
+ assert offsets.MonthEnd(2) + per == exp
+
+ exp = Period("2012-03", freq=freq)
+ assert per + offsets.MonthEnd(12) == exp
+ assert offsets.MonthEnd(12) + per == exp
+
+ msg = "|".join(
+ [
+ "Input has different freq",
+ "Input cannot be converted to Period",
+ ]
+ )
+
+ for off in [
+ offsets.YearBegin(2),
+ offsets.MonthBegin(1),
+ offsets.Minute(),
+ np.timedelta64(365, "D"),
+ timedelta(365),
+ ]:
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ per + off
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ off + per
+
+ # freq is Tick
+ for freq in ["D", "2D", "3D"]:
+ per = Period("2011-04-01", freq=freq)
+
+ exp = Period("2011-04-06", freq=freq)
+ assert per + offsets.Day(5) == exp
+ assert offsets.Day(5) + per == exp
+
+ exp = Period("2011-04-02", freq=freq)
+ assert per + offsets.Hour(24) == exp
+ assert offsets.Hour(24) + per == exp
+
+ exp = Period("2011-04-03", freq=freq)
+ assert per + np.timedelta64(2, "D") == exp
+ assert np.timedelta64(2, "D") + per == exp
+
+ exp = Period("2011-04-02", freq=freq)
+ assert per + np.timedelta64(3600 * 24, "s") == exp
+ assert np.timedelta64(3600 * 24, "s") + per == exp
+
+ exp = Period("2011-03-30", freq=freq)
+ assert per + timedelta(-2) == exp
+ assert timedelta(-2) + per == exp
+
+ exp = Period("2011-04-03", freq=freq)
+ assert per + timedelta(hours=48) == exp
+ assert timedelta(hours=48) + per == exp
+
+ msg = "|".join(
+ [
+ "Input has different freq",
+ "Input cannot be converted to Period",
+ ]
+ )
+
+ for off in [
+ offsets.YearBegin(2),
+ offsets.MonthBegin(1),
+ offsets.Minute(),
+ np.timedelta64(4, "h"),
+ timedelta(hours=23),
+ ]:
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ per + off
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ off + per
+
+ for freq in ["h", "2h", "3h"]:
+ per = Period("2011-04-01 09:00", freq=freq)
+
+ exp = Period("2011-04-03 09:00", freq=freq)
+ assert per + offsets.Day(2) == exp
+ assert offsets.Day(2) + per == exp
+
+ exp = Period("2011-04-01 12:00", freq=freq)
+ assert per + offsets.Hour(3) == exp
+ assert offsets.Hour(3) + per == exp
+
+ msg = "cannot use operands with types"
+ exp = Period("2011-04-01 12:00", freq=freq)
+ assert per + np.timedelta64(3, "h") == exp
+ assert np.timedelta64(3, "h") + per == exp
+
+ exp = Period("2011-04-01 10:00", freq=freq)
+ assert per + np.timedelta64(3600, "s") == exp
+ assert np.timedelta64(3600, "s") + per == exp
+
+ exp = Period("2011-04-01 11:00", freq=freq)
+ assert per + timedelta(minutes=120) == exp
+ assert timedelta(minutes=120) + per == exp
+
+ exp = Period("2011-04-05 12:00", freq=freq)
+ assert per + timedelta(days=4, minutes=180) == exp
+ assert timedelta(days=4, minutes=180) + per == exp
+
+ msg = "|".join(
+ [
+ "Input has different freq",
+ "Input cannot be converted to Period",
+ ]
+ )
+
+ for off in [
+ offsets.YearBegin(2),
+ offsets.MonthBegin(1),
+ offsets.Minute(),
+ np.timedelta64(3200, "s"),
+ timedelta(hours=23, minutes=30),
+ ]:
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ per + off
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ off + per
+
+ def test_period_sub_offset(self):
+ # freq is DateOffset
+ msg = "|".join(
+ [
+ "Input has different freq",
+ "Input cannot be converted to Period",
+ ]
+ )
+
+ for freq in ["Y", "2Y", "3Y"]:
+ per = Period("2011", freq=freq)
+ assert per - offsets.YearEnd(2) == Period("2009", freq=freq)
+
+ for off in [
+ offsets.YearBegin(2),
+ offsets.MonthBegin(1),
+ offsets.Minute(),
+ np.timedelta64(365, "D"),
+ timedelta(365),
+ ]:
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ per - off
+
+ for freq in ["M", "2M", "3M"]:
+ per = Period("2011-03", freq=freq)
+ assert per - offsets.MonthEnd(2) == Period("2011-01", freq=freq)
+ assert per - offsets.MonthEnd(12) == Period("2010-03", freq=freq)
+
+ for off in [
+ offsets.YearBegin(2),
+ offsets.MonthBegin(1),
+ offsets.Minute(),
+ np.timedelta64(365, "D"),
+ timedelta(365),
+ ]:
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ per - off
+
+ # freq is Tick
+ for freq in ["D", "2D", "3D"]:
+ per = Period("2011-04-01", freq=freq)
+ assert per - offsets.Day(5) == Period("2011-03-27", freq=freq)
+ assert per - offsets.Hour(24) == Period("2011-03-31", freq=freq)
+ assert per - np.timedelta64(2, "D") == Period("2011-03-30", freq=freq)
+ assert per - np.timedelta64(3600 * 24, "s") == Period(
+ "2011-03-31", freq=freq
+ )
+ assert per - timedelta(-2) == Period("2011-04-03", freq=freq)
+ assert per - timedelta(hours=48) == Period("2011-03-30", freq=freq)
+
+ for off in [
+ offsets.YearBegin(2),
+ offsets.MonthBegin(1),
+ offsets.Minute(),
+ np.timedelta64(4, "h"),
+ timedelta(hours=23),
+ ]:
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ per - off
+
+ for freq in ["h", "2h", "3h"]:
+ per = Period("2011-04-01 09:00", freq=freq)
+ assert per - offsets.Day(2) == Period("2011-03-30 09:00", freq=freq)
+ assert per - offsets.Hour(3) == Period("2011-04-01 06:00", freq=freq)
+ assert per - np.timedelta64(3, "h") == Period("2011-04-01 06:00", freq=freq)
+ assert per - np.timedelta64(3600, "s") == Period(
+ "2011-04-01 08:00", freq=freq
+ )
+ assert per - timedelta(minutes=120) == Period("2011-04-01 07:00", freq=freq)
+ assert per - timedelta(days=4, minutes=180) == Period(
+ "2011-03-28 06:00", freq=freq
+ )
+
+ for off in [
+ offsets.YearBegin(2),
+ offsets.MonthBegin(1),
+ offsets.Minute(),
+ np.timedelta64(3200, "s"),
+ timedelta(hours=23, minutes=30),
+ ]:
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ per - off
+
+ @pytest.mark.parametrize("freq", ["M", "2M", "3M"])
+ def test_period_addsub_nat(self, freq):
+ # GH#13071
+ per = Period("2011-01", freq=freq)
+
+ # For subtraction, NaT is treated as another Period object
+ assert NaT - per is NaT
+ assert per - NaT is NaT
+
+ # For addition, NaT is treated as offset-like
+ assert NaT + per is NaT
+ assert per + NaT is NaT
+
+ @pytest.mark.parametrize("unit", ["ns", "us", "ms", "s", "m"])
+ def test_period_add_sub_td64_nat(self, unit):
+ # GH#47196
+ per = Period("2022-06-01", "D")
+ nat = np.timedelta64("NaT", unit)
+
+ assert per + nat is NaT
+ assert nat + per is NaT
+ assert per - nat is NaT
+
+ with pytest.raises(TypeError, match="unsupported operand"):
+ nat - per
+
+ def test_period_ops_offset(self):
+ per = Period("2011-04-01", freq="D")
+ result = per + offsets.Day()
+ exp = Period("2011-04-02", freq="D")
+ assert result == exp
+
+ result = per - offsets.Day(2)
+ exp = Period("2011-03-30", freq="D")
+ assert result == exp
+
+ msg = r"Input cannot be converted to Period\(freq=D\)"
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ per + offsets.Hour(2)
+
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ per - offsets.Hour(2)
+
+ def test_period_add_timestamp_raises(self):
+ # GH#17983
+ ts = Timestamp("2017")
+ per = Period("2017", freq="M")
+
+ msg = r"unsupported operand type\(s\) for \+: 'Timestamp' and 'Period'"
+ with pytest.raises(TypeError, match=msg):
+ ts + per
+
+ msg = r"unsupported operand type\(s\) for \+: 'Period' and 'Timestamp'"
+ with pytest.raises(TypeError, match=msg):
+ per + ts
+
+
+class TestPeriodComparisons:
+ def test_period_comparison_same_freq(self):
+ jan = Period("2000-01", "M")
+ feb = Period("2000-02", "M")
+
+ assert not jan == feb
+ assert jan != feb
+ assert jan < feb
+ assert jan <= feb
+ assert not jan > feb
+ assert not jan >= feb
+
+ def test_period_comparison_same_period_different_object(self):
+ # Separate Period objects for the same period
+ left = Period("2000-01", "M")
+ right = Period("2000-01", "M")
+
+ assert left == right
+ assert left >= right
+ assert left <= right
+ assert not left < right
+ assert not left > right
+
+ def test_period_comparison_mismatched_freq(self):
+ jan = Period("2000-01", "M")
+ day = Period("2012-01-01", "D")
+
+ assert not jan == day
+ assert jan != day
+ msg = r"Input has different freq=D from Period\(freq=M\)"
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ jan < day
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ jan <= day
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ jan > day
+ with pytest.raises(IncompatibleFrequency, match=msg):
+ jan >= day
+
+ def test_period_comparison_invalid_type(self):
+ jan = Period("2000-01", "M")
+
+ assert not jan == 1
+ assert jan != 1
+
+ int_or_per = "'(Period|int)'"
+ msg = f"not supported between instances of {int_or_per} and {int_or_per}"
+ for left, right in [(jan, 1), (1, jan)]:
+ with pytest.raises(TypeError, match=msg):
+ left > right
+ with pytest.raises(TypeError, match=msg):
+ left >= right
+ with pytest.raises(TypeError, match=msg):
+ left < right
+ with pytest.raises(TypeError, match=msg):
+ left <= right
+
+ def test_period_comparison_nat(self):
+ per = Period("2011-01-01", freq="D")
+
+ ts = Timestamp("2011-01-01")
+ # confirm Period('NaT') work identical with Timestamp('NaT')
+ for left, right in [
+ (NaT, per),
+ (per, NaT),
+ (NaT, ts),
+ (ts, NaT),
+ ]:
+ assert not left < right
+ assert not left > right
+ assert not left == right
+ assert left != right
+ assert not left <= right
+ assert not left >= right
+
+ @pytest.mark.parametrize(
+ "scalar, expected",
+ ((0, False), (Period("2000-01", "M"), True)),
+ )
+ def test_period_comparison_numpy_zerodim_arr(self, scalar, expected):
+ zerodim_arr = np.array(scalar)
+ per = Period("2000-01", "M")
+
+ assert (per == zerodim_arr) is expected
+ assert (zerodim_arr == per) is expected