import datetime as dt
from datetime import datetime
from itertools import combinations

import dateutil
import numpy as np
import pytest

import pandas as pd
from pandas import DataFrame, Index, Series, Timestamp, concat, isna
import pandas._testing as tm


class TestAppend:
    def test_append(self, sort, float_frame):
        mixed_frame = float_frame.copy()
        mixed_frame["foo"] = "bar"

        begin_index = float_frame.index[:5]
        end_index = float_frame.index[5:]

        begin_frame = float_frame.reindex(begin_index)
        end_frame = float_frame.reindex(end_index)

        appended = begin_frame.append(end_frame)
        tm.assert_almost_equal(appended["A"], float_frame["A"])

        del end_frame["A"]
        partial_appended = begin_frame.append(end_frame, sort=sort)
        assert "A" in partial_appended

        partial_appended = end_frame.append(begin_frame, sort=sort)
        assert "A" in partial_appended

        # mixed type handling
        appended = mixed_frame[:5].append(mixed_frame[5:])
        tm.assert_frame_equal(appended, mixed_frame)

        # what to test here
        mixed_appended = mixed_frame[:5].append(float_frame[5:], sort=sort)
        mixed_appended2 = float_frame[:5].append(mixed_frame[5:], sort=sort)

        # all equal except 'foo' column
        tm.assert_frame_equal(
            mixed_appended.reindex(columns=["A", "B", "C", "D"]),
            mixed_appended2.reindex(columns=["A", "B", "C", "D"]),
        )

    def test_append_empty(self, float_frame):
        empty = DataFrame()

        appended = float_frame.append(empty)
        tm.assert_frame_equal(float_frame, appended)
        assert appended is not float_frame

        appended = empty.append(float_frame)
        tm.assert_frame_equal(float_frame, appended)
        assert appended is not float_frame

    def test_append_overlap_raises(self, float_frame):
        msg = "Indexes have overlapping values"
        with pytest.raises(ValueError, match=msg):
            float_frame.append(float_frame, verify_integrity=True)

    def test_append_new_columns(self):
        # see gh-6129: new columns
        df = DataFrame({"a": {"x": 1, "y": 2}, "b": {"x": 3, "y": 4}})
        row = Series([5, 6, 7], index=["a", "b", "c"], name="z")
        expected = DataFrame(
            {
                "a": {"x": 1, "y": 2, "z": 5},
                "b": {"x": 3, "y": 4, "z": 6},
                "c": {"z": 7},
            }
        )
        result = df.append(row)
        tm.assert_frame_equal(result, expected)

    def test_append_length0_frame(self, sort):
        df = DataFrame(columns=["A", "B", "C"])
        df3 = DataFrame(index=[0, 1], columns=["A", "B"])
        df5 = df.append(df3, sort=sort)

        expected = DataFrame(index=[0, 1], columns=["A", "B", "C"])
        tm.assert_frame_equal(df5, expected)

    def test_append_records(self):
        arr1 = np.zeros((2,), dtype=("i4,f4,a10"))
        arr1[:] = [(1, 2.0, "Hello"), (2, 3.0, "World")]

        arr2 = np.zeros((3,), dtype=("i4,f4,a10"))
        arr2[:] = [(3, 4.0, "foo"), (5, 6.0, "bar"), (7.0, 8.0, "baz")]

        df1 = DataFrame(arr1)
        df2 = DataFrame(arr2)

        result = df1.append(df2, ignore_index=True)
        expected = DataFrame(np.concatenate((arr1, arr2)))
        tm.assert_frame_equal(result, expected)

    # rewrite sort fixture, since we also want to test default of None
    def test_append_sorts(self, sort):
        df1 = DataFrame({"a": [1, 2], "b": [1, 2]}, columns=["b", "a"])
        df2 = DataFrame({"a": [1, 2], "c": [3, 4]}, index=[2, 3])

        with tm.assert_produces_warning(None):
            result = df1.append(df2, sort=sort)

        # for None / True
        expected = DataFrame(
            {"b": [1, 2, None, None], "a": [1, 2, 1, 2], "c": [None, None, 3, 4]},
            columns=["a", "b", "c"],
        )
        if sort is False:
            expected = expected[["b", "a", "c"]]
        tm.assert_frame_equal(result, expected)

    def test_append_different_columns(self, sort):
        df = DataFrame(
            {
                "bools": np.random.randn(10) > 0,
                "ints": np.random.randint(0, 10, 10),
                "floats": np.random.randn(10),
                "strings": ["foo", "bar"] * 5,
            }
        )

        a = df[:5].loc[:, ["bools", "ints", "floats"]]
        b = df[5:].loc[:, ["strings", "ints", "floats"]]

        appended = a.append(b, sort=sort)
        assert isna(appended["strings"][0:4]).all()
        assert isna(appended["bools"][5:]).all()

    def test_append_many(self, sort, float_frame):
        chunks = [
            float_frame[:5],
            float_frame[5:10],
            float_frame[10:15],
            float_frame[15:],
        ]

        result = chunks[0].append(chunks[1:])
        tm.assert_frame_equal(result, float_frame)

        chunks[-1] = chunks[-1].copy()
        chunks[-1]["foo"] = "bar"
        result = chunks[0].append(chunks[1:], sort=sort)
        tm.assert_frame_equal(result.loc[:, float_frame.columns], float_frame)
        assert (result["foo"][15:] == "bar").all()
        assert result["foo"][:15].isna().all()

    def test_append_preserve_index_name(self):
        # #980
        df1 = DataFrame(columns=["A", "B", "C"])
        df1 = df1.set_index(["A"])
        df2 = DataFrame(data=[[1, 4, 7], [2, 5, 8], [3, 6, 9]], columns=["A", "B", "C"])
        df2 = df2.set_index(["A"])

        result = df1.append(df2)
        assert result.index.name == "A"

    indexes_can_append = [
        pd.RangeIndex(3),
        Index([4, 5, 6]),
        Index([4.5, 5.5, 6.5]),
        Index(list("abc")),
        pd.CategoricalIndex("A B C".split()),
        pd.CategoricalIndex("D E F".split(), ordered=True),
        pd.IntervalIndex.from_breaks([7, 8, 9, 10]),
        pd.DatetimeIndex(
            [
                dt.datetime(2013, 1, 3, 0, 0),
                dt.datetime(2013, 1, 3, 6, 10),
                dt.datetime(2013, 1, 3, 7, 12),
            ]
        ),
    ]

    indexes_cannot_append_with_other = [
        pd.MultiIndex.from_arrays(["A B C".split(), "D E F".split()])
    ]

    all_indexes = indexes_can_append + indexes_cannot_append_with_other

    @pytest.mark.parametrize("index", all_indexes, ids=lambda x: type(x).__name__)
    def test_append_same_columns_type(self, index):
        # GH18359

        # df wider than ser
        df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=index)
        ser_index = index[:2]
        ser = Series([7, 8], index=ser_index, name=2)
        result = df.append(ser)
        expected = DataFrame(
            [[1.0, 2.0, 3.0], [4, 5, 6], [7, 8, np.nan]], index=[0, 1, 2], columns=index
        )
        tm.assert_frame_equal(result, expected)

        # ser wider than df
        ser_index = index
        index = index[:2]
        df = DataFrame([[1, 2], [4, 5]], columns=index)
        ser = Series([7, 8, 9], index=ser_index, name=2)
        result = df.append(ser)
        expected = DataFrame(
            [[1, 2, np.nan], [4, 5, np.nan], [7, 8, 9]],
            index=[0, 1, 2],
            columns=ser_index,
        )
        tm.assert_frame_equal(result, expected)

    @pytest.mark.parametrize(
        "df_columns, series_index",
        combinations(indexes_can_append, r=2),
        ids=lambda x: type(x).__name__,
    )
    def test_append_different_columns_types(self, df_columns, series_index):
        # GH18359
        # See also test 'test_append_different_columns_types_raises' below
        # for errors raised when appending

        df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=df_columns)
        ser = Series([7, 8, 9], index=series_index, name=2)

        result = df.append(ser)
        idx_diff = ser.index.difference(df_columns)
        combined_columns = Index(df_columns.tolist()).append(idx_diff)
        expected = DataFrame(
            [
                [1.0, 2.0, 3.0, np.nan, np.nan, np.nan],
                [4, 5, 6, np.nan, np.nan, np.nan],
                [np.nan, np.nan, np.nan, 7, 8, 9],
            ],
            index=[0, 1, 2],
            columns=combined_columns,
        )
        tm.assert_frame_equal(result, expected)

    @pytest.mark.parametrize(
        "index_can_append", indexes_can_append, ids=lambda x: type(x).__name__
    )
    @pytest.mark.parametrize(
        "index_cannot_append_with_other",
        indexes_cannot_append_with_other,
        ids=lambda x: type(x).__name__,
    )
    def test_append_different_columns_types_raises(
        self, index_can_append, index_cannot_append_with_other
    ):
        # GH18359
        # Dataframe.append will raise if MultiIndex appends
        # or is appended to a different index type
        #
        # See also test 'test_append_different_columns_types' above for
        # appending without raising.

        df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=index_can_append)
        ser = Series([7, 8, 9], index=index_cannot_append_with_other, name=2)
        msg = (
            r"Expected tuple, got (int|long|float|str|"
            r"pandas._libs.interval.Interval)|"
            r"object of type '(int|float|Timestamp|"
            r"pandas._libs.interval.Interval)' has no len\(\)|"
        )
        with pytest.raises(TypeError, match=msg):
            df.append(ser)

        df = DataFrame([[1, 2, 3], [4, 5, 6]], columns=index_cannot_append_with_other)
        ser = Series([7, 8, 9], index=index_can_append, name=2)

        with pytest.raises(TypeError, match=msg):
            df.append(ser)

    def test_append_dtype_coerce(self, sort):

        # GH 4993
        # appending with datetime will incorrectly convert datetime64

        df1 = DataFrame(
            index=[1, 2],
            data=[dt.datetime(2013, 1, 1, 0, 0), dt.datetime(2013, 1, 2, 0, 0)],
            columns=["start_time"],
        )
        df2 = DataFrame(
            index=[4, 5],
            data=[
                [dt.datetime(2013, 1, 3, 0, 0), dt.datetime(2013, 1, 3, 6, 10)],
                [dt.datetime(2013, 1, 4, 0, 0), dt.datetime(2013, 1, 4, 7, 10)],
            ],
            columns=["start_time", "end_time"],
        )

        expected = concat(
            [
                Series(
                    [
                        pd.NaT,
                        pd.NaT,
                        dt.datetime(2013, 1, 3, 6, 10),
                        dt.datetime(2013, 1, 4, 7, 10),
                    ],
                    name="end_time",
                ),
                Series(
                    [
                        dt.datetime(2013, 1, 1, 0, 0),
                        dt.datetime(2013, 1, 2, 0, 0),
                        dt.datetime(2013, 1, 3, 0, 0),
                        dt.datetime(2013, 1, 4, 0, 0),
                    ],
                    name="start_time",
                ),
            ],
            axis=1,
            sort=sort,
        )
        result = df1.append(df2, ignore_index=True, sort=sort)
        if sort:
            expected = expected[["end_time", "start_time"]]
        else:
            expected = expected[["start_time", "end_time"]]

        tm.assert_frame_equal(result, expected)

    def test_append_missing_column_proper_upcast(self, sort):
        df1 = DataFrame({"A": np.array([1, 2, 3, 4], dtype="i8")})
        df2 = DataFrame({"B": np.array([True, False, True, False], dtype=bool)})

        appended = df1.append(df2, ignore_index=True, sort=sort)
        assert appended["A"].dtype == "f8"
        assert appended["B"].dtype == "O"

    def test_append_empty_frame_to_series_with_dateutil_tz(self):
        # GH 23682
        date = Timestamp("2018-10-24 07:30:00", tz=dateutil.tz.tzutc())
        s = Series({"date": date, "a": 1.0, "b": 2.0})
        df = DataFrame(columns=["c", "d"])
        result_a = df.append(s, ignore_index=True)
        expected = DataFrame(
            [[np.nan, np.nan, 1.0, 2.0, date]], columns=["c", "d", "a", "b", "date"]
        )
        # These columns get cast to object after append
        expected["c"] = expected["c"].astype(object)
        expected["d"] = expected["d"].astype(object)
        tm.assert_frame_equal(result_a, expected)

        expected = DataFrame(
            [[np.nan, np.nan, 1.0, 2.0, date]] * 2, columns=["c", "d", "a", "b", "date"]
        )
        expected["c"] = expected["c"].astype(object)
        expected["d"] = expected["d"].astype(object)

        result_b = result_a.append(s, ignore_index=True)
        tm.assert_frame_equal(result_b, expected)

        # column order is different
        expected = expected[["c", "d", "date", "a", "b"]]
        result = df.append([s, s], ignore_index=True)
        tm.assert_frame_equal(result, expected)

    def test_append_empty_tz_frame_with_datetime64ns(self):
        # https://github.com/pandas-dev/pandas/issues/35460
        df = DataFrame(columns=["a"]).astype("datetime64[ns, UTC]")

        # pd.NaT gets inferred as tz-naive, so append result is tz-naive
        result = df.append({"a": pd.NaT}, ignore_index=True)
        expected = DataFrame({"a": [pd.NaT]}).astype("datetime64[ns]")
        tm.assert_frame_equal(result, expected)

        # also test with typed value to append
        df = DataFrame(columns=["a"]).astype("datetime64[ns, UTC]")
        result = df.append(
            Series({"a": pd.NaT}, dtype="datetime64[ns]"), ignore_index=True
        )
        expected = DataFrame({"a": [pd.NaT]}).astype("datetime64[ns]")
        tm.assert_frame_equal(result, expected)
