# -*- encoding: utf-8 -*-
#
# Copyright © 2014-2016 eNovance
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import functools
import math
import operator

import fixtures
import iso8601
import numpy

from gnocchi import carbonara
from gnocchi.tests import base


def datetime64(*args):
    return numpy.datetime64(datetime.datetime(*args))


class TestBoundTimeSerie(base.BaseTestCase):
    def test_benchmark(self):
        self.useFixture(fixtures.Timeout(300, gentle=True))
        carbonara.BoundTimeSerie.benchmark()

    @staticmethod
    def test_base():
        carbonara.BoundTimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [3, 5, 6])

    def test_block_size(self):
        ts = carbonara.BoundTimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 5),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [5, 6],
            block_size=numpy.timedelta64(5, 's'))
        self.assertEqual(2, len(ts))
        ts.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 10), 3),
                                   (datetime64(2014, 1, 1, 12, 0, 11), 4)],
                                  dtype=carbonara.TIMESERIES_ARRAY_DTYPE))
        self.assertEqual(2, len(ts))

    def test_block_size_back_window(self):
        ts = carbonara.BoundTimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [3, 5, 6],
            block_size=numpy.timedelta64(5, 's'),
            back_window=1)
        self.assertEqual(3, len(ts))
        ts.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 10), 3),
                                   (datetime64(2014, 1, 1, 12, 0, 11), 4)],
                                  dtype=carbonara.TIMESERIES_ARRAY_DTYPE))
        self.assertEqual(3, len(ts))

    def test_block_size_unordered(self):
        ts = carbonara.BoundTimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 5),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [5, 23],
            block_size=numpy.timedelta64(5, 's'))
        self.assertEqual(2, len(ts))
        ts.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 11), 3),
                                   (datetime64(2014, 1, 1, 12, 0, 10), 4)],
                                  dtype=carbonara.TIMESERIES_ARRAY_DTYPE))
        self.assertEqual(2, len(ts))

    def test_duplicate_timestamps(self):
        ts = carbonara.BoundTimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [10, 23])
        self.assertEqual(2, len(ts))
        self.assertEqual(10.0, ts[0][1])
        self.assertEqual(23.0, ts[1][1])

        ts.set_values(numpy.array([(datetime64(2014, 1, 1, 13, 0, 10), 3),
                                   (datetime64(2014, 1, 1, 13, 0, 11), 9),
                                   (datetime64(2014, 1, 1, 13, 0, 11), 8),
                                   (datetime64(2014, 1, 1, 13, 0, 11), 7),
                                   (datetime64(2014, 1, 1, 13, 0, 11), 4)],
                                  dtype=carbonara.TIMESERIES_ARRAY_DTYPE))
        self.assertEqual(4, len(ts))
        self.assertEqual(10.0, ts[0][1])
        self.assertEqual(23.0, ts[1][1])
        self.assertEqual(3.0, ts[2][1])
        self.assertEqual(9.0, ts[3][1])


class TestAggregatedTimeSerie(base.BaseTestCase):
    def test_benchmark(self):
        self.useFixture(fixtures.Timeout(300, gentle=True))
        carbonara.AggregatedTimeSerie.benchmark()

    def test_fetch_basic(self):
        ts = carbonara.AggregatedTimeSerie.from_data(
            timestamps=[datetime64(2014, 1, 1, 12, 0, 0),
                        datetime64(2014, 1, 1, 12, 0, 4),
                        datetime64(2014, 1, 1, 12, 0, 9)],
            values=[3, 5, 6],
            aggregation=carbonara.Aggregation(
                "mean", numpy.timedelta64(1, 's'), None))
        self.assertEqual(
            [(datetime64(2014, 1, 1, 12), 3),
             (datetime64(2014, 1, 1, 12, 0, 4), 5),
             (datetime64(2014, 1, 1, 12, 0, 9), 6)],
            list(ts.fetch()))
        self.assertEqual(
            [(datetime64(2014, 1, 1, 12, 0, 4), 5),
             (datetime64(2014, 1, 1, 12, 0, 9), 6)],
            list(ts.fetch(
                from_timestamp=datetime64(2014, 1, 1, 12, 0, 4))))
        self.assertEqual(
            [(datetime64(2014, 1, 1, 12, 0, 4), 5),
             (datetime64(2014, 1, 1, 12, 0, 9), 6)],
            list(ts.fetch(
                from_timestamp=numpy.datetime64(iso8601.parse_date(
                    "2014-01-01 12:00:04")))))
        self.assertEqual(
            [(datetime64(2014, 1, 1, 12, 0, 4), 5),
             (datetime64(2014, 1, 1, 12, 0, 9), 6)],
            list(ts.fetch(
                from_timestamp=numpy.datetime64(iso8601.parse_date(
                    "2014-01-01 13:00:04+01:00")))))

    def test_before_epoch(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(1950, 1, 1, 12),
             datetime64(2014, 1, 1, 12),
             datetime64(2014, 1, 1, 12)],
            [3, 5, 6])

        self.assertRaises(carbonara.BeforeEpochError,
                          ts.group_serie, 60)

    @staticmethod
    def _resample(ts, sampling, agg, derived=False):
        aggregation = carbonara.Aggregation(agg, sampling, None)
        grouped = ts.group_serie(sampling)
        if derived:
            grouped = grouped.derived()
        return carbonara.AggregatedTimeSerie.from_grouped_serie(
            grouped, aggregation)

    def test_derived_mean(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime.datetime(2014, 1, 1, 12, 0, 0),
             datetime.datetime(2014, 1, 1, 12, 0, 4),
             datetime.datetime(2014, 1, 1, 12, 1, 2),
             datetime.datetime(2014, 1, 1, 12, 1, 14),
             datetime.datetime(2014, 1, 1, 12, 1, 24),
             datetime.datetime(2014, 1, 1, 12, 2, 4),
             datetime.datetime(2014, 1, 1, 12, 2, 35),
             datetime.datetime(2014, 1, 1, 12, 2, 42),
             datetime.datetime(2014, 1, 1, 12, 3, 2),
             datetime.datetime(2014, 1, 1, 12, 3, 22),  # Counter reset
             datetime.datetime(2014, 1, 1, 12, 3, 42),
             datetime.datetime(2014, 1, 1, 12, 4, 9)],
            [50, 55, 65, 66, 70, 83, 92, 103, 105, 5, 7, 23])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), 'mean',
                            derived=True)

        self.assertEqual(5, len(ts))
        self.assertEqual(
            [(datetime64(2014, 1, 1, 12, 0, 0), 5),
             (datetime64(2014, 1, 1, 12, 1, 0), 5),
             (datetime64(2014, 1, 1, 12, 2, 0), 11),
             (datetime64(2014, 1, 1, 12, 3, 0), -32),
             (datetime64(2014, 1, 1, 12, 4, 0), 16)],
            list(ts.fetch(
                from_timestamp=datetime64(2014, 1, 1, 12))))

    def test_derived_hole(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime.datetime(2014, 1, 1, 12, 0, 0),
             datetime.datetime(2014, 1, 1, 12, 0, 4),
             datetime.datetime(2014, 1, 1, 12, 1, 2),
             datetime.datetime(2014, 1, 1, 12, 1, 14),
             datetime.datetime(2014, 1, 1, 12, 1, 24),
             datetime.datetime(2014, 1, 1, 12, 3, 2),
             datetime.datetime(2014, 1, 1, 12, 3, 22),
             datetime.datetime(2014, 1, 1, 12, 3, 42),
             datetime.datetime(2014, 1, 1, 12, 4, 9)],
            [50, 55, 65, 66, 70, 105, 108, 200, 202])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), 'last',
                            derived=True)

        self.assertEqual(4, len(ts))
        self.assertEqual(
            [(datetime64(2014, 1, 1, 12, 0, 0), 5),
             (datetime64(2014, 1, 1, 12, 1, 0), 4),
             (datetime64(2014, 1, 1, 12, 3, 0), 92),
             (datetime64(2014, 1, 1, 12, 4, 0), 2)],
            list(ts.fetch(
                from_timestamp=datetime64(2014, 1, 1, 12))))

    def test_74_percentile_serialized(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [3, 5, 6])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), '74pct')

        self.assertEqual(1, len(ts))
        self.assertEqual(5.48, ts[datetime64(2014, 1, 1, 12, 0, 0)][1])

        # Serialize and unserialize
        key = ts.get_split_key()
        o, s = ts.serialize(key)
        saved_ts = carbonara.AggregatedTimeSerie.unserialize(
            s, key, ts.aggregation)

        self.assertEqual(ts.aggregation, saved_ts.aggregation)

        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [3, 5, 6])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), '74pct')
        saved_ts.merge(ts)

        self.assertEqual(1, len(ts))
        self.assertEqual(5.48, ts[datetime64(2014, 1, 1, 12, 0, 0)][1])

    def test_95_percentile(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [3, 5, 6])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), '95pct')

        self.assertEqual(1, len(ts))
        self.assertEqual(5.9000000000000004,
                         ts[datetime64(2014, 1, 1, 12, 0, 0)][1])

    def _do_test_aggregation(self, name, v1, v2, v3):
        # NOTE(gordc): test data must have a group of odd count to properly
        # test 50pct test case.
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 10),
             datetime64(2014, 1, 1, 12, 0, 20),
             datetime64(2014, 1, 1, 12, 0, 30),
             datetime64(2014, 1, 1, 12, 0, 40),
             datetime64(2014, 1, 1, 12, 1, 0),
             datetime64(2014, 1, 1, 12, 1, 10),
             datetime64(2014, 1, 1, 12, 1, 20),
             datetime64(2014, 1, 1, 12, 1, 30),
             datetime64(2014, 1, 1, 12, 1, 40),
             datetime64(2014, 1, 1, 12, 1, 50),
             datetime64(2014, 1, 1, 12, 2, 0),
             datetime64(2014, 1, 1, 12, 2, 10)],
            [3, 5, 2, 3, 5, 8, 11, 22, 10, 42, 9, 4, 2])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), name)

        self.assertEqual(3, len(ts))
        self.assertEqual(v1, ts[datetime64(2014, 1, 1, 12, 0, 0)][1])
        self.assertEqual(v2, ts[datetime64(2014, 1, 1, 12, 1, 0)][1])
        self.assertEqual(v3, ts[datetime64(2014, 1, 1, 12, 2, 0)][1])

    def test_aggregation_first(self):
        self._do_test_aggregation('first', 3, 8, 4)

    def test_aggregation_last(self):
        self._do_test_aggregation('last', 5, 9, 2)

    def test_aggregation_count(self):
        self._do_test_aggregation('count', 5, 6, 2)

    def test_aggregation_sum(self):
        self._do_test_aggregation('sum', 18, 102, 6)

    def test_aggregation_mean(self):
        self._do_test_aggregation('mean', 3.6, 17, 3)

    def test_aggregation_median(self):
        self._do_test_aggregation('median', 3.0, 10.5, 3)

    def test_aggregation_50pct(self):
        self._do_test_aggregation('50pct', 3.0, 10.5, 3)

    def test_aggregation_56pct(self):
        self._do_test_aggregation('56pct', 3.4800000000000004,
                                  10.8, 3.120000000000001)

    def test_aggregation_min(self):
        self._do_test_aggregation('min', 2, 8, 2)

    def test_aggregation_max(self):
        self._do_test_aggregation('max', 5, 42, 4)

    def test_aggregation_std(self):
        self._do_test_aggregation('std', 1.3416407864998738,
                                  13.266499161421599, 1.4142135623730951)

    def test_aggregation_std_with_unique(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0)], [3])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), 'std')
        self.assertEqual(0, len(ts), str(ts.values))

        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9),
             datetime64(2014, 1, 1, 12, 1, 6)],
            [3, 6, 5, 9])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), "std")

        self.assertEqual(1, len(ts))
        self.assertEqual(1.5275252316519465,
                         ts[datetime64(2014, 1, 1, 12, 0, 0)][1])

    def test_different_length_in_timestamps_and_data(self):
        self.assertRaises(
            ValueError,
            carbonara.AggregatedTimeSerie.from_data,
            carbonara.Aggregation('mean', numpy.timedelta64(3, 's'), None),
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [3, 5])

    def test_truncate(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [3, 5, 6])
        ts = self._resample(ts, numpy.timedelta64(1, 's'), 'mean')

        ts.truncate(datetime64(2014, 1, 1, 12, 0, 0))

        self.assertEqual(2, len(ts))
        self.assertEqual(5, ts[0][1])
        self.assertEqual(6, ts[1][1])

    def test_down_sampling(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9)],
            [3, 5, 7])
        ts = self._resample(ts, numpy.timedelta64(300, 's'), 'mean')

        self.assertEqual(1, len(ts))
        self.assertEqual(5, ts[datetime64(2014, 1, 1, 12, 0, 0)][1])

    def test_down_sampling_and_truncate(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 1, 4),
             datetime64(2014, 1, 1, 12, 1, 9),
             datetime64(2014, 1, 1, 12, 2, 12)],
            [3, 5, 7, 1])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), 'mean')

        ts.truncate(datetime64(2014, 1, 1, 12, 0, 59))

        self.assertEqual(2, len(ts))
        self.assertEqual(6, ts[datetime64(2014, 1, 1, 12, 1, 0)][1])
        self.assertEqual(1, ts[datetime64(2014, 1, 1, 12, 2, 0)][1])

    def test_down_sampling_and_truncate_and_method_max(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 1, 4),
             datetime64(2014, 1, 1, 12, 1, 9),
             datetime64(2014, 1, 1, 12, 2, 12)],
            [3, 5, 70, 1])
        ts = self._resample(ts, numpy.timedelta64(60, 's'), 'max')

        ts.truncate(datetime64(2014, 1, 1, 12, 0, 59))

        self.assertEqual(2, len(ts))
        self.assertEqual(70, ts[datetime64(2014, 1, 1, 12, 1, 0)][1])
        self.assertEqual(1, ts[datetime64(2014, 1, 1, 12, 2, 0)][1])

    @staticmethod
    def _resample_and_merge(ts, agg_dict):
        """Helper method that mimics _compute_splits_operations workflow."""
        grouped = ts.group_serie(agg_dict['sampling'])
        existing = agg_dict.get('return')
        agg_dict['return'] = carbonara.AggregatedTimeSerie.from_grouped_serie(
            grouped, carbonara.Aggregation(
                agg_dict['agg'], agg_dict['sampling'], None))
        if existing:
            existing.merge(agg_dict['return'])
            agg_dict['return'] = existing

    def test_fetch(self):
        ts = {'sampling': numpy.timedelta64(60, 's'),
              'size': 10, 'agg': 'mean'}
        tsb = carbonara.BoundTimeSerie(block_size=ts['sampling'])

        tsb.set_values(numpy.array([
            (datetime64(2014, 1, 1, 11, 46, 4), 4),
            (datetime64(2014, 1, 1, 11, 47, 34), 8),
            (datetime64(2014, 1, 1, 11, 50, 54), 50),
            (datetime64(2014, 1, 1, 11, 54, 45), 4),
            (datetime64(2014, 1, 1, 11, 56, 49), 4),
            (datetime64(2014, 1, 1, 11, 57, 22), 6),
            (datetime64(2014, 1, 1, 11, 58, 22), 5),
            (datetime64(2014, 1, 1, 12, 1, 4), 4),
            (datetime64(2014, 1, 1, 12, 1, 9), 7),
            (datetime64(2014, 1, 1, 12, 2, 1), 15),
            (datetime64(2014, 1, 1, 12, 2, 12), 1),
            (datetime64(2014, 1, 1, 12, 3, 0), 3),
            (datetime64(2014, 1, 1, 12, 4, 9), 7),
            (datetime64(2014, 1, 1, 12, 5, 1), 15),
            (datetime64(2014, 1, 1, 12, 5, 12), 1),
            (datetime64(2014, 1, 1, 12, 6, 0, 2), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=ts))

        tsb.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 6), 5)],
                                   dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                       before_truncate_callback=functools.partial(
                           self._resample_and_merge, agg_dict=ts))

        self.assertEqual([
            (numpy.datetime64('2014-01-01T11:46:00.000000000'), 4.0),
            (numpy.datetime64('2014-01-01T11:47:00.000000000'), 8.0),
            (numpy.datetime64('2014-01-01T11:50:00.000000000'), 50.0),
            (datetime64(2014, 1, 1, 11, 54), 4.0),
            (datetime64(2014, 1, 1, 11, 56), 4.0),
            (datetime64(2014, 1, 1, 11, 57), 6.0),
            (datetime64(2014, 1, 1, 11, 58), 5.0),
            (datetime64(2014, 1, 1, 12, 1), 5.5),
            (datetime64(2014, 1, 1, 12, 2), 8.0),
            (datetime64(2014, 1, 1, 12, 3), 3.0),
            (datetime64(2014, 1, 1, 12, 4), 7.0),
            (datetime64(2014, 1, 1, 12, 5), 8.0),
            (datetime64(2014, 1, 1, 12, 6), 4.0)
        ], list(ts['return'].fetch()))

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 1), 5.5),
            (datetime64(2014, 1, 1, 12, 2), 8.0),
            (datetime64(2014, 1, 1, 12, 3), 3.0),
            (datetime64(2014, 1, 1, 12, 4), 7.0),
            (datetime64(2014, 1, 1, 12, 5), 8.0),
            (datetime64(2014, 1, 1, 12, 6), 4.0)
        ], list(ts['return'].fetch(datetime64(2014, 1, 1, 12, 0, 0))))

    def test_fetch_agg_pct(self):
        ts = {'sampling': numpy.timedelta64(1, 's'),
              'size': 3600 * 24, 'agg': '90pct'}
        tsb = carbonara.BoundTimeSerie(block_size=ts['sampling'])

        tsb.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 0), 3),
                                    (datetime64(2014, 1, 1, 12, 0, 0, 123), 4),
                                    (datetime64(2014, 1, 1, 12, 0, 2), 4)],
                                   dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                       before_truncate_callback=functools.partial(
                           self._resample_and_merge, agg_dict=ts))

        result = ts['return'].fetch(datetime64(2014, 1, 1, 12, 0, 0))
        reference = [
            (datetime64(
                2014, 1, 1, 12, 0, 0
            ), 3.9),
            (datetime64(
                2014, 1, 1, 12, 0, 2
            ), 4)
        ]

        self.assertEqual(len(reference), len(list(result)))

        for ref, res in zip(reference, result):
            self.assertEqual(ref[0], res[0])
            # Rounding \o/
            self.assertAlmostEqual(ref[1], res[1])

        tsb.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 2, 113), 110)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=ts))

        result = ts['return'].fetch(datetime64(2014, 1, 1, 12, 0, 0))
        reference = [
            (datetime64(
                2014, 1, 1, 12, 0, 0
            ), 3.9),
            (datetime64(
                2014, 1, 1, 12, 0, 2
            ), 99.4)
        ]

        self.assertEqual(len(reference), len(list(result)))

        for ref, res in zip(reference, result):
            self.assertEqual(ref[0], res[0])
            # Rounding \o/
            self.assertAlmostEqual(ref[1], res[1])

    def test_fetch_nano(self):
        ts = {'sampling': numpy.timedelta64(200, 'ms'),
              'size': 10, 'agg': 'mean'}
        tsb = carbonara.BoundTimeSerie(block_size=ts['sampling'])

        tsb.set_values(numpy.array([
            (datetime64(2014, 1, 1, 11, 46, 0, 200123), 4),
            (datetime64(2014, 1, 1, 11, 46, 0, 340000), 8),
            (datetime64(2014, 1, 1, 11, 47, 0, 323154), 50),
            (datetime64(2014, 1, 1, 11, 48, 0, 590903), 4),
            (datetime64(2014, 1, 1, 11, 48, 0, 903291), 4)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=ts))

        tsb.set_values(numpy.array([
            (datetime64(2014, 1, 1, 11, 48, 0, 821312), 5)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=ts))

        self.assertEqual([
            (datetime64(2014, 1, 1, 11, 46, 0, 200000), 6.0),
            (datetime64(2014, 1, 1, 11, 47, 0, 200000), 50.0),
            (datetime64(2014, 1, 1, 11, 48, 0, 400000), 4.0),
            (datetime64(2014, 1, 1, 11, 48, 0, 800000), 4.5)
        ], list(ts['return'].fetch()))
        self.assertEqual(numpy.timedelta64(200000000, 'ns'),
                         ts['return'].aggregation.granularity)

    def test_fetch_agg_std(self):
        # NOTE (gordc): this is a good test to ensure we drop NaN entries
        # 2014-01-01 12:00:00 will appear if we don't dropna()
        ts = {'sampling': numpy.timedelta64(60, 's'),
              'size': 60, 'agg': 'std'}
        tsb = carbonara.BoundTimeSerie(block_size=ts['sampling'])

        tsb.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 0), 3),
                                    (datetime64(2014, 1, 1, 12, 1, 4), 4),
                                    (datetime64(2014, 1, 1, 12, 1, 9), 7),
                                    (datetime64(2014, 1, 1, 12, 2, 1), 15),
                                    (datetime64(2014, 1, 1, 12, 2, 12), 1)],
                                   dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                       before_truncate_callback=functools.partial(
                           self._resample_and_merge, agg_dict=ts))

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 1, 0), 2.1213203435596424),
            (datetime64(2014, 1, 1, 12, 2, 0), 9.8994949366116654),
        ], list(ts['return'].fetch(datetime64(2014, 1, 1, 12, 0, 0))))

        tsb.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 2, 13), 110)],
                                   dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                       before_truncate_callback=functools.partial(
                           self._resample_and_merge, agg_dict=ts))

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 1, 0), 2.1213203435596424),
            (datetime64(2014, 1, 1, 12, 2, 0), 59.304300012730948),
        ], list(ts['return'].fetch(datetime64(2014, 1, 1, 12, 0, 0))))

    def test_fetch_agg_max(self):
        ts = {'sampling': numpy.timedelta64(60, 's'),
              'size': 60, 'agg': 'max'}
        tsb = carbonara.BoundTimeSerie(block_size=ts['sampling'])

        tsb.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 0), 3),
                                    (datetime64(2014, 1, 1, 12, 1, 4), 4),
                                    (datetime64(2014, 1, 1, 12, 1, 9), 7),
                                    (datetime64(2014, 1, 1, 12, 2, 1), 15),
                                    (datetime64(2014, 1, 1, 12, 2, 12), 1)],
                                   dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                       before_truncate_callback=functools.partial(
                           self._resample_and_merge, agg_dict=ts))

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0), 3),
            (datetime64(2014, 1, 1, 12, 1, 0), 7),
            (datetime64(2014, 1, 1, 12, 2, 0), 15),
        ], list(ts['return'].fetch(datetime64(2014, 1, 1, 12, 0, 0))))

        tsb.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 2, 13), 110)],
                                   dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                       before_truncate_callback=functools.partial(
                           self._resample_and_merge, agg_dict=ts))

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0), 3),
            (datetime64(2014, 1, 1, 12, 1, 0), 7),
            (datetime64(2014, 1, 1, 12, 2, 0), 110),
        ], list(ts['return'].fetch(datetime64(2014, 1, 1, 12, 0, 0))))

    def test_serialize(self):
        ts = {'sampling': numpy.timedelta64(500, 'ms'), 'agg': 'mean'}
        tsb = carbonara.BoundTimeSerie(block_size=ts['sampling'])

        tsb.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 0, 1234), 3),
            (datetime64(2014, 1, 1, 12, 0, 0, 321), 6),
            (datetime64(2014, 1, 1, 12, 1, 4, 234), 5),
            (datetime64(2014, 1, 1, 12, 1, 9, 32), 7),
            (datetime64(2014, 1, 1, 12, 2, 12, 532), 1)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=ts))

        key = ts['return'].get_split_key()
        o, s = ts['return'].serialize(key)
        self.assertEqual(ts['return'],
                         carbonara.AggregatedTimeSerie.unserialize(
                             s, key, ts['return'].aggregation))

    def test_no_truncation(self):
        ts = {'sampling': numpy.timedelta64(60, 's'), 'agg': 'mean'}
        tsb = carbonara.BoundTimeSerie()

        for i in range(1, 11):
            tsb.set_values(numpy.array([
                (datetime64(2014, 1, 1, 12, i, i), float(i))],
                dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                before_truncate_callback=functools.partial(
                    self._resample_and_merge, agg_dict=ts))
            tsb.set_values(numpy.array([
                (datetime64(2014, 1, 1, 12, i, i + 1), float(i + 1))],
                dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                before_truncate_callback=functools.partial(
                    self._resample_and_merge, agg_dict=ts))
            self.assertEqual(i, len(list(ts['return'].fetch())))

    def test_back_window(self):
        """Back window testing.

        Test the back window on an archive is not longer than the window we
        aggregate on.
        """
        ts = {'sampling': numpy.timedelta64(1, 's'), 'size': 60, 'agg': 'mean'}
        tsb = carbonara.BoundTimeSerie(block_size=ts['sampling'])

        tsb.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 1, 2300), 1),
            (datetime64(2014, 1, 1, 12, 0, 1, 4600), 2),
            (datetime64(2014, 1, 1, 12, 0, 2, 4500), 3),
            (datetime64(2014, 1, 1, 12, 0, 2, 7800), 4),
            (datetime64(2014, 1, 1, 12, 0, 3, 8), 2.5)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=ts))

        self.assertEqual(
            [
                (datetime64(2014, 1, 1, 12, 0, 1), 1.5),
                (datetime64(2014, 1, 1, 12, 0, 2), 3.5),
                (datetime64(2014, 1, 1, 12, 0, 3), 2.5),
            ],
            list(ts['return'].fetch()))

    def test_back_window_ignore(self):
        """Back window testing.

        Test the back window on an archive is not longer than the window we
        aggregate on.
        """
        ts = {'sampling': numpy.timedelta64(1, 's'), 'size': 60, 'agg': 'mean'}
        tsb = carbonara.BoundTimeSerie(block_size=ts['sampling'])

        tsb.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 1, 2300), 1),
            (datetime64(2014, 1, 1, 12, 0, 1, 4600), 2),
            (datetime64(2014, 1, 1, 12, 0, 2, 4500), 3),
            (datetime64(2014, 1, 1, 12, 0, 2, 7800), 4),
            (datetime64(2014, 1, 1, 12, 0, 3, 8), 2.5)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=ts))

        self.assertEqual(
            [
                (datetime64(2014, 1, 1, 12, 0, 1), 1.5),
                (datetime64(2014, 1, 1, 12, 0, 2), 3.5),
                (datetime64(2014, 1, 1, 12, 0, 3), 2.5),
            ],
            list(ts['return'].fetch()))

        tsb.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 2, 99), 9)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=ts))

        self.assertEqual(
            [
                (datetime64(2014, 1, 1, 12, 0, 1), 1.5),
                (datetime64(2014, 1, 1, 12, 0, 2), 3.5),
                (datetime64(2014, 1, 1, 12, 0, 3), 2.5),
            ],
            list(ts['return'].fetch()))

        tsb.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 2, 99), 9),
            (datetime64(2014, 1, 1, 12, 0, 3, 9), 4.5)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=ts))

        self.assertEqual(
            [
                (datetime64(2014, 1, 1, 12, 0, 1), 1.5),
                (datetime64(2014, 1, 1, 12, 0, 2), 3.5),
                (datetime64(2014, 1, 1, 12, 0, 3), 3.5),
            ],
            list(ts['return'].fetch()))

    def test_split_key(self):
        self.assertEqual(
            numpy.datetime64("2014-10-07"),
            carbonara.SplitKey.from_timestamp_and_sampling(
                numpy.datetime64("2015-01-01T15:03"),
                numpy.timedelta64(3600, 's')))
        self.assertEqual(
            numpy.datetime64("2014-12-31 18:00"),
            carbonara.SplitKey.from_timestamp_and_sampling(
                numpy.datetime64("2015-01-01 15:03:58"),
                numpy.timedelta64(58, 's')))

        key = carbonara.SplitKey.from_timestamp_and_sampling(
            numpy.datetime64("2015-01-01 15:03"),
            numpy.timedelta64(3600, 's'))

        self.assertGreater(key, numpy.datetime64("1970"))

        self.assertGreaterEqual(key, numpy.datetime64("1970"))

    def test_split_key_cmp(self):
        dt1 = numpy.datetime64("2015-01-01T15:03")
        dt1_1 = numpy.datetime64("2015-01-01T15:03")
        dt2 = numpy.datetime64("2015-01-05T15:03")
        td = numpy.timedelta64(60, 's')
        td2 = numpy.timedelta64(300, 's')

        self.assertEqual(
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td),
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td))
        self.assertEqual(
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td),
            carbonara.SplitKey.from_timestamp_and_sampling(dt1_1, td))
        self.assertNotEqual(
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td),
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td))
        self.assertNotEqual(
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td),
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td2))

        self.assertLess(
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td),
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td))
        self.assertLessEqual(
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td),
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td))

        self.assertGreater(
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td),
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td))
        self.assertGreaterEqual(
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td),
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td))

    def test_split_key_cmp_negative(self):
        dt1 = numpy.datetime64("2015-01-01T15:03")
        dt1_1 = numpy.datetime64("2015-01-01T15:03")
        dt2 = numpy.datetime64("2015-01-05T15:03")
        td = numpy.timedelta64(60, 's')
        td2 = numpy.timedelta64(300, 's')

        self.assertFalse(
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td) !=
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td))
        self.assertFalse(
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td) !=
            carbonara.SplitKey.from_timestamp_and_sampling(dt1_1, td))
        self.assertFalse(
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td) ==
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td))
        self.assertFalse(
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td) ==
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td2))
        self.assertRaises(
            TypeError,
            operator.le,
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td),
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td2))
        self.assertRaises(
            TypeError,
            operator.ge,
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td),
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td2))
        self.assertRaises(
            TypeError,
            operator.gt,
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td),
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td2))
        self.assertRaises(
            TypeError,
            operator.lt,
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td),
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td2))

        self.assertFalse(
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td) >=
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td))
        self.assertFalse(
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td) >
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td))

        self.assertFalse(
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td) <=
            carbonara.SplitKey.from_timestamp_and_sampling(dt1, td))
        self.assertFalse(
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td) <
            carbonara.SplitKey.from_timestamp_and_sampling(dt2, td))

    def test_split_key_next(self):
        self.assertEqual(
            numpy.datetime64("2015-03-06"),
            next(carbonara.SplitKey.from_timestamp_and_sampling(
                numpy.datetime64("2015-01-01 15:03"),
                numpy.timedelta64(3600, 's'))))
        self.assertEqual(
            numpy.datetime64("2015-08-03"),
            next(next(carbonara.SplitKey.from_timestamp_and_sampling(
                numpy.datetime64("2015-01-01T15:03"),
                numpy.timedelta64(3600, 's')))))

    def test_split(self):
        sampling = numpy.timedelta64(5, 's')
        points = 100000
        ts = carbonara.TimeSerie.from_data(
            timestamps=list(map(datetime.datetime.utcfromtimestamp,
                                range(points))),
            values=list(range(points)))
        agg = self._resample(ts, sampling, 'mean')

        grouped_points = list(agg.split())

        self.assertEqual(
            math.ceil((points / sampling.astype(float))
                      / carbonara.SplitKey.POINTS_PER_SPLIT),
            len(grouped_points))
        self.assertEqual("0.0",
                         str(carbonara.SplitKey(grouped_points[0][0], 0)))
        # 3600 × 5s = 5 hours
        self.assertEqual(datetime64(1970, 1, 1, 5),
                         grouped_points[1][0])
        self.assertEqual(carbonara.SplitKey.POINTS_PER_SPLIT,
                         len(grouped_points[0][1]))

    def test_from_timeseries(self):
        sampling = numpy.timedelta64(5, 's')
        points = 100000
        ts = carbonara.TimeSerie.from_data(
            timestamps=list(map(datetime.datetime.utcfromtimestamp,
                                range(points))),
            values=list(range(points)))
        agg = self._resample(ts, sampling, 'mean')

        split = [t[1] for t in list(agg.split())]

        self.assertEqual(agg,
                         carbonara.AggregatedTimeSerie.from_timeseries(
                             split, aggregation=agg.aggregation))

    def test_resample(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 4),
             datetime64(2014, 1, 1, 12, 0, 9),
             datetime64(2014, 1, 1, 12, 0, 11),
             datetime64(2014, 1, 1, 12, 0, 12)],
            [3, 5, 6, 2, 4])
        agg_ts = self._resample(ts, numpy.timedelta64(5, 's'), 'mean')
        self.assertEqual(3, len(agg_ts))

        agg_ts = agg_ts.resample(numpy.timedelta64(10, 's'))
        self.assertEqual(2, len(agg_ts))
        self.assertEqual(5, agg_ts[0][1])
        self.assertEqual(3, agg_ts[1][1])

    def test_iter(self):
        ts = carbonara.TimeSerie.from_data(
            [datetime64(2014, 1, 1, 12, 0, 0),
             datetime64(2014, 1, 1, 12, 0, 11),
             datetime64(2014, 1, 1, 12, 0, 12)],
            [3, 5, 6])
        self.assertEqual([
            (numpy.datetime64('2014-01-01T12:00:00'), 3.),
            (numpy.datetime64('2014-01-01T12:00:11'), 5.),
            (numpy.datetime64('2014-01-01T12:00:12'), 6.),
        ], list(ts))
