'''
Created on 20-Jan-2022

@author: bthakur
'''

import inspect
import traceback
import random
import numpy as np
import collections

from sympy import simplify, Rational, latex, Eq, symbols

from xv.km.util import latex_fraction
from sympy.core.expr import UnevaluatedExpr

from xv.km.ui import SvgHelper
from xv.util.data import CacheUtil
import matplotlib.pyplot as plt

from ...util import MathUtil
from xv.util import toIntOrFloat, list_to_str, html_list, html_table, html_table_rows

from xv.util import UtilClass

# _data_handling_helper
# DataHandlingHelper
class DataHandlingHelper(UtilClass):

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        pass

    def get_random_forcasting(self, **kwargs):
        x, y = symbols("x, y", real = True)

        title = kwargs.get('title', f'{x} vs {y}')
        x_label = kwargs.get('x_label', f'{x}')
        y_label = kwargs.get('y_label', f'{y}')

        size = random.randint(5, 10)
        mu = random.randint(11, 100)
        sigma = mu * random.randint(5, 10) / 100
        #define data
        min_x = random.randint(1, 100)
        xs = random.sample(range(min_x, min_x + size * 3), size)
        xs = np.array(xs)
        xs.sort()

        ys = [int(y_val)
              for y_val in np.random.normal(mu, sigma, size)]
        ys = np.array(ys)

        xy = np.column_stack((xs, ys))

        #find line of best fit
        a, b = np.polyfit(xs, ys, 1)
        eq = Eq(y, a * x + b)

        ys_predicted = a * xs + b
        ys_predicted = [int(round(y_val, 0)) for y_val in ys_predicted]
        ys_predicted = np.array(ys_predicted)

        xy_predicted = np.column_stack((xs, ys_predicted))

        svg_str = self.plot_forcasting(xs, ys, ys_predicted, a, b,
                                        x_label = x_label,
                                        y_label = y_label,
                                        title = title,
                                        )

        svg_str_annotated = self.plot_forcasting(xs, ys, ys_predicted, a, b,
                                        x_label = x_label,
                                        y_label = y_label,
                                        title = title,
                                        point_annotated = True
                                        )

        result = {
                'x': x,
                'y': y,
                'xs': xs,
                'ys': ys,
                'xy': xy,
                'ys_predicted': ys_predicted,
                'xy_predicted': xy_predicted,
                'x_label': x_label,
                'y_label': y_label,
                'title': title,
                'eq': eq,
                'svg_str': svg_str,
                'svg_str_annotated': svg_str_annotated,
            }

        qas = self.get_forcasting_qas(result)
        qas_format_result = self._format_qas_to_str(qas)
        result.update(qas_format_result)

        return result
        pass

    def get_forcasting_qas(self, result):
        qas = []

        x = result['x']
        y = result['y']
        xy = result['xy']
        xy_predicted = result['xy_predicted']

        pred_data = [[xy_val[0], xy_val[1], xy_predicted[idx][1]]
                     for idx, xy_val in enumerate(xy)]


        header = [f'${x}$', f'Actual ${y}$', f'Predicted ${y}$',
                  "Deviation",
                  "Absolute</br>Deviation",
                  ]
        pred_data = [(*item,
                      item[-1] - item[-2],
                      abs(item[-1] - item[-2]),
                       )
                     for item in pred_data]


        pred_data = np.array(pred_data)
        header = np.array(header)

        cols = [0, 1]
        xy_table = html_table(pred_data[:,cols], header = header[cols])

        cols = [0, 2]
        xy_predicted_table = html_table(pred_data[:,cols], header = header[cols])

        cols = [0, 1, 2]
        xy_y_predicted_table = html_table(pred_data[:,cols], header = header[cols])

        cols = [0, 1, 2, 3, 4]
        dev_table = html_table(pred_data[:,cols], header = header[cols])


        min_abs_dev = min([item[4] for item in pred_data])
        min_abs_dev_data = [item for item in pred_data if item[4] == min_abs_dev]
        min_abs_dev_data = np.array(min_abs_dev_data)
        cols = [0, 1, 2, 3, 4]
        min_abs_dev_table = html_table(min_abs_dev_data[:,cols], header = header[cols])


        max_abs_dev = max([item[4] for item in pred_data])
        max_abs_dev_data = [item for item in pred_data if item[4] == max_abs_dev]
        max_abs_dev_data = np.array(max_abs_dev_data)
        cols = [0, 1, 2, 3, 4]
        max_abs_dev_table = html_table(max_abs_dev_data[:,cols], header = header[cols])




        question = f"""
        Write actual values of ${y}$ for every ${x}$.
        """
        answer = f"""
        Each (${x}$, ${y}$) pair is as follows:
        {xy_table}
        """
        qas.append([question.strip(), answer.strip()])


        question = f"""
        Write predicted values of ${y}$ for every ${x}$.
        """
        answer = f"""
        Each predicted (${x}$, ${y}$) pair is as follows:
        {xy_predicted_table}
        """
        qas.append([question.strip(), answer.strip()])


        question = f"""
        Write actual and predicted values of ${y}$ for every ${x}$.
        """
        answer = f"""
        Each ${x}$, actual ${y}$ and predicted ${y}$ set is as follows:
        {xy_y_predicted_table}
        """
        qas.append([question.strip(), answer.strip()])


        question = f"""
        Calculate deviation in prediction.
        """
        answer = f"""
        Deviation:
        {dev_table}
        """
        qas.append([question.strip(), answer.strip()])


        question = f"""
        What is minimum absolute deviation?
        """
        answer = f"""
        minimum absolute deviation = {min_abs_dev}
        {min_abs_dev_table}
        """
        qas.append([question.strip(), answer.strip()])


        question = f"""
        What is maximum absolute deviation?
        """
        answer = f"""
        maximum absolute deviation = {max_abs_dev}
        {max_abs_dev_table}
        """
        qas.append([question.strip(), answer.strip()])


        question = f"""
        """
        answer = f"""

        """
        qas.append([question.strip(), answer.strip()])

        return qas



    def plot_forcasting(self, xs, ys, ys_predicted, a, b,
                         x_label = None,
                         y_label = None,
                         title = None,
                         point_annotated = False,
                         figsize = (10,6),
                         use_cache = False
                         ):
        try:

            verbose = self.verbose

            self.cacheUtil = CacheUtil(use_cache = use_cache, verbose = verbose)

            cache_file_path, exists_cached_file = self.cacheUtil.get_cached_path(
                    self.__class__, inspect.stack()[0][3],
                    xs, ys, ys_predicted, a, b,
                    title,
                    point_annotated,
                    x_label,
                    y_label,
                    figsize,
                    title,
                    file_prefix = f"{self.__class__.__name__}_{inspect.stack()[0][3]}_"
                    )

            if use_cache and exists_cached_file:
                return self.cacheUtil.load_file(cache_file_path)

            plt.ioff()
            plt.figure(figsize = figsize)

            ax = plt.gca()


            #add points to plot
            ax.scatter(xs, ys, color='blue')

            #add line of best fit to plot
            ax.plot(xs, ys_predicted, color='red', linestyle='-', linewidth=2)

            #add fitted regression equation to plot
            #ax.text(1, 17, 'y = ' + '{:.2f}'.format(b) + ' + {:.2f}'.format(a) + 'x', size=14)

            if point_annotated:
                ax.scatter(xs, ys_predicted, color='red')

                y_offset = 0.2 * (max(ys) - min(ys)) / len(ys)

                for x_idx, x_val in enumerate(xs):


                    y_val = ys[x_idx]
                    y_pedicted_val = ys_predicted[x_idx]

                    line_segment = [(x_val, x_val), (y_val, y_pedicted_val)]
                    ax.plot(*line_segment, color = "yellowgreen")

                    offset_mul = 1 + 0.5 *(x_idx % 2)
                    if y_val > y_pedicted_val:
                        offset1 = y_offset
                    else:
                        offset1 = -y_offset * offset_mul

                    offset2 = - offset1

                    ax.annotate(f"({x_val}, {y_val})",
                                xy=(x_val, y_val + offset1),
                                color = "blue"
                                )
                    ax.annotate(f"({x_val}, {y_pedicted_val})",
                                xy=(x_val, y_pedicted_val + offset2)
                                )

                    pass
                pass


            #ax.grid(b=True, which='major', color='#ff66ff', linestyle='-')
            ax.grid(which='major', color='#ff66ff', linestyle='-')

            ax.minorticks_on()
            #ax.grid(b=True, which='minor', color='#cccccc', linestyle='--')
            ax.grid(which='minor', color='#cccccc', linestyle='--')

            ax.set_xlabel(x_label)
            ax.set_ylabel(y_label)

            ax.set_title(title)
            #plt.legend()

            plt.tight_layout()

            svgHelper = SvgHelper()
            svg_str = svgHelper.plt_to_svg()

            if not svg_str:
                if self.verbose:
                    print(f"Error: svg_str could not be initialized")
            else:
                self.cacheUtil.save_file(svg_str, cache_file_path)

            #plt.figure().clear()
            plt.clf()
            plt.close()

            return svg_str
        except Exception as e:
            sError = f"""{inspect.stack()[0][3]} Error: {e}:
            xs = {xs}
            ys = {ys}
            x_label = {x_label}
            y_label = {y_label}
            title = {title}
            """
            self._log(sError)
            self._log(traceback.format_exc())
            return None
        pass


    def get_random_series(self, category = 'single_digit', size = None):
        if size is None:
            size = random.randint(5, 10)

        if category == 'single_digit':
            random_series = np.random.randint(0, 9, size)
        else:
            random_series = np.random.randint(0, 1000, size)
        return list(random_series)
        pass

    def get_random_data_set(self, category = 'single_digit', size = None):
        if size is None:
            size = random.randint(25, 50)

        data_set = []
        while len(data_set) < size:
            if category == 'single_digit':
                v = random.randint(0, 9)
            else:
                v = random.randint(0, 1000)

            f = random.randint(0, 10)
            data_set.extend([v for _ in range(f)])
            pass

        random.shuffle(data_set)
        return data_set
        pass

    def get_closest_pair(self, data_sets, data_labels = None):
        if data_labels is None:
            data_labels = list('ABCDE')[:len(data_sets)]

        data_sets_str = [f"<b>{data_labels[i]}</b> = " + list_to_str(data_set)
                         for i, data_set in enumerate(data_sets)]
        data_sets_str = "\n\n".join(data_sets_str)

        deviation_data_sets = self.get_deviation_data_sets(data_sets)

        deviation_html_tables = []
        deviation_summary = []
        for k, ds in deviation_data_sets.items():
            i, j = [int(x) for x in k.split(',')]

            ds_html_table = html_table(ds[1:], header = ds[0])

            ds_html_header = f"Deviation between <b>{data_labels[i]}</b> and <b>{data_labels[j]}</b>"
            deviation_html_tables.append(ds_html_header + ds_html_table)

            ds_summary = [f"<b>{data_labels[i]}{data_labels[j]}</b>"]
            ds_summary.extend(ds[-1][2:5])
            deviation_summary.append(ds_summary)
            pass

        deviation_summary_labels = ['']
        deviation_summary_labels.extend(list(deviation_data_sets.values())[0][0][2:5])

        deviation_html_tables = "\n\n".join(deviation_html_tables)
        deviation_summary_table = html_table(deviation_summary,
                                             header = deviation_summary_labels)

        abs_devations = [x[2] for x in deviation_summary]

        min_abs_devation = min(abs_devations)
        min_abs_devation_idxs = [i for i, x in enumerate(abs_devations)
                                if x == min_abs_devation]

        closet_by_abs_devation = [x[0] for i, x in enumerate(deviation_summary)
                         if i in min_abs_devation_idxs
                         ]
        closet_by_abs_devation_str = list_to_str(closet_by_abs_devation)

        sqr_devations = [x[3] for x in deviation_summary]

        min_sqr_devation = min(sqr_devations)
        min_sqr_devation_idxs = [i for i, x in enumerate(sqr_devations)
                                if x == min_sqr_devation]

        closet_by_sqr_devation = [x[0] for i, x in enumerate(deviation_summary)
                         if i in min_sqr_devation_idxs
                         ]
        closet_by_sqr_devation_str = list_to_str(closet_by_sqr_devation)


        result = {
            'data_sets': data_sets,
            'data_sets_str': data_sets_str,
            'deviation_data_sets': deviation_data_sets,
            'deviation_html_tables': deviation_html_tables,
            'deviation_summary_table': deviation_summary_table,
            'min_abs_devation': min_abs_devation,
            'closet_by_abs_devation': closet_by_abs_devation,
            'closet_by_abs_devation_str': closet_by_abs_devation_str,
            'min_sqr_devation': min_sqr_devation,
            'closet_by_sqr_devation': closet_by_sqr_devation,
            'closet_by_sqr_devation_str': closet_by_sqr_devation_str,
            }
        return result
        pass

    def get_deviation_data_sets(self, data_sets):
        deviation_data_sets = {}
        for i, data_set in enumerate(data_sets[:-1]):
            for j in range(i+1, len(data_sets)):
                deviation_data_set = self.get_deviation_data_set(data_set, data_sets[j])
                deviation_data_sets[f"{i},{j}"] = deviation_data_set
                pass
            pass
        return deviation_data_sets
        pass

    def get_deviation_data_set(self, data_set1, data_set2):
        combined_data_set = list(zip(data_set1, data_set2))

        cum_x_y = 0
        cum_abs_x_y = 0
        cum_sqr_x_y = 0

        deviation_label = ['x', 'y', 'Deviation<br/>x - y', 'Min abs dev<br/>abs(x - y)',
                           'Square dev<br/>(x - y)**2',
                           'Cumulative <br/>x - y',
                           'Cumulative <br/>abs(x - y)',
                           'Cumulative <br/>(x - y)**2'
                           ]
        deviation_data = [deviation_label]
        for x, y in combined_data_set:
            data_row = []
            data_row.append(x)
            data_row.append(y)
            data_row.append(x - y)
            data_row.append(abs(x - y))
            data_row.append((x - y) ** 2)

            cum_x_y = cum_x_y + (x - y)
            cum_abs_x_y = cum_abs_x_y + abs(x - y)
            cum_sqr_x_y = cum_sqr_x_y + (x - y) ** 2

            data_row.append(cum_x_y)
            data_row.append(cum_abs_x_y)
            data_row.append(cum_sqr_x_y)

            deviation_data.append(data_row)
            pass

        last_row = [f'Total',
                    '',
                    deviation_data[-1][5],  #'x - y',
                    deviation_data[-1][6], #'abs(x - y)',
                    deviation_data[-1][7], #'$(x - y)^2$',
                    '', #'Cumulative <br/>x - y',
                    '', #'Cumulative <br/>abs(x - y)',
                    '', #'Cumulative <br/>$(x - y)^2$'
                    ]

        deviation_data.append(last_row)
        return deviation_data
        pass

    def get_freq_dict(self, data_set):
        freq_dict = collections.Counter(data_set)
        freq_dict = collections.OrderedDict(sorted(freq_dict.items(),
                                                   key=lambda item: item[0]))
        return freq_dict

    def analyze_data(self, data_set):
        freq_dict = self.get_freq_dict(data_set)
        data_set_str = list_to_str(data_set)

        freq_list = list(freq_dict.items())
        freq_html = html_table(freq_list, header = ['Item (x)', 'Frequency (f)'])

        item_sum_list = [ [x[0], x[1], f"{x[0]} * {x[1]}", x[0] * x[1]] for x in freq_list]
        item_sum_html = html_table(item_sum_list,
                                   header = ['x', 'f', 'f * x', 'Sum (f * x)'])

        cum_sum_list = []
        cum_sum = 0
        cum_f = 0
        for k, f in freq_list:
            cum_sum = cum_sum + k * f
            cum_f = cum_f + f
            row_item = [k, f, cum_f,  k * f, cum_sum]
            cum_sum_list.append(row_item)
            pass

        ltd = f"""td style="font-size:1.5rem;" """

        cum_sum_rows = html_table_rows(cum_sum_list,
                                  header = ['x', 'f',
                                            'Cumulative f', "Sum (f * x)" , "Cumulative Sum"])
        last_row = f"<{ltd}>Total</td><{ltd}>{cum_f}</td><td></td><{ltd}>{cum_sum}</td><td></td>"
        cum_sum_rows.append(f"<tr>{last_row}</tr>")
        cum_sum_html = "<table>" + "".join(cum_sum_rows) + "</table>"



        result = {
            'data_set': data_set,
            'data_set_str': data_set_str,
            'freq_dict': freq_dict,
            'freq_html': freq_html,
            'item_sum_list': item_sum_list,
            'item_sum_html': item_sum_html,
            'cum_sum_html': cum_sum_html,
            'cum_sum': cum_sum,
            'cum_f': cum_f,
            'cum_sum_list': cum_sum_list,
            }

        qas = self.get_data_analysis_qas(result)
        qas_format_result = self._format_qas_to_str(qas)
        result.update(qas_format_result)
        return result
        pass



    def get_data_analysis_qas(self, result):
        data_set = result['data_set']
        freq_dict = result['freq_dict']

        items_sorted = sorted(list(data_set))
        items_sorted_str = list_to_str(items_sorted)

        highest_freq = max(list(freq_dict.values()))
        highest_freq_items = [k for k, v in freq_dict.items() if v == highest_freq ]
        highest_freq_items_str = list_to_str(highest_freq_items)

        lowest_freq = min(list(freq_dict.values()))
        lowest_freq_items = [k for k, v in freq_dict.items() if v == lowest_freq ]
        lowest_freq_items_str = list_to_str(lowest_freq_items)

        qas = []

        question = f"""
        How many times each item appears (it is also called frequency)?
        """
        answer = f"""
        The number of times an item appears is also called its frequency.
        {result['freq_html']}
        """
        qas.append([question.strip(), answer.strip()])

        question = f"""
        What is sum of each item separately?
        """
        answer = f"""
        Sum of each item:
        {result['item_sum_html']}
        """
        qas.append([question.strip(), answer.strip()])

        question = f"""
        Show cumulative frequency of each item.
        """
        answer = f"""
        Keep on adding row frequencies to get cumulative frequency.
        Keep on adding row sums to get cumulative sum
        {result['cum_sum_html']}
        """
        qas.append([question.strip(), answer.strip()])

        question = f"""
        What is total count of all items?
        """
        answer = f"""
        {result['cum_f']}
        """
        qas.append([question.strip(), answer.strip()])

        question = f"""
        What is total sum of all items?
        """
        answer = f"""
        {result['cum_sum']}
        """
        qas.append([question.strip(), answer.strip()])

        question = f"""
        What is the value of the lowest item?
        """
        answer = f"""
        {min(data_set)}
        """
        qas.append([question.strip(), answer.strip()])

        question = f"""
        What is the value of the highest item?
        """
        answer = f"""
        {max(data_set)}
        """
        qas.append([question.strip(), answer.strip()])

        question = f"""
        What is the difference of term with highest value and  the term with lowest value?
        """
        answer = f"""
        (term with highest value) - (term with lowest value)
        = {max(data_set)} - {min(data_set)} = {max(data_set) - min(data_set)}
        The the difference of term with highest value and  the term with lowest value is also called <span style="font-size: 2rem;">span of data or terms</span>.
        """
        qas.append([question.strip(), answer.strip()])

        question = f"""
        Which number comes highest number of times?
        """
        answer = f"""
        {highest_freq_items_str}
        It comes {highest_freq} times.
        The number that comes highest number of times is also called <span style="font-size: 2rem;">mode</span>.
        """
        qas.append([question.strip(), answer.strip()])

        question = f"""
        Which number comes lowest number of times?
        """
        answer = f"""
        {lowest_freq_items_str}
        It comes {lowest_freq} times.
        """
        qas.append([question.strip(), answer.strip()])



        avg_latex1 = latex_fraction(f"\\text{{Sum of numbers}}",
                                       f"\\text{{Count of numbers}}")

        avg_latex2 = latex_fraction(result['cum_sum'],
                                       result['cum_f'])

        avg = toIntOrFloat(result['cum_sum'] / result['cum_f'])

        question = f"""
        What is average of all numbers?
        """
        answer = f"""
        average of all numbers $\\displaystyle
        = {avg_latex1}
        = {avg_latex2}
        = {avg}
        $
        The average is also called <span style="font-size: 2rem;">mean</span>.
        """
        qas.append([question.strip(), answer.strip()])



        middle_item_idx_latex = latex_fraction(f"{len(data_set)} + 1", 2)
        middle_item_idx = toIntOrFloat((len(data_set) + 1) / 2)

        question = f"""
        Find the item in the middle.
        """

        ltd = f"""td style="font-size:1.5rem;" """
        lspan = f"""span style="color:red; font-size:1.5rem;" """
        cum_sum_list = result['cum_sum_list']
        cum_f = result['cum_f']
        cum_freq_list = []
        median_value = None
        for item in cum_sum_list:
            if median_value is None and item[2] >= middle_item_idx:
                cum_freq_row = [f"<{lspan}>{item[0]}</span>",
                                item[1],
                                f"<{lspan}>{item[2]}</span>",
                                ]
                median_value = item[0]
            else:
                cum_freq_row = item[0:3]
                pass
            cum_freq_list.append(cum_freq_row)
            pass

        cum_freq_rows = html_table_rows(cum_freq_list,
                                  header = ['x', 'f',
                                            'Cumulative f'])
        last_row = f"<{ltd}>Total</td><{ltd}>{cum_f}</td>"
        cum_freq_rows.append(f"<tr>{last_row}</tr>")
        cum_freq_html = "<table>" + "".join(cum_freq_rows) + "</table>"


        answer = []
        answer_item = f"""
        The total count of items = {len(data_set)}
        The index of middle item = ${middle_item_idx_latex} = {middle_item_idx}$

        The cumulative frequency table is:
        {cum_freq_html}
        """
        answer.append(answer_item.strip())



        answer_item = f"""
        See the cumulative frequency column of the table.
        The <{lspan}>{middle_item_idx}th</span> item is {median_value}
        The middle term is also called <span style="font-size: 2rem;">median</span>.
        """
        answer.append(answer_item.strip())

        answer = "\n\n".join(answer)


        qas.append([question.strip(), answer.strip()])

        qas = [qa for qa in qas if qa[0].strip()]

        return qas

    def _format_qas_to_str(self, qas):
        result = {}
        qas = [x for x in qas if x[0].strip()]
        questions = [x[0] for x in qas]
        questions_str = html_list(questions, list_type = "ol")

        qas_str = []
        answers = []
        for idx, (question, answer) in enumerate(qas):
            question = f"<b>Q {idx + 1}.</b> {question}"
            qas_str.append(f"{question}\n{answer}")
            answers.append(f"<b>Answer {idx + 1}.</b> {answer}")
            pass

        qas_str = list_to_str(qas_str, sep = "\n\n")
        answers_str = list_to_str(answers, sep = "\n\n")

        result['qas'] = qas
        result['qas_str'] = qas_str
        result['questions'] = questions
        result['questions_str'] = questions_str
        result['answers'] = answers
        result['answers_str'] = answers_str
        return result

    pass
#class DataHandlingHelper:
