# imports
import copy
import datetime as dt

import numpy as np
import pandas as pd

from umap import UMAP
from sklearn.cluster import DBSCAN

import mlxtend.frequent_patterns

import plotly
import plotly.express as px
from plotly import graph_objects as go

import ipywidgets as widgets
from ipywidgets import interactive
from IPython.core.display import display, HTML

# set this to False to just view images inside notebook only
# set this to True to also save the images them locally
SAVE_PLOTS = True

# "tag" to identify images produced during current run of the notbeook
IMAGES_TAG = str(int(dt.datetime.now().timestamp()))

# visualization utils
def create_hoverinfo(row):
    """
    Helper function for labelling data on plotly scatter3d
    For input one-hot encoded row of symptoms, returns a string that is the
    concatenation of the symptoms shown by this deployment, joined by "<br>"
    The "<br>" ensures that only one symptom is displayed per line in the hover info box in plotly
    """
    symptomlist = row[row != 0].index.tolist()
    return "<br>".join(sorted(symptomlist))


# color scheme
custom_colors = copy.deepcopy(px.colors.qualitative.Dark24)
primary = custom_colors[0]
to_remove = [primary] + [custom_colors[i] for i in [5, 7]]
for t in to_remove:
    custom_colors.remove(t)

# YYYYMMDD string that specifies the date from which we want data
DATE_PREFIX = "20201203"

# combine all reports to create clusters df
clusters_df = pd.read_parquet(
    f"../../data/processed/clusters_df_{DATE_PREFIX}.parquet"
)

print(clusters_df.shape)
clusters_df.head()

# combine symptoms data collected at all time intervals for the current date
clusters_symptoms_df = pd.read_parquet(
    f"../../data/processed/clusters_symptoms_df_{DATE_PREFIX}.parquet"
)

print(clusters_symptoms_df.shape)
clusters_symptoms_df.head()

# get known diagnosis labels
diag_names = [
    "ignored-symptoms",
    "sdn-issue",
    "kubelet-down",
    "BZ-1821905-DefaultSecurityContextConstraints_Mutated",
    "4.3-major-upgrade-autoscaler",
]
diagnoses_df = pd.read_parquet(
    f"../../data/processed/diagnoses_df_{DATE_PREFIX}.parquet"
)

print(diagnoses_df.shape)
diagnoses_df.head()

# how many ci/cd clusters matched the signature/definition of known diagnoses
diagnoses_df[diagnoses_df["symptom_id"].str.startswith("diagnosis")]

# mapping from maj.min to all maj.min.patch
patches_per_majmin = clusters_df.groupby("current_version_maj_min").apply(
    lambda g: sorted(g["current_version"].unique())
)
patches_per_majmin["ALL"] = sorted(clusters_df["current_version"].unique())

# how many depls of each maj.min
depls_per_majmin = clusters_df[
    clusters_df.cluster_id.isin(clusters_symptoms_df.cluster_id)
]["current_version_maj_min"].value_counts()
depls_per_majmin["ALL"] = depls_per_majmin.sum()

# add number of dpels info to maj.min names, sort by num depls
patches_per_majmin = patches_per_majmin.sort_index()
new_index = [
    "{0} ({1:4d} depls)".format(val, ct)
    for val, ct in depls_per_majmin.iteritems()
]
patches_per_majmin.index = sorted(new_index)


# func needed to ensure both select boxes are displayed
def do_nothing(dummy_input):
    return


# func to update pathces shown based on maj.min selected
def select_patch_version(maj_min_version):
    maj_min_patch_widget.options = patches_per_majmin[maj_min_version]
    maj_min_patch_widget.value = patches_per_majmin[maj_min_version]


# widget to select maj.min, showing in descending order of # depls
maj_min_widget_options = sorted(
    patches_per_majmin.index,
    key=lambda x: int(x.rsplit(" ", 1)[0].rsplit("(")[-1]),
    reverse=True,
)
maj_min_widget = widgets.Select(
    options=maj_min_widget_options,
    value=maj_min_widget_options[0],
    description="Major.Minor",
)

# widget to select maj.min.patch, all selected by default
maj_min_patch_widget = widgets.SelectMultiple(
    options=patches_per_majmin[maj_min_widget.value],
    value=patches_per_majmin[maj_min_widget.value],
    description="Patches",
)

# emails + count (how many depls of that email domain)
email_vc = clusters_df[
    clusters_df.cluster_id.isin(clusters_symptoms_df.cluster_id)
]["email_domain"].value_counts()
emails_with_ct = []
for i, v in email_vc.items():
    emails_with_ct.append(f"{i}  ({v} depls)")

# widget to select email domain, all selected by default
email_widget = widgets.Dropdown(
    options=[f"ALL ({(email_vc.sum())})"] + emails_with_ct,
    description="Email Domain",
)

# display all widgets
display(interactive(select_patch_version, maj_min_version=maj_min_widget))
display(interactive(do_nothing, dummy_input=maj_min_patch_widget))
display(interactive(do_nothing, dummy_input=email_widget))

# filter data and pivot so that each row is a deployment

# keep only deployments of selected version
X_df = clusters_symptoms_df[
    clusters_symptoms_df.cluster_id.isin(
        clusters_df["cluster_id"][
            clusters_df["current_version"].isin(maj_min_patch_widget.value)
        ]
    )
]

# keep only deployments of selected email
EMAIL = email_widget.value.split()[0]
if EMAIL.lower() != "all":
    X_df = X_df[
        X_df.cluster_id.isin(
            clusters_df[clusters_df["email_domain"] == EMAIL].cluster_id
        )
    ]

# drop nan rows, if any (ideally shouldnt exist)
nan_count = X_df.isna().sum().sum()
if nan_count != 0:
    print(f"Found {nan_count} nans in filtered X_df")
    X_df = X_df.dropna()

# ================================ checks for pivot ================================ #
# def check_pivot(X_df, X_df_new):
#     for n,g in X_df.groupby('cluster_id'):
#         res = X_df_new.loc[n]
#         if not res[res==1].index.difference(g['symptom_id'].values).empty:
#             pdb.set_trace()
#             return False
#     return True
# pre = X_df['cluster_id'].nunique()
# ================================================================================== #

# pivot so that each row represents a cluster id, and the columns are the symptoms
# NOTE: we need a numerical value column for pivot to work. assign value=1 dummy col
X_df = X_df.assign(value=1).pivot_table(
    index="cluster_id",
    columns=X_df.columns.drop("cluster_id").tolist(),
    values="value",
)

# ================================ checks for pivot ================================ #
# post = len(X_df_new)
# assert pre==post
# assert check_pivot(X_df, X_df_new)
# ================================================================================== #

X_df = X_df.fillna(value=0)

# # add UPI/IPI info
# # NOTE: if there are nans for install_type for some depl ids in clusters_df,
# # those depl ids would still show up here, but with 0's in is_UPI and is_IPI
# X_df = X_df.assign(
#     is_UPI=X_df.index.isin(
#         clusters_df["cluster_id"][
#             clusters_df["install_type"] == "UPI"
#         ].unique()
#     ).astype(np.float64)
# )
# X_df = X_df.assign(
#     is_IPI=X_df.index.isin(
#         clusters_df["cluster_id"][
#             clusters_df["install_type"] == "IPI"
#         ].unique()
#     ).astype(np.float64)
# )

print(
    "The following is a glimpse of the sliced/diced and formatted dataset that will be fed to ML models:"
)
print(f"Shape = {X_df.shape}")
X_df.head()

# drop columns ccx believes are not very informative
ignored_symptoms = [
    c
    for c in X_df.columns
    if any(i in c for i in ["ClusterOperatorDown", "ClusterOperatorDegraded"])
]
ignored_symptoms += [
    "rule|operators_check|OPERATOR_ISSUE",
    "rule|pods_check|POD_ISSUE",
    "rule|pods_check_containers|POD_CONTAINER_ISSUE",
]
ignored_symptoms += ["alert|AlertmanagerReceiversNotConfigured"]

print("Dropping symptoms:\n")
for i in ignored_symptoms:
    print(i)
X_df = X_df.drop(ignored_symptoms, axis=1)

# drop rows that are all 0s coz no useful data
X_df = X_df[X_df.sum(axis=1) != 0]
X_df.shape

# define umap instance
umap_instance = UMAP(
    n_components=3,
    metric="hamming",
    n_neighbors=64,
    min_dist=0.005,
    random_state=42,
)

# fit umap to our data and transform (reduce dimensions) the data using it
X_df_trans = umap_instance.fit_transform(X_df)

# visualize deployments and known diagnoses
fig = go.Figure()

# data to display when hover over - list of all symptoms
hoverdata = X_df.apply(create_hoverinfo, axis=1)

# add all deployments
fig.add_trace(
    go.Scatter3d(
        name="all_symptoms",
        mode="markers",
        x=X_df_trans[:, 0],
        y=X_df_trans[:, 1],
        z=X_df_trans[:, 2],
        hovertemplate="<b>_id</b>: %{customdata}<br>" + "<br>%{text}",
        customdata=hoverdata.index.tolist(),
        text=hoverdata.values,
        marker=dict(size=2, color=primary),
    )
)


for di, diag_name in enumerate(diag_names):
    # get deployments hit with current diagnosis
    labels = X_df.index.isin(
        diagnoses_df[diagnoses_df["symptom_id"] == f"diagnosis|{diag_name}"][
            "cluster_id"
        ].unique()
    )

    # add data points that have this diagnosis
    fig.add_trace(
        go.Scatter3d(
            name=diag_name,
            mode="markers",
            x=X_df_trans[labels, 0],
            y=X_df_trans[labels, 1],
            z=X_df_trans[labels, 2],
            hovertemplate="<b>_id</b>: %{customdata}<br>" + "<br>%{text}",
            customdata=hoverdata[labels].index.tolist(),
            text=hoverdata[labels].values,
            marker=dict(size=3, color=custom_colors[di]),
        )
    )

if SAVE_PLOTS:
    savedir = "../../reports/figures"
    fname = f"symptoms_{umap_instance.metric}_all_diagnoses"
    suffix = f"{DATE_PREFIX.replace('/', '_')}_{IMAGES_TAG}"

    print(f"Saving plot as '{savedir}_{fname}_{suffix}.html'")
    plotly.offline.plot(
        fig,
        filename=f"{savedir}/{fname}_{suffix}.html",
        auto_open=True,
    )
fig.show()

# fit a vanilla dbscan model
dbscan = DBSCAN(
    eps=0.325,
    n_jobs=-1,
)
dbscan.fit(X_df_trans)

# unique labels (clusters of deployments) found by dbscan
unique_dbscan_labels = np.unique(dbscan.labels_)

# visualize deployments, colored by dbscan cluster label
fig = go.Figure()

# data to display when hover over - list of all symptoms
hoverdata = X_df.apply(create_hoverinfo, axis=1)

for c in unique_dbscan_labels:
    # index where points are of this class
    c_pts = dbscan.labels_ == c

    # add those points
    fig.add_trace(
        go.Scatter3d(
            x=X_df_trans[c_pts, 0],
            y=X_df_trans[c_pts, 1],
            z=X_df_trans[c_pts, 2],
            name=f"Cluster ID {c}",
            mode="markers",
            hovertemplate="<b>_id</b>: %{customdata}<br>" + "<br>%{text}",
            customdata=hoverdata[c_pts].index.tolist(),
            text=hoverdata[c_pts].values,
            marker=dict(size=2),
        )
    )

if SAVE_PLOTS:
    savedir = "../../reports/figures"
    fname = f"dbscan_{dbscan.eps}_{dbscan.min_samples}"
    suffix = f"{DATE_PREFIX.replace('/', '_')}_{IMAGES_TAG}"

    print(f"Saving plot as '{savedir}/{fname}_{suffix}.html'")
    plotly.offline.plot(
        fig,
        filename=f"{savedir}/{fname}_{suffix}.html",
        auto_open=True,
    )
fig.show()

# for each cluster, what fraction of deployments have a particular diagnosis?
dbclust_diagnosis_df = pd.DataFrame(
    index=unique_dbscan_labels,
    columns=["cluster_size"] + [f"percent_{i}" for i in diag_names],
)
dbclust_diagnosis_df.index.rename("cluster_id", inplace=True)
for c in unique_dbscan_labels:
    clust_depls = X_df.index[dbscan.labels_ == c]
    dbclust_diagnosis_df.loc[c, "cluster_size"] = len(clust_depls)
    for diag in diag_names:
        does_clust_have_diag = clust_depls.isin(
            diagnoses_df[diagnoses_df["symptom_id"] == f"diagnosis|{diag}"][
                "cluster_id"
            ].unique()
        )
        try:
            dbclust_diagnosis_df.loc[
                c, f"percent_{diag}"
            ] = does_clust_have_diag.mean()
        except ZeroDivisionError:
            dbclust_diagnosis_df.loc[c, f"percent_{diag}"] = 0

pct_sorted_idx = (
    dbclust_diagnosis_df.drop("cluster_size", axis=1)
    .max(axis=1)
    .sort_values(ascending=False)
    .index
)

print(
    "The following table shows, what percent of deployments in a given cluster (group) had a particular diagnosis"
)
dbclust_diagnosis_df.reindex(pct_sorted_idx).head(10)

def get_frequent_symptom_combinations(
    symptom_onehot_df, algo="fpgrowth", min_support=0.9, drop_singles=False
):
    # calculate frequent patterns as per the algorithm name passed
    ret = getattr(mlxtend.frequent_patterns, algo)(
        df=symptom_onehot_df, min_support=min_support, use_colnames=True
    )

    # keep only those "combinations" whose length is >1
    if drop_singles:
        ret = ret[ret["itemsets"].apply(lambda x: len(x)) > 1]

    # friendlier index and column names
    ret = ret.rename(
        columns={
            "itemsets": "symptom_combination",
            "support": "percent_affected",
        }
    ).set_index("symptom_combination")

    # sort by percent_affected to bring attention to more frequent symptoms
    ret = ret.sort_values(by="percent_affected", ascending=False)

    return ret

# average value of each symptom in overall data
X_df.mean().sort_values(ascending=False).to_frame("percent_affected").head(10)

# find frequent combinations in the entire dataset
# NOTE: min_support = at least this % of depls should have this combination of symptoms
get_frequent_symptom_combinations(
    X_df, min_support=0.1, drop_singles=True
).head(10)

# specific cluster for which to extract frequent patterns
clustid = 9

# indexer into X_df. True where the deployment belongs to the cluster of id ==clustid
is_depl_in_clust = dbscan.labels_ == clustid

# how many deployments in this particular cluster
display(
    HTML(
        f"<h3> Num Deployments in this Cluster = {is_depl_in_clust.sum()}</h3>"
    )
)
display(
    HTML(
        f"<h3> Percent Deployments Assigned to this Cluster = {is_depl_in_clust.mean()}</h3>"
    )
)

# "average" symptom vector for this cluster
curr_clust_mean_symptom = (
    X_df[is_depl_in_clust].mean().to_frame("percent_affected")
)
display(HTML("<h3>Most Affecting Symptoms</h3>"))
display(
    curr_clust_mean_symptom.sort_values(
        "percent_affected", ascending=False
    ).head(10)
)

# "average" symptom vector for depls not in this cluster
other_clusts_mean_symptom = (
    X_df[~is_depl_in_clust].mean().to_frame("percent_affected")
)

# what symptoms have different occurence frequencies as compared to other clusters
diff = pd.merge(
    curr_clust_mean_symptom,
    other_clusts_mean_symptom,
    how="left",
    left_index=True,
    right_index=True,
    suffixes=("_this", "_others"),
)

# display in descending order of difference
display(HTML("<h3>Most <i>UNIQUELY</i> Affecting Symptoms</h3>"))
diff.reindex(
    (diff["percent_affected_this"] - diff["percent_affected_others"])
    .sort_values(ascending=False)
    .index
).head(10)

# frequent symptom cominations in depls of this cluster
curr_clust_combinations = get_frequent_symptom_combinations(
    X_df[is_depl_in_clust], min_support=0.25, drop_singles=False
)
display(HTML("<h3>Most Affecting Symptom Combinations</h3>"))
display(curr_clust_combinations.head(10))

# frequent symptom patterns in depls of all other clusters
# NOTE: since num total depls is high, and it can contain depls of different types, min_support is kept v low (~5%)
other_clusts_combinations = get_frequent_symptom_combinations(
    X_df[~is_depl_in_clust], min_support=0.01, drop_singles=False
)

# what symptom patterns have different frequencies in this cluster as compared to other clusters
diff = pd.merge(
    curr_clust_combinations,
    other_clusts_combinations,
    how="left",
    left_index=True,
    right_index=True,
    suffixes=("_this", "_others"),
).fillna(0.05)

# display in descending order of difference
display(HTML("<h3>Most <i>UNIQUELY</i> Affecting Symptom Combinations</h3>"))
diff.reindex(
    (diff["percent_affected_this"] - diff["percent_affected_others"])
    .sort_values(ascending=False)
    .index
).head(10)

# specific cluster for which to extract frequent patterns
clustid = 0

# indexer into X_df. True where the deployment belongs to the cluster of id ==clustid
is_depl_in_clust = dbscan.labels_ == clustid

# how many deployments in this particular cluster
display(
    HTML(
        f"<h3> Num Deployments in this Cluster = {is_depl_in_clust.sum()}</h3>"
    )
)
display(
    HTML(
        f"<h3> Percent Deployments Assigned to this Cluster = {is_depl_in_clust.mean()}</h3>"
    )
)

# "average" symptom vector for this cluster
curr_clust_mean_symptom = (
    X_df[is_depl_in_clust].mean().to_frame("percent_affected")
)
display(HTML("<h3>Most Affecting Symptoms</h3>"))
display(
    curr_clust_mean_symptom.sort_values(
        "percent_affected", ascending=False
    ).head(10)
)

# "average" symptom vector for depls not in this cluster
other_clusts_mean_symptom = (
    X_df[~is_depl_in_clust].mean().to_frame("percent_affected")
)

# what symptoms have different occurence frequencies as compared to other clusters
diff = pd.merge(
    curr_clust_mean_symptom,
    other_clusts_mean_symptom,
    how="left",
    left_index=True,
    right_index=True,
    suffixes=("_this", "_others"),
)

# display in descending order of difference
display(HTML("<h3>Most <i>UNIQUELY</i> Affecting Symptoms</h3>"))
diff.reindex(
    (diff["percent_affected_this"] - diff["percent_affected_others"])
    .sort_values(ascending=False)
    .index
).head(10)

# frequent symptom cominations in depls of this cluster
curr_clust_combinations = get_frequent_symptom_combinations(
    X_df[is_depl_in_clust],
    min_support=0.25,
    drop_singles=True,
)
display(HTML("<h3>Most Affecting Symptom Combinations</h3>"))
display(curr_clust_combinations.head(10))

# frequent symptom patterns in depls of all other clusters
# NOTE: since num total depls is high, and it can contain depls of different types, min_support is kept v low (~5%)
other_clusts_combinations = get_frequent_symptom_combinations(
    X_df[~is_depl_in_clust],
    min_support=0.05,
    drop_singles=True,
)

# what symptom patterns have different frequencies in this cluster as compared to other clusters
diff = pd.merge(
    curr_clust_combinations,
    other_clusts_combinations,
    how="left",
    left_index=True,
    right_index=True,
    suffixes=("_this", "_others"),
).fillna(0.05)

# display in descending order of difference
display(HTML("<h3>Most <i>UNIQUELY</i> Affecting Symptom Combinations</h3>"))
diff.reindex(
    (diff["percent_affected_this"] - diff["percent_affected_others"])
    .sort_values(ascending=False)
    .index
).head(10)