Quantify Time to Test

import os
import json
import datetime
import gzip
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

from ipynb.fs.defs.metric_template import testgrid_labelwise_encoding
from ipynb.fs.defs.metric_template import CephCommunication
from ipynb.fs.defs.metric_template import save_to_disk, read_from_disk
from dotenv import load_dotenv, find_dotenv

load_dotenv(find_dotenv())

## Specify variables

METRIC_NAME = "time_to_test"

# Specify the path for input grid data,
INPUT_DATA_PATH = "../../../../data/raw/testgrid_183.json.gz"

# Specify the path for output metric data
OUTPUT_DATA_PATH = f"../../../../data/processed/metrics/{METRIC_NAME}"

## CEPH Bucket variables
## Create a .env file on your local with the correct configs,
s3_endpoint_url = os.getenv("S3_ENDPOINT")
s3_access_key = os.getenv("S3_ACCESS_KEY")
s3_secret_key = os.getenv("S3_SECRET_KEY")
s3_bucket = os.getenv("S3_BUCKET")
s3_path = os.getenv("S3_PROJECT_KEY", "ai4ci/testgrid/metrics")
s3_input_data_path = "raw_data"
AUTOMATION = os.getenv("IN_AUTOMATION")

## Import data
timestamp = datetime.datetime.today()

if AUTOMATION:
    filename = f"testgrid_{timestamp.day}{timestamp.month}.json"
    cc = CephCommunication(s3_endpoint_url, s3_access_key, s3_secret_key, s3_bucket)
    s3_object = cc.s3_resource.Object(s3_bucket, f"{s3_input_data_path}/{filename}")
    file_content = s3_object.get()["Body"].read().decode("utf-8")
    testgrid_data = json.loads(file_content)

else:
    with gzip.open(INPUT_DATA_PATH, "rb") as read_file:
        testgrid_data = json.load(read_file)

# We pass status code as None since we are not filtering by Status here
testgrid_list = testgrid_labelwise_encoding(testgrid_data, None, False)

len(testgrid_list)

testgrid_list[0]

# Convert to dataframe
testgrid_df = pd.DataFrame(
    testgrid_list, columns=["timestamp", "tab", "grid", "test", "test_duration"]
)
testgrid_df.head()

testgrid_df = testgrid_df.loc[testgrid_df.test_duration.notnull()]

testgrid_df.head()

len(testgrid_df)

average_time = (
    testgrid_df.groupby(["tab", "grid", "test"])["test_duration"].mean().reset_index()
)

average_time.head()

df = (
    testgrid_df.groupby(["tab", "grid", "test"])
    .apply(lambda x: [list(x["test_duration"]), list(x["timestamp"])])
    .apply(pd.Series)
    .reset_index()
)
df.columns = ["tab", "grid", "test", "test_duration", "timestamp"]

df.head()

def plot_build_graph(tab, grid, dataframe):

    test_duration = dataframe.loc[
        (df["tab"] == tab) & (dataframe["grid"] == grid), "test_duration"
    ].iloc[0]
    timestamp = dataframe.loc[
        (df["tab"] == tab) & (dataframe["grid"] == grid), "timestamp"
    ].iloc[0]

    time_taken_df = pd.DataFrame(
        {"timestamp": timestamp, "time_duration": test_duration}
    )

    sns.set(rc={"figure.figsize": (17, 10)})
    sns.lineplot(x="timestamp", y="time_duration", data=time_taken_df)
    plt.xlabel("Timestamps")
    plt.ylabel("Time Taken to Run Job or Build (mins)")
    plt.title("Job/Build Duration over time ")
    plt.show()

plot_build_graph(
    '"redhat-assisted-installer"',
    "periodic-ci-openshift-release-master-nightly-4.7-e2e-metal-assisted",
    df,
)

timestamp = datetime.datetime.now()

if AUTOMATION:
    cc = CephCommunication(s3_endpoint_url, s3_access_key, s3_secret_key, s3_bucket)
    cc.upload_to_ceph(
        average_time,
        s3_path,
        f"{METRIC_NAME}/{METRIC_NAME}-{timestamp.year}-{timestamp.month}-{timestamp.day}.parquet",
    )
else:
    save_to_disk(
        average_time,
        OUTPUT_DATA_PATH,
        f"{METRIC_NAME}-{timestamp.year}-{timestamp.month}-{timestamp.day}.parquet",
    )

## Sanity check to see if the dataset is the same
if AUTOMATION:
    sanity_check = cc.read_from_ceph(
        s3_path,
        f"{METRIC_NAME}/{METRIC_NAME}-{timestamp.year}-{timestamp.month}-{timestamp.day}.parquet",
    )
else:
    sanity_check = read_from_disk(
        OUTPUT_DATA_PATH,
        f"{METRIC_NAME}-{timestamp.year}-{timestamp.month}-{timestamp.day}.parquet",
    )

sanity_check