Contributor Analysis

import pandas as pd
import os
import datetime
import re
from pathlib import Path
from dotenv import load_dotenv
import matplotlib.pyplot as plt
import seaborn as sns

load_dotenv("../../.env")

import sys

sys.path.append("../..")
from src import utils

BASE_PATH = os.getenv("LOCAL_DATA_PATH", "../../data/")

df = utils.load_dataset(f"{BASE_PATH}/interim/metadata/")

df.head()

# get all participants regradles of response
def match_threads(subject):
    return re.sub(r"^{0}".format(re.escape("Re: ")), "", subject)


# remove trailing emials
def remove_email(text):
    return re.sub(r" <.*", "", text)


# convert date string to datetime object
def parse_date(date):
    return pd.to_datetime(date)

# apply our transformations

df["Subject"] = df["Subject"].apply(match_threads)
df["Date"] = df["Date"].apply(parse_date)
df["Chunk"] = df["Date"].apply(lambda x: datetime.date(x.year, x.month, 1))
df = df.sort_values(by="Date")
df.reset_index(inplace=True, drop=True)
df.head()

# Quantify askers over the entire dataset
Monthly = df.groupby("Chunk").count()
Monthly.head(15)

sns.set(rc={"figure.figsize": (20, 10)})
plt.plot(Monthly.Subject)
plt.title("Number of emails per month")
plt.ylabel("Number of emails")
plt.xlabel("Date")
plt.show()

# plot # of threads

Subject = df.groupby(["Chunk"]).Subject.unique().apply(len)
Subject.head(5)

plt.plot(Subject)
plt.title("Number of threads per month")
plt.ylabel("Number of threads")
plt.xlabel("Date")
plt.show()

# number 1 subject per month

top_subjects = {}

for month in df.Chunk.unique():
    top_threads = [
        df[df["Chunk"] == month].Subject.value_counts().head(5).index.tolist()
    ]
    # thread_count = df[df["Chunk"] == month].Subject.value_counts().head(3).tolist()
    # current_month = list(zip(top_threads,thread_count))
    top_subjects[month] = top_threads

x = pd.DataFrame(top_subjects).T
x

x = x.join(Subject)
x = x.join(Monthly["Message-ID"])

x.columns = ["Top_5_Threads", "Number_of_threads", "Number_of_Messages"]
x

contributor_data_set = x

new_files = (
    (contributor_data_set, f"{BASE_PATH}/processed/contributors.csv"),
)

Path(f"{BASE_PATH}/processed").mkdir(parents=True, exist_ok=True)

contributor_data_set.to_csv(new_files[0][1], header=False)

if os.getenv("RUN_IN_AUTOMATION"):
    utils.upload_files(
        (f, f"processed/{Path(f).stem}/contributors.csv") for _, f in new_files
    )