raw to metadata

import mailbox
import os
import datetime
import pandas as pd
from pathlib import Path
from dotenv import load_dotenv

load_dotenv("../../.env")

import sys

sys.path.append("../..")
from src import utils

# collect paths of either the entire dataset, or only the most recent month if running in automation

BASE_PATH = os.getenv("LOCAL_DATA_PATH", "../../data")
path = Path(BASE_PATH).joinpath("raw/fedora-devel-list")
mboxes = [x.name for x in list(path.glob("*.mbox"))]

if os.getenv("RUN_IN_AUTOMATION"):
    LAST_MONTH_DATE = datetime.datetime.now().replace(
        day=1
    ) - datetime.timedelta(days=1)
    y = LAST_MONTH_DATE.year
    m = LAST_MONTH_DATE.month
    mboxes = [x.name for x in list(path.glob(f"*{y}-{m}.mbox"))]

# Define a function to convert mbox data into row,column format for analysis using pandas


def mbox_to_meta(mbox):

    csv = []
    for msg in mbox:
        msg_id = msg["Message-ID"]
        date = msg["Date"]
        subject = msg["Subject"]

        csv.append((msg_id, date, subject))
    df = pd.DataFrame(
        csv,
        columns=["Message-ID", "Date", "Subject"],
    )

    return df

# Ensure datset location exists
dataset_base_path = Path(f"{BASE_PATH}/interim/metadata")
dataset_base_path.mkdir(parents=True, exist_ok=True)

# Register all created dataset slices for later upload
new_files = []

# Save each dataset into its own monthly csv
for mbox in mboxes:
    output_path = dataset_base_path.joinpath(f"{mbox}.csv")
    monthly_mbox = mailbox.mbox(path.joinpath(mbox))
    df = mbox_to_meta(monthly_mbox)
    df.to_csv(output_path)
    new_files.append(output_path)
    print(f"{output_path} saved")

# Push to Ceph
if os.getenv("RUN_IN_AUTOMATION"):
    utils.upload_files(
        (f.as_posix(), f"interim/metadata/{Path(f).name}") for f in new_files
    )