Collect Data

import wget
import os
import datetime
import pandas as pd
from pathlib import Path
from dotenv import load_dotenv

load_dotenv("../../.env")

import sys

sys.path.append("../..")
from src import utils

BASE_PATH = os.getenv("LOCAL_DATA_PATH", "../../data")
dataset_base_path = Path(f"{BASE_PATH}/raw/fedora-devel-list/")
dataset_base_path.mkdir(parents=True, exist_ok=True)

CURRENT_DATE = datetime.datetime.now().replace(day=1)
START_DATE = datetime.date(2018, 1, 1)

dates = pd.date_range(START_DATE, CURRENT_DATE, freq="m")

if os.getenv("RUN_IN_AUTOMATION"):
    dates = [dates[-1]]

# iterate over dates and download the corresponding *.gz files
base_url = "https://lists.fedoraproject.org/archives/list/devel@lists.\
fedoraproject.org/export/devel@lists.fedoraproject.org.mbox.gz"

for i, date in enumerate(dates):
    url = f"{base_url}?start={date.year}-{date.month}-01&end={date.year}-{date.month}-{date.day}"
    print(url)
    wget.download(
        url,
        out=f"{dataset_base_path}/fedora-devel-{date.year}-{date.month}.mbox.gz",
    )

new_files = [
    f"{dataset_base_path}/fedora-devel-{date.year}-{date.month}.mbox.gz"
    for date in dates
]

if os.getenv("RUN_IN_AUTOMATION"):
    utils.upload_files((f, f"raw/{Path(f).name}") for f in new_files)