Email Data Scraping & Analytics with Python

Web scraping to extract top email senders

In this project, I’m writing a Python script that will be used for extracting email data from a Gmail account, creating a Pandas DataFrame, and performing some data analysis and visualization. In the end we will be able to identify the email IDs of top senders.

Step 1: Import Libraries

import imaplib
import email
import getpass
import pandas as pd
from datetime import datetime
from wordcloud import WordCloud
import matplotlib.pyplot as plt

Step 2: Connecting to the email account
In this step we are connecting the users gmail account.

username = "YourEmail@gmail.com"
password = "YourPassword"
mail = imaplib.IMAP4_SSL('imap.gmail.com')
mail.login(username, password)
mail.select("inbox")

Step 3: EDA
In this step we are connecting the users gmail account.

result, numbers = mail.uid('search', None, "ALL")
uids = numbers[0].split()
uids = [id.decode("utf-8") for id in uids]
uids = uids[-1:-3000:-1]
result, messages = mail.uid('fetch', ','.join(uids), '(BODY[HEADER.FIELDS (SUBJECT FROM DATE)])')

date_list = []
from_list = []
from_list1 = []
date_list1 = []
subject_text = []

for i, message in messages[::2]:
    msg = email.message_from_bytes(message)
    decode = email.header.decode_header(msg['Subject'])[0]
    if isinstance(decode[0], bytes):
        decoded = decode[0].decode(errors='ignore')
        subject_text.append(decoded)
    else:
        subject_text.append(decode[0])
    date_list.append(msg.get('date'))
    fromlist = msg.get('From')
    fromlist = fromlist.split("<")[0].replace('"', '')
    from_list1.append(fromlist)

date_list = pd.to_datetime(date_list)
date_list1 = [item.isoformat(' ')[:-6] for item in date_list]

Step 4: Count and display top senders
In this step we are connecting the users gmail account.

df = pd.DataFrame(data={'Date': date_list1, 'Sender': from_list1, 'Subject': subject_text})

sender_counts = df['Sender'].value_counts()
print(sender_counts.head(10))  # Display the top 10 senders

Step 5: Visualize the inbox
In this step we are using Wordcloud to see the most used words in the email subjects.

df.to_csv(r'C:\Users\User\Desktop\email_inbox.csv', index=False)

df_description = df.describe()
print("\nDataFrame Description:")
print(df_description)

text = ""
for item in df["Subject"]:
    if isinstance(item, str):
        text += " " + item
    text = text.replace("'", "")
    text = text.replace(",", "")
    text = text.replace('"', '')


wordcloud = WordCloud(width=800, height=800, background_color="white")

wordcloud.generate(text)
plt.figure(figsize=(8, 8))
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.margins(x=0, y=0)
plt.title("Most Used Subject Words", fontsize=20, ha="center", pad=20)
plt.show()