Task 2

notebooks/parse_slack_data.ipynb

@@ -24,11 +24,16 @@
     "from collections import Counter\n",
     "\n",
     "import pandas as pd\n",
-    "from matplotlib import pyplot as plt\n",
+    "import matplotlib.pyplot as plt\n",
     "import seaborn as sns\n",
     "\n",
+    "from textblob import TextBlob\n",
+    "\n",
     "from nltk.corpus import stopwords\n",
-    "from wordcloud import WordCloud"
+    "from wordcloud import WordCloud\n",
+    "\n",
+    "from sklearn.feature_extraction.text import CountVectorizer\n",
+    "from sklearn.decomposition import LatentDirichletAllocation"
    ]
   },
   {
@@ -218,6 +223,41 @@
     "    else: \n",
     "        print(f\"{column} not in data\")\n",
     "\n",
+    "\n",
+    "# Use SlackDataLoader to load data\n",
+    "data_loader = SlackDataLoader(\"path_to_slack_exported_data_folder\")\n",
+    "\n",
+    "# Get all channel messages\n",
+    "all_channel_msgs = pd.concat([data_loader.get_channel_messages(channel['name']) for channel in data_loader.channels])\n",
+    "\n",
+    "# Parse reaction data\n",
+    "reaction_data = parse_slack_reaction(\"path_to_reaction_data_folder\", \"General\")\n",
+    "\n",
+    "# Get community participation data\n",
+    "comm_participation = get_community_participation(\"path_to_community_participation_data_folder\")\n",
+    "\n",
+    "# Map user IDs to real names\n",
+    "user_profile = data_loader.get_users()\n",
+    "mapped_comm_dict = utils.map_userid_2_realname(user_profile, comm_participation, plot=True)\n",
+    "\n",
+    "# Get tagged users\n",
+    "tagged_users = utils.get_tagged_users(all_channel_msgs)\n",
+    "\n",
+    "# Get top 20 users\n",
+    "utils.get_top_20_user(all_channel_msgs, channel='General')\n",
+    "\n",
+    "# Draw average reply count\n",
+    "utils.draw_avg_reply_count(all_channel_msgs, channel='General')\n",
+    "\n",
+    "# Draw average reply users count\n",
+    "utils.draw_avg_reply_users_count(all_channel_msgs, channel='General')\n",
+    "\n",
+    "# Draw word cloud\n",
+    "utils.draw_wordcloud(all_channel_msgs['msg_content'], week='Week8-9 Combined')\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
     "def get_tagged_users(df):\n",
     "    \"\"\"get all @ in the messages\"\"\"\n",
     "\n",
@@ -259,6 +299,128 @@
     "    return ac_comm_dict"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Top and Bottom 10 Users by Reply Count\n",
+    "top_users_by_reply_count = all_channel_msgs.groupby('sender_name')['reply_count'].sum().sort_values(ascending=False)[:10]\n",
+    "bottom_users_by_reply_count = all_channel_msgs.groupby('sender_name')['reply_count'].sum().sort_values(ascending=True)[:10]\n",
+    "\n",
+    "# Print or visualize the results\n",
+    "print(\"Top Users by Reply Count:\")\n",
+    "print(top_users_by_reply_count)\n",
+    "\n",
+    "print(\"\\nBottom Users by Reply Count:\")\n",
+    "print(bottom_users_by_reply_count)\n",
+    "\n",
+    "all_channel_msgs['sender_name'].value_counts().sort_values(ascending=False)[:10]\n",
+    "all_channel_msgs['sender_name'].value_counts().sort_values(ascending=True)[:10]\n",
+    "\n",
+    "reaction_data.groupby('sender_name')['reaction_count'].sum().sort_values(ascending=False)[:10]\n",
+    "reaction_data.groupby('sender_name')['reaction_count'].sum().sort_values(ascending=True)[:10]\n",
+    "\n",
+    "\n",
+    "all_channel_msgs.sort_values(by='reply_count', ascending=False)[:10]\n",
+    "\n",
+    "\n",
+    "reaction_data.sort_values(by='reaction_count', ascending=False)[:10]\n",
+    "\n",
+    "# Assuming you have a column 'mentions' in your DataFrame\n",
+    "all_channel_msgs[all_channel_msgs['mentions'].apply(lambda x: len(x) if x else 0)].sort_values(by='mentions', ascending=False)[:10]\n",
+    "\n",
+    "all_channel_msgs['channel'].value_counts().idxmax()\n",
+    "\n",
+    "activity_df = pd.DataFrame({\n",
+    "    'Channel': all_channel_msgs['channel'],\n",
+    "    'Messages': all_channel_msgs.groupby('channel').size(),\n",
+    "    'Replies_Reactions_Sum': all_channel_msgs.groupby('channel')['reply_count'].sum() + reaction_data.groupby('channel')['reaction_count'].sum()\n",
+    "})\n",
+    "\n",
+    "plt.figure(figsize=(12, 8))\n",
+    "sns.scatterplot(x='Messages', y='Replies_Reactions_Sum', hue='Channel', data=activity_df)\n",
+    "plt.title('2D Scatter Plot of Channel Activity')\n",
+    "plt.show()\n",
+    "\n",
+    "# Convert to datetime\n",
+    "all_channel_msgs['msg_sent_time'] = pd.to_datetime(all_channel_msgs['msg_sent_time'])\n",
+    "all_channel_msgs['time_thread_start'] = pd.to_datetime(all_channel_msgs['time_thread_start'])\n",
+    "\n",
+    "# Calculate time difference\n",
+    "all_channel_msgs['time_to_reply'] = all_channel_msgs['time_thread_start'] - all_channel_msgs['msg_sent_time']\n",
+    "\n",
+    "# Calculate fraction replied within 5 minutes\n",
+    "fraction_replied_within_5mins = (all_channel_msgs['time_to_reply'] <= pd.Timedelta(minutes=5)).sum() / len(all_channel_msgs)\n",
+    "\n",
+    "plt.figure(figsize=(12, 8))\n",
+    "sns.scatterplot(x='time_to_reply', y=all_channel_msgs['msg_sent_time'].dt.hour, hue='channel', data=all_channel_msgs)\n",
+    "plt.title('2D Scatter Plot of Time Difference vs Time of Day')\n",
+    "plt.show()\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Assuming you have a DataFrame with columns 'msg_sent_time', 'time_thread_start', etc.\n",
+    "all_channel_msgs['time_to_reply'] = all_channel_msgs['time_thread_start'] - all_channel_msgs['msg_sent_time']\n",
+    "all_channel_msgs['time_to_reaction'] = reaction_data['reaction_timestamp'] - all_channel_msgs['msg_sent_time']\n",
+    "\n",
+    "# data_processing.py\n",
+    "def calculate_time_differences(all_channel_msgs, reaction_data):\n",
+    "    # Assuming you have a DataFrame with columns 'msg_sent_time', 'time_thread_start', etc.\n",
+    "    all_channel_msgs['time_to_reply'] = all_channel_msgs['time_thread_start'] - all_channel_msgs['msg_sent_time']\n",
+    "    all_channel_msgs['time_to_reaction'] = reaction_data['reaction_timestamp'] - all_channel_msgs['msg_sent_time']\n",
+    "\n",
+    "# Usage in another script or notebook\n",
+    "# from data_processing import calculate_time_differences\n",
+    "# calculate_time_differences(all_channel_msgs, reaction_data)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Add the new code for plotting histograms\n",
+    "plt.figure(figsize=(12, 8))\n",
+    "\n",
+    "# Consecutive Messages\n",
+    "plt.subplot(2, 2, 1)\n",
+    "all_channel_msgs['time_to_next_message'] = all_channel_msgs['msg_sent_time'].diff()\n",
+    "all_channel_msgs['time_to_next_message'].dropna().dt.total_seconds().hist(bins=50)\n",
+    "plt.title('Time Difference Between Consecutive Messages')\n",
+    "\n",
+    "# Consecutive Replies\n",
+    "plt.subplot(2, 2, 2)\n",
+    "all_channel_msgs['time_to_next_reply'] = all_channel_msgs.groupby('channel')['time_to_reply'].shift(-1)\n",
+    "all_channel_msgs['time_to_next_reply'].dropna().dt.total_seconds().hist(bins=50)\n",
+    "plt.title('Time Difference Between Consecutive Replies')\n",
+    "\n",
+    "# Consecutive Reactions\n",
+    "plt.subplot(2, 2, 3)\n",
+    "reaction_data['time_to_next_reaction'] = reaction_data.groupby('channel')['reaction_timestamp'].shift(-1)\n",
+    "reaction_data['time_to_next_reaction'].dropna().dt.total_seconds().hist(bins=50)\n",
+    "plt.title('Time Difference Between Consecutive Reactions')\n",
+    "\n",
+    "# Consecutive Events (Message, Reply, Reaction)\n",
+    "plt.subplot(2, 2, 4)\n",
+    "all_events = pd.concat([all_channel_msgs['msg_sent_time'], all_channel_msgs['time_thread_start'],\n",
+    "                       reaction_data['reaction_timestamp']]).sort_values()\n",
+    "all_events['time_to_next_event'] = all_events.diff()\n",
+    "all_events['time_to_next_event'].dropna().dt.total_seconds().hist(bins=50)\n",
+    "plt.title('Time Difference Between Consecutive Events (Message, Reply, Reaction)')\n",
+    "\n",
+    "plt.tight_layout()\n",
+    "plt.show()\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -321,6 +483,60 @@
     "    plt.show()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Assuming all_channel_msgs is your DataFrame with message and reply texts\n",
+    "# Concatenate message and reply texts\n",
+    "all_channel_msgs['combined_text'] = all_channel_msgs['msg_content'] + ' ' + all_channel_msgs['replies'].fillna('')\n",
+    "\n",
+    "# Vectorize the text data\n",
+    "vectorizer = CountVectorizer(stop_words='english', max_features=1000)\n",
+    "X = vectorizer.fit_transform(all_channel_msgs['combined_text'])\n",
+    "\n",
+    "# Apply Latent Dirichlet Allocation (LDA) for topic modeling\n",
+    "num_topics = 10  # You can adjust this based on your needs\n",
+    "lda = LatentDirichletAllocation(n_components=num_topics, random_state=42)\n",
+    "lda.fit(X)\n",
+    "\n",
+    "# Display the top words for each topic\n",
+    "feature_names = vectorizer.get_feature_names_out()\n",
+    "for topic_idx, topic in enumerate(lda.components_):\n",
+    "    top_words_idx = topic.argsort()[:-11:-1]\n",
+    "    top_words = [feature_names[i] for i in top_words_idx]\n",
+    "    print(f\"Topic #{topic_idx + 1}: {', '.join(top_words)}\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Assuming all_channel_msgs is your DataFrame with timestamp and message content\n",
+    "# Assuming 'msg_sent_time' is in datetime format\n",
+    "\n",
+    "# Group messages by day since the start of the training\n",
+    "all_channel_msgs['day_since_start'] = (all_channel_msgs['msg_sent_time'] - all_channel_msgs['msg_sent_time'].min()).dt.days\n",
+    "\n",
+    "# Concatenate all messages and replies in the same day as one big text\n",
+    "grouped_by_day = all_channel_msgs.groupby('day_since_start')['msg_content'].apply(' '.join).reset_index()\n",
+    "\n",
+    "# Perform sentiment analysis on aggregated messages\n",
+    "grouped_by_day['sentiment'] = grouped_by_day['msg_content'].apply(lambda x: TextBlob(x).sentiment.polarity)\n",
+    "\n",
+    "# Visualize the time series trend of sentiments\n",
+    "plt.figure(figsize=(12, 6))\n",
+    "plt.plot(grouped_by_day['day_since_start'], grouped_by_day['sentiment'], marker='o', linestyle='-')\n",
+    "plt.title('Sentiment Over Time')\n",
+    "plt.xlabel('Days Since Start of Training')\n",
+    "plt.ylabel('Sentiment Polarity')\n",
+    "plt.show()\n"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

src/loader.py

@@ -1,12 +1,9 @@
 import json
-import argparse
 import os
 import io
 import shutil
 import copy
 from datetime import datetime
-from pick import pick
-from time import sleep
 
 
 
@@ -76,9 +73,3 @@ def get_user_map(self):
 
 
 
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description='Export Slack history')
-
-
-    parser.add_argument('--zip', help="Name of a zip file to import")
-    args = parser.parse_args()

src/utils.py

@@ -4,12 +4,12 @@
 import json
 import datetime
 from collections import Counter
-from collections import Counter
 
 import pandas as pd
-from matplotlib import pyplot as plt
+import matplotlib.pyplot as plt
 import seaborn as sns
 from nltk.corpus import stopwords
+from wordcloud import WordCloud
 
 
 def break_combined_weeks(combined_weeks):
@@ -180,3 +180,65 @@ def convert_2_timestamp(column, data):
                 timestamp_.append(a.strftime('%Y-%m-%d %H:%M:%S'))
         return timestamp_
     else: print(f"{column} not in data")
+
+def get_tagged_users(df):
+    return df['msg_content'].str.findall(r'@U\w+').explode().unique().tolist()
+
+def map_userid_2_realname(user_profile: dict, comm_dict: dict, plot=False):
+    user_dict = {user['id']: user['real_name'] for user in user_profile['profile']}
+
+    mapped_comm_dict = {user_dict.get(user_id, user_id): count for user_id, count in comm_dict.items()}
+
+    mapped_comm_df = pd.DataFrame(list(mapped_comm_dict.items()), columns=['LearnerName', '# of Msg sent in Threads']) \
+        .sort_values(by='# of Msg sent in Threads', ascending=False)
+
+    if plot:
+        mapped_comm_df.plot.bar(figsize=(15, 7.5), x='LearnerName', y='# of Msg sent in Threads')
+        plt.title('Student based on Message sent in thread', size=20)
+        plt.show()
+
+    return mapped_comm_df
+
+def get_top_20_user(data, channel='Random'):
+    data['sender_name'].value_counts()[:20].plot.bar(figsize=(15, 7.5))
+    plt.title(f'Top 20 Message Senders in #{channel} channels', size=15, fontweight='bold')
+    plt.xlabel("Sender Name", size=18); plt.ylabel("Frequency", size=14);
+    plt.xticks(size=12); plt.yticks(size=12);
+    plt.show()
+
+def draw_avg_reply_count(data, channel='Random'):
+    data.groupby('sender_name')['reply_count'].mean().sort_values(ascending=False)[:20] \
+        .plot(kind='bar', figsize=(15, 7.5))
+    plt.title(f'Average Number of reply count per Sender in #{channel}', size=20, fontweight='bold')
+    plt.xlabel("Sender Name", size=18); plt.ylabel("Frequency", size=18);
+    plt.xticks(size=14); plt.yticks(size=14);
+    plt.show()
+
+def draw_avg_reply_users_count(data, channel='Random'):
+    data.groupby('sender_name')['reply_users_count'].mean().sort_values(ascending=False)[:20].plot(kind='bar',
+        figsize=(15, 7.5))
+    plt.title(f'Average Number of reply user count per Sender in #{channel}', size=20, fontweight='bold')
+    plt.xlabel("Sender Name", size=18); plt.ylabel("Frequency", size=18);
+    plt.xticks(size=14); plt.yticks(size=14);
+    plt.show()
+
+def draw_wordcloud(msg_content, week):    
+    all_words = ' '.join(msg_content)
+    wordcloud = WordCloud(background_color='#975429', width=500, height=300, random_state=21, max_words=500,
+                          mode='RGBA', max_font_size=140, stopwords=stopwords.words('english')).generate(all_words)
+    plt.figure(figsize=(15, 7.5))
+    plt.imshow(wordcloud, interpolation="bilinear")
+    plt.axis('off')
+    plt.tight_layout()
+    plt.title(f'WordCloud for {week}', size=30)
+    plt.show()
+
+def draw_user_reaction(data, channel='General'):
+    data.groupby('sender_name')[['reply_count', 'reply_users_count']].sum() \
+        .sort_values(by='reply_count', ascending=False)[:10].plot(kind='bar', figsize=(15, 7.5))
+    plt.title(f'User with the most reaction in #{channel}', size=25)
+    plt.xlabel("Sender Name", size=18); plt.ylabel("Frequency", size=18);
+    plt.xticks(size=14); plt.yticks(size=14);
+    plt.show()
+
+

tests/test_slack_loader.py

@@ -0,0 +1,15 @@
+import pandas as pd
+from src.loader import SlackDataLoader
+
+def test_slack_loader_columns():
+    # Initialize SlackDataLoader with a sample path
+    slack_loader = SlackDataLoader("path/to/slack/data")
+
+    # Get messages DataFrame
+    messages_df = slack_loader.get_channel_messages("sample_channel")
+
+    # Define expected columns
+    expected_columns = ["column1", "column2", ...]  # Add the actual expected column names
+
+    # Check if all expected columns are present in the DataFrame
+    assert set(expected_columns).issubset(messages_df.columns), f"Columns mismatch. Expected: {expected_columns}, Actual: {messages_df.columns}"