From 280d86d39a7b8277a94822a205c545a6aa0c10ff Mon Sep 17 00:00:00 2001 From: Collins Mokaya Date: Wed, 29 Nov 2023 10:13:07 +0300 Subject: [PATCH 1/8] Updated the utils.py --- src/loader.py | 3 --- src/utils.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/src/loader.py b/src/loader.py index c75b68d..a9278c2 100644 --- a/src/loader.py +++ b/src/loader.py @@ -1,12 +1,9 @@ import json -import argparse import os import io import shutil import copy from datetime import datetime -from pick import pick -from time import sleep diff --git a/src/utils.py b/src/utils.py index 45dda22..fc9714b 100644 --- a/src/utils.py +++ b/src/utils.py @@ -4,12 +4,12 @@ import json import datetime from collections import Counter -from collections import Counter import pandas as pd -from matplotlib import pyplot as plt +import matplotlib.pyplot as plt import seaborn as sns from nltk.corpus import stopwords +from wordcloud import WordCloud def break_combined_weeks(combined_weeks): @@ -180,3 +180,65 @@ def convert_2_timestamp(column, data): timestamp_.append(a.strftime('%Y-%m-%d %H:%M:%S')) return timestamp_ else: print(f"{column} not in data") + +def get_tagged_users(df): + return df['msg_content'].str.findall(r'@U\w+').explode().unique().tolist() + +def map_userid_2_realname(user_profile: dict, comm_dict: dict, plot=False): + user_dict = {user['id']: user['real_name'] for user in user_profile['profile']} + + mapped_comm_dict = {user_dict.get(user_id, user_id): count for user_id, count in comm_dict.items()} + + mapped_comm_df = pd.DataFrame(list(mapped_comm_dict.items()), columns=['LearnerName', '# of Msg sent in Threads']) \ + .sort_values(by='# of Msg sent in Threads', ascending=False) + + if plot: + mapped_comm_df.plot.bar(figsize=(15, 7.5), x='LearnerName', y='# of Msg sent in Threads') + plt.title('Student based on Message sent in thread', size=20) + plt.show() + + return mapped_comm_df + +def get_top_20_user(data, channel='Random'): + data['sender_name'].value_counts()[:20].plot.bar(figsize=(15, 7.5)) + plt.title(f'Top 20 Message Senders in #{channel} channels', size=15, fontweight='bold') + plt.xlabel("Sender Name", size=18); plt.ylabel("Frequency", size=14); + plt.xticks(size=12); plt.yticks(size=12); + plt.show() + +def draw_avg_reply_count(data, channel='Random'): + data.groupby('sender_name')['reply_count'].mean().sort_values(ascending=False)[:20] \ + .plot(kind='bar', figsize=(15, 7.5)) + plt.title(f'Average Number of reply count per Sender in #{channel}', size=20, fontweight='bold') + plt.xlabel("Sender Name", size=18); plt.ylabel("Frequency", size=18); + plt.xticks(size=14); plt.yticks(size=14); + plt.show() + +def draw_avg_reply_users_count(data, channel='Random'): + data.groupby('sender_name')['reply_users_count'].mean().sort_values(ascending=False)[:20].plot(kind='bar', + figsize=(15, 7.5)) + plt.title(f'Average Number of reply user count per Sender in #{channel}', size=20, fontweight='bold') + plt.xlabel("Sender Name", size=18); plt.ylabel("Frequency", size=18); + plt.xticks(size=14); plt.yticks(size=14); + plt.show() + +def draw_wordcloud(msg_content, week): + all_words = ' '.join(msg_content) + wordcloud = WordCloud(background_color='#975429', width=500, height=300, random_state=21, max_words=500, + mode='RGBA', max_font_size=140, stopwords=stopwords.words('english')).generate(all_words) + plt.figure(figsize=(15, 7.5)) + plt.imshow(wordcloud, interpolation="bilinear") + plt.axis('off') + plt.tight_layout() + plt.title(f'WordCloud for {week}', size=30) + plt.show() + +def draw_user_reaction(data, channel='General'): + data.groupby('sender_name')[['reply_count', 'reply_users_count']].sum() \ + .sort_values(by='reply_count', ascending=False)[:10].plot(kind='bar', figsize=(15, 7.5)) + plt.title(f'User with the most reaction in #{channel}', size=25) + plt.xlabel("Sender Name", size=18); plt.ylabel("Frequency", size=18); + plt.xticks(size=14); plt.yticks(size=14); + plt.show() + + From 1ea707c4e823a3a2e940b7780fed58d919ad8842 Mon Sep 17 00:00:00 2001 From: Collins Mokaya Date: Wed, 29 Nov 2023 10:20:35 +0300 Subject: [PATCH 2/8] Updated the loader.py file --- src/loader.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/loader.py b/src/loader.py index a9278c2..3526f5a 100644 --- a/src/loader.py +++ b/src/loader.py @@ -73,9 +73,3 @@ def get_user_map(self): -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Export Slack history') - - - parser.add_argument('--zip', help="Name of a zip file to import") - args = parser.parse_args() From b100c65c5338d6938b87499a5cd709557a80d364 Mon Sep 17 00:00:00 2001 From: Collins Mokaya Date: Wed, 29 Nov 2023 10:33:27 +0300 Subject: [PATCH 3/8] Updated the parse_slack file, with upload of data --- notebooks/parse_slack_data.ipynb | 37 +++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/notebooks/parse_slack_data.ipynb b/notebooks/parse_slack_data.ipynb index e3774f8..32726da 100644 --- a/notebooks/parse_slack_data.ipynb +++ b/notebooks/parse_slack_data.ipynb @@ -24,7 +24,7 @@ "from collections import Counter\n", "\n", "import pandas as pd\n", - "from matplotlib import pyplot as plt\n", + "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", "from nltk.corpus import stopwords\n", @@ -218,6 +218,41 @@ " else: \n", " print(f\"{column} not in data\")\n", "\n", + "\n", + "# Use SlackDataLoader to load data\n", + "data_loader = SlackDataLoader(\"path_to_slack_exported_data_folder\")\n", + "\n", + "# Get all channel messages\n", + "all_channel_msgs = pd.concat([data_loader.get_channel_messages(channel['name']) for channel in data_loader.channels])\n", + "\n", + "# Parse reaction data\n", + "reaction_data = parse_slack_reaction(\"path_to_reaction_data_folder\", \"General\")\n", + "\n", + "# Get community participation data\n", + "comm_participation = get_community_participation(\"path_to_community_participation_data_folder\")\n", + "\n", + "# Map user IDs to real names\n", + "user_profile = data_loader.get_users()\n", + "mapped_comm_dict = utils.map_userid_2_realname(user_profile, comm_participation, plot=True)\n", + "\n", + "# Get tagged users\n", + "tagged_users = utils.get_tagged_users(all_channel_msgs)\n", + "\n", + "# Get top 20 users\n", + "utils.get_top_20_user(all_channel_msgs, channel='General')\n", + "\n", + "# Draw average reply count\n", + "utils.draw_avg_reply_count(all_channel_msgs, channel='General')\n", + "\n", + "# Draw average reply users count\n", + "utils.draw_avg_reply_users_count(all_channel_msgs, channel='General')\n", + "\n", + "# Draw word cloud\n", + "utils.draw_wordcloud(all_channel_msgs['msg_content'], week='Week8-9 Combined')\n", + "\n", + "\n", + "\n", + "\n", "def get_tagged_users(df):\n", " \"\"\"get all @ in the messages\"\"\"\n", "\n", From f43cc3c27028b7580926cfe98ed34b08bf50bf74 Mon Sep 17 00:00:00 2001 From: Collins Mokaya Date: Wed, 29 Nov 2023 10:47:09 +0300 Subject: [PATCH 4/8] Updated parse_slack to perform EDA --- notebooks/parse_slack_data.ipynb | 62 ++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/notebooks/parse_slack_data.ipynb b/notebooks/parse_slack_data.ipynb index 32726da..ebfb57f 100644 --- a/notebooks/parse_slack_data.ipynb +++ b/notebooks/parse_slack_data.ipynb @@ -294,6 +294,68 @@ " return ac_comm_dict" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Top and Bottom 10 Users by Reply Count\n", + "top_users_by_reply_count = all_channel_msgs.groupby('sender_name')['reply_count'].sum().sort_values(ascending=False)[:10]\n", + "bottom_users_by_reply_count = all_channel_msgs.groupby('sender_name')['reply_count'].sum().sort_values(ascending=True)[:10]\n", + "\n", + "# Print or visualize the results\n", + "print(\"Top Users by Reply Count:\")\n", + "print(top_users_by_reply_count)\n", + "\n", + "print(\"\\nBottom Users by Reply Count:\")\n", + "print(bottom_users_by_reply_count)\n", + "\n", + "all_channel_msgs['sender_name'].value_counts().sort_values(ascending=False)[:10]\n", + "all_channel_msgs['sender_name'].value_counts().sort_values(ascending=True)[:10]\n", + "\n", + "reaction_data.groupby('sender_name')['reaction_count'].sum().sort_values(ascending=False)[:10]\n", + "reaction_data.groupby('sender_name')['reaction_count'].sum().sort_values(ascending=True)[:10]\n", + "\n", + "\n", + "all_channel_msgs.sort_values(by='reply_count', ascending=False)[:10]\n", + "\n", + "\n", + "reaction_data.sort_values(by='reaction_count', ascending=False)[:10]\n", + "\n", + "# Assuming you have a column 'mentions' in your DataFrame\n", + "all_channel_msgs[all_channel_msgs['mentions'].apply(lambda x: len(x) if x else 0)].sort_values(by='mentions', ascending=False)[:10]\n", + "\n", + "all_channel_msgs['channel'].value_counts().idxmax()\n", + "\n", + "activity_df = pd.DataFrame({\n", + " 'Channel': all_channel_msgs['channel'],\n", + " 'Messages': all_channel_msgs.groupby('channel').size(),\n", + " 'Replies_Reactions_Sum': all_channel_msgs.groupby('channel')['reply_count'].sum() + reaction_data.groupby('channel')['reaction_count'].sum()\n", + "})\n", + "\n", + "plt.figure(figsize=(12, 8))\n", + "sns.scatterplot(x='Messages', y='Replies_Reactions_Sum', hue='Channel', data=activity_df)\n", + "plt.title('2D Scatter Plot of Channel Activity')\n", + "plt.show()\n", + "\n", + "# Convert to datetime\n", + "all_channel_msgs['msg_sent_time'] = pd.to_datetime(all_channel_msgs['msg_sent_time'])\n", + "all_channel_msgs['time_thread_start'] = pd.to_datetime(all_channel_msgs['time_thread_start'])\n", + "\n", + "# Calculate time difference\n", + "all_channel_msgs['time_to_reply'] = all_channel_msgs['time_thread_start'] - all_channel_msgs['msg_sent_time']\n", + "\n", + "# Calculate fraction replied within 5 minutes\n", + "fraction_replied_within_5mins = (all_channel_msgs['time_to_reply'] <= pd.Timedelta(minutes=5)).sum() / len(all_channel_msgs)\n", + "\n", + "plt.figure(figsize=(12, 8))\n", + "sns.scatterplot(x='time_to_reply', y=all_channel_msgs['msg_sent_time'].dt.hour, hue='channel', data=all_channel_msgs)\n", + "plt.title('2D Scatter Plot of Time Difference vs Time of Day')\n", + "plt.show()\n", + "\n" + ] + }, { "cell_type": "code", "execution_count": null, From 48a3bf620c0983b4adf38093df03084ad1e70b06 Mon Sep 17 00:00:00 2001 From: Collins Mokaya Date: Wed, 29 Nov 2023 11:58:37 +0300 Subject: [PATCH 5/8] updated the test_slack_loader with testa --- tests/test_slack_loader.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/test_slack_loader.py diff --git a/tests/test_slack_loader.py b/tests/test_slack_loader.py new file mode 100644 index 0000000..68a16ef --- /dev/null +++ b/tests/test_slack_loader.py @@ -0,0 +1,15 @@ +import pandas as pd +from src.loader import SlackDataLoader + +def test_slack_loader_columns(): + # Initialize SlackDataLoader with a sample path + slack_loader = SlackDataLoader("path/to/slack/data") + + # Get messages DataFrame + messages_df = slack_loader.get_channel_messages("sample_channel") + + # Define expected columns + expected_columns = ["column1", "column2", ...] # Add the actual expected column names + + # Check if all expected columns are present in the DataFrame + assert set(expected_columns).issubset(messages_df.columns), f"Columns mismatch. Expected: {expected_columns}, Actual: {messages_df.columns}" From 4bc098b9ae38f6be4cb80d7b102d43fe57e1936f Mon Sep 17 00:00:00 2001 From: Collins Mokaya Date: Thu, 30 Nov 2023 09:28:47 +0300 Subject: [PATCH 6/8] updated the parse_slack file with calcution of time difference --- notebooks/parse_slack_data.ipynb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/notebooks/parse_slack_data.ipynb b/notebooks/parse_slack_data.ipynb index ebfb57f..d27cc29 100644 --- a/notebooks/parse_slack_data.ipynb +++ b/notebooks/parse_slack_data.ipynb @@ -356,6 +356,27 @@ "\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Assuming you have a DataFrame with columns 'msg_sent_time', 'time_thread_start', etc.\n", + "all_channel_msgs['time_to_reply'] = all_channel_msgs['time_thread_start'] - all_channel_msgs['msg_sent_time']\n", + "all_channel_msgs['time_to_reaction'] = reaction_data['reaction_timestamp'] - all_channel_msgs['msg_sent_time']\n", + "\n", + "# data_processing.py\n", + "def calculate_time_differences(all_channel_msgs, reaction_data):\n", + " # Assuming you have a DataFrame with columns 'msg_sent_time', 'time_thread_start', etc.\n", + " all_channel_msgs['time_to_reply'] = all_channel_msgs['time_thread_start'] - all_channel_msgs['msg_sent_time']\n", + " all_channel_msgs['time_to_reaction'] = reaction_data['reaction_timestamp'] - all_channel_msgs['msg_sent_time']\n", + "\n", + "# Usage in another script or notebook\n", + "# from data_processing import calculate_time_differences\n", + "# calculate_time_differences(all_channel_msgs, reaction_data)\n" + ] + }, { "cell_type": "code", "execution_count": null, From e9923864f9c8ffb84e0a92ac348a543d08d744e1 Mon Sep 17 00:00:00 2001 From: Collins Mokaya Date: Thu, 30 Nov 2023 09:32:29 +0300 Subject: [PATCH 7/8] updated the histograms plotting code --- notebooks/parse_slack_data.ipynb | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/notebooks/parse_slack_data.ipynb b/notebooks/parse_slack_data.ipynb index d27cc29..641175e 100644 --- a/notebooks/parse_slack_data.ipynb +++ b/notebooks/parse_slack_data.ipynb @@ -377,6 +377,45 @@ "# calculate_time_differences(all_channel_msgs, reaction_data)\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Add the new code for plotting histograms\n", + "plt.figure(figsize=(12, 8))\n", + "\n", + "# Consecutive Messages\n", + "plt.subplot(2, 2, 1)\n", + "all_channel_msgs['time_to_next_message'] = all_channel_msgs['msg_sent_time'].diff()\n", + "all_channel_msgs['time_to_next_message'].dropna().dt.total_seconds().hist(bins=50)\n", + "plt.title('Time Difference Between Consecutive Messages')\n", + "\n", + "# Consecutive Replies\n", + "plt.subplot(2, 2, 2)\n", + "all_channel_msgs['time_to_next_reply'] = all_channel_msgs.groupby('channel')['time_to_reply'].shift(-1)\n", + "all_channel_msgs['time_to_next_reply'].dropna().dt.total_seconds().hist(bins=50)\n", + "plt.title('Time Difference Between Consecutive Replies')\n", + "\n", + "# Consecutive Reactions\n", + "plt.subplot(2, 2, 3)\n", + "reaction_data['time_to_next_reaction'] = reaction_data.groupby('channel')['reaction_timestamp'].shift(-1)\n", + "reaction_data['time_to_next_reaction'].dropna().dt.total_seconds().hist(bins=50)\n", + "plt.title('Time Difference Between Consecutive Reactions')\n", + "\n", + "# Consecutive Events (Message, Reply, Reaction)\n", + "plt.subplot(2, 2, 4)\n", + "all_events = pd.concat([all_channel_msgs['msg_sent_time'], all_channel_msgs['time_thread_start'],\n", + " reaction_data['reaction_timestamp']]).sort_values()\n", + "all_events['time_to_next_event'] = all_events.diff()\n", + "all_events['time_to_next_event'].dropna().dt.total_seconds().hist(bins=50)\n", + "plt.title('Time Difference Between Consecutive Events (Message, Reply, Reaction)')\n", + "\n", + "plt.tight_layout()\n", + "plt.show()\n" + ] + }, { "cell_type": "code", "execution_count": null, From 11869ec829922ba3a4ae6eb0f1bb1b1ef5443700 Mon Sep 17 00:00:00 2001 From: Collins Mokaya Date: Thu, 30 Nov 2023 09:49:24 +0300 Subject: [PATCH 8/8] updated the sentiment analysis --- notebooks/parse_slack_data.ipynb | 61 +++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/notebooks/parse_slack_data.ipynb b/notebooks/parse_slack_data.ipynb index 641175e..f063d00 100644 --- a/notebooks/parse_slack_data.ipynb +++ b/notebooks/parse_slack_data.ipynb @@ -27,8 +27,13 @@ "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", + "from textblob import TextBlob\n", + "\n", "from nltk.corpus import stopwords\n", - "from wordcloud import WordCloud" + "from wordcloud import WordCloud\n", + "\n", + "from sklearn.feature_extraction.text import CountVectorizer\n", + "from sklearn.decomposition import LatentDirichletAllocation" ] }, { @@ -478,6 +483,60 @@ " plt.show()" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Assuming all_channel_msgs is your DataFrame with message and reply texts\n", + "# Concatenate message and reply texts\n", + "all_channel_msgs['combined_text'] = all_channel_msgs['msg_content'] + ' ' + all_channel_msgs['replies'].fillna('')\n", + "\n", + "# Vectorize the text data\n", + "vectorizer = CountVectorizer(stop_words='english', max_features=1000)\n", + "X = vectorizer.fit_transform(all_channel_msgs['combined_text'])\n", + "\n", + "# Apply Latent Dirichlet Allocation (LDA) for topic modeling\n", + "num_topics = 10 # You can adjust this based on your needs\n", + "lda = LatentDirichletAllocation(n_components=num_topics, random_state=42)\n", + "lda.fit(X)\n", + "\n", + "# Display the top words for each topic\n", + "feature_names = vectorizer.get_feature_names_out()\n", + "for topic_idx, topic in enumerate(lda.components_):\n", + " top_words_idx = topic.argsort()[:-11:-1]\n", + " top_words = [feature_names[i] for i in top_words_idx]\n", + " print(f\"Topic #{topic_idx + 1}: {', '.join(top_words)}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Assuming all_channel_msgs is your DataFrame with timestamp and message content\n", + "# Assuming 'msg_sent_time' is in datetime format\n", + "\n", + "# Group messages by day since the start of the training\n", + "all_channel_msgs['day_since_start'] = (all_channel_msgs['msg_sent_time'] - all_channel_msgs['msg_sent_time'].min()).dt.days\n", + "\n", + "# Concatenate all messages and replies in the same day as one big text\n", + "grouped_by_day = all_channel_msgs.groupby('day_since_start')['msg_content'].apply(' '.join).reset_index()\n", + "\n", + "# Perform sentiment analysis on aggregated messages\n", + "grouped_by_day['sentiment'] = grouped_by_day['msg_content'].apply(lambda x: TextBlob(x).sentiment.polarity)\n", + "\n", + "# Visualize the time series trend of sentiments\n", + "plt.figure(figsize=(12, 6))\n", + "plt.plot(grouped_by_day['day_since_start'], grouped_by_day['sentiment'], marker='o', linestyle='-')\n", + "plt.title('Sentiment Over Time')\n", + "plt.xlabel('Days Since Start of Training')\n", + "plt.ylabel('Sentiment Polarity')\n", + "plt.show()\n" + ] + }, { "cell_type": "markdown", "metadata": {},