diff --git a/notebooks/parse_slack_data.ipynb b/notebooks/parse_slack_data.ipynb index e3774f8..f063d00 100644 --- a/notebooks/parse_slack_data.ipynb +++ b/notebooks/parse_slack_data.ipynb @@ -24,11 +24,16 @@ "from collections import Counter\n", "\n", "import pandas as pd\n", - "from matplotlib import pyplot as plt\n", + "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", + "from textblob import TextBlob\n", + "\n", "from nltk.corpus import stopwords\n", - "from wordcloud import WordCloud" + "from wordcloud import WordCloud\n", + "\n", + "from sklearn.feature_extraction.text import CountVectorizer\n", + "from sklearn.decomposition import LatentDirichletAllocation" ] }, { @@ -218,6 +223,41 @@ " else: \n", " print(f\"{column} not in data\")\n", "\n", + "\n", + "# Use SlackDataLoader to load data\n", + "data_loader = SlackDataLoader(\"path_to_slack_exported_data_folder\")\n", + "\n", + "# Get all channel messages\n", + "all_channel_msgs = pd.concat([data_loader.get_channel_messages(channel['name']) for channel in data_loader.channels])\n", + "\n", + "# Parse reaction data\n", + "reaction_data = parse_slack_reaction(\"path_to_reaction_data_folder\", \"General\")\n", + "\n", + "# Get community participation data\n", + "comm_participation = get_community_participation(\"path_to_community_participation_data_folder\")\n", + "\n", + "# Map user IDs to real names\n", + "user_profile = data_loader.get_users()\n", + "mapped_comm_dict = utils.map_userid_2_realname(user_profile, comm_participation, plot=True)\n", + "\n", + "# Get tagged users\n", + "tagged_users = utils.get_tagged_users(all_channel_msgs)\n", + "\n", + "# Get top 20 users\n", + "utils.get_top_20_user(all_channel_msgs, channel='General')\n", + "\n", + "# Draw average reply count\n", + "utils.draw_avg_reply_count(all_channel_msgs, channel='General')\n", + "\n", + "# Draw average reply users count\n", + "utils.draw_avg_reply_users_count(all_channel_msgs, channel='General')\n", + "\n", + "# Draw word cloud\n", + "utils.draw_wordcloud(all_channel_msgs['msg_content'], week='Week8-9 Combined')\n", + "\n", + "\n", + "\n", + "\n", "def get_tagged_users(df):\n", " \"\"\"get all @ in the messages\"\"\"\n", "\n", @@ -259,6 +299,128 @@ " return ac_comm_dict" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Top and Bottom 10 Users by Reply Count\n", + "top_users_by_reply_count = all_channel_msgs.groupby('sender_name')['reply_count'].sum().sort_values(ascending=False)[:10]\n", + "bottom_users_by_reply_count = all_channel_msgs.groupby('sender_name')['reply_count'].sum().sort_values(ascending=True)[:10]\n", + "\n", + "# Print or visualize the results\n", + "print(\"Top Users by Reply Count:\")\n", + "print(top_users_by_reply_count)\n", + "\n", + "print(\"\\nBottom Users by Reply Count:\")\n", + "print(bottom_users_by_reply_count)\n", + "\n", + "all_channel_msgs['sender_name'].value_counts().sort_values(ascending=False)[:10]\n", + "all_channel_msgs['sender_name'].value_counts().sort_values(ascending=True)[:10]\n", + "\n", + "reaction_data.groupby('sender_name')['reaction_count'].sum().sort_values(ascending=False)[:10]\n", + "reaction_data.groupby('sender_name')['reaction_count'].sum().sort_values(ascending=True)[:10]\n", + "\n", + "\n", + "all_channel_msgs.sort_values(by='reply_count', ascending=False)[:10]\n", + "\n", + "\n", + "reaction_data.sort_values(by='reaction_count', ascending=False)[:10]\n", + "\n", + "# Assuming you have a column 'mentions' in your DataFrame\n", + "all_channel_msgs[all_channel_msgs['mentions'].apply(lambda x: len(x) if x else 0)].sort_values(by='mentions', ascending=False)[:10]\n", + "\n", + "all_channel_msgs['channel'].value_counts().idxmax()\n", + "\n", + "activity_df = pd.DataFrame({\n", + " 'Channel': all_channel_msgs['channel'],\n", + " 'Messages': all_channel_msgs.groupby('channel').size(),\n", + " 'Replies_Reactions_Sum': all_channel_msgs.groupby('channel')['reply_count'].sum() + reaction_data.groupby('channel')['reaction_count'].sum()\n", + "})\n", + "\n", + "plt.figure(figsize=(12, 8))\n", + "sns.scatterplot(x='Messages', y='Replies_Reactions_Sum', hue='Channel', data=activity_df)\n", + "plt.title('2D Scatter Plot of Channel Activity')\n", + "plt.show()\n", + "\n", + "# Convert to datetime\n", + "all_channel_msgs['msg_sent_time'] = pd.to_datetime(all_channel_msgs['msg_sent_time'])\n", + "all_channel_msgs['time_thread_start'] = pd.to_datetime(all_channel_msgs['time_thread_start'])\n", + "\n", + "# Calculate time difference\n", + "all_channel_msgs['time_to_reply'] = all_channel_msgs['time_thread_start'] - all_channel_msgs['msg_sent_time']\n", + "\n", + "# Calculate fraction replied within 5 minutes\n", + "fraction_replied_within_5mins = (all_channel_msgs['time_to_reply'] <= pd.Timedelta(minutes=5)).sum() / len(all_channel_msgs)\n", + "\n", + "plt.figure(figsize=(12, 8))\n", + "sns.scatterplot(x='time_to_reply', y=all_channel_msgs['msg_sent_time'].dt.hour, hue='channel', data=all_channel_msgs)\n", + "plt.title('2D Scatter Plot of Time Difference vs Time of Day')\n", + "plt.show()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Assuming you have a DataFrame with columns 'msg_sent_time', 'time_thread_start', etc.\n", + "all_channel_msgs['time_to_reply'] = all_channel_msgs['time_thread_start'] - all_channel_msgs['msg_sent_time']\n", + "all_channel_msgs['time_to_reaction'] = reaction_data['reaction_timestamp'] - all_channel_msgs['msg_sent_time']\n", + "\n", + "# data_processing.py\n", + "def calculate_time_differences(all_channel_msgs, reaction_data):\n", + " # Assuming you have a DataFrame with columns 'msg_sent_time', 'time_thread_start', etc.\n", + " all_channel_msgs['time_to_reply'] = all_channel_msgs['time_thread_start'] - all_channel_msgs['msg_sent_time']\n", + " all_channel_msgs['time_to_reaction'] = reaction_data['reaction_timestamp'] - all_channel_msgs['msg_sent_time']\n", + "\n", + "# Usage in another script or notebook\n", + "# from data_processing import calculate_time_differences\n", + "# calculate_time_differences(all_channel_msgs, reaction_data)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Add the new code for plotting histograms\n", + "plt.figure(figsize=(12, 8))\n", + "\n", + "# Consecutive Messages\n", + "plt.subplot(2, 2, 1)\n", + "all_channel_msgs['time_to_next_message'] = all_channel_msgs['msg_sent_time'].diff()\n", + "all_channel_msgs['time_to_next_message'].dropna().dt.total_seconds().hist(bins=50)\n", + "plt.title('Time Difference Between Consecutive Messages')\n", + "\n", + "# Consecutive Replies\n", + "plt.subplot(2, 2, 2)\n", + "all_channel_msgs['time_to_next_reply'] = all_channel_msgs.groupby('channel')['time_to_reply'].shift(-1)\n", + "all_channel_msgs['time_to_next_reply'].dropna().dt.total_seconds().hist(bins=50)\n", + "plt.title('Time Difference Between Consecutive Replies')\n", + "\n", + "# Consecutive Reactions\n", + "plt.subplot(2, 2, 3)\n", + "reaction_data['time_to_next_reaction'] = reaction_data.groupby('channel')['reaction_timestamp'].shift(-1)\n", + "reaction_data['time_to_next_reaction'].dropna().dt.total_seconds().hist(bins=50)\n", + "plt.title('Time Difference Between Consecutive Reactions')\n", + "\n", + "# Consecutive Events (Message, Reply, Reaction)\n", + "plt.subplot(2, 2, 4)\n", + "all_events = pd.concat([all_channel_msgs['msg_sent_time'], all_channel_msgs['time_thread_start'],\n", + " reaction_data['reaction_timestamp']]).sort_values()\n", + "all_events['time_to_next_event'] = all_events.diff()\n", + "all_events['time_to_next_event'].dropna().dt.total_seconds().hist(bins=50)\n", + "plt.title('Time Difference Between Consecutive Events (Message, Reply, Reaction)')\n", + "\n", + "plt.tight_layout()\n", + "plt.show()\n" + ] + }, { "cell_type": "code", "execution_count": null, @@ -321,6 +483,60 @@ " plt.show()" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Assuming all_channel_msgs is your DataFrame with message and reply texts\n", + "# Concatenate message and reply texts\n", + "all_channel_msgs['combined_text'] = all_channel_msgs['msg_content'] + ' ' + all_channel_msgs['replies'].fillna('')\n", + "\n", + "# Vectorize the text data\n", + "vectorizer = CountVectorizer(stop_words='english', max_features=1000)\n", + "X = vectorizer.fit_transform(all_channel_msgs['combined_text'])\n", + "\n", + "# Apply Latent Dirichlet Allocation (LDA) for topic modeling\n", + "num_topics = 10 # You can adjust this based on your needs\n", + "lda = LatentDirichletAllocation(n_components=num_topics, random_state=42)\n", + "lda.fit(X)\n", + "\n", + "# Display the top words for each topic\n", + "feature_names = vectorizer.get_feature_names_out()\n", + "for topic_idx, topic in enumerate(lda.components_):\n", + " top_words_idx = topic.argsort()[:-11:-1]\n", + " top_words = [feature_names[i] for i in top_words_idx]\n", + " print(f\"Topic #{topic_idx + 1}: {', '.join(top_words)}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Assuming all_channel_msgs is your DataFrame with timestamp and message content\n", + "# Assuming 'msg_sent_time' is in datetime format\n", + "\n", + "# Group messages by day since the start of the training\n", + "all_channel_msgs['day_since_start'] = (all_channel_msgs['msg_sent_time'] - all_channel_msgs['msg_sent_time'].min()).dt.days\n", + "\n", + "# Concatenate all messages and replies in the same day as one big text\n", + "grouped_by_day = all_channel_msgs.groupby('day_since_start')['msg_content'].apply(' '.join).reset_index()\n", + "\n", + "# Perform sentiment analysis on aggregated messages\n", + "grouped_by_day['sentiment'] = grouped_by_day['msg_content'].apply(lambda x: TextBlob(x).sentiment.polarity)\n", + "\n", + "# Visualize the time series trend of sentiments\n", + "plt.figure(figsize=(12, 6))\n", + "plt.plot(grouped_by_day['day_since_start'], grouped_by_day['sentiment'], marker='o', linestyle='-')\n", + "plt.title('Sentiment Over Time')\n", + "plt.xlabel('Days Since Start of Training')\n", + "plt.ylabel('Sentiment Polarity')\n", + "plt.show()\n" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/src/loader.py b/src/loader.py index c75b68d..3526f5a 100644 --- a/src/loader.py +++ b/src/loader.py @@ -1,12 +1,9 @@ import json -import argparse import os import io import shutil import copy from datetime import datetime -from pick import pick -from time import sleep @@ -76,9 +73,3 @@ def get_user_map(self): -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Export Slack history') - - - parser.add_argument('--zip', help="Name of a zip file to import") - args = parser.parse_args() diff --git a/src/utils.py b/src/utils.py index 45dda22..fc9714b 100644 --- a/src/utils.py +++ b/src/utils.py @@ -4,12 +4,12 @@ import json import datetime from collections import Counter -from collections import Counter import pandas as pd -from matplotlib import pyplot as plt +import matplotlib.pyplot as plt import seaborn as sns from nltk.corpus import stopwords +from wordcloud import WordCloud def break_combined_weeks(combined_weeks): @@ -180,3 +180,65 @@ def convert_2_timestamp(column, data): timestamp_.append(a.strftime('%Y-%m-%d %H:%M:%S')) return timestamp_ else: print(f"{column} not in data") + +def get_tagged_users(df): + return df['msg_content'].str.findall(r'@U\w+').explode().unique().tolist() + +def map_userid_2_realname(user_profile: dict, comm_dict: dict, plot=False): + user_dict = {user['id']: user['real_name'] for user in user_profile['profile']} + + mapped_comm_dict = {user_dict.get(user_id, user_id): count for user_id, count in comm_dict.items()} + + mapped_comm_df = pd.DataFrame(list(mapped_comm_dict.items()), columns=['LearnerName', '# of Msg sent in Threads']) \ + .sort_values(by='# of Msg sent in Threads', ascending=False) + + if plot: + mapped_comm_df.plot.bar(figsize=(15, 7.5), x='LearnerName', y='# of Msg sent in Threads') + plt.title('Student based on Message sent in thread', size=20) + plt.show() + + return mapped_comm_df + +def get_top_20_user(data, channel='Random'): + data['sender_name'].value_counts()[:20].plot.bar(figsize=(15, 7.5)) + plt.title(f'Top 20 Message Senders in #{channel} channels', size=15, fontweight='bold') + plt.xlabel("Sender Name", size=18); plt.ylabel("Frequency", size=14); + plt.xticks(size=12); plt.yticks(size=12); + plt.show() + +def draw_avg_reply_count(data, channel='Random'): + data.groupby('sender_name')['reply_count'].mean().sort_values(ascending=False)[:20] \ + .plot(kind='bar', figsize=(15, 7.5)) + plt.title(f'Average Number of reply count per Sender in #{channel}', size=20, fontweight='bold') + plt.xlabel("Sender Name", size=18); plt.ylabel("Frequency", size=18); + plt.xticks(size=14); plt.yticks(size=14); + plt.show() + +def draw_avg_reply_users_count(data, channel='Random'): + data.groupby('sender_name')['reply_users_count'].mean().sort_values(ascending=False)[:20].plot(kind='bar', + figsize=(15, 7.5)) + plt.title(f'Average Number of reply user count per Sender in #{channel}', size=20, fontweight='bold') + plt.xlabel("Sender Name", size=18); plt.ylabel("Frequency", size=18); + plt.xticks(size=14); plt.yticks(size=14); + plt.show() + +def draw_wordcloud(msg_content, week): + all_words = ' '.join(msg_content) + wordcloud = WordCloud(background_color='#975429', width=500, height=300, random_state=21, max_words=500, + mode='RGBA', max_font_size=140, stopwords=stopwords.words('english')).generate(all_words) + plt.figure(figsize=(15, 7.5)) + plt.imshow(wordcloud, interpolation="bilinear") + plt.axis('off') + plt.tight_layout() + plt.title(f'WordCloud for {week}', size=30) + plt.show() + +def draw_user_reaction(data, channel='General'): + data.groupby('sender_name')[['reply_count', 'reply_users_count']].sum() \ + .sort_values(by='reply_count', ascending=False)[:10].plot(kind='bar', figsize=(15, 7.5)) + plt.title(f'User with the most reaction in #{channel}', size=25) + plt.xlabel("Sender Name", size=18); plt.ylabel("Frequency", size=18); + plt.xticks(size=14); plt.yticks(size=14); + plt.show() + + diff --git a/tests/test_slack_loader.py b/tests/test_slack_loader.py new file mode 100644 index 0000000..68a16ef --- /dev/null +++ b/tests/test_slack_loader.py @@ -0,0 +1,15 @@ +import pandas as pd +from src.loader import SlackDataLoader + +def test_slack_loader_columns(): + # Initialize SlackDataLoader with a sample path + slack_loader = SlackDataLoader("path/to/slack/data") + + # Get messages DataFrame + messages_df = slack_loader.get_channel_messages("sample_channel") + + # Define expected columns + expected_columns = ["column1", "column2", ...] # Add the actual expected column names + + # Check if all expected columns are present in the DataFrame + assert set(expected_columns).issubset(messages_df.columns), f"Columns mismatch. Expected: {expected_columns}, Actual: {messages_df.columns}"