From 88d58e1aab74eb9ba1c1d81281a4bfb0742999a9 Mon Sep 17 00:00:00 2001 From: "Daniel Z. Geda" Date: Tue, 28 Nov 2023 00:08:10 +0300 Subject: [PATCH 1/3] change made for to configure to environment --- .gitignore | 2 +- src/config.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 8cd10b4..4b6712e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ conf/**/*credentials* # ignore everything in the following folders data/** logs/** - +src/anonymized/ # except their sub-folders !data/**/ !logs/**/ diff --git a/src/config.py b/src/config.py index 772d4ac..b588cb6 100644 --- a/src/config.py +++ b/src/config.py @@ -1,12 +1,12 @@ -from __future__ import print_function -import argparse - -parser = argparse.ArgumentParser(description='cmdArgs') -parser.add_argument('--output', type=str, default='slack_data.csv', - help='filename to write analysis output in CSV format') -parser.add_argument('--path', required=True, type=str, help='directory where slack data reside') -parser.add_argument('--channel', type=str, default='', help='which channel we parsing') -parser.add_argument('--userfile', type=str, default='users.json', help='users profile information') - -cfg = parser.parse_args() +from __future__ import print_function +import argparse + +parser = argparse.ArgumentParser(description='cmdArgs') +parser.add_argument('slack_data.csv', type=str, default='slack_data.csv', + help='filename to write analysis output in CSV format') +parser.add_argument('anonymized', required=True, type=str, help='directory where slack data reside') +parser.add_argument('--channel', type=str, default='', help='which channel we parsing') +parser.add_argument('--userfile', type=str, default='users.json', help='users profile information') + +cfg = parser.parse_args() # print(cfg) \ No newline at end of file From eea09314cba94379364e6b2f4ce5de088eee43bd Mon Sep 17 00:00:00 2001 From: "Daniel Z. Geda" Date: Wed, 29 Nov 2023 13:55:34 +0300 Subject: [PATCH 2/3] loading JSON file into loader from local machine --- .gitignore | 2 +- notebooks/parse_slack_data.ipynb | 26 +++-- src/loader.py | 168 +++++++++++++++---------------- 3 files changed, 104 insertions(+), 92 deletions(-) diff --git a/.gitignore b/.gitignore index 4b6712e..de6b3bb 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ conf/**/*credentials* # ignore everything in the following folders data/** logs/** -src/anonymized/ +anonymized/ # except their sub-folders !data/**/ !logs/**/ diff --git a/notebooks/parse_slack_data.ipynb b/notebooks/parse_slack_data.ipynb index e3774f8..3f86ee7 100644 --- a/notebooks/parse_slack_data.ipynb +++ b/notebooks/parse_slack_data.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -33,9 +33,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], "source": [ "# Add parent directory to path to import modules from src\n", "rpath = os.path.abspath('..')\n", @@ -43,7 +51,11 @@ " sys.path.insert(0, rpath)\n", "\n", "from src.loader import SlackDataLoader\n", - "import src.utils as utils" + "import src.utils as utils\n", + "\n", + "\n", + "data_loader = SlackDataLoader('C:/Users/zzz/Desktop/week0/week0_starter_network_analysis/anonymized')\n", + "print(data_loader)" ] }, { @@ -430,7 +442,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -444,7 +456,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.10.6" } }, "nbformat": 4, diff --git a/src/loader.py b/src/loader.py index c75b68d..cbb79fd 100644 --- a/src/loader.py +++ b/src/loader.py @@ -1,84 +1,84 @@ -import json -import argparse -import os -import io -import shutil -import copy -from datetime import datetime -from pick import pick -from time import sleep - - - -# Create wrapper classes for using slack_sdk in place of slacker -class SlackDataLoader: - ''' - Slack exported data IO class. - - When you open slack exported ZIP file, each channel or direct message - will have its own folder. Each folder will contain messages from the - conversation, organised by date in separate JSON files. - - You'll see reference files for different kinds of conversations: - users.json files for all types of users that exist in the slack workspace - channels.json files for public channels, - - These files contain metadata about the conversations, including their names and IDs. - - For secruity reason, we have annonymized names - the names you will see are generated using faker library. - - ''' - def __init__(self, path): - ''' - path: path to the slack exported data folder - ''' - self.path = path - self.channels = self.get_channels() - self.users = self.get_ussers() - - - def get_users(self): - ''' - write a function to get all the users from the json file - ''' - with open(os.path.join(self.path, 'users.json'), 'r') as f: - users = json.load(f) - - return users - - def get_channels(self): - ''' - write a function to get all the channels from the json file - ''' - with open(os.path.join(self.path, 'channels.json'), 'r') as f: - channels = json.load(f) - - return channels - - def get_channel_messages(self, channel_name): - ''' - write a function to get all the messages from a channel - - ''' - - # - def get_user_map(self): - ''' - write a function to get a map between user id and user name - ''' - userNamesById = {} - userIdsByName = {} - for user in self.users: - userNamesById[user['id']] = user['name'] - userIdsByName[user['name']] = user['id'] - return userNamesById, userIdsByName - - - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Export Slack history') - - - parser.add_argument('--zip', help="Name of a zip file to import") - args = parser.parse_args() +import json +import argparse +import os +import io +import shutil +import copy +from datetime import datetime +from pick import pick +from time import sleep + + + +# Create wrapper classes for using slack_sdk in place of slacker +class SlackDataLoader: + ''' + Slack exported data IO class. + + When you open slack exported ZIP file, each channel or direct message + will have its own folder. Each folder will contain messages from the + conversation, organised by date in separate JSON files. + + You'll see reference files for different kinds of conversations: + users.json files for all types of users that exist in the slack workspace + channels.json files for public channels, + + These files contain metadata about the conversations, including their names and IDs. + + For secruity reason, we have annonymized names - the names you will see are generated using faker library. + + ''' + def __init__(self, path): + ''' + path: path to the slack exported data folder + ''' + self.path = path + self.channels = self.get_channels() + self.users = self.get_users() + + + def get_users(self): + ''' + write a function to get all the users from the json file + ''' + with open(os.path.join(self.path, 'users.json'), 'r') as f: + users = json.load(f) + + return users + + def get_channels(self): + ''' + write a function to get all the channels from the json file + ''' + with open(os.path.join(self.path, 'channels.json'), 'r') as f: + channels = json.load(f) + + return channels + + def get_channel_messages(self, channel_name): + ''' + write a function to get all the messages from a channel + + ''' + + # + def get_user_map(self): + ''' + write a function to get a map between user id and user name + ''' + userNamesById = {} + userIdsByName = {} + for user in self.users: + userNamesById[user['id']] = user['name'] + userIdsByName[user['name']] = user['id'] + return userNamesById, userIdsByName + + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Export Slack history') + + + parser.add_argument('--zip', help="Name of a zip file to import") + args = parser.parse_args() From 440fae8ccf4adca68cbc0faf694d7215f5ec534d Mon Sep 17 00:00:00 2001 From: "Daniel Z. Geda" Date: Fri, 1 Dec 2023 18:36:04 +0300 Subject: [PATCH 3/3] creating unit test at test folder --- .gitignore | 2 +- tests/test1.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/test1.py diff --git a/.gitignore b/.gitignore index 4b6712e..de6b3bb 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ conf/**/*credentials* # ignore everything in the following folders data/** logs/** -src/anonymized/ +anonymized/ # except their sub-folders !data/**/ !logs/**/ diff --git a/tests/test1.py b/tests/test1.py new file mode 100644 index 0000000..b2a1834 --- /dev/null +++ b/tests/test1.py @@ -0,0 +1,26 @@ +# File: tests/test_slack_data_loader.py + +import pytest +from your_module import SlackDataLoader +import pandas as pd + +@pytest.fixture +def slack_data_loader(): + # Set up SlackDataLoader instance if needed + return SlackDataLoader() + +def test_load_data_columns(slack_data_loader): + + expected_columns = ['col1', 'col2', 'col3'] + data_frame = slack_data_loader.load_data() + assert isinstance(data_frame, pd.DataFrame) + assert all(column in data_frame.columns for column in expected_columns), "Columns do not match the expected columns" + +def test_process_data_columns(slack_data_loader): + expected_columns = ['processed_column1', 'processed_column2'] + + data_frame = slack_data_loader.load_data() + processed_data_frame = slack_data_loader.process_data(data_frame) + + assert isinstance(processed_data_frame, pd.DataFrame) + assert all(column in processed_data_frame.columns for column in expected_columns), "Processed columns do not match the expected columns" \ No newline at end of file