From ca4bbc7f6a2c710beede3d2a6500cf4fcdc2b270 Mon Sep 17 00:00:00 2001 From: "Steven (Szu-Han) Chen" Date: Mon, 15 Mar 2021 23:55:11 -0400 Subject: [PATCH 1/3] edit readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 4954905..4976c4d 100644 --- a/README.md +++ b/README.md @@ -168,3 +168,6 @@ A count for all the people who are currently infected for a given date. - Written by me (Aaron Ward - https://www.linkedin.com/in/aaronjward/) - A special thank you to the [JHU CSSE](https://systems.jhu.edu/) team for maintaining the data - Also a special thank you to @ajaymaity for bug fixes 🎉 + +## Fork +Forked this project by Steven Chen for York EECS 3311 class \ No newline at end of file From 2d60cc5d7400ba317a73262c4298f038afed9d64 Mon Sep 17 00:00:00 2001 From: "Steven (Szu-Han) Chen" Date: Thu, 15 Apr 2021 23:38:50 -0400 Subject: [PATCH 2/3] added a factory design pattern to generating reports in data prep for flexibility --- src/covidify/data_prep.py | 203 +++++++++++++++++++++++++++++--------- 1 file changed, 157 insertions(+), 46 deletions(-) diff --git a/src/covidify/data_prep.py b/src/covidify/data_prep.py index 6153d44..f210154 100644 --- a/src/covidify/data_prep.py +++ b/src/covidify/data_prep.py @@ -98,16 +98,6 @@ def check_specified_country(df, country): print('... No specific country specified') return df -df = check_specified_country(df, country) - -############ DAILY CASES ############ - -# sheets need to be sorted by date value -# print('Sorting by datetime...') -df = df.sort_values('datetime') - -current_date = str(datetime.date(datetime.now())) - ''' Get the difference of the sum totals for each date and plot them on a trendline graph @@ -137,6 +127,41 @@ def get_exp_moving_average(tmp, col): return df[col].ewm(span=2, adjust=True).mean() +def get_top_countries(data): + # Get top N infected countries + tmp_df = data.copy() + tmp_df = tmp_df[tmp_df.file_date == df.file_date.max()] + return tmp_df.groupby(['country']).agg({'confirmed': 'sum'}).sort_values('confirmed',ascending=False).head(top).index + +def get_day_counts(d, country): + ''' + For each country, get the days of the spread since 500 + cases + ''' + data = d.copy() + result_df = pd.DataFrame([]) + result_df = data.groupby(['file_date']).agg({'confirmed': 'sum', + 'recovered': 'sum', + 'deaths': 'sum'}) + result_df['date'] = data['file_date'].unique() + result_df['country'] = country + + result_df = result_df[result_df.confirmed >= 500] + result_df.insert(loc=0, column='day', value=np.arange(len(result_df))) + return result_df + +#### START SCRIPT #### + +df = check_specified_country(df, country) + +############ DAILY CASES ############ + +# sheets need to be sorted by date value +# print('Sorting by datetime...') +df = df.sort_values('datetime') + +current_date = str(datetime.date(datetime.now())) + print('... Calculating dataframe for new cases') daily_cases_df = pd.DataFrame([]) daily_cases_df['date'] = df.file_date.unique() @@ -163,46 +188,22 @@ def get_exp_moving_average(tmp, col): ############ LOG DATA ############ print('Calculating data for logarithmic plotting...') + if not country: print('... top infected countries: {}'.format(top)) -def get_top_countries(data): - # Get top N infected countries - tmp_df = data.copy() - tmp_df = tmp_df[tmp_df.file_date == df.file_date.max()] - return tmp_df.groupby(['country']).agg({'confirmed': 'sum'}).sort_values('confirmed',ascending=False).head(top).index - TOP_N_COUNTRIES = get_top_countries(df) tmp_df = df[df.country.isin(TOP_N_COUNTRIES)].copy() -def get_day_counts(d, country): - ''' - For each country, get the days of the spread since 500 - cases - ''' - data = d.copy() - result_df = pd.DataFrame([]) - result_df = data.groupby(['file_date']).agg({'confirmed': 'sum', - 'recovered': 'sum', - 'deaths': 'sum'}) - result_df['date'] = data['file_date'].unique() - result_df['country'] = country - - result_df = result_df[result_df.confirmed >= 500] - result_df.insert(loc=0, column='day', value=np.arange(len(result_df))) - return result_df - df_list = [] for country in TOP_N_COUNTRIES: - print(' ...', country + ': ' + str(tmp_df[(tmp_df.file_date == df.file_date.max()) & - (tmp_df.country == country)].confirmed.sum())) + print(' ...', country + ': ' + str(tmp_df[(tmp_df.file_date == df.file_date.max()) & (tmp_df.country == country)].confirmed.sum())) df_list.append(get_day_counts(tmp_df[tmp_df.country == country], country)) log_df = pd.concat(df_list, axis=0, ignore_index=True) - ############ SAVE DATA ############ #Create date of extraction folder data_folder = os.path.join('data', str(datetime.date(datetime.now()))) @@ -214,17 +215,127 @@ def get_day_counts(d, country): print('Creating subdirectory for data...') print('...', save_dir) -print('Saving...') -csv_file_name = 'agg_data_{}.csv'.format(datetime.date(datetime.now())) -df.astype(str).to_csv(os.path.join(save_dir, csv_file_name)) -print('...', csv_file_name) +# class Report(): +# """ +# """ +# def __init__(self, data, report_type, save_dir) -> None: +# self.df = data +# self.save_dir = save_dir +# self.report_type = report_type +# self.filename = self.create_filename() + +# def create_filename(self): +# self.filename = '{}_{}.csv'.format(self.report_type, datetime.date(datetime.now())) + +# def save_file(self): +# self.create_filename() +# self.df.astype(str).to_csv(os.path.join(self.save_dir, self.filename)) +# print('...', self.filename) + +# print('Saving...') + +# agg_report = Report(data=df, report_type="agg", save_dir=save_dir) +# agg_report.save_file() + +# trend_report = Report(data=daily_cases_df, report_type="trend", save_dir=save_dir) +# trend_report.save_file() + +# log_report = Report(data=log_df, report_type="log", save_dir=save_dir) +# log_report.save_file() + +# csv_file_name = 'agg_data_{}.csv'.format(datetime.date(datetime.now())) +# df.astype(str).to_csv(os.path.join(save_dir, csv_file_name)) +# print('...', csv_file_name) + +# daily_cases_file_name = 'trend_{}.csv'.format(datetime.date(datetime.now())) +# daily_cases_df.astype(str).to_csv(os.path.join(save_dir, daily_cases_file_name)) +# print('...', daily_cases_file_name) + +# log_file_name = 'log_{}.csv'.format(datetime.date(datetime.now())) +# log_df.astype(str).to_csv(os.path.join(save_dir, log_file_name)) +# print('...', log_file_name) -daily_cases_file_name = 'trend_{}.csv'.format(datetime.date(datetime.now())) -daily_cases_df.astype(str).to_csv(os.path.join(save_dir, daily_cases_file_name)) -print('...', daily_cases_file_name) +# print('Done!') -log_file_name = 'log_{}.csv'.format(datetime.date(datetime.now())) -log_df.astype(str).to_csv(os.path.join(save_dir, log_file_name)) -print('...', log_file_name) +from abc import ABC, abstractstaticmethod, abstractclassmethod, abstractmethod, abstractproperty +class ReportCreator(ABC): + + @abstractmethod + def create_report(self): + pass + + def save_csv(self): + report = self.create_report() + report.save_csv() + +class LogReportCreator(ReportCreator): + + def __init__(self, df): + self.df = df + + def create_report(self): + return LogReport(self.df) + +class TrendReportCreator(ReportCreator): + + def __init__(self, df): + self.df = df + + def create_report(self): + return TrendReport(self.df) + +class AggReportCreator(ReportCreator): + + def __init__(self, df): + self.df = df + + def create_report(self): + return AggReport(self.df) + +class Report(ABC): + + @abstractmethod + def save_csv(self): + pass + +class LogReport(Report): + + def __init__(self, df): + self.df = df + + def save_csv(self): + self.filename = 'log_{}.csv'.format(datetime.date(datetime.now())) + self.df.astype(str).to_csv(os.path.join(save_dir, self.filename)) + print('...', self.filename) + +class TrendReport(Report): + + def __init__(self, df): + self.df = df + + def save_csv(self): + self.filename = 'trend_{}.csv'.format(datetime.date(datetime.now())) + self.df.astype(str).to_csv(os.path.join(save_dir, self.filename)) + print('...', self.filename) + +class AggReport(Report): + + def __init__(self, df): + self.df = df + + def save_csv(self): + self.filename = 'agg_data_{}.csv'.format(datetime.date(datetime.now())) + self.df.astype(str).to_csv(os.path.join(save_dir, self.filename)) + print('...', self.filename) + +def create_report(creator: ReportCreator): + report = creator.create_report() + report.save_csv() + + +print('Saving...') +create_report(AggReportCreator(df=df)) +create_report(TrendReportCreator(df=daily_cases_df)) +create_report(LogReportCreator(df=log_df)) print('Done!') \ No newline at end of file From 51f9ede56714cf78b3e3e9c8702c1e711a5f715a Mon Sep 17 00:00:00 2001 From: "Steven (Szu-Han) Chen" Date: Fri, 16 Apr 2021 00:11:13 -0400 Subject: [PATCH 3/3] added adapter pattern to save files as json --- src/covidify/data_prep.py | 41 +++++++++++++-------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/src/covidify/data_prep.py b/src/covidify/data_prep.py index f210154..867ea7e 100644 --- a/src/covidify/data_prep.py +++ b/src/covidify/data_prep.py @@ -215,34 +215,8 @@ def get_day_counts(d, country): print('Creating subdirectory for data...') print('...', save_dir) -# class Report(): -# """ -# """ -# def __init__(self, data, report_type, save_dir) -> None: -# self.df = data -# self.save_dir = save_dir -# self.report_type = report_type -# self.filename = self.create_filename() - -# def create_filename(self): -# self.filename = '{}_{}.csv'.format(self.report_type, datetime.date(datetime.now())) - -# def save_file(self): -# self.create_filename() -# self.df.astype(str).to_csv(os.path.join(self.save_dir, self.filename)) -# print('...', self.filename) - # print('Saving...') -# agg_report = Report(data=df, report_type="agg", save_dir=save_dir) -# agg_report.save_file() - -# trend_report = Report(data=daily_cases_df, report_type="trend", save_dir=save_dir) -# trend_report.save_file() - -# log_report = Report(data=log_df, report_type="log", save_dir=save_dir) -# log_report.save_file() - # csv_file_name = 'agg_data_{}.csv'.format(datetime.date(datetime.now())) # df.astype(str).to_csv(os.path.join(save_dir, csv_file_name)) # print('...', csv_file_name) @@ -338,4 +312,17 @@ def create_report(creator: ReportCreator): create_report(AggReportCreator(df=df)) create_report(TrendReportCreator(df=daily_cases_df)) create_report(LogReportCreator(df=log_df)) -print('Done!') \ No newline at end of file +print('Done!') + +class Adaptee: + def save_json(self): + self.filename = 'agg_data_{}.json'.format(datetime.date(datetime.now())) + self.df.astype(str).to_json(os.path.join(save_dir, self.filename)) + print('...', self.filename) + +class Adapter(AggReport, Adaptee): + def save_csv(self): + self.save_json() + +aggReportJsonAdapter = Adapter(df=df) +aggReportJsonAdapter.save_csv() # but saves as json