From 9df8111ce6865f59d8ccb90b9ebead6df89d8486 Mon Sep 17 00:00:00 2001 From: Rao Lu Date: Thu, 15 Apr 2021 19:29:31 -0400 Subject: [PATCH 1/2] apply builder pattern --- builder/builder.py | 119 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 builder/builder.py diff --git a/builder/builder.py b/builder/builder.py new file mode 100644 index 0000000..eb205da --- /dev/null +++ b/builder/builder.py @@ -0,0 +1,119 @@ +class Plotter(ABC): + + @abstractmethod + def setTitle(self): + pass + + @abstractmethod + def setXlabel(self): + pass + + @abstractmethod + def setYlabel(self): + pass + + @abstractmethod + def getResult(self): + pass + + +class Graph(): + def __init__(self): + self.Xcol = None + self.Ycol = None + self.style = None + self.title = None + self.Xlabel = None + self.Ylabel = None + self.grid = None + + +class ScatterPlotterv1(Plotter): + def __init__(self): + self.graph = Graph() + self.x = 10 + self.y = 6 + self.style = 'o' + + def setXlabel(self, xl): + self.graph.Xlabel = xl + return self.graph + + def setYlabel(self, yl): + self.graph.Ylabel = yl + return self.graph + + def setTitle(self, t): + self.graph.title = t + return self.graph + + def getResult(self): + return self.graph + + +class ScatterPlotterv2(Plotter): + def __init__(self): + self.graph = Graph() + self.style = 'o' + + def setXcol(self, x): + self.graph.Xcol = x + return self.graph + + def setYcol(self, y): + self.graph.Ycol = y + return self.graph + + def setStyle(self, s): + self.graph.style = s + return self.graph + + def setTitle(self, t): + self.graph.title = t + return self.graph + + def setXlabel(self, xl): + self.graph.Xlabel = xl + return self.graph + + def setYlabel(self, yl): + self.graph.Ylabel = yl + return self.graph + + def getResult(self): + return self.graph + + +class SeriesPlotter(Plotter): + def __init__(self): + self.graph = Graph() + self.start = 0 + self.end = None + self.format = '-' + + def setTime(self): + self.time = time[start, end] + return self.graph + + def setXlabel(self, xl): + self.graph.setXlabel = xl + return self.graph + + def setYlabel(self, yl): + self.graph.setYlabel = yl + return self.graph + + def setGrid(self, g): + self.graph.grid = g + return self.graph + + def getResult(self): + return self.graph + + +class Director: + def __init__(self) -> None: + self._builder = None + + def setGraph(self, graph): + self.graph = graph From 7da373f6199e0279bd473be537d2f4447be18c0b Mon Sep 17 00:00:00 2001 From: Rao Lu Date: Thu, 15 Apr 2021 19:34:02 -0400 Subject: [PATCH 2/2] apply bridge pattern --- bridge/bridge.py | 328 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 328 insertions(+) create mode 100644 bridge/bridge.py diff --git a/bridge/bridge.py b/bridge/bridge.py new file mode 100644 index 0000000..a1cfd47 --- /dev/null +++ b/bridge/bridge.py @@ -0,0 +1,328 @@ +import os +import sys +import git +from datetime import datetime, date, time +from dateutil.parser import parse + +REPO = 'https://github.com/CSSEGISandData/COVID-19.git' +TMP_FOLDER = '/tmp/corona/' +TMP_GIT = os.path.join(TMP_FOLDER, 'COVID-19') +DATA = os.path.join(TMP_GIT, 'csse_covid_19_data/csse_covid_19_daily_reports/') +out = './' + +class DataAccessor: + def __init__(self, cleaner): + self.cleaner = cleaner + + def get_date(self, last_update): + pass + + def get_csv_date(self, file): + pass + + def get_data(self, cleaned_sheets): + pass + + +class AdvancedDataAccessor(DataAccessor): + def __init__(self, cleaner): + self.cleaner = cleaner + # self.REPO = 'https://github.com/CSSEGISandData/COVID-19.git' + # self.TMP_FOLDER = '/tmp/corona/' + # self.TMP_GIT = os.path.join(TMP_FOLDER, 'COVID-19') + # self.DATA = os.path.join(TMP_GIT, 'csse_covid_19_data/csse_covid_19_daily_reports/') + # self.out = './' + + def get_date(last_update): + return parse(str(last_update).split(' ')[0]).strftime("%Y-%m-%d") + + def get_csv_date(file): + return get_date(file.split('.')[0] + ' ') + + def get_data(cleaned_sheets): + all_csv = [] + # Import all CSV's + for file in tqdm(sorted(sheets), desc='... importing data: '): + if 'csv' in file: + # print('...', file) + tmp_df = pd.read_csv(os.path.join(DATA, file), index_col=None, header=0, parse_dates=['Last Update']) + tmp_df = tmp_df[keep_cols] + tmp_df[numeric_cols] = tmp_df[numeric_cols].fillna(0) + tmp_df[numeric_cols] = tmp_df[numeric_cols].astype(int) + tmp_df['Province/State'].fillna(tmp_df['Country/Region'], inplace=True) #If no region given, fill it with country + + tmp_df['Last Update'] = tmp_df['Last Update'].apply(clean_last_updated) + tmp_df['date'] = tmp_df['Last Update'].apply(get_date) + tmp_df['file_date'] = get_csv_date(file) + all_csv.append(tmp_df) + + # concatenate all csv's into one df + df_raw = pd.concat(all_csv, axis=0, ignore_index=True, sort=True) + df_raw = df_raw.sort_values(by=['Last Update']) + + frames = cleaner.drop_duplicates(df_raw) + tmp = pd.concat(frames, axis=0, ignore_index=True, sort=True) + + return tmp + + def get_clean_sheets(): + # Create Tmp Folder + if not os.path.isdir(TMP_FOLDER): + print('Creating folder...') + print('...', TMP_FOLDER) + os.mkdir(TMP_FOLDER) + + #Check if repo exists + #git pull if it does + if not os.path.isdir(TMP_GIT): + cleaner.clone_repo(TMP_FOLDER, REPO) + else: + try: + print('git pull from', REPO) + rep = git.Repo(TMP_GIT) + rep.remotes.origin.pull() + except: + print('Could not pull from', REPO) + sys.exit() + + sheets = os.listdir(DATA) + + # Clean the result to the sheet tabs we want + print('Getting sheets...') + cleaned_sheets = cleaner.clean_sheet_names(sheets) + + return cleaned_sheets + + def get_similar_countries(c, country_list): + pos_countries = get_close_matches(c, country_list) + + if len(pos_countries) > 0: + print(c, 'was not listed. did you mean', pos_countries[0].capitalize() + '?') + sys.exit() + else: + print(c, 'was not listed.') + sys.exit() + + def get_new_cases(tmp, col): + diff_list = [] + tmp_df_list = [] + df = tmp.copy() + + for i, day in enumerate(df.sort_values('date').date.unique()): + tmp_df = df[df.date == day] + tmp_df_list.append(tmp_df[col].sum()) + + if i == 0: + diff_list.append(tmp_df[col].sum()) + else: + diff_list.append(tmp_df[col].sum() - tmp_df_list[i-1]) + + return diff_list + + def get_moving_average(tmp, col): + df = tmp.copy() + return df[col].rolling(window=2).mean() + + def get_exp_moving_average(tmp, col): + df = tmp.copy() + return df[col].ewm(span=2, adjust=True).mean() + + +class Cleaner: + @abstractmethod + def clean_sheet_names(self, new_ranges): + pass + + @abstractmethod + def clean_last_updates(self, last_update): + pass + + @abstractmethod + def drop_duplicates(self, df_raw): + pass + + @abstractmethod + def clone_repo(self, TMP_FOLDER, REPO): + pass + + @abstractmethod + def clean_data(self, tmp_df): + pass + + @abstractmethod + def clean_sheets(self): + pass + + +class DataCleaner3(Cleaner): + + def clean_data(tmp_df): + if 'Demised' in tmp_df.columns: + tmp_df.rename(columns={'Demised':'Deaths'}, inplace=True) + + if 'Country/Region' in tmp_df.columns: + tmp_df.rename(columns={'Country/Region':'country'}, inplace=True) + + if 'Province/State' in tmp_df.columns: + tmp_df.rename(columns={'Province/State':'province'}, inplace=True) + + if 'Last Update' in tmp_df.columns: + tmp_df.rename(columns={'Last Update':'datetime'}, inplace=True) + + if 'Suspected' in tmp_df.columns: + tmp_df = tmp_df.drop(columns='Suspected') + + for col in tmp_df.columns: + tmp_df[col] = tmp_df[col].fillna(0) + + #Lower case all col names + tmp_df.columns = map(str.lower, tmp_df.columns) + return tmp_df + + def clean_last_updated(last_update): + ''' + convert date and time in YYYYMMDD HMS format + ''' + date = parse(str(last_update).split(' ')[0]).strftime("%Y-%m-%d") + time = parse(str(last_update).split(' ')[1]).strftime('%H:%M:%S') + parsed_date = str(date) + ' ' + str(time) + + return parsed_date + + def clean_sheet_names(new_ranges): + indices = [] + # Remove all sheets that dont have a numeric header + numeric_sheets = [x for x in new_ranges if re.search(r'\d', x)] + + return numeric_sheets + + def drop_duplicates(df_raw): + ''' + Take the max date value for each province for a given date + ''' + days_list = [] + + for datetime in df_raw.date.unique(): + tmp_df = df_raw[df_raw.date == datetime] + tmp_df = tmp_df.sort_values(['Last Update']).drop_duplicates('Province/State', keep='last') + days_list.append(tmp_df) + + return days_list + + def clone_repo(TMP_FOLDER, REPO): + print('Cloning Data Repo...') + git.Git(TMP_FOLDER).clone(REPO) + + def clean_sheets(self): + # Create Tmp Folder + if not os.path.isdir(TMP_FOLDER): + print('Creating folder...') + print('...', TMP_FOLDER) + os.mkdir(TMP_FOLDER) + + #Check if repo exists + #git pull if it does + if not os.path.isdir(TMP_GIT): + clone_repo(TMP_FOLDER, REPO) + else: + try: + print('git pull from', REPO) + rep = git.Repo(TMP_GIT) + rep.remotes.origin.pull() + except: + print('Could not pull from', REPO) + sys.exit() + + sheets = os.listdir(DATA) + + # Clean the result to the sheet tabs we want + print('Cleaning sheets...') + cleaned_sheets = clean_sheet_names(sheets) + return cleaned_sheets + + +class DataCleaner4(Cleaner): + + def clean_data(tmp_df): + if 'Demised' in tmp_df.columns: + tmp_df.rename(columns={'Demised':'Deaths'}, inplace=True) + + if 'Country/Region' in tmp_df.columns: + tmp_df.rename(columns={'Country/Region':'country'}, inplace=True) + + if 'Province/State' in tmp_df.columns: + tmp_df.rename(columns={'Province/State':'province'}, inplace=True) + + if 'Last Update' in tmp_df.columns: + tmp_df.rename(columns={'Last Update':'datetime'}, inplace=True) + + if 'Suspected' in tmp_df.columns: + tmp_df = tmp_df.drop(columns='Suspected') + + for col in tmp_df.columns: + tmp_df[col] = tmp_df[col].fillna(0) + + #Lower case all col names + tmp_df.columns = map(str.lower, tmp_df.columns) + return tmp_df + + def clean_last_updated(last_update): + ''' + convert date and time in YYYYMMDD HMS format + ''' + date = parse(str(last_update).split(' ')[0]).strftime("%Y-%m-%d") + time = parse(str(last_update).split(' ')[1]).strftime('%H:%M:%S') + parsed_date = str(date) + ' ' + str(time) + + return parsed_date + + def clean_sheet_names(new_ranges): + indices = [] + # Remove all sheets that dont have a numeric header + numeric_sheets = [x for x in new_ranges if re.search(r'\d', x)] + + return numeric_sheets + + def drop_duplicates(df_raw): + ''' + Take the max date value for each province for a given date + ''' + days_list = [] + + for datetime in df_raw.date.unique(): + tmp_df = df_raw[df_raw.date == datetime] + tmp_df = tmp_df.sort_values(['Last Update']).drop_duplicates('Province/State', keep='last') + days_list.append(tmp_df) + + return days_list + + def clone_repo(TMP_FOLDER, REPO): + print('Cloning Data Repo...') + git.Git(TMP_FOLDER).clone(REPO) + + def clean_sheets(self): + # Create Tmp Folder + if not os.path.isdir(TMP_FOLDER): + print('Creating folder...') + print('...', TMP_FOLDER) + os.mkdir(TMP_FOLDER) + + #Check if repo exists + #git pull if it does + if not os.path.isdir(TMP_GIT): + clone_repo(TMP_FOLDER, REPO) + else: + try: + print('git pull from', REPO) + rep = git.Repo(TMP_GIT) + rep.remotes.origin.pull() + except: + print('Could not pull from', REPO) + sys.exit() + + sheets = os.listdir(DATA) + + # Clean the result to the sheet tabs we want + print('Cleaning sheets...') + cleaned_sheets = clean_sheet_names(sheets) + return cleaned_sheets \ No newline at end of file