From 83c25c98a0af66ac21df51a9371a81cb842fbd19 Mon Sep 17 00:00:00 2001 From: Powell Date: Fri, 23 Dec 2022 16:36:15 -0500 Subject: [PATCH 01/15] Start refactoring code into separate modules, got tests running in docker container. --- Dockerfile | 2 +- mixmasta/cli.py | 308 ++++---- mixmasta/file_processor.py | 263 +++++++ mixmasta/mixmasta.py | 1433 ++++-------------------------------- mixmasta/normalizer.py | 1087 +++++++++++++++++++++++++++ 5 files changed, 1692 insertions(+), 1401 deletions(-) create mode 100644 mixmasta/file_processor.py create mode 100644 mixmasta/normalizer.py diff --git a/Dockerfile b/Dockerfile index 91fad9a..90e135c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,4 +32,4 @@ COPY . / RUN python3 setup.py install #RUN mixmasta download -ENTRYPOINT ["mixmasta"] +# ENTRYPOINT ["mixmasta"] diff --git a/mixmasta/cli.py b/mixmasta/cli.py index c6f2348..786fe1d 100644 --- a/mixmasta/cli.py +++ b/mixmasta/cli.py @@ -6,9 +6,11 @@ import pandas as pd from .download import download_and_clean -from .mixmasta import geocode, netcdf2df, process, raster2df, normalizer, optimize_df_types, mixdata -#from download import download_and_clean -#from mixmasta import geocode, netcdf2df, process, raster2df, normalizer, optimize_df_types, mixdata +from .mixmasta import geocode, process, normalizer, optimize_df_types, mixdata +from .file_processor import netcdf2df, raster2df + +# from download import download_and_clean +# from mixmasta import geocode, netcdf2df, process, raster2df, normalizer, optimize_df_types, mixdata from glob import glob import numpy as np @@ -19,37 +21,45 @@ # Constants CHUNK_SIZE = 100000 -DATA_TEMP_FILENAME = 'causmosify_multi_tmp' -PROCESSED_TEMP_FILENAME = 'processed_tmp' +DATA_TEMP_FILENAME = "causmosify_multi_tmp" +PROCESSED_TEMP_FILENAME = "processed_tmp" + @click.group() def cli(): pass -def chunk_normalize(input_file: str, mapper: dict, renamed_col_dict: dict, geo: str, gadm, df_geocode: pd.DataFrame) -> dict: +def chunk_normalize( + input_file: str, + mapper: dict, + renamed_col_dict: dict, + geo: str, + gadm, + df_geocode: pd.DataFrame, +) -> dict: """ - Description - ---------- - Normalize input_file in chunks written to pkl files. - - Parameters - ---------- - input_file: str - Filename of file to normalize. - mapper: dict - Mapper dict for filename. - renamed_col_dict: dict - Dict maintained to track columns renamed during normalize() - gadm: - GADM geopandas object passed as param so it is loaded only once - df_geocode: pd.DataFrame - lat/lng geocode library to cache geocoding. - - Returns - ------- - dict: - updated renamed_col_dict + Description + ---------- + Normalize input_file in chunks written to pkl files. + + Parameters + ---------- + input_file: str + Filename of file to normalize. + mapper: dict + Mapper dict for filename. + renamed_col_dict: dict + Dict maintained to track columns renamed during normalize() + gadm: + GADM geopandas object passed as param so it is loaded only once + df_geocode: pd.DataFrame + lat/lng geocode library to cache geocoding. + + Returns + ------- + dict: + updated renamed_col_dict """ @@ -57,12 +67,12 @@ def chunk_normalize(input_file: str, mapper: dict, renamed_col_dict: dict, geo: df_columns = [] first_norm_time = 0 - click.echo(f'\nLoading {input_file} ...') + click.echo(f"\nLoading {input_file} ...") start_time = timeit.default_timer() - + # Set the transformation type, and reduce the mapper to date, geo and feature keys. transform = mapper["meta"] - mapper = { k: mapper[k] for k in mapper.keys() & {"date", "geo", "feature"} } + mapper = {k: mapper[k] for k in mapper.keys() & {"date", "geo", "feature"}} # File type-specific pre-processing ftype = transform["ftype"] @@ -73,77 +83,89 @@ def chunk_normalize(input_file: str, mapper: dict, renamed_col_dict: dict, geo: d = transform["date"] df = raster2df( - InRaster = input_file, - feature_name = transform["feature_name"], - band = int(transform["band"] if "band" in transform and transform["band"] != "" else "0"), - nodataval = int(transform["null_val"]), - date = d, - band_name = transform["band_name"], - bands = transform["bands"] if "bands" in transform else None + InRaster=input_file, + feature_name=transform["feature_name"], + band=int( + transform["band"] + if "band" in transform and transform["band"] != "" + else "0" + ), + nodataval=int(transform["null_val"]), + date=d, + band_name=transform["band_name"], + bands=transform["bands"] if "bands" in transform else None, ) - elif ftype == 'excel': - df = pd.read_excel(input_file, transform['sheet']) + elif ftype == "excel": + df = pd.read_excel(input_file, transform["sheet"]) elif ftype != "csv": df = netcdf2df(input_file) else: df = pd.read_csv(input_file) df.reset_index(inplace=True, drop=True) - + # Set the number of chunks based on CHUNK_SIZE - chunks = 1 + (df.shape[0] // CHUNK_SIZE) - + chunks = 1 + (df.shape[0] // CHUNK_SIZE) + # Entire dataset is loaded into df. Write in chunks to pkl files. - for i in range(1, chunks+1): - filename = f'{DATA_TEMP_FILENAME}.{i}.pkl' - stop_row = i*CHUNK_SIZE + for i in range(1, chunks + 1): + filename = f"{DATA_TEMP_FILENAME}.{i}.pkl" + stop_row = i * CHUNK_SIZE start_row = stop_row - CHUNK_SIZE - df.iloc[start_row:stop_row,:].to_pickle(filename) - - click.echo(f'Processing {input_file} ...') + df.iloc[start_row:stop_row, :].to_pickle(filename) + + click.echo(f"Processing {input_file} ...") start_time = timeit.default_timer() - + # Iterate chunk tmp files and normalize each loaded df. - for i in range(1, chunks+1): - read_filename = f'{DATA_TEMP_FILENAME}.{i}.pkl' + for i in range(1, chunks + 1): + read_filename = f"{DATA_TEMP_FILENAME}.{i}.pkl" df_temp = pd.read_pickle(read_filename) ## Run normalizer. norm_start_time = timeit.default_timer() - norm, result_dict, df_geocode = normalizer(df_temp, mapper, geo, gadm=gadm, df_geocode=df_geocode) - + norm, result_dict, df_geocode = normalizer( + df_temp, mapper, geo, gadm=gadm, df_geocode=df_geocode + ) + # Normalizer will add NaN for missing values, e.g. when appending # dataframes with different columns. GADM will return None when geocoding # but not finding the entity (e.g. admin3 for United States). # Replace None with NaN for consistency. norm.fillna(value=np.nan, inplace=True) - # In edge cases where the input files have different columns, keep + # In edge cases where the input files have different columns, keep # list of all columns for reading the tmp file. for col in norm.columns.values.tolist(): if col not in df_columns: df_columns.append(col) - + if len(df_datatypes) == 0: # Record datatypes of normalized df for setting on read. norm = optimize_df_types(norm) df_datatypes = dict(norm.dtypes) - #norm.to_csv(chunked_temp_filename, mode='a', index=False, header=False) - write_filename = f'{PROCESSED_TEMP_FILENAME}.{os.path.basename(input_file)}.{i}.pkl' + # norm.to_csv(chunked_temp_filename, mode='a', index=False, header=False) + write_filename = ( + f"{PROCESSED_TEMP_FILENAME}.{os.path.basename(input_file)}.{i}.pkl" + ) norm.to_pickle(write_filename) # A little cleanup, probably ineffective. - del(norm) - del(df_temp) + del norm + del df_temp # Delete tmp files as they are read. os.remove(read_filename) # Combine dict to return single file. - renamed_col_dict = result_dict if not renamed_col_dict else {**renamed_col_dict, **result_dict} + renamed_col_dict = ( + result_dict if not renamed_col_dict else {**renamed_col_dict, **result_dict} + ) - click.echo(f'{input_file} processing completed in {timeit.default_timer() - start_time} seconds') + click.echo( + f"{input_file} processing completed in {timeit.default_timer() - start_time} seconds" + ) return renamed_col_dict @@ -151,20 +173,20 @@ def chunk_normalize(input_file: str, mapper: dict, renamed_col_dict: dict, geo: def get_gadm(geo: str): # Cache GADM for normalize() loops. md = mixdata() - if geo.lower() in ['admin0', 'country']: - click.echo('Loading GADM0 ...') + if geo.lower() in ["admin0", "country"]: + click.echo("Loading GADM0 ...") md.load_gadm2() gadm = md.gadm0 - elif geo.lower() == 'admin1': - click.echo('Loading GADM1 ...') + elif geo.lower() == "admin1": + click.echo("Loading GADM1 ...") md.load_gadm2() gadm = md.gadm1 - elif geo.lower() == 'admin2': - click.echo('Loading GADM2 ...') + elif geo.lower() == "admin2": + click.echo("Loading GADM2 ...") md.load_gadm2() gadm = md.gadm2 else: - click.echo('Loading GADM3 ...') + click.echo("Loading GADM3 ...") md.load_gadm3() gadm = md.gadm3 @@ -186,17 +208,20 @@ def glob_input_file(input_file: str) -> str: input_file = input_file return input_file + @cli.command() @click.option("--input_file", type=str, default=None) @click.option("--mapper", type=str, default=None) @click.option("--geo", type=str, default=None) @click.option("--output_file", type=str, default="mixmasta_output") -def causemosify(input_file, mapper, geo: str = None, output_file: str = "mixmasta_output"): +def causemosify( + input_file, mapper, geo: str = None, output_file: str = "mixmasta_output" +): """Processor for generating CauseMos compliant datasets.""" click.echo("Causemosifying data...") - input_file = glob_input_file(input_file) - + input_file = glob_input_file(input_file) + with open(mapper) as f: mapper = json.loads(f.read()) @@ -206,7 +231,9 @@ def causemosify(input_file, mapper, geo: str = None, output_file: str = "mixmast if "geocode_level" in transform: geo = transform["geocode_level"] else: - click.echo("No geo specified and no meta.geocode_level in mapper.json. Defaulting to admin2.") + click.echo( + "No geo specified and no meta.geocode_level in mapper.json. Defaulting to admin2." + ) geo = "admin2" gadm = get_gadm(geo) @@ -217,31 +244,33 @@ def causemosify(input_file, mapper, geo: str = None, output_file: str = "mixmast renamed_col_dict = chunk_normalize(input_file, mapper, {}, geo, gadm, df_geocode) # Reassemble tmp pkl files. - df_final = pd.concat([pd.read_pickle(fl) for fl in glob(f'{PROCESSED_TEMP_FILENAME}.*.pkl')]) + df_final = pd.concat( + [pd.read_pickle(fl) for fl in glob(f"{PROCESSED_TEMP_FILENAME}.*.pkl")] + ) df_final.reset_index(inplace=True, drop=True) - - # Clean up reassembly. - for fl in glob(f'{PROCESSED_TEMP_FILENAME}*'): + + # Clean up reassembly. + for fl in glob(f"{PROCESSED_TEMP_FILENAME}*"): os.remove(fl) - + # Write separate parquet files. - df_final['type'] = df_final[['value']].applymap(type) - df_final_str = df_final[df_final['type']==str] - df_final = df_final[df_final['type']!=str] - del(df_final_str['type']) - del(df_final['type']) + df_final["type"] = df_final[["value"]].applymap(type) + df_final_str = df_final[df_final["type"] == str] + df_final = df_final[df_final["type"] != str] + del df_final_str["type"] + del df_final["type"] df_final.to_parquet(f"{output_file}.parquet.gzip", compression="gzip") if not df_final_str.empty: df_final_str.to_parquet(f"{output_file}_str.parquet.gzip", compression="gzip") - + # Rebuild and reduce memory size of returned dataframe. df_final = df_final.append(df_final_str) df_final = optimize_df_types(df_final) df_final.reset_index(inplace=True, drop=True) return df_final, renamed_col_dict - + @cli.command() @click.option("--inputs", type=str, default=None) @@ -263,36 +292,36 @@ def causemosify_multi(inputs, geo: str = None, output_file: str = "mixmasta_outp Writes separate parquet files for each input file. """ - input_array = json.loads(inputs) + input_array = json.loads(inputs) click.echo(f"Causemosifying {len(input_array)} file(s) ...") - + # Setup variables. renamed_col_dict = {} df_geocode = pd.DataFrame() # Delete any previous tmp files. - for fl in glob(f'{PROCESSED_TEMP_FILENAME}*'): + for fl in glob(f"{PROCESSED_TEMP_FILENAME}*"): os.remove(fl) - - for fl in glob(f'{DATA_TEMP_FILENAME}*'): + + for fl in glob(f"{DATA_TEMP_FILENAME}*"): os.remove(fl) - + # Iterate each item_item in the --inputs JSON. item_counter = 0 # Geo control varibles to avoid reloading GADM. - item_geo = None # geo for item to be processed - last_geo = None # geo of last item processed + item_geo = None # geo for item to be processed + last_geo = None # geo of last item processed gadm = pd.DataFrame() # All output files should have the same columns. - output_columns = [] + output_columns = [] - for item_item in input_array: + for item_item in input_array: # Handle filename wildcards with glob_input_file(). - input_file = glob_input_file(item_item["input_file"]) - - with open(item_item ["mapper"]) as f: + input_file = glob_input_file(item_item["input_file"]) + + with open(item_item["mapper"]) as f: mapper = json.loads(f.read()) # Check transform for meta.geocode_level. This is overriden by geo parameter. @@ -301,15 +330,17 @@ def causemosify_multi(inputs, geo: str = None, output_file: str = "mixmasta_outp if "geocode_level" in transform: loop_geo = transform["geocode_level"] - if (loop_geo != last_geo or gadm.empty): + if loop_geo != last_geo or gadm.empty: gadm = get_gadm(loop_geo) last_geo = loop_geo else: - click.echo("No geo specified and no meta.geocode_level in mapper.json. Defaulting to admin2.") + click.echo( + "No geo specified and no meta.geocode_level in mapper.json. Defaulting to admin2." + ) loop_geo = "admin2" - if (loop_geo != last_geo or gadm.empty): + if loop_geo != last_geo or gadm.empty: gadm = get_gadm(loop_geo) last_geo = loop_geo @@ -318,66 +349,83 @@ def causemosify_multi(inputs, geo: str = None, output_file: str = "mixmasta_outp if gadm.empty: gadm = get_gadm(geo) - renamed_col_dict = chunk_normalize(input_file = input_file, - mapper = mapper, - renamed_col_dict=renamed_col_dict, + renamed_col_dict = chunk_normalize( + input_file=input_file, + mapper=mapper, + renamed_col_dict=renamed_col_dict, geo=geo, gadm=gadm, - df_geocode=df_geocode) + df_geocode=df_geocode, + ) - df_col = pd.read_pickle(f'{PROCESSED_TEMP_FILENAME}.{os.path.basename(input_file)}.1.pkl') + df_col = pd.read_pickle( + f"{PROCESSED_TEMP_FILENAME}.{os.path.basename(input_file)}.1.pkl" + ) # Maintain master list of output columns while respecting column order. if len(output_columns) == 0: output_columns = df_col.columns.values.tolist() else: # Only add the columns in the df_col not in output_columns. - output_columns = output_columns + list(set(df_col.columns.values.tolist()).difference(set(output_columns))) + output_columns = output_columns + list( + set(df_col.columns.values.tolist()).difference(set(output_columns)) + ) - # At this point all mixmasta.normalize() is completed for all input files. - # There will be multiple .pkl files with the prefix + # At this point all mixmasta.normalize() is completed for all input files. + # There will be multiple .pkl files with the prefix # {PROCESSED_TEMP_FILENAME}.{os.path.basename(input_file)} - # - # Processing of these pkl files is delayed so that the column names of - # output files were collated ensuring the parquet files have the same + # + # Processing of these pkl files is delayed so that the column names of + # output files were collated ensuring the parquet files have the same # columns even if they are full of NaN. # Reiterate the input file names. Load all tmp pkl files. Set colnames. for idx, item_item in enumerate(input_array): # Handle filename wildcards with glob_input_file(). - input_file = glob_input_file(item_item["input_file"]) - + input_file = glob_input_file(item_item["input_file"]) + # Reassemble tmp files into df ...) - df_final = pd.concat([pd.read_pickle(fl) for fl in glob(f'{PROCESSED_TEMP_FILENAME}.{os.path.basename(input_file)}.*.pkl')]) + df_final = pd.concat( + [ + pd.read_pickle(fl) + for fl in glob( + f"{PROCESSED_TEMP_FILENAME}.{os.path.basename(input_file)}.*.pkl" + ) + ] + ) # Add any columns in output_columns but not in this dataframe. - for col in list(set(output_columns).difference(set(df_final.columns.values.tolist()))): + for col in list( + set(output_columns).difference(set(df_final.columns.values.tolist())) + ): df_final[col] = np.nan # Ensure column order is the same across all output files. df_final = df_final[output_columns] df_final.reset_index(inplace=True, drop=True) - - # ... then clean up reassembly. - for fl in glob(f'{PROCESSED_TEMP_FILENAME}.{os.path.basename(input_file)}.*'): + + # ... then clean up reassembly. + for fl in glob(f"{PROCESSED_TEMP_FILENAME}.{os.path.basename(input_file)}.*"): os.remove(fl) - + # Write separate parquet files depending on type of value column ... # ... by creating a separting df for value col of type str ... - df_final['type'] = df_final[['value']].applymap(type) - df_final_str = df_final[df_final['type']==str] - df_final = df_final[df_final['type']!=str] - del(df_final_str['type']) - del(df_final['type']) + df_final["type"] = df_final[["value"]].applymap(type) + df_final_str = df_final[df_final["type"] == str] + df_final = df_final[df_final["type"] != str] + del df_final_str["type"] + del df_final["type"] # ... now write the files. df_final.to_parquet(f"{output_file}.{idx+1}.parquet.gzip", compression="gzip") if not df_final_str.empty: - df_final_str.to_parquet(f"{output_file}_str.{idx+1}.parquet.gzip", compression="gzip") + df_final_str.to_parquet( + f"{output_file}_str.{idx+1}.parquet.gzip", compression="gzip" + ) # Causemosify-multi does not return the dataframe or dict. - click.echo('Done.') + click.echo("Done.") @cli.command() @@ -450,10 +498,10 @@ def download(): download_and_clean("admin3") -if __name__ == "__main__": - +if __name__ == "__main__": + cli() - ''' + """ # Testing if os.name == 'nt': sep = '\\' @@ -485,4 +533,4 @@ def download(): inputs = inputs + f"./tests/inputs{sep}test3_qualifies.csv\",\"mapper\": \"./tests/inputs{sep}test3_qualifies.json\"" + "}]" causemosify_multi(inputs, geo='admin2', output_file='testing') # geo='admin1', - ''' \ No newline at end of file + """ diff --git a/mixmasta/file_processor.py b/mixmasta/file_processor.py new file mode 100644 index 0000000..b9015bc --- /dev/null +++ b/mixmasta/file_processor.py @@ -0,0 +1,263 @@ +"""File processor functions that return dataframes when fed a file in order to enable downstream processing. + +Raises: + Exception: _description_ + Exception: _description_ + +Returns: + pandas.Dataframe: A dataframe of the input data to be used downstream +""" + +import logging + +import numpy +from osgeo import gdal, gdalconst +import pandas +import xarray + + +def process_file_by_filetype(filepath, file_type, transformation_metadata): + dataframe = None + if file_type == "geotiff": + dataframe = raster2df( + InRaster=filepath, + feature_name=transformation_metadata["feature_name"], + band=int( + transformation_metadata["band"] + if "band" in transformation_metadata + and transformation_metadata["band"] != "" + else "0" + ), + nodataval=int(transformation_metadata["null_val"]), + date=transformation_metadata["date"] + if ( + "date" in transformation_metadata + and transformation_metadata["date"] != "" + ) + else None, + band_name=transformation_metadata["band_name"], + bands=transformation_metadata["bands"] + if "bands" in transformation_metadata + else None, + band_type=transformation_metadata["band_type"] + if "band_type" in transformation_metadata + else "category", + ) + elif file_type == "excel": + dataframe = pandas.read_excel(filepath, transformation_metadata["sheet"]) + elif file_type != "csv": + dataframe = netcdf2df(filepath) + else: + dataframe = pandas.read_csv(filepath) + + if dataframe is None: + raise TypeError( + "File failed to process, dataframe returned as None type object" + ) + return dataframe + + +def raster2df( + InRaster: str, + feature_name: str = "feature", + band: int = 0, + nodataval: int = -9999, + date: str = None, + band_name: str = "feature2", + bands: dict = None, + band_type: str = "category", +) -> pandas.DataFrame: + """ + Description + ----------- + Takes the path of a raster (.tiff) file and produces a Geopandas Data Frame. + + Parameters + ---------- + InRaster: str + the path of the inumpyut raster file + feature_name: str + the name of the feature represented by the pixel values + band: int, default 1 + the band to operate on + nodataval: int, default -9999 + the value for no data pixels + date: str, default None + date associated with the raster (if any) + band_name: str, default feature2 + the name of the band data e.g. head_count, flooding + bands: dict, default None + passed in meta; dictionary of band identifiers and specifies bands to + be processed. + band_type: str, default category + Specifies band type e.g. category or datetime. If datetime, this data goes into the date column. + + Examples + -------- + Converting a geotiff of rainfall data into a geopandas dataframe + + >>> df = raster2df('path_to_raster.geotiff', 'rainfall', band=1) + + """ + # open the raster and get some properties + data_source = gdal.OpenShared(InRaster, gdalconst.GA_ReadOnly) + GeoTrans = data_source.GetGeoTransform() + ColRange = range(data_source.RasterXSize) + RowRange = range(data_source.RasterYSize) + + # Cache the dataframe and value data type. + dataframe = pandas.DataFrame() + row_data_type = None + + for x in range(1, data_source.RasterCount + 1): + # If band has a value, then limit import to the single specified band. + if band > 0 and band != x: + continue + + # If no bands in meta, then single-band and use band_name + # If bands, then process only those in the meta. + if not bands: + band_value = band_name + logging.info( + f"Single band detected. Bands: {bands}, band_name: {band_name}, feature_name: {feature_name}" + ) + elif str(x) in bands: + band_value = bands[str(x)] + logging.info( + f"Multi-band detected Bands: {bands}, band_name: {band_name}, feature_name: {feature_name}" + ) + elif str(x) not in bands: + # Processing a band not specified in the meta, so skip it + logging.info(f"Skipping band {x} since it is not specified in {bands}.") + continue + else: + raise Exception( + f"Neither single nor multiple bands specified in meta. Current band: {x}, Bands: {bands}, band_name: {band_name}, feature_name: {feature_name}" + ) + + # Create columns for the dataframe. + if not bands: + columns = ["longitude", "latitude", feature_name] + logging.info(f"Single band detected. Columns are: {columns}") + elif band_type == "datetime": + columns = ["longitude", "latitude", "date", feature_name] + logging.info(f"Datetime multiband detected. Columns are: {columns}") + elif band_type == "category": + # categorical multi-band; add columns during processing. + columns = ["longitude", "latitude", band_value] + logging.info(f"Categorical multiband detected. Columns are: {columns}") + else: + raise Exception( + f"During column processing, neither single nor multiple bands specified in meta. Bands: {bands}, band_name: {band_name}, feature_name: {feature_name}" + ) + + rBand = data_source.GetRasterBand(x) + nData = rBand.GetNoDataValue() + + if nData == None: + logging.warning(f"No nodataval found, setting to {nodataval}") + nData = numpy.float32(nodataval) # set it to something if not set + else: + logging.info(f"Nodataval is: {nData} type is : {type(nData)}") + + # specify the center offset (takes the point in middle of pixel) + HalfX = GeoTrans[1] / 2 + HalfY = GeoTrans[5] / 2 + + # Check that NoDataValue is of the same type as the raster data + RowData = rBand.ReadAsArray(0, 0, data_source.RasterXSize, 1)[0] + row_data_type = type(RowData[0]) + if type(nData) != row_data_type: + logging.info( + f"NoData type mismatch: NoDataValue is type {type(nData)} and raster data is type {row_data_type}" + ) + # e.g. NoDataValue is type and raster data is type + # Fix float type mismatches so comparison works below (row_value != nData) + if row_data_type == numpy.float32: + nData = numpy.float32(nData) + elif row_data_type == numpy.float64: + nData = numpy.float64(nData) + elif row_data_type == numpy.float16: + nData = numpy.float16(nData) + + points = [] + + for ThisRow in RowRange: + RowData = rBand.ReadAsArray(0, ThisRow, data_source.RasterXSize, 1)[0] + for ThisCol in ColRange: + # need to exclude NaN values since there is no nodataval + row_value = RowData[ThisCol] + + if (row_value > nData) and not (numpy.isnan(row_value)): + + # TODO: implement filters on valid pixels + # for example, the below would ensure pixel values are between -100 and 100 + # if (RowData[ThisCol] <= 100) and (RowData[ThisCol] >= -100): + + X = GeoTrans[0] + (ThisCol * GeoTrans[1]) + Y = GeoTrans[3] + ( + ThisRow * GeoTrans[5] + ) # Y is negative so it's a minus + # this gives the upper left of the cell, offset by half a cell to get centre + X += HalfX + Y += HalfY + + # Add the data row to the dataframe. + if bands == None: + points.append([X, Y, row_value]) + elif band_type == "datetime": + points.append([X, Y, band_value, row_value]) + else: + points.append([X, Y, row_value]) + + # This will make all floats float64, but will be optimized in process(). + new_dataframe = pandas.DataFrame(points, columns=columns) + + if dataframe.empty: + dataframe = new_dataframe + else: + # df = df.merge(new_df, left_on=["longitude", "latitude"], right_on=["longitude", "latitude"]) + if bands and band_type != "datetime": + # df.join(new_df, on=["longitude", "latitude"]) + dataframe = dataframe.merge( + new_dataframe, + left_on=["longitude", "latitude"], + right_on=["longitude", "latitude"], + ) + else: + dataframe = dataframe.append(new_dataframe) + + # Add the date from the mapper. + if date and band_type != "datetime": + dataframe["date"] = date + + dataframe.sort_values(by=columns, inplace=True) + + return dataframe + + +def netcdf2df(netcdf: str) -> pandas.DataFrame: + """ + Produce a dataframe from a NetCDF4 file. + + Parameters + ---------- + netcdf: str + Path to the netcdf file + + Returns + ------- + DataFrame + The resultant dataframe + """ + try: + data_source = xarray.open_dataset(netcdf) + except: + raise AssertionError( + f"Improperly formatted netCDF file ({netcdf}), xarray could not convert it to a dataframe." + ) + + dataframe = data_source.to_dataframe() + final_dataframe = dataframe.reset_index() + + return final_dataframe diff --git a/mixmasta/mixmasta.py b/mixmasta/mixmasta.py index feec582..634c535 100644 --- a/mixmasta/mixmasta.py +++ b/mixmasta/mixmasta.py @@ -6,7 +6,6 @@ import sys from datetime import datetime from typing import List -from distutils.util import strtobool import geofeather as gf import geopandas as gpd @@ -17,43 +16,47 @@ import xarray as xr from osgeo import gdal, gdalconst from shapely import speedups -from shapely.geometry import Point -from pathlib import Path -import pkg_resources import fuzzywuzzy from fuzzywuzzy import fuzz from fuzzywuzzy import process import timeit -#from .spacetag_schema import SpaceModel -import click +from .file_processor import process_file_by_filetype +from .normalizer import normalizer + +# from .spacetag_schema import SpaceModel +# TODO Remove this into another file. # Constants COL_ORDER = [ - "timestamp", - "country", - "admin1", - "admin2", - "admin3", - "lat", - "lng", - "feature", - "value", - ] + "timestamp", + "country", + "admin1", + "admin2", + "admin3", + "lat", + "lng", + "feature", + "value", +] + +import click GEO_TYPE_COUNTRY = "country" -GEO_TYPE_ADMIN1 = "state/territory" -GEO_TYPE_ADMIN2 = "county/district" -GEO_TYPE_ADMIN3 = "municipality/town" +GEO_TYPE_ADMIN1 = "state/territory" +GEO_TYPE_ADMIN2 = "county/district" +GEO_TYPE_ADMIN3 = "municipality/town" if not sys.warnoptions: import warnings + warnings.simplefilter("ignore") logger = logging.getLogger(__name__) + def audit_renamed_col_dict(dct: dict) -> dict: """ Description @@ -86,439 +89,13 @@ def audit_renamed_col_dict(dct: dict) -> dict: return dct -def build_date_qualifies_field(qualified_col_dict: dict, assoc_fields: list) -> str: - """ - Description - ----------- - Handle edge case of each date field in assoc_fields qualifying the same - column e.g. day/month/year are associated and qualify a field. In this - case, the new_column_name. - - if assoc_fields is found as a value in qualified_col_dict, return the key - - Parameters - ---------- - qualified_col_dict: dict - {'pop': ['month_column', 'day_column', 'year_column']} - - assoc_fields: list - ['month_column', 'day_column', 'year_column'] - - """ - for k, v in qualified_col_dict.items(): - if v == assoc_fields: - return k - - return None - -def format_time(t: str, time_format: str, validate: bool = True) -> int: - """ - Description - ----------- - Converts a time feature (t) into epoch time using `time_format` which is a strftime definition - - Parameters - ---------- - t: str - the time string - time_format: str - the strftime format for the string t - validate: bool, default True - whether to error check the time string t. Is set to False, then no error is raised if the date fails to parse, but None is returned. - - Examples - -------- - - >>> epoch = format_time('5/12/20 12:20', '%m/%d/%y %H:%M') - """ - - try: - t_ = int(datetime.strptime(t, time_format).timestamp()) * 1000 # Want milliseonds - return t_ - except Exception as e: - if t.endswith(' 00:00:00'): - # Depending on the date format, pandas.read_excel will read the - # date as a Timestamp, so here it is a str with format - # '2021-03-26 00:00:00'. For now, handle this single case until - # there is time for a more comprehensive solution e.g. add a custom - # date_parser function that doesn't parse diddly/squat to - # pandas.read_excel() in process(). - return format_time(t.replace(' 00:00:00', ''), time_format, validate) - print(e) - if validate: - raise Exception(e) - else: - return None -def geocode( - admin: str, df: pd.DataFrame, x: str = "longitude", y: str = "latitude", gadm: gpd.GeoDataFrame = None, - df_geocode: pd.DataFrame = pd.DataFrame() +def match_geo_names( + admin: str, + df: pd.DataFrame, + resolve_to_gadm_geotypes: list, + gadm: gpd.GeoDataFrame = None, ) -> pd.DataFrame: - """ - Description - ----------- - Takes a dataframe containing coordinate data and geocodes it to GADM (https://gadm.org/) - - GEOCODES to ADMIN 0, 1, 2 OR 3 LEVEL - - Parameters - ---------- - admin: str - the level to geocode to. 'admin0' to 'admin3' - df: pd.DataFrame - a pandas dataframe containing point data - x: str, default 'longitude' - the name of the column containing longitude information - y: str, default 'latitude' - the name of the column containing latitude data - gadm: gpd.GeoDataFrame, default None - optional specification of a GeoDataFrame of GADM shapes of the appropriate - level (admin2/3) for geocoding - df_geocode: pd.DataFrame, default pd.DataFrame() - cached lat/long geocode library - - Examples - -------- - Geocoding a dataframe with columns named 'lat' and 'lon' - - >>> df = geocode(df, x='lon', y='lat') - - """ - - flag = speedups.available - if flag == True: - speedups.enable() - - cdir = os.path.expanduser("~") - download_data_folder = f"{cdir}/mixmasta_data" - - # Only load GADM if it wasn't explicitly passed to the function. - if gadm is not None: - logging.info("GADM geo dataframe has been provided.") - else: - logging.info("GADM has not been provided; loading now.") - - if admin in ['admin0','country']: - gadm_fn = f"gadm36_2.feather" - gadmDir = f"{download_data_folder}/{gadm_fn}" - gadm = gf.from_geofeather(gadmDir) - gadm["country"] = gadm["NAME_0"] - gadm = gadm[["geometry", "country"]] - - elif admin == "admin1": - gadm_fn = f"gadm36_2.feather" - gadmDir = f"{download_data_folder}/{gadm_fn}" - gadm = gf.from_geofeather(gadmDir) - gadm["country"] = gadm["NAME_0"] - #gadm["state"] = gadm["NAME_1"] - gadm["admin1"] = gadm["NAME_1"] - #gadm = gadm[["geometry", "country", "state", "admin1"]] - gadm = gadm[["geometry", "country", "admin1"]] - - elif admin == "admin2": - gadm_fn = f"gadm36_2.feather" - gadmDir = f"{download_data_folder}/{gadm_fn}" - gadm = gf.from_geofeather(gadmDir) - gadm["country"] = gadm["NAME_0"] - #gadm["state"] = gadm["NAME_1"] - gadm["admin1"] = gadm["NAME_1"] - gadm["admin2"] = gadm["NAME_2"] - #gadm = gadm[["geometry", "country", "state", "admin1", "admin2"]] - gadm = gadm[["geometry", "country", "admin1", "admin2"]] - - elif admin == "admin3": - gadm_fn = f"gadm36_3.feather" - gadmDir = f"{download_data_folder}/{gadm_fn}" - gadm = gf.from_geofeather(gadmDir) - gadm["country"] = gadm["NAME_0"] - #gadm["state"] = gadm["NAME_1"] - gadm["admin1"] = gadm["NAME_1"] - gadm["admin2"] = gadm["NAME_2"] - gadm["admin3"] = gadm["NAME_3"] - #gadm = gadm[["geometry", "country", "state", "admin1", "admin2", "admin3"]] - gadm = gadm[["geometry", "country", "admin1", "admin2", "admin3"]] - - start_time = timeit.default_timer() - - # 1) Drop x,y duplicates from data frame. - df_drop_dup_geo = df[[x,y]].drop_duplicates(subset=[x,y]) - - # 2) Get x,y not in df_geocode. - if not df_geocode.empty and not df_drop_dup_geo.empty: - df_drop_dup_geo = df_drop_dup_geo.merge(df_geocode, on=[x,y], how='left', indicator=True) - df_drop_dup_geo = df_drop_dup_geo[ df_drop_dup_geo['_merge'] == 'left_only'] - df_drop_dup_geo = df_drop_dup_geo[[x,y]] - - if not df_drop_dup_geo.empty: - # dr_drop_dup_geo contains x,y not in df_geocode; so, these need to be - # geocoded and added to the df_geocode library. - - # 3) Apply Point() to create the geometry col. - df_drop_dup_geo.loc[:, "geometry"] = df_drop_dup_geo.apply(lambda row: Point(row[x], row[y]), axis=1) - - # 4) Sjoin unique geometries with GADM. - gdf = gpd.GeoDataFrame(df_drop_dup_geo) - - # Spatial merge on GADM to obtain admin areas. - gdf = gpd.sjoin(gdf, gadm, how="left", op="within", lsuffix="mixmasta_left", rsuffix="mixmasta_geocoded") - del gdf["geometry"] - del gdf["index_mixmasta_geocoded"] - - # 5) Add the new geocoding to the df_geocode lat/long geocode library. - if not df_geocode.empty: - df_geocode = df_geocode.append(gdf) - else: - df_geocode = gdf - - # 6) Merge df and df_geocode on x,y - gdf = df.merge(df_geocode, how='left', on=[x,y]) - - return pd.DataFrame(gdf), df_geocode - -def generate_column_name(field_list: list) -> str: - """ - Description - ----------- - Contatenate a list of column fields into a single column name. - - Parameters - ---------- - field_list: list[str] of column names - - Returns - ------- - str: new column name - - """ - return ''.join(sorted(field_list)) - -def generate_timestamp_column(df: pd.DataFrame, date_mapper: dict, column_name: str) -> pd.DataFrame: - """ - Description - ----------- - Efficiently add a new timestamp column to a dataframe. It avoids the use of df.apply - which appears to be much slower for large dataframes. Defaults to 1/1/1970 for - missing day/month/year values. - - Parameters - ---------- - df: pd.DataFrame - our data - date_mapper: dict - a schema mapping (JSON) for the dataframe filtered for "date_type" equal to - Day, Month, or Year. The format is screwy for our purposes here and could - be reafactored. - column_name: str - name of the new column e.g. timestamp for primary_time, year1month1day1 - for a concatneated name from associated date fields. - - Examples - -------- - This example adds the generated series to the source dataframe. - >>> df = df.join(df.apply(generate_timestamp, date_mapper=date_mapper, - column_name="year1month1day", axis=1)) - """ - - # Identify which date values are passed. - dayCol = None - monthCol = None - yearCol = None - - for kk, vv in date_mapper.items(): - if vv and vv["date_type"] == "day": - dayCol = kk - elif vv and vv["date_type"] == "month": - monthCol = kk - elif vv and vv["date_type"] == "year": - yearCol = kk - - # For missing date values, add a column to the dataframe with the default - # value, then assign that to the day/month/year var. If the dataframe has - # the date value, assign day/month/year to it after casting as a str. - if dayCol: - day = df[dayCol].astype(str) - else: - df.loc[:, 'day_generate_timestamp_column'] = "1" - day = df['day_generate_timestamp_column'] - - if monthCol: - month = df[monthCol].astype(str) - else: - df.loc[:, 'month_generate_timestamp_column'] = "1" - month = df['month_generate_timestamp_column'] - - if yearCol: - year = df[yearCol].astype(str) - else: - df.loc[:, 'year_generate_timestamp_column'] = "01" - year = df['year_generate_timestamp_column'] - - # Add the new column - df.loc[:, column_name] = month + '/' + day + '/' + year - - # Delete the temporary columns - if not dayCol: - del(df['day_generate_timestamp_column']) - - if not monthCol: - del(df['month_generate_timestamp_column']) - - if not yearCol: - del(df['year_generate_timestamp_column']) - - return df - -def generate_timestamp_format(date_mapper: dict) -> str: - """ - Description - ----------- - Generates a the time format for day,month,year dates based on each's - specified time_format. - - Parameters - ---------- - date_mapper: dict - a dictionary for the schema mapping (JSON) for the dataframe filtered - for "date_type" equal to Day, Month, or Year. - - Output - ------ - e.g. "%m/%d/%Y" - """ - - day = "%d" - month = "%m" - year = "%y" - - for kk, vv in date_mapper.items(): - if vv["date_type"] == "day": - day = vv["time_format"] - elif vv["date_type"] == "month": - month = vv["time_format"] - elif vv["date_type"] == "year": - year = vv["time_format"] - - return str.format("{}/{}/{}", month, day, year) - -def get_iso_country_dict(iso_list: list) -> dict: - """ - Description - ----------- - iso2 or iso3 is used as primary_geo and therefore the country column. - Load the custom iso lookup table and return a dictionary of the iso codes - as keys and the country names as values. Assume all list items are the same - iso type. - - Parameters - ---------- - iso_list: - list of iso2 or iso3 codes - - Returns - ------- - dict: - key: iso code; value: country name - """ - - dct = {} - if iso_list: - iso_df = pd.DataFrame - try: - # The necessary code to load from pkg doesn't currently work in VS - # Code Debug, so wrap in try/except. - #iso_df = pd.read_csv(pkg_resources.resource_stream(__name__, 'data/iso_lookup.csv')) - with pkg_resources.resource_stream(__name__, 'data/iso_lookup.csv') as f: - iso_df = pd.read_csv(f) - #path = Path(__file__).parent / "data/iso_lookup.csv" - #iso_df = pd.read_csv(path) - except: - # Local VS Code load. - path = Path(__file__).parent / "data/iso_lookup.csv" - iso_df = pd.read_csv(path) - - if iso_df.empty: - return dct - - if len(iso_list[0]) == 2: - for iso in iso_list: - if iso in iso_df["iso2"].values: - dct[iso] = iso_df.loc[iso_df["iso2"] == iso]["country"].item() - else: - for iso in iso_list: - if iso in iso_df["iso3"].values: - dct[iso] = iso_df.loc[iso_df["iso3"] == iso]["country"].item() - - return dct - -def handle_colname_collisions(df: pd.DataFrame, mapper: dict, protected_cols: list) -> (pd.DataFrame, dict, dict): - """ - Description - ----------- - Identify mapper columns that match protected column names. When found, - update the mapper and dataframe, and keep a dict of these changes - to return to the caller e.g. SpaceTag. - - Parameters - ---------- - df: pd.DataFrame - submitted data - mapper: dict - a dictionary for the schema mapping (JSON) for the dataframe. - protected_cols: list - protected column names i.e. timestamp, country, admin1, feature, etc. - - Output - ------ - pd.DataFame: - The modified dataframe. - dict: - The modified mapper. - dict: - key: new column name e.g. "day1month1year1" or "country_non_primary" - value: list of old column names e.g. ['day1','month1','year1'] or ['country'] - """ - - # Get names of geo fields that collide and are not primary_geo = True - non_primary_geo_cols = [d["name"] for d in mapper["geo"] if d["name"] in protected_cols and ("primary_geo" not in d or d["primary_geo"] == False)] - - # Get names of date fields that collide and are not primary_date = True - non_primary_time_cols = [d['name'] for d in mapper['date'] if d["name"] in protected_cols and ('primary_date' not in d or d['primary_date'] == False)] - - # Only need to change a feature column name if it qualifies another field, - # and therefore will be appended as a column to the output. - feature_cols = [d["name"] for d in mapper['feature'] if d["name"] in protected_cols and "qualifies" in d and d["qualifies"]] - - # Verbose build of the collision_list, could have combined above. - collision_list = non_primary_geo_cols + non_primary_time_cols + feature_cols - - # Bail if no column name collisions. - if not collision_list: - return df, mapper, {} - - # Append any collision columns with the following suffix. - suffix = "_non_primary" - - # Build output dictionary and update df. - renamed_col_dict = {} - for col in collision_list: - df.rename(columns={col: col + suffix}, inplace=True) - renamed_col_dict[col + suffix] = [col] - - # Update mapper - for k, vlist in mapper.items(): - for dct in vlist: - if dct["name"] in collision_list: - dct["name"] = dct["name"] + suffix - elif "qualifies" in dct and dct["qualifies"]: - # change any instances of this column name qualified by another field - dct["qualifies"] = [w.replace(w, w + suffix) if w in collision_list else w for w in dct["qualifies"] ] - elif "associated_columns" in dct and dct["associated_columns"]: - # change any instances of this column name in an associated_columns dict - dct["associated_columns"] = {k: v.replace(v, v + suffix) if v in collision_list else v for k, v in dct["associated_columns"].items() } - - return df, mapper, renamed_col_dict - -def match_geo_names(admin: str, df: pd.DataFrame, resolve_to_gadm_geotypes: list, gadm: gpd.GeoDataFrame = None) -> pd.DataFrame: """ Assumption ---------- @@ -542,7 +119,7 @@ def match_geo_names(admin: str, df: pd.DataFrame, resolve_to_gadm_geotypes: list A pandas.Dataframe produced by modifying the parameter df. """ - print('geocoding ...') + print("geocoding ...") flag = speedups.available if flag == True: speedups.enable() @@ -552,7 +129,7 @@ def match_geo_names(admin: str, df: pd.DataFrame, resolve_to_gadm_geotypes: list # only load GADM if it wasn't explicitly passed to the function. if gadm is not None: - #logging.info("GADM geo dataframe has been provided.") + # logging.info("GADM geo dataframe has been provided.") pass else: logging.info("GADM has not been provided; loading now.") @@ -566,9 +143,9 @@ def match_geo_names(admin: str, df: pd.DataFrame, resolve_to_gadm_geotypes: list gadm = gf.from_geofeather(gadmDir) gadm["country"] = gadm["NAME_0"] - gadm["state"] = gadm["NAME_1"] - gadm["admin1"] = gadm["NAME_1"] - gadm["admin2"] = gadm["NAME_2"] + gadm["state"] = gadm["NAME_1"] + gadm["admin1"] = gadm["NAME_1"] + gadm["admin2"] = gadm["NAME_2"] if admin == "admin2": gadm = gadm[["country", "state", "admin1", "admin2"]] @@ -585,12 +162,14 @@ def match_geo_names(admin: str, df: pd.DataFrame, resolve_to_gadm_geotypes: list unknowns = df[~df.country.isin(gadm_country_list)].country.tolist() for unk in unknowns: try: - match = fuzzywuzzy.process.extractOne(unk, gadm_country_list, scorer=fuzz.partial_ratio) + match = fuzzywuzzy.process.extractOne( + unk, gadm_country_list, scorer=fuzz.partial_ratio + ) except Exception as e: match = None logging.error(f"Error in match_geo_names: {e}") if match != None: - df.loc[df.country == unk, 'country'] = match[0] + df.loc[df.country == unk, "country"] = match[0] # Filter GADM dicitonary for only those countries (ie. speed up) gadm = gadm[gadm["country"].isin(countries)] @@ -602,571 +181,57 @@ def match_geo_names(admin: str, df: pd.DataFrame, resolve_to_gadm_geotypes: list # Get list of admin1 values in df but not in gadm. Reduce list for country. if GEO_TYPE_ADMIN1 in resolve_to_gadm_geotypes: - admin1_list = gadm[gadm.country==c]["admin1"].unique() - if admin1_list is not None and all(admin1_list) and 'admin1' in df: - unknowns = df[(df.country == c) & ~df.admin1.isin(admin1_list)].admin1.tolist() - unknowns = [x for x in unknowns if pd.notnull(x) and x.strip()] # remove Nan + admin1_list = gadm[gadm.country == c]["admin1"].unique() + if admin1_list is not None and all(admin1_list) and "admin1" in df: + unknowns = df[ + (df.country == c) & ~df.admin1.isin(admin1_list) + ].admin1.tolist() + unknowns = [ + x for x in unknowns if pd.notnull(x) and x.strip() + ] # remove Nan for unk in unknowns: - match = fuzzywuzzy.process.extractOne(unk, admin1_list, scorer=fuzz.partial_ratio) + match = fuzzywuzzy.process.extractOne( + unk, admin1_list, scorer=fuzz.partial_ratio + ) if match != None: - df.loc[df.admin1 == unk, 'admin1'] = match[0] + df.loc[df.admin1 == unk, "admin1"] = match[0] # Get list of admin2 values in df but not in gadm. Reduce list for country. if GEO_TYPE_ADMIN2 in resolve_to_gadm_geotypes: - admin2_list = gadm[gadm.country==c ]["admin2"].unique() - if admin2_list is not None and all(admin2_list) and 'admin2' in df: - unknowns = df[(df.country == c) & ~df.admin2.isin(admin2_list)].admin2.tolist() - unknowns = [x for x in unknowns if pd.notnull(x) and x.strip()] # remove Nan + admin2_list = gadm[gadm.country == c]["admin2"].unique() + if admin2_list is not None and all(admin2_list) and "admin2" in df: + unknowns = df[ + (df.country == c) & ~df.admin2.isin(admin2_list) + ].admin2.tolist() + unknowns = [ + x for x in unknowns if pd.notnull(x) and x.strip() + ] # remove Nan for unk in unknowns: - match = fuzzywuzzy.process.extractOne(unk, admin2_list, scorer=fuzz.partial_ratio) + match = fuzzywuzzy.process.extractOne( + unk, admin2_list, scorer=fuzz.partial_ratio + ) if match != None: - df.loc[df.admin2 == unk, 'admin2'] = match[0] + df.loc[df.admin2 == unk, "admin2"] = match[0] - if admin =='admin3' and GEO_TYPE_ADMIN3 in resolve_to_gadm_geotypes: + if admin == "admin3" and GEO_TYPE_ADMIN3 in resolve_to_gadm_geotypes: # Get list of admin3 values in df but not in gadm. Reduce list for country. - admin3_list = gadm[gadm.country==c]["admin3"].unique() - if admin3_list is not None and all(admin3_list) and 'admin3' in df: - unknowns = df[(df.country == c) & ~df.admin3.isin(admin3_list)].admin3.tolist() - unknowns = [x for x in unknowns if pd.notnull(x) and x.strip()] # remove Nan + admin3_list = gadm[gadm.country == c]["admin3"].unique() + if admin3_list is not None and all(admin3_list) and "admin3" in df: + unknowns = df[ + (df.country == c) & ~df.admin3.isin(admin3_list) + ].admin3.tolist() + unknowns = [ + x for x in unknowns if pd.notnull(x) and x.strip() + ] # remove Nan for unk in unknowns: - match = fuzzywuzzy.process.extractOne(unk, admin3_list, scorer=fuzz.partial_ratio) + match = fuzzywuzzy.process.extractOne( + unk, admin3_list, scorer=fuzz.partial_ratio + ) if match != None: - df.loc[df.admin3 == unk, 'admin3'] = match[0] + df.loc[df.admin3 == unk, "admin3"] = match[0] return df -def netcdf2df(netcdf: str) -> pd.DataFrame: - """ - Produce a dataframe from a NetCDF4 file. - - Parameters - ---------- - netcdf: str - Path to the netcdf file - - Returns - ------- - DataFrame - The resultant dataframe - """ - try: - ds = xr.open_dataset(netcdf) - except: - raise AssertionError(f"improperly formatted netCDF file ({netcdf})") - - data = ds.to_dataframe() - df = data.reset_index() - - return df - -def normalizer(df: pd.DataFrame, mapper: dict, admin: str, gadm: gpd.GeoDataFrame = None, df_geocode: pd.DataFrame = pd.DataFrame()) -> (pd.DataFrame, dict, pd.DataFrame): - """ - Description - ----------- - Converts a dataframe into a CauseMos compliant format. - - Parameters - ---------- - df: pd.DataFrame - a pandas dataframe containing point data - mapper: dict - a schema mapping (JSON) for the dataframe - a dict where keys will be geo, feaure, date, and values will be lists of dict - example: - { 'geo': [ - {'name': 'country', 'type': 'geo', 'geo_type': 'country', 'primary_geo': False}, - {'name': 'state', 'type': 'geo', 'geo_type': 'state/territory', 'primary_geo': False} - ], - 'feature': [ - {'name': 'probabilty', 'type': 'feature', 'feature_type': 'float'}, - {'name': 'color', 'type': 'feature', 'feature_type': 'str'} - ], - 'date': [ - {'name': 'date_2', 'type': 'date', 'date_type': 'date', 'primary_date': False, 'time_format': '%m/%d/%y'}, - {'name': 'date', 'type': 'date', 'date_type': 'date', 'primary_date': True, 'time_format': '%m/%d/%y'} - ] - } - admin: str, default 'admin2' - the level to geocode to. Either 'admin2' or 'admin3' - gadm: gpd.GeoDataFrame, default None - optional specification of a GeoDataFrame of GADM shapes of the appropriate - level (admin2/3) for geocoding - df_gecode: pd.DataFrame, default pd.DataFrame() - lat,long geocode lookup library - - Returns - ------- - pd.DataFrame: CauseMos compliant format ready to be written to parquet. - dict: dictionary of modified column names; used by SpaceTag - pd.DataFRame: update lat,long geocode looup library - - Examples - -------- - >>> df_norm = normalizer(df, mapper, 'admin3') - """ - col_order = COL_ORDER.copy() - - required_cols = [ - "timestamp", - "country", - "admin1", - "admin2", - "admin3", - "lat", - "lng", - ] - - # List of date_types that be used to build a date. - MONTH_DAY_YEAR = ["day","month","year"] - - # Create a dictionary of list: colnames: new col name, and modify df and - # mapper for any column name collisions. - df, mapper, renamed_col_dict = handle_colname_collisions(df, mapper, col_order) - - ### mapper is a dictionary of lists of dictionaries. - click.echo("Raw dataframe:") - click.echo(df.head()) - - # list of names of datetime columns primary_date=True - primary_time_cols = [k['name'] for k in mapper['date'] if 'primary_date' in k and k['primary_date'] == True] - - # list of names of datetime columns no primary_date or primary_date = False - other_time_cols = [k['name'] for k in mapper['date'] if 'primary_date' not in k or k['primary_date'] == False] - - # list of names of geo columns primary_geo=True - primary_geo_cols = [k["name"] for k in mapper["geo"] if "primary_geo" in k and k["primary_geo"] == True] - - # list of geotypes of geo columns primary_geo=True (used for match_geo_names logic below) - primary_geo_types = [k["geo_type"] for k in mapper["geo"] if "primary_geo" in k and k["primary_geo"] == True] - - # qualified_col_dict: dictionary for columns qualified by another column. - # key: qualified column - # value: list of columns that qualify key column - qualified_col_dict = {} - - # subset dataframe for only columns specified in mapper schema. - # get all named objects in the date, feature, geo schema lists. - mapper_keys = [] - for k in mapper.items(): - mapper_keys.extend([l['name'] for l in k[1] if 'name' in l]) - - df = df[mapper_keys] - - # Rename protected columns - # and perform type conversion on the time column - features = [] - primary_date_group_mapper = {} - other_date_group_mapper = {} - - for date_dict in mapper["date"]: - kk = date_dict["name"] - if kk in primary_time_cols: - # There should only be a single epoch or date field, or a single - # group of year/month/day/minute/second marked as primary_time in - # the loaded schema. - if date_dict["date_type"] == "date": - # convert primary_time of date_type date to epochtime and rename as 'timestamp' - df.loc[:, kk] = df[kk].apply(lambda x: format_time(str(x), date_dict["time_format"], validate=False)) - staple_col_name = "timestamp" - df.rename(columns={kk: staple_col_name}, inplace=True) - # renamed_col_dict[ staple_col_name ] = [kk] # 7/2/2021 do not include primary cols - elif date_dict["date_type"] == "epoch": - # rename epoch time column as 'timestamp' - staple_col_name = "timestamp" - df.rename(columns={kk: staple_col_name}, inplace=True) - #renamed_col_dict[ staple_col_name ] = [kk] # 7/2/2021 do not include primary cols - elif date_dict["date_type"] in ["day","month","year"]: - primary_date_group_mapper[kk] = date_dict - - else: - if date_dict["date_type"] == "date": - # Convert all date/time to epoch time if not already. - df.loc[:, kk] = df[kk].apply(lambda x: format_time(str(x), date_dict["time_format"], validate=False)) - # If three are no assigned primary_time columns, make this the - # primary_time timestamp column, and keep as a feature so the - # column_name meaning is not lost. - if not primary_time_cols and not "timestamp" in df.columns: - df.rename(columns={kk: "timestamp"}, inplace=True) - staple_col_name ="timestamp" - renamed_col_dict[ staple_col_name ] = [kk] - # All not primary_time, not associated_columns fields are pushed to features. - features.append(kk) - - elif date_dict["date_type"] in MONTH_DAY_YEAR and 'associated_columns' in date_dict and date_dict["associated_columns"]: - # Various date columns have been associated by the user and are not primary_date. - # convert them to epoch then store them as a feature - # (instead of storing them as separate uncombined features). - # handle this dict after iterating all date fields - other_date_group_mapper[kk] = date_dict - - else: - features.append(kk) - - if "qualifies" in date_dict and date_dict["qualifies"]: - # Note that any "qualifier" column that is not primary geo/date - # will just be lopped on to the right as its own column. It's - # column name will just be the name and Uncharted will deal with - # it. The key takeaway is that qualifier columns grow the width, - # not the length of the dataset. - # Want to add the qualified col as the dictionary key. - # e.g. "name": "region", "qualifies": ["probability", "color"] - # should produce two dict entries for prob and color, with region - # in a list as the value for both. - for k in date_dict["qualifies"]: - if k in qualified_col_dict: - qualified_col_dict[k].append(kk) - else: - qualified_col_dict[k] = [kk] - - if primary_date_group_mapper: - # Applied when there were primary_date year,month,day fields above. - # These need to be combined - # into a date and then epoch time, and added as the timestamp field. - - # Create a separate df of the associated date fields. This avoids - # pandas upcasting the series dtypes on df.apply(); e.g., int to float, - # or a month 9 to 9.0, which breaks generate_timestamp() - assoc_fields = primary_date_group_mapper.keys() - date_df = df[ assoc_fields ] - - # Now generate the timestamp from date_df and add timestamp col to df. - df = generate_timestamp_column(df, primary_date_group_mapper, "timestamp") - - # Determine the correct time format for the new date column, and - # convert to epoch time. - time_formatter = generate_timestamp_format(primary_date_group_mapper) - df['timestamp'] = df["timestamp"].apply(lambda x: format_time(str(x), time_formatter, validate=False)) - - # Let SpaceTag know those date columns were renamed to timestamp. - #renamed_col_dict[ "timestamp" ] = assoc_fields # 7/2/2021 do not include primary cols - - while other_date_group_mapper: - # Various date columns have been associated by the user and are not primary_date. - # Convert to epoch time and store as a feature, do not store these separately in features. - # Exception is the group is only two of day, month, year: leave as date. - # Control for possibility of more than one set of assciated_columns. - - # Pop the first item in the mapper and begin building that date set. - date_field_tuple = other_date_group_mapper.popitem() - - # Build a list of column names associated with the the popped date field. - assoc_fields = [k[1] for k in date_field_tuple[1]['associated_columns'].items()] - - # Pop those mapper objects into a dict based on the column name keys in - # assocfields list. - assoc_columns_dict = { f : other_date_group_mapper.pop(f) for f in assoc_fields if f in other_date_group_mapper } - - # Add the first popped tuple into the assoc_columns dict where the key is the - # first part of the tuple; the value is the 2nd part. - assoc_columns_dict[date_field_tuple[0]] = date_field_tuple[1] - - # Add the first popped tuple column name to the list of associated fields. - assoc_fields.append(date_field_tuple[0]) - - # TODO: If day and year are associated to each other and month, but - # month is not associated to those fields, then at this point assoc_fields - # will be the three values, and assoc_columns will contain only day and - # year. This will error out below. It is assumed that SpaceTag will - # control for this instance. - - # If there is no primary_time column for timestamp, which would have - # been created above with primary_date_group_mapper, or farther above - # looping mapper["date"], attempt to generate from date_type = Month, - # Day, Year features. Otherwise, create a new column name from the - # concatenation of the associated date fields here. - if not "timestamp" in df.columns: - new_column_name = "timestamp" - else: - new_column_name = generate_column_name(assoc_fields) - - # Create a separate df of the associated date fields. This avoids - # pandas upcasting the series dtypes on df.apply(); e.g., int to float, - # or a month 9 to 9.0, which breaks generate_timestamp() - date_df = df[ assoc_fields ] - - # Now generate the timestamp from date_df and add timestamp col to df. - df = generate_timestamp_column(df, assoc_columns_dict, new_column_name) - - # Determine the correct time format for the new date column, and - # convert to epoch time only if all three date components (day, month, - # year) are present; otherwise leave as a date string. - date_types = [v["date_type"] for k,v in assoc_columns_dict.items()] - if len(frozenset(date_types).intersection(MONTH_DAY_YEAR)) == 3: - time_formatter = generate_timestamp_format(assoc_columns_dict) - df.loc[:, new_column_name] = df[new_column_name].apply(lambda x: format_time(str(x), time_formatter, validate=False)) - - # Let SpaceTag know those date columns were renamed to a new column. - renamed_col_dict[ new_column_name] = assoc_fields - - # timestamp is a protected column, so don't add to features. - if new_column_name != "timestamp": - # Handle edge case of each date field in assoc_fields qualifying - # the same column e.g. day/month/year are associated and qualify - # a field. In this case, the new_column_name - qualified_col = build_date_qualifies_field(qualified_col_dict, assoc_fields) - if qualified_col is None: - features.append(new_column_name) - else: - qualified_col_dict[qualified_col] = [new_column_name] - - for geo_dict in mapper["geo"]: - kk = geo_dict["name"] - if kk in primary_geo_cols: - if geo_dict["geo_type"] == "latitude": - staple_col_name = "lat" - df.rename(columns={kk: staple_col_name}, inplace=True) - #renamed_col_dict[staple_col_name] = [kk] # 7/2/2021 do not include primary cols - elif geo_dict["geo_type"] == "longitude": - staple_col_name = "lng" - df.rename(columns={kk: staple_col_name}, inplace=True) - #renamed_col_dict[staple_col_name] = [kk] # 7/2/2021 do not include primary cols - elif geo_dict["geo_type"] == "coordinates": - c_f = geo_dict["coord_format"] - coords = df[kk].values - if c_f == "lonlat": - lats = [x for x in coords.split(",")[1]] - longs = [x for x in coords.split(",")[0]] - else: - lats = [x for x in coords.split(",")[0]] - longs = [x for x in coords.split(",")[1]] - df["lng"] = longs - df["lat"] = lats - del df[kk] - elif geo_dict["geo_type"] == GEO_TYPE_COUNTRY and kk != "country": - # force the country column to be named country - staple_col_name = "country" - df.rename(columns={kk: staple_col_name}, inplace=True) - #renamed_col_dict[staple_col_name] = [kk] # 7/2/2021 do not include primary cols - elif geo_dict["geo_type"] == GEO_TYPE_ADMIN1 and kk != "admin1": - # force the country column to be named country - staple_col_name = "admin1" - df.rename(columns={kk: staple_col_name}, inplace=True) - elif geo_dict["geo_type"] == GEO_TYPE_ADMIN2 and kk != "admin2": - # force the country column to be named country - staple_col_name = "admin2" - df.rename(columns={kk: staple_col_name}, inplace=True) - elif geo_dict["geo_type"] == GEO_TYPE_ADMIN3 and kk != "admin2": - # force the country column to be named country - staple_col_name = "admin3" - df.rename(columns={kk: staple_col_name}, inplace=True) - - elif str(geo_dict["geo_type"]).lower() in ["iso2", "iso3"]: - # use the ISO2 or ISO3 column as country - - # use ISO2/3 lookup dictionary to change ISO to country name. - iso_list = df[kk].unique().tolist() - dct = get_iso_country_dict(iso_list) - df.loc[:, kk] = df[kk].apply(lambda x: dct[x] if x in dct else x) - - # now rename that column as "country" - staple_col_name = "country" - df.rename(columns={kk: staple_col_name}, inplace=True) - #renamed_col_dict[staple_col_name] = [kk] # 7/2/2021 do not include primary cols - - elif "qualifies" in geo_dict and geo_dict["qualifies"]: - # Note that any "qualifier" column that is not primary geo/date - # will just be lopped on to the right as its own column. It'’'s - # column name will just be the name and Uncharted will deal with - # it. The key takeaway is that qualifier columns grow the width, - # not the length of the dataset. - # Want to add the qualified col as the dictionary key. - # e.g. "name": "region", "qualifies": ["probability", "color"] - # should produce two dict entries for prob and color, with region - # in a list as the value for both. - for k in geo_dict["qualifies"]: - if k in qualified_col_dict: - qualified_col_dict[k].append(kk) - else: - qualified_col_dict[k] = [kk] - else: - # only push geo columns to the named columns - # in the event there is no primary geo - # otherwise they are features and we geocode lat/lng - if len(primary_geo_cols) == 0: - if geo_dict["geo_type"] == GEO_TYPE_COUNTRY: - df["country"] = df[kk] - renamed_col_dict["country"] = [kk] - continue - if geo_dict["geo_type"] == GEO_TYPE_ADMIN1: - df["admin1"] = df[kk] - renamed_col_dict["admin1"] = [kk] - continue - if geo_dict["geo_type"] == GEO_TYPE_ADMIN2: - df["admin2"] = df[kk] - renamed_col_dict["admin2"] = [kk] - continue - if geo_dict["geo_type"] == GEO_TYPE_ADMIN3: - df["admin3"] = df[kk] - renamed_col_dict["admin3"] = [kk] - continue - features.append(kk) - - # Append columns annotated in feature dict to features list (if not a - # qualifies column) - #features.extend([k["name"] for k in mapper["feature"]]) - for feature_dict in mapper["feature"]: - if "qualifies" not in feature_dict or not feature_dict["qualifies"]: - features.append(feature_dict["name"]) - elif "qualifies" in feature_dict and feature_dict["qualifies"]: - # Note that any "qualifier" column that is not primary geo/date - # will just be lopped on to the right as its own column. It's - # column name will just be the name and Uncharted will deal with - # it. The key takeaway is that qualifier columns grow the width, - # not the length of the dataset. - # Want to add the qualified col as the dictionary key. - # e.g. "name": "region", "qualifies": ["probability", "color"] - # should produce two dict entries for prob and color, with region - # in a list as the value for both. - for k in feature_dict["qualifies"]: - kk = feature_dict["name"] - if k in qualified_col_dict: - qualified_col_dict[k].append(kk) - else: - qualified_col_dict[k] = [kk] - - # Convert aliases based on user annotations - aliases = feature_dict.get("aliases", {}) - if aliases: - click.echo(f"Pre-processed aliases are: {aliases}") - type_ = df[feature_dict["name"]].dtype.type - click.echo(f"Detected column type is: {type_}") - aliases_ = {} - # The goal below is to identify the data type and then to cast the - # alias key from string into that type so that it will match - # if that fails, just cast it as a string - for kk, vv in aliases.items(): - try: - if issubclass(type_, (int, np.integer)): - click.echo("Aliasing: integer detected") - aliases_[int(kk)] = vv - elif issubclass(type_, (float, np.float16, np.float32, np.float64, np.float128)): - click.echo("Aliasing: float detected") - aliases_[float(kk)] = vv - elif issubclass(type_, (bool, np.bool, np.bool_)): - click.echo("Aliasing: boolean detected") - if strtobool(kk) == 1: - aliases_[True] = vv - click.echo("Converted true string to boolean") - else: - click.echo("Converted false string to boolean") - aliases_[False] = vv - # Fall back on string - else: - click.echo("Aliasing: string detected") - aliases_[kk] = vv - except ValueError as e: - # Fall back on string - click.echo(f"Error: {e}") - aliases_[kk] = vv - click.echo(f"Aliases for {feature_dict['name']} are {aliases_}.") - df[[feature_dict["name"]]] = df[[feature_dict["name"]]].replace(aliases_) - - # Since the user has decided to apply categorical aliases to this feature, we must coerce - # the entire feature to a string, even if they did not alias every value within the feature - # the reason for this is to avoid mixed types within the feature (e.g. half int/half string) - # since this makes it difficult to visualize - df[[feature_dict["name"]]] = df[[feature_dict["name"]]].astype(str) - - # perform geocoding if lat/lng are present - if "lat" in df and "lng" in df: - df, df_geocode = geocode(admin, df, x="lng", y="lat", gadm=gadm, df_geocode=df_geocode) - elif "country" in primary_geo_types or ("country" in df and not primary_geo_types): - # Correct any misspellings etc. in state and admin areas when not - # geocoding lat and lng above, and country is the primary_geo. - # This doesn't match names if iso2/iso3 are primary, and when country - # admin1-3 are moved to features. Exception is when country is present, - # but nothing is marked as primary. - - # Only geo_code resolve_to_gadm = True fields. - # Used below when match_geocode_names - resolve_to_gadm_geotypes = [k["geo_type"] for k in mapper["geo"] if "resolve_to_gadm" in k and k["resolve_to_gadm"] == True] - if resolve_to_gadm_geotypes: - df = match_geo_names(admin, df, resolve_to_gadm_geotypes) - - df_geo_cols = [i for i in df.columns if 'mixmasta_geocoded' in i] - for c in df_geo_cols: - df.rename(columns={c: c.replace('_mixmasta_geocoded','')}, inplace=True) - - # protected_cols are the required_cols present in the submitted dataframe. - protected_cols = list(set(required_cols) & set(df.columns)) - - # if a field qualifies a protected field like country, it should have data - # in each row, unlike features below where the qualifying data appears - # only on those rows. - # k: qualified column (str) - # v: list of columns (str) that qualify k - for k,v in qualified_col_dict.items(): - if k in protected_cols: - # k is qualified by the columns in v, and k is a protected column, - # so extend the width of the output dataset with v for each row. - protected_cols.extend(v) - col_order.extend(v) - - # Prepare output by - # 1. if there are no features, simply reduce the dataframe. - # or, 2.iterating features to add to feature adn value columns. - if not features: - df_out = df[protected_cols] - else: - df_out = pd.DataFrame() - for feat in features: - using_cols = protected_cols.copy() - - if feat in qualified_col_dict: - # dict value is a list, so extend. - using_cols.extend(qualified_col_dict[feat]) - - # add a qualifying column name only if not in col_order already - for c in qualified_col_dict[feat]: - if c not in col_order: - col_order.append(c) - - join_overlap = False - try: - df_ = df[using_cols + [feat+'_mixmasta_left']].copy() - join_overlap = True - except: - df_ = df[using_cols + [feat]].copy() - - try: - if mapper[feat]["new_col_name"] == None: - df_["feature"] = feat - else: - df_["feature"] = mapper[feat]["new_col_name"] - except: - df_["feature"] = feat - - if join_overlap: - df_.rename(columns={f"{feat}_mixmasta_left": "value"}, inplace=True) - else: - df_.rename(columns={feat: "value"}, inplace=True) - - # Add feature/value for epochtime as object adds it without decimal - # places, but it is still saved as a double in the parquet file. - if len(df_out) == 0: - if feat in other_time_cols: - df_out = df_.astype({'value': object}) - else: - df_out = df_ - else: - if feat in other_time_cols: - df_out = df_out.append(df_.astype({'value': object})) - else: - df_out = df_out.append(df_) - - for c in col_order: - if c not in df_out: - df_out[c] = None - - # Drop rows with nulls in value column. - df_out.dropna(axis=0, subset=['value'], inplace=True) - - # Handle any renamed cols being renamed. - renamed_col_dict = audit_renamed_col_dict(renamed_col_dict) - - click.echo("Processed dataframe:") - click.echo(df_out.head()) - return df_out[col_order], renamed_col_dict, df_geocode def optimize_df_types(df: pd.DataFrame): """ @@ -1177,13 +242,13 @@ def optimize_df_types(df: pd.DataFrame): For very large dataframes the memory reduction should translate into increased efficieny. """ - floats = df.select_dtypes(include=['float64']).columns.tolist() - df[floats] = df[floats].apply(pd.to_numeric, downcast='float') + floats = df.select_dtypes(include=["float64"]).columns.tolist() + df[floats] = df[floats].apply(pd.to_numeric, downcast="float") - ints = df.select_dtypes(include=['int64']).columns.tolist() - df[ints] = df[ints].apply(pd.to_numeric, downcast='integer') + ints = df.select_dtypes(include=["int64"]).columns.tolist() + df[ints] = df[ints].apply(pd.to_numeric, downcast="integer") - #for col in df.select_dtypes(include=['object']): + # for col in df.select_dtypes(include=['object']): # num_unique_values = len(df[col].unique()) # num_total_values = len(df[col]) # if float(num_unique_values) / num_total_values < 0.5: @@ -1191,7 +256,10 @@ def optimize_df_types(df: pd.DataFrame): return df -def process(fp: str, mp: str, admin: str, output_file: str, write_output = True, gadm=None): + +def process( + fp: str, mp: str, admin: str, output_file: str, write_output=True, gadm=None +): """ Parameters ---------- @@ -1199,7 +267,7 @@ def process(fp: str, mp: str, admin: str, output_file: str, write_output = True, Filename for JSON mapper from spacetag. Schema: https://github.com/jataware/spacetag/blob/schema/schema.py Example: https://github.com/jataware/spacetag/blob/schema/example.json - + gadm: gpd.GeoDataFrame, default None optional specification of a GeoDataFrame of GADM shapes of the appropriate level (admin2/3) for geocoding @@ -1211,36 +279,22 @@ def process(fp: str, mp: str, admin: str, output_file: str, write_output = True, mapper = json.loads(f.read()) # Validate JSON mapper schema against SpaceTag schema.py model. - #model = SpaceModel(geo=mapper['geo'], date=mapper['date'], feature=mapper['feature'], meta=mapper['meta']) + # model = SpaceModel(geo=mapper['geo'], date=mapper['date'], feature=mapper['feature'], meta=mapper['meta']) # "meta" portion of schema specifies transformation type transform = mapper["meta"] # Check transform for meta.geocode_level. Update admin to this if present. - if (admin == None and "geocode_level" in transform): + if admin == None and "geocode_level" in transform: admin = transform["geocode_level"] ftype = transform["ftype"] - if ftype == "geotiff": - df = raster2df( - InRaster = fp, - feature_name = transform["feature_name"], - band = int(transform["band"] if "band" in transform and transform["band"] != "" else "0"), - nodataval = int(transform["null_val"]), - date = transform["date"] if ("date" in transform and transform["date"] != "") else None, - band_name = transform["band_name"], - bands = transform["bands"] if "bands" in transform else None, - band_type = transform["band_type"] if "band_type" in transform else "category" - ) - elif ftype == 'excel': - df = pd.read_excel(fp, transform['sheet']) - elif ftype != "csv": - df = netcdf2df(fp) - else: - df = pd.read_csv(fp) + df = process_file_by_filetype( + filepath=fp, file_type=ftype, transformation_metadata=transform + ) ## Make mapper contain only keys for date, geo, and feature. - mapper = { k: mapper[k] for k in mapper.keys() & {"date", "geo", "feature"} } + mapper = {k: mapper[k] for k in mapper.keys() & {"date", "geo", "feature"}} ## To speed up normalize(), reduce the memory size of the dataframe by: # 1. Optimize the dataframe types. @@ -1263,18 +317,21 @@ def process(fp: str, mp: str, admin: str, output_file: str, write_output = True, qualify_cols = set(norm.columns).difference(set(COL_ORDER)) for col in qualify_cols: for feature_dict in mapper["feature"]: - if feature_dict["name"] == col and feature_dict["feature_type"] == 'string': + if ( + feature_dict["name"] == col + and feature_dict["feature_type"] == "string" + ): norm[col] = norm[col].astype(str) # Separate string from other dtypes in value column. # This is predicated on the assumption that qualifying feature columns # are of a single dtype. - norm['type'] = norm[['value']].applymap(type) - norm_str = norm[norm['type']==str] - norm = norm[norm['type']!=str] - del(norm_str['type']) - del(norm['type']) + norm["type"] = norm[["value"]].applymap(type) + norm_str = norm[norm["type"] == str] + norm = norm[norm["type"] != str] + del norm_str["type"] + del norm["type"] # Write parquet files norm.to_parquet(f"{output_file}.parquet.gzip", compression="gzip") @@ -1289,170 +346,6 @@ def process(fp: str, mp: str, admin: str, output_file: str, write_output = True, return norm, renamed_col_dict -def raster2df( - InRaster: str, - feature_name: str = "feature", - band: int = 0, - nodataval: int = -9999, - date: str = None, - band_name: str = "feature2", - bands: dict = None, - band_type: str = 'category' - -) -> pd.DataFrame: - """ - Description - ----------- - Takes the path of a raster (.tiff) file and produces a Geopandas Data Frame. - - Parameters - ---------- - InRaster: str - the path of the input raster file - feature_name: str - the name of the feature represented by the pixel values - band: int, default 1 - the band to operate on - nodataval: int, default -9999 - the value for no data pixels - date: str, default None - date associated with the raster (if any) - band_name: str, default feature2 - the name of the band data e.g. head_count, flooding - bands: dict, default None - passed in meta; dictionary of band identifiers and specifies bands to - be processed. - band_type: str, default category - Specifies band type e.g. category or datetime. If datetime, this data goes into the date column. - - Examples - -------- - Converting a geotiff of rainfall data into a geopandas dataframe - - >>> df = raster2df('path_to_raster.geotiff', 'rainfall', band=1) - - """ - # open the raster and get some properties - ds = gdal.OpenShared(InRaster, gdalconst.GA_ReadOnly) - GeoTrans = ds.GetGeoTransform() - ColRange = range(ds.RasterXSize) - RowRange = range(ds.RasterYSize) - - # Cache the dataframe and value data type. - df = pd.DataFrame() - row_data_type = None - - for x in range(1, ds.RasterCount+1): - # If band has a value, then limit import to the single specified band. - if band > 0 and band != x: - continue - - # If no bands in meta, then single-band and use band_name - # If bands, then process only those in the meta. - if not bands: - band_value = band_name - logging.info(f"Single band detected. Bands: {bands}, band_name: {band_name}, feature_name: {feature_name}") - elif str(x) in bands: - band_value = bands[str(x)] - logging.info(f"Multi-band detected Bands: {bands}, band_name: {band_name}, feature_name: {feature_name}") - elif str(x) not in bands: - # Processing a band not specified in the meta, so skip it - logging.info(f"Skipping band {x} since it is not specified in {bands}.") - continue - else: - raise Exception(f"Neither single nor multiple bands specified in meta. Current band: {x}, Bands: {bands}, band_name: {band_name}, feature_name: {feature_name}") - - # Create columns for the dataframe. - if not bands: - columns = ["longitude", "latitude", feature_name] - logging.info(f"Single band detected. Columns are: {columns}") - elif band_type == 'datetime': - columns=["longitude", "latitude", 'date', feature_name] - logging.info(f"Datetime multiband detected. Columns are: {columns}") - elif band_type == 'category': - # categorical multi-band; add columns during processing. - columns=["longitude", "latitude", band_value] - logging.info(f"Categorical multiband detected. Columns are: {columns}") - else: - raise Exception(f"During column processing, neither single nor multiple bands specified in meta. Bands: {bands}, band_name: {band_name}, feature_name: {feature_name}") - - rBand = ds.GetRasterBand(x) - nData = rBand.GetNoDataValue() - - if nData == None: - logging.warning(f"No nodataval found, setting to {nodataval}") - nData = np.float32(nodataval) # set it to something if not set - else: - logging.info(f"Nodataval is: {nData} type is : {type(nData)}") - - # specify the center offset (takes the point in middle of pixel) - HalfX = GeoTrans[1] / 2 - HalfY = GeoTrans[5] / 2 - - # Check that NoDataValue is of the same type as the raster data - RowData = rBand.ReadAsArray(0, 0, ds.RasterXSize, 1)[0] - row_data_type = type(RowData[0]) - if type(nData) != row_data_type: - logging.info( - f"NoData type mismatch: NoDataValue is type {type(nData)} and raster data is type {row_data_type}" - ) - # e.g. NoDataValue is type and raster data is type - # Fix float type mismatches so comparison works below (row_value != nData) - if row_data_type == np.float32: - nData = np.float32(nData) - elif row_data_type == np.float64: - nData = np.float64(nData) - elif row_data_type == np.float16: - nData = np.float16(nData) - - points = [] - - for ThisRow in RowRange: - RowData = rBand.ReadAsArray(0, ThisRow, ds.RasterXSize, 1)[0] - for ThisCol in ColRange: - # need to exclude NaN values since there is no nodataval - row_value = RowData[ThisCol] - - if (row_value > nData) and not (np.isnan(row_value)): - - # TODO: implement filters on valid pixels - # for example, the below would ensure pixel values are between -100 and 100 - # if (RowData[ThisCol] <= 100) and (RowData[ThisCol] >= -100): - - X = GeoTrans[0] + (ThisCol * GeoTrans[1]) - Y = GeoTrans[3] + (ThisRow * GeoTrans[5]) # Y is negative so it's a minus - # this gives the upper left of the cell, offset by half a cell to get centre - X += HalfX - Y += HalfY - - # Add the data row to the dataframe. - if bands == None: - points.append([X, Y, row_value]) - elif band_type == 'datetime': - points.append([X, Y, band_value, row_value]) - else: - points.append([X, Y, row_value]) - - # This will make all floats float64, but will be optimized in process(). - new_df = pd.DataFrame(points, columns=columns) - - if df.empty: - df = new_df - else: - #df = df.merge(new_df, left_on=["longitude", "latitude"], right_on=["longitude", "latitude"]) - if (bands and band_type != 'datetime'): - #df.join(new_df, on=["longitude", "latitude"]) - df = df.merge(new_df, left_on=["longitude", "latitude"], right_on=["longitude", "latitude"]) - else: - df = df.append(new_df) - - # Add the date from the mapper. - if (date and band_type != 'datetime'): - df["date"] = date - - df.sort_values(by=columns, inplace=True) - - return df class mixdata: def load_gadm2(self): @@ -1464,15 +357,15 @@ def load_gadm2(self): gadmDir = f"{download_data_folder}/{gadm_fn}" gadm = gf.from_geofeather(gadmDir) gadm["country"] = gadm["NAME_0"] - #gadm["state"] = gadm["NAME_1"] + # gadm["state"] = gadm["NAME_1"] gadm["admin1"] = gadm["NAME_1"] - gadm["admin2"] = gadm["NAME_2"] + gadm["admin2"] = gadm["NAME_2"] gadm0 = gadm[["geometry", "country"]] - #gadm1 = gadm[["geometry", "country", "state", "admin1"]] - #gadm2 = gadm[["geometry", "country", "state", "admin1", "admin2"]] + # gadm1 = gadm[["geometry", "country", "state", "admin1"]] + # gadm2 = gadm[["geometry", "country", "state", "admin1", "admin2"]] gadm1 = gadm[["geometry", "country", "admin1"]] gadm2 = gadm[["geometry", "country", "admin1", "admin2"]] - + self.gadm0 = gadm0 self.gadm1 = gadm1 self.gadm2 = gadm2 @@ -1480,78 +373,78 @@ def load_gadm2(self): def load_gadm3(self): # Admin 3 cdir = os.path.expanduser("~") - download_data_folder = f"{cdir}/mixmasta_data" + download_data_folder = f"{cdir}/mixmasta_data" gadm_fn = f"gadm36_3.feather" gadmDir = f"{download_data_folder}/{gadm_fn}" gadm3 = gf.from_geofeather(gadmDir) gadm3["country"] = gadm3["NAME_0"] - #gadm3["state"] = gadm3["NAME_1"] + # gadm3["state"] = gadm3["NAME_1"] gadm3["admin1"] = gadm3["NAME_1"] gadm3["admin2"] = gadm3["NAME_2"] gadm3["admin3"] = gadm3["NAME_3"] - #gadm3 = gadm3[["geometry", "country", "state", "admin1", "admin2", "admin3"]] + # gadm3 = gadm3[["geometry", "country", "state", "admin1", "admin2", "admin3"]] gadm3 = gadm3[["geometry", "country", "admin1", "admin2", "admin3"]] self.gadm3 = gadm3 - + # Testing # iso testing: -#mp = 'examples/causemosify-tests/mixmasta_ready_annotations_timestampfeature.json' -#fp = 'examples/causemosify-tests/raw_excel_timestampfeature.xlsx' +# mp = 'examples/causemosify-tests/mixmasta_ready_annotations_timestampfeature.json' +# fp = 'examples/causemosify-tests/raw_excel_timestampfeature.xlsx' # build a date qualifier -#mp = 'examples/causemosify-tests/build-a-date-qualifier.json' -#fp = 'examples/causemosify-tests/build-a-date-qualifier_xyzz.csv' - -#fp = "examples/causemosify-tests/november_tests_atlasai_assetwealth_allyears_2km.tif" -#mp = "examples/causemosify-tests/november_tests_atlasai_assetwealth_allyears_2km.json" - -#fp = "examples/causemosify-tests/flood_monthly.tif" -#mp = "examples/causemosify-tests/flood_monthly.json" - -#fp = "examples/causemosify-tests/Kenya - Admin1_Pasture_NDVI_2019-2021 converted.csv" -#mp = "examples/causemosify-tests/kenya_pasture.json" - -#fp = "examples/causemosify-tests/rainfall_error.xlsx" -#mp = "examples/causemosify-tests/rainfall_error.json" -#geo = 'admin2' -#outf = 'examples/causemosify-tests/testing' - -#mp = 'tests/inputs/test3_qualifies.json' -#fp = 'tests/inputs/test3_qualifies.csv' -#geo = 'admin2' -#outf = 'tests/outputs/unittests' - -#fp = "examples/causemosify-tests/hoa_conflict.csv" -#mp = "examples/causemosify-tests/hoa_conflict.json" -#geo = 'admin2' -#outf = 'examples/causemosify-tests' - -#fp = "examples/causemosify-tests/maxent_Ethiopia_precipChange.0.8tempChange.-0.3.tif" -#mp = "examples/causemosify-tests/maxent_Ethiopia_precipChange.0.8tempChange.-0.3.json" - -#fp = 'examples/causemosify-tests/SouthSudan_2017_Apr_hires_masked_malnut.tiff' -#mp = 'examples/causemosify-tests/SouthSudan_2017_Apr_hires_masked_malnut.json' -#geo = 'admin2' -#outf = 'examples/causemosify-tests' - -#fp = "examples/causemosify-tests/example.csv" -#mp = "examples/causemosify-tests/example.json" -#geo = 'admin2' -#outf = 'examples/causemosify-tests' - -#start_time = timeit.default_timer() -#df = pd.DataFrame() -#print('processing...') -#df, dct = process(fp, mp, geo, outf) -#print('process time', timeit.default_timer() - start_time) -#cols = ['timestamp','country','admin1','admin2','admin3','lat','lng','feature','value'] -#df.sort_values(by=cols, inplace=True) -#df.reset_index(drop=True, inplace=True) -#df.to_csv("tests/outputs/test7_single_band_tif_output.csv", index = False) -#print('\n', df.head()) -#print('\n', df.tail()) -#print('\n', df.shape) -#print('\nrenamed column dictionary\n', dct) -#print(f"shape\n{df.shape}\ninfo\n{df.info()}\nmemory usage\n{df.memory_usage(index=True, deep=True)}\n{df.memory_usage(index=True, deep=True).sum()}") +# mp = 'examples/causemosify-tests/build-a-date-qualifier.json' +# fp = 'examples/causemosify-tests/build-a-date-qualifier_xyzz.csv' + +# fp = "examples/causemosify-tests/november_tests_atlasai_assetwealth_allyears_2km.tif" +# mp = "examples/causemosify-tests/november_tests_atlasai_assetwealth_allyears_2km.json" + +# fp = "examples/causemosify-tests/flood_monthly.tif" +# mp = "examples/causemosify-tests/flood_monthly.json" + +# fp = "examples/causemosify-tests/Kenya - Admin1_Pasture_NDVI_2019-2021 converted.csv" +# mp = "examples/causemosify-tests/kenya_pasture.json" + +# fp = "examples/causemosify-tests/rainfall_error.xlsx" +# mp = "examples/causemosify-tests/rainfall_error.json" +# geo = 'admin2' +# outf = 'examples/causemosify-tests/testing' + +# mp = 'tests/inputs/test3_qualifies.json' +# fp = 'tests/inputs/test3_qualifies.csv' +# geo = 'admin2' +# outf = 'tests/outputs/unittests' + +# fp = "examples/causemosify-tests/hoa_conflict.csv" +# mp = "examples/causemosify-tests/hoa_conflict.json" +# geo = 'admin2' +# outf = 'examples/causemosify-tests' + +# fp = "examples/causemosify-tests/maxent_Ethiopia_precipChange.0.8tempChange.-0.3.tif" +# mp = "examples/causemosify-tests/maxent_Ethiopia_precipChange.0.8tempChange.-0.3.json" + +# fp = 'examples/causemosify-tests/SouthSudan_2017_Apr_hires_masked_malnut.tiff' +# mp = 'examples/causemosify-tests/SouthSudan_2017_Apr_hires_masked_malnut.json' +# geo = 'admin2' +# outf = 'examples/causemosify-tests' + +# fp = "examples/causemosify-tests/example.csv" +# mp = "examples/causemosify-tests/example.json" +# geo = 'admin2' +# outf = 'examples/causemosify-tests' + +# start_time = timeit.default_timer() +# df = pd.DataFrame() +# print('processing...') +# df, dct = process(fp, mp, geo, outf) +# print('process time', timeit.default_timer() - start_time) +# cols = ['timestamp','country','admin1','admin2','admin3','lat','lng','feature','value'] +# df.sort_values(by=cols, inplace=True) +# df.reset_index(drop=True, inplace=True) +# df.to_csv("tests/outputs/test7_single_band_tif_output.csv", index = False) +# print('\n', df.head()) +# print('\n', df.tail()) +# print('\n', df.shape) +# print('\nrenamed column dictionary\n', dct) +# print(f"shape\n{df.shape}\ninfo\n{df.info()}\nmemory usage\n{df.memory_usage(index=True, deep=True)}\n{df.memory_usage(index=True, deep=True).sum()}") diff --git a/mixmasta/normalizer.py b/mixmasta/normalizer.py new file mode 100644 index 0000000..aadfb0e --- /dev/null +++ b/mixmasta/normalizer.py @@ -0,0 +1,1087 @@ +"""Normalizer module to complete the normalization on dataframes. + +Returns: + Tuple(pandas.Dataframe, dict, pandas.Dataframe): _description_ +""" +import logging +import os + +import click +import geopandas as gpd +import numpy +import pandas +import pkg_resources +import timeit + +from distutils.util import strtobool +import geofeather as gf +from pathlib import Path +from shapely import speedups +from shapely.geometry import Point + +# TODO remove to another file. +# Constants +COL_ORDER = [ + "timestamp", + "country", + "admin1", + "admin2", + "admin3", + "lat", + "lng", + "feature", + "value", +] + +GEO_TYPE_COUNTRY = "country" +GEO_TYPE_ADMIN1 = "state/territory" +GEO_TYPE_ADMIN2 = "county/district" +GEO_TYPE_ADMIN3 = "municipality/town" + + +def normalizer( + df: pandas.DataFrame, + mapper: dict, + admin: str, + gadm: gpd.GeoDataFrame = None, + df_geocode: pandas.DataFrame = pandas.DataFrame(), +) -> (pandas.DataFrame, dict, pandas.DataFrame): + """ + Description + ----------- + Converts a dataframe into a CauseMos compliant format. + + Parameters + ---------- + df: pandas.DataFrame + a pandas dataframe containing point data + mapper: dict + a schema mapping (JSON) for the dataframe + a dict where keys will be geo, feaure, date, and values will be lists of dict + example: + { 'geo': [ + {'name': 'country', 'type': 'geo', 'geo_type': 'country', 'primary_geo': False}, + {'name': 'state', 'type': 'geo', 'geo_type': 'state/territory', 'primary_geo': False} + ], + 'feature': [ + {'name': 'probabilty', 'type': 'feature', 'feature_type': 'float'}, + {'name': 'color', 'type': 'feature', 'feature_type': 'str'} + ], + 'date': [ + {'name': 'date_2', 'type': 'date', 'date_type': 'date', 'primary_date': False, 'time_format': '%m/%d/%y'}, + {'name': 'date', 'type': 'date', 'date_type': 'date', 'primary_date': True, 'time_format': '%m/%d/%y'} + ] + } + admin: str, default 'admin2' + the level to geocode to. Either 'admin2' or 'admin3' + gadm: gpd.GeoDataFrame, default None + optional specification of a GeoDataFrame of GADM shapes of the appropriate + level (admin2/3) for geocoding + df_gecode: pandas.DataFrame, default pandas.DataFrame() + lat,long geocode lookup library + + Returns + ------- + pandas.DataFrame: CauseMos compliant format ready to be written to parquet. + dict: dictionary of modified column names; used by SpaceTag + pandas.DataFRame: update lat,long geocode looup library + + Examples + -------- + >>> df_norm = normalizer(df, mapper, 'admin3') + """ + col_order = COL_ORDER.copy() + + required_cols = [ + "timestamp", + "country", + "admin1", + "admin2", + "admin3", + "lat", + "lng", + ] + + # List of date_types that be used to build a date. + MONTH_DAY_YEAR = ["day", "month", "year"] + + # Create a dictionary of list: colnames: new col name, and modify df and + # mapper for any column name collisions. + df, mapper, renamed_col_dict = handle_colname_collisions(df, mapper, col_order) + + ### mapper is a dictionary of lists of dictionaries. + click.echo("Raw dataframe:") + click.echo(df.head()) + + # list of names of datetime columns primary_date=True + primary_time_cols = [ + k["name"] + for k in mapper["date"] + if "primary_date" in k and k["primary_date"] == True + ] + + # list of names of datetime columns no primary_date or primary_date = False + other_time_cols = [ + k["name"] + for k in mapper["date"] + if "primary_date" not in k or k["primary_date"] == False + ] + + # list of names of geo columns primary_geo=True + primary_geo_cols = [ + k["name"] + for k in mapper["geo"] + if "primary_geo" in k and k["primary_geo"] == True + ] + + # list of geotypes of geo columns primary_geo=True (used for match_geo_names logic below) + primary_geo_types = [ + k["geo_type"] + for k in mapper["geo"] + if "primary_geo" in k and k["primary_geo"] == True + ] + + # qualified_col_dict: dictionary for columns qualified by another column. + # key: qualified column + # value: list of columns that qualify key column + qualified_col_dict = {} + + # subset dataframe for only columns specified in mapper schema. + # get all named objects in the date, feature, geo schema lists. + mapper_keys = [] + for k in mapper.items(): + mapper_keys.extend([l["name"] for l in k[1] if "name" in l]) + + df = df[mapper_keys] + + # Rename protected columns + # and perform type conversion on the time column + features = [] + primary_date_group_mapper = {} + other_date_group_mapper = {} + + for date_dict in mapper["date"]: + kk = date_dict["name"] + if kk in primary_time_cols: + # There should only be a single epoch or date field, or a single + # group of year/month/day/minute/second marked as primary_time in + # the loaded schema. + if date_dict["date_type"] == "date": + # convert primary_time of date_type date to epochtime and rename as 'timestamp' + df.loc[:, kk] = df[kk].apply( + lambda x: format_time( + str(x), date_dict["time_format"], validate=False + ) + ) + staple_col_name = "timestamp" + df.rename(columns={kk: staple_col_name}, inplace=True) + # renamed_col_dict[ staple_col_name ] = [kk] # 7/2/2021 do not include primary cols + elif date_dict["date_type"] == "epoch": + # rename epoch time column as 'timestamp' + staple_col_name = "timestamp" + df.rename(columns={kk: staple_col_name}, inplace=True) + # renamed_col_dict[ staple_col_name ] = [kk] # 7/2/2021 do not include primary cols + elif date_dict["date_type"] in ["day", "month", "year"]: + primary_date_group_mapper[kk] = date_dict + + else: + if date_dict["date_type"] == "date": + # Convert all date/time to epoch time if not already. + df.loc[:, kk] = df[kk].apply( + lambda x: format_time( + str(x), date_dict["time_format"], validate=False + ) + ) + # If three are no assigned primary_time columns, make this the + # primary_time timestamp column, and keep as a feature so the + # column_name meaning is not lost. + if not primary_time_cols and not "timestamp" in df.columns: + df.rename(columns={kk: "timestamp"}, inplace=True) + staple_col_name = "timestamp" + renamed_col_dict[staple_col_name] = [kk] + # All not primary_time, not associated_columns fields are pushed to features. + features.append(kk) + + elif ( + date_dict["date_type"] in MONTH_DAY_YEAR + and "associated_columns" in date_dict + and date_dict["associated_columns"] + ): + # Various date columns have been associated by the user and are not primary_date. + # convert them to epoch then store them as a feature + # (instead of storing them as separate uncombined features). + # handle this dict after iterating all date fields + other_date_group_mapper[kk] = date_dict + + else: + features.append(kk) + + if "qualifies" in date_dict and date_dict["qualifies"]: + # Note that any "qualifier" column that is not primary geo/date + # will just be lopped on to the right as its own column. It's + # column name will just be the name and Uncharted will deal with + # it. The key takeaway is that qualifier columns grow the width, + # not the length of the dataset. + # Want to add the qualified col as the dictionary key. + # e.g. "name": "region", "qualifies": ["probability", "color"] + # should produce two dict entries for prob and color, with region + # in a list as the value for both. + for k in date_dict["qualifies"]: + if k in qualified_col_dict: + qualified_col_dict[k].append(kk) + else: + qualified_col_dict[k] = [kk] + + if primary_date_group_mapper: + # Applied when there were primary_date year,month,day fields above. + # These need to be combined + # into a date and then epoch time, and added as the timestamp field. + + # Create a separate df of the associated date fields. This avoids + # pandas upcasting the series dtypes on df.apply(); e.g., int to float, + # or a month 9 to 9.0, which breaks generate_timestamp() + assoc_fields = primary_date_group_mapper.keys() + date_df = df[assoc_fields] + + # Now generate the timestamp from date_df and add timestamp col to df. + df = generate_timestamp_column(df, primary_date_group_mapper, "timestamp") + + # Determine the correct time format for the new date column, and + # convert to epoch time. + time_formatter = generate_timestamp_format(primary_date_group_mapper) + df["timestamp"] = df["timestamp"].apply( + lambda x: format_time(str(x), time_formatter, validate=False) + ) + + # Let SpaceTag know those date columns were renamed to timestamp. + # renamed_col_dict[ "timestamp" ] = assoc_fields # 7/2/2021 do not include primary cols + + while other_date_group_mapper: + # Various date columns have been associated by the user and are not primary_date. + # Convert to epoch time and store as a feature, do not store these separately in features. + # Exception is the group is only two of day, month, year: leave as date. + # Control for possibility of more than one set of assciated_columns. + + # Pop the first item in the mapper and begin building that date set. + date_field_tuple = other_date_group_mapper.popitem() + + # Build a list of column names associated with the the popped date field. + assoc_fields = [k[1] for k in date_field_tuple[1]["associated_columns"].items()] + + # Pop those mapper objects into a dict based on the column name keys in + # assocfields list. + assoc_columns_dict = { + f: other_date_group_mapper.pop(f) + for f in assoc_fields + if f in other_date_group_mapper + } + + # Add the first popped tuple into the assoc_columns dict where the key is the + # first part of the tuple; the value is the 2nd part. + assoc_columns_dict[date_field_tuple[0]] = date_field_tuple[1] + + # Add the first popped tuple column name to the list of associated fields. + assoc_fields.append(date_field_tuple[0]) + + # TODO: If day and year are associated to each other and month, but + # month is not associated to those fields, then at this point assoc_fields + # will be the three values, and assoc_columns will contain only day and + # year. This will error out below. It is assumed that SpaceTag will + # control for this instance. + + # If there is no primary_time column for timestamp, which would have + # been created above with primary_date_group_mapper, or farther above + # looping mapper["date"], attempt to generate from date_type = Month, + # Day, Year features. Otherwise, create a new column name from the + # concatenation of the associated date fields here. + if not "timestamp" in df.columns: + new_column_name = "timestamp" + else: + new_column_name = generate_column_name(assoc_fields) + + # Create a separate df of the associated date fields. This avoids + # pandas upcasting the series dtypes on df.apply(); e.g., int to float, + # or a month 9 to 9.0, which breaks generate_timestamp() + date_df = df[assoc_fields] + + # Now generate the timestamp from date_df and add timestamp col to df. + df = generate_timestamp_column(df, assoc_columns_dict, new_column_name) + + # Determine the correct time format for the new date column, and + # convert to epoch time only if all three date components (day, month, + # year) are present; otherwise leave as a date string. + date_types = [v["date_type"] for k, v in assoc_columns_dict.items()] + if len(frozenset(date_types).intersection(MONTH_DAY_YEAR)) == 3: + time_formatter = generate_timestamp_format(assoc_columns_dict) + df.loc[:, new_column_name] = df[new_column_name].apply( + lambda x: format_time(str(x), time_formatter, validate=False) + ) + + # Let SpaceTag know those date columns were renamed to a new column. + renamed_col_dict[new_column_name] = assoc_fields + + # timestamp is a protected column, so don't add to features. + if new_column_name != "timestamp": + # Handle edge case of each date field in assoc_fields qualifying + # the same column e.g. day/month/year are associated and qualify + # a field. In this case, the new_column_name + qualified_col = build_date_qualifies_field(qualified_col_dict, assoc_fields) + if qualified_col is None: + features.append(new_column_name) + else: + qualified_col_dict[qualified_col] = [new_column_name] + + for geo_dict in mapper["geo"]: + kk = geo_dict["name"] + if kk in primary_geo_cols: + if geo_dict["geo_type"] == "latitude": + staple_col_name = "lat" + df.rename(columns={kk: staple_col_name}, inplace=True) + # renamed_col_dict[staple_col_name] = [kk] # 7/2/2021 do not include primary cols + elif geo_dict["geo_type"] == "longitude": + staple_col_name = "lng" + df.rename(columns={kk: staple_col_name}, inplace=True) + # renamed_col_dict[staple_col_name] = [kk] # 7/2/2021 do not include primary cols + elif geo_dict["geo_type"] == "coordinates": + c_f = geo_dict["coord_format"] + coords = df[kk].values + if c_f == "lonlat": + lats = [x for x in coords.split(",")[1]] + longs = [x for x in coords.split(",")[0]] + else: + lats = [x for x in coords.split(",")[0]] + longs = [x for x in coords.split(",")[1]] + df["lng"] = longs + df["lat"] = lats + del df[kk] + elif geo_dict["geo_type"] == GEO_TYPE_COUNTRY and kk != "country": + # force the country column to be named country + staple_col_name = "country" + df.rename(columns={kk: staple_col_name}, inplace=True) + # renamed_col_dict[staple_col_name] = [kk] # 7/2/2021 do not include primary cols + elif geo_dict["geo_type"] == GEO_TYPE_ADMIN1 and kk != "admin1": + # force the country column to be named country + staple_col_name = "admin1" + df.rename(columns={kk: staple_col_name}, inplace=True) + elif geo_dict["geo_type"] == GEO_TYPE_ADMIN2 and kk != "admin2": + # force the country column to be named country + staple_col_name = "admin2" + df.rename(columns={kk: staple_col_name}, inplace=True) + elif geo_dict["geo_type"] == GEO_TYPE_ADMIN3 and kk != "admin2": + # force the country column to be named country + staple_col_name = "admin3" + df.rename(columns={kk: staple_col_name}, inplace=True) + + elif str(geo_dict["geo_type"]).lower() in ["iso2", "iso3"]: + # use the ISO2 or ISO3 column as country + + # use ISO2/3 lookup dictionary to change ISO to country name. + iso_list = df[kk].unique().tolist() + dct = get_iso_country_dict(iso_list) + df.loc[:, kk] = df[kk].apply(lambda x: dct[x] if x in dct else x) + + # now rename that column as "country" + staple_col_name = "country" + df.rename(columns={kk: staple_col_name}, inplace=True) + # renamed_col_dict[staple_col_name] = [kk] # 7/2/2021 do not include primary cols + + elif "qualifies" in geo_dict and geo_dict["qualifies"]: + # Note that any "qualifier" column that is not primary geo/date + # will just be lopped on to the right as its own column. It'’'s + # column name will just be the name and Uncharted will deal with + # it. The key takeaway is that qualifier columns grow the width, + # not the length of the dataset. + # Want to add the qualified col as the dictionary key. + # e.g. "name": "region", "qualifies": ["probability", "color"] + # should produce two dict entries for prob and color, with region + # in a list as the value for both. + for k in geo_dict["qualifies"]: + if k in qualified_col_dict: + qualified_col_dict[k].append(kk) + else: + qualified_col_dict[k] = [kk] + else: + # only push geo columns to the named columns + # in the event there is no primary geo + # otherwise they are features and we geocode lat/lng + if len(primary_geo_cols) == 0: + if geo_dict["geo_type"] == GEO_TYPE_COUNTRY: + df["country"] = df[kk] + renamed_col_dict["country"] = [kk] + continue + if geo_dict["geo_type"] == GEO_TYPE_ADMIN1: + df["admin1"] = df[kk] + renamed_col_dict["admin1"] = [kk] + continue + if geo_dict["geo_type"] == GEO_TYPE_ADMIN2: + df["admin2"] = df[kk] + renamed_col_dict["admin2"] = [kk] + continue + if geo_dict["geo_type"] == GEO_TYPE_ADMIN3: + df["admin3"] = df[kk] + renamed_col_dict["admin3"] = [kk] + continue + features.append(kk) + + # Append columns annotated in feature dict to features list (if not a + # qualifies column) + # features.extend([k["name"] for k in mapper["feature"]]) + for feature_dict in mapper["feature"]: + if "qualifies" not in feature_dict or not feature_dict["qualifies"]: + features.append(feature_dict["name"]) + elif "qualifies" in feature_dict and feature_dict["qualifies"]: + # Note that any "qualifier" column that is not primary geo/date + # will just be lopped on to the right as its own column. It's + # column name will just be the name and Uncharted will deal with + # it. The key takeaway is that qualifier columns grow the width, + # not the length of the dataset. + # Want to add the qualified col as the dictionary key. + # e.g. "name": "region", "qualifies": ["probability", "color"] + # should produce two dict entries for prob and color, with region + # in a list as the value for both. + for k in feature_dict["qualifies"]: + kk = feature_dict["name"] + if k in qualified_col_dict: + qualified_col_dict[k].append(kk) + else: + qualified_col_dict[k] = [kk] + + # Convert aliases based on user annotations + aliases = feature_dict.get("aliases", {}) + if aliases: + click.echo(f"Pre-processed aliases are: {aliases}") + type_ = df[feature_dict["name"]].dtype.type + click.echo(f"Detected column type is: {type_}") + aliases_ = {} + # The goal below is to identify the data type and then to cast the + # alias key from string into that type so that it will match + # if that fails, just cast it as a string + for kk, vv in aliases.items(): + try: + if issubclass(type_, (int, numpy.integer)): + click.echo("Aliasing: integer detected") + aliases_[int(kk)] = vv + elif issubclass( + type_, + ( + float, + numpy.float16, + numpy.float32, + numpy.float64, + numpy.float128, + ), + ): + click.echo("Aliasing: float detected") + aliases_[float(kk)] = vv + elif issubclass(type_, (bool, numpy.bool, numpy.bool_)): + click.echo("Aliasing: boolean detected") + if strtobool(kk) == 1: + aliases_[True] = vv + click.echo("Converted true string to boolean") + else: + click.echo("Converted false string to boolean") + aliases_[False] = vv + # Fall back on string + else: + click.echo("Aliasing: string detected") + aliases_[kk] = vv + except ValueError as e: + # Fall back on string + click.echo(f"Error: {e}") + aliases_[kk] = vv + click.echo(f"Aliases for {feature_dict['name']} are {aliases_}.") + df[[feature_dict["name"]]] = df[[feature_dict["name"]]].replace(aliases_) + + # Since the user has decided to apply categorical aliases to this feature, we must coerce + # the entire feature to a string, even if they did not alias every value within the feature + # the reason for this is to avoid mixed types within the feature (e.g. half int/half string) + # since this makes it difficult to visualize + df[[feature_dict["name"]]] = df[[feature_dict["name"]]].astype(str) + + # perform geocoding if lat/lng are present + if "lat" in df and "lng" in df: + df, df_geocode = geocode( + admin, df, x="lng", y="lat", gadm=gadm, df_geocode=df_geocode + ) + elif "country" in primary_geo_types or ("country" in df and not primary_geo_types): + # Correct any misspellings etc. in state and admin areas when not + # geocoding lat and lng above, and country is the primary_geo. + # This doesn't match names if iso2/iso3 are primary, and when country + # admin1-3 are moved to features. Exception is when country is present, + # but nothing is marked as primary. + + # Only geo_code resolve_to_gadm = True fields. + # Used below when match_geocode_names + resolve_to_gadm_geotypes = [ + k["geo_type"] + for k in mapper["geo"] + if "resolve_to_gadm" in k and k["resolve_to_gadm"] == True + ] + if resolve_to_gadm_geotypes: + df = match_geo_names(admin, df, resolve_to_gadm_geotypes) + + df_geo_cols = [i for i in df.columns if "mixmasta_geocoded" in i] + for c in df_geo_cols: + df.rename(columns={c: c.replace("_mixmasta_geocoded", "")}, inplace=True) + + # protected_cols are the required_cols present in the submitted dataframe. + protected_cols = list(set(required_cols) & set(df.columns)) + + # if a field qualifies a protected field like country, it should have data + # in each row, unlike features below where the qualifying data appears + # only on those rows. + # k: qualified column (str) + # v: list of columns (str) that qualify k + for k, v in qualified_col_dict.items(): + if k in protected_cols: + # k is qualified by the columns in v, and k is a protected column, + # so extend the width of the output dataset with v for each row. + protected_cols.extend(v) + col_order.extend(v) + + # Prepare output by + # 1. if there are no features, simply reduce the dataframe. + # or, 2.iterating features to add to feature adn value columns. + if not features: + df_out = df[protected_cols] + else: + df_out = pandas.DataFrame() + for feat in features: + using_cols = protected_cols.copy() + + if feat in qualified_col_dict: + # dict value is a list, so extend. + using_cols.extend(qualified_col_dict[feat]) + + # add a qualifying column name only if not in col_order already + for c in qualified_col_dict[feat]: + if c not in col_order: + col_order.append(c) + + join_overlap = False + try: + df_ = df[using_cols + [feat + "_mixmasta_left"]].copy() + join_overlap = True + except: + df_ = df[using_cols + [feat]].copy() + + try: + if mapper[feat]["new_col_name"] == None: + df_["feature"] = feat + else: + df_["feature"] = mapper[feat]["new_col_name"] + except: + df_["feature"] = feat + + if join_overlap: + df_.rename(columns={f"{feat}_mixmasta_left": "value"}, inplace=True) + else: + df_.rename(columns={feat: "value"}, inplace=True) + + # Add feature/value for epochtime as object adds it without decimal + # places, but it is still saved as a double in the parquet file. + if len(df_out) == 0: + if feat in other_time_cols: + df_out = df_.astype({"value": object}) + else: + df_out = df_ + else: + if feat in other_time_cols: + df_out = df_out.append(df_.astype({"value": object})) + else: + df_out = df_out.append(df_) + + for c in col_order: + if c not in df_out: + df_out[c] = None + + # Drop rows with nulls in value column. + df_out.dropna(axis=0, subset=["value"], inplace=True) + + # Handle any renamed cols being renamed. + renamed_col_dict = audit_renamed_col_dict(renamed_col_dict) + + click.echo("Processed dataframe:") + click.echo(df_out.head()) + return df_out[col_order], renamed_col_dict, df_geocode + + +def handle_colname_collisions( + df: pandas.DataFrame, mapper: dict, protected_cols: list +) -> (pandas.DataFrame, dict, dict): + """ + Description + ----------- + Identify mapper columns that match protected column names. When found, + update the mapper and dataframe, and keep a dict of these changes + to return to the caller e.g. SpaceTag. + + Parameters + ---------- + df: pandas.DataFrame + submitted data + mapper: dict + a dictionary for the schema mapping (JSON) for the dataframe. + protected_cols: list + protected column names i.e. timestamp, country, admin1, feature, etc. + + Output + ------ + pandas.DataFame: + The modified dataframe. + dict: + The modified mapper. + dict: + key: new column name e.g. "day1month1year1" or "country_non_primary" + value: list of old column names e.g. ['day1','month1','year1'] or ['country'] + """ + + # Get names of geo fields that collide and are not primary_geo = True + non_primary_geo_cols = [ + d["name"] + for d in mapper["geo"] + if d["name"] in protected_cols + and ("primary_geo" not in d or d["primary_geo"] == False) + ] + + # Get names of date fields that collide and are not primary_date = True + non_primary_time_cols = [ + d["name"] + for d in mapper["date"] + if d["name"] in protected_cols + and ("primary_date" not in d or d["primary_date"] == False) + ] + + # Only need to change a feature column name if it qualifies another field, + # and therefore will be appended as a column to the output. + feature_cols = [ + d["name"] + for d in mapper["feature"] + if d["name"] in protected_cols and "qualifies" in d and d["qualifies"] + ] + + # Verbose build of the collision_list, could have combined above. + collision_list = non_primary_geo_cols + non_primary_time_cols + feature_cols + + # Bail if no column name collisions. + if not collision_list: + return df, mapper, {} + + # Append any collision columns with the following suffix. + suffix = "_non_primary" + + # Build output dictionary and update df. + renamed_col_dict = {} + for col in collision_list: + df.rename(columns={col: col + suffix}, inplace=True) + renamed_col_dict[col + suffix] = [col] + + # Update mapper + for k, vlist in mapper.items(): + for dct in vlist: + if dct["name"] in collision_list: + dct["name"] = dct["name"] + suffix + elif "qualifies" in dct and dct["qualifies"]: + # change any instances of this column name qualified by another field + dct["qualifies"] = [ + w.replace(w, w + suffix) if w in collision_list else w + for w in dct["qualifies"] + ] + elif "associated_columns" in dct and dct["associated_columns"]: + # change any instances of this column name in an associated_columns dict + dct["associated_columns"] = { + k: v.replace(v, v + suffix) if v in collision_list else v + for k, v in dct["associated_columns"].items() + } + + return df, mapper, renamed_col_dict + + +def format_time(t: str, time_format: str, validate: bool = True) -> int: + """ + Description + ----------- + Converts a time feature (t) into epoch time using `time_format` which is a strftime definition + + Parameters + ---------- + t: str + the time string + time_format: str + the strftime format for the string t + validate: bool, default True + whether to error check the time string t. Is set to False, then no error is raised if the date fails to parse, but None is returned. + + Examples + -------- + + >>> epoch = format_time('5/12/20 12:20', '%m/%d/%y %H:%M') + """ + + try: + t_ = ( + int(datetime.strptime(t, time_format).timestamp()) * 1000 + ) # Want milliseonds + return t_ + except Exception as e: + if t.endswith(" 00:00:00"): + # Depending on the date format, pandas.read_excel will read the + # date as a Timestamp, so here it is a str with format + # '2021-03-26 00:00:00'. For now, handle this single case until + # there is time for a more comprehensive solution e.g. add a custom + # date_parser function that doesn't parse diddly/squat to + # pandas.read_excel() in process(). + return format_time(t.replace(" 00:00:00", ""), time_format, validate) + print(e) + if validate: + raise Exception(e) + else: + return None + + +def generate_timestamp_column( + df: pandas.DataFrame, date_mapper: dict, column_name: str +) -> pandas.DataFrame: + """ + Description + ----------- + Efficiently add a new timestamp column to a dataframe. It avoids the use of df.apply + which appears to be much slower for large dataframes. Defaults to 1/1/1970 for + missing day/month/year values. + + Parameters + ---------- + df: pandas.DataFrame + our data + date_mapper: dict + a schema mapping (JSON) for the dataframe filtered for "date_type" equal to + Day, Month, or Year. The format is screwy for our purposes here and could + be reafactored. + column_name: str + name of the new column e.g. timestamp for primary_time, year1month1day1 + for a concatneated name from associated date fields. + + Examples + -------- + This example adds the generated series to the source dataframe. + >>> df = df.join(df.apply(generate_timestamp, date_mapper=date_mapper, + column_name="year1month1day", axis=1)) + """ + + # Identify which date values are passed. + dayCol = None + monthCol = None + yearCol = None + + for kk, vv in date_mapper.items(): + if vv and vv["date_type"] == "day": + dayCol = kk + elif vv and vv["date_type"] == "month": + monthCol = kk + elif vv and vv["date_type"] == "year": + yearCol = kk + + # For missing date values, add a column to the dataframe with the default + # value, then assign that to the day/month/year var. If the dataframe has + # the date value, assign day/month/year to it after casting as a str. + if dayCol: + day = df[dayCol].astype(str) + else: + df.loc[:, "day_generate_timestamp_column"] = "1" + day = df["day_generate_timestamp_column"] + + if monthCol: + month = df[monthCol].astype(str) + else: + df.loc[:, "month_generate_timestamp_column"] = "1" + month = df["month_generate_timestamp_column"] + + if yearCol: + year = df[yearCol].astype(str) + else: + df.loc[:, "year_generate_timestamp_column"] = "01" + year = df["year_generate_timestamp_column"] + + # Add the new column + df.loc[:, column_name] = month + "/" + day + "/" + year + + # Delete the temporary columns + if not dayCol: + del df["day_generate_timestamp_column"] + + if not monthCol: + del df["month_generate_timestamp_column"] + + if not yearCol: + del df["year_generate_timestamp_column"] + + return df + + +def generate_column_name(field_list: list) -> str: + """ + Description + ----------- + Contatenate a list of column fields into a single column name. + + Parameters + ---------- + field_list: list[str] of column names + + Returns + ------- + str: new column name + + """ + return "".join(sorted(field_list)) + + +def generate_timestamp_format(date_mapper: dict) -> str: + """ + Description + ----------- + Generates a the time format for day,month,year dates based on each's + specified time_format. + + Parameters + ---------- + date_mapper: dict + a dictionary for the schema mapping (JSON) for the dataframe filtered + for "date_type" equal to Day, Month, or Year. + + Output + ------ + e.g. "%m/%d/%Y" + """ + + day = "%d" + month = "%m" + year = "%y" + + for kk, vv in date_mapper.items(): + if vv["date_type"] == "day": + day = vv["time_format"] + elif vv["date_type"] == "month": + month = vv["time_format"] + elif vv["date_type"] == "year": + year = vv["time_format"] + + return str.format("{}/{}/{}", month, day, year) + + +def build_date_qualifies_field(qualified_col_dict: dict, assoc_fields: list) -> str: + """ + Description + ----------- + Handle edge case of each date field in assoc_fields qualifying the same + column e.g. day/month/year are associated and qualify a field. In this + case, the new_column_name. + + if assoc_fields is found as a value in qualified_col_dict, return the key + + Parameters + ---------- + qualified_col_dict: dict + {'pop': ['month_column', 'day_column', 'year_column']} + + assoc_fields: list + ['month_column', 'day_column', 'year_column'] + + """ + for k, v in qualified_col_dict.items(): + if v == assoc_fields: + return k + + return None + + +def get_iso_country_dict(iso_list: list) -> dict: + """ + Description + ----------- + iso2 or iso3 is used as primary_geo and therefore the country column. + Load the custom iso lookup table and return a dictionary of the iso codes + as keys and the country names as values. Assume all list items are the same + iso type. + + Parameters + ---------- + iso_list: + list of iso2 or iso3 codes + + Returns + ------- + dict: + key: iso code; value: country name + """ + + dct = {} + if iso_list: + iso_df = pandas.DataFrame + try: + # The necessary code to load from pkg doesn't currently work in VS + # Code Debug, so wrap in try/except. + # iso_df = pandas.read_csv(pkg_resources.resource_stream(__name__, 'data/iso_lookup.csv')) + with pkg_resources.resource_stream(__name__, "data/iso_lookup.csv") as f: + iso_df = pandas.read_csv(f) + # path = Path(__file__).parent / "data/iso_lookup.csv" + # iso_df = pandas.read_csv(path) + except: + # Local VS Code load. + path = Path(__file__).parent / "data/iso_lookup.csv" + iso_df = pandas.read_csv(path) + + if iso_df.empty: + return dct + + if len(iso_list[0]) == 2: + for iso in iso_list: + if iso in iso_df["iso2"].values: + dct[iso] = iso_df.loc[iso_df["iso2"] == iso]["country"].item() + else: + for iso in iso_list: + if iso in iso_df["iso3"].values: + dct[iso] = iso_df.loc[iso_df["iso3"] == iso]["country"].item() + + return dct + + +def geocode( + admin: str, + df: pandas.DataFrame, + x: str = "longitude", + y: str = "latitude", + gadm: gpd.GeoDataFrame = None, + df_geocode: pandas.DataFrame = pandas.DataFrame(), +) -> pandas.DataFrame: + """ + Description + ----------- + Takes a dataframe containing coordinate data and geocodes it to GADM (https://gadm.org/) + + GEOCODES to ADMIN 0, 1, 2 OR 3 LEVEL + + Parameters + ---------- + admin: str + the level to geocode to. 'admin0' to 'admin3' + df: pandas.DataFrame + a pandas dataframe containing point data + x: str, default 'longitude' + the name of the column containing longitude information + y: str, default 'latitude' + the name of the column containing latitude data + gadm: gpd.GeoDataFrame, default None + optional specification of a GeoDataFrame of GADM shapes of the appropriate + level (admin2/3) for geocoding + df_geocode: pandas.DataFrame, default pandas.DataFrame() + cached lat/long geocode library + + Examples + -------- + Geocoding a dataframe with columns named 'lat' and 'lon' + + >>> df = geocode(df, x='lon', y='lat') + + """ + + flag = speedups.available + if flag == True: + speedups.enable() + + cdir = os.path.expanduser("~") + download_data_folder = f"{cdir}/mixmasta_data" + + # Only load GADM if it wasn't explicitly passed to the function. + if gadm is not None: + logging.info("GADM geo dataframe has been provided.") + else: + logging.info("GADM has not been provided; loading now.") + + if admin in ["admin0", "country"]: + gadm_fn = f"gadm36_2.feather" + gadmDir = f"{download_data_folder}/{gadm_fn}" + gadm = gf.from_geofeather(gadmDir) + gadm["country"] = gadm["NAME_0"] + gadm = gadm[["geometry", "country"]] + + elif admin == "admin1": + gadm_fn = f"gadm36_2.feather" + gadmDir = f"{download_data_folder}/{gadm_fn}" + gadm = gf.from_geofeather(gadmDir) + gadm["country"] = gadm["NAME_0"] + # gadm["state"] = gadm["NAME_1"] + gadm["admin1"] = gadm["NAME_1"] + # gadm = gadm[["geometry", "country", "state", "admin1"]] + gadm = gadm[["geometry", "country", "admin1"]] + + elif admin == "admin2": + gadm_fn = f"gadm36_2.feather" + gadmDir = f"{download_data_folder}/{gadm_fn}" + gadm = gf.from_geofeather(gadmDir) + gadm["country"] = gadm["NAME_0"] + # gadm["state"] = gadm["NAME_1"] + gadm["admin1"] = gadm["NAME_1"] + gadm["admin2"] = gadm["NAME_2"] + # gadm = gadm[["geometry", "country", "state", "admin1", "admin2"]] + gadm = gadm[["geometry", "country", "admin1", "admin2"]] + + elif admin == "admin3": + gadm_fn = f"gadm36_3.feather" + gadmDir = f"{download_data_folder}/{gadm_fn}" + gadm = gf.from_geofeather(gadmDir) + gadm["country"] = gadm["NAME_0"] + # gadm["state"] = gadm["NAME_1"] + gadm["admin1"] = gadm["NAME_1"] + gadm["admin2"] = gadm["NAME_2"] + gadm["admin3"] = gadm["NAME_3"] + # gadm = gadm[["geometry", "country", "state", "admin1", "admin2", "admin3"]] + gadm = gadm[["geometry", "country", "admin1", "admin2", "admin3"]] + + start_time = timeit.default_timer() + + # 1) Drop x,y duplicates from data frame. + df_drop_dup_geo = df[[x, y]].drop_duplicates(subset=[x, y]) + + # 2) Get x,y not in df_geocode. + if not df_geocode.empty and not df_drop_dup_geo.empty: + df_drop_dup_geo = df_drop_dup_geo.merge( + df_geocode, on=[x, y], how="left", indicator=True + ) + df_drop_dup_geo = df_drop_dup_geo[df_drop_dup_geo["_merge"] == "left_only"] + df_drop_dup_geo = df_drop_dup_geo[[x, y]] + + if not df_drop_dup_geo.empty: + # dr_drop_dup_geo contains x,y not in df_geocode; so, these need to be + # geocoded and added to the df_geocode library. + + # 3) Apply Point() to create the geometry col. + df_drop_dup_geo.loc[:, "geometry"] = df_drop_dup_geo.apply( + lambda row: Point(row[x], row[y]), axis=1 + ) + + # 4) Sjoin unique geometries with GADM. + gdf = gpd.GeoDataFrame(df_drop_dup_geo) + + # Spatial merge on GADM to obtain admin areas. + gdf = gpd.sjoin( + gdf, + gadm, + how="left", + op="within", + lsuffix="mixmasta_left", + rsuffix="mixmasta_geocoded", + ) + del gdf["geometry"] + del gdf["index_mixmasta_geocoded"] + + # 5) Add the new geocoding to the df_geocode lat/long geocode library. + if not df_geocode.empty: + df_geocode = df_geocode.append(gdf) + else: + df_geocode = gdf + + # 6) Merge df and df_geocode on x,y + gdf = df.merge(df_geocode, how="left", on=[x, y]) + + return pandas.DataFrame(gdf), df_geocode From 4ccd2624755829ab8b488e10f8b2e887e49d64fe Mon Sep 17 00:00:00 2001 From: Powell Date: Fri, 6 Jan 2023 14:39:29 -0500 Subject: [PATCH 02/15] General clean up continued, Docker improved, tests running in docker container --- Dockerfile | 44 +++-- docker-compose.yml | 8 + mixmasta/cli.py | 6 +- mixmasta/constants.py | 17 ++ mixmasta/mixmasta.py | 217 +---------------------- mixmasta/normalizer.py | 215 ++++++++++++++++++++--- tests/test_mixmasta.py | 384 ++++++++++++++++++++++++++++------------- 7 files changed, 498 insertions(+), 393 deletions(-) create mode 100644 docker-compose.yml create mode 100644 mixmasta/constants.py diff --git a/Dockerfile b/Dockerfile index 90e135c..7be9f00 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,35 +1,33 @@ FROM ubuntu:20.04 -RUN apt-get update && \ - apt-get -y install sudo -RUN apt-get install -y software-properties-common -RUN apt-get update +RUN apt-get update && apt-get install -y \ + software-properties-common \ + sudo \ + unzip \ + wget -RUN apt-get install wget unzip -RUN wget https://jataware-world-modelers.s3.amazonaws.com/gadm/gadm36_2.feather.zip -RUN wget https://jataware-world-modelers.s3.amazonaws.com/gadm/gadm36_3.feather.zip -RUN mkdir ~/mixmasta_data && \ +RUN wget https://jataware-world-modelers.s3.amazonaws.com/gadm/gadm36_2.feather.zip && \ + wget https://jataware-world-modelers.s3.amazonaws.com/gadm/gadm36_3.feather.zip && \ + mkdir ~/mixmasta_data && \ unzip gadm36_2.feather.zip -d ~/mixmasta_data/ && \ unzip gadm36_3.feather.zip -d ~/mixmasta_data/ && \ rm gadm36_?.feather.zip -RUN add-apt-repository ppa:ubuntugis/ubuntugis-unstable -RUN apt-get update -RUN apt-get install -y python3.8-dev -RUN apt-get install -y gdal-bin -RUN apt-get install -y libgdal-dev -RUN apt-get install -y python3-pip -RUN apt install -y python3-rtree +RUN add-apt-repository ppa:ubuntugis/ubuntugis-unstable && apt-get update && \ + apt-get install -y \ + gdal-bin \ + libgdal-dev \ + python3-pip \ + python3-rtree \ + python3.8-dev && \ + apt-get -y autoremove && apt-get clean autoclean ENV CPLUS_INCLUDE_PATH=/usr/include/gdal ENV C_INCLUDE_PATH=/usr/include/gdal -RUN pip3 install numpy==1.22 +COPY . /mixmasta -WORKDIR / -COPY requirements.txt /requirements.txt -RUN pip3 install -r requirements.txt -COPY . / -RUN python3 setup.py install -#RUN mixmasta download +WORKDIR /mixmasta -# ENTRYPOINT ["mixmasta"] +RUN pip3 install numpy==1.22 && \ + pip3 install -r requirements.txt && \ + python3 setup.py install diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a39addc --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,8 @@ +version: "3.9" +services: + mixmasta: + container_name: mixmasta-container + command: tail -f /dev/null #Keeps container up on start, no other function + build: ./ + volumes: + - $PWD:/mixmasta diff --git a/mixmasta/cli.py b/mixmasta/cli.py index 786fe1d..cc27747 100644 --- a/mixmasta/cli.py +++ b/mixmasta/cli.py @@ -6,11 +6,9 @@ import pandas as pd from .download import download_and_clean -from .mixmasta import geocode, process, normalizer, optimize_df_types, mixdata +from .mixmasta import normalizer, optimize_df_types, mixdata from .file_processor import netcdf2df, raster2df - -# from download import download_and_clean -# from mixmasta import geocode, netcdf2df, process, raster2df, normalizer, optimize_df_types, mixdata +from .normalizer import geocode from glob import glob import numpy as np diff --git a/mixmasta/constants.py b/mixmasta/constants.py new file mode 100644 index 0000000..352dc20 --- /dev/null +++ b/mixmasta/constants.py @@ -0,0 +1,17 @@ +# Constants used in multiple files +COL_ORDER = [ + "timestamp", + "country", + "admin1", + "admin2", + "admin3", + "lat", + "lng", + "feature", + "value", +] + +GEO_TYPE_COUNTRY = "country" +GEO_TYPE_ADMIN1 = "state/territory" +GEO_TYPE_ADMIN2 = "county/district" +GEO_TYPE_ADMIN3 = "municipality/town" diff --git a/mixmasta/mixmasta.py b/mixmasta/mixmasta.py index 634c535..1ae8897 100644 --- a/mixmasta/mixmasta.py +++ b/mixmasta/mixmasta.py @@ -1,54 +1,17 @@ """Main module.""" import json import logging -import click import os import sys -from datetime import datetime -from typing import List import geofeather as gf -import geopandas as gpd import numpy as np import pandas as pd -from pandas.core.frame import DataFrame -import requests -import xarray as xr -from osgeo import gdal, gdalconst -from shapely import speedups - - -import fuzzywuzzy -from fuzzywuzzy import fuzz -from fuzzywuzzy import process -import timeit +from . import constants from .file_processor import process_file_by_filetype from .normalizer import normalizer -# from .spacetag_schema import SpaceModel - -# TODO Remove this into another file. -# Constants -COL_ORDER = [ - "timestamp", - "country", - "admin1", - "admin2", - "admin3", - "lat", - "lng", - "feature", - "value", -] - -import click - -GEO_TYPE_COUNTRY = "country" -GEO_TYPE_ADMIN1 = "state/territory" -GEO_TYPE_ADMIN2 = "county/district" -GEO_TYPE_ADMIN3 = "municipality/town" - if not sys.warnoptions: import warnings @@ -57,182 +20,6 @@ logger = logging.getLogger(__name__) -def audit_renamed_col_dict(dct: dict) -> dict: - """ - Description - ----------- - Handle edge cases where a col could be renamed back to itself. - example: no primary_geo, but country is present. Because it is a protected - col name, it would be renamed country_non_primary. Later, it would be set - as primary_geo country, and the pair added to renamed_col_dict again: - {'country_non_primary' : ['country'], "country": ['country_non_primary'] } - - Parameters - ---------- - dct: dict - renamed_col_dict of key: new column name, value: list of old columns - - Output - ------ - dict: - The modified parameter dict. - """ - remove_these = set() - for k, v in dct.items(): - vstr = "".join(v) - if vstr in dct.keys() and [k] in dct.values(): - remove_these.add(vstr) - remove_these.add(k) - - for k in remove_these: - dct.pop(k, None) - - return dct - - -def match_geo_names( - admin: str, - df: pd.DataFrame, - resolve_to_gadm_geotypes: list, - gadm: gpd.GeoDataFrame = None, -) -> pd.DataFrame: - """ - Assumption - ---------- - Country was selected by drop-down on file submission, column "country" - is present in the data frame, and lng/lat is not being used for geocoding. - - Parameters - ---------- - admin: str - the level to geocode to. Either 'admin2' or 'admin3' - df: pandas.DataFrame - the uploaded dataframe - resolve_to_gadm_geotypes: - list of geotypes marked resolve_to_gadm = True e.g. ["admin1", "country"] - gadm: gpd.GeoDataFrame, default None - optional specification of a GeoDataFrame of GADM shapes of the appropriate - level (admin2/3) for geocoding - - Result - ------ - A pandas.Dataframe produced by modifying the parameter df. - - """ - print("geocoding ...") - flag = speedups.available - if flag == True: - speedups.enable() - - cdir = os.path.expanduser("~") - download_data_folder = f"{cdir}/mixmasta_data" - - # only load GADM if it wasn't explicitly passed to the function. - if gadm is not None: - # logging.info("GADM geo dataframe has been provided.") - pass - else: - logging.info("GADM has not been provided; loading now.") - - if admin == "admin2": - gadm_fn = f"gadm36_2.feather" - else: - gadm_fn = f"gadm36_3.feather" - - gadmDir = f"{download_data_folder}/{gadm_fn}" - gadm = gf.from_geofeather(gadmDir) - - gadm["country"] = gadm["NAME_0"] - gadm["state"] = gadm["NAME_1"] - gadm["admin1"] = gadm["NAME_1"] - gadm["admin2"] = gadm["NAME_2"] - - if admin == "admin2": - gadm = gadm[["country", "state", "admin1", "admin2"]] - else: - gadm["admin3"] = gadm["NAME_3"] - gadm = gadm[["country", "state", "admin1", "admin2", "admin3"]] - - # Filter GADM for countries in df. - countries = df["country"].unique() - - # Correct country names. - if GEO_TYPE_COUNTRY in resolve_to_gadm_geotypes: - gadm_country_list = gadm["country"].unique() - unknowns = df[~df.country.isin(gadm_country_list)].country.tolist() - for unk in unknowns: - try: - match = fuzzywuzzy.process.extractOne( - unk, gadm_country_list, scorer=fuzz.partial_ratio - ) - except Exception as e: - match = None - logging.error(f"Error in match_geo_names: {e}") - if match != None: - df.loc[df.country == unk, "country"] = match[0] - - # Filter GADM dicitonary for only those countries (ie. speed up) - gadm = gadm[gadm["country"].isin(countries)] - - # Loop by country using gadm dict filtered for that country. - for c in countries: - # The following ignores admin1 / admin2 pairs; it only cares if those - # values exist for the appropriate country. - - # Get list of admin1 values in df but not in gadm. Reduce list for country. - if GEO_TYPE_ADMIN1 in resolve_to_gadm_geotypes: - admin1_list = gadm[gadm.country == c]["admin1"].unique() - if admin1_list is not None and all(admin1_list) and "admin1" in df: - unknowns = df[ - (df.country == c) & ~df.admin1.isin(admin1_list) - ].admin1.tolist() - unknowns = [ - x for x in unknowns if pd.notnull(x) and x.strip() - ] # remove Nan - for unk in unknowns: - match = fuzzywuzzy.process.extractOne( - unk, admin1_list, scorer=fuzz.partial_ratio - ) - if match != None: - df.loc[df.admin1 == unk, "admin1"] = match[0] - - # Get list of admin2 values in df but not in gadm. Reduce list for country. - if GEO_TYPE_ADMIN2 in resolve_to_gadm_geotypes: - admin2_list = gadm[gadm.country == c]["admin2"].unique() - if admin2_list is not None and all(admin2_list) and "admin2" in df: - unknowns = df[ - (df.country == c) & ~df.admin2.isin(admin2_list) - ].admin2.tolist() - unknowns = [ - x for x in unknowns if pd.notnull(x) and x.strip() - ] # remove Nan - for unk in unknowns: - match = fuzzywuzzy.process.extractOne( - unk, admin2_list, scorer=fuzz.partial_ratio - ) - if match != None: - df.loc[df.admin2 == unk, "admin2"] = match[0] - - if admin == "admin3" and GEO_TYPE_ADMIN3 in resolve_to_gadm_geotypes: - # Get list of admin3 values in df but not in gadm. Reduce list for country. - admin3_list = gadm[gadm.country == c]["admin3"].unique() - if admin3_list is not None and all(admin3_list) and "admin3" in df: - unknowns = df[ - (df.country == c) & ~df.admin3.isin(admin3_list) - ].admin3.tolist() - unknowns = [ - x for x in unknowns if pd.notnull(x) and x.strip() - ] # remove Nan - for unk in unknowns: - match = fuzzywuzzy.process.extractOne( - unk, admin3_list, scorer=fuzz.partial_ratio - ) - if match != None: - df.loc[df.admin3 == unk, "admin3"] = match[0] - - return df - - def optimize_df_types(df: pd.DataFrame): """ Pandas will upcast essentially everything. This will use the built-in @@ -314,7 +101,7 @@ def process( if write_output: # If any qualify columns were added, the feature_type must be enforced # here because pandas will have cast strings as ints etc. - qualify_cols = set(norm.columns).difference(set(COL_ORDER)) + qualify_cols = set(norm.columns).difference(set(constants.COL_ORDER)) for col in qualify_cols: for feature_dict in mapper["feature"]: if ( diff --git a/mixmasta/normalizer.py b/mixmasta/normalizer.py index aadfb0e..d488567 100644 --- a/mixmasta/normalizer.py +++ b/mixmasta/normalizer.py @@ -14,29 +14,14 @@ import timeit from distutils.util import strtobool +from fuzzywuzzy import fuzz +from fuzzywuzzy import process as fuzzyprocess import geofeather as gf from pathlib import Path from shapely import speedups from shapely.geometry import Point -# TODO remove to another file. -# Constants -COL_ORDER = [ - "timestamp", - "country", - "admin1", - "admin2", - "admin3", - "lat", - "lng", - "feature", - "value", -] - -GEO_TYPE_COUNTRY = "country" -GEO_TYPE_ADMIN1 = "state/territory" -GEO_TYPE_ADMIN2 = "county/district" -GEO_TYPE_ADMIN3 = "municipality/town" +from . import constants def normalizer( @@ -90,7 +75,7 @@ def normalizer( -------- >>> df_norm = normalizer(df, mapper, 'admin3') """ - col_order = COL_ORDER.copy() + col_order = constants.COL_ORDER.copy() required_cols = [ "timestamp", @@ -354,20 +339,20 @@ def normalizer( df["lng"] = longs df["lat"] = lats del df[kk] - elif geo_dict["geo_type"] == GEO_TYPE_COUNTRY and kk != "country": + elif geo_dict["geo_type"] == constants.GEO_TYPE_COUNTRY and kk != "country": # force the country column to be named country staple_col_name = "country" df.rename(columns={kk: staple_col_name}, inplace=True) # renamed_col_dict[staple_col_name] = [kk] # 7/2/2021 do not include primary cols - elif geo_dict["geo_type"] == GEO_TYPE_ADMIN1 and kk != "admin1": + elif geo_dict["geo_type"] == constants.GEO_TYPE_ADMIN1 and kk != "admin1": # force the country column to be named country staple_col_name = "admin1" df.rename(columns={kk: staple_col_name}, inplace=True) - elif geo_dict["geo_type"] == GEO_TYPE_ADMIN2 and kk != "admin2": + elif geo_dict["geo_type"] == constants.GEO_TYPE_ADMIN2 and kk != "admin2": # force the country column to be named country staple_col_name = "admin2" df.rename(columns={kk: staple_col_name}, inplace=True) - elif geo_dict["geo_type"] == GEO_TYPE_ADMIN3 and kk != "admin2": + elif geo_dict["geo_type"] == constants.GEO_TYPE_ADMIN3 and kk != "admin2": # force the country column to be named country staple_col_name = "admin3" df.rename(columns={kk: staple_col_name}, inplace=True) @@ -405,19 +390,19 @@ def normalizer( # in the event there is no primary geo # otherwise they are features and we geocode lat/lng if len(primary_geo_cols) == 0: - if geo_dict["geo_type"] == GEO_TYPE_COUNTRY: + if geo_dict["geo_type"] == constants.GEO_TYPE_COUNTRY: df["country"] = df[kk] renamed_col_dict["country"] = [kk] continue - if geo_dict["geo_type"] == GEO_TYPE_ADMIN1: + if geo_dict["geo_type"] == constants.GEO_TYPE_ADMIN1: df["admin1"] = df[kk] renamed_col_dict["admin1"] = [kk] continue - if geo_dict["geo_type"] == GEO_TYPE_ADMIN2: + if geo_dict["geo_type"] == constants.GEO_TYPE_ADMIN2: df["admin2"] = df[kk] renamed_col_dict["admin2"] = [kk] continue - if geo_dict["geo_type"] == GEO_TYPE_ADMIN3: + if geo_dict["geo_type"] == constants.GEO_TYPE_ADMIN3: df["admin3"] = df[kk] renamed_col_dict["admin3"] = [kk] continue @@ -1085,3 +1070,179 @@ def geocode( gdf = df.merge(df_geocode, how="left", on=[x, y]) return pandas.DataFrame(gdf), df_geocode + + +def match_geo_names( + admin: str, + df: pandas.DataFrame, + resolve_to_gadm_geotypes: list, + gadm: gpd.GeoDataFrame = None, +) -> pandas.DataFrame: + """ + Assumption + ---------- + Country was selected by drop-down on file submission, column "country" + is present in the data frame, and lng/lat is not being used for geocoding. + + Parameters + ---------- + admin: str + the level to geocode to. Either 'admin2' or 'admin3' + df: pandas.DataFrame + the uploaded dataframe + resolve_to_gadm_geotypes: + list of geotypes marked resolve_to_gadm = True e.g. ["admin1", "country"] + gadm: gpd.GeoDataFrame, default None + optional specification of a GeoDataFrame of GADM shapes of the appropriate + level (admin2/3) for geocoding + + Result + ------ + A pandas.Dataframe produced by modifying the parameter df. + + """ + print("geocoding ...") + flag = speedups.available + if flag == True: + speedups.enable() + + cdir = os.path.expanduser("~") + download_data_folder = f"{cdir}/mixmasta_data" + + # only load GADM if it wasn't explicitly passed to the function. + if gadm is not None: + # logging.info("GADM geo dataframe has been provided.") + pass + else: + logging.info("GADM has not been provided; loading now.") + + if admin == "admin2": + gadm_fn = f"gadm36_2.feather" + else: + gadm_fn = f"gadm36_3.feather" + + gadmDir = f"{download_data_folder}/{gadm_fn}" + gadm = gf.from_geofeather(gadmDir) + + gadm["country"] = gadm["NAME_0"] + gadm["state"] = gadm["NAME_1"] + gadm["admin1"] = gadm["NAME_1"] + gadm["admin2"] = gadm["NAME_2"] + + if admin == "admin2": + gadm = gadm[["country", "state", "admin1", "admin2"]] + else: + gadm["admin3"] = gadm["NAME_3"] + gadm = gadm[["country", "state", "admin1", "admin2", "admin3"]] + + # Filter GADM for countries in df. + countries = df["country"].unique() + + # Correct country names. + if constants.GEO_TYPE_COUNTRY in resolve_to_gadm_geotypes: + gadm_country_list = gadm["country"].unique() + unknowns = df[~df.country.isin(gadm_country_list)].country.tolist() + for unk in unknowns: + try: + match = fuzzyprocess.extractOne( + unk, gadm_country_list, scorer=fuzz.partial_ratio + ) + except Exception as e: + match = None + logging.error(f"Error in match_geo_names: {e}") + if match != None: + df.loc[df.country == unk, "country"] = match[0] + + # Filter GADM dicitonary for only those countries (ie. speed up) + gadm = gadm[gadm["country"].isin(countries)] + + # Loop by country using gadm dict filtered for that country. + for c in countries: + # The following ignores admin1 / admin2 pairs; it only cares if those + # values exist for the appropriate country. + + # Get list of admin1 values in df but not in gadm. Reduce list for country. + if constants.GEO_TYPE_ADMIN1 in resolve_to_gadm_geotypes: + admin1_list = gadm[gadm.country == c]["admin1"].unique() + if admin1_list is not None and all(admin1_list) and "admin1" in df: + unknowns = df[ + (df.country == c) & ~df.admin1.isin(admin1_list) + ].admin1.tolist() + unknowns = [ + x for x in unknowns if pandas.notnull(x) and x.strip() + ] # remove Nan + for unk in unknowns: + match = fuzzyprocess.extractOne( + unk, admin1_list, scorer=fuzz.partial_ratio + ) + if match != None: + df.loc[df.admin1 == unk, "admin1"] = match[0] + + # Get list of admin2 values in df but not in gadm. Reduce list for country. + if constants.GEO_TYPE_ADMIN2 in resolve_to_gadm_geotypes: + admin2_list = gadm[gadm.country == c]["admin2"].unique() + if admin2_list is not None and all(admin2_list) and "admin2" in df: + unknowns = df[ + (df.country == c) & ~df.admin2.isin(admin2_list) + ].admin2.tolist() + unknowns = [ + x for x in unknowns if pandas.notnull(x) and x.strip() + ] # remove Nan + for unk in unknowns: + match = fuzzyprocess.extractOne( + unk, admin2_list, scorer=fuzz.partial_ratio + ) + if match != None: + df.loc[df.admin2 == unk, "admin2"] = match[0] + + if admin == "admin3" and constants.GEO_TYPE_ADMIN3 in resolve_to_gadm_geotypes: + # Get list of admin3 values in df but not in gadm. Reduce list for country. + admin3_list = gadm[gadm.country == c]["admin3"].unique() + if admin3_list is not None and all(admin3_list) and "admin3" in df: + unknowns = df[ + (df.country == c) & ~df.admin3.isin(admin3_list) + ].admin3.tolist() + unknowns = [ + x for x in unknowns if pandas.notnull(x) and x.strip() + ] # remove Nan + for unk in unknowns: + match = fuzzyprocess.extractOne( + unk, admin3_list, scorer=fuzz.partial_ratio + ) + if match != None: + df.loc[df.admin3 == unk, "admin3"] = match[0] + + return df + + +def audit_renamed_col_dict(dct: dict) -> dict: + """ + Description + ----------- + Handle edge cases where a col could be renamed back to itself. + example: no primary_geo, but country is present. Because it is a protected + col name, it would be renamed country_non_primary. Later, it would be set + as primary_geo country, and the pair added to renamed_col_dict again: + {'country_non_primary' : ['country'], "country": ['country_non_primary'] } + + Parameters + ---------- + dct: dict + renamed_col_dict of key: new column name, value: list of old columns + + Output + ------ + dict: + The modified parameter dict. + """ + remove_these = set() + for k, v in dct.items(): + vstr = "".join(v) + if vstr in dct.keys() and [k] in dct.values(): + remove_these.add(vstr) + remove_these.add(k) + + for k in remove_these: + dct.pop(k, None) + + return dct diff --git a/tests/test_mixmasta.py b/tests/test_mixmasta.py index 725f407..19450cb 100644 --- a/tests/test_mixmasta.py +++ b/tests/test_mixmasta.py @@ -5,24 +5,24 @@ import unittest import warnings - -from click.testing import CliRunner import subprocess import logging import json -from mixmasta import cli, mixmasta -from pandas.util.testing import assert_frame_equal, assert_dict_equal -import pandas as pd import gc import os -if os.name == 'nt': - sep = '\\' +from mixmasta import mixmasta +from pandas.util.testing import assert_frame_equal, assert_dict_equal +import pandas as pd + +if os.name == "nt": + sep = "\\" else: - sep = '/' + sep = "/" logger = logging.getLogger(__name__) + class TestMixmaster(unittest.TestCase): """Tests for `mixmasta` package.""" @@ -30,7 +30,7 @@ def setUp(self): """Set up test fixtures, if any.""" # turn off warnings unless refactoring. - warnings.simplefilter('ignore') + warnings.simplefilter("ignore") def tearDown(self): """Tear down test fixtures, if any.""" @@ -42,7 +42,7 @@ def tearDown(self): pass try: - os.remove(f"outputs{sep}unittests_str.parquet.gzip") + os.remove(f"outputs{sep}unittests_str.parquet.gzip") except: pass @@ -52,24 +52,34 @@ def test_001_process(self): # Define mixmasta inputs: mp = f"inputs{sep}test1_input.json" fp = f"inputs{sep}test1_input.csv" - geo = 'admin2' + geo = "admin2" outf = f"outputs{sep}unittests" # Process: df, dct = mixmasta.process(fp, mp, geo, outf) # Load expected output: - output_df = pd.read_csv(f'outputs{sep}test1_output.csv', index_col=False) + output_df = pd.read_csv(f"outputs{sep}test1_output.csv", index_col=False) output_df = mixmasta.optimize_df_types(output_df) - with open(f'outputs{sep}test1_dict.json') as f: + with open(f"outputs{sep}test1_dict.json") as f: output_dict = json.loads(f.read()) # Sort both data frames and reindex for comparison,. - cols = ['timestamp','country','admin1','admin2','admin3','lat','lng','feature','value'] + cols = [ + "timestamp", + "country", + "admin1", + "admin2", + "admin3", + "lat", + "lng", + "feature", + "value", + ] df.sort_values(by=cols, inplace=True) output_df.sort_values(by=cols, inplace=True) df.reset_index(drop=True, inplace=True) - output_df.reset_index(drop =True, inplace=True) + output_df.reset_index(drop=True, inplace=True) # Assertions assert_frame_equal(df, output_df) @@ -83,122 +93,159 @@ def test_002_process(self): """ # Define mixmasta inputs: - mp = f'inputs{sep}test2_assetwealth_input.json' - fp = f'inputs{sep}test2_assetwealth_input.tif' - geo = 'admin2' - outf = f'outputs{sep}unittests' + mp = f"inputs{sep}test2_assetwealth_input.json" + fp = f"inputs{sep}test2_assetwealth_input.tif" + geo = "admin2" + outf = f"outputs{sep}unittests" # Process: df, dct = mixmasta.process(fp, mp, geo, outf) - #categories = df.select_dtypes(include=['category']).columns.tolist() - df['value'] = df['value'].astype('str') + # categories = df.select_dtypes(include=['category']).columns.tolist() + df["value"] = df["value"].astype("str") # Load expected output: - output_df = pd.read_csv(f'outputs{sep}test2_assetwealth_output.csv', index_col=False) + output_df = pd.read_csv( + f"outputs{sep}test2_assetwealth_output.csv", index_col=False + ) - with open(f'outputs{sep}test2_assetwealth_dict.json') as f: + with open(f"outputs{sep}test2_assetwealth_dict.json") as f: output_dict = json.loads(f.read()) # Sort both data frames and reindex for comparison,. - cols = ['timestamp','country','admin1','admin2','admin3','lat','lng','feature','value'] + cols = [ + "timestamp", + "country", + "admin1", + "admin2", + "admin3", + "lat", + "lng", + "feature", + "value", + ] df = df[cols] output_df = output_df[cols] # Optimize datatypes for output_df. - floats = output_df.select_dtypes(include=['float64']).columns.tolist() - output_df[floats] = output_df[floats].apply(pd.to_numeric, downcast='float') + floats = output_df.select_dtypes(include=["float64"]).columns.tolist() + output_df[floats] = output_df[floats].apply(pd.to_numeric, downcast="float") - ints = output_df.select_dtypes(include=['int64']).columns.tolist() - output_df[ints] = output_df[ints].apply(pd.to_numeric, downcast='integer') + ints = output_df.select_dtypes(include=["int64"]).columns.tolist() + output_df[ints] = output_df[ints].apply(pd.to_numeric, downcast="integer") # Standardize value and feature columns to str for comparison. - df['value'] = df['value'].astype('str') - df['feature'] = df['feature'].astype('str') - output_df['value'] = output_df['value'].astype('str') - output_df['feature'] = output_df['feature'].astype('str') + df["value"] = df["value"].astype("str") + df["feature"] = df["feature"].astype("str") + output_df["value"] = output_df["value"].astype("str") + output_df["feature"] = output_df["feature"].astype("str") # Sort and reindex. df.sort_values(by=cols, inplace=True) df.reset_index(drop=True, inplace=True) output_df.sort_values(by=cols, inplace=True) - output_df.reset_index(drop =True, inplace=True) + output_df.reset_index(drop=True, inplace=True) # Assertions - assert_frame_equal(df, output_df, check_categorical = False) + assert_frame_equal(df, output_df, check_categorical=False) assert_dict_equal(dct, output_dict) def test_003_process(self): """Test qualifies, lat/lng primary geo.""" # Define mixmasta inputs: - mp = f'inputs{sep}test3_qualifies.json' - fp = f'inputs{sep}test3_qualifies.csv' - geo = 'admin2' - outf = f'outputs{sep}unittests' + mp = f"inputs{sep}test3_qualifies.json" + fp = f"inputs{sep}test3_qualifies.csv" + geo = "admin2" + outf = f"outputs{sep}unittests" # Process: df, dct = mixmasta.process(fp, mp, geo, outf) # Load expected output: - output_df = pd.read_csv(f'outputs{sep}test3_qualifies_output.csv', index_col=False) + output_df = pd.read_csv( + f"outputs{sep}test3_qualifies_output.csv", index_col=False + ) output_df = mixmasta.optimize_df_types(output_df) - with open(f'outputs{sep}test3_qualifies_dict.json') as f: + with open(f"outputs{sep}test3_qualifies_dict.json") as f: output_dict = json.loads(f.read()) # Sort both data frames and reindex for comparison,. - cols = ['timestamp','country','admin1','admin2','admin3','lat','lng','feature','value'] + cols = [ + "timestamp", + "country", + "admin1", + "admin2", + "admin3", + "lat", + "lng", + "feature", + "value", + ] df.sort_values(by=cols, inplace=True) output_df.sort_values(by=cols, inplace=True) df.reset_index(drop=True, inplace=True) - output_df.reset_index(drop =True, inplace=True) + output_df.reset_index(drop=True, inplace=True) # Make the datatypes the same for value/feature columns. - df['value'] = df['value'].astype('str') - df['feature'] = df['feature'].astype('str') - output_df['value'] = output_df['value'].astype('str') - output_df['feature'] = output_df['feature'].astype('str') + df["value"] = df["value"].astype("str") + df["feature"] = df["feature"].astype("str") + output_df["value"] = output_df["value"].astype("str") + output_df["feature"] = output_df["feature"].astype("str") # Assertions - assert_frame_equal(df, output_df, check_categorical = False) + assert_frame_equal(df, output_df, check_categorical=False) assert_dict_equal(dct, output_dict) def test_004_process(self): """Test .xlxs file, qualifies col with multi dtypes.""" # Define mixmasta inputs: - mp = f'inputs{sep}test4_rainfall_error.json' - fp = f'inputs{sep}test4_rainfall_error.xlsx' - geo = 'admin2' - outf = f'outputs{sep}unittests' + mp = f"inputs{sep}test4_rainfall_error.json" + fp = f"inputs{sep}test4_rainfall_error.xlsx" + geo = "admin2" + outf = f"outputs{sep}unittests" # Process: df, dct = mixmasta.process(fp, mp, geo, outf) # Load expected output: - output_df = pd.read_csv(f'outputs{sep}test4_rainfall_error_output.csv', index_col=False) + output_df = pd.read_csv( + f"outputs{sep}test4_rainfall_error_output.csv", index_col=False + ) output_df = mixmasta.optimize_df_types(output_df) - with open(f'outputs{sep}test4_rainfall_error_dict.json') as f: + with open(f"outputs{sep}test4_rainfall_error_dict.json") as f: output_dict = json.loads(f.read()) # Sort both data frames and reindex for comparison,. - cols = ['timestamp','country','admin1','admin2','admin3','lat','lng','feature','value','MainCause'] + cols = [ + "timestamp", + "country", + "admin1", + "admin2", + "admin3", + "lat", + "lng", + "feature", + "value", + "MainCause", + ] df.sort_values(by=cols, inplace=True) output_df.sort_values(by=cols, inplace=True) df.reset_index(drop=True, inplace=True) - output_df.reset_index(drop =True, inplace=True) + output_df.reset_index(drop=True, inplace=True) # Make the datatypes the same for value/feature and qualifying columns. - df['value'] = df['value'].astype('str') - df['feature'] = df['feature'].astype('str') - df['MainCause'] = df['MainCause'].astype('str') - output_df['value'] = output_df['value'].astype('str') - output_df['feature'] = output_df['feature'].astype('str') - output_df['MainCause'] = output_df['MainCause'].astype('str') + df["value"] = df["value"].astype("str") + df["feature"] = df["feature"].astype("str") + df["MainCause"] = df["MainCause"].astype("str") + output_df["value"] = output_df["value"].astype("str") + output_df["feature"] = output_df["feature"].astype("str") + output_df["MainCause"] = output_df["MainCause"].astype("str") # Assertions - assert_frame_equal(df, output_df, check_categorical = False) + assert_frame_equal(df, output_df, check_categorical=False) assert_dict_equal(dct, output_dict) def test_005__command_line_interface(self): @@ -215,26 +262,57 @@ def test_005__command_line_interface(self): """ # Confirm CLI --help is available. - result = subprocess.run(['mixmasta', '--help'], capture_output=True, encoding='utf-8') - self.assertIn('Processor for generating CauseMos compliant datasets.', result.stdout) - self.assertIn('--help Show this message and exit.', result.stdout) + result = subprocess.run( + ["mixmasta", "--help"], capture_output=True, encoding="utf-8" + ) + self.assertIn( + "Processor for generating CauseMos compliant datasets.", result.stdout + ) + self.assertIn("--help Show this message and exit.", result.stdout) self.assertEqual(result.returncode, 0) - self.assertIn('causemosify-multi Process multiple input files to generate a single',result.stdout) - + self.assertIn( + "causemosify-multi Process multiple input files to generate a single", + result.stdout, + ) # Confirm CLI causemosify-multi --help is available. - result = subprocess.run(['mixmasta', "causemosify-multi", "--help"], capture_output=True, encoding='utf-8') + result = subprocess.run( + ["mixmasta", "causemosify-multi", "--help"], + capture_output=True, + encoding="utf-8", + ) self.assertEqual(result.returncode, 0) - self.assertIn('Process multiple input files to generate a single CauseMos compliant',result.stdout) + self.assertIn( + "Process multiple input files to generate a single CauseMos compliant", + result.stdout, + ) # Run causemosify-multi - inputs = "--inputs=[{\"input_file\": \"inputs" + f"{sep}test1_input.csv\",\"mapper\": \"inputs{sep}test1_input.json\"" + "},{\"input_file\": \"" - inputs = inputs + f"inputs{sep}test3_qualifies.csv\",\"mapper\": \"inputs{sep}test3_qualifies.json\"" + "}]" - result = subprocess.run(['mixmasta', "causemosify-multi", inputs, "--geo=admin2", f"--output-file=outputs{sep}unittests"], capture_output=True, encoding='utf-8') - if (result.returncode != 0): + inputs = ( + '--inputs=[{"input_file": "inputs' + + f'{sep}test1_input.csv","mapper": "inputs{sep}test1_input.json"' + + '},{"input_file": "' + ) + inputs = ( + inputs + + f'inputs{sep}test3_qualifies.csv","mapper": "inputs{sep}test3_qualifies.json"' + + "}]" + ) + result = subprocess.run( + [ + "mixmasta", + "causemosify-multi", + inputs, + "--geo=admin2", + f"--output-file=outputs{sep}unittests", + ], + capture_output=True, + encoding="utf-8", + ) + if result.returncode != 0: print(result) self.assertEqual(result.returncode, 0) - + ## Compare parquet files. df1 = pd.read_parquet(f"outputs{sep}unittests.1.parquet.gzip") df2 = pd.read_parquet(f"outputs{sep}unittests.2.parquet.gzip") @@ -242,105 +320,149 @@ def test_005__command_line_interface(self): output_df = pd.read_parquet(f"outputs{sep}test5.parquet.gzip") # Sort both data frames and reindex for comparison,. - cols = ['timestamp','country','admin1','admin2','admin3','lat','lng','feature','value'] + cols = [ + "timestamp", + "country", + "admin1", + "admin2", + "admin3", + "lat", + "lng", + "feature", + "value", + ] df.sort_values(by=cols, inplace=True) output_df.sort_values(by=cols, inplace=True) df.reset_index(drop=True, inplace=True) - output_df.reset_index(drop =True, inplace=True) + output_df.reset_index(drop=True, inplace=True) logger.info(df.shape) logger.info(output_df.shape) # Assertion - assert_frame_equal(df, output_df, check_categorical = False) + assert_frame_equal(df, output_df, check_categorical=False) ## Compare str.parquet file. df = pd.read_parquet(f"outputs{sep}unittests_str.2.parquet.gzip") output_df = pd.read_parquet(f"outputs{sep}test5_str.parquet.gzip") # Sort both data frames and reindex for comparison,. - cols = ['timestamp','country','admin1','admin2','admin3','lat','lng','feature','value'] + cols = [ + "timestamp", + "country", + "admin1", + "admin2", + "admin3", + "lat", + "lng", + "feature", + "value", + ] df.sort_values(by=cols, inplace=True) output_df.sort_values(by=cols, inplace=True) df.reset_index(drop=True, inplace=True) - output_df.reset_index(drop =True, inplace=True) + output_df.reset_index(drop=True, inplace=True) # Assertion - assert_frame_equal(df, output_df, check_categorical = False) + assert_frame_equal(df, output_df, check_categorical=False) def test_006_process(self): """Test multi primary_geo, resolve_to_gadm""" # Define mixmasta inputs: - mp = f'inputs{sep}test6_hoa_conflict_input.json' - fp = f'inputs{sep}test6_hoa_conflict_input.csv' - geo = 'admin2' - outf = f'outputs{sep}unittests' + mp = f"inputs{sep}test6_hoa_conflict_input.json" + fp = f"inputs{sep}test6_hoa_conflict_input.csv" + geo = "admin2" + outf = f"outputs{sep}unittests" # Process: df, dct = mixmasta.process(fp, mp, geo, outf) # Load expected output: - output_df = pd.read_csv(f'outputs{sep}test6_hoa_conflict_output.csv', index_col=False) + output_df = pd.read_csv( + f"outputs{sep}test6_hoa_conflict_output.csv", index_col=False + ) output_df = mixmasta.optimize_df_types(output_df) - with open(f'outputs{sep}test6_hoa_conflict_dict.json') as f: + with open(f"outputs{sep}test6_hoa_conflict_dict.json") as f: output_dict = json.loads(f.read()) # Sort both data frames and reindex for comparison,. - cols = ['timestamp','country','admin1','admin2','admin3','lat','lng','feature','value'] + cols = [ + "timestamp", + "country", + "admin1", + "admin2", + "admin3", + "lat", + "lng", + "feature", + "value", + ] df.sort_values(by=cols, inplace=True) output_df.sort_values(by=cols, inplace=True) df.reset_index(drop=True, inplace=True) - output_df.reset_index(drop =True, inplace=True) + output_df.reset_index(drop=True, inplace=True) # Make the datatypes the same for value/feature and qualifying columns. - df['value'] = df['value'].astype('str') - df['feature'] = df['feature'].astype('str') - output_df['value'] = output_df['value'].astype('str') - output_df['feature'] = output_df['feature'].astype('str') + df["value"] = df["value"].astype("str") + df["feature"] = df["feature"].astype("str") + output_df["value"] = output_df["value"].astype("str") + output_df["feature"] = output_df["feature"].astype("str") # Assertions - assert_frame_equal(df, output_df, check_categorical = False) + assert_frame_equal(df, output_df, check_categorical=False) assert_dict_equal(dct, output_dict) def test_007_single_band_tif(self): - """ This tests single-band geotiff processing.""" + """This tests single-band geotiff processing.""" # Define mixmasta inputs: - mp = f'inputs{sep}test7_single_band_tif_input.json' - fp = f'inputs{sep}test7_single_band_tif_input.tif' - geo = 'admin2' - outf = f'outputs{sep}unittests' + mp = f"inputs{sep}test7_single_band_tif_input.json" + fp = f"inputs{sep}test7_single_band_tif_input.tif" + geo = "admin2" + outf = f"outputs{sep}unittests" # Process: df, dct = mixmasta.process(fp, mp, geo, outf) - #categories = df.select_dtypes(include=['category']).columns.tolist() - df['value'] = df['value'].astype('str') + # categories = df.select_dtypes(include=['category']).columns.tolist() + df["value"] = df["value"].astype("str") # Load expected output: - output_df = pd.read_csv(f'outputs{sep}test7_single_band_tif_output.csv', index_col=False) + output_df = pd.read_csv( + f"outputs{sep}test7_single_band_tif_output.csv", index_col=False + ) - with open(f'outputs{sep}test7_single_band_tif_dict.json') as f: + with open(f"outputs{sep}test7_single_band_tif_dict.json") as f: output_dict = json.loads(f.read()) # Sort both data frames and reindex for comparison,. - cols = ['timestamp','country','admin1','admin2','admin3','lat','lng','feature','value'] + cols = [ + "timestamp", + "country", + "admin1", + "admin2", + "admin3", + "lat", + "lng", + "feature", + "value", + ] df = df[cols] output_df = output_df[cols] # Optimize datatypes for output_df. - floats = output_df.select_dtypes(include=['float64']).columns.tolist() - output_df[floats] = output_df[floats].apply(pd.to_numeric, downcast='float') + floats = output_df.select_dtypes(include=["float64"]).columns.tolist() + output_df[floats] = output_df[floats].apply(pd.to_numeric, downcast="float") - ints = output_df.select_dtypes(include=['int64']).columns.tolist() - output_df[ints] = output_df[ints].apply(pd.to_numeric, downcast='integer') + ints = output_df.select_dtypes(include=["int64"]).columns.tolist() + output_df[ints] = output_df[ints].apply(pd.to_numeric, downcast="integer") # Standardize value and feature columns to str for comparison. - df['value'] = df['value'].astype('str') - df['feature'] = df['feature'].astype('str') - output_df['value'] = output_df['value'].astype('str') - output_df['feature'] = output_df['feature'].astype('str') + df["value"] = df["value"].astype("str") + df["feature"] = df["feature"].astype("str") + output_df["value"] = output_df["value"].astype("str") + output_df["feature"] = output_df["feature"].astype("str") # Sort and reindex. df.sort_values(by=cols, inplace=True) @@ -350,22 +472,36 @@ def test_007_single_band_tif(self): output_df.reset_index(drop=True, inplace=True) # Assertions - assert_frame_equal(df, output_df, check_categorical = False) + assert_frame_equal(df, output_df, check_categorical=False) assert_dict_equal(dct, output_dict) def test_008_aliases(self): - """ This tests feature name aliases.""" + """This tests feature name aliases.""" # Define mixmasta inputs: - mp = f'inputs{sep}test8_aliases_input.json' - fp = f'inputs{sep}test8_aliases_input.csv' - geo = 'admin2' - outf = f'outputs{sep}unittests' - - inputs = "--inputs=[{\"input_file\": \"inputs" + f"{sep}test8_aliases_input.csv\",\"mapper\": \"inputs{sep}test8_aliases_input.json\"" + "}]" - result = subprocess.run(['mixmasta', "causemosify-multi", inputs, "--geo=admin2", f"--output-file=outputs{sep}unittests"], capture_output=True, encoding='utf-8') + mp = f"inputs{sep}test8_aliases_input.json" + fp = f"inputs{sep}test8_aliases_input.csv" + geo = "admin2" + outf = f"outputs{sep}unittests" - if (result.returncode != 0): + inputs = ( + '--inputs=[{"input_file": "inputs' + + f'{sep}test8_aliases_input.csv","mapper": "inputs{sep}test8_aliases_input.json"' + + "}]" + ) + result = subprocess.run( + [ + "mixmasta", + "causemosify-multi", + inputs, + "--geo=admin2", + f"--output-file=outputs{sep}unittests", + ], + capture_output=True, + encoding="utf-8", + ) + + if result.returncode != 0: print(result) self.assertEqual(result.returncode, 0) @@ -379,12 +515,12 @@ def test_008_aliases(self): output_df = output_df_1.append(output_df_2) # Assertions - assert_frame_equal(df, output_df, check_categorical = False) + assert_frame_equal(df, output_df, check_categorical=False) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() """ Test by: > /usr/bin/python3.8 -m unittest /workspaces/mixmasta/tests/test_mixmasta.py -v -""" \ No newline at end of file +""" From 895f31bf6c3be9d6634eb763ef3955c17dd05f94 Mon Sep 17 00:00:00 2001 From: Powell Date: Fri, 6 Jan 2023 17:14:56 -0500 Subject: [PATCH 03/15] Readability changes, normalizer improvements --- mixmasta/normalizer.py | 62 +++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/mixmasta/normalizer.py b/mixmasta/normalizer.py index d488567..b834ab8 100644 --- a/mixmasta/normalizer.py +++ b/mixmasta/normalizer.py @@ -3,6 +3,7 @@ Returns: Tuple(pandas.Dataframe, dict, pandas.Dataframe): _description_ """ +from datetime import datetime import logging import os @@ -100,30 +101,34 @@ def normalizer( # list of names of datetime columns primary_date=True primary_time_cols = [ - k["name"] - for k in mapper["date"] - if "primary_date" in k and k["primary_date"] == True + date_annotation_dict["name"] + for date_annotation_dict in mapper["date"] + if "primary_date" in date_annotation_dict + and date_annotation_dict["primary_date"] == True ] # list of names of datetime columns no primary_date or primary_date = False other_time_cols = [ - k["name"] - for k in mapper["date"] - if "primary_date" not in k or k["primary_date"] == False + date_annotation_dict["name"] + for date_annotation_dict in mapper["date"] + if "primary_date" not in date_annotation_dict + or date_annotation_dict["primary_date"] == False ] # list of names of geo columns primary_geo=True primary_geo_cols = [ - k["name"] - for k in mapper["geo"] - if "primary_geo" in k and k["primary_geo"] == True + geo_annotation_dict["name"] + for geo_annotation_dict in mapper["geo"] + if "primary_geo" in geo_annotation_dict + and geo_annotation_dict["primary_geo"] == True ] # list of geotypes of geo columns primary_geo=True (used for match_geo_names logic below) primary_geo_types = [ - k["geo_type"] - for k in mapper["geo"] - if "primary_geo" in k and k["primary_geo"] == True + geo_annotation_dict["geo_type"] + for geo_annotation_dict in mapper["geo"] + if "primary_geo" in geo_annotation_dict + and geo_annotation_dict["primary_geo"] == True ] # qualified_col_dict: dictionary for columns qualified by another column. @@ -146,33 +151,33 @@ def normalizer( other_date_group_mapper = {} for date_dict in mapper["date"]: - kk = date_dict["name"] - if kk in primary_time_cols: + date_annotation_name = date_dict["name"] + if date_annotation_name in primary_time_cols: # There should only be a single epoch or date field, or a single # group of year/month/day/minute/second marked as primary_time in # the loaded schema. if date_dict["date_type"] == "date": # convert primary_time of date_type date to epochtime and rename as 'timestamp' - df.loc[:, kk] = df[kk].apply( + df.loc[:, date_annotation_name] = df[date_annotation_name].apply( lambda x: format_time( str(x), date_dict["time_format"], validate=False ) ) staple_col_name = "timestamp" - df.rename(columns={kk: staple_col_name}, inplace=True) + df.rename(columns={date_annotation_name: staple_col_name}, inplace=True) # renamed_col_dict[ staple_col_name ] = [kk] # 7/2/2021 do not include primary cols elif date_dict["date_type"] == "epoch": # rename epoch time column as 'timestamp' staple_col_name = "timestamp" - df.rename(columns={kk: staple_col_name}, inplace=True) + df.rename(columns={date_annotation_name: staple_col_name}, inplace=True) # renamed_col_dict[ staple_col_name ] = [kk] # 7/2/2021 do not include primary cols elif date_dict["date_type"] in ["day", "month", "year"]: - primary_date_group_mapper[kk] = date_dict + primary_date_group_mapper[date_annotation_name] = date_dict else: if date_dict["date_type"] == "date": # Convert all date/time to epoch time if not already. - df.loc[:, kk] = df[kk].apply( + df.loc[:, date_annotation_name] = df[date_annotation_name].apply( lambda x: format_time( str(x), date_dict["time_format"], validate=False ) @@ -181,11 +186,11 @@ def normalizer( # primary_time timestamp column, and keep as a feature so the # column_name meaning is not lost. if not primary_time_cols and not "timestamp" in df.columns: - df.rename(columns={kk: "timestamp"}, inplace=True) + df.rename(columns={date_annotation_name: "timestamp"}, inplace=True) staple_col_name = "timestamp" - renamed_col_dict[staple_col_name] = [kk] + renamed_col_dict[staple_col_name] = [date_annotation_name] # All not primary_time, not associated_columns fields are pushed to features. - features.append(kk) + features.append(date_annotation_name) elif ( date_dict["date_type"] in MONTH_DAY_YEAR @@ -196,10 +201,10 @@ def normalizer( # convert them to epoch then store them as a feature # (instead of storing them as separate uncombined features). # handle this dict after iterating all date fields - other_date_group_mapper[kk] = date_dict + other_date_group_mapper[date_annotation_name] = date_dict else: - features.append(kk) + features.append(date_annotation_name) if "qualifies" in date_dict and date_dict["qualifies"]: # Note that any "qualifier" column that is not primary geo/date @@ -211,11 +216,11 @@ def normalizer( # e.g. "name": "region", "qualifies": ["probability", "color"] # should produce two dict entries for prob and color, with region # in a list as the value for both. - for k in date_dict["qualifies"]: - if k in qualified_col_dict: - qualified_col_dict[k].append(kk) + for qualified_column in date_dict["qualifies"]: + if qualified_column in qualified_col_dict: + qualified_col_dict[qualified_column].append(date_annotation_name) else: - qualified_col_dict[k] = [kk] + qualified_col_dict[qualified_column] = [date_annotation_name] if primary_date_group_mapper: # Applied when there were primary_date year,month,day fields above. @@ -249,6 +254,7 @@ def normalizer( # Pop the first item in the mapper and begin building that date set. date_field_tuple = other_date_group_mapper.popitem() + print(f"DEBUG: {date_field_tuple}") # Build a list of column names associated with the the popped date field. assoc_fields = [k[1] for k in date_field_tuple[1]["associated_columns"].items()] From 67bb25b41e3591d8e3de20185cf2d61a209de761 Mon Sep 17 00:00:00 2001 From: Powell Date: Fri, 6 Jan 2023 17:53:24 -0500 Subject: [PATCH 04/15] Breaking up files more, more improvements --- Dockerfile | 2 +- mixmasta/cli.py | 4 +- mixmasta/file_processor.py | 3 +- mixmasta/geo_processor.py | 350 ++++++++++++++++++++++++ mixmasta/normalizer.py | 532 +----------------------------------- mixmasta/spacetag_schema.py | 180 ------------ mixmasta/time_processor.py | 183 +++++++++++++ 7 files changed, 542 insertions(+), 712 deletions(-) create mode 100644 mixmasta/geo_processor.py delete mode 100644 mixmasta/spacetag_schema.py create mode 100644 mixmasta/time_processor.py diff --git a/Dockerfile b/Dockerfile index 7be9f00..145bd22 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ RUN apt-get update && apt-get install -y \ software-properties-common \ sudo \ unzip \ - wget + wget RUN wget https://jataware-world-modelers.s3.amazonaws.com/gadm/gadm36_2.feather.zip && \ wget https://jataware-world-modelers.s3.amazonaws.com/gadm/gadm36_3.feather.zip && \ diff --git a/mixmasta/cli.py b/mixmasta/cli.py index cc27747..dfa9734 100644 --- a/mixmasta/cli.py +++ b/mixmasta/cli.py @@ -1,6 +1,4 @@ import os -import sys -from datetime import datetime import click import pandas as pd @@ -8,7 +6,7 @@ from .download import download_and_clean from .mixmasta import normalizer, optimize_df_types, mixdata from .file_processor import netcdf2df, raster2df -from .normalizer import geocode +from .geo_processor import geocode from glob import glob import numpy as np diff --git a/mixmasta/file_processor.py b/mixmasta/file_processor.py index b9015bc..ce504be 100644 --- a/mixmasta/file_processor.py +++ b/mixmasta/file_processor.py @@ -105,9 +105,8 @@ def raster2df( ColRange = range(data_source.RasterXSize) RowRange = range(data_source.RasterYSize) - # Cache the dataframe and value data type. + # Creating variables for the dataframe and value data type. dataframe = pandas.DataFrame() - row_data_type = None for x in range(1, data_source.RasterCount + 1): # If band has a value, then limit import to the single specified band. diff --git a/mixmasta/geo_processor.py b/mixmasta/geo_processor.py new file mode 100644 index 0000000..6bd0e22 --- /dev/null +++ b/mixmasta/geo_processor.py @@ -0,0 +1,350 @@ +import logging +import os + +from fuzzywuzzy import fuzz +from fuzzywuzzy import process as fuzzyprocess +import geopandas as gpd +import geofeather as gf +import pandas +from pathlib import Path +import pkg_resources +from shapely import speedups +from shapely.geometry import Point +import timeit + +from . import constants + + +def match_geo_names( + admin: str, + df: pandas.DataFrame, + resolve_to_gadm_geotypes: list, + gadm: gpd.GeoDataFrame = None, +) -> pandas.DataFrame: + """ + Assumption + ---------- + Country was selected by drop-down on file submission, column "country" + is present in the data frame, and lng/lat is not being used for geocoding. + + Parameters + ---------- + admin: str + the level to geocode to. Either 'admin2' or 'admin3' + df: pandas.DataFrame + the uploaded dataframe + resolve_to_gadm_geotypes: + list of geotypes marked resolve_to_gadm = True e.g. ["admin1", "country"] + gadm: gpd.GeoDataFrame, default None + optional specification of a GeoDataFrame of GADM shapes of the appropriate + level (admin2/3) for geocoding + + Result + ------ + A pandas.Dataframe produced by modifying the parameter df. + + """ + print("geocoding ...") + flag = speedups.available + if flag == True: + speedups.enable() + + cdir = os.path.expanduser("~") + download_data_folder = f"{cdir}/mixmasta_data" + + # only load GADM if it wasn't explicitly passed to the function. + if gadm is not None: + # logging.info("GADM geo dataframe has been provided.") + pass + else: + logging.info("GADM has not been provided; loading now.") + + if admin == "admin2": + gadm_fn = f"gadm36_2.feather" + else: + gadm_fn = f"gadm36_3.feather" + + gadmDir = f"{download_data_folder}/{gadm_fn}" + gadm = gf.from_geofeather(gadmDir) + + gadm["country"] = gadm["NAME_0"] + gadm["state"] = gadm["NAME_1"] + gadm["admin1"] = gadm["NAME_1"] + gadm["admin2"] = gadm["NAME_2"] + + if admin == "admin2": + gadm = gadm[["country", "state", "admin1", "admin2"]] + else: + gadm["admin3"] = gadm["NAME_3"] + gadm = gadm[["country", "state", "admin1", "admin2", "admin3"]] + + # Filter GADM for countries in df. + countries = df["country"].unique() + + # Correct country names. + if constants.GEO_TYPE_COUNTRY in resolve_to_gadm_geotypes: + gadm_country_list = gadm["country"].unique() + unknowns = df[~df.country.isin(gadm_country_list)].country.tolist() + for unk in unknowns: + try: + match = fuzzyprocess.extractOne( + unk, gadm_country_list, scorer=fuzz.partial_ratio + ) + except Exception as e: + match = None + logging.error(f"Error in match_geo_names: {e}") + if match != None: + df.loc[df.country == unk, "country"] = match[0] + + # Filter GADM dicitonary for only those countries (ie. speed up) + gadm = gadm[gadm["country"].isin(countries)] + + # Loop by country using gadm dict filtered for that country. + for c in countries: + # The following ignores admin1 / admin2 pairs; it only cares if those + # values exist for the appropriate country. + + # Get list of admin1 values in df but not in gadm. Reduce list for country. + if constants.GEO_TYPE_ADMIN1 in resolve_to_gadm_geotypes: + admin1_list = gadm[gadm.country == c]["admin1"].unique() + if admin1_list is not None and all(admin1_list) and "admin1" in df: + unknowns = df[ + (df.country == c) & ~df.admin1.isin(admin1_list) + ].admin1.tolist() + unknowns = [ + x for x in unknowns if pandas.notnull(x) and x.strip() + ] # remove Nan + for unk in unknowns: + match = fuzzyprocess.extractOne( + unk, admin1_list, scorer=fuzz.partial_ratio + ) + if match != None: + df.loc[df.admin1 == unk, "admin1"] = match[0] + + # Get list of admin2 values in df but not in gadm. Reduce list for country. + if constants.GEO_TYPE_ADMIN2 in resolve_to_gadm_geotypes: + admin2_list = gadm[gadm.country == c]["admin2"].unique() + if admin2_list is not None and all(admin2_list) and "admin2" in df: + unknowns = df[ + (df.country == c) & ~df.admin2.isin(admin2_list) + ].admin2.tolist() + unknowns = [ + x for x in unknowns if pandas.notnull(x) and x.strip() + ] # remove Nan + for unk in unknowns: + match = fuzzyprocess.extractOne( + unk, admin2_list, scorer=fuzz.partial_ratio + ) + if match != None: + df.loc[df.admin2 == unk, "admin2"] = match[0] + + if admin == "admin3" and constants.GEO_TYPE_ADMIN3 in resolve_to_gadm_geotypes: + # Get list of admin3 values in df but not in gadm. Reduce list for country. + admin3_list = gadm[gadm.country == c]["admin3"].unique() + if admin3_list is not None and all(admin3_list) and "admin3" in df: + unknowns = df[ + (df.country == c) & ~df.admin3.isin(admin3_list) + ].admin3.tolist() + unknowns = [ + x for x in unknowns if pandas.notnull(x) and x.strip() + ] # remove Nan + for unk in unknowns: + match = fuzzyprocess.extractOne( + unk, admin3_list, scorer=fuzz.partial_ratio + ) + if match != None: + df.loc[df.admin3 == unk, "admin3"] = match[0] + + return df + + +def geocode( + admin: str, + df: pandas.DataFrame, + x: str = "longitude", + y: str = "latitude", + gadm: gpd.GeoDataFrame = None, + df_geocode: pandas.DataFrame = pandas.DataFrame(), +) -> pandas.DataFrame: + """ + Description + ----------- + Takes a dataframe containing coordinate data and geocodes it to GADM (https://gadm.org/) + + GEOCODES to ADMIN 0, 1, 2 OR 3 LEVEL + + Parameters + ---------- + admin: str + the level to geocode to. 'admin0' to 'admin3' + df: pandas.DataFrame + a pandas dataframe containing point data + x: str, default 'longitude' + the name of the column containing longitude information + y: str, default 'latitude' + the name of the column containing latitude data + gadm: gpd.GeoDataFrame, default None + optional specification of a GeoDataFrame of GADM shapes of the appropriate + level (admin2/3) for geocoding + df_geocode: pandas.DataFrame, default pandas.DataFrame() + cached lat/long geocode library + + Examples + -------- + Geocoding a dataframe with columns named 'lat' and 'lon' + + >>> df = geocode(df, x='lon', y='lat') + + """ + + flag = speedups.available + if flag == True: + speedups.enable() + + cdir = os.path.expanduser("~") + download_data_folder = f"{cdir}/mixmasta_data" + + # Only load GADM if it wasn't explicitly passed to the function. + if gadm is not None: + logging.info("GADM geo dataframe has been provided.") + else: + logging.info("GADM has not been provided; loading now.") + + if admin in ["admin0", "country"]: + gadm_fn = f"gadm36_2.feather" + gadmDir = f"{download_data_folder}/{gadm_fn}" + gadm = gf.from_geofeather(gadmDir) + gadm["country"] = gadm["NAME_0"] + gadm = gadm[["geometry", "country"]] + + elif admin == "admin1": + gadm_fn = f"gadm36_2.feather" + gadmDir = f"{download_data_folder}/{gadm_fn}" + gadm = gf.from_geofeather(gadmDir) + gadm["country"] = gadm["NAME_0"] + # gadm["state"] = gadm["NAME_1"] + gadm["admin1"] = gadm["NAME_1"] + # gadm = gadm[["geometry", "country", "state", "admin1"]] + gadm = gadm[["geometry", "country", "admin1"]] + + elif admin == "admin2": + gadm_fn = f"gadm36_2.feather" + gadmDir = f"{download_data_folder}/{gadm_fn}" + gadm = gf.from_geofeather(gadmDir) + gadm["country"] = gadm["NAME_0"] + # gadm["state"] = gadm["NAME_1"] + gadm["admin1"] = gadm["NAME_1"] + gadm["admin2"] = gadm["NAME_2"] + # gadm = gadm[["geometry", "country", "state", "admin1", "admin2"]] + gadm = gadm[["geometry", "country", "admin1", "admin2"]] + + elif admin == "admin3": + gadm_fn = f"gadm36_3.feather" + gadmDir = f"{download_data_folder}/{gadm_fn}" + gadm = gf.from_geofeather(gadmDir) + gadm["country"] = gadm["NAME_0"] + # gadm["state"] = gadm["NAME_1"] + gadm["admin1"] = gadm["NAME_1"] + gadm["admin2"] = gadm["NAME_2"] + gadm["admin3"] = gadm["NAME_3"] + # gadm = gadm[["geometry", "country", "state", "admin1", "admin2", "admin3"]] + gadm = gadm[["geometry", "country", "admin1", "admin2", "admin3"]] + + start_time = timeit.default_timer() + + # 1) Drop x,y duplicates from data frame. + df_drop_dup_geo = df[[x, y]].drop_duplicates(subset=[x, y]) + + # 2) Get x,y not in df_geocode. + if not df_geocode.empty and not df_drop_dup_geo.empty: + df_drop_dup_geo = df_drop_dup_geo.merge( + df_geocode, on=[x, y], how="left", indicator=True + ) + df_drop_dup_geo = df_drop_dup_geo[df_drop_dup_geo["_merge"] == "left_only"] + df_drop_dup_geo = df_drop_dup_geo[[x, y]] + + if not df_drop_dup_geo.empty: + # dr_drop_dup_geo contains x,y not in df_geocode; so, these need to be + # geocoded and added to the df_geocode library. + + # 3) Apply Point() to create the geometry col. + df_drop_dup_geo.loc[:, "geometry"] = df_drop_dup_geo.apply( + lambda row: Point(row[x], row[y]), axis=1 + ) + + # 4) Sjoin unique geometries with GADM. + gdf = gpd.GeoDataFrame(df_drop_dup_geo) + + # Spatial merge on GADM to obtain admin areas. + gdf = gpd.sjoin( + gdf, + gadm, + how="left", + op="within", + lsuffix="mixmasta_left", + rsuffix="mixmasta_geocoded", + ) + del gdf["geometry"] + del gdf["index_mixmasta_geocoded"] + + # 5) Add the new geocoding to the df_geocode lat/long geocode library. + if not df_geocode.empty: + df_geocode = df_geocode.append(gdf) + else: + df_geocode = gdf + + # 6) Merge df and df_geocode on x,y + gdf = df.merge(df_geocode, how="left", on=[x, y]) + + return pandas.DataFrame(gdf), df_geocode + + +def get_iso_country_dict(iso_list: list) -> dict: + """ + Description + ----------- + iso2 or iso3 is used as primary_geo and therefore the country column. + Load the custom iso lookup table and return a dictionary of the iso codes + as keys and the country names as values. Assume all list items are the same + iso type. + + Parameters + ---------- + iso_list: + list of iso2 or iso3 codes + + Returns + ------- + dict: + key: iso code; value: country name + """ + + dct = {} + if iso_list: + iso_df = pandas.DataFrame + try: + # The necessary code to load from pkg doesn't currently work in VS + # Code Debug, so wrap in try/except. + # iso_df = pandas.read_csv(pkg_resources.resource_stream(__name__, 'data/iso_lookup.csv')) + with pkg_resources.resource_stream(__name__, "data/iso_lookup.csv") as f: + iso_df = pandas.read_csv(f) + # path = Path(__file__).parent / "data/iso_lookup.csv" + # iso_df = pandas.read_csv(path) + except: + # Local VS Code load. + path = Path(__file__).parent / "data/iso_lookup.csv" + iso_df = pandas.read_csv(path) + + if iso_df.empty: + return dct + + if len(iso_list[0]) == 2: + for iso in iso_list: + if iso in iso_df["iso2"].values: + dct[iso] = iso_df.loc[iso_df["iso2"] == iso]["country"].item() + else: + for iso in iso_list: + if iso in iso_df["iso3"].values: + dct[iso] = iso_df.loc[iso_df["iso3"] == iso]["country"].item() + + return dct diff --git a/mixmasta/normalizer.py b/mixmasta/normalizer.py index b834ab8..1710989 100644 --- a/mixmasta/normalizer.py +++ b/mixmasta/normalizer.py @@ -3,26 +3,21 @@ Returns: Tuple(pandas.Dataframe, dict, pandas.Dataframe): _description_ """ -from datetime import datetime -import logging -import os import click import geopandas as gpd import numpy import pandas -import pkg_resources -import timeit from distutils.util import strtobool -from fuzzywuzzy import fuzz -from fuzzywuzzy import process as fuzzyprocess -import geofeather as gf -from pathlib import Path -from shapely import speedups -from shapely.geometry import Point from . import constants +from .time_processor import ( + format_time, + generate_timestamp_column, + generate_timestamp_format, +) +from .geo_processor import geocode, get_iso_country_dict, match_geo_names def normalizer( @@ -688,127 +683,6 @@ def handle_colname_collisions( return df, mapper, renamed_col_dict -def format_time(t: str, time_format: str, validate: bool = True) -> int: - """ - Description - ----------- - Converts a time feature (t) into epoch time using `time_format` which is a strftime definition - - Parameters - ---------- - t: str - the time string - time_format: str - the strftime format for the string t - validate: bool, default True - whether to error check the time string t. Is set to False, then no error is raised if the date fails to parse, but None is returned. - - Examples - -------- - - >>> epoch = format_time('5/12/20 12:20', '%m/%d/%y %H:%M') - """ - - try: - t_ = ( - int(datetime.strptime(t, time_format).timestamp()) * 1000 - ) # Want milliseonds - return t_ - except Exception as e: - if t.endswith(" 00:00:00"): - # Depending on the date format, pandas.read_excel will read the - # date as a Timestamp, so here it is a str with format - # '2021-03-26 00:00:00'. For now, handle this single case until - # there is time for a more comprehensive solution e.g. add a custom - # date_parser function that doesn't parse diddly/squat to - # pandas.read_excel() in process(). - return format_time(t.replace(" 00:00:00", ""), time_format, validate) - print(e) - if validate: - raise Exception(e) - else: - return None - - -def generate_timestamp_column( - df: pandas.DataFrame, date_mapper: dict, column_name: str -) -> pandas.DataFrame: - """ - Description - ----------- - Efficiently add a new timestamp column to a dataframe. It avoids the use of df.apply - which appears to be much slower for large dataframes. Defaults to 1/1/1970 for - missing day/month/year values. - - Parameters - ---------- - df: pandas.DataFrame - our data - date_mapper: dict - a schema mapping (JSON) for the dataframe filtered for "date_type" equal to - Day, Month, or Year. The format is screwy for our purposes here and could - be reafactored. - column_name: str - name of the new column e.g. timestamp for primary_time, year1month1day1 - for a concatneated name from associated date fields. - - Examples - -------- - This example adds the generated series to the source dataframe. - >>> df = df.join(df.apply(generate_timestamp, date_mapper=date_mapper, - column_name="year1month1day", axis=1)) - """ - - # Identify which date values are passed. - dayCol = None - monthCol = None - yearCol = None - - for kk, vv in date_mapper.items(): - if vv and vv["date_type"] == "day": - dayCol = kk - elif vv and vv["date_type"] == "month": - monthCol = kk - elif vv and vv["date_type"] == "year": - yearCol = kk - - # For missing date values, add a column to the dataframe with the default - # value, then assign that to the day/month/year var. If the dataframe has - # the date value, assign day/month/year to it after casting as a str. - if dayCol: - day = df[dayCol].astype(str) - else: - df.loc[:, "day_generate_timestamp_column"] = "1" - day = df["day_generate_timestamp_column"] - - if monthCol: - month = df[monthCol].astype(str) - else: - df.loc[:, "month_generate_timestamp_column"] = "1" - month = df["month_generate_timestamp_column"] - - if yearCol: - year = df[yearCol].astype(str) - else: - df.loc[:, "year_generate_timestamp_column"] = "01" - year = df["year_generate_timestamp_column"] - - # Add the new column - df.loc[:, column_name] = month + "/" + day + "/" + year - - # Delete the temporary columns - if not dayCol: - del df["day_generate_timestamp_column"] - - if not monthCol: - del df["month_generate_timestamp_column"] - - if not yearCol: - del df["year_generate_timestamp_column"] - - return df - - def generate_column_name(field_list: list) -> str: """ Description @@ -827,400 +701,6 @@ def generate_column_name(field_list: list) -> str: return "".join(sorted(field_list)) -def generate_timestamp_format(date_mapper: dict) -> str: - """ - Description - ----------- - Generates a the time format for day,month,year dates based on each's - specified time_format. - - Parameters - ---------- - date_mapper: dict - a dictionary for the schema mapping (JSON) for the dataframe filtered - for "date_type" equal to Day, Month, or Year. - - Output - ------ - e.g. "%m/%d/%Y" - """ - - day = "%d" - month = "%m" - year = "%y" - - for kk, vv in date_mapper.items(): - if vv["date_type"] == "day": - day = vv["time_format"] - elif vv["date_type"] == "month": - month = vv["time_format"] - elif vv["date_type"] == "year": - year = vv["time_format"] - - return str.format("{}/{}/{}", month, day, year) - - -def build_date_qualifies_field(qualified_col_dict: dict, assoc_fields: list) -> str: - """ - Description - ----------- - Handle edge case of each date field in assoc_fields qualifying the same - column e.g. day/month/year are associated and qualify a field. In this - case, the new_column_name. - - if assoc_fields is found as a value in qualified_col_dict, return the key - - Parameters - ---------- - qualified_col_dict: dict - {'pop': ['month_column', 'day_column', 'year_column']} - - assoc_fields: list - ['month_column', 'day_column', 'year_column'] - - """ - for k, v in qualified_col_dict.items(): - if v == assoc_fields: - return k - - return None - - -def get_iso_country_dict(iso_list: list) -> dict: - """ - Description - ----------- - iso2 or iso3 is used as primary_geo and therefore the country column. - Load the custom iso lookup table and return a dictionary of the iso codes - as keys and the country names as values. Assume all list items are the same - iso type. - - Parameters - ---------- - iso_list: - list of iso2 or iso3 codes - - Returns - ------- - dict: - key: iso code; value: country name - """ - - dct = {} - if iso_list: - iso_df = pandas.DataFrame - try: - # The necessary code to load from pkg doesn't currently work in VS - # Code Debug, so wrap in try/except. - # iso_df = pandas.read_csv(pkg_resources.resource_stream(__name__, 'data/iso_lookup.csv')) - with pkg_resources.resource_stream(__name__, "data/iso_lookup.csv") as f: - iso_df = pandas.read_csv(f) - # path = Path(__file__).parent / "data/iso_lookup.csv" - # iso_df = pandas.read_csv(path) - except: - # Local VS Code load. - path = Path(__file__).parent / "data/iso_lookup.csv" - iso_df = pandas.read_csv(path) - - if iso_df.empty: - return dct - - if len(iso_list[0]) == 2: - for iso in iso_list: - if iso in iso_df["iso2"].values: - dct[iso] = iso_df.loc[iso_df["iso2"] == iso]["country"].item() - else: - for iso in iso_list: - if iso in iso_df["iso3"].values: - dct[iso] = iso_df.loc[iso_df["iso3"] == iso]["country"].item() - - return dct - - -def geocode( - admin: str, - df: pandas.DataFrame, - x: str = "longitude", - y: str = "latitude", - gadm: gpd.GeoDataFrame = None, - df_geocode: pandas.DataFrame = pandas.DataFrame(), -) -> pandas.DataFrame: - """ - Description - ----------- - Takes a dataframe containing coordinate data and geocodes it to GADM (https://gadm.org/) - - GEOCODES to ADMIN 0, 1, 2 OR 3 LEVEL - - Parameters - ---------- - admin: str - the level to geocode to. 'admin0' to 'admin3' - df: pandas.DataFrame - a pandas dataframe containing point data - x: str, default 'longitude' - the name of the column containing longitude information - y: str, default 'latitude' - the name of the column containing latitude data - gadm: gpd.GeoDataFrame, default None - optional specification of a GeoDataFrame of GADM shapes of the appropriate - level (admin2/3) for geocoding - df_geocode: pandas.DataFrame, default pandas.DataFrame() - cached lat/long geocode library - - Examples - -------- - Geocoding a dataframe with columns named 'lat' and 'lon' - - >>> df = geocode(df, x='lon', y='lat') - - """ - - flag = speedups.available - if flag == True: - speedups.enable() - - cdir = os.path.expanduser("~") - download_data_folder = f"{cdir}/mixmasta_data" - - # Only load GADM if it wasn't explicitly passed to the function. - if gadm is not None: - logging.info("GADM geo dataframe has been provided.") - else: - logging.info("GADM has not been provided; loading now.") - - if admin in ["admin0", "country"]: - gadm_fn = f"gadm36_2.feather" - gadmDir = f"{download_data_folder}/{gadm_fn}" - gadm = gf.from_geofeather(gadmDir) - gadm["country"] = gadm["NAME_0"] - gadm = gadm[["geometry", "country"]] - - elif admin == "admin1": - gadm_fn = f"gadm36_2.feather" - gadmDir = f"{download_data_folder}/{gadm_fn}" - gadm = gf.from_geofeather(gadmDir) - gadm["country"] = gadm["NAME_0"] - # gadm["state"] = gadm["NAME_1"] - gadm["admin1"] = gadm["NAME_1"] - # gadm = gadm[["geometry", "country", "state", "admin1"]] - gadm = gadm[["geometry", "country", "admin1"]] - - elif admin == "admin2": - gadm_fn = f"gadm36_2.feather" - gadmDir = f"{download_data_folder}/{gadm_fn}" - gadm = gf.from_geofeather(gadmDir) - gadm["country"] = gadm["NAME_0"] - # gadm["state"] = gadm["NAME_1"] - gadm["admin1"] = gadm["NAME_1"] - gadm["admin2"] = gadm["NAME_2"] - # gadm = gadm[["geometry", "country", "state", "admin1", "admin2"]] - gadm = gadm[["geometry", "country", "admin1", "admin2"]] - - elif admin == "admin3": - gadm_fn = f"gadm36_3.feather" - gadmDir = f"{download_data_folder}/{gadm_fn}" - gadm = gf.from_geofeather(gadmDir) - gadm["country"] = gadm["NAME_0"] - # gadm["state"] = gadm["NAME_1"] - gadm["admin1"] = gadm["NAME_1"] - gadm["admin2"] = gadm["NAME_2"] - gadm["admin3"] = gadm["NAME_3"] - # gadm = gadm[["geometry", "country", "state", "admin1", "admin2", "admin3"]] - gadm = gadm[["geometry", "country", "admin1", "admin2", "admin3"]] - - start_time = timeit.default_timer() - - # 1) Drop x,y duplicates from data frame. - df_drop_dup_geo = df[[x, y]].drop_duplicates(subset=[x, y]) - - # 2) Get x,y not in df_geocode. - if not df_geocode.empty and not df_drop_dup_geo.empty: - df_drop_dup_geo = df_drop_dup_geo.merge( - df_geocode, on=[x, y], how="left", indicator=True - ) - df_drop_dup_geo = df_drop_dup_geo[df_drop_dup_geo["_merge"] == "left_only"] - df_drop_dup_geo = df_drop_dup_geo[[x, y]] - - if not df_drop_dup_geo.empty: - # dr_drop_dup_geo contains x,y not in df_geocode; so, these need to be - # geocoded and added to the df_geocode library. - - # 3) Apply Point() to create the geometry col. - df_drop_dup_geo.loc[:, "geometry"] = df_drop_dup_geo.apply( - lambda row: Point(row[x], row[y]), axis=1 - ) - - # 4) Sjoin unique geometries with GADM. - gdf = gpd.GeoDataFrame(df_drop_dup_geo) - - # Spatial merge on GADM to obtain admin areas. - gdf = gpd.sjoin( - gdf, - gadm, - how="left", - op="within", - lsuffix="mixmasta_left", - rsuffix="mixmasta_geocoded", - ) - del gdf["geometry"] - del gdf["index_mixmasta_geocoded"] - - # 5) Add the new geocoding to the df_geocode lat/long geocode library. - if not df_geocode.empty: - df_geocode = df_geocode.append(gdf) - else: - df_geocode = gdf - - # 6) Merge df and df_geocode on x,y - gdf = df.merge(df_geocode, how="left", on=[x, y]) - - return pandas.DataFrame(gdf), df_geocode - - -def match_geo_names( - admin: str, - df: pandas.DataFrame, - resolve_to_gadm_geotypes: list, - gadm: gpd.GeoDataFrame = None, -) -> pandas.DataFrame: - """ - Assumption - ---------- - Country was selected by drop-down on file submission, column "country" - is present in the data frame, and lng/lat is not being used for geocoding. - - Parameters - ---------- - admin: str - the level to geocode to. Either 'admin2' or 'admin3' - df: pandas.DataFrame - the uploaded dataframe - resolve_to_gadm_geotypes: - list of geotypes marked resolve_to_gadm = True e.g. ["admin1", "country"] - gadm: gpd.GeoDataFrame, default None - optional specification of a GeoDataFrame of GADM shapes of the appropriate - level (admin2/3) for geocoding - - Result - ------ - A pandas.Dataframe produced by modifying the parameter df. - - """ - print("geocoding ...") - flag = speedups.available - if flag == True: - speedups.enable() - - cdir = os.path.expanduser("~") - download_data_folder = f"{cdir}/mixmasta_data" - - # only load GADM if it wasn't explicitly passed to the function. - if gadm is not None: - # logging.info("GADM geo dataframe has been provided.") - pass - else: - logging.info("GADM has not been provided; loading now.") - - if admin == "admin2": - gadm_fn = f"gadm36_2.feather" - else: - gadm_fn = f"gadm36_3.feather" - - gadmDir = f"{download_data_folder}/{gadm_fn}" - gadm = gf.from_geofeather(gadmDir) - - gadm["country"] = gadm["NAME_0"] - gadm["state"] = gadm["NAME_1"] - gadm["admin1"] = gadm["NAME_1"] - gadm["admin2"] = gadm["NAME_2"] - - if admin == "admin2": - gadm = gadm[["country", "state", "admin1", "admin2"]] - else: - gadm["admin3"] = gadm["NAME_3"] - gadm = gadm[["country", "state", "admin1", "admin2", "admin3"]] - - # Filter GADM for countries in df. - countries = df["country"].unique() - - # Correct country names. - if constants.GEO_TYPE_COUNTRY in resolve_to_gadm_geotypes: - gadm_country_list = gadm["country"].unique() - unknowns = df[~df.country.isin(gadm_country_list)].country.tolist() - for unk in unknowns: - try: - match = fuzzyprocess.extractOne( - unk, gadm_country_list, scorer=fuzz.partial_ratio - ) - except Exception as e: - match = None - logging.error(f"Error in match_geo_names: {e}") - if match != None: - df.loc[df.country == unk, "country"] = match[0] - - # Filter GADM dicitonary for only those countries (ie. speed up) - gadm = gadm[gadm["country"].isin(countries)] - - # Loop by country using gadm dict filtered for that country. - for c in countries: - # The following ignores admin1 / admin2 pairs; it only cares if those - # values exist for the appropriate country. - - # Get list of admin1 values in df but not in gadm. Reduce list for country. - if constants.GEO_TYPE_ADMIN1 in resolve_to_gadm_geotypes: - admin1_list = gadm[gadm.country == c]["admin1"].unique() - if admin1_list is not None and all(admin1_list) and "admin1" in df: - unknowns = df[ - (df.country == c) & ~df.admin1.isin(admin1_list) - ].admin1.tolist() - unknowns = [ - x for x in unknowns if pandas.notnull(x) and x.strip() - ] # remove Nan - for unk in unknowns: - match = fuzzyprocess.extractOne( - unk, admin1_list, scorer=fuzz.partial_ratio - ) - if match != None: - df.loc[df.admin1 == unk, "admin1"] = match[0] - - # Get list of admin2 values in df but not in gadm. Reduce list for country. - if constants.GEO_TYPE_ADMIN2 in resolve_to_gadm_geotypes: - admin2_list = gadm[gadm.country == c]["admin2"].unique() - if admin2_list is not None and all(admin2_list) and "admin2" in df: - unknowns = df[ - (df.country == c) & ~df.admin2.isin(admin2_list) - ].admin2.tolist() - unknowns = [ - x for x in unknowns if pandas.notnull(x) and x.strip() - ] # remove Nan - for unk in unknowns: - match = fuzzyprocess.extractOne( - unk, admin2_list, scorer=fuzz.partial_ratio - ) - if match != None: - df.loc[df.admin2 == unk, "admin2"] = match[0] - - if admin == "admin3" and constants.GEO_TYPE_ADMIN3 in resolve_to_gadm_geotypes: - # Get list of admin3 values in df but not in gadm. Reduce list for country. - admin3_list = gadm[gadm.country == c]["admin3"].unique() - if admin3_list is not None and all(admin3_list) and "admin3" in df: - unknowns = df[ - (df.country == c) & ~df.admin3.isin(admin3_list) - ].admin3.tolist() - unknowns = [ - x for x in unknowns if pandas.notnull(x) and x.strip() - ] # remove Nan - for unk in unknowns: - match = fuzzyprocess.extractOne( - unk, admin3_list, scorer=fuzz.partial_ratio - ) - if match != None: - df.loc[df.admin3 == unk, "admin3"] = match[0] - - return df - - def audit_renamed_col_dict(dct: dict) -> dict: """ Description diff --git a/mixmasta/spacetag_schema.py b/mixmasta/spacetag_schema.py deleted file mode 100644 index 8cc7e06..0000000 --- a/mixmasta/spacetag_schema.py +++ /dev/null @@ -1,180 +0,0 @@ -from typing import Optional, List, Union -from enum import Enum -from pydantic import BaseModel, Field, validator -from typing_extensions import TypedDict - - - -############################# -#### DEFINE ENUMERATIONS #### -############################# -class FileType(str, Enum): - CSV = "csv" - EXCEL = "excel" - NETCDF = "netcdf" - GEOTIFF = "geotiff" - - -class ColumnType(str, Enum): - DATE = "date" - GEO = "geo" - FEATURE = "feature" - - -class DateType(str, Enum): - YEAR = "year" - MONTH = "month" - DAY = "day" - EPOCH = "epoch" - DATE = "date" - - -class GeoType(str, Enum): - LATITUDE = "latitude" - LONGITUDE = "longitude" - COORDINATES = "coordinates" - COUNTRY = "country" - ISO2 = "iso2" - ISO3 = "iso3" - STATE = "state/territory" - COUNTY = "county/district" - CITY = "municipality/town" - - -class FeatureType(str, Enum): - INT = "int" - FLOAT = "float" - STR = "str" - BINARY = "binary" - BOOLEAN = "boolean" - - -class CoordFormat(str, Enum): - LONLAT = "lonlat" - LATLON = "latlon" - - -################################# -#### DEFINE ANNOTATION TYPES #### -################################# -class GeoAnnotation(BaseModel): - name: str - display_name: Optional[str] - description: Optional[str] - type: ColumnType = "geo" - geo_type: GeoType - primary_geo: Optional[bool] - resolve_to_gadm: Optional[bool] - is_geo_pair: Optional[str] = Field( - title="Geo Pair", - description="If present, this is the name of a paired coordinate column.", - example="Lon_", - ) - coord_format: Optional[CoordFormat] = Field( - title="Coordinate Format", - description="If geo type is COORDINATES, then provide the coordinate format", - ) - qualifies: Optional[List[str]] = Field( - title="Qualifies Columns", - description="An array of the column names which this qualifies", - example=["crop_production", "malnutrition_rate"], - ) - -class TimeField(str, Enum): - YEAR = "year" - MONTH = "month" - DAY = "day" - HOUR = "hour" - MINUTE = "minute" - -class DateAnnotation(BaseModel): - name: str = Field(example="year_column") - display_name: Optional[str] - description: Optional[str] - type: ColumnType = "date" - primary_date: Optional[bool] - time_format: Optional[str] = Field( - title="Time Format", - description="The strftime formatter for this field", - example="%y", - ) - date_type: DateType - # The following line - # associated_columns: Optional[dict[TimeField, str]] = Field( - # throws Exception "TypeError: 'type' object is not subscriptable" - # Solve by not subscripting the dicionary; instead, add a validator. - associated_columns: Optional[dict] = Field( - title="Associated datetime column", - description="the type of time as the key with the column name being the value", - example={"day": "day_column", "hour": "hour_column"}, - ) - qualifies: Optional[List[str]] = Field( - title="Qualifies Columns", - description="An array of the column names which this qualifies", - example=["crop_production", "malnutrition_rate"], - ) - - #@validator('date_type') - #def time_format_optional(cls, v, values): - # if v == DateType.DATE and values["time_format"] == None: - # raise ValueError('time_format is required for when date_type is date') - # return v - - @validator('associated_columns') - def validate_associated_columns(cls, v): - # v is type dict - # validate it is dict[TimeField, str] - timefield_list = [e.value for e in TimeField] - for kk, vv in v.items(): - if kk not in timefield_list: - raise ValueError(str.format('Date associated_columns dictionary key must be of type TimeField: {} in {}', kk, v)) - elif type(vv) is not str: - raise ValueError(str.format('Date associated_columns dictionary value must be of type string: {} in {}', vv, v)) - -class FeatureAnnotation(BaseModel): - name: str - display_name: Optional[str] - description: Optional[str] - type: ColumnType = "feature" - feature_type: FeatureType - units: Optional[str] - units_description: Optional[str] - qualifies: Optional[List[str]] = Field( - title="Qualifies Columns", - description="An array of the column names which this qualifies", - example=["crop_production", "malnutrition_rate"], - ) - - -######################### -#### DEFINE METADATA #### -######################### -class Meta(BaseModel): - ftype: FileType - band: Optional[str] - sheet: Optional[str] - date: Optional[str] - # info needed for multi-band geotiffs: - feature_name: Optional[str] # what the values represent e.g. wealth, flooding, headcount - null_value: Optional[float] # usually mixmasta will just determine this from the .tif file - band_name: Optional[str] # this will be the column name for the bands e.g. "year", "month_year" - bands: Optional[dict] = Field( - title="Geotiff Bands", - description="dictionary str:object of band number:band value (value can be float, str, etc.)", - example={ - "1": 2018, - "2": 2019, - "3": 2020, - "4": 2021 - } - - ) - -############################ -#### DEFINE FINAL MODEL #### -############################ -class SpaceModel(BaseModel): - geo: List[Optional[GeoAnnotation]] - date: List[Optional[DateAnnotation]] - feature: List[FeatureAnnotation] - meta: Meta \ No newline at end of file diff --git a/mixmasta/time_processor.py b/mixmasta/time_processor.py new file mode 100644 index 0000000..ba8d004 --- /dev/null +++ b/mixmasta/time_processor.py @@ -0,0 +1,183 @@ +from datetime import datetime + +import pandas + + +def format_time(t: str, time_format: str, validate: bool = True) -> int: + """ + Description + ----------- + Converts a time feature (t) into epoch time using `time_format` which is a strftime definition + + Parameters + ---------- + t: str + the time string + time_format: str + the strftime format for the string t + validate: bool, default True + whether to error check the time string t. Is set to False, then no error is raised if the date fails to parse, but None is returned. + + Examples + -------- + + >>> epoch = format_time('5/12/20 12:20', '%m/%d/%y %H:%M') + """ + + try: + t_ = ( + int(datetime.strptime(t, time_format).timestamp()) * 1000 + ) # Want milliseonds + return t_ + except Exception as e: + if t.endswith(" 00:00:00"): + # Depending on the date format, pandas.read_excel will read the + # date as a Timestamp, so here it is a str with format + # '2021-03-26 00:00:00'. For now, handle this single case until + # there is time for a more comprehensive solution e.g. add a custom + # date_parser function that doesn't parse diddly/squat to + # pandas.read_excel() in process(). + return format_time(t.replace(" 00:00:00", ""), time_format, validate) + print(e) + if validate: + raise Exception(e) + else: + return None + + +def generate_timestamp_column( + df: pandas.DataFrame, date_mapper: dict, column_name: str +) -> pandas.DataFrame: + """ + Description + ----------- + Efficiently add a new timestamp column to a dataframe. It avoids the use of df.apply + which appears to be much slower for large dataframes. Defaults to 1/1/1970 for + missing day/month/year values. + + Parameters + ---------- + df: pandas.DataFrame + our data + date_mapper: dict + a schema mapping (JSON) for the dataframe filtered for "date_type" equal to + Day, Month, or Year. The format is screwy for our purposes here and could + be reafactored. + column_name: str + name of the new column e.g. timestamp for primary_time, year1month1day1 + for a concatneated name from associated date fields. + + Examples + -------- + This example adds the generated series to the source dataframe. + >>> df = df.join(df.apply(generate_timestamp, date_mapper=date_mapper, + column_name="year1month1day", axis=1)) + """ + + # Identify which date values are passed. + dayCol = None + monthCol = None + yearCol = None + + for kk, vv in date_mapper.items(): + if vv and vv["date_type"] == "day": + dayCol = kk + elif vv and vv["date_type"] == "month": + monthCol = kk + elif vv and vv["date_type"] == "year": + yearCol = kk + + # For missing date values, add a column to the dataframe with the default + # value, then assign that to the day/month/year var. If the dataframe has + # the date value, assign day/month/year to it after casting as a str. + if dayCol: + day = df[dayCol].astype(str) + else: + df.loc[:, "day_generate_timestamp_column"] = "1" + day = df["day_generate_timestamp_column"] + + if monthCol: + month = df[monthCol].astype(str) + else: + df.loc[:, "month_generate_timestamp_column"] = "1" + month = df["month_generate_timestamp_column"] + + if yearCol: + year = df[yearCol].astype(str) + else: + df.loc[:, "year_generate_timestamp_column"] = "01" + year = df["year_generate_timestamp_column"] + + # Add the new column + df.loc[:, column_name] = month + "/" + day + "/" + year + + # Delete the temporary columns + if not dayCol: + del df["day_generate_timestamp_column"] + + if not monthCol: + del df["month_generate_timestamp_column"] + + if not yearCol: + del df["year_generate_timestamp_column"] + + return df + + +def generate_timestamp_format(date_mapper: dict) -> str: + """ + Description + ----------- + Generates a the time format for day,month,year dates based on each's + specified time_format. + + Parameters + ---------- + date_mapper: dict + a dictionary for the schema mapping (JSON) for the dataframe filtered + for "date_type" equal to Day, Month, or Year. + + Output + ------ + e.g. "%m/%d/%Y" + """ + + day = "%d" + month = "%m" + year = "%y" + + for kk, vv in date_mapper.items(): + if vv["date_type"] == "day": + day = vv["time_format"] + elif vv["date_type"] == "month": + month = vv["time_format"] + elif vv["date_type"] == "year": + year = vv["time_format"] + + return str.format("{}/{}/{}", month, day, year) + + +def build_date_qualifies_field(qualified_col_dict: dict, assoc_fields: list) -> str: + """ + Description + ----------- + Handle edge case of each date field in assoc_fields qualifying the same + column e.g. day/month/year are associated and qualify a field. In this + case, the new_column_name. + + if assoc_fields is found as a value in qualified_col_dict, return the key + + Parameters + ---------- + qualified_col_dict: dict + {'pop': ['month_column', 'day_column', 'year_column']} + + assoc_fields: list + ['month_column', 'day_column', 'year_column'] + + """ + for k, v in qualified_col_dict.items(): + if v == assoc_fields: + return k + + return None From 848302e08d78de6829f08fc6bbcbe64d9723d34f Mon Sep 17 00:00:00 2001 From: Powell Date: Mon, 9 Jan 2023 17:47:03 -0500 Subject: [PATCH 05/15] Updated refactor, WIP on time conversion code --- mixmasta/constants.py | 3 + mixmasta/normalizer.py | 8 +- mixmasta/time_processor.py | 117 ++++++++++++++++++++++++++--- mixmasta/zero_to_one_normalizer.py | 23 ++++++ 4 files changed, 136 insertions(+), 15 deletions(-) create mode 100644 mixmasta/zero_to_one_normalizer.py diff --git a/mixmasta/constants.py b/mixmasta/constants.py index 352dc20..3af8d05 100644 --- a/mixmasta/constants.py +++ b/mixmasta/constants.py @@ -15,3 +15,6 @@ GEO_TYPE_ADMIN1 = "state/territory" GEO_TYPE_ADMIN2 = "county/district" GEO_TYPE_ADMIN3 = "municipality/town" + +# List of date_types that be used to build a date. +MONTH_DAY_YEAR = ["day", "month", "year"] diff --git a/mixmasta/normalizer.py b/mixmasta/normalizer.py index 1710989..b972558 100644 --- a/mixmasta/normalizer.py +++ b/mixmasta/normalizer.py @@ -18,6 +18,7 @@ generate_timestamp_format, ) from .geo_processor import geocode, get_iso_country_dict, match_geo_names +from .time_processor import build_date_qualifies_field def normalizer( @@ -83,9 +84,6 @@ def normalizer( "lng", ] - # List of date_types that be used to build a date. - MONTH_DAY_YEAR = ["day", "month", "year"] - # Create a dictionary of list: colnames: new col name, and modify df and # mapper for any column name collisions. df, mapper, renamed_col_dict = handle_colname_collisions(df, mapper, col_order) @@ -188,7 +186,7 @@ def normalizer( features.append(date_annotation_name) elif ( - date_dict["date_type"] in MONTH_DAY_YEAR + date_dict["date_type"] in constants.MONTH_DAY_YEAR and "associated_columns" in date_dict and date_dict["associated_columns"] ): @@ -297,7 +295,7 @@ def normalizer( # convert to epoch time only if all three date components (day, month, # year) are present; otherwise leave as a date string. date_types = [v["date_type"] for k, v in assoc_columns_dict.items()] - if len(frozenset(date_types).intersection(MONTH_DAY_YEAR)) == 3: + if len(frozenset(date_types).intersection(constants.MONTH_DAY_YEAR)) == 3: time_formatter = generate_timestamp_format(assoc_columns_dict) df.loc[:, new_column_name] = df[new_column_name].apply( lambda x: format_time(str(x), time_formatter, validate=False) diff --git a/mixmasta/time_processor.py b/mixmasta/time_processor.py index ba8d004..76c427f 100644 --- a/mixmasta/time_processor.py +++ b/mixmasta/time_processor.py @@ -2,21 +2,23 @@ import pandas +from . import constants -def format_time(t: str, time_format: str, validate: bool = True) -> int: + +def format_time(time: str, time_format: str, validate: bool = True) -> int: """ Description ----------- - Converts a time feature (t) into epoch time using `time_format` which is a strftime definition + Converts a time feature (time) into epoch time using `time_format` which is a strftime definition Parameters ---------- - t: str + time: str the time string time_format: str - the strftime format for the string t + the strftime format for the string 'time' validate: bool, default True - whether to error check the time string t. Is set to False, then no error is raised if the date fails to parse, but None is returned. + whether to error check the time string 'time'. Is set to False, then no error is raised if the date fails to parse, but None is returned. Examples -------- @@ -25,19 +27,19 @@ def format_time(t: str, time_format: str, validate: bool = True) -> int: """ try: - t_ = ( - int(datetime.strptime(t, time_format).timestamp()) * 1000 + time_ = ( + int(datetime.strptime(time, time_format).timestamp()) * 1000 ) # Want milliseonds - return t_ + return time_ except Exception as e: - if t.endswith(" 00:00:00"): + if time.endswith(" 00:00:00"): # Depending on the date format, pandas.read_excel will read the # date as a Timestamp, so here it is a str with format # '2021-03-26 00:00:00'. For now, handle this single case until # there is time for a more comprehensive solution e.g. add a custom # date_parser function that doesn't parse diddly/squat to # pandas.read_excel() in process(). - return format_time(t.replace(" 00:00:00", ""), time_format, validate) + return format_time(time.replace(" 00:00:00", ""), time_format, validate) print(e) if validate: raise Exception(e) @@ -181,3 +183,98 @@ def build_date_qualifies_field(qualified_col_dict: dict, assoc_fields: list) -> return k return None + + +def day_month_year_converter(other_date_group_mapper): + # Various date columns have been associated by the user and are not primary_date. + # Convert to epoch time and store as a feature, do not store these separately in features. + # Exception is the group is only two of day, month, year: leave as date. + # Control for possibility of more than one set of assciated_columns. + + # Pop the first item in the mapper and begin building that date set. + date_field_tuple = other_date_group_mapper.popitem() + print(f"DEBUG: {date_field_tuple}") + + # Build a list of column names associated with the the popped date field. + assoc_fields = [k[1] for k in date_field_tuple[1]["associated_columns"].items()] + + # Pop those mapper objects into a dict based on the column name keys in + # assocfields list. + assoc_columns_dict = { + f: other_date_group_mapper.pop(f) + for f in assoc_fields + if f in other_date_group_mapper + } + + # Add the first popped tuple into the assoc_columns dict where the key is the + # first part of the tuple; the value is the 2nd part. + assoc_columns_dict[date_field_tuple[0]] = date_field_tuple[1] + + # Add the first popped tuple column name to the list of associated fields. + assoc_fields.append(date_field_tuple[0]) + + # TODO: If day and year are associated to each other and month, but + # month is not associated to those fields, then at this point assoc_fields + # will be the three values, and assoc_columns will contain only day and + # year. This will error out below. It is assumed that SpaceTag will + # control for this instance. + + # If there is no primary_time column for timestamp, which would have + # been created above with primary_date_group_mapper, or farther above + # looping mapper["date"], attempt to generate from date_type = Month, + # Day, Year features. Otherwise, create a new column name from the + # concatenation of the associated date fields here. + if not "timestamp" in df.columns: + new_column_name = "timestamp" + else: + new_column_name = generate_column_name(assoc_fields) + + # Create a separate df of the associated date fields. This avoids + # pandas upcasting the series dtypes on df.apply(); e.g., int to float, + # or a month 9 to 9.0, which breaks generate_timestamp() + date_df = df[assoc_fields] + + # Now generate the timestamp from date_df and add timestamp col to df. + df = generate_timestamp_column(df, assoc_columns_dict, new_column_name) + + # Determine the correct time format for the new date column, and + # convert to epoch time only if all three date components (day, month, + # year) are present; otherwise leave as a date string. + date_types = [v["date_type"] for k, v in assoc_columns_dict.items()] + if len(frozenset(date_types).intersection(constants.MONTH_DAY_YEAR)) == 3: + time_formatter = generate_timestamp_format(assoc_columns_dict) + df.loc[:, new_column_name] = df[new_column_name].apply( + lambda x: format_time(str(x), time_formatter, validate=False) + ) + + # Let SpaceTag know those date columns were renamed to a new column. + renamed_col_dict[new_column_name] = assoc_fields + + # timestamp is a protected column, so don't add to features. + if new_column_name != "timestamp": + # Handle edge case of each date field in assoc_fields qualifying + # the same column e.g. day/month/year are associated and qualify + # a field. In this case, the new_column_name + qualified_col = build_date_qualifies_field(qualified_col_dict, assoc_fields) + if qualified_col is None: + features.append(new_column_name) + else: + qualified_col_dict[qualified_col] = [new_column_name] + + +def generate_column_name(field_list: list) -> str: + """ + Description + ----------- + Contatenate a list of column fields into a single column name. + + Parameters + ---------- + field_list: list[str] of column names + + Returns + ------- + str: new column name + + """ + return "".join(sorted(field_list)) diff --git a/mixmasta/zero_to_one_normalizer.py b/mixmasta/zero_to_one_normalizer.py new file mode 100644 index 0000000..b40bc3c --- /dev/null +++ b/mixmasta/zero_to_one_normalizer.py @@ -0,0 +1,23 @@ +import numpy as np + + +def normalize_dataframe(dataframe): + """ + This function accepts a dataframe in the canonical format + and min/max normalizes each feature to between 0 to 1 + """ + dataframe_normalized = dataframe.copy(deep=True) + features = dataframe.feature.unique() + + for f in features: + feat = dataframe_normalized[dataframe_normalized["feature"] == f] + dataframe_normalized.loc[feat.index, "value"] = normalize_data(feat["value"]) + return dataframe_normalized + + +def normalize_data(data): + """ + This function takes in an array and performs 0 to 1 normalization on it. + It is robust to NaN values and ignores them (leaves as NaN). + """ + return (data - np.min(data)) / (np.max(data) - np.min(data)) From 367f26e4b040b05593062d8fed39d144409d0afb Mon Sep 17 00:00:00 2001 From: Powell Date: Tue, 17 Jan 2023 15:04:29 -0500 Subject: [PATCH 06/15] Renamed 0 to 1 scaling, added entrypoint from main file to access feature scaling, ready for intermediate release. --- ...o_to_one_normalizer.py => feature_scaling.py} | 16 +++++++++------- mixmasta/mixmasta.py | 16 ++++++++++++++++ mixmasta/normalizer.py | 9 ++++++--- mixmasta/time_processor.py | 15 +++++++++++++++ 4 files changed, 46 insertions(+), 10 deletions(-) rename mixmasta/{zero_to_one_normalizer.py => feature_scaling.py} (51%) diff --git a/mixmasta/zero_to_one_normalizer.py b/mixmasta/feature_scaling.py similarity index 51% rename from mixmasta/zero_to_one_normalizer.py rename to mixmasta/feature_scaling.py index b40bc3c..2722f04 100644 --- a/mixmasta/zero_to_one_normalizer.py +++ b/mixmasta/feature_scaling.py @@ -1,21 +1,23 @@ import numpy as np +import pandas -def normalize_dataframe(dataframe): +def scale_dataframe(dataframe): """ This function accepts a dataframe in the canonical format - and min/max normalizes each feature to between 0 to 1 + and min/max scales each feature to between 0 to 1 """ - dataframe_normalized = dataframe.copy(deep=True) + dfs = [] features = dataframe.feature.unique() for f in features: - feat = dataframe_normalized[dataframe_normalized["feature"] == f] - dataframe_normalized.loc[feat.index, "value"] = normalize_data(feat["value"]) - return dataframe_normalized + feat = dataframe[dataframe["feature"] == f].copy() + feat["value"] = scale_data(feat["value"]) + dfs.append(feat) + return pandas.concat(dfs) -def normalize_data(data): +def scale_data(data): """ This function takes in an array and performs 0 to 1 normalization on it. It is robust to NaN values and ignores them (leaves as NaN). diff --git a/mixmasta/mixmasta.py b/mixmasta/mixmasta.py index 1ae8897..66b2263 100644 --- a/mixmasta/mixmasta.py +++ b/mixmasta/mixmasta.py @@ -11,6 +11,7 @@ from . import constants from .file_processor import process_file_by_filetype from .normalizer import normalizer +from .feature_scaling import scale_dataframe if not sys.warnoptions: import warnings @@ -134,6 +135,21 @@ def process( return norm, renamed_col_dict +def scale_features(dataframe): + """scale_features takes a dataframe and scales all numerical features on a 0 to 1 scale. + This normalizes the data for comparison and visualization. + + Args: + dataframe (pandas.Dataframe): A pandas dataframe with a "feature" and "value" column. + Will scale numerical values in the "value" column from 0 to 1. + + Returns: + pandas.Dataframe: Returns a pandas Dataframe with numerical features scaled from 0 to 1. + """ + df = scale_dataframe(dataframe) + return df + + class mixdata: def load_gadm2(self): cdir = os.path.expanduser("~") diff --git a/mixmasta/normalizer.py b/mixmasta/normalizer.py index b972558..265e891 100644 --- a/mixmasta/normalizer.py +++ b/mixmasta/normalizer.py @@ -18,7 +18,7 @@ generate_timestamp_format, ) from .geo_processor import geocode, get_iso_country_dict, match_geo_names -from .time_processor import build_date_qualifies_field +from .time_processor import build_date_qualifies_field, add_date_to_dataframe_as_epoch def normalizer( @@ -150,15 +150,18 @@ def normalizer( # group of year/month/day/minute/second marked as primary_time in # the loaded schema. if date_dict["date_type"] == "date": - # convert primary_time of date_type date to epochtime and rename as 'timestamp' + # df = add_date_to_dataframe_as_epoch( + # dataframe=df, original_date_column_name=date_annotation_name + # ) df.loc[:, date_annotation_name] = df[date_annotation_name].apply( lambda x: format_time( str(x), date_dict["time_format"], validate=False ) ) + staple_col_name = "timestamp" df.rename(columns={date_annotation_name: staple_col_name}, inplace=True) - # renamed_col_dict[ staple_col_name ] = [kk] # 7/2/2021 do not include primary cols + elif date_dict["date_type"] == "epoch": # rename epoch time column as 'timestamp' staple_col_name = "timestamp" diff --git a/mixmasta/time_processor.py b/mixmasta/time_processor.py index 76c427f..0f83751 100644 --- a/mixmasta/time_processor.py +++ b/mixmasta/time_processor.py @@ -185,6 +185,21 @@ def build_date_qualifies_field(qualified_col_dict: dict, assoc_fields: list) -> return None +def add_date_to_dataframe_as_epoch(dataframe, original_date_column_name): + # convert value of date_type date annotations to epochtime and rename column as 'timestamp' + dataframe.loc[:, original_date_column_name] = dataframe[ + original_date_column_name + ].apply( + lambda x: format_time( + str(x), original_date_column_name["time_format"], validate=False + ) + ) + + staple_col_name = "timestamp" + dataframe.rename(columns={original_date_column_name: staple_col_name}, inplace=True) + return dataframe + + def day_month_year_converter(other_date_group_mapper): # Various date columns have been associated by the user and are not primary_date. # Convert to epoch time and store as a feature, do not store these separately in features. From 1e4a7c63b1027f1634f463a2e95f7bffb1de1773 Mon Sep 17 00:00:00 2001 From: Powell Date: Tue, 17 Jan 2023 18:53:43 -0500 Subject: [PATCH 07/15] Changes to enable the 0 to 1 feature scaling to be used in dojo-stack. --- mixmasta/mixmasta.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mixmasta/mixmasta.py b/mixmasta/mixmasta.py index 66b2263..e4e3b3a 100644 --- a/mixmasta/mixmasta.py +++ b/mixmasta/mixmasta.py @@ -135,7 +135,7 @@ def process( return norm, renamed_col_dict -def scale_features(dataframe): +def scale_features(dataframe, output_file: str = None): """scale_features takes a dataframe and scales all numerical features on a 0 to 1 scale. This normalizes the data for comparison and visualization. @@ -147,6 +147,9 @@ def scale_features(dataframe): pandas.Dataframe: Returns a pandas Dataframe with numerical features scaled from 0 to 1. """ df = scale_dataframe(dataframe) + + if output_file: + df.to_parquet(f"{output_file}_normalized.parquet.gzip", compression="gzip") return df From b51e11a0b0c6b4e7dc0d8b137cfdb8ab9a2aa8b3 Mon Sep 17 00:00:00 2001 From: Powell Date: Mon, 23 Jan 2023 10:47:22 -0500 Subject: [PATCH 08/15] WIP almost complete time refactor. --- mixmasta/geo_processor.py | 2 +- mixmasta/mixmasta.py | 2 +- mixmasta/normalizer.py | 32 +++++++++++++++---- mixmasta/time_helpers.py | 46 +++++++++++++++++++++++++++ mixmasta/time_processor.py | 64 +++++++++++++++++++++++++++----------- 5 files changed, 120 insertions(+), 26 deletions(-) create mode 100644 mixmasta/time_helpers.py diff --git a/mixmasta/geo_processor.py b/mixmasta/geo_processor.py index 6bd0e22..de0b8c0 100644 --- a/mixmasta/geo_processor.py +++ b/mixmasta/geo_processor.py @@ -296,7 +296,7 @@ def geocode( # 6) Merge df and df_geocode on x,y gdf = df.merge(df_geocode, how="left", on=[x, y]) - return pandas.DataFrame(gdf), df_geocode + return pandas.DataFrame(gdf) def get_iso_country_dict(iso_list: list) -> dict: diff --git a/mixmasta/mixmasta.py b/mixmasta/mixmasta.py index e4e3b3a..59eb140 100644 --- a/mixmasta/mixmasta.py +++ b/mixmasta/mixmasta.py @@ -91,7 +91,7 @@ def process( df.reset_index(inplace=True, drop=True) ## Run normalizer. - norm, renamed_col_dict, df_geocode = normalizer(df, mapper, admin, gadm=gadm) + norm, renamed_col_dict = normalizer(df, mapper, admin, gadm=gadm) # Normalizer will add NaN for missing values, e.g. when appending # dataframes with different columns. GADM will return None when geocoding diff --git a/mixmasta/normalizer.py b/mixmasta/normalizer.py index 265e891..a1dc7ac 100644 --- a/mixmasta/normalizer.py +++ b/mixmasta/normalizer.py @@ -18,7 +18,8 @@ generate_timestamp_format, ) from .geo_processor import geocode, get_iso_country_dict, match_geo_names -from .time_processor import build_date_qualifies_field, add_date_to_dataframe_as_epoch +from .time_processor import build_date_qualifies_field +from .time_helpers import date_type_handler, build_a_date_handler def normalizer( @@ -143,6 +144,28 @@ def normalizer( primary_date_group_mapper = {} other_date_group_mapper = {} + # next((primary for primary in mapper["date"] if primary["primary_date"] == true), None) + mapper_date_list = mapper["date"] + # For every annotated date + for date_dict in mapper_date_list: + # Determine type of date and how to proceed, sometimes this spits out a full dataframe. + result = date_type_handler(date_dict=date_dict, dataframe=df) + if result is "build-a-date": + # Special case to handle triplet date of day, month, year column. + mapper_date_list.remove(date_dict) + build_date_components = [ + date_dict + for date_dict in mapper_date_list + if date_dict["name"] + in [value for key, value in date_dict["associated_colums"]] + ] + mapper_date_list.remove(item for item in build_date_components) + build_date_components.append(date_dict) + result = build_a_date_handler( + date_dict=date_dict, date_mapper=build_date_components, dataframe=df + ) + df = result + for date_dict in mapper["date"]: date_annotation_name = date_dict["name"] if date_annotation_name in primary_time_cols: @@ -227,7 +250,6 @@ def normalizer( # pandas upcasting the series dtypes on df.apply(); e.g., int to float, # or a month 9 to 9.0, which breaks generate_timestamp() assoc_fields = primary_date_group_mapper.keys() - date_df = df[assoc_fields] # Now generate the timestamp from date_df and add timestamp col to df. df = generate_timestamp_column(df, primary_date_group_mapper, "timestamp") @@ -487,9 +509,7 @@ def normalizer( # perform geocoding if lat/lng are present if "lat" in df and "lng" in df: - df, df_geocode = geocode( - admin, df, x="lng", y="lat", gadm=gadm, df_geocode=df_geocode - ) + df = geocode(admin, df, x="lng", y="lat", gadm=gadm, df_geocode=df_geocode) elif "country" in primary_geo_types or ("country" in df and not primary_geo_types): # Correct any misspellings etc. in state and admin areas when not # geocoding lat and lng above, and country is the primary_geo. @@ -590,7 +610,7 @@ def normalizer( click.echo("Processed dataframe:") click.echo(df_out.head()) - return df_out[col_order], renamed_col_dict, df_geocode + return df_out[col_order], renamed_col_dict def handle_colname_collisions( diff --git a/mixmasta/time_helpers.py b/mixmasta/time_helpers.py new file mode 100644 index 0000000..3dd5838 --- /dev/null +++ b/mixmasta/time_helpers.py @@ -0,0 +1,46 @@ +from .time_processor import ( + add_date_to_dataframe_as_epoch, + rename_column_to_timestamp, + generate_timestamp_column, + generate_timestamp_format, + format_time, +) + + +def date_type_handler(dataframe, date_dict): + """Takes the target dataframe and a date_dict from the annotation and correctly processes it. + + Args: + dataframe (pandas.Dataframe): Target dataframe of the annotation passed to mixmasta + date_dict (dict): A dictionary containing all the information about the date annotation + + Returns: + pandas.Dataframe or str: Returns a processed pandas.Dataframe if the time was a date or + epoch time, returns a str flag if the date was day month year + """ + date_type = date_dict["date_type"] + primary_date = date_dict.get("primary_date") + date_column_name = date_dict["name"] + match [date_type, primary_date]: + case ["date", True]: + return add_date_to_dataframe_as_epoch(dataframe, date_column_name) + case ["epoch", True]: + return rename_column_to_timestamp( + dataframe=dataframe, original_date_column_name=date_column_name + ) + case ["day" | "month" | "year", True]: + return "build-a-date" + + +def build_a_date_handler(date_mapper, dataframe): + + # Now generate the timestamp from date_df and add timestamp col to df. + result = generate_timestamp_column(dataframe, date_mapper, "timestamp") + + # Determine the correct time format for the new date column, and + # convert to epoch time. + time_formatter = generate_timestamp_format(date_mapper) + result["timestamp"] = result["timestamp"].apply( + lambda x: format_time(str(x), time_formatter, validate=False) + ) + return result diff --git a/mixmasta/time_processor.py b/mixmasta/time_processor.py index 0f83751..9d18e5c 100644 --- a/mixmasta/time_processor.py +++ b/mixmasta/time_processor.py @@ -81,13 +81,13 @@ def generate_timestamp_column( monthCol = None yearCol = None - for kk, vv in date_mapper.items(): - if vv and vv["date_type"] == "day": - dayCol = kk - elif vv and vv["date_type"] == "month": - monthCol = kk - elif vv and vv["date_type"] == "year": - yearCol = kk + for date_column_name, date_ann_dict in date_mapper.items(): + if date_ann_dict and date_ann_dict["date_type"] == "day": + dayCol = date_column_name + elif date_ann_dict and date_ann_dict["date_type"] == "month": + monthCol = date_column_name + elif date_ann_dict and date_ann_dict["date_type"] == "year": + yearCol = date_column_name # For missing date values, add a column to the dataframe with the default # value, then assign that to the day/month/year var. If the dataframe has @@ -110,7 +110,7 @@ def generate_timestamp_column( df.loc[:, "year_generate_timestamp_column"] = "01" year = df["year_generate_timestamp_column"] - # Add the new column + # Add the new column COLUMN NAME IS TIMESTAMP df.loc[:, column_name] = month + "/" + day + "/" + year # Delete the temporary columns @@ -148,13 +148,13 @@ def generate_timestamp_format(date_mapper: dict) -> str: month = "%m" year = "%y" - for kk, vv in date_mapper.items(): - if vv["date_type"] == "day": - day = vv["time_format"] - elif vv["date_type"] == "month": - month = vv["time_format"] - elif vv["date_type"] == "year": - year = vv["time_format"] + for date_column_name, date_ann_dict in date_mapper.items(): + if date_ann_dict["date_type"] == "day": + day = date_ann_dict["time_format"] + elif date_ann_dict["date_type"] == "month": + month = date_ann_dict["time_format"] + elif date_ann_dict["date_type"] == "year": + year = date_ann_dict["time_format"] return str.format("{}/{}/{}", month, day, year) @@ -195,9 +195,37 @@ def add_date_to_dataframe_as_epoch(dataframe, original_date_column_name): ) ) - staple_col_name = "timestamp" - dataframe.rename(columns={original_date_column_name: staple_col_name}, inplace=True) - return dataframe + return rename_column_to_timestamp( + dataframe=dataframe, original_date_column_name=original_date_column_name + ) + + +def rename_column_to_timestamp(dataframe, original_date_column_name): + return dataframe.rename( + columns={original_date_column_name: "timestamp"}, inplace=True + ) + + +def primary_day_month_year(primary_date_list): + # Applied when there were primary_date year,month,day fields above. + # These need to be combined + # into a date and then epoch time, and added as the timestamp field. + + # Create a separate df of the associated date fields. This avoids + # pandas upcasting the series dtypes on df.apply(); e.g., int to float, + # or a month 9 to 9.0, which breaks generate_timestamp() + assoc_fields = primary_date_group_mapper.keys() + date_df = df[assoc_fields] + + # Now generate the timestamp from date_df and add timestamp col to df. + df = generate_timestamp_column(df, primary_date_group_mapper, "timestamp") + + # Determine the correct time format for the new date column, and + # convert to epoch time. + time_formatter = generate_timestamp_format(primary_date_group_mapper) + df["timestamp"] = df["timestamp"].apply( + lambda x: format_time(str(x), time_formatter, validate=False) + ) def day_month_year_converter(other_date_group_mapper): From 37251e493b609f6464edd8b8968828dff7f61023 Mon Sep 17 00:00:00 2001 From: Powell Date: Mon, 30 Jan 2023 17:06:57 -0500 Subject: [PATCH 09/15] Continued WIP, removed time conversion in favor of my simplified version and still getting some issues, but progress being made. --- mixmasta/normalizer.py | 367 +++++++++++++++++++------------------ mixmasta/time_helpers.py | 27 ++- mixmasta/time_processor.py | 13 +- requirements.txt | 2 +- 4 files changed, 211 insertions(+), 198 deletions(-) diff --git a/mixmasta/normalizer.py b/mixmasta/normalizer.py index a1dc7ac..de2eae3 100644 --- a/mixmasta/normalizer.py +++ b/mixmasta/normalizer.py @@ -150,195 +150,204 @@ def normalizer( for date_dict in mapper_date_list: # Determine type of date and how to proceed, sometimes this spits out a full dataframe. result = date_type_handler(date_dict=date_dict, dataframe=df) - if result is "build-a-date": + print(f"date type results: {result}") + if result is None: # Special case to handle triplet date of day, month, year column. mapper_date_list.remove(date_dict) + print([value for value in date_dict["associated_columns"].values()]) + print([date_assoc for date_assoc in mapper_date_list]) build_date_components = [ - date_dict - for date_dict in mapper_date_list - if date_dict["name"] - in [value for key, value in date_dict["associated_colums"]] + date_assoc + for date_assoc in mapper_date_list + if date_assoc["name"] + in [value for value in date_dict["associated_columns"].values()] ] - mapper_date_list.remove(item for item in build_date_components) + print(f"BUILD COMPONENTS: {build_date_components}") + print( + f"mapper date {mapper_date_list}, ITEMS: {[item for item in build_date_components]}" + ) + for item in build_date_components: + mapper_date_list.remove(item) build_date_components.append(date_dict) result = build_a_date_handler( date_dict=date_dict, date_mapper=build_date_components, dataframe=df ) + print(f"date type results after build a date: {result}") df = result - for date_dict in mapper["date"]: - date_annotation_name = date_dict["name"] - if date_annotation_name in primary_time_cols: - # There should only be a single epoch or date field, or a single - # group of year/month/day/minute/second marked as primary_time in - # the loaded schema. - if date_dict["date_type"] == "date": - # df = add_date_to_dataframe_as_epoch( - # dataframe=df, original_date_column_name=date_annotation_name - # ) - df.loc[:, date_annotation_name] = df[date_annotation_name].apply( - lambda x: format_time( - str(x), date_dict["time_format"], validate=False - ) - ) - - staple_col_name = "timestamp" - df.rename(columns={date_annotation_name: staple_col_name}, inplace=True) - - elif date_dict["date_type"] == "epoch": - # rename epoch time column as 'timestamp' - staple_col_name = "timestamp" - df.rename(columns={date_annotation_name: staple_col_name}, inplace=True) - # renamed_col_dict[ staple_col_name ] = [kk] # 7/2/2021 do not include primary cols - elif date_dict["date_type"] in ["day", "month", "year"]: - primary_date_group_mapper[date_annotation_name] = date_dict - - else: - if date_dict["date_type"] == "date": - # Convert all date/time to epoch time if not already. - df.loc[:, date_annotation_name] = df[date_annotation_name].apply( - lambda x: format_time( - str(x), date_dict["time_format"], validate=False - ) - ) - # If three are no assigned primary_time columns, make this the - # primary_time timestamp column, and keep as a feature so the - # column_name meaning is not lost. - if not primary_time_cols and not "timestamp" in df.columns: - df.rename(columns={date_annotation_name: "timestamp"}, inplace=True) - staple_col_name = "timestamp" - renamed_col_dict[staple_col_name] = [date_annotation_name] - # All not primary_time, not associated_columns fields are pushed to features. - features.append(date_annotation_name) - - elif ( - date_dict["date_type"] in constants.MONTH_DAY_YEAR - and "associated_columns" in date_dict - and date_dict["associated_columns"] - ): - # Various date columns have been associated by the user and are not primary_date. - # convert them to epoch then store them as a feature - # (instead of storing them as separate uncombined features). - # handle this dict after iterating all date fields - other_date_group_mapper[date_annotation_name] = date_dict - - else: - features.append(date_annotation_name) - - if "qualifies" in date_dict and date_dict["qualifies"]: - # Note that any "qualifier" column that is not primary geo/date - # will just be lopped on to the right as its own column. It's - # column name will just be the name and Uncharted will deal with - # it. The key takeaway is that qualifier columns grow the width, - # not the length of the dataset. - # Want to add the qualified col as the dictionary key. - # e.g. "name": "region", "qualifies": ["probability", "color"] - # should produce two dict entries for prob and color, with region - # in a list as the value for both. - for qualified_column in date_dict["qualifies"]: - if qualified_column in qualified_col_dict: - qualified_col_dict[qualified_column].append(date_annotation_name) - else: - qualified_col_dict[qualified_column] = [date_annotation_name] - - if primary_date_group_mapper: - # Applied when there were primary_date year,month,day fields above. - # These need to be combined - # into a date and then epoch time, and added as the timestamp field. - - # Create a separate df of the associated date fields. This avoids - # pandas upcasting the series dtypes on df.apply(); e.g., int to float, - # or a month 9 to 9.0, which breaks generate_timestamp() - assoc_fields = primary_date_group_mapper.keys() - - # Now generate the timestamp from date_df and add timestamp col to df. - df = generate_timestamp_column(df, primary_date_group_mapper, "timestamp") - - # Determine the correct time format for the new date column, and - # convert to epoch time. - time_formatter = generate_timestamp_format(primary_date_group_mapper) - df["timestamp"] = df["timestamp"].apply( - lambda x: format_time(str(x), time_formatter, validate=False) - ) - - # Let SpaceTag know those date columns were renamed to timestamp. - # renamed_col_dict[ "timestamp" ] = assoc_fields # 7/2/2021 do not include primary cols - - while other_date_group_mapper: - # Various date columns have been associated by the user and are not primary_date. - # Convert to epoch time and store as a feature, do not store these separately in features. - # Exception is the group is only two of day, month, year: leave as date. - # Control for possibility of more than one set of assciated_columns. - - # Pop the first item in the mapper and begin building that date set. - date_field_tuple = other_date_group_mapper.popitem() - print(f"DEBUG: {date_field_tuple}") - - # Build a list of column names associated with the the popped date field. - assoc_fields = [k[1] for k in date_field_tuple[1]["associated_columns"].items()] - - # Pop those mapper objects into a dict based on the column name keys in - # assocfields list. - assoc_columns_dict = { - f: other_date_group_mapper.pop(f) - for f in assoc_fields - if f in other_date_group_mapper - } - - # Add the first popped tuple into the assoc_columns dict where the key is the - # first part of the tuple; the value is the 2nd part. - assoc_columns_dict[date_field_tuple[0]] = date_field_tuple[1] - - # Add the first popped tuple column name to the list of associated fields. - assoc_fields.append(date_field_tuple[0]) - - # TODO: If day and year are associated to each other and month, but - # month is not associated to those fields, then at this point assoc_fields - # will be the three values, and assoc_columns will contain only day and - # year. This will error out below. It is assumed that SpaceTag will - # control for this instance. - - # If there is no primary_time column for timestamp, which would have - # been created above with primary_date_group_mapper, or farther above - # looping mapper["date"], attempt to generate from date_type = Month, - # Day, Year features. Otherwise, create a new column name from the - # concatenation of the associated date fields here. - if not "timestamp" in df.columns: - new_column_name = "timestamp" - else: - new_column_name = generate_column_name(assoc_fields) - - # Create a separate df of the associated date fields. This avoids - # pandas upcasting the series dtypes on df.apply(); e.g., int to float, - # or a month 9 to 9.0, which breaks generate_timestamp() - date_df = df[assoc_fields] - - # Now generate the timestamp from date_df and add timestamp col to df. - df = generate_timestamp_column(df, assoc_columns_dict, new_column_name) - - # Determine the correct time format for the new date column, and - # convert to epoch time only if all three date components (day, month, - # year) are present; otherwise leave as a date string. - date_types = [v["date_type"] for k, v in assoc_columns_dict.items()] - if len(frozenset(date_types).intersection(constants.MONTH_DAY_YEAR)) == 3: - time_formatter = generate_timestamp_format(assoc_columns_dict) - df.loc[:, new_column_name] = df[new_column_name].apply( - lambda x: format_time(str(x), time_formatter, validate=False) - ) - - # Let SpaceTag know those date columns were renamed to a new column. - renamed_col_dict[new_column_name] = assoc_fields - - # timestamp is a protected column, so don't add to features. - if new_column_name != "timestamp": - # Handle edge case of each date field in assoc_fields qualifying - # the same column e.g. day/month/year are associated and qualify - # a field. In this case, the new_column_name - qualified_col = build_date_qualifies_field(qualified_col_dict, assoc_fields) - if qualified_col is None: - features.append(new_column_name) - else: - qualified_col_dict[qualified_col] = [new_column_name] + # for date_dict in mapper["date"]: + # date_annotation_name = date_dict["name"] + # if date_annotation_name in primary_time_cols: + # # There should only be a single epoch or date field, or a single + # # group of year/month/day/minute/second marked as primary_time in + # # the loaded schema. + # if date_dict["date_type"] == "date": + # # df = add_date_to_dataframe_as_epoch( + # # dataframe=df, original_date_column_name=date_annotation_name + # # ) + # df.loc[:, date_annotation_name] = df[date_annotation_name].apply( + # lambda x: format_time( + # str(x), date_dict["time_format"], validate=False + # ) + # ) + + # staple_col_name = "timestamp" + # df.rename(columns={date_annotation_name: staple_col_name}, inplace=True) + + # elif date_dict["date_type"] == "epoch": + # # rename epoch time column as 'timestamp' + # staple_col_name = "timestamp" + # df.rename(columns={date_annotation_name: staple_col_name}, inplace=True) + # # renamed_col_dict[ staple_col_name ] = [kk] # 7/2/2021 do not include primary cols + # elif date_dict["date_type"] in ["day", "month", "year"]: + # primary_date_group_mapper[date_annotation_name] = date_dict + + # else: + # if date_dict["date_type"] == "date": + # # Convert all date/time to epoch time if not already. + # df.loc[:, date_annotation_name] = df[date_annotation_name].apply( + # lambda x: format_time( + # str(x), date_dict["time_format"], validate=False + # ) + # ) + # # If three are no assigned primary_time columns, make this the + # # primary_time timestamp column, and keep as a feature so the + # # column_name meaning is not lost. + # if not primary_time_cols and not "timestamp" in df.columns: + # df.rename(columns={date_annotation_name: "timestamp"}, inplace=True) + # staple_col_name = "timestamp" + # renamed_col_dict[staple_col_name] = [date_annotation_name] + # # All not primary_time, not associated_columns fields are pushed to features. + # features.append(date_annotation_name) + + # elif ( + # date_dict["date_type"] in constants.MONTH_DAY_YEAR + # and "associated_columns" in date_dict + # and date_dict["associated_columns"] + # ): + # # Various date columns have been associated by the user and are not primary_date. + # # convert them to epoch then store them as a feature + # # (instead of storing them as separate uncombined features). + # # handle this dict after iterating all date fields + # other_date_group_mapper[date_annotation_name] = date_dict + + # else: + # features.append(date_annotation_name) + + # if "qualifies" in date_dict and date_dict["qualifies"]: + # # Note that any "qualifier" column that is not primary geo/date + # # will just be lopped on to the right as its own column. It's + # # column name will just be the name and Uncharted will deal with + # # it. The key takeaway is that qualifier columns grow the width, + # # not the length of the dataset. + # # Want to add the qualified col as the dictionary key. + # # e.g. "name": "region", "qualifies": ["probability", "color"] + # # should produce two dict entries for prob and color, with region + # # in a list as the value for both. + # for qualified_column in date_dict["qualifies"]: + # if qualified_column in qualified_col_dict: + # qualified_col_dict[qualified_column].append(date_annotation_name) + # else: + # qualified_col_dict[qualified_column] = [date_annotation_name] + + # if primary_date_group_mapper: + # # Applied when there were primary_date year,month,day fields above. + # # These need to be combined + # # into a date and then epoch time, and added as the timestamp field. + + # # Create a separate df of the associated date fields. This avoids + # # pandas upcasting the series dtypes on df.apply(); e.g., int to float, + # # or a month 9 to 9.0, which breaks generate_timestamp() + # assoc_fields = primary_date_group_mapper.keys() + + # # Now generate the timestamp from date_df and add timestamp col to df. + # df = generate_timestamp_column(df, primary_date_group_mapper, "timestamp") + + # # Determine the correct time format for the new date column, and + # # convert to epoch time. + # time_formatter = generate_timestamp_format(primary_date_group_mapper) + # df["timestamp"] = df["timestamp"].apply( + # lambda x: format_time(str(x), time_formatter, validate=False) + # ) + + # # Let SpaceTag know those date columns were renamed to timestamp. + # # renamed_col_dict[ "timestamp" ] = assoc_fields # 7/2/2021 do not include primary cols + + # while other_date_group_mapper: + # # Various date columns have been associated by the user and are not primary_date. + # # Convert to epoch time and store as a feature, do not store these separately in features. + # # Exception is the group is only two of day, month, year: leave as date. + # # Control for possibility of more than one set of assciated_columns. + + # # Pop the first item in the mapper and begin building that date set. + # date_field_tuple = other_date_group_mapper.popitem() + # print(f"DEBUG: {date_field_tuple}") + + # # Build a list of column names associated with the the popped date field. + # assoc_fields = [k[1] for k in date_field_tuple[1]["associated_columns"].items()] + + # # Pop those mapper objects into a dict based on the column name keys in + # # assocfields list. + # assoc_columns_dict = { + # f: other_date_group_mapper.pop(f) + # for f in assoc_fields + # if f in other_date_group_mapper + # } + + # # Add the first popped tuple into the assoc_columns dict where the key is the + # # first part of the tuple; the value is the 2nd part. + # assoc_columns_dict[date_field_tuple[0]] = date_field_tuple[1] + + # # Add the first popped tuple column name to the list of associated fields. + # assoc_fields.append(date_field_tuple[0]) + + # # TODO: If day and year are associated to each other and month, but + # # month is not associated to those fields, then at this point assoc_fields + # # will be the three values, and assoc_columns will contain only day and + # # year. This will error out below. It is assumed that SpaceTag will + # # control for this instance. + + # # If there is no primary_time column for timestamp, which would have + # # been created above with primary_date_group_mapper, or farther above + # # looping mapper["date"], attempt to generate from date_type = Month, + # # Day, Year features. Otherwise, create a new column name from the + # # concatenation of the associated date fields here. + # if not "timestamp" in df.columns: + # new_column_name = "timestamp" + # else: + # new_column_name = generate_column_name(assoc_fields) + + # # Create a separate df of the associated date fields. This avoids + # # pandas upcasting the series dtypes on df.apply(); e.g., int to float, + # # or a month 9 to 9.0, which breaks generate_timestamp() + # date_df = df[assoc_fields] + + # # Now generate the timestamp from date_df and add timestamp col to df. + # df = generate_timestamp_column(df, assoc_columns_dict, new_column_name) + + # # Determine the correct time format for the new date column, and + # # convert to epoch time only if all three date components (day, month, + # # year) are present; otherwise leave as a date string. + # date_types = [v["date_type"] for k, v in assoc_columns_dict.items()] + # if len(frozenset(date_types).intersection(constants.MONTH_DAY_YEAR)) == 3: + # time_formatter = generate_timestamp_format(assoc_columns_dict) + # df.loc[:, new_column_name] = df[new_column_name].apply( + # lambda x: format_time(str(x), time_formatter, validate=False) + # ) + + # # Let SpaceTag know those date columns were renamed to a new column. + # renamed_col_dict[new_column_name] = assoc_fields + + # # timestamp is a protected column, so don't add to features. + # if new_column_name != "timestamp": + # # Handle edge case of each date field in assoc_fields qualifying + # # the same column e.g. day/month/year are associated and qualify + # # a field. In this case, the new_column_name + # qualified_col = build_date_qualifies_field(qualified_col_dict, assoc_fields) + # if qualified_col is None: + # features.append(new_column_name) + # else: + # qualified_col_dict[qualified_col] = [new_column_name] for geo_dict in mapper["geo"]: kk = geo_dict["name"] diff --git a/mixmasta/time_helpers.py b/mixmasta/time_helpers.py index 3dd5838..580122d 100644 --- a/mixmasta/time_helpers.py +++ b/mixmasta/time_helpers.py @@ -19,17 +19,26 @@ def date_type_handler(dataframe, date_dict): epoch time, returns a str flag if the date was day month year """ date_type = date_dict["date_type"] + print(f"DATE TYPE: {date_type}") primary_date = date_dict.get("primary_date") date_column_name = date_dict["name"] - match [date_type, primary_date]: - case ["date", True]: - return add_date_to_dataframe_as_epoch(dataframe, date_column_name) - case ["epoch", True]: - return rename_column_to_timestamp( - dataframe=dataframe, original_date_column_name=date_column_name - ) - case ["day" | "month" | "year", True]: - return "build-a-date" + if date_type == "date": + return add_date_to_dataframe_as_epoch(dataframe, date_dict, date_column_name) + if date_type == "epoch": + return rename_column_to_timestamp( + dataframe=dataframe, original_date_column_name=date_column_name + ) + if date_type == "day" or date_type == "month" or date_type == "year": + return None + # match [date_type, primary_date]: + # case ["date", True]: + # return add_date_to_dataframe_as_epoch(dataframe, date_column_name) + # case ["epoch", True]: + # return rename_column_to_timestamp( + # dataframe=dataframe, original_date_column_name=date_column_name + # ) + # case ["day" | "month" | "year", True]: + # return "build-a-date" def build_a_date_handler(date_mapper, dataframe): diff --git a/mixmasta/time_processor.py b/mixmasta/time_processor.py index 9d18e5c..c6319ba 100644 --- a/mixmasta/time_processor.py +++ b/mixmasta/time_processor.py @@ -185,15 +185,11 @@ def build_date_qualifies_field(qualified_col_dict: dict, assoc_fields: list) -> return None -def add_date_to_dataframe_as_epoch(dataframe, original_date_column_name): +def add_date_to_dataframe_as_epoch(dataframe, date_dict, original_date_column_name): # convert value of date_type date annotations to epochtime and rename column as 'timestamp' dataframe.loc[:, original_date_column_name] = dataframe[ original_date_column_name - ].apply( - lambda x: format_time( - str(x), original_date_column_name["time_format"], validate=False - ) - ) + ].apply(lambda x: format_time(str(x), date_dict["time_format"], validate=False)) return rename_column_to_timestamp( dataframe=dataframe, original_date_column_name=original_date_column_name @@ -201,9 +197,8 @@ def add_date_to_dataframe_as_epoch(dataframe, original_date_column_name): def rename_column_to_timestamp(dataframe, original_date_column_name): - return dataframe.rename( - columns={original_date_column_name: "timestamp"}, inplace=True - ) + dataframe.rename(columns={original_date_column_name: "timestamp"}, inplace=True) + return dataframe def primary_day_month_year(primary_date_list): diff --git a/requirements.txt b/requirements.txt index 3e3c827..1067087 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ bump2version==1.0.1 -Click>=7.0,<8 +Click==8.0 coverage==4.5.4 Cython==0.29.23 flake8==3.7.8 From 9afd84052b60e175fc246a0e7dafeb319ad6f011 Mon Sep 17 00:00:00 2001 From: Powell Date: Wed, 1 Feb 2023 12:19:56 -0500 Subject: [PATCH 10/15] Added clipping transformation code and job, main flow still broken. --- mixmasta/mixmasta.py | 18 ++++++++++++++++ mixmasta/normalizer.py | 2 +- mixmasta/time_helpers.py | 3 ++- mixmasta/transformations/__init__.py | 0 mixmasta/transformations/clipping.py | 32 ++++++++++++++++++++++++++++ requirements.txt | 3 ++- 6 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 mixmasta/transformations/__init__.py create mode 100644 mixmasta/transformations/clipping.py diff --git a/mixmasta/mixmasta.py b/mixmasta/mixmasta.py index 59eb140..1b9cb29 100644 --- a/mixmasta/mixmasta.py +++ b/mixmasta/mixmasta.py @@ -12,6 +12,7 @@ from .file_processor import process_file_by_filetype from .normalizer import normalizer from .feature_scaling import scale_dataframe +from .transformations.clipping import construct_multipolygon, clip_dataframe if not sys.warnoptions: import warnings @@ -153,6 +154,23 @@ def scale_features(dataframe, output_file: str = None): return df +def clip_data(dataframe, geo_columns, polygons_list): + """Clips data based on geographical shape(s) or shapefile (NOT IMPLEMENTED). + + Args: + dataframe (pandas.Dataframe): A pandas dataframe containing geographical data. + geo_columns (list): A list containing the two column names for the lat/lon columns in the dataframe. + polygons_list (list[list[obj]]): A list containing lists of objects that represent polygon shapes to clip to. + + Returns: + pandas.Dataframe: A pandas dataframe only containing the clipped data. + """ + + mask = construct_multipolygon(polygons_list=polygons_list) + + return clip_dataframe(dataframe=dataframe, geo_columns=geo_columns, mask=mask) + + class mixdata: def load_gadm2(self): cdir = os.path.expanduser("~") diff --git a/mixmasta/normalizer.py b/mixmasta/normalizer.py index de2eae3..346f2c5 100644 --- a/mixmasta/normalizer.py +++ b/mixmasta/normalizer.py @@ -170,7 +170,7 @@ def normalizer( mapper_date_list.remove(item) build_date_components.append(date_dict) result = build_a_date_handler( - date_dict=date_dict, date_mapper=build_date_components, dataframe=df + date_mapper=build_date_components, dataframe=df ) print(f"date type results after build a date: {result}") df = result diff --git a/mixmasta/time_helpers.py b/mixmasta/time_helpers.py index 580122d..2f13b20 100644 --- a/mixmasta/time_helpers.py +++ b/mixmasta/time_helpers.py @@ -19,7 +19,7 @@ def date_type_handler(dataframe, date_dict): epoch time, returns a str flag if the date was day month year """ date_type = date_dict["date_type"] - print(f"DATE TYPE: {date_type}") + # print(f"DATE TYPE: {date_type}") primary_date = date_dict.get("primary_date") date_column_name = date_dict["name"] if date_type == "date": @@ -44,6 +44,7 @@ def date_type_handler(dataframe, date_dict): def build_a_date_handler(date_mapper, dataframe): # Now generate the timestamp from date_df and add timestamp col to df. + print(f"BUILD A DATE MAPPER {date_mapper}") result = generate_timestamp_column(dataframe, date_mapper, "timestamp") # Determine the correct time format for the new date column, and diff --git a/mixmasta/transformations/__init__.py b/mixmasta/transformations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mixmasta/transformations/clipping.py b/mixmasta/transformations/clipping.py new file mode 100644 index 0000000..52f86de --- /dev/null +++ b/mixmasta/transformations/clipping.py @@ -0,0 +1,32 @@ +import geopandas +import pandas +from shapely import MultiPolygon, Polygon + + +def construct_multipolygon(polygons_list): + """Constructs a shapely Multipolygon to be used as a clipping file. + + Args: + polygons_list (list): This should be a list of lists, where each + list contains a dictionary with a "lat" key and a "lng" key representing the edges of the shape(s) + """ + final_multipolygon_list = [] + for polygon_list in polygons_list: + intermediate_list = Polygon( + [edge.get("lat"), edge.get("lng")] for edge in polygon_list + ) + final_multipolygon_list.append(intermediate_list) + + shape = MultiPolygon(final_multipolygon_list) + + return shape + + +def clip_dataframe(dataframe, geo_columns, mask): + x_geo = dataframe[geo_columns[0]] + y_geo = dataframe[geo_columns[1]] + geo_dataframe = geopandas.GeoDataFrame( + dataframe, geometry=geopandas.points_from_xy(x_geo, y_geo) + ) + + return pandas.DataFrame(geopandas.clip(geo_dataframe, mask)) diff --git a/requirements.txt b/requirements.txt index 1067087..8dddb92 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ bump2version==1.0.1 +cdo==1.5.5 Click==8.0 coverage==4.5.4 Cython==0.29.23 @@ -24,4 +25,4 @@ twine==1.14.0 watchdog==0.9.0 wheel==0.33.6 xarray==0.16.1 -xlrd==2.0.1 +xlrd==2.0.1 \ No newline at end of file From a799bc7264fbeb548cd0ca4c07529bc4d518666a Mon Sep 17 00:00:00 2001 From: Powell Date: Fri, 3 Feb 2023 09:13:52 -0500 Subject: [PATCH 11/15] Bugfixes to clipping transformation. --- mixmasta/transformations/clipping.py | 2 +- requirements.txt | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mixmasta/transformations/clipping.py b/mixmasta/transformations/clipping.py index 52f86de..4e2074b 100644 --- a/mixmasta/transformations/clipping.py +++ b/mixmasta/transformations/clipping.py @@ -1,6 +1,6 @@ import geopandas import pandas -from shapely import MultiPolygon, Polygon +from shapely.geometry import MultiPolygon, Polygon def construct_multipolygon(polygons_list): diff --git a/requirements.txt b/requirements.txt index 8dddb92..797f6f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ bump2version==1.0.1 -cdo==1.5.5 Click==8.0 coverage==4.5.4 Cython==0.29.23 @@ -17,7 +16,7 @@ pyproj==2.6.1.post1 python-Levenshtein>=0.12.2 rasterio>=1.1.0 Rtree==0.8.3 -Shapely==1.7.1 +shapely Sphinx==1.8.5 tox==3.14.0 tqdm>=4.41.1,<5.0.0 From 4131d18acfb422c3792bed6438f90eb2c18acd0a Mon Sep 17 00:00:00 2001 From: Powell Date: Fri, 3 Feb 2023 17:30:14 -0500 Subject: [PATCH 12/15] WIP main standardization time flow almost without assertion errors on tests. --- mixmasta/normalizer.py | 15 +++++---------- mixmasta/time_helpers.py | 4 +++- requirements.txt | 1 - 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/mixmasta/normalizer.py b/mixmasta/normalizer.py index 346f2c5..090d86e 100644 --- a/mixmasta/normalizer.py +++ b/mixmasta/normalizer.py @@ -154,21 +154,16 @@ def normalizer( if result is None: # Special case to handle triplet date of day, month, year column. mapper_date_list.remove(date_dict) - print([value for value in date_dict["associated_columns"].values()]) - print([date_assoc for date_assoc in mapper_date_list]) - build_date_components = [ - date_assoc + build_date_components = { + date_assoc["name"]: date_assoc for date_assoc in mapper_date_list if date_assoc["name"] in [value for value in date_dict["associated_columns"].values()] - ] + } print(f"BUILD COMPONENTS: {build_date_components}") - print( - f"mapper date {mapper_date_list}, ITEMS: {[item for item in build_date_components]}" - ) - for item in build_date_components: + for item in build_date_components.values(): mapper_date_list.remove(item) - build_date_components.append(date_dict) + build_date_components[date_dict["name"]] = date_dict result = build_a_date_handler( date_mapper=build_date_components, dataframe=df ) diff --git a/mixmasta/time_helpers.py b/mixmasta/time_helpers.py index 2f13b20..ac94aa6 100644 --- a/mixmasta/time_helpers.py +++ b/mixmasta/time_helpers.py @@ -45,7 +45,9 @@ def build_a_date_handler(date_mapper, dataframe): # Now generate the timestamp from date_df and add timestamp col to df. print(f"BUILD A DATE MAPPER {date_mapper}") - result = generate_timestamp_column(dataframe, date_mapper, "timestamp") + result = generate_timestamp_column( + df=dataframe, date_mapper=date_mapper, column_name="timestamp" + ) # Determine the correct time format for the new date column, and # convert to epoch time. diff --git a/requirements.txt b/requirements.txt index 797f6f0..e755823 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,6 @@ pip>=21.1 pydantic>=1.8.2 pyproj==2.6.1.post1 python-Levenshtein>=0.12.2 -rasterio>=1.1.0 Rtree==0.8.3 shapely Sphinx==1.8.5 From 1759fe09ff48ee37be519278fbaf9d7ef9ce1016 Mon Sep 17 00:00:00 2001 From: Powell Date: Tue, 7 Feb 2023 10:23:18 -0500 Subject: [PATCH 13/15] 3 more transformations added, general improvements to main flow. --- mixmasta/mixmasta.py | 42 ++++++++++- mixmasta/normalizer.py | 4 +- mixmasta/transformations/clipping.py | 23 ++++++ mixmasta/transformations/regridding.py | 38 ++++++++++ mixmasta/transformations/scaling.py | 26 +++++++ tests/inputs/test4_rainfall_error.json | 96 +++++++++++++++++++++++++- 6 files changed, 226 insertions(+), 3 deletions(-) create mode 100644 mixmasta/transformations/regridding.py create mode 100644 mixmasta/transformations/scaling.py diff --git a/mixmasta/mixmasta.py b/mixmasta/mixmasta.py index 1b9cb29..b03892b 100644 --- a/mixmasta/mixmasta.py +++ b/mixmasta/mixmasta.py @@ -12,7 +12,8 @@ from .file_processor import process_file_by_filetype from .normalizer import normalizer from .feature_scaling import scale_dataframe -from .transformations.clipping import construct_multipolygon, clip_dataframe +from .transformations.clipping import construct_multipolygon, clip_dataframe, clip_time +from .transformations.scaling import scale_time if not sys.warnoptions: import warnings @@ -171,6 +172,45 @@ def clip_data(dataframe, geo_columns, polygons_list): return clip_dataframe(dataframe=dataframe, geo_columns=geo_columns, mask=mask) +def clip_dataframe_time(dataframe, time_column, time_ranges): + """Clips data in a dataframe based on a list of time ranges. + + Args: + dataframe (pandas.Dataframe): Dataframe with some time column that is the target for the clip + time_column (string): Name of target time column + time_ranges (List[Dict]): List of dictionaries containing "start" and "end" datetime values + + Returns: + _type_: _description_ + """ + + return clip_time( + dataframe=dataframe, time_column=time_column, time_ranges=time_ranges + ) + + +def rescale_dataframe_time( + dataframe, time_column, time_bucket, aggregation_function_list +): + """Rescales a dataframes time periodicity using aggregation functions. + + Args: + dataframe (pandas.Dataframe): A dataframe containing a column of time values to be rescaled + time_column (string): Name of target time column + time_bucket (DateOffset, Timedelta or str): Some time bucketing rule to lump the time in to. ex. 'M', 'A', '2H' + aggregation_function_list (List[strings]): List of aggregation functions to apply to the data. ex. ['sum'] or ['sum', 'min', 'max'] + + Returns: + _type_: _description_ + """ + return scale_time( + dataframe=dataframe, + time_column=time_column, + time_bucket=time_bucket, + aggregation_function_list=aggregation_function_list, + ) + + class mixdata: def load_gadm2(self): cdir = os.path.expanduser("~") diff --git a/mixmasta/normalizer.py b/mixmasta/normalizer.py index 090d86e..384309f 100644 --- a/mixmasta/normalizer.py +++ b/mixmasta/normalizer.py @@ -600,7 +600,9 @@ def normalizer( if feat in other_time_cols: df_out = df_out.append(df_.astype({"value": object})) else: - df_out = df_out.append(df_) + print(f"DF OUT: {df_out}") + print(f"DF PRIME: {df_}") + df_out = df_out.append(df_.reset_index(drop=True)) for c in col_order: if c not in df_out: diff --git a/mixmasta/transformations/clipping.py b/mixmasta/transformations/clipping.py index 4e2074b..0fa8d50 100644 --- a/mixmasta/transformations/clipping.py +++ b/mixmasta/transformations/clipping.py @@ -30,3 +30,26 @@ def clip_dataframe(dataframe, geo_columns, mask): ) return pandas.DataFrame(geopandas.clip(geo_dataframe, mask)) + + +def clip_time(dataframe, time_column, time_ranges): + """Removes rows in a dataset that lie outside of a specified time range. + + Args: + dataframe (pandas.Dataframe): Dataframe to drop rows from + time_column (string): name of the column in the dataframe that represents the time to check. + time_ranges (List[Object[start: datetime, end: datetime]]): A list of objects containing a start and end datetime to make a range. + """ + dataframe[time_column] = pandas.to_datetime(dataframe[time_column]) + + print(dataframe.dtypes) + + final_dataframe = pandas.DataFrame() + for start_end_datetime in time_ranges: + mask = (dataframe[time_column] > start_end_datetime["start"]) & ( + dataframe[time_column] <= start_end_datetime["end"] + ) + intermediate_frame = dataframe.loc[mask] + final_dataframe = final_dataframe.append(intermediate_frame) + + return final_dataframe diff --git a/mixmasta/transformations/regridding.py b/mixmasta/transformations/regridding.py new file mode 100644 index 0000000..fee60e8 --- /dev/null +++ b/mixmasta/transformations/regridding.py @@ -0,0 +1,38 @@ +import geopandas +import numpy +import pandas +from shapely.geometry import MultiPolygon, Polygon, box + + +def regrid_dataframe(dataframe, geo_columns): + x_geo = dataframe[geo_columns[0]] + y_geo = dataframe[geo_columns[1]] + geo_dataframe = geopandas.GeoDataFrame( + dataframe, geometry=geopandas.points_from_xy(x_geo, y_geo) + ) + + xmin, ymin, xmax, ymax = geo_dataframe.total_bounds + + n_cells = 30 + cell_size = (xmax - xmin) / n_cells + + grid_cells = [] + + for x0 in numpy.arange(xmin, xmax + cell_size, cell_size): + for y0 in numpy.arange(ymin, ymax + cell_size, cell_size): + # bounds + x1 = x0 - cell_size + y1 = y0 + cell_size + grid_cells.append(box(x0, y0, x1, y1)) + + cell = geopandas.GeoDataFrame(grid_cells, columns=["geometry"]) + + merged = geopandas.sjoin(geo_dataframe, cell, how="left", op="within") + print(f"MERGED: {merged}") + + dissolve = merged.dissolve(by="index_right", aggfunc="sum") + print(f"DISSOLVED: {dissolve}") + + cell.loc[dissolve.index, "fatalities"] = dissolve.fatalities.values + + print(f"REGRID DATA: {cell}") diff --git a/mixmasta/transformations/scaling.py b/mixmasta/transformations/scaling.py new file mode 100644 index 0000000..caddd0e --- /dev/null +++ b/mixmasta/transformations/scaling.py @@ -0,0 +1,26 @@ +import pandas + + +def scale_time(dataframe, time_column, time_bucket, aggregation_function_list): + """Scales timestamp data in a dataframe to a less granular time frequency + + Args: + dataframe (pandas.Dataframe): pandas dataframe with a timestamp field. + time_column (string): Name of the time column to scale in the dataframe. + time_bucket (DateOffset, Timedelta or str): The offset string or object representing target conversion. ex. "2H", "M" + aggregation_function (List[string]): A list of aggregation functions like sum, average, median, mean, mode, etc. Example: ['sum'], or ['min', 'max', 'sum'] + """ + + dataframe[time_column] = pandas.to_datetime(dataframe[time_column]) + + print(f"dataframe after datetime: {dataframe}") + print(dataframe.dtypes) + + # dataframe.set_index(time_column).resample(time_bucket).agg( + # aggregation_function_list + # ) + scaled_frame = dataframe.resample(time_bucket, on=time_column).agg( + aggregation_function_list + ) + + return scaled_frame diff --git a/tests/inputs/test4_rainfall_error.json b/tests/inputs/test4_rainfall_error.json index 574b842..1767b83 100644 --- a/tests/inputs/test4_rainfall_error.json +++ b/tests/inputs/test4_rainfall_error.json @@ -1 +1,95 @@ -{"geo": [{"name": "lat", "display_name": "lat", "description": "longitude of flood event", "type": "geo", "geo_type": "latitude", "primary_geo": true, "is_geo_pair": "long"}, {"name": "long", "display_name": "Longitude", "description": "longitude of flood event", "type": "geo", "geo_type": "longitude", "primary_geo": true, "is_geo_pair": "lat"}], "date": [{"name": "Began", "display_name": "Start Date", "description": "date that the flood event started", "type": "date", "date_type": "date", "time_format": "%Y-%m-%d", "primary_date": true}, {"name": "Ended", "display_name": "End Date", "description": "date that the flood ended", "type": "date", "date_type": "date", "time_format": "%Y-%m-%d"}], "feature": [{"name": "Area", "display_name": "Area Flooded", "description": "the amount of area flooded in square kilometers", "type": "feature", "feature_type": "float", "units": "square kilometers", "units_description": "square kilometers (area)"}, {"name": "Displaced", "display_name": "Number Displaced", "description": "number displaced by the flood event", "type": "feature", "feature_type": "float", "units": "people", "units_description": "people"}, {"name": "Severity", "display_name": "Severity of the Flood", "description": "severity of the flood event - higher number is worse", "type": "feature", "feature_type": "float", "units": "number", "units_description": "number"}, {"name": "Dead", "display_name": "Number of Dead", "description": "number of people that died from the flood event", "type": "feature", "feature_type": "float", "units": "people", "units_description": "people"}, {"name": "MainCause", "display_name": "Main Cause of the Flood", "description": "describes the main cause of the flood event", "type": "feature", "feature_type": "string", "units": "none", "units_description": "none", "qualifies": ["Displaced"]}], "meta": {"ftype": "excel", "sheet": "FloodArchive"}} \ No newline at end of file +{ + "geo": [ + { + "name": "lat", + "display_name": "lat", + "description": "longitude of flood event", + "type": "geo", + "geo_type": "latitude", + "primary_geo": true, + "is_geo_pair": "long" + }, + { + "name": "long", + "display_name": "Longitude", + "description": "longitude of flood event", + "type": "geo", + "geo_type": "longitude", + "primary_geo": true, + "is_geo_pair": "lat" + } + ], + "date": [ + { + "name": "Began", + "display_name": "Start Date", + "description": "date that the flood event started", + "type": "date", + "date_type": "date", + "time_format": "%Y-%m-%d", + "primary_date": true + }, + { + "name": "Ended", + "display_name": "End Date", + "description": "date that the flood ended", + "type": "date", + "date_type": "date", + "time_format": "%Y-%m-%d" + } + ], + "feature": [ + { + "name": "Area", + "display_name": "Area Flooded", + "description": "the amount of area flooded in square kilometers", + "type": "feature", + "feature_type": "float", + "units": "square kilometers", + "units_description": "square kilometers (area)" + }, + { + "name": "Displaced", + "display_name": "Number Displaced", + "description": "number displaced by the flood event", + "type": "feature", + "feature_type": "float", + "units": "people", + "units_description": "people" + }, + { + "name": "Severity", + "display_name": "Severity of the Flood", + "description": "severity of the flood event - higher number is worse", + "type": "feature", + "feature_type": "float", + "units": "number", + "units_description": "number" + }, + { + "name": "Dead", + "display_name": "Number of Dead", + "description": "number of people that died from the flood event", + "type": "feature", + "feature_type": "float", + "units": "people", + "units_description": "people" + }, + { + "name": "MainCause", + "display_name": "Main Cause of the Flood", + "description": "describes the main cause of the flood event", + "type": "feature", + "feature_type": "string", + "units": "none", + "units_description": "none", + "qualifies": [ + "Displaced" + ] + } + ], + "meta": { + "ftype": "excel", + "sheet": "FloodArchive" + } +} \ No newline at end of file From 308ac318269a9861c66a875d74915261475835b4 Mon Sep 17 00:00:00 2001 From: Powell Date: Wed, 8 Feb 2023 10:51:58 -0500 Subject: [PATCH 14/15] Added function to get boundary for UI --- mixmasta/mixmasta.py | 15 +++++++++++++++ mixmasta/transformations/geo_utils.py | 14 ++++++++++++++ mixmasta/transformations/regridding.py | 2 +- 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 mixmasta/transformations/geo_utils.py diff --git a/mixmasta/mixmasta.py b/mixmasta/mixmasta.py index b03892b..0ccecf0 100644 --- a/mixmasta/mixmasta.py +++ b/mixmasta/mixmasta.py @@ -14,6 +14,7 @@ from .feature_scaling import scale_dataframe from .transformations.clipping import construct_multipolygon, clip_dataframe, clip_time from .transformations.scaling import scale_time +from .transformations.geo_utils import calculate_boundary_box if not sys.warnoptions: import warnings @@ -211,6 +212,20 @@ def rescale_dataframe_time( ) +def get_boundary_box(dataframe, geo_columns): + """Returns the minimum and maximum x,y coordinates for a geographical dataset with latitude and longitude. + + Args: + dataframe (pandas.Dataframe): Pandas dataframe with latitude and longitude + geo_columns (List[string]): A list of the two column names that represent latitude and longitude. + + Returns: + Dict: An object containing key value pairs for xmin, xmax, ymin, and ymax. + """ + + return calculate_boundary_box(dataframe=dataframe, geo_columns=geo_columns) + + class mixdata: def load_gadm2(self): cdir = os.path.expanduser("~") diff --git a/mixmasta/transformations/geo_utils.py b/mixmasta/transformations/geo_utils.py new file mode 100644 index 0000000..b186c71 --- /dev/null +++ b/mixmasta/transformations/geo_utils.py @@ -0,0 +1,14 @@ +import geopandas + + +def calculate_boundary_box(dataframe, geo_columns): + x_geo = dataframe[geo_columns[0]] + y_geo = dataframe[geo_columns[1]] + geo_dataframe = geopandas.GeoDataFrame( + dataframe, geometry=geopandas.points_from_xy(x_geo, y_geo) + ) + + xmin, ymin, xmax, ymax = geo_dataframe.total_bounds + + boundary_dict = {"xmin": xmin, "xmax": xmax, "ymin": ymin, "ymax": ymax} + return boundary_dict diff --git a/mixmasta/transformations/regridding.py b/mixmasta/transformations/regridding.py index fee60e8..73654a7 100644 --- a/mixmasta/transformations/regridding.py +++ b/mixmasta/transformations/regridding.py @@ -1,7 +1,7 @@ import geopandas import numpy import pandas -from shapely.geometry import MultiPolygon, Polygon, box +from shapely.geometry import box def regrid_dataframe(dataframe, geo_columns): From 445c25a48cff9e26396d3a16141391a6eec67327 Mon Sep 17 00:00:00 2001 From: Powell Date: Wed, 15 Feb 2023 11:31:25 -0700 Subject: [PATCH 15/15] transformation upgrades, additional cleanup of normalization flow. --- mixmasta/mixmasta.py | 2 + mixmasta/time_helpers.py | 6 +- mixmasta/time_processor.py | 14 ++-- mixmasta/transformations/regridding.py | 98 +++++++++++++++++++------- tests/test_mixmasta.py | 4 ++ 5 files changed, 93 insertions(+), 31 deletions(-) diff --git a/mixmasta/mixmasta.py b/mixmasta/mixmasta.py index 0ccecf0..d437be0 100644 --- a/mixmasta/mixmasta.py +++ b/mixmasta/mixmasta.py @@ -168,6 +168,8 @@ def clip_data(dataframe, geo_columns, polygons_list): pandas.Dataframe: A pandas dataframe only containing the clipped data. """ + print(f"POLY LIST {polygons_list}") + mask = construct_multipolygon(polygons_list=polygons_list) return clip_dataframe(dataframe=dataframe, geo_columns=geo_columns, mask=mask) diff --git a/mixmasta/time_helpers.py b/mixmasta/time_helpers.py index ac94aa6..9fd2644 100644 --- a/mixmasta/time_helpers.py +++ b/mixmasta/time_helpers.py @@ -20,10 +20,12 @@ def date_type_handler(dataframe, date_dict): """ date_type = date_dict["date_type"] # print(f"DATE TYPE: {date_type}") - primary_date = date_dict.get("primary_date") + primary_date = date_dict.get("primary_date", False) date_column_name = date_dict["name"] if date_type == "date": - return add_date_to_dataframe_as_epoch(dataframe, date_dict, date_column_name) + return add_date_to_dataframe_as_epoch( + dataframe, date_dict, date_column_name, primary_date=primary_date + ) if date_type == "epoch": return rename_column_to_timestamp( dataframe=dataframe, original_date_column_name=date_column_name diff --git a/mixmasta/time_processor.py b/mixmasta/time_processor.py index c6319ba..50929cd 100644 --- a/mixmasta/time_processor.py +++ b/mixmasta/time_processor.py @@ -185,15 +185,19 @@ def build_date_qualifies_field(qualified_col_dict: dict, assoc_fields: list) -> return None -def add_date_to_dataframe_as_epoch(dataframe, date_dict, original_date_column_name): - # convert value of date_type date annotations to epochtime and rename column as 'timestamp' +def add_date_to_dataframe_as_epoch( + dataframe, date_dict, original_date_column_name, primary_date +): + # convert value of date_type date annotations to epochtime and rename column as 'timestamp' if it is primary date. dataframe.loc[:, original_date_column_name] = dataframe[ original_date_column_name ].apply(lambda x: format_time(str(x), date_dict["time_format"], validate=False)) - return rename_column_to_timestamp( - dataframe=dataframe, original_date_column_name=original_date_column_name - ) + if primary_date: + return rename_column_to_timestamp( + dataframe=dataframe, original_date_column_name=original_date_column_name + ) + return dataframe def rename_column_to_timestamp(dataframe, original_date_column_name): diff --git a/mixmasta/transformations/regridding.py b/mixmasta/transformations/regridding.py index 73654a7..ab91748 100644 --- a/mixmasta/transformations/regridding.py +++ b/mixmasta/transformations/regridding.py @@ -1,38 +1,88 @@ -import geopandas +import math +import xarray import numpy -import pandas -from shapely.geometry import box def regrid_dataframe(dataframe, geo_columns): - x_geo = dataframe[geo_columns[0]] - y_geo = dataframe[geo_columns[1]] - geo_dataframe = geopandas.GeoDataFrame( - dataframe, geometry=geopandas.points_from_xy(x_geo, y_geo) + """Uses xarray interpolation to regrid geography in a dataframe. + + Args: + dataframe (_type_): _description_ + geo_columns (_type_): _description_ + + Returns: + _type_: _description_ + """ + + ds = xarray.Dataset.from_dataframe(dataframe) + + geo_dim_0 = geo_columns[0] + "_dim" + geo_dim_1 = geo_columns[1] + "_dim" + + coord_dict = { + geo_columns[0]: (geo_dim_0, ds[geo_columns[0]].data), + geo_columns[1]: (geo_dim_1, ds[geo_columns[1]].data), + } + + ds = ds.assign_coords(**coord_dict) + + print(f"Xarray DS: {ds}") + + print( + f"SCALE VALUES: {ds[geo_columns[0]][0]} , {ds[geo_columns[1]][0]} , {ds[geo_columns[0]][1]} , {ds[geo_columns[1]][1]}" + ) + + ds_scale = getScale( + ds[geo_columns[0]][0], + ds[geo_columns[1]][0], + ds[geo_columns[0]][1], + ds[geo_columns[1]][1], + ) + + print(f"SCALE {ds_scale}") + + multiplier = ds_scale / 500 + print(f"MULTIPLIER {multiplier}") + + new_0 = numpy.linspace( + ds[geo_columns[0]][0], + ds[geo_columns[0]][-1], + round(ds.dims[geo_dim_0] * multiplier), + ) + new_1 = numpy.linspace( + ds[geo_columns[1]][0], + ds[geo_columns[1]][-1], + round(ds.dims[geo_dim_1] * multiplier), ) - xmin, ymin, xmax, ymax = geo_dataframe.total_bounds + interpolation = {geo_dim_0: new_0, geo_dim_1: new_1} - n_cells = 30 - cell_size = (xmax - xmin) / n_cells + ds2 = ds.interp(**interpolation) + print(ds2) - grid_cells = [] + p_dataframe = ds2.to_dataframe() - for x0 in numpy.arange(xmin, xmax + cell_size, cell_size): - for y0 in numpy.arange(ymin, ymax + cell_size, cell_size): - # bounds - x1 = x0 - cell_size - y1 = y0 + cell_size - grid_cells.append(box(x0, y0, x1, y1)) + return p_dataframe - cell = geopandas.GeoDataFrame(grid_cells, columns=["geometry"]) - merged = geopandas.sjoin(geo_dataframe, cell, how="left", op="within") - print(f"MERGED: {merged}") +def getScale(lat0, lon0, lat1, lon1): + """ + Description + ----------- + Return an estimation of the scale in km of a netcdf dataset. + The estimate is based on the first two data points, and returns + the scale distance at that lat/lon. - dissolve = merged.dissolve(by="index_right", aggfunc="sum") - print(f"DISSOLVED: {dissolve}") + """ + r = 6371 # Radius of the earth in km + dLat = numpy.radians(lat1 - lat0) + dLon = numpy.radians(lon1 - lon0) - cell.loc[dissolve.index, "fatalities"] = dissolve.fatalities.values + a = math.sin(dLat / 2) * math.sin(dLat / 2) + math.cos( + numpy.radians(lat0) + ) * math.cos(numpy.radians(lat1)) * math.sin(dLon / 2) * math.sin(dLon / 2) - print(f"REGRID DATA: {cell}") + c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) + d = r * c + # Distance in km + return d diff --git a/tests/test_mixmasta.py b/tests/test_mixmasta.py index 19450cb..b106da7 100644 --- a/tests/test_mixmasta.py +++ b/tests/test_mixmasta.py @@ -200,6 +200,8 @@ def test_003_process(self): def test_004_process(self): """Test .xlxs file, qualifies col with multi dtypes.""" + print("TEST 4 START") + # Define mixmasta inputs: mp = f"inputs{sep}test4_rainfall_error.json" fp = f"inputs{sep}test4_rainfall_error.xlsx" @@ -248,6 +250,8 @@ def test_004_process(self): assert_frame_equal(df, output_df, check_categorical=False) assert_dict_equal(dct, output_dict) + print("TEST 4 FINISH") + def test_005__command_line_interface(self): """Test the CLI and causemosify-multi.""" """