main.py

#DATA CLEANING APPLICATION

"""
This app takes datasets and cleans the data
It will ask for:
-Datasets path and name
-It should check the number of duplicates and remove all the duplicates
-It should keep a copy of all the duplicate
-It should check for missing values
-Of any columns is numeric it should replace nulls with mean else it would drop it
-At the end it would save the data as clean data and return duplicates records
"""

#importing necessary depedencies
import pandas as pd
import numpy as np
import time
import openpyxl
import xlrd
import os
import  random

#data_path='Datasets/sales.xlsx'
#data_name='sales_data'

output_folder='Results'

def data_cleaner(data_path,data_name):

    print("Thank you for using this app")



    #Print delay message
    print(f"Checking file type.Please wait {random.randint(1,5)} seconds...")
    time.sleep(random.randint(1,5))

    if not os.path.exists(data_path):
        print("Wrong path")

    else:
        #Checking the file type
        if data_path.endswith('.csv'):
            print("Dataset type csv")
            data=pd.read_csv(data_path)
        elif data_path.endswith('.xlsx'):
            print("Dataset type excel")
            data=pd.read_excel(data_path)
        else:
            print("Dataset type unknown")

    print(f"Checking columns.Please wait {random.randint(1,5)} seconds...")
    time.sleep(random.randint(1,5))

    #Showing number of records

    print(f"Dataset contains total rows: {data.shape[0]} \n total columns: {data.shape[1]}")


    #Data cleaning

    duplicates=data.duplicated()
    total_duplicate=data.duplicated().sum()

    print(f"Total duplicate rows: {total_duplicate}")

    #Saving the duplicates
    if total_duplicate > 0:
        duplicates_records=data[duplicates]
        file_path_duplicates = os.path.join(output_folder, f'{data_name}_duplicates.csv')
        duplicates_records.to_csv(file_path_duplicates,index=None)

    #Deleting duplicates

    print(f"Cleaning duplicates.Please wait {random.randint(1,5)} seconds...")
    time.sleep(random.randint(1,5))

    df = data.drop_duplicates()

    #Find missing values
    total_missing_values = df.isnull().sum().sum()
    missing_value_by_columns = df.isnull().sum()

    print(f"Total missing values: {total_missing_values}")
    print(f"Dataset contains missing values: {missing_value_by_columns}")

    #Dealing with missing values
    #fillna -- int and float
    #dropna -- any object

    print(f"Cleaning the dataset.Please wait {random.randint(1,5)} seconds...")
    time.sleep(random.randint(1,5))

    columns = df.columns

    for col in columns:

        #Filling mean for numeric columns all rows
        if df[col].dtype in (float,int):
            df[col].fillna(df[col].mean())

        else:
            #Dropping all rows with missing records for non number columns
            df.dropna(subset=[col],inplace=True)

    #Data is cleaned
    print(f"Dataset cleaned! Number of rows {df.shape[0]} and number of columns {df.shape[1]}")

    #Saving the clean dataset
    file_path_cleaned = os.path.join(output_folder, f'{data_name}_cleaned.csv')
    df.to_csv(file_path_cleaned,index=None)
    print("Dataset saved!")

if __name__ == '__main__':
    #Ask the path and file name

    data_path=input("Please enter dataset path")
    data_name=input("Please enter dataset name")

    #Calling the function
    data_cleaner(data_path,data_name)