From f416f8064bc9229585ca78c23f864dd39b71c36a Mon Sep 17 00:00:00 2001 From: Hari Date: Sat, 27 Jun 2026 21:05:48 +0530 Subject: [PATCH] add requirements.txt , eda-checks, pending setup queries --- ecommerce_dbt/analyses/EDA-checks.sql | 219 ++++++++++++++++++++++++++ requirements.txt | Bin 0 -> 3072 bytes snowflake/04_frankfurter_setup.sql | 21 +++ 3 files changed, 240 insertions(+) create mode 100644 ecommerce_dbt/analyses/EDA-checks.sql create mode 100644 requirements.txt create mode 100644 snowflake/04_frankfurter_setup.sql diff --git a/ecommerce_dbt/analyses/EDA-checks.sql b/ecommerce_dbt/analyses/EDA-checks.sql new file mode 100644 index 0000000..4ed0bb2 --- /dev/null +++ b/ecommerce_dbt/analyses/EDA-checks.sql @@ -0,0 +1,219 @@ +LIST @ECOMMERCE_BRONZE_DB.OLIST.s3_olist_stage; + + +SELECT CURRENT_REGION(); + +USE ROLE ACCOUNTADMIN; +LIST @ECOMMERCE_BRONZE_DB.OLIST.s3_olist_stage; + + +SHOW PIPES; + + + + + +USE ROLE ACCOUNTADMIN; + +ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_CUSTOMERS REFRESH; +ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_ORDERS REFRESH; +ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_ORDER_ITEMS REFRESH; +ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_ORDER_PAYMENTS REFRESH; +ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_ORDER_REVIEWS REFRESH; +ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_PRODUCTS REFRESH; +ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_SELLERS REFRESH; + + + + +SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.CUSTOMERS_RAW; +SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.ORDERS_RAW; +SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.PRODUCTS_RAW; + + + + +SELECT + _raw_data:customer_id::TEXT AS customer_id, + _raw_data:customer_city::TEXT AS customer_city, + _raw_data:customer_state::TEXT AS customer_state, + _stg_file_name, + _stg_load_time +FROM ECOMMERCE_BRONZE_DB.OLIST.CUSTOMERS_RAW +LIMIT 5; + + + + + +USE ROLE ACCOUNTADMIN; +USE DATABASE ECOMMERCE_BRONZE_DB; +USE SCHEMA OLIST; + +TRUNCATE TABLE CUSTOMERS_RAW; +TRUNCATE TABLE ORDERS_RAW; +TRUNCATE TABLE ORDER_ITEMS_RAW; +TRUNCATE TABLE ORDER_PAYMENTS_RAW; +TRUNCATE TABLE ORDER_REVIEWS_RAW; +TRUNCATE TABLE PRODUCTS_RAW; +TRUNCATE TABLE SELLERS_RAW; + + +SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.CUSTOMERS_RAW; +SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.ORDERS_RAW; +SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.ORDER_ITEMS_RAW; +SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.ORDER_PAYMENTS_RAW; +SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.ORDER_REVIEWS_RAW; +SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.PRODUCTS_RAW; +SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.SELLERS_RAW; + + +-- TRUNCATE TABLE ECOMMERCE_BRONZE_DB.OLIST.ORDER_ITEMS_RAW; +-- TRUNCATE TABLE ECOMMERCE_BRONZE_DB.OLIST.ORDER_PAYMENTS_RAW; + + +USE ROLE ACCOUNTADMIN; +SELECT * FROM ECOMMERCE_BRONZE_DB.OLIST.STG_CUSTOMERS LIMIT 5; + + +SHOW VIEWS IN DATABASE ECOMMERCE_BRONZE_DB; + + +USE ROLE ACCOUNTADMIN; + +DROP SCHEMA IF EXISTS ECOMMERCE_BRONZE_DB.COMMERCE; +DROP SCHEMA IF EXISTS ECOMMERCE_BRONZE_DB.COMMERCE_OLIST; + + + +SELECT * FROM ECOMMERCE_SILVER_DB.COMMERCE.CUSTOMERS LIMIT 5; + + + +SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.CUSTOMERS; +SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDERS; +SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDER_ITEMS; +SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDER_PAYMENTS; + +-- TRUNCATE TABLE ECOMMERCE_SILVER_DB.COMMERCE.ORDER_ITEMS; + + +-- TRUNCATE TABLE ECOMMERCE_SILVER_DB.COMMERCE.ORDER_ITEMS; +-- TRUNCATE TABLE ECOMMERCE_SILVER_DB.COMMERCE.ORDER_PAYMENTS; + + + +SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.ORDER_ITEMS_RAW; +SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.ORDER_PAYMENTS_RAW; + + +USE ROLE ACCOUNTADMIN; + +ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_ORDER_ITEMS REFRESH; +ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_ORDER_PAYMENTS REFRESH; + + + + +USE ROLE ACCOUNTADMIN; + +TRUNCATE TABLE ECOMMERCE_BRONZE_DB.OLIST.ORDER_ITEMS_RAW; +COPY INTO ECOMMERCE_BRONZE_DB.OLIST.ORDER_ITEMS_RAW (_raw_data, _stg_file_name) +FROM ( + SELECT $1, METADATA$FILENAME + FROM @ECOMMERCE_BRONZE_DB.OLIST.s3_olist_stage/order_items/ +) +FILE_FORMAT = (TYPE = PARQUET); + +TRUNCATE TABLE ECOMMERCE_BRONZE_DB.OLIST.ORDER_PAYMENTS_RAW; +COPY INTO ECOMMERCE_BRONZE_DB.OLIST.ORDER_PAYMENTS_RAW (_raw_data, _stg_file_name) +FROM ( + SELECT $1, METADATA$FILENAME + FROM @ECOMMERCE_BRONZE_DB.OLIST.s3_olist_stage/order_payments/ +) +FILE_FORMAT = (TYPE = PARQUET); + + + + +SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.CUSTOMERS; +SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDERS; +SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDER_ITEMS; +SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDER_PAYMENTS; +SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDER_REVIEWS; +SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.PRODUCTS; +SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.SELLERS; + + + +SELECT * FROM ECOMMERCE_GOLD_DB.SALES.FACT_ORDERS LIMIT 5; + + + +SELECT customer_segment, COUNT(*) as customer_count +FROM ECOMMERCE_GOLD_DB.SALES.DIM_CUSTOMERS +GROUP BY customer_segment +ORDER BY customer_count DESC; + + +SELECT performance_tier, COUNT(*) as product_count, ROUND(SUM(total_revenue), 2) as total_revenue +FROM ECOMMERCE_GOLD_DB.PRODUCT.FACT_PRODUCT_PERFORMANCE +GROUP BY performance_tier +ORDER BY total_revenue DESC; + + +USE ROLE ECOMMERCE_ENGINEER; +USE DATABASE ECOMMERCE_BRONZE_DB; + +CREATE TABLE IF NOT EXISTS FRANKFURTER.EXCHANGE_RATES_RAW ( + rate_date DATE, + base_currency VARCHAR(10), + target_currency VARCHAR(10), + rate FLOAT, + _loaded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP() +); + + +GRANT USAGE ON WAREHOUSE ECOMMERCE_BRONZE_WH TO ROLE ECOMMERCE_ENGINEER; + +SELECT * FROM ECOMMERCE_BRONZE_DB.FRANKFURTER.EXCHANGE_RATES_RAW +ORDER BY rate_date DESC +LIMIT 10; + + +SELECT + order_date, + COUNT(*) AS orders, + ROUND(SUM(total_brl), 2) AS total_brl, + ROUND(SUM(total_usd), 2) AS total_usd +FROM ECOMMERCE_GOLD_DB.SALES.FACT_REVENUE_USD +GROUP BY order_date +ORDER BY order_date DESC +LIMIT 100; + + + +TRUNCATE TABLE ECOMMERCE_BRONZE_DB.FRANKFURTER.EXCHANGE_RATES_RAW; + +SELECT + order_date, + COUNT(*) AS orders, + ROUND(SUM(total_brl), 2) AS total_brl, + ROUND(SUM(total_usd), 2) AS total_usd +FROM ECOMMERCE_GOLD_DB.SALES.FACT_REVENUE_USD +GROUP BY order_date +ORDER BY order_date DESC +LIMIT 10; + + + +SELECT DISTINCT order_status +FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDERS; + + +ALTER USER HARISNOWFLAKEACC SET MINS_TO_UNLOCK = 0; + +SHOW GRANTS TO USER HARISNOWFLAKEACC; + + +GRANT ROLE ECOMMERCE_DBT_RUNNER TO USER HARISNOWFLAKEACC; \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..dfaf0715c775d7ec8fac1822fc9a1494e08ab671 GIT binary patch literal 3072 zcmZvePmkk75XJk9#7D_!<79W?um>(2R;)l;B*bZyILT~g633fAK<2{(?^n+??FqCFq0C-z=X$== z{Vo{fdMmr*D5lp|tyYfWuq_+iFl-b-Q_K-G$KPtbTEH+_w&ja>62n?JlgQGs?DcLZ z#OJ^V7jARi?}J?%t8IA-pG3u4Gprr73hPDSup56^IJR+CAHuE|i)&!bqiU7kN+@v8 z>{mZ4Wf#Vaf=#bya^Y%H{bs}|ZOP_d*FkY6;;huFjP1r*pN5|JMNMh7E`P}@akly= z;*1p~SV?KyYZd(+MhuDatuW`WP^kp@?_|9d@cLsUDvoD9;nX8fpKuEs&{(C9)t$fs42 z8aDr1Ycv5aHN|&!^Z#LQ%0z5oZL$~WQ2Gz;TlsWO&GaEC3*tn{O^QyR!@NXqzb1$*BC{1!KNRk5YxI=l3n!6QI)=mlV>Fu*i<`e ziGNO_aSm%wVbO(xt0HS&SoI`kq7gR}fWE*NoZP#qnXECZpiINYmFPqLXH@|9d->c3 zHnu3p8MM;sY+5P{p0#kEMcvZpNwG6OdqvkM3(JTR#-7~pvzFX3gR%C9f^GFC=;EkDcaS{^^^`f0Foq}g?N zq#x<1N${f%m#9QN$vug;3O?(yE zTBs_)E@y`4s@XpWf;)Vyy{kF^UCyF@Ot0H7ATsJ6C6~XKAIhg8PvBIgQUQ9Kf97CS zyKU;kz~ZZ7V+NmfXO(Be93#4;^0?C4Bs5_QzGCo6cUjRVG4pB5Otg1wt^9gQoW#Z* z?%Y%^|2-Kep(nl#nUy}n%_3`q2l~bF!!}XqFwR-NBz@{IKdthtigm$!=A>IIN;(L| zFCu5y*PP%dcYGE4>q--+=M5}Cx1F$