add requirements.txt , eda-checks, pending setup queries

ecommerce_dbt/analyses/EDA-checks.sql

@@ -0,0 +1,219 @@
+LIST @ECOMMERCE_BRONZE_DB.OLIST.s3_olist_stage;
+
+
+SELECT CURRENT_REGION();
+
+USE ROLE ACCOUNTADMIN;
+LIST @ECOMMERCE_BRONZE_DB.OLIST.s3_olist_stage;
+
+
+SHOW PIPES;
+
+
+
+
+
+USE ROLE ACCOUNTADMIN;
+
+ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_CUSTOMERS REFRESH;
+ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_ORDERS REFRESH;
+ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_ORDER_ITEMS REFRESH;
+ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_ORDER_PAYMENTS REFRESH;
+ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_ORDER_REVIEWS REFRESH;
+ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_PRODUCTS REFRESH;
+ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_SELLERS REFRESH;
+
+
+
+
+SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.CUSTOMERS_RAW;
+SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.ORDERS_RAW;
+SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.PRODUCTS_RAW;
+
+
+
+
+SELECT 
+  _raw_data:customer_id::TEXT AS customer_id,
+  _raw_data:customer_city::TEXT AS customer_city,
+  _raw_data:customer_state::TEXT AS customer_state,
+  _stg_file_name,
+  _stg_load_time
+FROM ECOMMERCE_BRONZE_DB.OLIST.CUSTOMERS_RAW
+LIMIT 5;
+
+
+
+
+
+USE ROLE ACCOUNTADMIN;
+USE DATABASE ECOMMERCE_BRONZE_DB;
+USE SCHEMA OLIST;
+
+TRUNCATE TABLE CUSTOMERS_RAW;
+TRUNCATE TABLE ORDERS_RAW;
+TRUNCATE TABLE ORDER_ITEMS_RAW;
+TRUNCATE TABLE ORDER_PAYMENTS_RAW;
+TRUNCATE TABLE ORDER_REVIEWS_RAW;
+TRUNCATE TABLE PRODUCTS_RAW;
+TRUNCATE TABLE SELLERS_RAW;
+
+
+SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.CUSTOMERS_RAW;
+SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.ORDERS_RAW;
+SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.ORDER_ITEMS_RAW;
+SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.ORDER_PAYMENTS_RAW;
+SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.ORDER_REVIEWS_RAW;
+SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.PRODUCTS_RAW;
+SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.SELLERS_RAW;
+
+
+-- TRUNCATE TABLE ECOMMERCE_BRONZE_DB.OLIST.ORDER_ITEMS_RAW;
+-- TRUNCATE TABLE ECOMMERCE_BRONZE_DB.OLIST.ORDER_PAYMENTS_RAW;
+
+
+USE ROLE ACCOUNTADMIN;
+SELECT * FROM ECOMMERCE_BRONZE_DB.OLIST.STG_CUSTOMERS LIMIT 5;
+
+
+SHOW VIEWS IN DATABASE ECOMMERCE_BRONZE_DB;
+
+
+USE ROLE ACCOUNTADMIN;
+
+DROP SCHEMA IF EXISTS ECOMMERCE_BRONZE_DB.COMMERCE;
+DROP SCHEMA IF EXISTS ECOMMERCE_BRONZE_DB.COMMERCE_OLIST;
+
+
+
+SELECT * FROM ECOMMERCE_SILVER_DB.COMMERCE.CUSTOMERS LIMIT 5;
+
+
+
+SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.CUSTOMERS;
+SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDERS;
+SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDER_ITEMS;
+SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDER_PAYMENTS;
+
+-- TRUNCATE TABLE ECOMMERCE_SILVER_DB.COMMERCE.ORDER_ITEMS;
+
+
+-- TRUNCATE TABLE ECOMMERCE_SILVER_DB.COMMERCE.ORDER_ITEMS;
+-- TRUNCATE TABLE ECOMMERCE_SILVER_DB.COMMERCE.ORDER_PAYMENTS;
+
+
+
+SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.ORDER_ITEMS_RAW;
+SELECT COUNT(*) FROM ECOMMERCE_BRONZE_DB.OLIST.ORDER_PAYMENTS_RAW;
+
+
+USE ROLE ACCOUNTADMIN;
+
+ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_ORDER_ITEMS REFRESH;
+ALTER PIPE ECOMMERCE_BRONZE_DB.OLIST.PIPE_ORDER_PAYMENTS REFRESH;
+
+
+
+
+USE ROLE ACCOUNTADMIN;
+
+TRUNCATE TABLE ECOMMERCE_BRONZE_DB.OLIST.ORDER_ITEMS_RAW;
+COPY INTO ECOMMERCE_BRONZE_DB.OLIST.ORDER_ITEMS_RAW (_raw_data, _stg_file_name)
+FROM (
+  SELECT $1, METADATA$FILENAME
+  FROM @ECOMMERCE_BRONZE_DB.OLIST.s3_olist_stage/order_items/
+)
+FILE_FORMAT = (TYPE = PARQUET);
+
+TRUNCATE TABLE ECOMMERCE_BRONZE_DB.OLIST.ORDER_PAYMENTS_RAW;
+COPY INTO ECOMMERCE_BRONZE_DB.OLIST.ORDER_PAYMENTS_RAW (_raw_data, _stg_file_name)
+FROM (
+  SELECT $1, METADATA$FILENAME
+  FROM @ECOMMERCE_BRONZE_DB.OLIST.s3_olist_stage/order_payments/
+)
+FILE_FORMAT = (TYPE = PARQUET);
+
+
+
+
+SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.CUSTOMERS;
+SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDERS;
+SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDER_ITEMS;
+SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDER_PAYMENTS;
+SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDER_REVIEWS;
+SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.PRODUCTS;
+SELECT COUNT(*) FROM ECOMMERCE_SILVER_DB.COMMERCE.SELLERS;
+
+
+
+SELECT * FROM ECOMMERCE_GOLD_DB.SALES.FACT_ORDERS LIMIT 5;
+
+
+
+SELECT customer_segment, COUNT(*) as customer_count
+FROM ECOMMERCE_GOLD_DB.SALES.DIM_CUSTOMERS
+GROUP BY customer_segment
+ORDER BY customer_count DESC;
+
+
+SELECT performance_tier, COUNT(*) as product_count, ROUND(SUM(total_revenue), 2) as total_revenue
+FROM ECOMMERCE_GOLD_DB.PRODUCT.FACT_PRODUCT_PERFORMANCE
+GROUP BY performance_tier
+ORDER BY total_revenue DESC;
+
+
+USE ROLE ECOMMERCE_ENGINEER;
+USE DATABASE ECOMMERCE_BRONZE_DB;
+
+CREATE TABLE IF NOT EXISTS FRANKFURTER.EXCHANGE_RATES_RAW (
+    rate_date       DATE,
+    base_currency   VARCHAR(10),
+    target_currency VARCHAR(10),
+    rate            FLOAT,
+    _loaded_at      TIMESTAMP DEFAULT CURRENT_TIMESTAMP()
+);
+
+
+GRANT USAGE ON WAREHOUSE ECOMMERCE_BRONZE_WH TO ROLE ECOMMERCE_ENGINEER;
+
+SELECT * FROM ECOMMERCE_BRONZE_DB.FRANKFURTER.EXCHANGE_RATES_RAW
+ORDER BY rate_date DESC
+LIMIT 10;
+
+
+SELECT
+    order_date,
+    COUNT(*) AS orders,
+    ROUND(SUM(total_brl), 2) AS total_brl,
+    ROUND(SUM(total_usd), 2) AS total_usd
+FROM ECOMMERCE_GOLD_DB.SALES.FACT_REVENUE_USD
+GROUP BY order_date
+ORDER BY order_date DESC
+LIMIT 100;
+
+
+
+TRUNCATE TABLE ECOMMERCE_BRONZE_DB.FRANKFURTER.EXCHANGE_RATES_RAW;
+
+SELECT
+    order_date,
+    COUNT(*) AS orders,
+    ROUND(SUM(total_brl), 2) AS total_brl,
+    ROUND(SUM(total_usd), 2) AS total_usd
+FROM ECOMMERCE_GOLD_DB.SALES.FACT_REVENUE_USD
+GROUP BY order_date
+ORDER BY order_date DESC
+LIMIT 10;
+
+
+
+SELECT DISTINCT order_status
+FROM ECOMMERCE_SILVER_DB.COMMERCE.ORDERS;
+
+
+ALTER USER HARISNOWFLAKEACC SET MINS_TO_UNLOCK = 0;
+
+SHOW GRANTS TO USER HARISNOWFLAKEACC;
+
+
+GRANT ROLE ECOMMERCE_DBT_RUNNER TO USER HARISNOWFLAKEACC;

snowflake/04_frankfurter_setup.sql

@@ -0,0 +1,21 @@
+
+USE ROLE ACCOUNTADMIN;
+
+
+CREATE SCHEMA IF NOT EXISTS ECOMMERCE_BRONZE_DB.FRANKFURTER;
+
+
+GRANT USAGE ON SCHEMA ECOMMERCE_BRONZE_DB.FRANKFURTER TO ROLE ECOMMERCE_ENGINEER;
+GRANT CREATE TABLE ON SCHEMA ECOMMERCE_BRONZE_DB.FRANKFURTER TO ROLE ECOMMERCE_ENGINEER;
+
+
+USE ROLE ECOMMERCE_ENGINEER;
+USE DATABASE ECOMMERCE_BRONZE_DB;
+
+CREATE TABLE IF NOT EXISTS FRANKFURTER.EXCHANGE_RATES_RAW (
+    rate_date       DATE,
+    base_currency   VARCHAR(10),
+    target_currency VARCHAR(10),
+    rate            FLOAT,
+    _loaded_at      TIMESTAMP DEFAULT CURRENT_TIMESTAMP()
+);