riskpr/Makefile at main · felipesassi/riskpr · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
include .env

n_customers := 1000
created_at_start_str := 2024-01-01 00:00:00
created_at_end_str := 2025-01-01 00:00:00


.PHONY: format
format:
	echo 'Formating Python scripts using Ruff'
	uvx ruff check --select I --fix
	uvx ruff format scripts/ analytics/ processor/


.PHONY: data
data:
	echo 'Running customers generation script: number of customers: $(n_customers)'
	uv run scripts/generate_customers.py             \
	--n_customers=$(n_customers) 		             \
	--created_at_start_str='$(created_at_start_str)' \
	--created_at_end_str='$(created_at_end_str)'

	echo 'Running transactions generation script'
	uv run scripts/generate_transactions.py


.PHONY: topics
topics:
	echo 'Creating Kafka topics'
	uv run scripts/create_topics.py


.PHONY: experiments
experiments:
	echo 'Loading experiments to database'
	uv run scripts/generate_experiments.py


.PHONY: rules
rules:
	echo 'Loading rules to database'
	uv run scripts/generate_rules.py


.PHONY: data_lake
data_lake:
	echo 'Running MinIO to generate credentials that needs to be updated in the .env file'
	docker exec riskpr_minio_1 mc alias set minio http://localhost:9000 ${MINIO_USER_NAME} ${MINIO_PASSWORD}
	docker exec riskpr_minio_1 mc admin accesskey create --access-key '${S3_ACCESS_KEY}' --secret-key '${S3_SECRET_ACCESS_KEY}' minio
	docker exec riskpr_minio_1 mc mb minio/raw


.PHONY: processors
processors:
	echo 'Starting Flink risk processor'
	docker exec -it riskpr_jobmanager_1 ./bin/flink run --detached --pyModule jobs.processor.main --pyFiles ./jobs/processor
	echo 'Starting Flink analytics processor'
	docker exec -it riskpr_jobmanager_1 ./bin/flink run --detached --pyModule jobs.analytics.main --pyFiles ./jobs/analytics


.PHONY: setup
setup:
	make data n_customers=$(n_customers) created_at_start_str='$(created_at_start_str)' created_at_end_str='$(created_at_end_str)'
	make clear
	make topics
	make experiments
	make rules
	-make data_lake
	make processors


.PHONY: stream
stream:
	uv run scripts/producer.py


.PHONY: clear
clear:
	echo 'Cleaning Kafka broker (removing data and topics)'
	uv run scripts/delete_topics.py
	echo 'Cleaning Postgres using a trucate command to drop all the data'
	psql "postgresql://risk_database:risk_database@localhost:5432/postgres" -c 'truncate transactions, postprocessed_transactions, features, rules, experiments;'
	echo 'Cleaning data lake'
	rm -rf data_lake/raw/artifacts


.PHONY: stop
stop:
	make clear
	echo 'Deactivating containers'
	docker-compose down