From f28b6fbba02ecbcff69debde442d332f1629d2b1 Mon Sep 17 00:00:00 2001 From: Alex Loosley Date: Mon, 5 Dec 2022 22:13:57 +0100 Subject: [PATCH 1/4] typer dependency --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 3c6cdd9..15dc056 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,7 @@ python_requires = >=3.7 install_requires = pot >= 0.8.0 pandas >= 1.0.0 + typer>=0.7.0 [options.packages.find] include=faim* From ea78ad0cfbdf47ace44a35a555539eede5dd88f4 Mon Sep 17 00:00:00 2001 From: Alex Loosley Date: Mon, 5 Dec 2022 22:14:02 +0100 Subject: [PATCH 2/4] version bump --- faim/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/faim/__init__.py b/faim/__init__.py index 27fdca4..3dc1f76 100644 --- a/faim/__init__.py +++ b/faim/__init__.py @@ -1 +1 @@ -__version__ = "0.0.3" +__version__ = "0.1.0" From 00875b8faaf41c6ae8eaee9bc4f8321830b60317 Mon Sep 17 00:00:00 2001 From: Alex Loosley Date: Mon, 5 Dec 2022 23:26:44 +0100 Subject: [PATCH 3/4] skeleton CLI --- faim/cli.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 faim/cli.py diff --git a/faim/cli.py b/faim/cli.py new file mode 100644 index 0000000..0958a4f --- /dev/null +++ b/faim/cli.py @@ -0,0 +1,46 @@ +from pathlib import Path +from typing import Tuple + +import typer + +app = typer.Typer(name="faim", help="FAIM CLI") +experiment_app = typer.Typer(name="experiment", help="reproduce experiments found in FAIM paper by Zehlike et al.") +app.add_typer(experiment_app) + +synthetic_two_group_normal_experiment_app = typer.Typer( + name="synthetic-two-group-normal", + help="experiment with synthetic two-group data generated with prediction and ground truth scores sampled " + "from a two dimensional normal distribution (synthetic dataset from in paper)", +) +experiment_app.add_typer(synthetic_two_group_normal_experiment_app) +compas_experiment_app = typer.Typer(name="compas", help="experiment with compas data") +experiment_app.add_typer(compas_experiment_app) +zalando_experiment_app = typer.Typer(name="zalando", help="experiment with Zalando data (coming soon)") +experiment_app.add_typer(zalando_experiment_app, name="zalando") + +train_app = typer.Typer(name="train", help="train a FAIM post processing model") +app.add_typer(train_app) + +post_process_app = typer.Typer(name="transform-scores", help="transform scores (with group-ids) given a FAIM model") +app.add_typer(post_process_app) + + +@synthetic_two_group_normal_experiment_app.command( + "prepare-data", help="Generate two group synthetic distribution sampling from a 2-D correlated normal distribution." +) +def prepare_synthetic_two_group_normal_experiment_dataset( + output_dir: Path = typer.Option(Path("prepared-data/synthetic-2groups"), help="prepared data output directory"), + group1: Tuple[str, int, float, float, float] = typer.Option( + ("disadvantaged", 100000, -1, -3, 0.8), + help="group 1 statistics: number of examples, mean ground truth score, mean predicted score, and covariance between the two", + ), + group2: Tuple[str, int, float, float, float] = typer.Option( + ("privileged", 100000, 1, 2, 0.8), + help="group 2 statistics: number of examples, mean ground truth score, mean predicted score, and covariance between the two", + ), +) -> None: + ... + + +if __name__ == "__main__": + app() From 6be7f7c9f6f84da7f29f9dadec87b88625f6bcb9 Mon Sep 17 00:00:00 2001 From: Alex Loosley Date: Tue, 6 Dec 2022 11:14:00 +0100 Subject: [PATCH 4/4] cli simplification --- faim/cli.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/faim/cli.py b/faim/cli.py index 0958a4f..8fc702c 100644 --- a/faim/cli.py +++ b/faim/cli.py @@ -7,12 +7,11 @@ experiment_app = typer.Typer(name="experiment", help="reproduce experiments found in FAIM paper by Zehlike et al.") app.add_typer(experiment_app) -synthetic_two_group_normal_experiment_app = typer.Typer( - name="synthetic-two-group-normal", - help="experiment with synthetic two-group data generated with prediction and ground truth scores sampled " - "from a two dimensional normal distribution (synthetic dataset from in paper)", +synthetic_two_group_binormal_experiment_app = typer.Typer( + name="synthetic-two-group-binormal", + help='experiment with post-processing synthetic prediction and ground truth scores for two groups, each sampled from a corresponding binormal distribution (corresponds to "experiment with synthetic" data in paper)', ) -experiment_app.add_typer(synthetic_two_group_normal_experiment_app) +experiment_app.add_typer(synthetic_two_group_binormal_experiment_app) compas_experiment_app = typer.Typer(name="compas", help="experiment with compas data") experiment_app.add_typer(compas_experiment_app) zalando_experiment_app = typer.Typer(name="zalando", help="experiment with Zalando data (coming soon)") @@ -25,18 +24,18 @@ app.add_typer(post_process_app) -@synthetic_two_group_normal_experiment_app.command( - "prepare-data", help="Generate two group synthetic distribution sampling from a 2-D correlated normal distribution." +@synthetic_two_group_binormal_experiment_app.command( + "prepare-data", help="generate synthetic prediction and ground truth scores for two groups, each sampled from a corresponding binormal distribution" ) def prepare_synthetic_two_group_normal_experiment_dataset( output_dir: Path = typer.Option(Path("prepared-data/synthetic-2groups"), help="prepared data output directory"), group1: Tuple[str, int, float, float, float] = typer.Option( ("disadvantaged", 100000, -1, -3, 0.8), - help="group 1 statistics: number of examples, mean ground truth score, mean predicted score, and covariance between the two", + help="NAME, N_EXAMPLES, MEAN GROUND TRUTH SCORE, MEAN PREDICTION SCORE, COV(GT, PRED)", ), group2: Tuple[str, int, float, float, float] = typer.Option( ("privileged", 100000, 1, 2, 0.8), - help="group 2 statistics: number of examples, mean ground truth score, mean predicted score, and covariance between the two", + help="NAME, N_EXAMPLES, MEAN GROUND TRUTH SCORE, MEAN PREDICTION SCORE, COV(GT, PRED)", ), ) -> None: ...