diff --git a/backend/protzilla/data_preprocessing/normalisation.py b/backend/protzilla/data_preprocessing/normalisation.py index 6c70ebf8..1de85485 100644 --- a/backend/protzilla/data_preprocessing/normalisation.py +++ b/backend/protzilla/data_preprocessing/normalisation.py @@ -1,6 +1,7 @@ import logging import pandas as pd +import numpy as np from sklearn.preprocessing import StandardScaler from backend.protzilla.data_preprocessing.plots import ( @@ -51,6 +52,7 @@ def by_z_score(protein_df: pd.DataFrame) -> dict: def by_median( protein_df: pd.DataFrame, + log: bool, percentile=0.5, # quartile, default is median ) -> dict: """ @@ -62,6 +64,8 @@ def by_median( :param protein_df: the dataframe that should be filtered in long format :type protein_df: pandas DataFrame + :param log: whether the data was log transformed before the normalisation or not + :type log: bool :param percentile: the chosen quartile of the sample intensities for normalisation :type percentile: float @@ -84,17 +88,33 @@ def by_median( samples = protein_df["Sample"].unique().tolist() zeroed_samples = [] + if log: + valid_mask = np.isfinite(protein_df[intensity_name]) + valid_data = protein_df.loc[valid_mask, intensity_name] + global_median = valid_data.median() + for sample in samples: df_sample = protein_df.loc[protein_df["Sample"] == sample,] quantile = df_sample[intensity_name].quantile(q=percentile) - if quantile != 0: - df_sample[f"Normalised {intensity_name}"] = df_sample[intensity_name].div( - quantile - ) + if log: + if np.isfinite(quantile): + # without adding the global median our data would be zero centered and therefore one half would be + # negative which can lead to problems later down the workflow + df_sample[f"Normalised {intensity_name}"] = ( + df_sample[intensity_name] - quantile + global_median + ) + else: + df_sample[f"Normalised {intensity_name}"] = 0 + zeroed_samples.append(sample) else: - df_sample[f"Normalised {intensity_name}"] = 0 - zeroed_samples.append(sample) + if quantile != 0: + df_sample[f"Normalised {intensity_name}"] = df_sample[ + intensity_name + ].div(quantile) + else: + df_sample[f"Normalised {intensity_name}"] = 0 + zeroed_samples.append(sample) df_sample.drop(axis=1, labels=[intensity_name], inplace=True) scaled_df = pd.concat([scaled_df, df_sample], ignore_index=True) diff --git a/backend/protzilla/methods/data_preprocessing.py b/backend/protzilla/methods/data_preprocessing.py index 56cd14c7..e233f9c2 100644 --- a/backend/protzilla/methods/data_preprocessing.py +++ b/backend/protzilla/methods/data_preprocessing.py @@ -644,6 +644,16 @@ def create_form(self): max=1, step=0.1, ), + CheckboxField( + name="log", + label="Data was log-transformed before normalization", + value=False, + ), + InfoField( + name="log_before_normalisation_info", + label="The normalisation is calculated differently for log-transformed data, " + "using subtraction instead of division.", + ), FormDivider("Plot settings"), DropdownField( name="graph_type", @@ -673,6 +683,14 @@ def create_form(self): ], ) + def modify_form(self, run): + if self.form["log"].value: + self.form["visual_transformation"].value = ( + VisualTransformations.LINEAR.value + ) + elif not self.form["log"].value: + self.form["visual_transformation"].value = VisualTransformations.LOG10.value + calc_method = staticmethod(normalisation.by_median) plot_method = staticmethod(normalisation.by_median_plot) diff --git a/backend/tests/protzilla/data_preprocessing/test_normalisation.py b/backend/tests/protzilla/data_preprocessing/test_normalisation.py index 7df93e30..4db915f3 100644 --- a/backend/tests/protzilla/data_preprocessing/test_normalisation.py +++ b/backend/tests/protzilla/data_preprocessing/test_normalisation.py @@ -49,6 +49,26 @@ def normalisation_df(): ) +@pytest.fixture +def input_log_normalisation_df(): + """Provides a mocked log-transformed dataset with -inf for failed runs.""" + input_df = pd.DataFrame( + data=( + ["Sample_1", "Gene_1", -np.inf, -np.inf, -np.inf], + ["Sample_2", "Gene_2", 12.0, 14.0, 16.0], + ["Sample_3", "Gene_3", 14.0, 16.0, 18.0], + ["Sample_4", "Gene_4", -np.inf, -np.inf, -np.inf], + ), + columns=["Sample", "Gene", "Protein_1", "Protein_2", "Protein_3"], + ) + return pd.melt( + input_df, + id_vars=["Sample", "Gene"], + var_name="Protein ID", + value_name="Intensity", + ).sort_values(by=["Sample", "Protein ID"], ignore_index=True) + + @pytest.fixture def normalisation_by_ref_protein_df(): intensity_df = pd.DataFrame( @@ -208,6 +228,25 @@ def expected_df_by_median_normalisation(): ).sort_values(by=["Sample", "Protein ID"], ignore_index=True) +@pytest.fixture +def expected_df_by_median_log_normalisation(): + expected_df = pd.DataFrame( + data=( + ["Sample_1", "Gene_1", 0.0, 0.0, 0.0], + ["Sample_2", "Gene_2", 13.0, 15.0, 17.0], + ["Sample_3", "Gene_3", 13.0, 15.0, 17.0], + ["Sample_4", "Gene_4", 0.0, 0.0, 0.0], + ), + columns=["Sample", "Gene", "Protein_1", "Protein_2", "Protein_3"], + ) + return pd.melt( + expected_df, + id_vars=["Sample", "Gene"], + var_name="Protein ID", + value_name="Normalised Intensity", + ).sort_values(by=["Sample", "Protein ID"], ignore_index=True) + + @pytest.fixture def expected_df_by_totalsum_normalisation(): expected_df = pd.DataFrame( @@ -335,7 +374,7 @@ def test_normalisation_by_z_score( def test_normalisation_by_median( normalisation_df, expected_df_by_median_normalisation, show_figures ): - method_outputs = by_median(normalisation_df) + method_outputs = by_median(normalisation_df, log=False) fig = by_median_plot( normalisation_df, @@ -359,9 +398,35 @@ def test_normalisation_by_median( def test_normalisation_by_median_invalid_percentile(normalisation_df): with pytest.raises(AssertionError): - by_median(normalisation_df, percentile=-1) + by_median(normalisation_df, log=False, percentile=-1) with pytest.raises(AssertionError): - by_median(normalisation_df, percentile=1.1) + by_median(normalisation_df, log=False, percentile=1.1) + + +def test_normalisation_by_median_log( + input_log_normalisation_df, expected_df_by_median_log_normalisation, show_figures +): + method_outputs = by_median(input_log_normalisation_df, log=True) + + fig = by_median_plot( + input_log_normalisation_df, + method_outputs[DataKey.PROTEIN_DF], + "Boxplot", + "Sample", + "log10", + True, + )[0] + if show_figures: + fig.show() + + result_df = method_outputs[DataKey.PROTEIN_DF] + + assert result_df.round(3).equals( + expected_df_by_median_log_normalisation + ), f"Log median normalisation does not match! Should be \ + \n{expected_df_by_median_log_normalisation}\nbut is\n{result_df}" + + assert method_outputs["zeroed_samples"] == ["Sample_1", "Sample_4"] def test_totalsum_normalisation( diff --git a/backend/tests/protzilla/test_runner.py b/backend/tests/protzilla/test_runner.py index 636c9975..4656eedb 100644 --- a/backend/tests/protzilla/test_runner.py +++ b/backend/tests/protzilla/test_runner.py @@ -258,6 +258,7 @@ def test_runner_imports( }, { "percentile": 0.5, + "log": False, "graph_type": "Boxplot", "group_by": "None", "visual_transformation": "log10",