From f5f0addc57b350f013a96a2809407f711f657abe Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Tue, 20 Sep 2022 18:04:31 +1000 Subject: [PATCH 01/20] CTF parameter conversion (messy, half finished) --- pyem/metadata.py | 54 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index b414dc4..aeb8feb 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -293,22 +293,47 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): if df is None: df = pd.DataFrame() if 'ctf/tilt_A' in cs.dtype.names: - log.debug("Recovering beam tilt") - df[star.Relion.BEAMTILTX] = cs['ctf/tilt_A'][:, 0] - df[star.Relion.BEAMTILTY] = cs['ctf/tilt_A'][:, 1] + log.debug("Recovering beam tilt and converting to mrad") + df[star.Relion.BEAMTILTX] = np.arcsin(cs['ctf/tilt_A'][:, 0] / cs['ctf/cs_mm'] * 1e-7) * 1e3 + df[star.Relion.BEAMTILTY] = np.arcsin(cs['ctf/tilt_A'][:, 1] / cs['ctf/cs_mm'] * 1e-7) * 1e3 + z_neg13 = np.array(df[star.Relion.BEAMTILTX]) + z_13 = np.array(df[star.Relion.BEAMTILTY]) if 'ctf/shift_A' in cs.dtype.names: - pass + z_neg11 = (2 * np.pi) + cs['ctf/shift_A'][:, 0] + z_11 = (2 * np.pi) + cs['ctf/shift_A'][:, 1] if 'ctf/trefoil_A' in cs.dtype.names: - pass - # df[star.Relion.ODDZERNIKE] = cs['ctf/trefoil_A'] - if 'ctf/tetrafoil_A' in cs.dtype.names: - pass - # df[star.Relion.EVENZERNIKE] = cs['ctf/tetra_A'] - if 'ctf/anisomag' in cs.dtype.names: - df[star.Relion.MAGMAT00] = cs['ctf/anisomag'][:, 0] - df[star.Relion.MAGMAT01] = cs['ctf/anisomag'][:, 1] - df[star.Relion.MAGMAT10] = cs['ctf/anisomag'][:, 2] - df[star.Relion.MAGMAT11] = cs['ctf/anisomag'][:, 3] + # https://www.jeol.co.jp/en/words/emterms/search_result.html?keyword=wavelength%20of%20electron + E = cs['ctf/accel_kv'] # convert units? + wavelength = 1.23e3 / np.sqrt(E*(1 + 9.78e-7 * E)) # ? TODO: check this + unit_conversion = (2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2) + z_neg33 = unit_conversion * cs['ctf/trefoil_A'][:, 0] + z_33 = unit_conversion * cs['ctf/trefoil_A'][:, 1] + + # odd_zernike = [z_neg11, z_11, z_neg33, z_neg13, z_13, z_33] + # df.at[0, star.Relion.ODDZERNIKE] = odd_zernike + # if 'ctf/tetra_A' in cs.dtype.names: + + # z_00 = cs['ctf/amp_contrast'] - np.arccos(phi) + # unit_factor = np.pi * wavelength + # unit_conversion_odd = -np.pi/2 * (wavelength**3) + # z_neg22 = + # z_02 = + # z_22 = + # z_neg44 = + # z_neg24 = + # z_04 = + # z_24 = + # z_44 = + + # cs['ctf/tetra_A'] + + # even_zernike = [z_00, z_neg22, z_02, z_22, z_neg44, z_neg24, z_04, z_24, z_44] + # df.at[0, star.Relion.EVENZERNIKE] = even_zernike + # if 'ctf/anisomag' in cs.dtype.names: + # df[star.Relion.MAGMAT00] = cs['ctf/anisomag'][:, 0] + # df[star.Relion.MAGMAT01] = cs['ctf/anisomag'][:, 1] + # df[star.Relion.MAGMAT10] = cs['ctf/anisomag'][:, 2] + # df[star.Relion.MAGMAT11] = cs['ctf/anisomag'][:, 3] return df @@ -430,6 +455,7 @@ def parse_cryosparc_2_cs(csfile, passthroughs=None, minphic=0, boxsize=None, df = cryosparc_2_cs_model_parameters(cs, df, minphic=minphic) df = cryosparc_2_cs_array_parameters(cs, df) df = cryosparc_2_cs_filament_parameters(cs, df) + df = cryosparc_2_cs_ctf_parameters(cs, df) if passthroughs is not None: for passthrough in passthroughs: if type(passthrough) is np.ndarray: From 9cfc465eea5f82530113008221438d2a31250d2a Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Wed, 21 Sep 2022 14:52:46 +1000 Subject: [PATCH 02/20] More work on CTF parameter metadata --- pyem/metadata.py | 106 ++++++++++++++++++++++++++++++----------------- pyem/star.py | 17 ++++++++ 2 files changed, 86 insertions(+), 37 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index aeb8feb..6354668 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -292,48 +292,80 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): log = logging.getLogger('root') if df is None: df = pd.DataFrame() + if 'ctf/accel_kv' in cs.dtype.names: + # Calculate electron wavelentgh + # https://www.jeol.co.jp/en/words/emterms/search_result.html?keyword=wavelength%20of%20electron + E = cs['ctf/accel_kv'] # convert units? + wavelength = 1.23e3 / np.sqrt(E*(1 + 9.78e-7 * E)) # ? TODO: check this + else: + wavelength = None + if 'ctf/anisomag' in cs.dtype.names: + df[star.Relion.MAGMAT00] = cs['ctf/anisomag'][:, 0] + df[star.Relion.MAGMAT01] = cs['ctf/anisomag'][:, 1] + df[star.Relion.MAGMAT10] = cs['ctf/anisomag'][:, 2] + df[star.Relion.MAGMAT11] = cs['ctf/anisomag'][:, 3] if 'ctf/tilt_A' in cs.dtype.names: log.debug("Recovering beam tilt and converting to mrad") df[star.Relion.BEAMTILTX] = np.arcsin(cs['ctf/tilt_A'][:, 0] / cs['ctf/cs_mm'] * 1e-7) * 1e3 df[star.Relion.BEAMTILTY] = np.arcsin(cs['ctf/tilt_A'][:, 1] / cs['ctf/cs_mm'] * 1e-7) * 1e3 - z_neg13 = np.array(df[star.Relion.BEAMTILTX]) - z_13 = np.array(df[star.Relion.BEAMTILTY]) + df[star.Relion.Z_neg1_3] = df[star.Relion.BEAMTILTX] # does this also require a conversion factor of 2π * Cs * λ^2 + df[star.Relion.Z_pos1_3] = df[star.Relion.BEAMTILTY] # does this also require a conversion factor of 2π * Cs * λ^2 if 'ctf/shift_A' in cs.dtype.names: - z_neg11 = (2 * np.pi) + cs['ctf/shift_A'][:, 0] - z_11 = (2 * np.pi) + cs['ctf/shift_A'][:, 1] - if 'ctf/trefoil_A' in cs.dtype.names: - # https://www.jeol.co.jp/en/words/emterms/search_result.html?keyword=wavelength%20of%20electron - E = cs['ctf/accel_kv'] # convert units? - wavelength = 1.23e3 / np.sqrt(E*(1 + 9.78e-7 * E)) # ? TODO: check this - unit_conversion = (2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2) - z_neg33 = unit_conversion * cs['ctf/trefoil_A'][:, 0] - z_33 = unit_conversion * cs['ctf/trefoil_A'][:, 1] - - # odd_zernike = [z_neg11, z_11, z_neg33, z_neg13, z_13, z_33] - # df.at[0, star.Relion.ODDZERNIKE] = odd_zernike - # if 'ctf/tetra_A' in cs.dtype.names: - - # z_00 = cs['ctf/amp_contrast'] - np.arccos(phi) - # unit_factor = np.pi * wavelength - # unit_conversion_odd = -np.pi/2 * (wavelength**3) - # z_neg22 = - # z_02 = - # z_22 = - # z_neg44 = - # z_neg24 = - # z_04 = - # z_24 = - # z_44 = - - # cs['ctf/tetra_A'] - - # even_zernike = [z_00, z_neg22, z_02, z_22, z_neg44, z_neg24, z_04, z_24, z_44] - # df.at[0, star.Relion.EVENZERNIKE] = even_zernike - # if 'ctf/anisomag' in cs.dtype.names: - # df[star.Relion.MAGMAT00] = cs['ctf/anisomag'][:, 0] - # df[star.Relion.MAGMAT01] = cs['ctf/anisomag'][:, 1] - # df[star.Relion.MAGMAT10] = cs['ctf/anisomag'][:, 2] - # df[star.Relion.MAGMAT11] = cs['ctf/anisomag'][:, 3] + df[star.Relion.Z_neg1_1] = (2 * np.pi) + cs['ctf/shift_A'][:, 0] + df[star.Relion.Z_pos1_1] = (2 * np.pi) + cs['ctf/shift_A'][:, 1] + if 'ctf/trefoil_A' in cs.dtype.names and 'ctf/cs_mm' in cs.dtype.names and wavelength is not None: + unit_conversion_even = (2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2) + df[star.Relion.Z_neg3_3] = unit_conversion_even * cs['ctf/trefoil_A'][:, 0] + df[star.Relion.Z_pos3_3] = unit_conversion_even * cs['ctf/trefoil_A'][:, 1] + if 'ctf/amp_contrast' in cs.dtype.names: + df[star.Relion.Z_0_0] = cs['ctf/amp_contrast'] - np.arccos(phi) # what is phi? Could it be cs['ctf/phas_shift_rad'] ? + if 'ctf/df1_A' in cs.dtype.names and 'ctf/df2_A' in cs.dtype.names and wavelength is not None: + df[star.Relion.Z_neg2_2] = np.pi * wavelength * cs['ctf/df1_A'] # defocus asigmatism (X) Z1 + # df[star.Relion.Z_0_2] = np.pi * wavelength * # average defocus # how to find this? I have ctf/df_angle_rad ctf/bfactor ctf/scale & ctf/scale_const + df[star.Relion.Z_pos2_2] = np.pi * wavelength * cs['ctf/df2_A'] # defocus astigmatism (Y) Z2 + if 'ctf/cs_mm' in cs.dtype.names and wavelength is not None: + unit_conversion_odd = -np.pi/2 * (wavelength**3) + df[star.Relion.Z_0_4] = unit_conversion_odd * cs['ctf/cs_mm'] # spherical abberation + if 'ctf/tetra_A' in cs.dtype.names and wavelength is not None: + unit_conversion_odd = -np.pi/2 * (wavelength**3) + df[star.Relion.Z_neg4_4] = unit_conversion_odd * cs['ctf/tetra'][:, 0] + df[star.Relion.Z_neg2_4] = unit_conversion_odd * cs['ctf/tetra'][:, 1] + df[star.Relion.Z_pos2_4] = unit_conversion_odd * cs['ctf/tetra'][:, 2] + df[star.Relion.Z_pos4_4] = unit_conversion_odd * cs['ctf/tetra'][:, 3] + # Combine the odd Zernike coefficients into one column + if pd.Series([star.Relion.Z_neg1_1, + star.Relion.Z_pos1_1, + star.Relion.Z_neg3_3, + star.Relion.Z_neg1_3, + star.Relion.Z_pos1_3, + star.Relion.Z_pos3_3]).isin(df.columns).all(): + df[star.Relion.ODDZERNIKE] = "[" + \ + df[star.Relion.Z_neg1_1] + ", " + \ + df[star.Relion.Z_pos1_1] + ", " + \ + df[star.Relion.Z_neg3_3] + ", " + \ + df[star.Relion.Z_neg1_3] + ", " + \ + df[star.Relion.Z_pos1_3] + ", " + \ + df[star.Relion.Z_pos3_3] + "]" + # Combine the even Zernike coefficients into one column + if pd.Series([star.Relion.Z_0_0, + star.Relion.Z_neg2_2, + star.Relion.Z_0_2, + star.Relion.Z_neg4_4, + star.Relion.Z_neg2_4, + star.Relion.Z_0_4, + star.Relion.Z_2_4, + star.Relion.Z_4_4, + star.Relion.Z_pos3_3]).isin(df.columns).all(): + df[star.Relion.EVENZERNIKE] = "[" + \ + df[star.Relion.Z_0_0] + ", " + \ + df[star.Relion.Z_neg2_2] + ", " + \ + df[star.Relion.Z_0_2] + ", " + \ + df[star.Relion.Z_pos2_2] + ", " + \ + df[star.Relion.Z_neg4_4] + ", " + \ + df[star.Relion.Z_neg2_4] + ", " + \ + df[star.Relion.Z_0_4] + ", " + \ + df[star.Relion.Z_2_4] + ", " + \ + df[star.Relion.Z_4_4] + "]" return df diff --git a/pyem/star.py b/pyem/star.py index f63e75f..b1c7a01 100755 --- a/pyem/star.py +++ b/pyem/star.py @@ -103,6 +103,23 @@ class Relion: MTFFILENAME = "rlnMtfFileName" HELICALTUBEID = "rlnHelicalTubeID" + # Zernike coefficients + Z_0_0 = "rlnZernike_0_0" + Z_neg1_1 = "rlnZernike_neg1_1" + Z_pos1_1 = "rlnZernike_pos1_1" + Z_neg2_2 = "rlnZernike_neg2_2" + Z_0_2 = "rlnZernike_0_2" + Z_pos2_2 = "rlnZernike_pos2_2" + Z_neg3_3 = "rlnZernike_neg3_3" + Z_neg1_3 = "rlnZernike_neg1_3" + Z_pos1_3 = "rlnZernike_pos1_3" + Z_pos3_3 = "rlnZernike_pos3_3" + Z_neg4_4 = "rlnZernike_neg4_4" + Z_neg2_4 = "rlnZernike_neg2_4" + Z_0_4 = "rlnZernike_0_4" + Z_pos2_4 = "rlnZernike_pos2_4" + Z_pos4_4 = "rlnZernike_pos4_4" + # Field lists. COORDS = [COORDX, COORDY] ORIGINS = [ORIGINX, ORIGINY] From baee79a71e92bb28c4a16ab1f1cf2d1220b28809 Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Mon, 26 Sep 2022 16:27:24 +1000 Subject: [PATCH 03/20] Calculate electron wavelength with more precision --- pyem/metadata.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index 6354668..9b5184c 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -294,9 +294,8 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): df = pd.DataFrame() if 'ctf/accel_kv' in cs.dtype.names: # Calculate electron wavelentgh - # https://www.jeol.co.jp/en/words/emterms/search_result.html?keyword=wavelength%20of%20electron - E = cs['ctf/accel_kv'] # convert units? - wavelength = 1.23e3 / np.sqrt(E*(1 + 9.78e-7 * E)) # ? TODO: check this + kv = cs['ctf/accel_kv'] + wavelength = 12.2643247 / np.sqrt(kv * (1 + (kv * 0.978466e-6))) else: wavelength = None if 'ctf/anisomag' in cs.dtype.names: From 2ee74df77700358e4e8b9a0155ecce13660e5f79 Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Mon, 26 Sep 2022 16:56:22 +1000 Subject: [PATCH 04/20] Fix equations related to defocus --- pyem/metadata.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index 9b5184c..efdd420 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -318,10 +318,13 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): df[star.Relion.Z_pos3_3] = unit_conversion_even * cs['ctf/trefoil_A'][:, 1] if 'ctf/amp_contrast' in cs.dtype.names: df[star.Relion.Z_0_0] = cs['ctf/amp_contrast'] - np.arccos(phi) # what is phi? Could it be cs['ctf/phas_shift_rad'] ? - if 'ctf/df1_A' in cs.dtype.names and 'ctf/df2_A' in cs.dtype.names and wavelength is not None: - df[star.Relion.Z_neg2_2] = np.pi * wavelength * cs['ctf/df1_A'] # defocus asigmatism (X) Z1 - # df[star.Relion.Z_0_2] = np.pi * wavelength * # average defocus # how to find this? I have ctf/df_angle_rad ctf/bfactor ctf/scale & ctf/scale_const - df[star.Relion.Z_pos2_2] = np.pi * wavelength * cs['ctf/df2_A'] # defocus astigmatism (Y) Z2 + if 'ctf/df1_A' in cs.dtype.names and 'ctf/df2_A' in cs.dtype.names and 'ctf/df_angle_rad' in cs.dtype.names and wavelength is not None: + average_defocus = (cs['ctf/df1_A'] + cs['ctf/df2_A']) / 2 + defocus_deviation = (cs['ctf/df1_A'] - cs['ctf/df2_A']) / 2 + astigmatism_angle = cs['ctf/df_angle_rad'] + df[star.Relion.Z_0_2] = (np.pi * wavelength) * average_defocus + df[star.Relion.Z_neg2_2] = (np.pi * wavelength) * np.cos(2 * astigmatism_angle) * defocus_deviation # defocus asigmatism in X (Z1) + df[star.Relion.Z_pos2_2] = (np.pi * wavelength) * np.sin(2 * astigmatism_angle) * defocus_deviation # defocus astigmatism in Y (Z2) if 'ctf/cs_mm' in cs.dtype.names and wavelength is not None: unit_conversion_odd = -np.pi/2 * (wavelength**3) df[star.Relion.Z_0_4] = unit_conversion_odd * cs['ctf/cs_mm'] # spherical abberation From 3efd6946201e53d82801bc318771fd40bd26df2a Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Mon, 26 Sep 2022 17:06:23 +1000 Subject: [PATCH 05/20] Remove variable name 'unit_conversion_odd', simplifies things slightly --- pyem/metadata.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index efdd420..eabb88f 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -326,14 +326,12 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): df[star.Relion.Z_neg2_2] = (np.pi * wavelength) * np.cos(2 * astigmatism_angle) * defocus_deviation # defocus asigmatism in X (Z1) df[star.Relion.Z_pos2_2] = (np.pi * wavelength) * np.sin(2 * astigmatism_angle) * defocus_deviation # defocus astigmatism in Y (Z2) if 'ctf/cs_mm' in cs.dtype.names and wavelength is not None: - unit_conversion_odd = -np.pi/2 * (wavelength**3) - df[star.Relion.Z_0_4] = unit_conversion_odd * cs['ctf/cs_mm'] # spherical abberation + df[star.Relion.Z_0_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/cs_mm'] # spherical abberation if 'ctf/tetra_A' in cs.dtype.names and wavelength is not None: - unit_conversion_odd = -np.pi/2 * (wavelength**3) - df[star.Relion.Z_neg4_4] = unit_conversion_odd * cs['ctf/tetra'][:, 0] - df[star.Relion.Z_neg2_4] = unit_conversion_odd * cs['ctf/tetra'][:, 1] - df[star.Relion.Z_pos2_4] = unit_conversion_odd * cs['ctf/tetra'][:, 2] - df[star.Relion.Z_pos4_4] = unit_conversion_odd * cs['ctf/tetra'][:, 3] + df[star.Relion.Z_neg4_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra'][:, 0] + df[star.Relion.Z_neg2_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra'][:, 1] + df[star.Relion.Z_pos2_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra'][:, 2] + df[star.Relion.Z_pos4_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra'][:, 3] # Combine the odd Zernike coefficients into one column if pd.Series([star.Relion.Z_neg1_1, star.Relion.Z_pos1_1, From e5d4c02f6d61d472530cfe4ebdf00c73827f8dc6 Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Mon, 26 Sep 2022 17:19:52 +1000 Subject: [PATCH 06/20] Update Zernike coefficient equations for tilt and trefoil --- pyem/metadata.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index eabb88f..456f8af 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -307,15 +307,15 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): log.debug("Recovering beam tilt and converting to mrad") df[star.Relion.BEAMTILTX] = np.arcsin(cs['ctf/tilt_A'][:, 0] / cs['ctf/cs_mm'] * 1e-7) * 1e3 df[star.Relion.BEAMTILTY] = np.arcsin(cs['ctf/tilt_A'][:, 1] / cs['ctf/cs_mm'] * 1e-7) * 1e3 - df[star.Relion.Z_neg1_3] = df[star.Relion.BEAMTILTX] # does this also require a conversion factor of 2π * Cs * λ^2 - df[star.Relion.Z_pos1_3] = df[star.Relion.BEAMTILTY] # does this also require a conversion factor of 2π * Cs * λ^2 + if 'ctf/tilt_A' in cs.dtype.names and wavelength is not None: + df[star.Relion.Z_neg1_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * df[star.Relion.BEAMTILTX] + df[star.Relion.Z_pos1_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * df[star.Relion.BEAMTILTY] if 'ctf/shift_A' in cs.dtype.names: df[star.Relion.Z_neg1_1] = (2 * np.pi) + cs['ctf/shift_A'][:, 0] df[star.Relion.Z_pos1_1] = (2 * np.pi) + cs['ctf/shift_A'][:, 1] if 'ctf/trefoil_A' in cs.dtype.names and 'ctf/cs_mm' in cs.dtype.names and wavelength is not None: - unit_conversion_even = (2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2) - df[star.Relion.Z_neg3_3] = unit_conversion_even * cs['ctf/trefoil_A'][:, 0] - df[star.Relion.Z_pos3_3] = unit_conversion_even * cs['ctf/trefoil_A'][:, 1] + df[star.Relion.Z_neg3_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * cs['ctf/trefoil_A'][:, 0] + df[star.Relion.Z_pos3_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * cs['ctf/trefoil_A'][:, 1] if 'ctf/amp_contrast' in cs.dtype.names: df[star.Relion.Z_0_0] = cs['ctf/amp_contrast'] - np.arccos(phi) # what is phi? Could it be cs['ctf/phas_shift_rad'] ? if 'ctf/df1_A' in cs.dtype.names and 'ctf/df2_A' in cs.dtype.names and 'ctf/df_angle_rad' in cs.dtype.names and wavelength is not None: From 2dbbda98a61348a81c2fc5cc96176ec5a5f386f1 Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Mon, 26 Sep 2022 17:23:21 +1000 Subject: [PATCH 07/20] phi = cs['ctf/phase_shift_rad'] --- pyem/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index 456f8af..37b3b4b 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -317,7 +317,7 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): df[star.Relion.Z_neg3_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * cs['ctf/trefoil_A'][:, 0] df[star.Relion.Z_pos3_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * cs['ctf/trefoil_A'][:, 1] if 'ctf/amp_contrast' in cs.dtype.names: - df[star.Relion.Z_0_0] = cs['ctf/amp_contrast'] - np.arccos(phi) # what is phi? Could it be cs['ctf/phas_shift_rad'] ? + df[star.Relion.Z_0_0] = cs['ctf/amp_contrast'] - np.arccos(cs['ctf/phase_shift_rad']) if 'ctf/df1_A' in cs.dtype.names and 'ctf/df2_A' in cs.dtype.names and 'ctf/df_angle_rad' in cs.dtype.names and wavelength is not None: average_defocus = (cs['ctf/df1_A'] + cs['ctf/df2_A']) / 2 defocus_deviation = (cs['ctf/df1_A'] - cs['ctf/df2_A']) / 2 From e9f666825c4867213bf6e1e65fddf89985ff0b1e Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Mon, 26 Sep 2022 17:43:48 +1000 Subject: [PATCH 08/20] Fix typo in name 'ctf/tetra_A' --- pyem/metadata.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index 37b3b4b..adfa332 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -328,10 +328,10 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): if 'ctf/cs_mm' in cs.dtype.names and wavelength is not None: df[star.Relion.Z_0_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/cs_mm'] # spherical abberation if 'ctf/tetra_A' in cs.dtype.names and wavelength is not None: - df[star.Relion.Z_neg4_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra'][:, 0] - df[star.Relion.Z_neg2_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra'][:, 1] - df[star.Relion.Z_pos2_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra'][:, 2] - df[star.Relion.Z_pos4_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra'][:, 3] + df[star.Relion.Z_neg4_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 0] + df[star.Relion.Z_neg2_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 1] + df[star.Relion.Z_pos2_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 2] + df[star.Relion.Z_pos4_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 3] # Combine the odd Zernike coefficients into one column if pd.Series([star.Relion.Z_neg1_1, star.Relion.Z_pos1_1, From c86cdd62c1a31bdf0ea78cf3513e1f1a5babe39f Mon Sep 17 00:00:00 2001 From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com> Date: Mon, 26 Sep 2022 17:54:15 +1000 Subject: [PATCH 09/20] Convert dataframe columns to string before concatenating Zernike coeffs --- pyem/metadata.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index adfa332..676950d 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -340,12 +340,12 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): star.Relion.Z_pos1_3, star.Relion.Z_pos3_3]).isin(df.columns).all(): df[star.Relion.ODDZERNIKE] = "[" + \ - df[star.Relion.Z_neg1_1] + ", " + \ - df[star.Relion.Z_pos1_1] + ", " + \ - df[star.Relion.Z_neg3_3] + ", " + \ - df[star.Relion.Z_neg1_3] + ", " + \ - df[star.Relion.Z_pos1_3] + ", " + \ - df[star.Relion.Z_pos3_3] + "]" + df[star.Relion.Z_neg1_1].astype(str) + ", " + \ + df[star.Relion.Z_pos1_1].astype(str) + ", " + \ + df[star.Relion.Z_neg3_3].astype(str) + ", " + \ + df[star.Relion.Z_neg1_3].astype(str) + ", " + \ + df[star.Relion.Z_pos1_3].astype(str) + ", " + \ + df[star.Relion.Z_pos3_3].astype(str) + "]" # Combine the even Zernike coefficients into one column if pd.Series([star.Relion.Z_0_0, star.Relion.Z_neg2_2, @@ -353,19 +353,19 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): star.Relion.Z_neg4_4, star.Relion.Z_neg2_4, star.Relion.Z_0_4, - star.Relion.Z_2_4, - star.Relion.Z_4_4, + star.Relion.Z_pos2_4, + star.Relion.Z_pos4_4, star.Relion.Z_pos3_3]).isin(df.columns).all(): df[star.Relion.EVENZERNIKE] = "[" + \ - df[star.Relion.Z_0_0] + ", " + \ - df[star.Relion.Z_neg2_2] + ", " + \ - df[star.Relion.Z_0_2] + ", " + \ - df[star.Relion.Z_pos2_2] + ", " + \ - df[star.Relion.Z_neg4_4] + ", " + \ - df[star.Relion.Z_neg2_4] + ", " + \ - df[star.Relion.Z_0_4] + ", " + \ - df[star.Relion.Z_2_4] + ", " + \ - df[star.Relion.Z_4_4] + "]" + df[star.Relion.Z_0_0].astype(str) + ", " + \ + df[star.Relion.Z_neg2_2].astype(str) + ", " + \ + df[star.Relion.Z_0_2].astype(str) + ", " + \ + df[star.Relion.Z_pos2_2].astype(str) + ", " + \ + df[star.Relion.Z_neg4_4].astype(str) + ", " + \ + df[star.Relion.Z_neg2_4].astype(str) + ", " + \ + df[star.Relion.Z_0_4].astype(str) + ", " + \ + df[star.Relion.Z_pos2_4].astype(str) + ", " + \ + df[star.Relion.Z_pos4_4].astype(str) + "]" return df From dbbd4272b97ac8226c11582c7b43ebb5f1c13284 Mon Sep 17 00:00:00 2001 From: Daniel Asarnow Date: Mon, 26 Sep 2022 13:06:26 -0700 Subject: [PATCH 10/20] Zernike coefs concatenated when writing star file --- pyem/metadata.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index 676950d..889d817 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -332,40 +332,6 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): df[star.Relion.Z_neg2_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 1] df[star.Relion.Z_pos2_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 2] df[star.Relion.Z_pos4_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 3] - # Combine the odd Zernike coefficients into one column - if pd.Series([star.Relion.Z_neg1_1, - star.Relion.Z_pos1_1, - star.Relion.Z_neg3_3, - star.Relion.Z_neg1_3, - star.Relion.Z_pos1_3, - star.Relion.Z_pos3_3]).isin(df.columns).all(): - df[star.Relion.ODDZERNIKE] = "[" + \ - df[star.Relion.Z_neg1_1].astype(str) + ", " + \ - df[star.Relion.Z_pos1_1].astype(str) + ", " + \ - df[star.Relion.Z_neg3_3].astype(str) + ", " + \ - df[star.Relion.Z_neg1_3].astype(str) + ", " + \ - df[star.Relion.Z_pos1_3].astype(str) + ", " + \ - df[star.Relion.Z_pos3_3].astype(str) + "]" - # Combine the even Zernike coefficients into one column - if pd.Series([star.Relion.Z_0_0, - star.Relion.Z_neg2_2, - star.Relion.Z_0_2, - star.Relion.Z_neg4_4, - star.Relion.Z_neg2_4, - star.Relion.Z_0_4, - star.Relion.Z_pos2_4, - star.Relion.Z_pos4_4, - star.Relion.Z_pos3_3]).isin(df.columns).all(): - df[star.Relion.EVENZERNIKE] = "[" + \ - df[star.Relion.Z_0_0].astype(str) + ", " + \ - df[star.Relion.Z_neg2_2].astype(str) + ", " + \ - df[star.Relion.Z_0_2].astype(str) + ", " + \ - df[star.Relion.Z_pos2_2].astype(str) + ", " + \ - df[star.Relion.Z_neg4_4].astype(str) + ", " + \ - df[star.Relion.Z_neg2_4].astype(str) + ", " + \ - df[star.Relion.Z_0_4].astype(str) + ", " + \ - df[star.Relion.Z_pos2_4].astype(str) + ", " + \ - df[star.Relion.Z_pos4_4].astype(str) + "]" return df From 9b4ae522fe96199cfdd0f16ea81b4e2e3ba94dcc Mon Sep 17 00:00:00 2001 From: Daniel Asarnow Date: Mon, 26 Sep 2022 13:07:34 -0700 Subject: [PATCH 11/20] Force 1 to float, remove extra parens --- pyem/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index 889d817..918fa1f 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -295,7 +295,7 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): if 'ctf/accel_kv' in cs.dtype.names: # Calculate electron wavelentgh kv = cs['ctf/accel_kv'] - wavelength = 12.2643247 / np.sqrt(kv * (1 + (kv * 0.978466e-6))) + wavelength = 12.2643247 / np.sqrt(kv * (1. + kv * 0.978466e-6)) else: wavelength = None if 'ctf/anisomag' in cs.dtype.names: From 8a255b21fd80841051d41ed06e33927b3e35419d Mon Sep 17 00:00:00 2001 From: Daniel Asarnow Date: Mon, 26 Sep 2022 13:33:02 -0700 Subject: [PATCH 12/20] Checkout pyem/star.py from master --- pyem/star.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/pyem/star.py b/pyem/star.py index 83d83c8..d0e4f28 100755 --- a/pyem/star.py +++ b/pyem/star.py @@ -104,23 +104,6 @@ class Relion: ODDZERNIKE = "rlnOddZernike" EVENZERNIKE = "rlnEvenZernike" - # Zernike coefficients - Z_0_0 = "rlnZernike_0_0" - Z_neg1_1 = "rlnZernike_neg1_1" - Z_pos1_1 = "rlnZernike_pos1_1" - Z_neg2_2 = "rlnZernike_neg2_2" - Z_0_2 = "rlnZernike_0_2" - Z_pos2_2 = "rlnZernike_pos2_2" - Z_neg3_3 = "rlnZernike_neg3_3" - Z_neg1_3 = "rlnZernike_neg1_3" - Z_pos1_3 = "rlnZernike_pos1_3" - Z_pos3_3 = "rlnZernike_pos3_3" - Z_neg4_4 = "rlnZernike_neg4_4" - Z_neg2_4 = "rlnZernike_neg2_4" - Z_0_4 = "rlnZernike_0_4" - Z_pos2_4 = "rlnZernike_pos2_4" - Z_pos4_4 = "rlnZernike_pos4_4" - # Field lists. COORDS = [COORDX, COORDY] COORDS3D = [COORDX, COORDY, COORDZ] From 183787c19118658428cdb75df80adc0449babc83 Mon Sep 17 00:00:00 2001 From: Daniel Asarnow Date: Mon, 26 Sep 2022 13:34:15 -0700 Subject: [PATCH 13/20] Fix constant references --- pyem/metadata.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index 918fa1f..5c56e87 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -308,30 +308,30 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): df[star.Relion.BEAMTILTX] = np.arcsin(cs['ctf/tilt_A'][:, 0] / cs['ctf/cs_mm'] * 1e-7) * 1e3 df[star.Relion.BEAMTILTY] = np.arcsin(cs['ctf/tilt_A'][:, 1] / cs['ctf/cs_mm'] * 1e-7) * 1e3 if 'ctf/tilt_A' in cs.dtype.names and wavelength is not None: - df[star.Relion.Z_neg1_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * df[star.Relion.BEAMTILTX] - df[star.Relion.Z_pos1_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * df[star.Relion.BEAMTILTY] + df[star.UCSF.Z_neg1_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * df[star.Relion.BEAMTILTX] + df[star.UCSF.Z_1_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * df[star.Relion.BEAMTILTY] if 'ctf/shift_A' in cs.dtype.names: - df[star.Relion.Z_neg1_1] = (2 * np.pi) + cs['ctf/shift_A'][:, 0] - df[star.Relion.Z_pos1_1] = (2 * np.pi) + cs['ctf/shift_A'][:, 1] + df[star.UCSF.Z_neg1_1] = (2 * np.pi) + cs['ctf/shift_A'][:, 0] + df[star.UCSF.Z_1_1] = (2 * np.pi) + cs['ctf/shift_A'][:, 1] if 'ctf/trefoil_A' in cs.dtype.names and 'ctf/cs_mm' in cs.dtype.names and wavelength is not None: - df[star.Relion.Z_neg3_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * cs['ctf/trefoil_A'][:, 0] - df[star.Relion.Z_pos3_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * cs['ctf/trefoil_A'][:, 1] + df[star.UCSF.Z_neg3_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * cs['ctf/trefoil_A'][:, 0] + df[star.UCSF.Z_3_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * cs['ctf/trefoil_A'][:, 1] if 'ctf/amp_contrast' in cs.dtype.names: - df[star.Relion.Z_0_0] = cs['ctf/amp_contrast'] - np.arccos(cs['ctf/phase_shift_rad']) + df[star.UCSF.Z_0_0] = cs['ctf/amp_contrast'] - np.arccos(cs['ctf/phase_shift_rad']) if 'ctf/df1_A' in cs.dtype.names and 'ctf/df2_A' in cs.dtype.names and 'ctf/df_angle_rad' in cs.dtype.names and wavelength is not None: average_defocus = (cs['ctf/df1_A'] + cs['ctf/df2_A']) / 2 defocus_deviation = (cs['ctf/df1_A'] - cs['ctf/df2_A']) / 2 astigmatism_angle = cs['ctf/df_angle_rad'] - df[star.Relion.Z_0_2] = (np.pi * wavelength) * average_defocus - df[star.Relion.Z_neg2_2] = (np.pi * wavelength) * np.cos(2 * astigmatism_angle) * defocus_deviation # defocus asigmatism in X (Z1) - df[star.Relion.Z_pos2_2] = (np.pi * wavelength) * np.sin(2 * astigmatism_angle) * defocus_deviation # defocus astigmatism in Y (Z2) + df[star.UCSF.Z_0_2] = (np.pi * wavelength) * average_defocus + df[star.UCSF.Z_neg2_2] = (np.pi * wavelength) * np.cos(2 * astigmatism_angle) * defocus_deviation # defocus asigmatism in X (Z1) + df[star.UCSF.Z_2_2] = (np.pi * wavelength) * np.sin(2 * astigmatism_angle) * defocus_deviation # defocus astigmatism in Y (Z2) if 'ctf/cs_mm' in cs.dtype.names and wavelength is not None: - df[star.Relion.Z_0_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/cs_mm'] # spherical abberation + df[star.UCSF.Z_0_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/cs_mm'] # spherical abberation if 'ctf/tetra_A' in cs.dtype.names and wavelength is not None: - df[star.Relion.Z_neg4_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 0] - df[star.Relion.Z_neg2_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 1] - df[star.Relion.Z_pos2_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 2] - df[star.Relion.Z_pos4_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 3] + df[star.UCSF.Z_neg4_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 0] + df[star.UCSF.Z_neg2_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 1] + df[star.UCSF.Z_2_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 2] + df[star.UCSF.Z_4_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 3] return df From 9dcf99e32d4ed515dd4c2ffa47fa85e08f55ed3b Mon Sep 17 00:00:00 2001 From: Daniel Asarnow Date: Mon, 26 Sep 2022 13:40:19 -0700 Subject: [PATCH 14/20] Refactor to match other modules (pyem.ctf) --- pyem/metadata.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index 5c56e87..f166ced 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -308,30 +308,30 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): df[star.Relion.BEAMTILTX] = np.arcsin(cs['ctf/tilt_A'][:, 0] / cs['ctf/cs_mm'] * 1e-7) * 1e3 df[star.Relion.BEAMTILTY] = np.arcsin(cs['ctf/tilt_A'][:, 1] / cs['ctf/cs_mm'] * 1e-7) * 1e3 if 'ctf/tilt_A' in cs.dtype.names and wavelength is not None: - df[star.UCSF.Z_neg1_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * df[star.Relion.BEAMTILTX] - df[star.UCSF.Z_1_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * df[star.Relion.BEAMTILTY] + df[star.UCSF.Z_neg1_3] = 2 * np.pi * cs['ctf/cs_mm'] * wavelength**2 * df[star.Relion.BEAMTILTX] + df[star.UCSF.Z_1_3] = 2 * np.pi * cs['ctf/cs_mm'] * wavelength**2 * df[star.Relion.BEAMTILTY] if 'ctf/shift_A' in cs.dtype.names: - df[star.UCSF.Z_neg1_1] = (2 * np.pi) + cs['ctf/shift_A'][:, 0] - df[star.UCSF.Z_1_1] = (2 * np.pi) + cs['ctf/shift_A'][:, 1] + df[star.UCSF.Z_neg1_1] = 2 * np.pi + cs['ctf/shift_A'][:, 0] + df[star.UCSF.Z_1_1] = 2 * np.pi + cs['ctf/shift_A'][:, 1] if 'ctf/trefoil_A' in cs.dtype.names and 'ctf/cs_mm' in cs.dtype.names and wavelength is not None: - df[star.UCSF.Z_neg3_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * cs['ctf/trefoil_A'][:, 0] - df[star.UCSF.Z_3_3] = ((2*np.pi) * cs['ctf/cs_mm'] * (wavelength**2)) * cs['ctf/trefoil_A'][:, 1] + df[star.UCSF.Z_neg3_3] = 2*np.pi * cs['ctf/cs_mm'] * wavelength**2 * cs['ctf/trefoil_A'][:, 0] + df[star.UCSF.Z_3_3] = 2*np.pi * cs['ctf/cs_mm'] * wavelength**2 * cs['ctf/trefoil_A'][:, 1] if 'ctf/amp_contrast' in cs.dtype.names: df[star.UCSF.Z_0_0] = cs['ctf/amp_contrast'] - np.arccos(cs['ctf/phase_shift_rad']) if 'ctf/df1_A' in cs.dtype.names and 'ctf/df2_A' in cs.dtype.names and 'ctf/df_angle_rad' in cs.dtype.names and wavelength is not None: - average_defocus = (cs['ctf/df1_A'] + cs['ctf/df2_A']) / 2 - defocus_deviation = (cs['ctf/df1_A'] - cs['ctf/df2_A']) / 2 - astigmatism_angle = cs['ctf/df_angle_rad'] - df[star.UCSF.Z_0_2] = (np.pi * wavelength) * average_defocus - df[star.UCSF.Z_neg2_2] = (np.pi * wavelength) * np.cos(2 * astigmatism_angle) * defocus_deviation # defocus asigmatism in X (Z1) - df[star.UCSF.Z_2_2] = (np.pi * wavelength) * np.sin(2 * astigmatism_angle) * defocus_deviation # defocus astigmatism in Y (Z2) + df_avg = (cs['ctf/df1_A'] + cs['ctf/df2_A']) / 2 + df_dev = (cs['ctf/df1_A'] - cs['ctf/df2_A']) / 2 + angast = cs['ctf/df_angle_rad'] + df[star.UCSF.Z_0_2] = np.pi * wavelength * df_avg + df[star.UCSF.Z_neg2_2] = np.pi * wavelength * np.cos(2 * angast) * df_dev # defocus asigmatism in X (Z1) + df[star.UCSF.Z_2_2] = np.pi * wavelength * np.sin(2 * angast) * df_dev # defocus astigmatism in Y (Z2) if 'ctf/cs_mm' in cs.dtype.names and wavelength is not None: - df[star.UCSF.Z_0_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/cs_mm'] # spherical abberation + df[star.UCSF.Z_0_4] = -np.pi/2 * wavelength**3 * cs['ctf/cs_mm'] # spherical abberation if 'ctf/tetra_A' in cs.dtype.names and wavelength is not None: - df[star.UCSF.Z_neg4_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 0] - df[star.UCSF.Z_neg2_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 1] - df[star.UCSF.Z_2_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 2] - df[star.UCSF.Z_4_4] = (-np.pi/2 * (wavelength**3)) * cs['ctf/tetra_A'][:, 3] + df[star.UCSF.Z_neg4_4] = -np.pi/2 * wavelength**3 * cs['ctf/tetra_A'][:, 0] + df[star.UCSF.Z_neg2_4] = -np.pi/2 * wavelength**3 * cs['ctf/tetra_A'][:, 1] + df[star.UCSF.Z_2_4] = -np.pi/2 * wavelength**3 * cs['ctf/tetra_A'][:, 2] + df[star.UCSF.Z_4_4] = -np.pi/2 * wavelength**3 * cs['ctf/tetra_A'][:, 3] return df From 226fc6aa51f8bb10af01519382fd0f1d97f50304 Mon Sep 17 00:00:00 2001 From: Daniel Asarnow Date: Mon, 26 Sep 2022 15:11:43 -0700 Subject: [PATCH 15/20] Arccos to convert % to rad --- pyem/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index f166ced..19737ee 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -317,7 +317,7 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): df[star.UCSF.Z_neg3_3] = 2*np.pi * cs['ctf/cs_mm'] * wavelength**2 * cs['ctf/trefoil_A'][:, 0] df[star.UCSF.Z_3_3] = 2*np.pi * cs['ctf/cs_mm'] * wavelength**2 * cs['ctf/trefoil_A'][:, 1] if 'ctf/amp_contrast' in cs.dtype.names: - df[star.UCSF.Z_0_0] = cs['ctf/amp_contrast'] - np.arccos(cs['ctf/phase_shift_rad']) + df[star.UCSF.Z_0_0] = cs['ctf/phase_shift_rad'] - np.arccos(cs['ctf/amp_contrast']) if 'ctf/df1_A' in cs.dtype.names and 'ctf/df2_A' in cs.dtype.names and 'ctf/df_angle_rad' in cs.dtype.names and wavelength is not None: df_avg = (cs['ctf/df1_A'] + cs['ctf/df2_A']) / 2 df_dev = (cs['ctf/df1_A'] - cs['ctf/df2_A']) / 2 From 63533a5b1522c8de472691a84f38c8a7d18737f9 Mon Sep 17 00:00:00 2001 From: Daniel Asarnow Date: Mon, 26 Sep 2022 15:26:19 -0700 Subject: [PATCH 16/20] Refactored conversions - Precompute 2*pi, pi/2, wl**2, wl**3, 2*angast - assert kV is available (and therefore wl) - Cs in A for Z(0,4) --- pyem/metadata.py | 54 +++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index 19737ee..e4d9f4f 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -292,12 +292,14 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): log = logging.getLogger('root') if df is None: df = pd.DataFrame() - if 'ctf/accel_kv' in cs.dtype.names: - # Calculate electron wavelentgh - kv = cs['ctf/accel_kv'] - wavelength = 12.2643247 / np.sqrt(kv * (1. + kv * 0.978466e-6)) - else: - wavelength = None + assert 'ctf/accel_kv' in cs.dtype.names + # Calculate electron wavelentgh + kv = cs['ctf/accel_kv'] + wl = 12.2643247 / np.sqrt(kv * (1. + kv * 0.978466e-6)) + wl_2 = wl**2 + wl_3 = wl**3 + half_pi = 0.5 * np.pi + two_pi = 2 * np.pi if 'ctf/anisomag' in cs.dtype.names: df[star.Relion.MAGMAT00] = cs['ctf/anisomag'][:, 0] df[star.Relion.MAGMAT01] = cs['ctf/anisomag'][:, 1] @@ -307,31 +309,31 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): log.debug("Recovering beam tilt and converting to mrad") df[star.Relion.BEAMTILTX] = np.arcsin(cs['ctf/tilt_A'][:, 0] / cs['ctf/cs_mm'] * 1e-7) * 1e3 df[star.Relion.BEAMTILTY] = np.arcsin(cs['ctf/tilt_A'][:, 1] / cs['ctf/cs_mm'] * 1e-7) * 1e3 - if 'ctf/tilt_A' in cs.dtype.names and wavelength is not None: - df[star.UCSF.Z_neg1_3] = 2 * np.pi * cs['ctf/cs_mm'] * wavelength**2 * df[star.Relion.BEAMTILTX] - df[star.UCSF.Z_1_3] = 2 * np.pi * cs['ctf/cs_mm'] * wavelength**2 * df[star.Relion.BEAMTILTY] + if 'ctf/tilt_A' in cs.dtype.names: + df[star.UCSF.Z_neg1_3] = two_pi * wl_2 * cs['ctf/tilt_A'][:, 0] + df[star.UCSF.Z_1_3] = two_pi * wl_2 * cs['ctf/tilt_A'][:, 1] if 'ctf/shift_A' in cs.dtype.names: - df[star.UCSF.Z_neg1_1] = 2 * np.pi + cs['ctf/shift_A'][:, 0] - df[star.UCSF.Z_1_1] = 2 * np.pi + cs['ctf/shift_A'][:, 1] - if 'ctf/trefoil_A' in cs.dtype.names and 'ctf/cs_mm' in cs.dtype.names and wavelength is not None: - df[star.UCSF.Z_neg3_3] = 2*np.pi * cs['ctf/cs_mm'] * wavelength**2 * cs['ctf/trefoil_A'][:, 0] - df[star.UCSF.Z_3_3] = 2*np.pi * cs['ctf/cs_mm'] * wavelength**2 * cs['ctf/trefoil_A'][:, 1] + df[star.UCSF.Z_neg1_1] = two_pi + cs['ctf/shift_A'][:, 0] + df[star.UCSF.Z_1_1] = two_pi + cs['ctf/shift_A'][:, 1] + if 'ctf/trefoil_A' in cs.dtype.names and 'ctf/cs_mm' in cs.dtype.names: + df[star.UCSF.Z_neg3_3] = two_pi * cs['ctf/cs_mm'] * wl_2 * cs['ctf/trefoil_A'][:, 0] + df[star.UCSF.Z_3_3] = two_pi * cs['ctf/cs_mm'] * wl_2 * cs['ctf/trefoil_A'][:, 1] if 'ctf/amp_contrast' in cs.dtype.names: df[star.UCSF.Z_0_0] = cs['ctf/phase_shift_rad'] - np.arccos(cs['ctf/amp_contrast']) - if 'ctf/df1_A' in cs.dtype.names and 'ctf/df2_A' in cs.dtype.names and 'ctf/df_angle_rad' in cs.dtype.names and wavelength is not None: + if 'ctf/df1_A' in cs.dtype.names and 'ctf/df2_A' in cs.dtype.names and 'ctf/df_angle_rad' in cs.dtype.names: df_avg = (cs['ctf/df1_A'] + cs['ctf/df2_A']) / 2 df_dev = (cs['ctf/df1_A'] - cs['ctf/df2_A']) / 2 - angast = cs['ctf/df_angle_rad'] - df[star.UCSF.Z_0_2] = np.pi * wavelength * df_avg - df[star.UCSF.Z_neg2_2] = np.pi * wavelength * np.cos(2 * angast) * df_dev # defocus asigmatism in X (Z1) - df[star.UCSF.Z_2_2] = np.pi * wavelength * np.sin(2 * angast) * df_dev # defocus astigmatism in Y (Z2) - if 'ctf/cs_mm' in cs.dtype.names and wavelength is not None: - df[star.UCSF.Z_0_4] = -np.pi/2 * wavelength**3 * cs['ctf/cs_mm'] # spherical abberation - if 'ctf/tetra_A' in cs.dtype.names and wavelength is not None: - df[star.UCSF.Z_neg4_4] = -np.pi/2 * wavelength**3 * cs['ctf/tetra_A'][:, 0] - df[star.UCSF.Z_neg2_4] = -np.pi/2 * wavelength**3 * cs['ctf/tetra_A'][:, 1] - df[star.UCSF.Z_2_4] = -np.pi/2 * wavelength**3 * cs['ctf/tetra_A'][:, 2] - df[star.UCSF.Z_4_4] = -np.pi/2 * wavelength**3 * cs['ctf/tetra_A'][:, 3] + two_angast = 2 * cs['ctf/df_angle_rad'] + df[star.UCSF.Z_0_2] = np.pi * wl * df_avg + df[star.UCSF.Z_neg2_2] = np.pi * wl * np.cos(two_angast) * df_dev # defocus asigmatism in X (Z1) + df[star.UCSF.Z_2_2] = np.pi * wl * np.sin(two_angast) * df_dev # defocus astigmatism in Y (Z2) + if 'ctf/cs_mm' in cs.dtype.names: + df[star.UCSF.Z_0_4] = -half_pi * wl_3 * cs['ctf/cs_mm'] * 1e7 # spherical abberation + if 'ctf/tetra_A' in cs.dtype.names: + df[star.UCSF.Z_neg4_4] = -half_pi * np.pi * wl_3 * cs['ctf/tetra_A'][:, 0] + df[star.UCSF.Z_neg2_4] = -half_pi * np.pi * wl_3 * cs['ctf/tetra_A'][:, 1] + df[star.UCSF.Z_2_4] = -half_pi * np.pi * wl_3 * cs['ctf/tetra_A'][:, 2] + df[star.UCSF.Z_4_4] = -half_pi * np.pi * wl_3 * cs['ctf/tetra_A'][:, 3] return df From 8e6faddae562dc510b5709453c8e6250b2cf449b Mon Sep 17 00:00:00 2001 From: Daniel Asarnow Date: Mon, 26 Sep 2022 15:29:32 -0700 Subject: [PATCH 17/20] -2 * pi for Zernike shift --- pyem/metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index e4d9f4f..ed6b236 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -313,8 +313,8 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): df[star.UCSF.Z_neg1_3] = two_pi * wl_2 * cs['ctf/tilt_A'][:, 0] df[star.UCSF.Z_1_3] = two_pi * wl_2 * cs['ctf/tilt_A'][:, 1] if 'ctf/shift_A' in cs.dtype.names: - df[star.UCSF.Z_neg1_1] = two_pi + cs['ctf/shift_A'][:, 0] - df[star.UCSF.Z_1_1] = two_pi + cs['ctf/shift_A'][:, 1] + df[star.UCSF.Z_neg1_1] = -two_pi + cs['ctf/shift_A'][:, 0] + df[star.UCSF.Z_1_1] = -two_pi + cs['ctf/shift_A'][:, 1] if 'ctf/trefoil_A' in cs.dtype.names and 'ctf/cs_mm' in cs.dtype.names: df[star.UCSF.Z_neg3_3] = two_pi * cs['ctf/cs_mm'] * wl_2 * cs['ctf/trefoil_A'][:, 0] df[star.UCSF.Z_3_3] = two_pi * cs['ctf/cs_mm'] * wl_2 * cs['ctf/trefoil_A'][:, 1] From d09478717df8199e8c88956273a63a8a92560920 Mon Sep 17 00:00:00 2001 From: Daniel Asarnow Date: Mon, 26 Sep 2022 15:35:15 -0700 Subject: [PATCH 18/20] Conversion formulas reviewed --- pyem/metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index ed6b236..e19e008 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -325,10 +325,10 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): df_dev = (cs['ctf/df1_A'] - cs['ctf/df2_A']) / 2 two_angast = 2 * cs['ctf/df_angle_rad'] df[star.UCSF.Z_0_2] = np.pi * wl * df_avg - df[star.UCSF.Z_neg2_2] = np.pi * wl * np.cos(two_angast) * df_dev # defocus asigmatism in X (Z1) - df[star.UCSF.Z_2_2] = np.pi * wl * np.sin(two_angast) * df_dev # defocus astigmatism in Y (Z2) + df[star.UCSF.Z_neg2_2] = np.pi * wl * np.cos(two_angast) * df_dev # Defocus asigmatism major (Z1) + df[star.UCSF.Z_2_2] = np.pi * wl * np.sin(two_angast) * df_dev # Defocus astigmatism minor (Z2) if 'ctf/cs_mm' in cs.dtype.names: - df[star.UCSF.Z_0_4] = -half_pi * wl_3 * cs['ctf/cs_mm'] * 1e7 # spherical abberation + df[star.UCSF.Z_0_4] = -half_pi * wl_3 * cs['ctf/cs_mm'] * 1e7 # Spherical aberration in Å. if 'ctf/tetra_A' in cs.dtype.names: df[star.UCSF.Z_neg4_4] = -half_pi * np.pi * wl_3 * cs['ctf/tetra_A'][:, 0] df[star.UCSF.Z_neg2_4] = -half_pi * np.pi * wl_3 * cs['ctf/tetra_A'][:, 1] From d87ada13e011fa99cf32c31a86f2c1c6697518e3 Mon Sep 17 00:00:00 2001 From: Daniel Asarnow Date: Mon, 26 Sep 2022 15:36:20 -0700 Subject: [PATCH 19/20] Moved shift conversion before tilt to match e.g. table from wikipedia --- pyem/metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index e19e008..1fe3ab9 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -309,12 +309,12 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): log.debug("Recovering beam tilt and converting to mrad") df[star.Relion.BEAMTILTX] = np.arcsin(cs['ctf/tilt_A'][:, 0] / cs['ctf/cs_mm'] * 1e-7) * 1e3 df[star.Relion.BEAMTILTY] = np.arcsin(cs['ctf/tilt_A'][:, 1] / cs['ctf/cs_mm'] * 1e-7) * 1e3 - if 'ctf/tilt_A' in cs.dtype.names: - df[star.UCSF.Z_neg1_3] = two_pi * wl_2 * cs['ctf/tilt_A'][:, 0] - df[star.UCSF.Z_1_3] = two_pi * wl_2 * cs['ctf/tilt_A'][:, 1] if 'ctf/shift_A' in cs.dtype.names: df[star.UCSF.Z_neg1_1] = -two_pi + cs['ctf/shift_A'][:, 0] df[star.UCSF.Z_1_1] = -two_pi + cs['ctf/shift_A'][:, 1] + if 'ctf/tilt_A' in cs.dtype.names: + df[star.UCSF.Z_neg1_3] = two_pi * wl_2 * cs['ctf/tilt_A'][:, 0] + df[star.UCSF.Z_1_3] = two_pi * wl_2 * cs['ctf/tilt_A'][:, 1] if 'ctf/trefoil_A' in cs.dtype.names and 'ctf/cs_mm' in cs.dtype.names: df[star.UCSF.Z_neg3_3] = two_pi * cs['ctf/cs_mm'] * wl_2 * cs['ctf/trefoil_A'][:, 0] df[star.UCSF.Z_3_3] = two_pi * cs['ctf/cs_mm'] * wl_2 * cs['ctf/trefoil_A'][:, 1] From 536751387a4f293e7ba781083cc24135f13a0c92 Mon Sep 17 00:00:00 2001 From: Daniel Asarnow Date: Mon, 26 Sep 2022 15:47:13 -0700 Subject: [PATCH 20/20] Precomputed pre-factors for all conversions - Also some additional corrections --- pyem/metadata.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/pyem/metadata.py b/pyem/metadata.py index 1fe3ab9..cfd198e 100644 --- a/pyem/metadata.py +++ b/pyem/metadata.py @@ -296,10 +296,13 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): # Calculate electron wavelentgh kv = cs['ctf/accel_kv'] wl = 12.2643247 / np.sqrt(kv * (1. + kv * 0.978466e-6)) - wl_2 = wl**2 - wl_3 = wl**3 + wl2 = wl**2 + wl3 = wl**3 half_pi = 0.5 * np.pi two_pi = 2 * np.pi + pi_wl = np.pi * wl + two_pi_wl2 = two_pi * wl2 + neg_half_pi_wl3 = -half_pi * wl3 if 'ctf/anisomag' in cs.dtype.names: df[star.Relion.MAGMAT00] = cs['ctf/anisomag'][:, 0] df[star.Relion.MAGMAT01] = cs['ctf/anisomag'][:, 1] @@ -309,31 +312,33 @@ def cryosparc_2_cs_ctf_parameters(cs, df=None): log.debug("Recovering beam tilt and converting to mrad") df[star.Relion.BEAMTILTX] = np.arcsin(cs['ctf/tilt_A'][:, 0] / cs['ctf/cs_mm'] * 1e-7) * 1e3 df[star.Relion.BEAMTILTY] = np.arcsin(cs['ctf/tilt_A'][:, 1] / cs['ctf/cs_mm'] * 1e-7) * 1e3 + log.debug("Converting odd Zernike moments") if 'ctf/shift_A' in cs.dtype.names: - df[star.UCSF.Z_neg1_1] = -two_pi + cs['ctf/shift_A'][:, 0] - df[star.UCSF.Z_1_1] = -two_pi + cs['ctf/shift_A'][:, 1] + df[star.UCSF.Z_neg1_1] = -two_pi * cs['ctf/shift_A'][:, 0] + df[star.UCSF.Z_1_1] = -two_pi * cs['ctf/shift_A'][:, 1] if 'ctf/tilt_A' in cs.dtype.names: - df[star.UCSF.Z_neg1_3] = two_pi * wl_2 * cs['ctf/tilt_A'][:, 0] - df[star.UCSF.Z_1_3] = two_pi * wl_2 * cs['ctf/tilt_A'][:, 1] + df[star.UCSF.Z_neg1_3] = two_pi_wl2 * cs['ctf/tilt_A'][:, 0] + df[star.UCSF.Z_1_3] = two_pi_wl2 * cs['ctf/tilt_A'][:, 1] if 'ctf/trefoil_A' in cs.dtype.names and 'ctf/cs_mm' in cs.dtype.names: - df[star.UCSF.Z_neg3_3] = two_pi * cs['ctf/cs_mm'] * wl_2 * cs['ctf/trefoil_A'][:, 0] - df[star.UCSF.Z_3_3] = two_pi * cs['ctf/cs_mm'] * wl_2 * cs['ctf/trefoil_A'][:, 1] + df[star.UCSF.Z_neg3_3] = two_pi_wl2 * cs['ctf/trefoil_A'][:, 0] + df[star.UCSF.Z_3_3] = two_pi_wl2 * cs['ctf/trefoil_A'][:, 1] + log.debug("Converting even Zernike moments") if 'ctf/amp_contrast' in cs.dtype.names: df[star.UCSF.Z_0_0] = cs['ctf/phase_shift_rad'] - np.arccos(cs['ctf/amp_contrast']) if 'ctf/df1_A' in cs.dtype.names and 'ctf/df2_A' in cs.dtype.names and 'ctf/df_angle_rad' in cs.dtype.names: df_avg = (cs['ctf/df1_A'] + cs['ctf/df2_A']) / 2 df_dev = (cs['ctf/df1_A'] - cs['ctf/df2_A']) / 2 two_angast = 2 * cs['ctf/df_angle_rad'] - df[star.UCSF.Z_0_2] = np.pi * wl * df_avg - df[star.UCSF.Z_neg2_2] = np.pi * wl * np.cos(two_angast) * df_dev # Defocus asigmatism major (Z1) - df[star.UCSF.Z_2_2] = np.pi * wl * np.sin(two_angast) * df_dev # Defocus astigmatism minor (Z2) + df[star.UCSF.Z_0_2] = pi_wl * df_avg + df[star.UCSF.Z_neg2_2] = pi_wl * np.cos(two_angast) * df_dev # Defocus asigmatism major (Z1). + df[star.UCSF.Z_2_2] = pi_wl * np.sin(two_angast) * df_dev # Defocus astigmatism minor (Z2). if 'ctf/cs_mm' in cs.dtype.names: - df[star.UCSF.Z_0_4] = -half_pi * wl_3 * cs['ctf/cs_mm'] * 1e7 # Spherical aberration in Å. + df[star.UCSF.Z_0_4] = neg_half_pi_wl3 * cs['ctf/cs_mm'] * 1e7 # Spherical aberration in Å. if 'ctf/tetra_A' in cs.dtype.names: - df[star.UCSF.Z_neg4_4] = -half_pi * np.pi * wl_3 * cs['ctf/tetra_A'][:, 0] - df[star.UCSF.Z_neg2_4] = -half_pi * np.pi * wl_3 * cs['ctf/tetra_A'][:, 1] - df[star.UCSF.Z_2_4] = -half_pi * np.pi * wl_3 * cs['ctf/tetra_A'][:, 2] - df[star.UCSF.Z_4_4] = -half_pi * np.pi * wl_3 * cs['ctf/tetra_A'][:, 3] + df[star.UCSF.Z_neg4_4] = neg_half_pi_wl3 * cs['ctf/tetra_A'][:, 0] + df[star.UCSF.Z_neg2_4] = neg_half_pi_wl3 * cs['ctf/tetra_A'][:, 1] + df[star.UCSF.Z_2_4] = neg_half_pi_wl3 * cs['ctf/tetra_A'][:, 2] + df[star.UCSF.Z_4_4] = neg_half_pi_wl3 * cs['ctf/tetra_A'][:, 3] return df