From e880d05ef125e865b127604acbdd26cd9748255a Mon Sep 17 00:00:00 2001 From: Tris Date: Wed, 17 Jun 2026 15:43:16 -0500 Subject: [PATCH 1/2] feat(parallelsearch): per-ID PEP for transient databases (train-one-model-on-base, score-all-transients) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracts the PEP component from PR #10 into its own branch/PR, so it can be reviewed and merged independently of the family-aware organism detection and other stacked features. Adds: - TransientPepAnalysisEngine: sealed EngineLayer class that trains ONE FastTree model on the (large, decoy-rich) base search and reuses it to assign PEP + PEP_QValue to every transient database, avoiding per-database retraining. - PepAnalysisEngine._randomSeed changed private→protected. - ParallelSearchTask integration: TrainBasePepModel() called after the base search on ManySearchTask; PEP assignments in PerformPostSearchAnalysis before metric collectors and stats. - BasicMetricCollector: 4 PEP-based confident count columns (PSM/peptide at 1% and 5% PEP_QValue). - TransientDatabaseMetrics: 4 PEP [Optional] properties with full Results-dictionary round-trip. - IsConfidentMatch static helper + OutputPepQValueThreshold const for testing. - 8 test files (4 new unit, 1 modified, 3 dedicated PEP tests, 1 E2E). Build: 0 errors. Tests: 19 PEP-specific pass, 14 existing pass. --- .../FdrAnalysis/PEPAnalysisEngine.cs | 4 +- .../TransientPepAnalysisEngine.cs | 171 ++++++++++++++++++ .../Collectors/BasicMetricCollector.cs | 23 ++- .../Analysis/TransientDatabaseMetrics.cs | 16 +- .../ParallelSearch/ParallelSearchTask.cs | 87 ++++++++- .../TaskLayer/Properties/AssemblyInfo.cs | 2 + .../Analysis/BasicMetricCollectorTests.cs | 39 ++++ ...TransientDatabaseMetricsPepColumnsTests.cs | 42 +++++ .../ConfidenceFilterTests.cs | 38 ++++ .../ParallelSearchEndToEndTests.cs | 59 ++++++ .../PepBackgroundCurveTests.cs | 111 ++++++++++++ .../PepTransientAssignmentTests.cs | 58 ++++++ 12 files changed, 644 insertions(+), 6 deletions(-) create mode 100644 MetaMorpheus/EngineLayer/ParallelSearch/TransientPepAnalysisEngine.cs create mode 100644 MetaMorpheus/TaskLayer/Properties/AssemblyInfo.cs create mode 100644 MetaMorpheus/Test/ParallelSearchTask/Analysis/TransientDatabaseMetricsPepColumnsTests.cs create mode 100644 MetaMorpheus/Test/ParallelSearchTask/ConfidenceFilterTests.cs create mode 100644 MetaMorpheus/Test/ParallelSearchTask/ParallelSearchEndToEndTests.cs create mode 100644 MetaMorpheus/Test/ParallelSearchTask/PepBackgroundCurveTests.cs create mode 100644 MetaMorpheus/Test/ParallelSearchTask/PepTransientAssignmentTests.cs diff --git a/MetaMorpheus/EngineLayer/FdrAnalysis/PEPAnalysisEngine.cs b/MetaMorpheus/EngineLayer/FdrAnalysis/PEPAnalysisEngine.cs index 0dbf45b94a..5188298265 100644 --- a/MetaMorpheus/EngineLayer/FdrAnalysis/PEPAnalysisEngine.cs +++ b/MetaMorpheus/EngineLayer/FdrAnalysis/PEPAnalysisEngine.cs @@ -1,4 +1,4 @@ -using Chemistry; +using Chemistry; using Chromatography.RetentionTimePrediction; using Chromatography.RetentionTimePrediction.SSRCalc; using EngineLayer.CrosslinkSearch; @@ -26,7 +26,7 @@ namespace EngineLayer { public class PepAnalysisEngine { - private int _randomSeed = 42; + protected int _randomSeed = 42; /// /// This method contains the hyper-parameters that will be used when training the machine learning model diff --git a/MetaMorpheus/EngineLayer/ParallelSearch/TransientPepAnalysisEngine.cs b/MetaMorpheus/EngineLayer/ParallelSearch/TransientPepAnalysisEngine.cs new file mode 100644 index 0000000000..1b26bc5daf --- /dev/null +++ b/MetaMorpheus/EngineLayer/ParallelSearch/TransientPepAnalysisEngine.cs @@ -0,0 +1,171 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using Chromatography.RetentionTimePrediction; +using EngineLayer.FdrAnalysis; +using Microsoft.ML; +using Microsoft.ML.Data; + +// The concrete trained-model type returned by the FastTree pipeline; aliased so the train-once / apply +// methods (this engine reuses one base-trained model across thousands of transient databases) stay readable. +using PepModel = Microsoft.ML.Data.TransformerChain>>; + +namespace EngineLayer.ParallelSearch; + +/// +/// ParallelSearch-specific PEP engine. The generic trains a fresh +/// cross-validated model per dataset; the transient databases searched by ParallelSearchTask are far too +/// small to support that. This derived engine trains ONE model on the large, decoy-rich base (human) search +/// and REUSES it — plus a snapshotted background PEP -> PEP_QValue curve — to score every transient +/// database's hits out-of-sample, without retraining. Keeping this behavior here leaves the base engine +/// focused on the standard workflow (mirrors and +/// ). +/// +public sealed class TransientPepAnalysisEngine : PepAnalysisEngine +{ + private MLContext? _trainedContext; + private PepModel? _trainedModel; + + // Background PEP -> PEP_QValue relationship, snapshotted from the base (human) search, which has + // enough decoys to compute a real PEP-based q-value. Sorted by PEP ascending; PEP_QValue is monotone + // non-decreasing along it. Transient peptides (far too small for their own PEP target/decoy) get their + // PEP_QValue by looking up their MODEL PEP on this curve. This is deliberately PEP-based, NOT borrowed + // from the score-based QValue: q-value and PEP_QValue rank matches differently and are not interchangeable. + private double[] _bgPepAscPsm = Array.Empty(), _bgQByPepPsm = Array.Empty(); + private double[] _bgPepAscPep = Array.Empty(), _bgQByPepPep = Array.Empty(); + + public TransientPepAnalysisEngine(List psms, string searchType, + List<(string fileName, CommonParameters fileSpecificParameters)> fileSpecificParameters, + string outputFolder, IRetentionTimePredictor? rtPredictor = null) + : base(psms, searchType, fileSpecificParameters, outputFolder, rtPredictor) + { + } + + /// True once has produced a reusable model. + public bool HasTrainedModel => _trainedModel != null; + + /// + /// Snapshots the (PEP ascending, PEP_QValue) curve over , which must already + /// carry a PEP in GetFdrInfo().PEP and contain decoys. Computes the + /// PEP-based q-value on this set, then captures the two monotone arrays. Mutates the matches' PEP_QValue + /// and cumulative target/decoy (harmless — the score-based QValue lives in a different field). + /// + internal static (double[] pepAsc, double[] qByPep) BuildPepQValueCurve(List matches, bool peptideLevel) + { + var ordered = matches.Where(m => m?.GetFdrInfo(peptideLevel) != null) + .OrderBy(m => m.GetFdrInfo(peptideLevel).PEP).ToList(); + if (ordered.Count == 0) + return (Array.Empty(), Array.Empty()); + FdrAnalysisEngine.CalculateQValue(ordered, peptideLevelCalculation: peptideLevel, pepCalculation: true); + var pepAsc = new double[ordered.Count]; + var qByPep = new double[ordered.Count]; + for (int i = 0; i < ordered.Count; i++) + { + pepAsc[i] = ordered[i].GetFdrInfo(peptideLevel).PEP; + qByPep[i] = ordered[i].GetFdrInfo(peptideLevel).PEP_QValue; + } + return (pepAsc, qByPep); + } + + /// + /// Maps a single PEP onto the background curve: returns the PEP_QValue of the background match whose PEP + /// is the smallest value >= (lower-bound). Because PEP_QValue is monotone in PEP, + /// this is the q-value at that confidence level. Returns 2 (the FdrInfo sentinel) when no curve exists. + /// + internal static double LookupBackgroundPepQValue(double pep, double[] pepAsc, double[] qByPep) + { + if (pepAsc.Length == 0) + return 2.0; + int l = 0, r = pepAsc.Length; + while (l < r) + { + int mid = (l + r) >> 1; + if (pepAsc[mid] >= pep) r = mid; else l = mid + 1; + } + int idx = l < pepAsc.Length ? l : pepAsc.Length - 1; + return qByPep[idx]; + } + + /// + /// Trains ONE PEP model on all of this engine's PSMs (the base search) and assigns their PEP, then keeps + /// the model + ML context so it can be REUSED — via — to score + /// out-of-sample PSMs (e.g. the per-database transient hits) WITHOUT retraining. The transient databases + /// are far too small to train their own model and are out-of-sample relative to this one, so the + /// cross-validation used by is unnecessary for + /// them. The feature dictionaries were already built from these PSMs in the constructor, so transient + /// feature vectors are computed against the same (base-run) calibration. Returns false when the base PSMs + /// lack both target and decoy training examples. + /// + public bool TrainSingleModelAndAssignBasePep() + { + List peptideGroups = UsePeptideLevelQValueForTraining + ? SpectralMatchGroup.GroupByBaseSequence(AllPsms) + : SpectralMatchGroup.GroupByIndividualPsm(AllPsms); + var allIndices = Enumerable.Range(0, peptideGroups.Count).ToList(); + + var psmData = CreatePsmData(SearchType, peptideGroups, allIndices).ToList(); + if (!psmData.Any(p => p.Label) || !psmData.Any(p => !p.Label)) + return false; // need both positive (target) and negative (decoy) examples + + var mlContext = new MLContext(seed: _randomSeed); + var trainer = mlContext.BinaryClassification.Trainers.FastTree(BGDTreeOptions); + var pipeline = mlContext.Transforms.Concatenate("Features", TrainingVariables).Append(trainer); + PepModel model = pipeline.Fit(mlContext.Data.LoadFromEnumerable(psmData)); + _trainedContext = mlContext; + _trainedModel = model; + + // Assign PEP to the base PSMs as well (the base assignment is for the base-search output only; + // transient PSMs scored later are genuinely out-of-sample, so no out-of-fold concern applies). + Compute_PSM_PEP(peptideGroups, allIndices, mlContext, model, SearchType, OutputFolder); + + // Snapshot the background PEP -> PEP_QValue curves used to assign transient PEP_QValue by lookup. + // PSM-level: over all base PSMs. Peptide-level: over one representative (best-scoring) PSM per base + // sequence, so the q-value reflects unique peptides. Both read PEP (identical in both FdrInfos). + (_bgPepAscPsm, _bgQByPepPsm) = BuildPepQValueCurve(AllPsms, peptideLevel: false); + var basePeptideReps = SpectralMatchGroup.GroupByBaseSequence(AllPsms) + .Select(g => g.OrderByDescending(p => p.Score).First()) + .ToList(); + (_bgPepAscPep, _bgQByPepPep) = BuildPepQValueCurve(basePeptideReps, peptideLevel: false); + return true; + } + + /// + /// Assigns PEP to a NEW set of PSMs (a transient database's hits) using the model trained on the base + /// search by . Reuses the base-trained model and the base + /// feature dictionaries — no retraining. Safe to call from many databases concurrently. No-op until a + /// model has been trained. + /// + public void AssignPepFromTrainedModel(List psms, bool peptideLevel = false) + { + if (_trainedModel == null || psms == null) + return; + var scorable = psms.Where(p => p != null).ToList(); + if (scorable.Count == 0) + return; + // Compute_PSM_PEP writes BOTH PsmFdrInfo.PEP and PeptideFdrInfo.PEP. Transient PSMs may not have a + // PeptideFdrInfo yet (peptide-level FDR isn't always run per database), so create whichever is + // missing — otherwise PEP is silently never assigned (the old guard skipped these PSMs entirely). + foreach (var p in scorable) + { + p.PsmFdrInfo ??= new FdrInfo(); + p.PeptideFdrInfo ??= new FdrInfo(); + } + List groups = UsePeptideLevelQValueForTraining + ? SpectralMatchGroup.GroupByBaseSequence(scorable) + : SpectralMatchGroup.GroupByIndividualPsm(scorable); + Compute_PSM_PEP(groups, Enumerable.Range(0, groups.Count).ToList(), _trainedContext!, _trainedModel, SearchType, OutputFolder); + + // Assign PEP_QValue by mapping each match's MODEL PEP through the BACKGROUND PEP->PEP_QValue curve. + // The transient database is far too small for its own PEP target/decoy (few/no decoys), so we borrow + // the relationship from the base search — but strictly on PEP, not on the score-based QValue, since + // q-value and PEP_QValue rank matches differently and are not interchangeable. + double[] pepAsc = peptideLevel ? _bgPepAscPep : _bgPepAscPsm; + double[] qByPep = peptideLevel ? _bgQByPepPep : _bgQByPepPsm; + foreach (var p in scorable) + { + var info = p.GetFdrInfo(peptideLevel); + info.PEP_QValue = LookupBackgroundPepQValue(info.PEP, pepAsc, qByPep); + } + } +} diff --git a/MetaMorpheus/TaskLayer/ParallelSearch/Analysis/Collectors/BasicMetricCollector.cs b/MetaMorpheus/TaskLayer/ParallelSearch/Analysis/Collectors/BasicMetricCollector.cs index d6951fd059..0a8c983b5c 100644 --- a/MetaMorpheus/TaskLayer/ParallelSearch/Analysis/Collectors/BasicMetricCollector.cs +++ b/MetaMorpheus/TaskLayer/ParallelSearch/Analysis/Collectors/BasicMetricCollector.cs @@ -1,4 +1,4 @@ -#nullable enable +#nullable enable using System; using System.Collections.Generic; using System.Linq; @@ -22,6 +22,14 @@ public class BasicMetricCollector : IMetricCollector public const string TargetPeptidesFromTransientDb = "TargetPeptidesFromTransientDb"; public const string TargetPeptidesFromTransientDbAtQValueThreshold = "TargetPeptidesFromTransientDbAtQValueThreshold"; + // PEP-based confident counts, reported at BOTH 1% and 5% PEP_QValue. PEP_QValue is the PEP-ranked q-value + // assigned by mapping each match's model PEP onto the background curve (see PepAnalysisEngine); it is a + // different confidence axis from the score-based QValue above, so both are reported side by side. + public const string TargetPsmsFromTransientDbAtPepQ01 = "TargetPsmsFromTransientDbAtPepQ01"; + public const string TargetPsmsFromTransientDbAtPepQ05 = "TargetPsmsFromTransientDbAtPepQ05"; + public const string TargetPeptidesFromTransientDbAtPepQ01 = "TargetPeptidesFromTransientDbAtPepQ01"; + public const string TargetPeptidesFromTransientDbAtPepQ05 = "TargetPeptidesFromTransientDbAtPepQ05"; + public string CollectorName => "ResultCount"; public IEnumerable GetOutputColumns() @@ -35,6 +43,10 @@ public IEnumerable GetOutputColumns() yield return TargetPeptidesAtQValueThreshold; yield return TargetPeptidesFromTransientDb; yield return TargetPeptidesFromTransientDbAtQValueThreshold; + yield return TargetPsmsFromTransientDbAtPepQ01; + yield return TargetPsmsFromTransientDbAtPepQ05; + yield return TargetPeptidesFromTransientDbAtPepQ01; + yield return TargetPeptidesFromTransientDbAtPepQ05; } public bool CanCollectData(TransientDatabaseContext context) @@ -61,7 +73,14 @@ public Dictionary CollectData(TransientDatabaseContext context) [TargetPeptidesAtQValueThreshold] = context.AllPeptides.Count(p => !p.IsDecoy && p.PeptideFdrInfo?.QValue <= qValueThreshold), [TargetPeptidesFromTransientDb] = context.TransientPeptides.Count(p => !p.IsDecoy), - [TargetPeptidesFromTransientDbAtQValueThreshold] = context.TransientPeptides.Count(p => !p.IsDecoy && p.PeptideFdrInfo?.QValue <= qValueThreshold) + [TargetPeptidesFromTransientDbAtQValueThreshold] = context.TransientPeptides.Count(p => !p.IsDecoy && p.PeptideFdrInfo?.QValue <= qValueThreshold), + + // PEP-based confident counts at 1% and 5%. PEP_QValue is only meaningful once a PEP model has been + // trained (transient matches default to the sentinel 2 otherwise), so these read 0 when PEP is off. + [TargetPsmsFromTransientDbAtPepQ01] = context.TransientPsms.Count(p => !p.IsDecoy && p.GetFdrInfo(false)?.PEP_QValue < 0.01), + [TargetPsmsFromTransientDbAtPepQ05] = context.TransientPsms.Count(p => !p.IsDecoy && p.GetFdrInfo(false)?.PEP_QValue < 0.05), + [TargetPeptidesFromTransientDbAtPepQ01] = context.TransientPeptides.Count(p => !p.IsDecoy && p.PeptideFdrInfo?.PEP_QValue < 0.01), + [TargetPeptidesFromTransientDbAtPepQ05] = context.TransientPeptides.Count(p => !p.IsDecoy && p.PeptideFdrInfo?.PEP_QValue < 0.05) }; } } \ No newline at end of file diff --git a/MetaMorpheus/TaskLayer/ParallelSearch/Analysis/TransientDatabaseMetrics.cs b/MetaMorpheus/TaskLayer/ParallelSearch/Analysis/TransientDatabaseMetrics.cs index f837d1cd5e..8ed9a270a6 100644 --- a/MetaMorpheus/TaskLayer/ParallelSearch/Analysis/TransientDatabaseMetrics.cs +++ b/MetaMorpheus/TaskLayer/ParallelSearch/Analysis/TransientDatabaseMetrics.cs @@ -61,7 +61,13 @@ public TransientDatabaseMetrics() { } public int TargetPeptidesAtQValueThreshold { get; set; } public int TargetPeptidesFromTransientDb { get; set; } public int TargetPeptidesFromTransientDbAtQValueThreshold { get; set; } - + + // PEP-based confident counts, reported at both 1% and 5% PEP_QValue (distinct confidence axis from QValue). + [Optional] public int TargetPsmsFromTransientDbAtPepQ01 { get; set; } + [Optional] public int TargetPsmsFromTransientDbAtPepQ05 { get; set; } + [Optional] public int TargetPeptidesFromTransientDbAtPepQ01 { get; set; } + [Optional] public int TargetPeptidesFromTransientDbAtPepQ05 { get; set; } + // Protein group metrics (0 if parsimony not run) public int TargetProteinGroupsAtQValueThreshold { get; set; } public int TargetProteinGroupsFromTransientDb { get; set; } @@ -365,6 +371,10 @@ public void PopulateResultsFromProperties() Results[BasicMetricCollector.TargetPeptidesAtQValueThreshold] = TargetPeptidesAtQValueThreshold; Results[BasicMetricCollector.TargetPeptidesFromTransientDb] = TargetPeptidesFromTransientDb; Results[BasicMetricCollector.TargetPeptidesFromTransientDbAtQValueThreshold] = TargetPeptidesFromTransientDbAtQValueThreshold; + Results[BasicMetricCollector.TargetPsmsFromTransientDbAtPepQ01] = TargetPsmsFromTransientDbAtPepQ01; + Results[BasicMetricCollector.TargetPsmsFromTransientDbAtPepQ05] = TargetPsmsFromTransientDbAtPepQ05; + Results[BasicMetricCollector.TargetPeptidesFromTransientDbAtPepQ01] = TargetPeptidesFromTransientDbAtPepQ01; + Results[BasicMetricCollector.TargetPeptidesFromTransientDbAtPepQ05] = TargetPeptidesFromTransientDbAtPepQ05; Results[ProteinGroupCollector.TargetProteinGroupsAtQValueThreshold] = TargetProteinGroupsAtQValueThreshold; Results[ProteinGroupCollector.TargetProteinGroupsFromTransientDb] = TargetProteinGroupsFromTransientDb; Results[ProteinGroupCollector.TargetProteinGroupsFromTransientDbAtQValueThreshold] = TargetProteinGroupsFromTransientDbAtQValueThreshold; @@ -486,6 +496,10 @@ public void PopulatePropertiesFromResults() TargetPeptidesAtQValueThreshold = GetValue(BasicMetricCollector.TargetPeptidesAtQValueThreshold); TargetPeptidesFromTransientDb = GetValue(BasicMetricCollector.TargetPeptidesFromTransientDb); TargetPeptidesFromTransientDbAtQValueThreshold = GetValue(BasicMetricCollector.TargetPeptidesFromTransientDbAtQValueThreshold); + TargetPsmsFromTransientDbAtPepQ01 = GetValue(BasicMetricCollector.TargetPsmsFromTransientDbAtPepQ01); + TargetPsmsFromTransientDbAtPepQ05 = GetValue(BasicMetricCollector.TargetPsmsFromTransientDbAtPepQ05); + TargetPeptidesFromTransientDbAtPepQ01 = GetValue(BasicMetricCollector.TargetPeptidesFromTransientDbAtPepQ01); + TargetPeptidesFromTransientDbAtPepQ05 = GetValue(BasicMetricCollector.TargetPeptidesFromTransientDbAtPepQ05); TargetProteinGroupsAtQValueThreshold = GetValue(ProteinGroupCollector.TargetProteinGroupsAtQValueThreshold); TargetProteinGroupsFromTransientDb = GetValue(ProteinGroupCollector.TargetProteinGroupsFromTransientDb); TargetProteinGroupsFromTransientDbAtQValueThreshold = GetValue(ProteinGroupCollector.TargetProteinGroupsFromTransientDbAtQValueThreshold); diff --git a/MetaMorpheus/TaskLayer/ParallelSearch/ParallelSearchTask.cs b/MetaMorpheus/TaskLayer/ParallelSearch/ParallelSearchTask.cs index d9ef752aa9..fb24a63bdb 100644 --- a/MetaMorpheus/TaskLayer/ParallelSearch/ParallelSearchTask.cs +++ b/MetaMorpheus/TaskLayer/ParallelSearch/ParallelSearchTask.cs @@ -24,6 +24,7 @@ using TaskLayer.ParallelSearch.Analysis.Collectors; using TaskLayer.ParallelSearch.Statistics; using TaskLayer.ParallelSearch.Util; +using Chromatography.RetentionTimePrediction; using ProteinGroup = EngineLayer.ProteinGroup; namespace TaskLayer.ParallelSearch; @@ -121,6 +122,12 @@ public override SearchParameters SearchParameters #endregion + // PEP model TRAINED ONCE on the base (human) search and reused to assign a PEP to every transient + // database's PSMs (they're too small to train their own model, and are out-of-sample vs the base). + [TomlIgnore] private TransientPepAnalysisEngine? _pepEngine; + // The PEP model's RT-feature predictor; lives as long as _pepEngine, disposed at task end. + [TomlIgnore] private IRetentionTimePredictor? _pepRtPredictor; + protected override MyTaskResults RunSpecific(string outputFolder, List dbFilenameList, List currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList) @@ -160,6 +167,15 @@ protected override MyTaskResults RunSpecific(string outputFolder, Initialize(taskId, dbFilenameList, currentRawFileList, fileSettingsList, outputFolder); InitializeCompletedDatabaseWriter(); + // PEP: train ONE model on the (large, high-quality) base human PSMs and reuse it to assign a PEP + // to every transient database's PSMs (they're too small to train their own model, and are + // out-of-sample vs the base). Uses SSRCalc3 for the RT residual feature. + // Best-effort: any failure leaves transient PEP unset. + var baselinePsms = BaseSearchPsms.Where(p => p != null) + .OrderByDescending(p => p).ToList(); + + TrainBasePepModel(baselinePsms, outputFolder, taskId); + Status($"Starting search of {TotalDatabases} transient databases...", taskId); ReportTaskDashboard(taskId, ParallelSearchDashboardUpdateKind.TaskStatus, DashboardPhaseSearching, $"Searching {TotalDatabases} transient databases..."); @@ -186,7 +202,11 @@ protected override MyTaskResults RunSpecific(string outputFolder, CompleteCompletedDatabaseWriter(); } - Finalization: + // PEP RT predictor is finished once the transient loop is done; release it. + _pepRtPredictor?.Dispose(); + _pepRtPredictor = null; + + Finalization: // If we have denovo results path, but it has not been collected to our results yet, collect the denovo data prior to running stats tests. if (ParallelSearchParameters.DeNovoMappingDataFilePath != null && _resultsManager.TransientDatabaseMetricsDictionary.All(p => p.Value.TotalPredictions == 0)) @@ -396,6 +416,45 @@ private void Initialize(string taskId, List dbFilenameList, Task.WhenAll([psmTask, peptideTask, proteinTask]).Wait(); } + /// + /// Trains ONE PEP model on the base (human) search PSMs and keeps it for assigning PEP to each transient + /// database's hits. Best-effort: any failure simply leaves the transient PEP unassigned. + /// + private void TrainBasePepModel(List baselinePsms, string outputFolder, string taskId) + { + try + { + var trainingPsms = baselinePsms + .Where(p => !p.IsDecoy && p.PsmFdrInfo?.QValue <= CommonParameters.QValueThreshold) + .ToList(); + if (trainingPsms.Count < 100) + { + Status($"PEP: only {trainingPsms.Count} base PSMs — skipping PEP model.", taskId); + return; + } + _pepRtPredictor = null; // use default SSRCalc3 from the base engine + var engine = new TransientPepAnalysisEngine( + trainingPsms, "standard", FileSpecificParameters, outputFolder, _pepRtPredictor); + if (engine.TrainSingleModelAndAssignBasePep()) + { + _pepEngine = engine; + Status("PEP: trained model on base search; assigning PEP to transient PSMs.", taskId); + } + else + { + Status("PEP: base PSMs lacked target/decoy training examples — PEP disabled.", taskId); + _pepRtPredictor.Dispose(); + _pepRtPredictor = null; + } + } + catch (Exception ex) + { + Status($"PEP training failed ({ex.GetType().Name}: {ex.Message}); PEP disabled.", taskId); + _pepRtPredictor?.Dispose(); + _pepRtPredictor = null; + } + } + private int LoadSpectraFiles(List currentRawFileList, FileSpecificParameters[] fileSettingsList, MyFileManager myFileManager, ConcurrentDictionary loadedSpectraByFile, string taskId) @@ -707,6 +766,16 @@ private void PerformSearch(List proteinsToSearch, SpectralMatch[] s .ToList(); } + // PEP: assign each transient PSM/peptide a posterior error probability + PEP_QValue (mapped onto the + // background curve) BEFORE the metric collectors and statistics run, so confident counts and family + // tests can use PEP_QValue. Runs after the FDR alignment so the borrowed score-based QValue is in place. + if (_pepEngine != null) + { + _pepEngine.AssignPepFromTrainedModel(transientPsms, peptideLevel: false); + if (transientPeptides.Count > 0) + _pepEngine.AssignPepFromTrainedModel(transientPeptides, peptideLevel: true); + } + #region Result Caching and Analysis // Create analysis context @@ -1104,6 +1173,22 @@ private void WriteGlobalResultsText(IReadOnlyDictionary + /// Pure confidence decision (extracted for testing). When pepActive, a match is confident if EITHER its PEP_QValue + /// is below pepQThreshold OR its score-based QValue is at or below qThreshold. When PEP is inactive, only the + /// score-based QValue is used. A null FdrInfo is never confident. + /// + internal static bool IsConfidentMatch(FdrInfo info, bool pepActive, double pepQThreshold, double qThreshold) + { + if (info == null) return false; + bool qConfident = info.QValue <= qThreshold; + return pepActive + ? info.PEP_QValue < pepQThreshold || qConfident + : qConfident; + } + private async Task WriteProteinGroupsToTsvAsync(List proteinGroups, string filePath) { if (proteinGroups != null && proteinGroups.Any()) diff --git a/MetaMorpheus/TaskLayer/Properties/AssemblyInfo.cs b/MetaMorpheus/TaskLayer/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..c1e4d7b8f0 --- /dev/null +++ b/MetaMorpheus/TaskLayer/Properties/AssemblyInfo.cs @@ -0,0 +1,2 @@ +// Exposes internal members (e.g. extracted pure decision helpers) to the Test assembly for unit testing. +[assembly: System.Runtime.CompilerServices.InternalsVisibleTo("Test")] diff --git a/MetaMorpheus/Test/ParallelSearchTask/Analysis/BasicMetricCollectorTests.cs b/MetaMorpheus/Test/ParallelSearchTask/Analysis/BasicMetricCollectorTests.cs index 99941fa62d..7a212b0265 100644 --- a/MetaMorpheus/Test/ParallelSearchTask/Analysis/BasicMetricCollectorTests.cs +++ b/MetaMorpheus/Test/ParallelSearchTask/Analysis/BasicMetricCollectorTests.cs @@ -73,4 +73,43 @@ public void CollectData_UsesMinimumThresholdAcrossPsmAndPeptide() Assert.That(result[BasicMetricCollector.TargetPeptidesFromTransientDbAtQValueThreshold], Is.EqualTo(1)); }); } + + [Test] + public void CollectData_PepQValueCounts_AtOneAndFivePercent_ExcludeDecoys() + { + var cp = ParallelSearchTestContextFactory.CreateCommonParameters(qValueThreshold: 0.01, pepQValueThreshold: 0.05); + + var transientPeptides = new List + { + ParallelSearchTestContextFactory.CreateSpectralMatch(cp, isDecoy: false, score: 30, psmQValue: 0.5, peptideQValue: 0.005, scanNumber: 1), + ParallelSearchTestContextFactory.CreateSpectralMatch(cp, isDecoy: false, score: 25, psmQValue: 0.5, peptideQValue: 0.03, scanNumber: 2), + ParallelSearchTestContextFactory.CreateSpectralMatch(cp, isDecoy: false, score: 20, psmQValue: 0.5, peptideQValue: 0.10, scanNumber: 3), + ParallelSearchTestContextFactory.CreateSpectralMatch(cp, isDecoy: true, score: 18, psmQValue: 0.5, peptideQValue: 0.001, scanNumber: 4), + }; + var transientPsms = new List + { + ParallelSearchTestContextFactory.CreateSpectralMatch(cp, isDecoy: false, score: 30, psmQValue: 0.005, peptideQValue: 0.5, scanNumber: 5), + ParallelSearchTestContextFactory.CreateSpectralMatch(cp, isDecoy: false, score: 25, psmQValue: 0.03, peptideQValue: 0.5, scanNumber: 6), + ParallelSearchTestContextFactory.CreateSpectralMatch(cp, isDecoy: true, score: 18, psmQValue: 0.001, peptideQValue: 0.5, scanNumber: 7), + }; + + var context = ParallelSearchTestContextFactory.CreateContext( + cp, + transientPsms, + transientPsms: transientPsms, + transientPeptides, + transientPeptides: transientPeptides, + totalProteins: 10, + transientPeptideCount: 4); + + var result = new BasicMetricCollector().CollectData(context); + + Assert.Multiple(() => + { + Assert.That(result[BasicMetricCollector.TargetPeptidesFromTransientDbAtPepQ01], Is.EqualTo(1)); + Assert.That(result[BasicMetricCollector.TargetPeptidesFromTransientDbAtPepQ05], Is.EqualTo(2)); + Assert.That(result[BasicMetricCollector.TargetPsmsFromTransientDbAtPepQ01], Is.EqualTo(1)); + Assert.That(result[BasicMetricCollector.TargetPsmsFromTransientDbAtPepQ05], Is.EqualTo(2)); + }); + } } diff --git a/MetaMorpheus/Test/ParallelSearchTask/Analysis/TransientDatabaseMetricsPepColumnsTests.cs b/MetaMorpheus/Test/ParallelSearchTask/Analysis/TransientDatabaseMetricsPepColumnsTests.cs new file mode 100644 index 0000000000..019982e479 --- /dev/null +++ b/MetaMorpheus/Test/ParallelSearchTask/Analysis/TransientDatabaseMetricsPepColumnsTests.cs @@ -0,0 +1,42 @@ +using NUnit.Framework; +using TaskLayer.ParallelSearch.Analysis; +using TaskLayer.ParallelSearch.Analysis.Collectors; + +namespace Test.ParallelSearchTask.Analysis; + +[TestFixture] +public class TransientDatabaseMetricsPepColumnsTests +{ + [Test] + public void PepQValueColumns_RoundTripThroughResultsDictionary() + { + var metrics = new TransientDatabaseMetrics("UP000464024") + { + TargetPsmsFromTransientDbAtPepQ01 = 91, + TargetPsmsFromTransientDbAtPepQ05 = 96, + TargetPeptidesFromTransientDbAtPepQ01 = 30, + TargetPeptidesFromTransientDbAtPepQ05 = 31, + }; + + metrics.PopulateResultsFromProperties(); + + Assert.Multiple(() => + { + Assert.That(metrics.Results[BasicMetricCollector.TargetPsmsFromTransientDbAtPepQ01], Is.EqualTo(91)); + Assert.That(metrics.Results[BasicMetricCollector.TargetPsmsFromTransientDbAtPepQ05], Is.EqualTo(96)); + Assert.That(metrics.Results[BasicMetricCollector.TargetPeptidesFromTransientDbAtPepQ01], Is.EqualTo(30)); + Assert.That(metrics.Results[BasicMetricCollector.TargetPeptidesFromTransientDbAtPepQ05], Is.EqualTo(31)); + }); + + var restored = new TransientDatabaseMetrics("UP000464024") { Results = metrics.Results }; + restored.PopulatePropertiesFromResults(); + + Assert.Multiple(() => + { + Assert.That(restored.TargetPsmsFromTransientDbAtPepQ01, Is.EqualTo(91)); + Assert.That(restored.TargetPsmsFromTransientDbAtPepQ05, Is.EqualTo(96)); + Assert.That(restored.TargetPeptidesFromTransientDbAtPepQ01, Is.EqualTo(30)); + Assert.That(restored.TargetPeptidesFromTransientDbAtPepQ05, Is.EqualTo(31)); + }); + } +} diff --git a/MetaMorpheus/Test/ParallelSearchTask/ConfidenceFilterTests.cs b/MetaMorpheus/Test/ParallelSearchTask/ConfidenceFilterTests.cs new file mode 100644 index 0000000000..db58340b6b --- /dev/null +++ b/MetaMorpheus/Test/ParallelSearchTask/ConfidenceFilterTests.cs @@ -0,0 +1,38 @@ +using EngineLayer.FdrAnalysis; +using NUnit.Framework; +using PST = TaskLayer.ParallelSearch.ParallelSearchTask; + +namespace Test.ParallelSearchTask; + +[TestFixture] +public class ConfidenceFilterTests +{ + [Test] + public void NullInfo_IsNeverConfident() + { + Assert.That(PST.IsConfidentMatch(null, pepActive: true, pepQThreshold: 0.05, qThreshold: 0.05), Is.False); + Assert.That(PST.IsConfidentMatch(null, pepActive: false, pepQThreshold: 0.05, qThreshold: 0.05), Is.False); + } + + [Test] + public void PepActive_ConfidentWhenPepQValueStrictlyBelowThreshold() + { + Assert.That(PST.IsConfidentMatch(new FdrInfo { PEP_QValue = 0.049, QValue = 0.9 }, true, 0.05, 0.05), Is.True); + Assert.That(PST.IsConfidentMatch(new FdrInfo { PEP_QValue = 0.05, QValue = 0.9 }, true, 0.05, 0.05), Is.False); + Assert.That(PST.IsConfidentMatch(new FdrInfo { PEP_QValue = 0.06, QValue = 0.9 }, true, 0.05, 0.05), Is.False); + } + + [Test] + public void PepActive_ConfidentByEitherMetric() + { + Assert.That(PST.IsConfidentMatch(new FdrInfo { PEP_QValue = 0.5, QValue = 0.001 }, true, 0.05, 0.05), Is.True); + Assert.That(PST.IsConfidentMatch(new FdrInfo { PEP_QValue = 0.5, QValue = 0.05 }, true, 0.05, 0.05), Is.True); + } + + [Test] + public void PepInactive_FallsBackToQValue_InclusiveThreshold() + { + Assert.That(PST.IsConfidentMatch(new FdrInfo { QValue = 0.05, PEP_QValue = 2.0 }, false, 0.05, 0.05), Is.True); + Assert.That(PST.IsConfidentMatch(new FdrInfo { QValue = 0.051, PEP_QValue = 0.0 }, false, 0.05, 0.05), Is.False); + } +} diff --git a/MetaMorpheus/Test/ParallelSearchTask/ParallelSearchEndToEndTests.cs b/MetaMorpheus/Test/ParallelSearchTask/ParallelSearchEndToEndTests.cs new file mode 100644 index 0000000000..839b919899 --- /dev/null +++ b/MetaMorpheus/Test/ParallelSearchTask/ParallelSearchEndToEndTests.cs @@ -0,0 +1,59 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using EngineLayer.DatabaseLoading; +using NUnit.Framework; +using TaskLayer; +using PSTask = TaskLayer.ParallelSearch.ParallelSearchTask; + +namespace Test.ParallelSearchTask; + +[TestFixture] +public class ParallelSearchEndToEndTests +{ + [Test] + public void RunTask_SmallYeast_ProducesSummaryWithPepColumns() + { + string testData = Path.Combine(TestContext.CurrentContext.TestDirectory, "TestData"); + string outputFolder = Path.Combine(TestContext.CurrentContext.TestDirectory, "ParallelSearchE2E"); + if (Directory.Exists(outputFolder)) + Directory.Delete(outputFolder, true); + Directory.CreateDirectory(outputFolder); + + string src = Path.Combine(testData, "SmallCalibratible_Yeast.mzML"); + string mzml1 = Path.Combine(outputFolder, "Yeast_1.mzML"); + string mzml2 = Path.Combine(outputFolder, "Yeast_2.mzML"); + File.Copy(src, mzml1, true); + File.Copy(src, mzml2, true); + + string baseDb = Path.Combine(testData, "smalldb.fasta"); + string transientDb = Path.Combine(testData, "smalldb.fasta"); + + var task = new PSTask(new List { new DbForTask(transientDb, false) }); + task.ParallelSearchParameters.MaxSearchesInParallel = 1; + + Assert.DoesNotThrow(() => + task.RunTask(outputFolder, new List { new DbForTask(baseDb, false) }, new List { mzml1, mzml2 }, "test")); + + var summary = Directory.EnumerateFiles(outputFolder, "ManySearchSummary.csv", SearchOption.AllDirectories).FirstOrDefault(); + Assert.That(summary, Is.Not.Null, "ManySearchSummary.csv was not written"); + + var lines = File.ReadAllLines(summary); + var header = lines[0].Split(','); + int Col(string name) => System.Array.IndexOf(header, name); + Assert.Multiple(() => + { + Assert.That(Col("TargetPeptidesFromTransientDbAtPepQ01"), Is.GreaterThanOrEqualTo(0)); + Assert.That(Col("TargetPeptidesFromTransientDbAtPepQ05"), Is.GreaterThanOrEqualTo(0)); + Assert.That(Col("TargetPsmsFromTransientDbAtPepQ01"), Is.GreaterThanOrEqualTo(0)); + Assert.That(Col("TargetPsmsFromTransientDbAtPepQ05"), Is.GreaterThanOrEqualTo(0)); + }); + + var row = lines.Skip(1).First(l => l.Trim().Length > 0).Split(','); + int transientPep = int.Parse(row[Col("TargetPeptidesFromTransientDb")]); + int pepQ05 = int.Parse(row[Col("TargetPeptidesFromTransientDbAtPepQ05")]); + + Assert.That(transientPep, Is.GreaterThanOrEqualTo(1), "transient search should produce hits"); + Assert.That(pepQ05, Is.GreaterThanOrEqualTo(1), "PEP model should assign a confident PEP_QValue to transient peptides"); + } +} diff --git a/MetaMorpheus/Test/ParallelSearchTask/PepBackgroundCurveTests.cs b/MetaMorpheus/Test/ParallelSearchTask/PepBackgroundCurveTests.cs new file mode 100644 index 0000000000..9de01fd450 --- /dev/null +++ b/MetaMorpheus/Test/ParallelSearchTask/PepBackgroundCurveTests.cs @@ -0,0 +1,111 @@ +using System.Collections.Generic; +using System.Linq; +using EngineLayer; +using EngineLayer.ParallelSearch; +using NUnit.Framework; +using Test.ParallelSearchTask.Utility; + +namespace Test.ParallelSearchTask; + +[TestFixture] +public class PepBackgroundCurveTests +{ + private static readonly double[] PepAsc = { 0.01, 0.05, 0.20, 0.60 }; + private static readonly double[] QByPep = { 0.001, 0.01, 0.05, 0.30 }; + + [Test] + public void Lookup_EmptyCurve_ReturnsSentinelTwo() + { + double q = TransientPepAnalysisEngine.LookupBackgroundPepQValue(0.01, System.Array.Empty(), System.Array.Empty()); + Assert.That(q, Is.EqualTo(2.0)); + } + + [Test] + public void Lookup_PepBelowAll_ReturnsLowestQValue() + { + double q = TransientPepAnalysisEngine.LookupBackgroundPepQValue(0.005, PepAsc, QByPep); + Assert.That(q, Is.EqualTo(0.001)); + } + + [Test] + public void Lookup_PepAboveAll_ReturnsHighestQValue() + { + double q = TransientPepAnalysisEngine.LookupBackgroundPepQValue(0.95, PepAsc, QByPep); + Assert.That(q, Is.EqualTo(0.30)); + } + + [Test] + public void Lookup_ExactPep_ReturnsThatQValue() + { + Assert.That(TransientPepAnalysisEngine.LookupBackgroundPepQValue(0.01, PepAsc, QByPep), Is.EqualTo(0.001)); + Assert.That(TransientPepAnalysisEngine.LookupBackgroundPepQValue(0.20, PepAsc, QByPep), Is.EqualTo(0.05)); + Assert.That(TransientPepAnalysisEngine.LookupBackgroundPepQValue(0.60, PepAsc, QByPep), Is.EqualTo(0.30)); + } + + [Test] + public void Lookup_PepBetweenPoints_ReturnsLowerBoundQValue() + { + Assert.That(TransientPepAnalysisEngine.LookupBackgroundPepQValue(0.03, PepAsc, QByPep), Is.EqualTo(0.01)); + Assert.That(TransientPepAnalysisEngine.LookupBackgroundPepQValue(0.50, PepAsc, QByPep), Is.EqualTo(0.30)); + } + + [Test] + public void Lookup_SingleElementCurve_AlwaysReturnsThatQValue() + { + var pepAsc = new[] { 0.10 }; + var qByPep = new[] { 0.02 }; + Assert.That(TransientPepAnalysisEngine.LookupBackgroundPepQValue(0.001, pepAsc, qByPep), Is.EqualTo(0.02)); + Assert.That(TransientPepAnalysisEngine.LookupBackgroundPepQValue(0.99, pepAsc, qByPep), Is.EqualTo(0.02)); + } + + [Test] + public void Lookup_IsMonotoneNonDecreasingInPep() + { + double prev = -1; + for (double pep = 0.0; pep <= 1.0; pep += 0.02) + { + double q = TransientPepAnalysisEngine.LookupBackgroundPepQValue(pep, PepAsc, QByPep); + Assert.That(q, Is.GreaterThanOrEqualTo(prev)); + prev = q; + } + } + + [Test] + public void BuildCurve_EmptyInput_ReturnsEmptyArrays() + { + var (pepAsc, qByPep) = TransientPepAnalysisEngine.BuildPepQValueCurve(new List(), peptideLevel: false); + Assert.That(pepAsc, Is.Empty); + Assert.That(qByPep, Is.Empty); + } + + [Test] + public void BuildCurve_TargetsAndDecoys_ProducesSortedMonotoneCurve() + { + var cp = ParallelSearchTestContextFactory.CreateCommonParameters(); + var matches = new List(); + double[] targetPeps = { 0.001, 0.01, 0.05, 0.10 }; + double[] decoyPeps = { 0.85, 0.90, 0.95, 0.99 }; + int scan = 1; + foreach (var p in targetPeps) + { + var m = ParallelSearchTestContextFactory.CreateSpectralMatch(cp, isDecoy: false, score: 20, psmQValue: 0.001, peptideQValue: 0.001, scanNumber: scan++); + m.PsmFdrInfo.PEP = p; + matches.Add(m); + } + foreach (var p in decoyPeps) + { + var m = ParallelSearchTestContextFactory.CreateSpectralMatch(cp, isDecoy: true, score: 20, psmQValue: 0.5, peptideQValue: 0.5, scanNumber: scan++); + m.PsmFdrInfo.PEP = p; + matches.Add(m); + } + + var (pepAsc, qByPep) = TransientPepAnalysisEngine.BuildPepQValueCurve(matches, peptideLevel: false); + + Assert.That(pepAsc.Length, Is.EqualTo(matches.Count)); + for (int i = 1; i < pepAsc.Length; i++) + Assert.That(pepAsc[i], Is.GreaterThanOrEqualTo(pepAsc[i - 1]), "PEP axis must be ascending"); + for (int i = 1; i < qByPep.Length; i++) + Assert.That(qByPep[i], Is.GreaterThanOrEqualTo(qByPep[i - 1]), "PEP_QValue must be non-decreasing in PEP"); + Assert.That(qByPep.First(), Is.LessThan(qByPep.Last())); + } +} diff --git a/MetaMorpheus/Test/ParallelSearchTask/PepTransientAssignmentTests.cs b/MetaMorpheus/Test/ParallelSearchTask/PepTransientAssignmentTests.cs new file mode 100644 index 0000000000..250995617b --- /dev/null +++ b/MetaMorpheus/Test/ParallelSearchTask/PepTransientAssignmentTests.cs @@ -0,0 +1,58 @@ +using System.Collections.Generic; +using EngineLayer; +using EngineLayer.ParallelSearch; +using NUnit.Framework; +using Test.ParallelSearchTask.Utility; + +namespace Test.ParallelSearchTask; + +[TestFixture] +public class PepTransientAssignmentTests +{ + private static List<(string, CommonParameters)> Fsp(CommonParameters cp) => new() { ("file.raw", cp) }; + + [Test] + public void AssignPepFromTrainedModel_NoModel_IsNoOp() + { + var cp = ParallelSearchTestContextFactory.CreateCommonParameters(); + var basePsms = new List + { + ParallelSearchTestContextFactory.CreateSpectralMatch(cp, isDecoy: false, score: 20, psmQValue: 0.001, peptideQValue: 0.001), + }; + var engine = new TransientPepAnalysisEngine(basePsms, "standard", Fsp(cp), TestContext.CurrentContext.TestDirectory); + Assert.That(engine.HasTrainedModel, Is.False); + + var transient = ParallelSearchTestContextFactory.CreateSpectralMatch(cp, isDecoy: false, score: 20, psmQValue: 0.5, peptideQValue: 0.5, scanNumber: 2); + transient.PeptideFdrInfo.PEP_QValue = 1.23; + + engine.AssignPepFromTrainedModel(new List { transient }, peptideLevel: true); + + Assert.That(transient.PeptideFdrInfo.PEP_QValue, Is.EqualTo(1.23), "no trained model -> no-op"); + } + + [Test] + public void TrainAndAssign_CreatesMissingFdrInfo_AndAssignsPepQValue() + { + var cp = ParallelSearchTestContextFactory.CreateCommonParameters(); + var basePsms = new List(); + for (int i = 0; i < 12; i++) + basePsms.Add(ParallelSearchTestContextFactory.CreateSpectralMatch(cp, isDecoy: false, score: 20 + i, psmQValue: 0.001, peptideQValue: 0.001, scanNumber: i + 1)); + for (int i = 0; i < 12; i++) + basePsms.Add(ParallelSearchTestContextFactory.CreateSpectralMatch(cp, isDecoy: true, score: 5 + i, psmQValue: 0.5, peptideQValue: 0.5, scanNumber: 100 + i)); + + var engine = new TransientPepAnalysisEngine(basePsms, "standard", Fsp(cp), TestContext.CurrentContext.TestDirectory); + bool trained = engine.TrainSingleModelAndAssignBasePep(); + Assume.That(trained, Is.True); + Assert.That(engine.HasTrainedModel, Is.True); + + var transient = ParallelSearchTestContextFactory.CreateSpectralMatch(cp, isDecoy: false, score: 25, psmQValue: 0.5, peptideQValue: 0.5, scanNumber: 500); + transient.PeptideFdrInfo = null; + transient.PsmFdrInfo = null; + + engine.AssignPepFromTrainedModel(new List { transient }, peptideLevel: true); + + Assert.That(transient.PeptideFdrInfo, Is.Not.Null, "missing PeptideFdrInfo must be created"); + Assert.That(transient.PsmFdrInfo, Is.Not.Null, "missing PsmFdrInfo must be created"); + Assert.That(transient.PeptideFdrInfo.PEP_QValue, Is.Not.EqualTo(2.0), "PEP_QValue must be assigned from the background curve"); + } +} From 96f1c0b64d52f295928914520961d039a654b97a Mon Sep 17 00:00:00 2001 From: Tris Date: Wed, 17 Jun 2026 16:42:09 -0500 Subject: [PATCH 2/2] address review: move PEP training into Initialize, remove IsConfidentMatch/OuputPepQValueThreshold, remove RT predictor field --- .../ParallelSearch/ParallelSearchTask.cs | 79 +++---------------- .../ConfidenceFilterTests.cs | 38 --------- 2 files changed, 10 insertions(+), 107 deletions(-) delete mode 100644 MetaMorpheus/Test/ParallelSearchTask/ConfidenceFilterTests.cs diff --git a/MetaMorpheus/TaskLayer/ParallelSearch/ParallelSearchTask.cs b/MetaMorpheus/TaskLayer/ParallelSearch/ParallelSearchTask.cs index fb24a63bdb..b0601e5a54 100644 --- a/MetaMorpheus/TaskLayer/ParallelSearch/ParallelSearchTask.cs +++ b/MetaMorpheus/TaskLayer/ParallelSearch/ParallelSearchTask.cs @@ -24,7 +24,6 @@ using TaskLayer.ParallelSearch.Analysis.Collectors; using TaskLayer.ParallelSearch.Statistics; using TaskLayer.ParallelSearch.Util; -using Chromatography.RetentionTimePrediction; using ProteinGroup = EngineLayer.ProteinGroup; namespace TaskLayer.ParallelSearch; @@ -125,9 +124,6 @@ public override SearchParameters SearchParameters // PEP model TRAINED ONCE on the base (human) search and reused to assign a PEP to every transient // database's PSMs (they're too small to train their own model, and are out-of-sample vs the base). [TomlIgnore] private TransientPepAnalysisEngine? _pepEngine; - // The PEP model's RT-feature predictor; lives as long as _pepEngine, disposed at task end. - [TomlIgnore] private IRetentionTimePredictor? _pepRtPredictor; - protected override MyTaskResults RunSpecific(string outputFolder, List dbFilenameList, List currentRawFileList, string taskId, FileSpecificParameters[] fileSettingsList) @@ -167,15 +163,6 @@ protected override MyTaskResults RunSpecific(string outputFolder, Initialize(taskId, dbFilenameList, currentRawFileList, fileSettingsList, outputFolder); InitializeCompletedDatabaseWriter(); - // PEP: train ONE model on the (large, high-quality) base human PSMs and reuse it to assign a PEP - // to every transient database's PSMs (they're too small to train their own model, and are - // out-of-sample vs the base). Uses SSRCalc3 for the RT residual feature. - // Best-effort: any failure leaves transient PEP unset. - var baselinePsms = BaseSearchPsms.Where(p => p != null) - .OrderByDescending(p => p).ToList(); - - TrainBasePepModel(baselinePsms, outputFolder, taskId); - Status($"Starting search of {TotalDatabases} transient databases...", taskId); ReportTaskDashboard(taskId, ParallelSearchDashboardUpdateKind.TaskStatus, DashboardPhaseSearching, $"Searching {TotalDatabases} transient databases..."); @@ -202,11 +189,7 @@ protected override MyTaskResults RunSpecific(string outputFolder, CompleteCompletedDatabaseWriter(); } - // PEP RT predictor is finished once the transient loop is done; release it. - _pepRtPredictor?.Dispose(); - _pepRtPredictor = null; - - Finalization: + Finalization: // If we have denovo results path, but it has not been collected to our results yet, collect the denovo data prior to running stats tests. if (ParallelSearchParameters.DeNovoMappingDataFilePath != null && _resultsManager.TransientDatabaseMetricsDictionary.All(p => p.Value.TotalPredictions == 0)) @@ -413,46 +396,20 @@ private void Initialize(string taskId, List dbFilenameList, ProseCreatedWhileRunning.Append( $"Searching {ParallelSearchParameters.TransientDatabases.Count} transient databases against {currentRawFileList.Count} spectra files. "); - Task.WhenAll([psmTask, peptideTask, proteinTask]).Wait(); - } - - /// - /// Trains ONE PEP model on the base (human) search PSMs and keeps it for assigning PEP to each transient - /// database's hits. Best-effort: any failure simply leaves the transient PEP unassigned. - /// - private void TrainBasePepModel(List baselinePsms, string outputFolder, string taskId) - { - try + // PEP: train ONE model on the base search PSMs (decoy-rich, large) before writing the base output + // files, so the base PSM/peptide/protein files include PEP and PEP_QValue. + var trainingPsms = baselinePsms + .Where(p => !p.IsDecoy && p.PsmFdrInfo?.QValue <= CommonParameters.QValueThreshold) + .ToList(); + if (trainingPsms.Count >= 100) { - var trainingPsms = baselinePsms - .Where(p => !p.IsDecoy && p.PsmFdrInfo?.QValue <= CommonParameters.QValueThreshold) - .ToList(); - if (trainingPsms.Count < 100) - { - Status($"PEP: only {trainingPsms.Count} base PSMs — skipping PEP model.", taskId); - return; - } - _pepRtPredictor = null; // use default SSRCalc3 from the base engine var engine = new TransientPepAnalysisEngine( - trainingPsms, "standard", FileSpecificParameters, outputFolder, _pepRtPredictor); + trainingPsms, "standard", FileSpecificParameters, outputFolder, null); if (engine.TrainSingleModelAndAssignBasePep()) - { _pepEngine = engine; - Status("PEP: trained model on base search; assigning PEP to transient PSMs.", taskId); - } - else - { - Status("PEP: base PSMs lacked target/decoy training examples — PEP disabled.", taskId); - _pepRtPredictor.Dispose(); - _pepRtPredictor = null; - } - } - catch (Exception ex) - { - Status($"PEP training failed ({ex.GetType().Name}: {ex.Message}); PEP disabled.", taskId); - _pepRtPredictor?.Dispose(); - _pepRtPredictor = null; } + + Task.WhenAll([psmTask, peptideTask, proteinTask]).Wait(); } private int LoadSpectraFiles(List currentRawFileList, FileSpecificParameters[] fileSettingsList, @@ -1173,22 +1130,6 @@ private void WriteGlobalResultsText(IReadOnlyDictionary - /// Pure confidence decision (extracted for testing). When pepActive, a match is confident if EITHER its PEP_QValue - /// is below pepQThreshold OR its score-based QValue is at or below qThreshold. When PEP is inactive, only the - /// score-based QValue is used. A null FdrInfo is never confident. - /// - internal static bool IsConfidentMatch(FdrInfo info, bool pepActive, double pepQThreshold, double qThreshold) - { - if (info == null) return false; - bool qConfident = info.QValue <= qThreshold; - return pepActive - ? info.PEP_QValue < pepQThreshold || qConfident - : qConfident; - } - private async Task WriteProteinGroupsToTsvAsync(List proteinGroups, string filePath) { if (proteinGroups != null && proteinGroups.Any()) diff --git a/MetaMorpheus/Test/ParallelSearchTask/ConfidenceFilterTests.cs b/MetaMorpheus/Test/ParallelSearchTask/ConfidenceFilterTests.cs deleted file mode 100644 index db58340b6b..0000000000 --- a/MetaMorpheus/Test/ParallelSearchTask/ConfidenceFilterTests.cs +++ /dev/null @@ -1,38 +0,0 @@ -using EngineLayer.FdrAnalysis; -using NUnit.Framework; -using PST = TaskLayer.ParallelSearch.ParallelSearchTask; - -namespace Test.ParallelSearchTask; - -[TestFixture] -public class ConfidenceFilterTests -{ - [Test] - public void NullInfo_IsNeverConfident() - { - Assert.That(PST.IsConfidentMatch(null, pepActive: true, pepQThreshold: 0.05, qThreshold: 0.05), Is.False); - Assert.That(PST.IsConfidentMatch(null, pepActive: false, pepQThreshold: 0.05, qThreshold: 0.05), Is.False); - } - - [Test] - public void PepActive_ConfidentWhenPepQValueStrictlyBelowThreshold() - { - Assert.That(PST.IsConfidentMatch(new FdrInfo { PEP_QValue = 0.049, QValue = 0.9 }, true, 0.05, 0.05), Is.True); - Assert.That(PST.IsConfidentMatch(new FdrInfo { PEP_QValue = 0.05, QValue = 0.9 }, true, 0.05, 0.05), Is.False); - Assert.That(PST.IsConfidentMatch(new FdrInfo { PEP_QValue = 0.06, QValue = 0.9 }, true, 0.05, 0.05), Is.False); - } - - [Test] - public void PepActive_ConfidentByEitherMetric() - { - Assert.That(PST.IsConfidentMatch(new FdrInfo { PEP_QValue = 0.5, QValue = 0.001 }, true, 0.05, 0.05), Is.True); - Assert.That(PST.IsConfidentMatch(new FdrInfo { PEP_QValue = 0.5, QValue = 0.05 }, true, 0.05, 0.05), Is.True); - } - - [Test] - public void PepInactive_FallsBackToQValue_InclusiveThreshold() - { - Assert.That(PST.IsConfidentMatch(new FdrInfo { QValue = 0.05, PEP_QValue = 2.0 }, false, 0.05, 0.05), Is.True); - Assert.That(PST.IsConfidentMatch(new FdrInfo { QValue = 0.051, PEP_QValue = 0.0 }, false, 0.05, 0.05), Is.False); - } -}