From 762c9782fbdc8345da85c79fefaa7cd4191fdb3e Mon Sep 17 00:00:00 2001 From: Frank West Date: Mon, 23 Jan 2017 16:52:11 +0000 Subject: [PATCH 1/4] Allows recalculation of patient caches In order to recalculate patient caches we want to use the recalculate option in the calculation job. Using the recalculate flag we can force an expiration of results. We also added an expired_at value to the patient caches in order for them not to be included in future calculations and results. These can be purged at any time in the future. --- lib/qme/map/map_reduce_executor.rb | 5 ++-- lib/qme/map/measure_calculation_job.rb | 5 ++-- lib/qme/quality_report.rb | 5 ++++ test/unit/qme/quality_report_test.rb | 41 +++++++++++++++++++++++++- 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/lib/qme/map/map_reduce_executor.rb b/lib/qme/map/map_reduce_executor.rb index 9ca138a..b75c30e 100644 --- a/lib/qme/map/map_reduce_executor.rb +++ b/lib/qme/map/map_reduce_executor.rb @@ -35,10 +35,11 @@ def build_query filters = @parameter_values["filters"] - match = {'value.measure_id' => @measure_id, + match = {'value.measure_id' => @measure_id, 'value.sub_id' => @sub_id, 'value.effective_date' => @parameter_values['effective_date'], 'value.test_id' => @parameter_values['test_id'], + 'value.expired_at' => nil, 'value.manual_exclusion' => {'$in' => [nil, false]}} if(filters) @@ -90,7 +91,7 @@ def calculate_supplemental_data_elements supplemental_data = Hash[*keys.map{|k| [k,{QME::QualityReport::RACE => {}, QME::QualityReport::ETHNICITY => {}, QME::QualityReport::SEX => {}, - QME::QualityReport::PAYER => {}}]}.flatten] + QME::QualityReport::PAYER => {}}]}.flatten] keys.each do |pop_id| pline = build_query diff --git a/lib/qme/map/measure_calculation_job.rb b/lib/qme/map/measure_calculation_job.rb index 6f41439..0624b3d 100644 --- a/lib/qme/map/measure_calculation_job.rb +++ b/lib/qme/map/measure_calculation_job.rb @@ -19,9 +19,10 @@ def initialize(options) end def perform - if !@quality_report.calculated? + if !@quality_report.calculated? || @options['recalculate'] map = QME::MapReduce::Executor.new(@quality_report.measure_id,@quality_report.sub_id, @options.merge('start_time' => Time.now.to_i)) - if !@quality_report.patients_cached? + if @quality_report.patients_cached? || @options['recalculate'] + @quality_report.expire_patient_results tick('Starting MapReduce') map.map_records_into_measure_groups(@options['prefilter']) tick('MapReduce complete') diff --git a/lib/qme/quality_report.rb b/lib/qme/quality_report.rb index c318b50..cc19449 100644 --- a/lib/qme/quality_report.rb +++ b/lib/qme/quality_report.rb @@ -166,6 +166,10 @@ def patient_results QME::PatientCache.where(patient_cache_matcher) end + def expire_patient_results + patient_results.update_all("value.expired_at" => Time.now) + end + def measure QME::QualityMeasure.where({"hqmf_id"=>self.measure_id, "sub_id" => self.sub_id}).first end @@ -189,6 +193,7 @@ def patient_cache_matcher 'value.sub_id' => self.sub_id, 'value.effective_date' => self.effective_date, 'value.test_id' => test_id, + 'value.expired_at' => nil, 'value.manual_exclusion' => {'$in' => [nil, false]}} if(filters) diff --git a/test/unit/qme/quality_report_test.rb b/test/unit/qme/quality_report_test.rb index d4f8f22..fc9ee75 100644 --- a/test/unit/qme/quality_report_test.rb +++ b/test/unit/qme/quality_report_test.rb @@ -45,7 +45,8 @@ def setup }, "measure_id" => "test2", "sub_id" => "b", - "effective_date" => Time.gm(2010, 9, 19).to_i + "effective_date" => Time.gm(2010, 9, 19).to_i, + "expired_at" => nil }} ) collection_fixtures(get_db(), 'delayed_backend_mongoid_jobs', '_id') @@ -129,4 +130,42 @@ def test_rollup_buffering assert_equal 0, Mongoid.default_client["rollup_buffer"].count({}) end + + def test_expire_patient_results + qr = QME::QualityReport.find_or_create( + 'test2', + 'b', + 'effective_date' => Time.gm(2010, 9, 19).to_i + ) + + + assert_equal 1, qr.patient_results.count + + qr.expire_patient_results + + assert_equal 0, qr.patient_results.count + end + + def test_patient_cache_matcher + effective_date = Time.gm(2010, 9, 19).to_i + qr = QME::QualityReport.find_or_create( + 'test2', + 'b', + 'effective_date' => effective_date + ) + + expected_results = { + 'value.measure_id' => 'test2', + 'value.sub_id' => 'b', + 'value.effective_date' => effective_date, + 'value.test_id' => nil, + 'value.facility_id' => nil, + 'value.expired_at' => nil, + 'value.manual_exclusion' => { + '$in' => [nil, false] + } + } + + assert_equal expected_results, qr.patient_cache_matcher + end end From a028320e5e49d79b731fde1826d39dd0826486b0 Mon Sep 17 00:00:00 2001 From: Frank West Date: Mon, 23 Jan 2017 17:29:28 +0000 Subject: [PATCH 2/4] Adds tests for recalculation This tests that recalculation only happens when we pass in the recalculation option. It also tests that on a second run the original results are expired and we only have the new results attached to the quality report. --- .../qme/map/measure_calculation_job_test.rb | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/test/unit/qme/map/measure_calculation_job_test.rb b/test/unit/qme/map/measure_calculation_job_test.rb index 2d96804..cd16b87 100644 --- a/test/unit/qme/map/measure_calculation_job_test.rb +++ b/test/unit/qme/map/measure_calculation_job_test.rb @@ -21,4 +21,47 @@ def test_perform job = Delayed::Job.enqueue(QME::MapReduce::MeasureCalculationJob.new({'quality_report_id' => qr.id,"oid_dictionary"=>{}})) assert job end -end \ No newline at end of file + + def test_perform_recalculate + options = { + 'measure_id' => '2E679CD2-3FEC-4A75-A75A-61403E5EFEE8', + 'effective_date' => Time.gm(2011, 1, 15).to_i, + } + + qr = QME::QualityReport.find_or_create_by(options) + job = QME::MapReduce::MeasureCalculationJob.new( + { + 'quality_report_id' => qr.id, + 'oid_dictionary' => {}, + 'recalculate' => true + } + ) + job.perform + job.perform # Perform second time to expire the originals + + assert_equal 4, qr.patient_results.count + assert_equal( + 4, QME::PatientCache.where(:'value.expired_at'.exists => true).count + ) + end + + def test_perform_no_recalculate + options = { + 'measure_id' => '2E679CD2-3FEC-4A75-A75A-61403E5EFEE8', + 'effective_date' => Time.gm(2011, 1, 15).to_i, + 'status.state' => 'complete' + } + + qr = QME::QualityReport.find_or_create_by(options) + job = QME::MapReduce::MeasureCalculationJob.new( + { + 'quality_report_id' => qr.id, + 'oid_dictionary' => {}, + 'recalculate' => false + } + ) + job.perform + + assert_equal 0, qr.patient_results.count + end +end From 9df8ff322a25306828f8978b0227ff57f7bcb7df Mon Sep 17 00:00:00 2001 From: Dillon Welch Date: Fri, 27 Jan 2017 14:43:52 -0800 Subject: [PATCH 3/4] Add the exclamation point back in --- lib/qme/map/measure_calculation_job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/qme/map/measure_calculation_job.rb b/lib/qme/map/measure_calculation_job.rb index 0624b3d..8a99145 100644 --- a/lib/qme/map/measure_calculation_job.rb +++ b/lib/qme/map/measure_calculation_job.rb @@ -21,7 +21,7 @@ def initialize(options) def perform if !@quality_report.calculated? || @options['recalculate'] map = QME::MapReduce::Executor.new(@quality_report.measure_id,@quality_report.sub_id, @options.merge('start_time' => Time.now.to_i)) - if @quality_report.patients_cached? || @options['recalculate'] + if !@quality_report.patients_cached? || @options['recalculate'] @quality_report.expire_patient_results tick('Starting MapReduce') map.map_records_into_measure_groups(@options['prefilter']) From e9aa6e7ad91cc3e0c268039c20b74aa85ee876fc Mon Sep 17 00:00:00 2001 From: Dillon Welch Date: Fri, 27 Jan 2017 15:01:40 -0800 Subject: [PATCH 4/4] Change test to look for non-existence of expired reports --- test/unit/qme/map/measure_calculation_job_test.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/unit/qme/map/measure_calculation_job_test.rb b/test/unit/qme/map/measure_calculation_job_test.rb index cd16b87..945bcc1 100644 --- a/test/unit/qme/map/measure_calculation_job_test.rb +++ b/test/unit/qme/map/measure_calculation_job_test.rb @@ -61,7 +61,11 @@ def test_perform_no_recalculate } ) job.perform + job.perform - assert_equal 0, qr.patient_results.count + assert_equal 4, qr.patient_results.count + assert_equal( + 0, QME::PatientCache.where(:'value.expired_at'.exists => true).count + ) end end