From ff80b5763400182858a0ec65b45200340cbcbdf5 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 25 Oct 2022 10:16:52 +0100 Subject: [PATCH 1/4] with ProvenanceReader() as db --- .../redundant_packet_count_report.py | 34 +++++++++---------- .../test_many_boards/many_boards.py | 3 +- .../test_onchip_compressor/many_routes.py | 3 +- .../test_onchip_compressor/one_route.py | 3 +- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/spynnaker/pyNN/extra_algorithms/redundant_packet_count_report.py b/spynnaker/pyNN/extra_algorithms/redundant_packet_count_report.py index 6180f116d48..a939e684827 100644 --- a/spynnaker/pyNN/extra_algorithms/redundant_packet_count_report.py +++ b/spynnaker/pyNN/extra_algorithms/redundant_packet_count_report.py @@ -95,23 +95,23 @@ def _create_views(): def _write_report(output): - reader = ProvenanceReader() - for data in reader.run_query("select * from redundancy_by_core"): - (_, _, _, source, _, filtered, invalid, _, - redundant, total, percent) = data - output.write(f"\ncore {source} \n") - output.write(f" {total} packets received. \n") - output.write(f" {redundant} were detected as " - "redundant packets by the bitfield filter. \n") - output.write( - f" {filtered} were detected as having no targets " - f"after the DMA stage. \n") - output.write( - f" {invalid} were detected as packets which " - f"we should not have received in the first place. \n") - output.write(f" Overall this makes a redundant percentage of " - f"{percent}\n") - data = reader.run_query("select * from redundancy_summary") + with ProvenanceReader() as db: + for data in db.run_query("select * from redundancy_by_core"): + (_, _, _, source, _, filtered, invalid, _, + redundant, total, percent) = data + output.write(f"\ncore {source} \n") + output.write(f" {total} packets received. \n") + output.write(f" {redundant} were detected as " + "redundant packets by the bitfield filter. \n") + output.write( + f" {filtered} were detected as having no targets " + f"after the DMA stage. \n") + output.write( + f" {invalid} were detected as packets which " + f"we should not have received in the first place. \n") + output.write(f" Overall this makes a redundant percentage of " + f"{percent}\n") + data = db.run_query("select * from redundancy_summary") (sum_total, max_total, min_total, avg_total, sum_reduant, max_redundant, min_redundant, avg_redundant, max_percent, min_percent, avg_percent, global_percent) = data[0] diff --git a/spynnaker_integration_tests/test_many_boards/many_boards.py b/spynnaker_integration_tests/test_many_boards/many_boards.py index f3a6714e8c9..a1cb41ce8b3 100644 --- a/spynnaker_integration_tests/test_many_boards/many_boards.py +++ b/spynnaker_integration_tests/test_many_boards/many_boards.py @@ -87,7 +87,8 @@ def do_run(self): for pop in self._pops: check_data(pop, self._expected_spikes, self.simtime) t_after_check = time.time() - results = ProvenanceReader().get_run_time_of_BufferExtractor() + with ProvenanceReader() as db: + results = db.get_run_time_of_BufferExtractor() self.report(results, report_file) self.report( "machine run time was: {} seconds\n".format( diff --git a/spynnaker_integration_tests/test_onchip_compressor/many_routes.py b/spynnaker_integration_tests/test_onchip_compressor/many_routes.py index 829217b776b..047c6d0a65a 100644 --- a/spynnaker_integration_tests/test_onchip_compressor/many_routes.py +++ b/spynnaker_integration_tests/test_onchip_compressor/many_routes.py @@ -76,6 +76,7 @@ def do_run(): receptor_type="inhibitory") sim.run(1) - t = ProvenanceReader().get_timer_provenance("Routing table loader") + with ProvenanceReader() as db: + t = db.get_timer_provenance("Routing table loader") assert t == "", "Routing table loader should not have run" sim.end() diff --git a/spynnaker_integration_tests/test_onchip_compressor/one_route.py b/spynnaker_integration_tests/test_onchip_compressor/one_route.py index e87fa74bc4d..0b39dab1d7b 100644 --- a/spynnaker_integration_tests/test_onchip_compressor/one_route.py +++ b/spynnaker_integration_tests/test_onchip_compressor/one_route.py @@ -80,6 +80,7 @@ def do_one_run(): receptor_type="inhibitory") sim.run(1) - t = ProvenanceReader().get_timer_provenance("Routing table loader") + with ProvenanceReader() as db: + t = db.get_timer_provenance("Routing table loader") assert t == "", "Routing table loader should not have run" sim.end() From f1e06d7cc5119d704a48ced54d540e547bcf0626 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 25 Oct 2022 13:13:48 +0100 Subject: [PATCH 2/4] global provenance --- spynnaker/pyNN/spinnaker.py | 4 ++-- spynnaker_integration_tests/test_many_boards/many_boards.py | 4 ++-- .../test_onchip_compressor/many_routes.py | 4 ++-- .../test_onchip_compressor/one_route.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/spynnaker/pyNN/spinnaker.py b/spynnaker/pyNN/spinnaker.py index 84552e0be01..03b2c77cc22 100644 --- a/spynnaker/pyNN/spinnaker.py +++ b/spynnaker/pyNN/spinnaker.py @@ -29,7 +29,7 @@ from spinn_front_end_common.interface.abstract_spinnaker_base import ( AbstractSpinnakerBase) from spinn_front_end_common.interface.provenance import ( - FecTimer, ProvenanceWriter, TimerCategory, TimerWork) + FecTimer, GlobalProvenance, TimerCategory, TimerWork) from spinn_front_end_common.utilities.constants import ( MICRO_TO_MILLISECOND_CONVERSION) from spinn_front_end_common.utilities.exceptions import ConfigurationException @@ -118,7 +118,7 @@ def __init__( # set up machine targeted data self._set_up_timings(timestep, min_delay, time_scale_factor) - with ProvenanceWriter() as db: + with GlobalProvenance() as db: db.insert_version("sPyNNaker_version", _version.__version__) db.insert_version("pyNN_version", pynn_version) db.insert_version("quantities_version", quantities_version) diff --git a/spynnaker_integration_tests/test_many_boards/many_boards.py b/spynnaker_integration_tests/test_many_boards/many_boards.py index a1cb41ce8b3..59dce292deb 100644 --- a/spynnaker_integration_tests/test_many_boards/many_boards.py +++ b/spynnaker_integration_tests/test_many_boards/many_boards.py @@ -15,7 +15,7 @@ import time from unittest import SkipTest from spinn_utilities.config_holder import get_config_bool -from spinn_front_end_common.interface.provenance import ProvenanceReader +from spinn_front_end_common.interface.provenance import GlobalProvenance from spynnaker.pyNN.exceptions import ConfigurationException import pyNN.spiNNaker as sim from spynnaker_integration_tests.scripts import check_data @@ -87,7 +87,7 @@ def do_run(self): for pop in self._pops: check_data(pop, self._expected_spikes, self.simtime) t_after_check = time.time() - with ProvenanceReader() as db: + with GlobalProvenance() as db: results = db.get_run_time_of_BufferExtractor() self.report(results, report_file) self.report( diff --git a/spynnaker_integration_tests/test_onchip_compressor/many_routes.py b/spynnaker_integration_tests/test_onchip_compressor/many_routes.py index 047c6d0a65a..5d36cb9565d 100644 --- a/spynnaker_integration_tests/test_onchip_compressor/many_routes.py +++ b/spynnaker_integration_tests/test_onchip_compressor/many_routes.py @@ -14,7 +14,7 @@ # along with this program. If not, see . import math from unittest import SkipTest -from spinn_front_end_common.interface.provenance import ProvenanceReader +from spinn_front_end_common.interface.provenance import GlobalProvenance from spynnaker.pyNN.exceptions import ConfigurationException import pyNN.spiNNaker as sim from spynnaker.pyNN.extra_algorithms.splitter_components import ( @@ -76,7 +76,7 @@ def do_run(): receptor_type="inhibitory") sim.run(1) - with ProvenanceReader() as db: + with GlobalProvenance() as db: t = db.get_timer_provenance("Routing table loader") assert t == "", "Routing table loader should not have run" sim.end() diff --git a/spynnaker_integration_tests/test_onchip_compressor/one_route.py b/spynnaker_integration_tests/test_onchip_compressor/one_route.py index 0b39dab1d7b..35aa45a6c9e 100644 --- a/spynnaker_integration_tests/test_onchip_compressor/one_route.py +++ b/spynnaker_integration_tests/test_onchip_compressor/one_route.py @@ -14,7 +14,7 @@ # along with this program. If not, see . import math from unittest import SkipTest -from spinn_front_end_common.interface.provenance import ProvenanceReader +from spinn_front_end_common.interface.provenance import GlobalProvenance from spynnaker.pyNN.exceptions import ConfigurationException import pyNN.spiNNaker as sim from spynnaker.pyNN.extra_algorithms.splitter_components import ( @@ -80,7 +80,7 @@ def do_one_run(): receptor_type="inhibitory") sim.run(1) - with ProvenanceReader() as db: + with GlobalProvenance() as db: t = db.get_timer_provenance("Routing table loader") assert t == "", "Routing table loader should not have run" sim.end() From 26a3311e7845638d66b50cf2f4083f6b0d1ae1b9 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 26 Oct 2022 11:10:22 +0100 Subject: [PATCH 3/4] check for data sqlite3 files --- spynnaker_integration_tests/test_debug_mode/check_debug.py | 4 ++++ .../test_using_virtual_board/test_debug_mode/test_debug.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/spynnaker_integration_tests/test_debug_mode/check_debug.py b/spynnaker_integration_tests/test_debug_mode/check_debug.py index d79c4ce4dfe..35ba18f8ef5 100644 --- a/spynnaker_integration_tests/test_debug_mode/check_debug.py +++ b/spynnaker_integration_tests/test_debug_mode/check_debug.py @@ -105,6 +105,7 @@ def debug(self): found = os.listdir(SpynnakerDataView.get_run_dir_path()) for report in reports: self.assertIn(report, found) + self.assertIn("data.sqlite3", found) sim.run(10) pop.get_data("v") @@ -115,6 +116,8 @@ def debug(self): SpynnakerDataView.set_requires_data_generation() sim.run(10) pop.get_data("v") + found = os.listdir(SpynnakerDataView.get_run_dir_path()) + self.assertIn("data1.sqlite3", found) # No point in checking files they are already there sim.reset() @@ -124,5 +127,6 @@ def debug(self): found = os.listdir(SpynnakerDataView.get_run_dir_path()) for report in reports: self.assertIn(report, found) + self.assertIn("data2.sqlite3", found) sim.end() diff --git a/unittests/test_using_virtual_board/test_debug_mode/test_debug.py b/unittests/test_using_virtual_board/test_debug_mode/test_debug.py index e01e676ec71..c22b07d8bc3 100644 --- a/unittests/test_using_virtual_board/test_debug_mode/test_debug.py +++ b/unittests/test_using_virtual_board/test_debug_mode/test_debug.py @@ -64,7 +64,7 @@ def debug(self): # write_network_specification_report network_specification_file_name, # write_provenance_data - "provenance_data", + "data.sqlite3", # write_tag_allocation_reports reports_names._TAGS_FILENAME, # write_algorithm_timings From 92c7c601f00fb1c686d54a59a1cd5b89e9ea075b Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Fri, 28 Oct 2022 10:55:12 +0100 Subject: [PATCH 4/4] dont overwrite the redundant_packet_count.rpt --- .../pyNN/extra_algorithms/redundant_packet_count_report.py | 5 +++-- spynnaker_integration_tests/test_debug_mode/check_debug.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/spynnaker/pyNN/extra_algorithms/redundant_packet_count_report.py b/spynnaker/pyNN/extra_algorithms/redundant_packet_count_report.py index a939e684827..aef2ae2dd45 100644 --- a/spynnaker/pyNN/extra_algorithms/redundant_packet_count_report.py +++ b/spynnaker/pyNN/extra_algorithms/redundant_packet_count_report.py @@ -66,7 +66,6 @@ "FROM redundancy_by_core") -_FILE_NAME = "redundant_packet_count.rpt" _N_PROV_ITEMS_NEEDED = 4 _MAX = 100 @@ -76,7 +75,9 @@ def redundant_packet_count_report(): :return: """ - file_name = os.path.join(SpynnakerDataView.get_run_dir_path(), _FILE_NAME) + file_name = os.path.join( + SpynnakerDataView.get_run_dir_path(), + f"redundant_packet_count{SpynnakerDataView.get_reset_str()}.rpt") try: _create_views() diff --git a/spynnaker_integration_tests/test_debug_mode/check_debug.py b/spynnaker_integration_tests/test_debug_mode/check_debug.py index 35ba18f8ef5..8a0456d7a49 100644 --- a/spynnaker_integration_tests/test_debug_mode/check_debug.py +++ b/spynnaker_integration_tests/test_debug_mode/check_debug.py @@ -106,6 +106,7 @@ def debug(self): for report in reports: self.assertIn(report, found) self.assertIn("data.sqlite3", found) + self.assertIn("redundant_packet_count.rpt", found) sim.run(10) pop.get_data("v") @@ -118,6 +119,7 @@ def debug(self): pop.get_data("v") found = os.listdir(SpynnakerDataView.get_run_dir_path()) self.assertIn("data1.sqlite3", found) + self.assertIn("redundant_packet_count1.rpt", found) # No point in checking files they are already there sim.reset() @@ -128,5 +130,6 @@ def debug(self): for report in reports: self.assertIn(report, found) self.assertIn("data2.sqlite3", found) + self.assertIn("redundant_packet_count2.rpt", found) sim.end()