From b70b0c5d46fff896ff251b69fa1b826224006cac Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 27 Oct 2022 09:45:45 +0100 Subject: [PATCH 1/9] test timer report runs --- spynnaker_integration_tests/test_debug_mode/check_debug.py | 4 ++++ .../test_using_virtual_board/test_debug_mode/test_debug.py | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/spynnaker_integration_tests/test_debug_mode/check_debug.py b/spynnaker_integration_tests/test_debug_mode/check_debug.py index 35ba18f8ef5..de12f22e629 100644 --- a/spynnaker_integration_tests/test_debug_mode/check_debug.py +++ b/spynnaker_integration_tests/test_debug_mode/check_debug.py @@ -31,6 +31,8 @@ # import EnergyReport from spinn_front_end_common.utilities.report_functions.board_chip_report \ import AREA_CODE_REPORT_NAME +from spinn_front_end_common.utilities.report_functions.timer_report import ( + TIMER_FILENAME) from spinn_front_end_common.utility_models import \ DataSpeedUpPacketGatherMachineVertex from spinnaker_testbase import BaseTestCase @@ -130,3 +132,5 @@ def debug(self): self.assertIn("data2.sqlite3", found) sim.end() + found = os.listdir(SpynnakerDataView.get_timestamp_dir_path()) + self.assertIn(TIMER_FILENAME, found) diff --git a/unittests/test_using_virtual_board/test_debug_mode/test_debug.py b/unittests/test_using_virtual_board/test_debug_mode/test_debug.py index c22b07d8bc3..d8171328966 100644 --- a/unittests/test_using_virtual_board/test_debug_mode/test_debug.py +++ b/unittests/test_using_virtual_board/test_debug_mode/test_debug.py @@ -19,6 +19,8 @@ reports_names from spinn_front_end_common.utilities.report_functions.network_specification \ import _FILENAME as network_specification_file_name +from spinn_front_end_common.utilities.report_functions.timer_report import ( + TIMER_FILENAME) from spinnaker_testbase import BaseTestCase from spynnaker.pyNN.data import SpynnakerDataView from spynnaker.pyNN.extra_algorithms.\ @@ -89,9 +91,10 @@ def debug(self): sim.end() found = os.listdir(SpynnakerDataView.get_run_dir_path()) - print(found) for report in reports: self.assertIn(report, found) + found = os.listdir(SpynnakerDataView.get_timestamp_dir_path()) + self.assertIn(TIMER_FILENAME, found) def test_debug(self): self.runsafe(self.debug) From a6509ff56e30a4392c9132b4ce1d953b4088b481 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 18 Jan 2023 07:58:14 +0000 Subject: [PATCH 2/9] speedup by converting lists to sets to do intersection --- spynnaker/pyNN/models/common/neuron_recorder.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/spynnaker/pyNN/models/common/neuron_recorder.py b/spynnaker/pyNN/models/common/neuron_recorder.py index 5be99b4ab5f..d8ac178fd74 100644 --- a/spynnaker/pyNN/models/common/neuron_recorder.py +++ b/spynnaker/pyNN/models/common/neuron_recorder.py @@ -231,10 +231,11 @@ def neurons_recording(self, variable, vertex_slice, atoms_shape): return [] if self.__indexes[variable] is None: return vertex_slice.get_raster_ids(atoms_shape) - indexes = self.__indexes[variable] - return [ - i for i in vertex_slice.get_raster_ids(atoms_shape) - if i in indexes] + all_set = set(self.__indexes[variable]) + slice_set = set(vertex_slice.get_raster_ids(atoms_shape)) + local_list = list(all_set.intersection(slice_set)) + local_list.sort() + return local_list def _convert_placement_matrix_data( self, row_data, n_rows, data_row_length, n_neurons, data_type): From 55dd2f1dbca92370dfd1ee46a15c10e7eb8d08cd Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 18 Jan 2023 13:08:47 +0000 Subject: [PATCH 3/9] Pop timer categories --- spynnaker/pyNN/models/populations/population.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/spynnaker/pyNN/models/populations/population.py b/spynnaker/pyNN/models/populations/population.py index a9f4aee82df..c73e66ae943 100644 --- a/spynnaker/pyNN/models/populations/population.py +++ b/spynnaker/pyNN/models/populations/population.py @@ -22,6 +22,8 @@ from spinn_utilities.log import FormatAdapter from spinn_utilities.logger_utils import warn_once from spinn_utilities.overrides import overrides +from spinn_front_end_common.interface.provenance import ( + FecTimer, TimerCategory) from spinn_front_end_common.utilities.exceptions import ConfigurationException from spynnaker.pyNN.data import SpynnakerDataView from spynnaker.pyNN.exceptions import SpynnakerException @@ -178,8 +180,10 @@ def record(self, variables, to_file=None, sampling_interval=None): :param int sampling_interval: a value in milliseconds, and an integer multiple of the simulation timestep. """ + FecTimer.start_category(TimerCategory.POP_RECORD) self.__recorder.record( variables, to_file, sampling_interval, indexes=None) + FecTimer.end_category(TimerCategory.POP_RECORD) def sample(self, n, rng=None): """ Randomly sample `n` cells from the Population, and return a\ @@ -227,6 +231,7 @@ def write_data(self, io, variables='all', gather=True, clear=False, record. """ # pylint: disable=too-many-arguments + FecTimer.start_category(TimerCategory.POP_GET_DATA_DATA) self._check_params(gather, annotations) if isinstance(io, str): @@ -234,8 +239,11 @@ def write_data(self, io, variables='all', gather=True, clear=False, data = self.__recorder.extract_neo_block( variables, None, clear, annotations) + FecTimer.end_category(TimerCategory.POP_GET_DATA_DATA) + FecTimer.start_category(TimerCategory.POP_WRITE_DATA) # write the neo block to the file io.write(data) + FecTimer.end_category(TimerCategory.POP_WRITE_DATA) def describe(self, template='population_default.txt', engine='default'): """ Returns a human-readable description of the population. @@ -308,9 +316,11 @@ def get_data( If the variable or variables have not been previously set to record. """ + FecTimer.start_category(TimerCategory.POP_GET_DATA) self._check_params(gather, annotations) return self.__recorder.extract_neo_block( variables, None, clear, annotations) + FecTimer.end_category(TimerCategory.POP_GET_DATA) def spinnaker_get_data(self, variable, as_matrix=False, view_indexes=None): """ Public accessor for getting data as a numpy array, instead of\ @@ -324,12 +334,14 @@ def spinnaker_get_data(self, variable, as_matrix=False, view_indexes=None): :return: array of the data :rtype: ~numpy.ndarray """ + FecTimer.start_category(TimerCategory.POP_GET_DATA) warn_once( logger, "spinnaker_get_data is non-standard PyNN and therefore " "will not be portable to other simulators.") with NeoBufferDatabase() as db: return db.spinnaker_get_data(self.__recorder.recording_label, variable, as_matrix, view_indexes) + FecTimer.end_category(TimerCategory.POP_GET_DATA) @overrides(PopulationBase.get_spike_counts, extend_doc=False) def get_spike_counts(self, gather=True): @@ -337,9 +349,11 @@ def get_spike_counts(self, gather=True): :rtype: ~numpy.ndarray """ + FecTimer.start_category(TimerCategory.POP_GET_DATA) self._check_params(gather) with NeoBufferDatabase() as db: return db.get_spike_counts(self.__recorder.recording_label) + FecTimer.end_category(TimerCategory.POP_GET_DATA) def find_units(self, variable): """ Get the units of a variable From 6626598b8dcb7a6cdb09d2b806757d08ad71103f Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 18 Jan 2023 13:46:08 +0000 Subject: [PATCH 4/9] Pop timer categories --- spynnaker/pyNN/models/populations/population_view.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/spynnaker/pyNN/models/populations/population_view.py b/spynnaker/pyNN/models/populations/population_view.py index 08043b02487..3790e4685e5 100644 --- a/spynnaker/pyNN/models/populations/population_view.py +++ b/spynnaker/pyNN/models/populations/population_view.py @@ -24,6 +24,8 @@ from .idmixin import IDMixin from .population_base import PopulationBase from spinn_utilities.overrides import overrides +from spinn_front_end_common.interface.provenance import ( + FecTimer, TimerCategory) from spynnaker.pyNN.utilities.neo_buffer_database import NeoBufferDatabase logger = FormatAdapter(logging.getLogger(__name__)) @@ -324,6 +326,7 @@ def get_data( If the variable or variables have not been previously set to record. """ + FecTimer.start_category(TimerCategory.POP_GET_DATA) if not gather: logger.warning("SpiNNaker only supports gather=True. We will run " "as if gather was set to True.") @@ -331,6 +334,7 @@ def get_data( warn_once( logger, "Annotations parameter is not standard PyNN so may " "not be supported by all platforms.") + FecTimer.end_category(TimerCategory.POP_GET_DATA) return self.__recorder.extract_neo_block( variables, self.__indexes, clear, annotations) @@ -480,8 +484,10 @@ def record(self, variables, to_file=None, sampling_interval=None): should be a value in milliseconds, and an integer multiple of the simulation timestep. """ + FecTimer.start_category(TimerCategory.POP_RECORD) self.__recorder.record( variables, to_file, sampling_interval, self.__indexes) + FecTimer.end_category(TimerCategory.POP_RECORD) def sample(self, n, rng=None): """ Randomly sample `n` cells from the Population view, and return a\ @@ -556,17 +562,21 @@ def write_data(self, io, variables='all', gather=True, clear=False, If the variable or variables have not been previously set to record. """ + FecTimer.start_category(TimerCategory.POP_GET_DATA) if not gather: logger.warning("SpiNNaker only supports gather=True. We will run " "as if gather was set to True.") data = self.__recorder.extract_neo_block( variables, self.__indexes, clear, annotations) + FecTimer.end_category(TimerCategory.POP_GET_DATA) + FecTimer.start_category(TimerCategory.POP_WRITE_DATA) if isinstance(io, str): io = neo.get_io(io) # write the neo block to the file io.write(data) + FecTimer.end_category(TimerCategory.POP_WRITE_DATA) @property @overrides(PopulationBase._vertex) From 0e7897fba746d12141ba358421bce623b363ce5f Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 18 Jan 2023 14:10:53 +0000 Subject: [PATCH 5/9] fix category --- spynnaker/pyNN/models/populations/population.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spynnaker/pyNN/models/populations/population.py b/spynnaker/pyNN/models/populations/population.py index c73e66ae943..47218e6870f 100644 --- a/spynnaker/pyNN/models/populations/population.py +++ b/spynnaker/pyNN/models/populations/population.py @@ -231,7 +231,7 @@ def write_data(self, io, variables='all', gather=True, clear=False, record. """ # pylint: disable=too-many-arguments - FecTimer.start_category(TimerCategory.POP_GET_DATA_DATA) + FecTimer.start_category(TimerCategory.POP_GET_DATA) self._check_params(gather, annotations) if isinstance(io, str): @@ -239,7 +239,7 @@ def write_data(self, io, variables='all', gather=True, clear=False, data = self.__recorder.extract_neo_block( variables, None, clear, annotations) - FecTimer.end_category(TimerCategory.POP_GET_DATA_DATA) + FecTimer.end_category(TimerCategory.POP_GET_DATA) FecTimer.start_category(TimerCategory.POP_WRITE_DATA) # write the neo block to the file io.write(data) From 155677bb6d34ba688c6ac4ac240c8ddf4f91e8f6 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Thu, 19 Jan 2023 06:40:33 +0000 Subject: [PATCH 6/9] more use of set --- spynnaker/pyNN/utilities/neo_buffer_database.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spynnaker/pyNN/utilities/neo_buffer_database.py b/spynnaker/pyNN/utilities/neo_buffer_database.py index 073d60b625e..0d67878d844 100644 --- a/spynnaker/pyNN/utilities/neo_buffer_database.py +++ b/spynnaker/pyNN/utilities/neo_buffer_database.py @@ -722,7 +722,8 @@ def __combine_indexes(self, view_indexes, data_indexes, variable): :return: """ # keep just the view indexes in the data - indexes = [i for i in view_indexes if i in data_indexes] + data_set = set(data_indexes) + indexes = [i for i in view_indexes if i in data_set] # check for missing and report view_set = set(view_indexes) missing = view_set.difference(data_indexes) From 11ca33b47f01b8b28b8bceac6bf3cfac500aaf04 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Fri, 3 Feb 2023 10:22:27 +0000 Subject: [PATCH 7/9] make sure end timer is reachable --- spynnaker/pyNN/models/populations/population.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spynnaker/pyNN/models/populations/population.py b/spynnaker/pyNN/models/populations/population.py index 5f614f61134..d42052032b4 100644 --- a/spynnaker/pyNN/models/populations/population.py +++ b/spynnaker/pyNN/models/populations/population.py @@ -328,9 +328,10 @@ def get_data( """ FecTimer.start_category(TimerCategory.POP_GET_DATA) self._check_params(gather, annotations) - return self.__recorder.extract_neo_block( + neo = self.__recorder.extract_neo_block( variables, None, clear, annotations) FecTimer.end_category(TimerCategory.POP_GET_DATA) + return neo def spinnaker_get_data(self, variable, as_matrix=False, view_indexes=None): """ Public accessor for getting data as a numpy array, instead of\ From 7edbf4d135ecff15a2c8eddfe8c16ed128240f94 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 26 Apr 2023 09:53:04 +0100 Subject: [PATCH 8/9] ds.sqlite3 is in the run dir --- .../test_using_virtual_board/test_debug_mode/test_debug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/test_using_virtual_board/test_debug_mode/test_debug.py b/unittests/test_using_virtual_board/test_debug_mode/test_debug.py index 3ab499256cb..a3887cdd008 100644 --- a/unittests/test_using_virtual_board/test_debug_mode/test_debug.py +++ b/unittests/test_using_virtual_board/test_debug_mode/test_debug.py @@ -92,9 +92,9 @@ def debug(self): found = os.listdir(SpynnakerDataView.get_run_dir_path()) for report in reports: self.assertIn(report, found) + self.assertIn("ds.sqlite3", found) found = os.listdir(SpynnakerDataView.get_timestamp_dir_path()) self.assertIn(TIMER_FILENAME, found) - self.assertIn("ds.sqlite3", found) def test_debug(self): self.runsafe(self.debug) From 71c87219fd724e94ad4db788911f43e399cdcdee Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 26 Apr 2023 10:08:02 +0100 Subject: [PATCH 9/9] rename variable hinding an import --- spynnaker/pyNN/models/populations/population.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spynnaker/pyNN/models/populations/population.py b/spynnaker/pyNN/models/populations/population.py index 1a473d7f4be..a78f939b20b 100644 --- a/spynnaker/pyNN/models/populations/population.py +++ b/spynnaker/pyNN/models/populations/population.py @@ -335,10 +335,10 @@ def get_data( """ FecTimer.start_category(TimerCategory.POP_GET_DATA) self._check_params(gather, annotations) - neo = self.__recorder.extract_neo_block( + neo_data = self.__recorder.extract_neo_block( variables, None, clear, annotations) FecTimer.end_category(TimerCategory.POP_GET_DATA) - return neo + return neo_data def spinnaker_get_data(self, variable, as_matrix=False, view_indexes=None): """