From c02dc478d2b8dd02660d5e7900c44fa08b057bb8 Mon Sep 17 00:00:00 2001 From: Petrut Bogdan Date: Mon, 9 Mar 2020 14:07:44 +0000 Subject: [PATCH 1/9] using pandas to save csv of all interesting provenance info --- .../placements_provenance_gatherer.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py b/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py index 74f670af4d..261a752307 100644 --- a/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py +++ b/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py @@ -17,6 +17,7 @@ from spinn_utilities.progress_bar import ProgressBar from spinn_front_end_common.interface.provenance import ( AbstractProvidesProvenanceDataFromMachine) +import pandas as pd logger = logging.getLogger(__name__) @@ -31,6 +32,7 @@ def __call__(self, transceiver, placements): """ prov_items = list() + prov_placement = list() progress = ProgressBar( placements.n_placements, "Getting provenance data") @@ -43,5 +45,44 @@ def __call__(self, transceiver, placements): prov_items.extend( placement.vertex.get_provenance_data_from_machine( transceiver, placement)) + prov_placement.append(placement) + + # write provenance to file here in a useful way + columns = ['pop', 'label', 'min_atom', 'max_atom', 'no_atoms', + 'x', 'y', 'p', + 'prov_name', 'prov_value', + 'fixed_sdram', 'sdram_per_timestep', + 'cpu_cycles', 'dtcm'] + + structured_provenance = list() + for i, (provenance, placement) in enumerate(zip(prov_items, prov_placement)): + prov_name = provenance.names[1] + prov_value = provenance.value + pop = placement.vertex.label.split(":")[0] + x = placement.x + y = placement.y + p = placement.p + fixed_sdram = placement.vertex.resources_required.sdram.fixed + sdram_per_timestep = placement.vertex.resources_required.sdram.per_timestep + cpu_cycles = placement.vertex.resources_required.cpu_cycles.get_value() + dtcm = placement.vertex.resources_required.dtcm.get_value() + + label = placement.vertex.label + slices = label.split(":") + max_atom = int(slices[-1]) + min_atom = int(slices[-2]) + no_atoms = max_atom - min_atom + 1 + + structured_provenance.append( + [pop, label, min_atom, max_atom, no_atoms, + x, y, p, + prov_name, prov_value, + fixed_sdram, sdram_per_timestep, + cpu_cycles, dtcm] + ) + + structured_provenance_df = pd.DataFrame.from_records( + structured_provenance, columns=columns) + structured_provenance_df.to_csv("structured_provenance.csv") return prov_items From 0fd046b36254b997726bfcb1499ebe044fe15464 Mon Sep 17 00:00:00 2001 From: Petrut Bogdan Date: Tue, 10 Mar 2020 18:38:39 +0000 Subject: [PATCH 2/9] fix bug in matching sizes between provenance and placements. added assert --- .../placements_provenance_gatherer.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py b/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py index 261a752307..0078da2b9a 100644 --- a/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py +++ b/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py @@ -42,10 +42,12 @@ def __call__(self, transceiver, placements): if isinstance(placement.vertex, AbstractProvidesProvenanceDataFromMachine): # get data + new_prov = placement.vertex.get_provenance_data_from_machine( + transceiver, placement) prov_items.extend( - placement.vertex.get_provenance_data_from_machine( - transceiver, placement)) - prov_placement.append(placement) + new_prov + ) + prov_placement.extend([placement]*len(new_prov)) # write provenance to file here in a useful way columns = ['pop', 'label', 'min_atom', 'max_atom', 'no_atoms', @@ -53,7 +55,7 @@ def __call__(self, transceiver, placements): 'prov_name', 'prov_value', 'fixed_sdram', 'sdram_per_timestep', 'cpu_cycles', 'dtcm'] - + assert (len(prov_placement) == len(prov_items)) structured_provenance = list() for i, (provenance, placement) in enumerate(zip(prov_items, prov_placement)): prov_name = provenance.names[1] From 812d530c20009391a48953608e251bd647562b8b Mon Sep 17 00:00:00 2001 From: Andrew Gait Date: Mon, 6 Dec 2021 10:49:41 +0000 Subject: [PATCH 3/9] Add LUT provenance to database --- .../interface/provenance/db.sql | 10 ++++++++- .../interface/provenance/provenance_writer.py | 22 +++++++++++++++++++ .../provenance/test_provenance_database.py | 8 +++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/spinn_front_end_common/interface/provenance/db.sql b/spinn_front_end_common/interface/provenance/db.sql index 4312ee71f9..2579bf862a 100644 --- a/spinn_front_end_common/interface/provenance/db.sql +++ b/spinn_front_end_common/interface/provenance/db.sql @@ -202,4 +202,12 @@ CREATE TABLE IF NOT EXISTS connector_provenance( description STRING NOT NULL, the_value INTEGER NOT NULL); - +-- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +-- A table LUT provenance +CREATE TABLE IF NOT EXISTS lut_provenance( + connector_id INTEGER PRIMARY KEY AUTOINCREMENT, + pre_population STRING NOT NULL, + post_population STRING NOT NULL, + the_type STRING NOT NULL, + description STRING NOT NULL, + the_value FLOAT NOT NULL); diff --git a/spinn_front_end_common/interface/provenance/provenance_writer.py b/spinn_front_end_common/interface/provenance/provenance_writer.py index 5773db3868..9a57faf26c 100644 --- a/spinn_front_end_common/interface/provenance/provenance_writer.py +++ b/spinn_front_end_common/interface/provenance/provenance_writer.py @@ -276,3 +276,25 @@ def insert_connector( """, [pre_population, post_population, the_type, description, the_value]) + + def insert_lut( + self, pre_population, post_population, the_type, description, + the_value): + """ + Inserts edge data into the lut_provenance + :param str pre_population: Name of the pre population / vertex + :param str post_population: Name of the post population / vertex + :param str the_type: Class of the rule of the LUT + :param str description: type of value + :param float the_value: data + """ + with self.transaction() as cur: + cur.execute( + """ + INSERT OR IGNORE INTO lut_provenance( + pre_population, post_population, the_type, description, + the_value) + VALUES(?, ?, ?, ?, ?) + """, + [pre_population, post_population, the_type, description, + the_value]) diff --git a/unittests/interface/provenance/test_provenance_database.py b/unittests/interface/provenance/test_provenance_database.py index 2a6f5befbe..95abca81f1 100644 --- a/unittests/interface/provenance/test_provenance_database.py +++ b/unittests/interface/provenance/test_provenance_database.py @@ -158,3 +158,11 @@ def test_connector(self): data = reader.run_query("Select * from connector_provenance") expected = [(1, 'the pre', 'A post', 'OneToOne', 'foo', 12)] self.assertListEqual(expected, data) + + def test_lut(self): + with ProvenanceWriter() as db: + db.insert_lut("the pre", "A post", "OneToOne", "foo", 0.5) + reader = ProvenanceReader() + data = reader.run_query("Select * from lut_provenance") + expected = [(1, 'the pre', 'A post', 'OneToOne', 'foo', 0.5)] + self.assertListEqual(expected, data) From 56936729c2324d9b84021348923007849b5c8095 Mon Sep 17 00:00:00 2001 From: Andrew Gait Date: Tue, 7 Dec 2021 09:59:24 +0000 Subject: [PATCH 4/9] Fix structured provenance to use vertex slice attributes, not label --- .../placements_provenance_gatherer.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py b/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py index 7f4c1015ed..6b146049c6 100644 --- a/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py +++ b/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py @@ -88,15 +88,10 @@ def _add_structured_provenance(placement, errors): dtcm = placement.vertex.resources_required.dtcm.get_value() label = placement.vertex.label - slices = label.split(":") - if len(slices) == 2: # a non-population vertex e.g. extra monitor - max_atom = 0 - min_atom = 0 - no_atoms = 0 - else: - max_atom = int(slices[-1]) - min_atom = int(slices[-2]) - no_atoms = max_atom - min_atom + 1 + slice = placement.vertex.vertex_slice + max_atom = slice.hi_atom + min_atom = slice.lo_atom + no_atoms = slice.n_atoms structured_provenance = [ pop, label, min_atom, max_atom, no_atoms, From 684955580f7da7f25f94e6b8eb37a1daf6b7d67d Mon Sep 17 00:00:00 2001 From: Andrew Gait Date: Tue, 19 Jul 2022 14:07:58 +0100 Subject: [PATCH 5/9] Merge weirdness --- ...r_compressor.py => test_host_bit_field_router_compressor.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename unittests/interface/interface_functions/{test_host_bIt_field_router_compressor.py => test_host_bit_field_router_compressor.py} (96%) diff --git a/unittests/interface/interface_functions/test_host_bIt_field_router_compressor.py b/unittests/interface/interface_functions/test_host_bit_field_router_compressor.py similarity index 96% rename from unittests/interface/interface_functions/test_host_bIt_field_router_compressor.py rename to unittests/interface/interface_functions/test_host_bit_field_router_compressor.py index a0395ae671..41779622a9 100644 --- a/unittests/interface/interface_functions/test_host_bIt_field_router_compressor.py +++ b/unittests/interface/interface_functions/test_host_bit_field_router_compressor.py @@ -26,7 +26,7 @@ def setUp(self): def test_bit_field_as_bit_array(self): data = _BitFieldData( - 1, [1], None, None, None) + 1, [1], None, None, None, None, None) as_array = data.bit_field_as_bit_array() self.assertListEqual( [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, From d5104e7cace9d6a870a7ade46fd03f9c724a3adf Mon Sep 17 00:00:00 2001 From: Andrew Gait Date: Mon, 22 Aug 2022 12:06:11 +0100 Subject: [PATCH 6/9] update structured provenance --- .../placements_provenance_gatherer.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py b/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py index ca02245ddc..49f731f167 100644 --- a/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py +++ b/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py @@ -75,15 +75,11 @@ def _add_structured_provenance(placement, errors): # Custom provenance presentation from SpiNNCer # write provenance to file here in a useful way columns = ['pop', 'label', 'min_atom', 'max_atom', 'no_atoms', - 'fixed_sdram', 'sdram_per_timestep', 'cpu_cycles', 'dtcm'] + 'fixed_sdram', 'sdram_per_timestep'] pop = placement.vertex.label.split(":")[0] - fixed_sdram = placement.vertex.resources_required.sdram.fixed - sdram_per_timestep = placement.vertex.resources_required.\ - sdram.per_timestep - cpu_cycles = placement.vertex.resources_required.cpu_cycles.\ - get_value() - dtcm = placement.vertex.resources_required.dtcm.get_value() + fixed_sdram = placement.vertex.sdram_required.fixed + sdram_per_timestep = placement.vertex.sdram_required.per_timestep label = placement.vertex.label slice = placement.vertex.vertex_slice @@ -93,7 +89,7 @@ def _add_structured_provenance(placement, errors): structured_provenance = [ pop, label, min_atom, max_atom, no_atoms, - fixed_sdram, sdram_per_timestep, cpu_cycles, dtcm] + fixed_sdram, sdram_per_timestep] # get data try: From ef8103bf7b9d50e61887fa0c2d495e3ffcd6dfba Mon Sep 17 00:00:00 2001 From: Andrew Gait Date: Mon, 5 Dec 2022 13:40:48 +0000 Subject: [PATCH 7/9] Save ITCM --- c_common/front_end_common_lib/src/recording.c | 4 ++-- c_common/front_end_common_lib/src/simulation.c | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/c_common/front_end_common_lib/src/recording.c b/c_common/front_end_common_lib/src/recording.c index 1bf84d35a5..e99173fa21 100644 --- a/c_common/front_end_common_lib/src/recording.c +++ b/c_common/front_end_common_lib/src/recording.c @@ -196,8 +196,8 @@ bool recording_initialize( sark_heap_max(sv->sdram_heap, 0)); return false; } - log_info("Allocated %u bytes for recording channel %u at 0x%08x", - space, i, region->data); +// log_info("Allocated %u bytes for recording channel %u at 0x%08x", +// space, i, region->data); if (recording_flags != NULL) { *recording_flags = (*recording_flags | (1 << i)); } diff --git a/c_common/front_end_common_lib/src/simulation.c b/c_common/front_end_common_lib/src/simulation.c index a20c137159..02801e13f7 100644 --- a/c_common/front_end_common_lib/src/simulation.c +++ b/c_common/front_end_common_lib/src/simulation.c @@ -198,15 +198,15 @@ static void simulation_control_scp_callback(uint mailbox, UNUSED uint port) { log_info("Calling pre-exit function"); stored_exit_function(); } - log_info("Exiting"); +// log_info("Exiting"); spin1_exit(0); break; case CMD_RUNTIME: - log_info("Setting the runtime of this model to %d starting at %d", - msg->arg1, msg->arg3); - log_info("Setting the flag of infinite run for this model to %d", - msg->arg2); + log_info("Setting the runtime of this model to %d starting at %d (flag %d)", + msg->arg1, msg->arg3, msg->arg2); +// log_info("Setting the flag of infinite run for this model to %d", +// msg->arg2); // resetting the simulation time pointer *pointer_to_simulation_time = msg->arg1; From 1c41975154a1d8e6623cc0857e15dbc6b37483c8 Mon Sep 17 00:00:00 2001 From: Andrew Gait Date: Tue, 17 Jan 2023 11:49:32 +0000 Subject: [PATCH 8/9] pylint rename variable --- .../interface_functions/placements_provenance_gatherer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py b/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py index 49f731f167..93874b362c 100644 --- a/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py +++ b/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py @@ -82,10 +82,10 @@ def _add_structured_provenance(placement, errors): sdram_per_timestep = placement.vertex.sdram_required.per_timestep label = placement.vertex.label - slice = placement.vertex.vertex_slice - max_atom = slice.hi_atom - min_atom = slice.lo_atom - no_atoms = slice.n_atoms + vertex_slice = placement.vertex.vertex_slice + max_atom = vertex_slice.hi_atom + min_atom = vertex_slice.lo_atom + no_atoms = vertex_slice.n_atoms structured_provenance = [ pop, label, min_atom, max_atom, no_atoms, From f7b25324ecba977b878d27079f9ab5327d606873 Mon Sep 17 00:00:00 2001 From: Andrew Gait Date: Thu, 23 Feb 2023 10:51:29 +0000 Subject: [PATCH 9/9] Missing import --- .../interface_functions/placements_provenance_gatherer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py b/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py index 7bea111284..fba6ab9614 100644 --- a/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py +++ b/spinn_front_end_common/interface/interface_functions/placements_provenance_gatherer.py @@ -17,7 +17,7 @@ from spinn_utilities.log import FormatAdapter from spinn_utilities.progress_bar import ProgressBar from spinn_front_end_common.interface.provenance import ( - AbstractProvidesProvenanceDataFromMachine) + AbstractProvidesProvenanceDataFromMachine, ProvenanceWriter) logger = FormatAdapter(logging.getLogger(__name__))