From d5bad2952c2c0da0b8f6f7a6369ccd5e3ef29dcf Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 18 Oct 2022 11:56:36 +0100 Subject: [PATCH 01/12] SqlLiteDatabase handles file path --- .../buffered_receiving_data.py | 19 +++++-------------- .../storage_objects/sqllite_database.py | 14 ++++++++++++-- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/spinn_front_end_common/interface/buffer_management/storage_objects/buffered_receiving_data.py b/spinn_front_end_common/interface/buffer_management/storage_objects/buffered_receiving_data.py index 18667499fa..a165c39e0d 100644 --- a/spinn_front_end_common/interface/buffer_management/storage_objects/buffered_receiving_data.py +++ b/spinn_front_end_common/interface/buffer_management/storage_objects/buffered_receiving_data.py @@ -14,10 +14,6 @@ # along with this program. If not, see . import os from .sqllite_database import SqlLiteDatabase -from spinn_front_end_common.data import FecDataView - -#: Name of the database in the data folder -DB_FILE_NAME = "buffer.sqlite3" class BufferedReceivingData(object): @@ -29,9 +25,6 @@ class BufferedReceivingData(object): #: the AbstractDatabase holding the data to store "_db", - #: the path to the database - "_db_file", - #: the (size, address) of each region "__sizes_and_addresses", @@ -40,8 +33,6 @@ class BufferedReceivingData(object): ] def __init__(self): - self._db_file = os.path.join( - FecDataView.get_run_dir_path(), DB_FILE_NAME) self._db = None self.__sizes_and_addresses = None self.__data_flushed = None @@ -50,11 +41,11 @@ def __init__(self): def reset(self): """ Perform tasks to restart recording from time=0 """ - if os.path.exists(self._db_file): - if self._db: - self._db.close() - os.remove(self._db_file) - self._db = SqlLiteDatabase(self._db_file) + if self._db: + self._db.close() + if os.path.exists(self._db.default_database_file()): + os.remove(self._db.default_database_file()) + self._db = SqlLiteDatabase() self.__sizes_and_addresses = dict() self.__data_flushed = set() diff --git a/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py b/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py index 2f9448e828..0d15ef6bb6 100644 --- a/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py +++ b/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py @@ -18,12 +18,15 @@ import time from spinn_utilities.abstract_context_manager import AbstractContextManager from spinn_utilities.overrides import overrides +from spinn_front_end_common.data import FecDataView from spinn_front_end_common.utilities.sqlite_db import SQLiteDB from .abstract_database import AbstractDatabase _DDL_FILE = os.path.join(os.path.dirname(__file__), "db.sql") _SECONDS_TO_MICRO_SECONDS_CONVERSION = 1000 +#: Name of the database in the data folder +DB_FILE_NAME = "buffer.sqlite3" def _timestamp(): return int(time.time() * _SECONDS_TO_MICRO_SECONDS_CONVERSION) @@ -43,11 +46,18 @@ def __init__(self, database_file=None): """ :param str database_file: The name of a file that contains (or will contain) an SQLite - database holding the data. If omitted, an unshared in-memory - database will be used. + database holding the data. + If omitted the default location will be used """ + if database_file is None: + database_file = self.default_database_file + super().__init__(database_file, ddl_file=_DDL_FILE) + def default_database_file(self): + return os.path.join( + FecDataView.get_run_dir_path(), DB_FILE_NAME) + @overrides(AbstractDatabase.clear) def clear(self): with self.transaction() as cursor: From 6535fd65268d07d1dc2eea38c4e34e1a19b6dfc6 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 18 Oct 2022 12:17:07 +0100 Subject: [PATCH 02/12] store vertex labels in buffer sql --- .../interface/abstract_spinnaker_base.py | 31 ++++++++++++++++++- .../buffer_management/storage_objects/db.sql | 3 +- .../storage_objects/sqllite_database.py | 12 ++++++- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/spinn_front_end_common/interface/abstract_spinnaker_base.py b/spinn_front_end_common/interface/abstract_spinnaker_base.py index b23d8d3160..5e24d3f616 100644 --- a/spinn_front_end_common/interface/abstract_spinnaker_base.py +++ b/spinn_front_end_common/interface/abstract_spinnaker_base.py @@ -67,6 +67,8 @@ from spinn_front_end_common.abstract_models import ( AbstractVertexWithEdgeToDependentVertices, AbstractCanReset) +from spinn_front_end_common.interface.buffer_management.storage_objects \ + import (SqlLiteDatabase) from spinn_front_end_common.interface.config_handler import ConfigHandler from spinn_front_end_common.interface.interface_functions import ( application_finisher, application_runner, @@ -2270,10 +2272,32 @@ def _print_iobuf(errors, warnings): for error in errors: logger.error(error) + def _execute_prepare_chip_power(self): + with FecTimer("Prepare Chip Power", TimerWork.REPORT) as timer: + if timer.skip_if_cfg_false("Reports", "write_energy_report"): + return + if timer.skip_if_virtual_board(): + return + db = SqlLiteDatabase() + db.store_placements() + + def _report_chip_active(self): + with FecTimer("Prepare Chip Power", TimerWork.REPORT) as timer: + if timer.skip_if_cfg_false("Reports", "write_energy_report"): + return + if timer.skip_if_virtual_board(): + return + write_chip_active_report() + + def _do_end_of_run(self): + if not self._data_writer.is_ran_last(): + return + self._execute_prepare_chip_power() + #self._report_chip_active() + def reset(self): """ Code that puts the simulation back at time zero """ - FecTimer.start_category(TimerCategory.RESETTING) if not self._data_writer.is_ran_last(): if not self._data_writer.is_ran_ever(): logger.error("Ignoring the reset before the run") @@ -2281,8 +2305,11 @@ def reset(self): logger.error("Ignoring the repeated reset call") return + FecTimer.start_category(TimerCategory.RESETTING) logger.info("Resetting") + self._do_end_of_run() + # rewind the buffers from the buffer manager, to start at the beginning # of the simulation again and clear buffered out if self._data_writer.has_buffer_manager(): @@ -2367,6 +2394,8 @@ def stop(self): set_config("Reports", "read_provenance_data", "True") self._do_read_provenance() + self._do_end_of_run() + except Exception as e: self._recover_from_error(e) self.write_errored_file() diff --git a/spinn_front_end_common/interface/buffer_management/storage_objects/db.sql b/spinn_front_end_common/interface/buffer_management/storage_objects/db.sql index b75bd93cb6..2789f5be6a 100644 --- a/spinn_front_end_common/interface/buffer_management/storage_objects/db.sql +++ b/spinn_front_end_common/interface/buffer_management/storage_objects/db.sql @@ -22,7 +22,8 @@ CREATE TABLE IF NOT EXISTS core( core_id INTEGER PRIMARY KEY AUTOINCREMENT, x INTEGER NOT NULL, y INTEGER NOT NULL, - processor INTEGER NOT NULL); + processor INTEGER NOT NULL, + label STRING); -- Every processor has a unique ID CREATE UNIQUE INDEX IF NOT EXISTS coreSanity ON core( x ASC, y ASC, processor ASC); diff --git a/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py b/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py index 0d15ef6bb6..7342b56cdb 100644 --- a/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py +++ b/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py @@ -50,7 +50,7 @@ def __init__(self, database_file=None): If omitted the default location will be used """ if database_file is None: - database_file = self.default_database_file + database_file = self.default_database_file() super().__init__(database_file, ddl_file=_DDL_FILE) @@ -246,3 +246,13 @@ def get_region_data(self, x, y, p, region): return data, False except LookupError: return memoryview(b''), True + + def store_placements(self): + with self.transaction() as cursor: + for placement in FecDataView.iterate_placemements(): + core_id = self.__get_core_id( + cursor, placement.x, placement.y, placement.p) + cursor.execute( + "UPDATE core SET label = ? WHERE core_id = ?", + (placement.vertex.label, core_id)) + assert cursor.rowcount == 1 From 36c01ab8739467438652e6d0f531cd1e4fab7acd Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 18 Oct 2022 14:56:02 +0100 Subject: [PATCH 03/12] save extra chip power monitor stuff to the database --- .../interface/abstract_spinnaker_base.py | 9 ++- .../buffer_management/storage_objects/db.sql | 3 +- .../storage_objects/sqllite_database.py | 60 ++++++++++++++++++- .../utilities/report_functions/__init__.py | 2 + 4 files changed, 68 insertions(+), 6 deletions(-) diff --git a/spinn_front_end_common/interface/abstract_spinnaker_base.py b/spinn_front_end_common/interface/abstract_spinnaker_base.py index 5e24d3f616..f6ea838dd6 100644 --- a/spinn_front_end_common/interface/abstract_spinnaker_base.py +++ b/spinn_front_end_common/interface/abstract_spinnaker_base.py @@ -110,7 +110,7 @@ memory_map_on_host_chip_report, network_specification, router_collision_potential_report, routing_table_from_machine_report, tags_from_machine_report, - write_json_machine, write_json_placements, + write_chip_active_report, write_json_machine, write_json_placements, write_json_routing_tables, drift_report) from spinn_front_end_common.utilities.iobuf_extractor import IOBufExtractor from spinn_front_end_common.utilities.utility_objs import ExecutableType @@ -2280,6 +2280,9 @@ def _execute_prepare_chip_power(self): return db = SqlLiteDatabase() db.store_placements() + db.store_chip_power_monitors() + #data = list(db.iterate_chip_power_monitor_cores()) + db.close() def _report_chip_active(self): with FecTimer("Prepare Chip Power", TimerWork.REPORT) as timer: @@ -2287,13 +2290,13 @@ def _report_chip_active(self): return if timer.skip_if_virtual_board(): return - write_chip_active_report() + #write_chip_active_report() def _do_end_of_run(self): if not self._data_writer.is_ran_last(): return self._execute_prepare_chip_power() - #self._report_chip_active() + self._report_chip_active() def reset(self): """ Code that puts the simulation back at time zero diff --git a/spinn_front_end_common/interface/buffer_management/storage_objects/db.sql b/spinn_front_end_common/interface/buffer_management/storage_objects/db.sql index 2789f5be6a..b75bd93cb6 100644 --- a/spinn_front_end_common/interface/buffer_management/storage_objects/db.sql +++ b/spinn_front_end_common/interface/buffer_management/storage_objects/db.sql @@ -22,8 +22,7 @@ CREATE TABLE IF NOT EXISTS core( core_id INTEGER PRIMARY KEY AUTOINCREMENT, x INTEGER NOT NULL, y INTEGER NOT NULL, - processor INTEGER NOT NULL, - label STRING); + processor INTEGER NOT NULL); -- Every processor has a unique ID CREATE UNIQUE INDEX IF NOT EXISTS coreSanity ON core( x ASC, y ASC, processor ASC); diff --git a/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py b/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py index 7342b56cdb..a0672b319a 100644 --- a/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py +++ b/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py @@ -220,7 +220,7 @@ def store_data_in_region_buffer(self, x, y, p, region, missing, data): region_id, content, content_len) VALUES (?, CAST(? AS BLOB), ?) """, (region_id, datablob, len(data))) - assert cursor.rowcount == 1 + assert cursor.rowcount == 1 def __use_main_table(self, cursor, region_id): """ @@ -248,7 +248,18 @@ def get_region_data(self, x, y, p, region): return memoryview(b''), True def store_placements(self): + exists = False with self.transaction() as cursor: + for row in cursor.execute("PRAGMA TABLE_INFO(core)"): + if row["name"] == "label": + exists = True + + if exists: + return + # already done so no need to repeat + + cursor.execute("ALTER TABLE core ADD COLUMN label STRING") + for placement in FecDataView.iterate_placemements(): core_id = self.__get_core_id( cursor, placement.x, placement.y, placement.p) @@ -256,3 +267,50 @@ def store_placements(self): "UPDATE core SET label = ? WHERE core_id = ?", (placement.vertex.label, core_id)) assert cursor.rowcount == 1 + + def store_chip_power_monitors(self): + # delayed import due to circular refrences + from spinn_front_end_common.utility_models.\ + chip_power_monitor_machine_vertex import ( + ChipPowerMonitorMachineVertex) + + with self.transaction() as cursor: + for row in cursor.execute( + """ + SELECT name FROM sqlite_master + WHERE type='table' AND name='chip_power_monitor' + """): + # Already exists so no need to run again + return + + cursor.execute( + """ + CREATE TABLE chip_power_monitors( + cpm_id INTEGER PRIMARY KEY autoincrement, + core_id INTEGER NOT NULL + REFERENCES core(core_id) ON DELETE RESTRICT, + sampling_frequency FLOAT NOT NULL) + """) + + for placement in FecDataView.iterate_placements_by_vertex_type( + ChipPowerMonitorMachineVertex): + core_id = self.__get_core_id( + cursor, placement.x, placement.y, placement.p) + cursor.execute( + """ + INSERT INTO chip_power_monitors( + core_id, sampling_frequency) + VALUES (?, ?) + """, (core_id, placement.vertex.sampling_frequency)) + assert cursor.rowcount == 1 + + def iterate_chip_power_monitor_cores(self): + with self.transaction() as cursor: + for row in cursor.execute( + """ + SELECT core_id, sampling_frequency + FROM chip_power_monitors + ORDER BY core_id + """): + yield row + diff --git a/spinn_front_end_common/utilities/report_functions/__init__.py b/spinn_front_end_common/utilities/report_functions/__init__.py index 2a28071ecc..c66d0aa9e2 100644 --- a/spinn_front_end_common/utilities/report_functions/__init__.py +++ b/spinn_front_end_common/utilities/report_functions/__init__.py @@ -16,6 +16,7 @@ from .bit_field_compressor_report import bitfield_compressor_report from .bit_field_summary import BitFieldSummary from .board_chip_report import board_chip_report +from .chip_active_report import write_chip_active_report from .energy_report import EnergyReport from .fixed_route_from_machine_report import fixed_route_from_machine_report from .memory_map_on_host_chip_report import memory_map_on_host_chip_report @@ -44,6 +45,7 @@ "router_collision_potential_report", "routing_table_from_machine_report", "tags_from_machine_report", + "write_chip_active_report", "write_json_machine", "write_json_placements", "write_json_routing_tables", From d7957edaa65cecc5763a0134b885fc3a8596d5f9 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 18 Oct 2022 16:33:23 +0100 Subject: [PATCH 04/12] chip_active_report --- .../interface/abstract_spinnaker_base.py | 3 +- .../storage_objects/sqllite_database.py | 68 ++++++++++++++++++- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/spinn_front_end_common/interface/abstract_spinnaker_base.py b/spinn_front_end_common/interface/abstract_spinnaker_base.py index f6ea838dd6..b7deb832b0 100644 --- a/spinn_front_end_common/interface/abstract_spinnaker_base.py +++ b/spinn_front_end_common/interface/abstract_spinnaker_base.py @@ -2281,7 +2281,6 @@ def _execute_prepare_chip_power(self): db = SqlLiteDatabase() db.store_placements() db.store_chip_power_monitors() - #data = list(db.iterate_chip_power_monitor_cores()) db.close() def _report_chip_active(self): @@ -2290,7 +2289,7 @@ def _report_chip_active(self): return if timer.skip_if_virtual_board(): return - #write_chip_active_report() + write_chip_active_report() def _do_end_of_run(self): if not self._data_writer.is_ran_last(): diff --git a/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py b/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py index a0672b319a..55b1ea5729 100644 --- a/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py +++ b/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py @@ -141,6 +141,52 @@ def __read_contents(self, cursor, x, y, p, region): data = c_buffer return memoryview(data) + def __read_contents(self, cursor, x, y, p, region): + """ + :param ~sqlite3.Cursor cursor: + :param int x: + :param int y: + :param int p: + :param int region: + :rtype: memoryview + """ + for row in cursor.execute( + """ + SELECT region_id, content, have_extra + FROM region_view + WHERE x = ? AND y = ? AND processor = ? + AND local_region_index = ? LIMIT 1 + """, (x, y, p, region)): + r_id, data, extra = ( + row["region_id"], row["content"], row["have_extra"]) + break + else: + raise LookupError("no record for region ({},{},{}:{})".format( + x, y, p, region)) + if extra: + c_buffer = None + for row in cursor.execute( + """ + SELECT r.content_len + ( + SELECT SUM(x.content_len) + FROM region_extra AS x + WHERE x.region_id = r.region_id) AS len + FROM region AS r WHERE region_id = ? LIMIT 1 + """, (r_id, )): + c_buffer = bytearray(row["len"]) + c_buffer[:len(data)] = data + idx = len(data) + for row in cursor.execute( + """ + SELECT content FROM region_extra + WHERE region_id = ? ORDER BY extra_id ASC + """, (r_id, )): + item = row["content"] + c_buffer[idx:idx + len(item)] = item + idx += len(item) + data = c_buffer + return memoryview(data) + @staticmethod def __get_core_id(cursor, x, y, p): """ @@ -268,6 +314,17 @@ def store_placements(self): (placement.vertex.label, core_id)) assert cursor.rowcount == 1 + def get_label(self, x, y, p): + with self.transaction() as cursor: + for row in cursor.execute( + """ + SELECT label + FROM core + WHERE x = ? AND y = ? and processor = ? + """, (x, y, p)): + return str(row["label"], 'utf8') + return "" + def store_chip_power_monitors(self): # delayed import due to circular refrences from spinn_front_end_common.utility_models.\ @@ -292,6 +349,13 @@ def store_chip_power_monitors(self): sampling_frequency FLOAT NOT NULL) """) + cursor.execute( + """ + CREATE VIEW chip_power_monitors_view AS + SELECT core_id, x, y, processor, sampling_frequency + FROM core NATURAL JOIN chip_power_monitors + """) + for placement in FecDataView.iterate_placements_by_vertex_type( ChipPowerMonitorMachineVertex): core_id = self.__get_core_id( @@ -308,8 +372,8 @@ def iterate_chip_power_monitor_cores(self): with self.transaction() as cursor: for row in cursor.execute( """ - SELECT core_id, sampling_frequency - FROM chip_power_monitors + SELECT x, y, processor, sampling_frequency + FROM chip_power_monitors_view ORDER BY core_id """): yield row From d0556bd9b80f9cda1140890bd0b264c33057a1c0 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 19 Oct 2022 06:28:14 +0100 Subject: [PATCH 05/12] fix import --- .../buffer_management/test_buffered_receiver_with_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/interface/buffer_management/test_buffered_receiver_with_db.py b/unittests/interface/buffer_management/test_buffered_receiver_with_db.py index 50d30f0c60..d2ebe0d463 100644 --- a/unittests/interface/buffer_management/test_buffered_receiver_with_db.py +++ b/unittests/interface/buffer_management/test_buffered_receiver_with_db.py @@ -19,7 +19,7 @@ from spinn_front_end_common.interface.buffer_management.storage_objects \ import BufferedReceivingData from spinn_front_end_common.interface.buffer_management.storage_objects\ - .buffered_receiving_data import DB_FILE_NAME + .sqllite_database import DB_FILE_NAME from spinn_front_end_common.interface.config_setup import unittest_setup From f701a29654a0e6cb6380685851597048c0ce834d Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 19 Oct 2022 06:36:41 +0100 Subject: [PATCH 06/12] write_chip_active_report --- .../report_functions/chip_active_report.py | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 spinn_front_end_common/utilities/report_functions/chip_active_report.py diff --git a/spinn_front_end_common/utilities/report_functions/chip_active_report.py b/spinn_front_end_common/utilities/report_functions/chip_active_report.py new file mode 100644 index 0000000000..3fd05249db --- /dev/null +++ b/spinn_front_end_common/utilities/report_functions/chip_active_report.py @@ -0,0 +1,92 @@ +# Copyright (c) 2017-2019 The University of Manchester +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from collections import defaultdict +import logging +import numpy +import os +from spinn_utilities.config_holder import (get_config_int, get_config_str) +from spinn_utilities.exceptions import SpiNNUtilsException +from spinn_utilities.log import FormatAdapter +from spinn_front_end_common.data import FecDataView +from spinn_front_end_common.interface.buffer_management.storage_objects \ + import (SqlLiteDatabase) +from spinn_front_end_common.interface.provenance import ( + FecTimer, ProvenanceReader, TimerCategory) +from spinn_front_end_common.utility_models import ChipPowerMonitorMachineVertex +from spinn_front_end_common.utilities.exceptions import ConfigurationException +from spinn_front_end_common.interface.interface_functions.compute_energy_used\ + import (JOULES_PER_SPIKE, MILLIWATTS_PER_CHIP_ACTIVE_OVERHEAD, + MILLIWATTS_PER_FRAME_ACTIVE_COST, MILLIWATTS_PER_FPGA, + MILLIWATTS_PER_IDLE_CHIP) +from spinn_machine.machine import Machine + +logger = FormatAdapter(logging.getLogger(__name__)) + +#: converter between joules to kilowatt hours +JOULES_TO_KILOWATT_HOURS = 3600000 + +# energy report file name +CHIP_ACTIVE_FILENAME = "chip_active_report.rpt" + +def write_chip_active_report(report_path=None, buffer_path=None): + """ Writes the report. + + :param report_path: Where to write the report if not using the default + :type report_path: None or str + :param buffer_path: Where the provenance sqlite3 files is located + if not using the default. + :type buffer_path: None or str + :rtype: None + """ + if report_path is None: + try: + report_dir = FecDataView.get_run_dir_path() + report_path = os.path.join( + report_dir, CHIP_ACTIVE_FILENAME) + except SpiNNUtilsException: + report_path = os.path.join( + os.path.curdir, CHIP_ACTIVE_FILENAME) + logger.warning(f"no report_path so writing to {report_path}") + + # create detailed report + with open(report_path, "w", encoding="utf-8") as f: + __write_report(f, buffer_path) + +def __write_report(f, buffer_path): + db = SqlLiteDatabase(buffer_path) + n_samples_per_recording = get_config_int( + "EnergyMonitor", "n_samples_per_recording_entry") + + for row in db.iterate_chip_power_monitor_cores(): + record_raw, data_missing = db.get_region_data( + row["x"], row["y"], row["processor"], 0) + results = ( + numpy.frombuffer(record_raw, dtype="uint32").reshape(-1, 18) / + n_samples_per_recording) + active_sums = numpy.sum(results, axis=0) + activity_count = numpy.sum(results) + time_for_recorded_sample =\ + (row["sampling_frequency"] * n_samples_per_recording) / 1000 + + for core in range(0, 18): + label = db.get_label(row["x"], row["y"], core) + if (active_sums[core] > 0) or label: + energy = (active_sums[core] * time_for_recorded_sample * + MILLIWATTS_PER_CHIP_ACTIVE_OVERHEAD / 18) + f.write( + f"processor {row['x']}:{row['y']}:{core}({label})" + f" was active for {active_sums[core]} " + f" using {energy} Joules\n") From 0178b7967ad1301f7f19e7127cb0ec0217b9aa22 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 19 Oct 2022 06:45:24 +0100 Subject: [PATCH 07/12] flake8 --- .../storage_objects/sqllite_database.py | 52 ++----------------- .../report_functions/chip_active_report.py | 9 +--- 2 files changed, 4 insertions(+), 57 deletions(-) diff --git a/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py b/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py index 55b1ea5729..951d5d56b5 100644 --- a/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py +++ b/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py @@ -28,6 +28,7 @@ #: Name of the database in the data folder DB_FILE_NAME = "buffer.sqlite3" + def _timestamp(): return int(time.time() * _SECONDS_TO_MICRO_SECONDS_CONVERSION) @@ -141,52 +142,6 @@ def __read_contents(self, cursor, x, y, p, region): data = c_buffer return memoryview(data) - def __read_contents(self, cursor, x, y, p, region): - """ - :param ~sqlite3.Cursor cursor: - :param int x: - :param int y: - :param int p: - :param int region: - :rtype: memoryview - """ - for row in cursor.execute( - """ - SELECT region_id, content, have_extra - FROM region_view - WHERE x = ? AND y = ? AND processor = ? - AND local_region_index = ? LIMIT 1 - """, (x, y, p, region)): - r_id, data, extra = ( - row["region_id"], row["content"], row["have_extra"]) - break - else: - raise LookupError("no record for region ({},{},{}:{})".format( - x, y, p, region)) - if extra: - c_buffer = None - for row in cursor.execute( - """ - SELECT r.content_len + ( - SELECT SUM(x.content_len) - FROM region_extra AS x - WHERE x.region_id = r.region_id) AS len - FROM region AS r WHERE region_id = ? LIMIT 1 - """, (r_id, )): - c_buffer = bytearray(row["len"]) - c_buffer[:len(data)] = data - idx = len(data) - for row in cursor.execute( - """ - SELECT content FROM region_extra - WHERE region_id = ? ORDER BY extra_id ASC - """, (r_id, )): - item = row["content"] - c_buffer[idx:idx + len(item)] = item - idx += len(item) - data = c_buffer - return memoryview(data) - @staticmethod def __get_core_id(cursor, x, y, p): """ @@ -329,7 +284,7 @@ def store_chip_power_monitors(self): # delayed import due to circular refrences from spinn_front_end_common.utility_models.\ chip_power_monitor_machine_vertex import ( - ChipPowerMonitorMachineVertex) + ChipPowerMonitorMachineVertex) with self.transaction() as cursor: for row in cursor.execute( @@ -352,7 +307,7 @@ def store_chip_power_monitors(self): cursor.execute( """ CREATE VIEW chip_power_monitors_view AS - SELECT core_id, x, y, processor, sampling_frequency + SELECT core_id, x, y, processor, sampling_frequency FROM core NATURAL JOIN chip_power_monitors """) @@ -377,4 +332,3 @@ def iterate_chip_power_monitor_cores(self): ORDER BY core_id """): yield row - diff --git a/spinn_front_end_common/utilities/report_functions/chip_active_report.py b/spinn_front_end_common/utilities/report_functions/chip_active_report.py index 3fd05249db..61cc29c61f 100644 --- a/spinn_front_end_common/utilities/report_functions/chip_active_report.py +++ b/spinn_front_end_common/utilities/report_functions/chip_active_report.py @@ -23,15 +23,8 @@ from spinn_front_end_common.data import FecDataView from spinn_front_end_common.interface.buffer_management.storage_objects \ import (SqlLiteDatabase) -from spinn_front_end_common.interface.provenance import ( - FecTimer, ProvenanceReader, TimerCategory) -from spinn_front_end_common.utility_models import ChipPowerMonitorMachineVertex -from spinn_front_end_common.utilities.exceptions import ConfigurationException from spinn_front_end_common.interface.interface_functions.compute_energy_used\ - import (JOULES_PER_SPIKE, MILLIWATTS_PER_CHIP_ACTIVE_OVERHEAD, - MILLIWATTS_PER_FRAME_ACTIVE_COST, MILLIWATTS_PER_FPGA, - MILLIWATTS_PER_IDLE_CHIP) -from spinn_machine.machine import Machine + import (MILLIWATTS_PER_CHIP_ACTIVE_OVERHEAD) logger = FormatAdapter(logging.getLogger(__name__)) From 66bcaf8865dd67836c07d065b486ec8f355303ea Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 19 Oct 2022 07:40:17 +0100 Subject: [PATCH 08/12] totals and track monitors --- .../storage_objects/sqllite_database.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py b/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py index 951d5d56b5..eb0d87b944 100644 --- a/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py +++ b/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py @@ -269,6 +269,18 @@ def store_placements(self): (placement.vertex.label, core_id)) assert cursor.rowcount == 1 + for chip in FecDataView.get_machine().chips: + for processor in chip.processors: + if processor.is_monitor: + core_id = self.__get_core_id( + cursor, chip.x, chip.y, processor.processor_id) + cursor.execute( + """ + UPDATE core SET label = 'MONITOR' + WHERE core_id = ? + """, [core_id]) + assert cursor.rowcount == 1 + def get_label(self, x, y, p): with self.transaction() as cursor: for row in cursor.execute( @@ -287,7 +299,7 @@ def store_chip_power_monitors(self): ChipPowerMonitorMachineVertex) with self.transaction() as cursor: - for row in cursor.execute( + for _ in cursor.execute( """ SELECT name FROM sqlite_master WHERE type='table' AND name='chip_power_monitor' From 8db050df54d0370c1f550d19ec520145a3a27fe4 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 19 Oct 2022 07:44:20 +0100 Subject: [PATCH 09/12] totals and track monitors --- .../storage_objects/sqllite_database.py | 2 +- .../report_functions/chip_active_report.py | 28 +++++++++++++++---- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py b/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py index eb0d87b944..8cc50aee12 100644 --- a/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py +++ b/spinn_front_end_common/interface/buffer_management/storage_objects/sqllite_database.py @@ -276,7 +276,7 @@ def store_placements(self): cursor, chip.x, chip.y, processor.processor_id) cursor.execute( """ - UPDATE core SET label = 'MONITOR' + UPDATE core SET label = 'MONITOR' WHERE core_id = ? """, [core_id]) assert cursor.rowcount == 1 diff --git a/spinn_front_end_common/utilities/report_functions/chip_active_report.py b/spinn_front_end_common/utilities/report_functions/chip_active_report.py index 61cc29c61f..297970be5a 100644 --- a/spinn_front_end_common/utilities/report_functions/chip_active_report.py +++ b/spinn_front_end_common/utilities/report_functions/chip_active_report.py @@ -13,11 +13,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from collections import defaultdict import logging import numpy import os -from spinn_utilities.config_holder import (get_config_int, get_config_str) +from spinn_utilities.config_holder import get_config_int from spinn_utilities.exceptions import SpiNNUtilsException from spinn_utilities.log import FormatAdapter from spinn_front_end_common.data import FecDataView @@ -34,6 +33,7 @@ # energy report file name CHIP_ACTIVE_FILENAME = "chip_active_report.rpt" + def write_chip_active_report(report_path=None, buffer_path=None): """ Writes the report. @@ -58,11 +58,16 @@ def write_chip_active_report(report_path=None, buffer_path=None): with open(report_path, "w", encoding="utf-8") as f: __write_report(f, buffer_path) + def __write_report(f, buffer_path): db = SqlLiteDatabase(buffer_path) n_samples_per_recording = get_config_int( "EnergyMonitor", "n_samples_per_recording_entry") + milliwatts = MILLIWATTS_PER_CHIP_ACTIVE_OVERHEAD / 18 + activity_total = 0 + energy_total = 0 + for row in db.iterate_chip_power_monitor_cores(): record_raw, data_missing = db.get_region_data( row["x"], row["y"], row["processor"], 0) @@ -73,13 +78,24 @@ def __write_report(f, buffer_path): activity_count = numpy.sum(results) time_for_recorded_sample =\ (row["sampling_frequency"] * n_samples_per_recording) / 1000 + energy_factor = time_for_recorded_sample * milliwatts for core in range(0, 18): label = db.get_label(row["x"], row["y"], core) if (active_sums[core] > 0) or label: - energy = (active_sums[core] * time_for_recorded_sample * - MILLIWATTS_PER_CHIP_ACTIVE_OVERHEAD / 18) f.write( f"processor {row['x']}:{row['y']}:{core}({label})" - f" was active for {active_sums[core]} " - f" using {energy} Joules\n") + f" was active for {active_sums[core]}ms " + f" using { active_sums[core] * energy_factor} Joules\n") + + energy = activity_count * energy_factor + activity_total += activity_count + energy_total += energy + f.write( + f"Total for chip {row['x']}:{row['y']} " + f" was {activity_count}ms of activity " + f" using {energy} Joules\n\n") + f.write( + f"Total " + f" was {activity_total}ms of activity " + f" using {energy_total} Joules\n\n") From d3ce5afac32522b597a73deb10f9ec8899a285c2 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 19 Oct 2022 09:49:22 +0100 Subject: [PATCH 10/12] write_chip_active_report standalone --- unittests/utilities/.gitignore | 1 + unittests/utilities/buffer.sqlite3 | Bin 0 -> 32768 bytes unittests/utilities/test_chip_active.py | 63 ++++++++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 unittests/utilities/.gitignore create mode 100644 unittests/utilities/buffer.sqlite3 create mode 100644 unittests/utilities/test_chip_active.py diff --git a/unittests/utilities/.gitignore b/unittests/utilities/.gitignore new file mode 100644 index 0000000000..b911337edd --- /dev/null +++ b/unittests/utilities/.gitignore @@ -0,0 +1 @@ +*.rpt \ No newline at end of file diff --git a/unittests/utilities/buffer.sqlite3 b/unittests/utilities/buffer.sqlite3 new file mode 100644 index 0000000000000000000000000000000000000000..ff14f71d5850c34fb3dda823e78a84bfa9c1728e GIT binary patch literal 32768 zcmeI4O>7%Q6o7Yj*Iw_o_8=-_(n6W2ifXG?aR?x_P*gYFL;=T6Z3hS!ck65$3&(aH zw@r>o8*$*&3*rU`-~vb-;07n)zyZMpsh1u=P=wS9A-Dh+-t3RPan^Q9)2e|tQZ#>W z-h1=So3Y~^ZQ)d=(lXiEdUMfeu`%HRfe_(1V}c+=;V%S#ZqlG2yA{ZZr`)R;6%If0 zQJ9_$|RLH`Gs+Z=ewq58GR{;@A7_15aiLYO)XGhB(X6$S{-bRu?cv#6k!5Ey8DC)zA+Jy9c|AL>7c3NUhiZc5vg{;) zh6T#&g<^hcyy$OnGi)H6$)&-rV>_{Wh$yjWl&l@G@qv%rjJOdHwAe%(7us|1B1q2` zVU1wmw4Mcr)aAC6YYiG#{2JTwvAL5~jWcGI@mXdk6Ke-UN_2QQbmdVVyykMNX;@+0 zDR~38)9Qw5cX&3m)N84YM|1YE>4^ zBy$8Ug@?=aS_>iq9Dp^niq2R+sZ!Ofc_evN>#-B;;Tv~{SLJG5QKAP9gf1sIDW|8a z=w;=R;IPy_q&DqoA3n@Qqp7MpZx;ZcR+61HTjd3_iycVEoeuUK8%=N+3&usWWWnsu zVoeMxv5^sSdD4nxFAQ&4*$~;@p17vELC}uF4e?zVXWcRGjQ1Q0Zmh#pJp0#{i?_x7 z_ppnKoyg=)wD&0N^@;RsrpOK+gbn1-v_i!_h+KhGq7eWFLuB>w3J+7;T)}_?f5k2b zo!QT3r%ui4FwK+t%TB=BV*G4zoaMHxV;ojtJjph%GH*|;y&@~Ip&_zPJ=kpx0Uy)= z+3xIhkaiPxeQ@XM&zDQ}KOa>e|GRYpUMFtd0O$b!*J}bQ+>q|J*IwBax7;~HTldv@_E83|NsjSH=K>gW?s9!c!c>_)j!Z zAJzF!rqlKlfW0AZvPYl7?fdr)2P{KrSLcjTe${ME8m$GhX*T1jWGb;==4RDn6i2>{fT~dKTjNeLjp(u2_OL^fCP{L z56ldPD3OZfO`)l3>nmRE07Tq*k$LkrhqIVq6#^*iy>&| zb;a(Nf_5$jxK5SGT&GGTu2Usq0FwUnkx_zFw9(Q7*ak2aoSBu~YR7 z+TH*6d2JV;d!@&T@@Lk|VkgSQt*+v|@)G^op3_9^ucrOkGC*wLd&0K5UiZo$C(55* zKSMOQmu-Jz@j5xkk$ynDdNLq*xSw#JKChmKL2l6I>ZQu7=b>IgTWH^@dIoXZ>b!18 z&%Qo9Z%5CAcc8}!@|5*bKYBx;f5ZL%TTsA+1dsp{Kmter2_OL^fCP{L5_a01`j~NB{{S0VIF~kN^@u0!RP}Ac6ld0qg!h z{a&Dd(%. + +import os +import unittest +from spinn_utilities.exceptions import InvalidDirectory +from spinn_front_end_common.data.fec_data_writer import FecDataWriter +from spinn_front_end_common.interface.config_setup import unittest_setup +from spinn_front_end_common.utilities.report_functions import ( + write_chip_active_report) + + +class TestChipActive(unittest.TestCase): + + def setUp(cls): + unittest_setup() + + def test_no_params(self): + try: + write_chip_active_report() + failed = False + except Exception as ex: + self.assertIn("no such table", str(ex)) + failed = True + self.assertTrue(failed) + + def test_db_only(self): + # make sure there is not run_dir_path so falls back on default + writer = FecDataWriter.setup() + try: + writer.set_run_dir_path("THIS DIRECTORY DOES NOT EXIST") + except InvalidDirectory: + pass + db_path = os.path.join(os.path.dirname(__file__), "buffer.sqlite3") + write_chip_active_report(buffer_path=db_path) + + + def test_all_params(self): + # make sure there is not run_dir_path so falls back on default + writer = FecDataWriter.setup() + try: + writer.set_run_dir_path("THIS DIRECTORY DOES NOT EXIST") + except InvalidDirectory: + pass + db_path = os.path.join(os.path.dirname(__file__), "buffer.sqlite3") + report = os.path.join(os.path.dirname(__file__), "my_active.rpt") + write_chip_active_report(report_path=report, buffer_path=db_path) + + +if __name__ == '__main__': + unittest.main() From 235d16cbd31201dd6500a29eeafcbef9cc7b1c5d Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 19 Oct 2022 11:04:22 +0100 Subject: [PATCH 11/12] flake8 --- unittests/utilities/test_chip_active.py | 1 - 1 file changed, 1 deletion(-) diff --git a/unittests/utilities/test_chip_active.py b/unittests/utilities/test_chip_active.py index 5b43c4c6c6..422396e9c1 100644 --- a/unittests/utilities/test_chip_active.py +++ b/unittests/utilities/test_chip_active.py @@ -46,7 +46,6 @@ def test_db_only(self): db_path = os.path.join(os.path.dirname(__file__), "buffer.sqlite3") write_chip_active_report(buffer_path=db_path) - def test_all_params(self): # make sure there is not run_dir_path so falls back on default writer = FecDataWriter.setup() From 39fce52eeef9e440fb19ff6e9833f96af5e817fb Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Wed, 19 Oct 2022 15:13:21 +0100 Subject: [PATCH 12/12] Dont rat reports created in unittests --- .ratexcludes | 1 + 1 file changed, 1 insertion(+) diff --git a/.ratexcludes b/.ratexcludes index 8acb1c02ea..964a183a24 100644 --- a/.ratexcludes +++ b/.ratexcludes @@ -17,3 +17,4 @@ **/PACMAN/** **/DataSpecification/** **/spalloc/** +**/unittests/**/*.rpt