From 6e120fc67fb67fd339d1704e14892f584c30eaee Mon Sep 17 00:00:00 2001 From: Sudheer Chunduri Date: Sun, 30 Jan 2022 22:25:50 -0500 Subject: [PATCH 01/12] apss: correct the duplicate declaration and correct the first APSS counter --- apss/lib/darshan-apss-utils.h | 1 - apss/lib/darshan-apss.c | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/apss/lib/darshan-apss-utils.h b/apss/lib/darshan-apss-utils.h index 8ce7378..40c6b39 100644 --- a/apss/lib/darshan-apss-utils.h +++ b/apss/lib/darshan-apss-utils.h @@ -40,7 +40,6 @@ static int sstopo_get_mycoords(int *rack, int *chassis, int *blade, int *node) char a, b, c, d, e; int layer_temp; - int rack, chassis, blade, anode; /* format example: c1-0c1s2n1 c3-0c2s15n3 */ /* format example: x3012c0s13b0n0 */ sscanf(hostname, diff --git a/apss/lib/darshan-apss.c b/apss/lib/darshan-apss.c index 459ce9e..5e0f86a 100644 --- a/apss/lib/darshan-apss.c +++ b/apss/lib/darshan-apss.c @@ -105,7 +105,7 @@ static void initialize_counters (void) PAPI_create_eventset(&apss_runtime->PAPI_event_set); /* start with first PAPI counter */ - for (i = AR_RTR_0_0_INQ_PRF_INCOMING_FLIT_VC0; + for (i = CQ_CQ_OXE_NUM_STALLS; strcmp(PAPI_events[i], "APSS_NUM_INDICES") != 0; i++) { @@ -136,7 +136,7 @@ static void capture(struct darshan_apss_perf_record *rec, darshan_record_id rec_id) { PAPI_stop(apss_runtime->PAPI_event_set, - (long long*) &rec->counters[AR_RTR_0_0_INQ_PRF_INCOMING_FLIT_VC0]); + (long long*) &rec->counters[CQ_CQ_OXE_NUM_STALLS]); PAPI_reset(apss_runtime->PAPI_event_set); rec->group = apss_runtime->group; From 4604d5153252efdee1af7df8d0868ab0bed31f9d Mon Sep 17 00:00:00 2001 From: Sudheer Chunduri Date: Sun, 30 Jan 2022 22:53:05 -0500 Subject: [PATCH 02/12] apss: correct the variable declarations --- apss/lib/darshan-apss-utils.h | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/apss/lib/darshan-apss-utils.h b/apss/lib/darshan-apss-utils.h index 40c6b39..d344526 100644 --- a/apss/lib/darshan-apss-utils.h +++ b/apss/lib/darshan-apss-utils.h @@ -44,17 +44,12 @@ static int sstopo_get_mycoords(int *rack, int *chassis, int *blade, int *node) /* format example: x3012c0s13b0n0 */ sscanf(hostname, "%c%d%c%d%c%d%c%d%c%d", - &a, &rack, &b, &layer_temp, &c, &chassis, &d, &blade, &e, &anode); + &a, &rack, &b, &layer_temp, &c, &chassis, &d, &blade, &e, &node); #ifdef DEBUG - fprintf(stderr, "coords = (%d,%d, %d, %d,%d,%d,%d) \n", layer_rank, blade_rank, node_rank, rack, chassis, blade, anode); + fprintf(stderr, "coords = (%d,%d, %d, %d,%d,%d,%d) \n", layer_rank, blade_rank, node_rank, rack, chassis, blade, node); #endif - rack = rack; - chassis = chassis; - blade = blade; - node = anode; - return 0; } From b2546905fdced502a973684812be8ed23f85925f Mon Sep 17 00:00:00 2001 From: Sudheer Chunduri Date: Sun, 30 Jan 2022 23:48:26 -0500 Subject: [PATCH 03/12] apss: correct a typo --- apss/lib/darshan-apss.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apss/lib/darshan-apss.c b/apss/lib/darshan-apss.c index 5e0f86a..6a380f8 100644 --- a/apss/lib/darshan-apss.c +++ b/apss/lib/darshan-apss.c @@ -220,7 +220,7 @@ void apss_runtime_initialize() apss_runtime->header_record->magic = APSS_MAGIC; } - get_xc_coords(&apss_runtime->group, + get_ss_coords(&apss_runtime->group, &apss_runtime->chassis, &apss_runtime->blade, &apss_runtime->node); From 7224528223fbef07dc4f520ca9b8d4c948f434ec Mon Sep 17 00:00:00 2001 From: Sudheer Chunduri Date: Sun, 30 Jan 2022 23:48:26 -0500 Subject: [PATCH 04/12] apss: correct a typo --- apss/lib/darshan-apss.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apss/lib/darshan-apss.c b/apss/lib/darshan-apss.c index 5e0f86a..3d19669 100644 --- a/apss/lib/darshan-apss.c +++ b/apss/lib/darshan-apss.c @@ -220,7 +220,7 @@ void apss_runtime_initialize() apss_runtime->header_record->magic = APSS_MAGIC; } - get_xc_coords(&apss_runtime->group, + sstopo_get_mycoords(&apss_runtime->group, &apss_runtime->chassis, &apss_runtime->blade, &apss_runtime->node); From 7a65651acd053a8dcd826bc2d828c402ba4b7073 Mon Sep 17 00:00:00 2001 From: Sudheer Chunduri Date: Mon, 31 Jan 2022 13:22:36 -0500 Subject: [PATCH 05/12] apss: adding more counters and removing the counter reduction logic --- apss/darshan-apss-log-format.h | 44 ++++++++++++++++++++++++++++++++++ apss/lib/darshan-apss.c | 16 ++++++------- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/apss/darshan-apss-log-format.h b/apss/darshan-apss-log-format.h index 123a22f..de9680e 100644 --- a/apss/darshan-apss-log-format.h +++ b/apss/darshan-apss-log-format.h @@ -27,6 +27,50 @@ X(CQ_CYCLES_BLOCKED_1) \ X(CQ_CYCLES_BLOCKED_2) \ X(CQ_CYCLES_BLOCKED_3) \ + X(HNI_RX_PAUSED_0) \ + X(HNI_RX_PAUSED_1) \ + X(HNI_RX_PAUSED_2) \ + X(HNI_RX_PAUSED_3) \ + X(HNI_RX_PAUSED_4) \ + X(HNI_RX_PAUSED_5) \ + X(HNI_RX_PAUSED_6) \ + X(HNI_RX_PAUSED_7) \ + X(HNI_TX_PAUSED_0) \ + X(HNI_TX_PAUSED_1) \ + X(HNI_TX_PAUSED_2) \ + X(HNI_TX_PAUSED_3) \ + X(HNI_TX_PAUSED_4) \ + X(HNI_TX_PAUSED_5) \ + X(HNI_TX_PAUSED_6) \ + X(HNI_TX_PAUSED_7) \ + X(HNI_PKTS_SENT_BY_TC_0) \ + X(HNI_PKTS_SENT_BY_TC_1) \ + X(HNI_PKTS_SENT_BY_TC_2) \ + X(HNI_PKTS_SENT_BY_TC_3) \ + X(HNI_PKTS_SENT_BY_TC_4) \ + X(HNI_PKTS_SENT_BY_TC_5) \ + X(HNI_PKTS_SENT_BY_TC_6) \ + X(HNI_PKTS_SENT_BY_TC_7) \ + X(HNI_PKTS_RECV_BY_TC_0) \ + X(HNI_PKTS_RECV_BY_TC_1) \ + X(HNI_PKTS_RECV_BY_TC_2) \ + X(HNI_PKTS_RECV_BY_TC_3) \ + X(HNI_PKTS_RECV_BY_TC_4) \ + X(HNI_PKTS_RECV_BY_TC_5) \ + X(HNI_PKTS_RECV_BY_TC_6) \ + X(HNI_PKTS_RECV_BY_TC_7) \ + X(HNI_TX_OK_27) \ + X(HNI_TX_OK_35) \ + X(HNI_TX_OK_36_TO_63) \ + X(HNI_TX_OK_64) \ + X(HNI_TX_OK_65_TO_127) \ + X(HNI_TX_OK_128_TO_255) \ + X(HNI_TX_OK_256_TO_511) \ + X(HNI_TX_OK_512_TO_1023) \ + X(HNI_TX_OK_1024_TO_2047) \ + X(HNI_TX_OK_2048_TO_4095) \ + X(HNI_TX_OK_4096_TO_8191) \ + X(HNI_TX_OK_8192_TO_MAX) \ /* end of counters */\ Z(APSS_NUM_INDICES) diff --git a/apss/lib/darshan-apss.c b/apss/lib/darshan-apss.c index 3d19669..bb45703 100644 --- a/apss/lib/darshan-apss.c +++ b/apss/lib/darshan-apss.c @@ -36,7 +36,7 @@ static char* PAPI_events[] = #undef Z #define MAX_GROUPS (128) -#define MAX_CHASSIS (MAX_GROUPS*6) +#define MAX_CHASSIS (MAX_GROUPS*16) #define MAX_BLADES (MAX_CHASSIS*16) /* @@ -152,7 +152,7 @@ static void capture(struct darshan_apss_perf_record *rec, void apss_runtime_initialize() { size_t apss_buf_size; - char rtr_rec_name[128]; + //char rtr_rec_name[128]; darshan_module_funcs mod_funcs = { //#ifdef HAVE_MPI @@ -225,8 +225,8 @@ void apss_runtime_initialize() &apss_runtime->blade, &apss_runtime->node); - sprintf(rtr_rec_name, "darshan-apss-rtr-%d-%d-%d", - apss_runtime->group, apss_runtime->chassis, apss_runtime->blade); + //sprintf(rtr_rec_name, "darshan-apss-rtr-%d-%d-%d", + // apss_runtime->group, apss_runtime->chassis, apss_runtime->blade); //apss_runtime->rtr_id = darshan_core_gen_record_id(rtr_rec_name); apss_runtime->rtr_id = darshan_core_gen_record_id("APSS"); apss_runtime->perf_record = darshan_core_register_record( @@ -295,7 +295,7 @@ static void apss_mpi_redux( { apss_runtime->header_record->appid = atoi((char*)getenv( csJOBID_ENV_STR )); } - +#if 0 /* count network dimensions */ if (bitvec) { @@ -346,7 +346,7 @@ static void apss_mpi_redux( if (my_rank == 0) { - apss_runtime->header_record->nblades = router_count; + apss_runtime->header_record->nblades = router_count; apss_runtime->header_record->nchassis = chassis_count; apss_runtime->header_record->ngroups = group_count; } @@ -354,7 +354,7 @@ static void apss_mpi_redux( } else { - apss_runtime->header_record->nblades = 0; + apss_runtime->header_record->nblades = 0; apss_runtime->header_record->nchassis = 0; apss_runtime->header_record->ngroups = 0; } @@ -391,7 +391,7 @@ static void apss_mpi_redux( PMPI_Comm_free(&router_comm); APSS_UNLOCK(); - +#endif return; } From 3e58894bcd6f9b485a5948598d5854be215506ea Mon Sep 17 00:00:00 2001 From: Sudheer Chunduri Date: Mon, 31 Jan 2022 14:09:43 -0500 Subject: [PATCH 06/12] correct the variable dereference --- apss/lib/darshan-apss-utils.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apss/lib/darshan-apss-utils.h b/apss/lib/darshan-apss-utils.h index d344526..598716b 100644 --- a/apss/lib/darshan-apss-utils.h +++ b/apss/lib/darshan-apss-utils.h @@ -44,10 +44,10 @@ static int sstopo_get_mycoords(int *rack, int *chassis, int *blade, int *node) /* format example: x3012c0s13b0n0 */ sscanf(hostname, "%c%d%c%d%c%d%c%d%c%d", - &a, &rack, &b, &layer_temp, &c, &chassis, &d, &blade, &e, &node); + &a, rack, &b, &layer_temp, &c, chassis, &d, blade, &e, node); #ifdef DEBUG - fprintf(stderr, "coords = (%d,%d, %d, %d,%d,%d,%d) \n", layer_rank, blade_rank, node_rank, rack, chassis, blade, node); + fprintf(stderr, "coords = (%d,%d, %d, %d,%d,%d,%d) \n", layer_rank, blade_rank, node_rank, *rack, *chassis, *blade, *node); #endif return 0; From 14cbe4ad1381cdd0b15ddcd571c0dbe0212359d3 Mon Sep 17 00:00:00 2001 From: Sudheer Chunduri Date: Wed, 2 Feb 2022 19:46:20 -0500 Subject: [PATCH 07/12] APSS: correcting the macro names --- apss/util/darshan-apss-logutils.c | 228 +++++++++++++++--------------- 1 file changed, 114 insertions(+), 114 deletions(-) diff --git a/apss/util/darshan-apss-logutils.c b/apss/util/darshan-apss-logutils.c index 2de463f..1291328 100644 --- a/apss/util/darshan-apss-logutils.c +++ b/apss/util/darshan-apss-logutils.c @@ -22,10 +22,10 @@ /* counter name strings for the apss module */ #define Y(a) #a, -#define X(a) Y(apss_ ## a) +#define X(a) Y(APSS_ ## a) #define Z(a) #a char *apss_counter_names[] = { - apss_PERF_COUNTERS + APSS_PERF_COUNTERS }; #undef Y #undef X @@ -59,14 +59,14 @@ static int darshan_log_get_apss_rec(darshan_fd fd, void** buf_p) int ret = -1; static int first_rec = 1; - if(fd->mod_map[DARSHAN_apss_MOD].len == 0) + if(fd->mod_map[DARSHAN_APSS_MOD].len == 0) return(0); - if(fd->mod_ver[DARSHAN_apss_MOD] == 0 || - fd->mod_ver[DARSHAN_apss_MOD] > apss_VER) + if(fd->mod_ver[DARSHAN_APSS_MOD] == 0 || + fd->mod_ver[DARSHAN_APSS_MOD] > APSS_VER) { - fprintf(stderr, "Error: Invalid apss module version number (got %d)\n", - fd->mod_ver[DARSHAN_apss_MOD]); + fprintf(stderr, "Error: Invalid APSS module version number (got %d)\n", + fd->mod_ver[DARSHAN_APSS_MOD]); return(-1); } @@ -84,13 +84,13 @@ static int darshan_log_get_apss_rec(darshan_fd fd, void** buf_p) buffer = *buf_p; } - if (fd->mod_ver[DARSHAN_apss_MOD] < apss_VER) + if (fd->mod_ver[DARSHAN_APSS_MOD] < APSS_VER) { /* perform conversion as needed */ } /* v1, current version */ - if (fd->mod_ver[DARSHAN_apss_MOD] == apss_VER) + if (fd->mod_ver[DARSHAN_APSS_MOD] == APSS_VER) { if (first_rec) { @@ -100,7 +100,7 @@ static int darshan_log_get_apss_rec(darshan_fd fd, void** buf_p) else rec_len = sizeof(struct darshan_apss_perf_record); - ret = darshan_log_get_mod(fd, DARSHAN_apss_MOD, buffer, rec_len); + ret = darshan_log_get_mod(fd, DARSHAN_APSS_MOD, buffer, rec_len); } if (ret == rec_len) @@ -128,7 +128,7 @@ static int darshan_log_get_apss_rec(darshan_fd fd, void** buf_p) DARSHAN_BSWAP64(&(prf_rec->chassis)); DARSHAN_BSWAP64(&(prf_rec->blade)); DARSHAN_BSWAP64(&(prf_rec->node)); - for (i = 0; i < apss_NUM_INDICES; i++) + for (i = 0; i < APSS_NUM_INDICES; i++) { DARSHAN_BSWAP64(&prf_rec->counters[i]); } @@ -167,8 +167,8 @@ static int darshan_log_put_apss_rec(darshan_fd fd, void* buf) else rec_len = sizeof(struct darshan_apss_perf_record); - ret = darshan_log_put_mod(fd, DARSHAN_apss_MOD, buf, - rec_len, apss_VER); + ret = darshan_log_put_mod(fd, DARSHAN_APSS_MOD, buf, + rec_len, APSS_VER); if(ret < 0) return(-1); @@ -186,41 +186,41 @@ static void darshan_log_print_apss_rec(void *rec, char *file_name, if (first_rec) { hdr_rec = rec; - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "apss_GROUPS", hdr_rec->ngroups, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + "APSS_GROUPS", hdr_rec->ngroups, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "apss_CHASSIS", hdr_rec->nchassis, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + "APSS_CHASSIS", hdr_rec->nchassis, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "apss_BLADES", hdr_rec->nblades, "", "", ""); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + "APSS_BLADES", hdr_rec->nblades, "", "", ""); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "apss_APPLICATION_ID", hdr_rec->appid, "", "", ""); + "APSS_APPLICATION_ID", hdr_rec->appid, "", "", ""); first_rec = 0; } else { prf_rec = rec; - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec->base_rec.rank, prf_rec->base_rec.id, - "apss_GROUP", prf_rec->group, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + "APSS_GROUP", prf_rec->group, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec->base_rec.rank, prf_rec->base_rec.id, - "apss_CHASSIS", prf_rec->chassis, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + "APSS_CHASSIS", prf_rec->chassis, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec->base_rec.rank, prf_rec->base_rec.id, - "apss_BLADE", prf_rec->blade, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + "APSS_BLADE", prf_rec->blade, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec->base_rec.rank, prf_rec->base_rec.id, - "apss_NODE", prf_rec->node, "", "", ""); + "APSS_NODE", prf_rec->node, "", "", ""); - for(i = 0; i < apss_NUM_INDICES; i++) + for(i = 0; i < APSS_NUM_INDICES; i++) { - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec->base_rec.rank, prf_rec->base_rec.id, apss_counter_names[i], prf_rec->counters[i], "", "", ""); @@ -234,20 +234,20 @@ static void darshan_log_print_apss_description(int ver) { printf("\n# description of apss counters:\n"); printf("# global summary stats for the apss module:\n"); - printf("# apss_GROUPS: total number of groups\n"); - printf("# apss_CHASSIS: total number of chassis\n"); - printf("# apss_BLADES: total number of blades\n"); + printf("# APSS_GROUPS: total number of groups\n"); + printf("# APSS_CHASSIS: total number of chassis\n"); + printf("# APSS_BLADES: total number of blades\n"); printf("# per-router statistics for the apss module:\n"); - printf("# apss_GROUP: group this router is on\n"); - printf("# apss_CHASSIS: chassis this router is on\n"); - printf("# apss_BLADE: blade this router is on\n"); - printf("# apss_NODE: node connected to this router\n"); - printf("# apss_AR_RTR_* port counters for the 40 router-router ports\n"); - printf("# apss_AR_RTR_x_y_INQ_PRF_INCOMING_FLIT_VC[0-7]: flits on VCs of x y tile\n"); - printf("# apss_AR_RTR_x_y_INQ_PRF_ROWBUS_STALL_CNT: stalls on x y tile\n"); - printf("# apss_AR_RTR_PT_* port counters for the 8 router-nic ports\n"); - printf("# apss_AR_RTR_PT_x_y_INQ_PRF_INCOMING_FLIT_VC[0,4]: flits on VCs of x y tile\n"); - printf("# apss_AR_RTR_PT_x_y_INQ_PRF_REQ_ROWBUS_STALL_CNT: stalls on x y tile\n"); + printf("# APSS_GROUP: group this router is on\n"); + printf("# APSS_CHASSIS: chassis this router is on\n"); + printf("# APSS_BLADE: blade this router is on\n"); + printf("# APSS_NODE: node connected to this router\n"); + printf("# APSS_AR_RTR_* port counters for the 40 router-router ports\n"); + printf("# APSS_AR_RTR_x_y_INQ_PRF_INCOMING_FLIT_VC[0-7]: flits on VCs of x y tile\n"); + printf("# APSS_AR_RTR_x_y_INQ_PRF_ROWBUS_STALL_CNT: stalls on x y tile\n"); + printf("# APSS_AR_RTR_PT_* port counters for the 8 router-nic ports\n"); + printf("# APSS_AR_RTR_PT_x_y_INQ_PRF_INCOMING_FLIT_VC[0,4]: flits on VCs of x y tile\n"); + printf("# APSS_AR_RTR_PT_x_y_INQ_PRF_REQ_ROWBUS_STALL_CNT: stalls on x y tile\n"); return; } @@ -265,94 +265,94 @@ static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, prf_rec1 = (struct darshan_apss_perf_record*) file_rec1; prf_rec2 = (struct darshan_apss_perf_record*) file_rec2; - if (hdr_rec1->magic == apss_MAGIC) + if (hdr_rec1->magic == APSS_MAGIC) { /* this is the header record */ if (!hdr_rec2) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_GROUPS", hdr_rec1->ngroups, "", "", ""); + "APSS_GROUPS", hdr_rec1->ngroups, "", "", ""); printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_CHASSIS", hdr_rec1->nchassis, "", "", ""); + "APSS_CHASSIS", hdr_rec1->nchassis, "", "", ""); printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_BLADES", hdr_rec1->nblades, "", "", ""); + "APSS_BLADES", hdr_rec1->nblades, "", "", ""); printf("- "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_APPLICATION_ID", hdr_rec1->appid, "", "", ""); + "APSS_APPLICATION_ID", hdr_rec1->appid, "", "", ""); } else if (!hdr_rec1) { printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_GROUPS", hdr_rec2->ngroups, "", "", ""); + "APSS_GROUPS", hdr_rec2->ngroups, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_CHASSIS", hdr_rec2->nchassis, "", "", ""); + "APSS_CHASSIS", hdr_rec2->nchassis, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_BLADES", hdr_rec2->nblades, "", "", ""); + "APSS_BLADES", hdr_rec2->nblades, "", "", ""); printf("+ "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_APPLICATION_ID", hdr_rec2->appid, "", "", ""); + "APSS_APPLICATION_ID", hdr_rec2->appid, "", "", ""); } else { if (hdr_rec1->ngroups != hdr_rec2->ngroups) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_GROUPS", hdr_rec1->ngroups, "", "", ""); + "APSS_GROUPS", hdr_rec1->ngroups, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_GROUPS", hdr_rec2->ngroups, "", "", ""); + "APSS_GROUPS", hdr_rec2->ngroups, "", "", ""); } if (hdr_rec1->nchassis != hdr_rec2->nchassis) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_CHASSIS", hdr_rec1->nchassis, "", "", ""); + "APSS_CHASSIS", hdr_rec1->nchassis, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_CHASSIS", hdr_rec2->nchassis, "", "", ""); + "APSS_CHASSIS", hdr_rec2->nchassis, "", "", ""); } if (hdr_rec1->nblades != hdr_rec2->nblades) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_BLADES", hdr_rec1->nblades, "", "", ""); + "APSS_BLADES", hdr_rec1->nblades, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_BLADES", hdr_rec2->nblades, "", "", ""); + "APSS_BLADES", hdr_rec2->nblades, "", "", ""); } if (hdr_rec1->appid != hdr_rec2->appid) { printf("- "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_APPLICATION_ID", hdr_rec1->appid, "", "", ""); + "APSS_APPLICATION_ID", hdr_rec1->appid, "", "", ""); printf("+ "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_APPLICATION_ID", hdr_rec2->appid, "", "", ""); + "APSS_APPLICATION_ID", hdr_rec2->appid, "", "", ""); } } } @@ -361,96 +361,96 @@ static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, if (!prf_rec2) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_GROUP", prf_rec1->group, "", "", ""); + "APSS_GROUP", prf_rec1->group, "", "", ""); printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_CHASSIS", prf_rec1->chassis, "", "", ""); + "APSS_CHASSIS", prf_rec1->chassis, "", "", ""); printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_BLADE", prf_rec1->blade, "", "", ""); + "APSS_BLADE", prf_rec1->blade, "", "", ""); printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_NODE", prf_rec1->node, "", "", ""); + "APSS_NODE", prf_rec1->node, "", "", ""); } else if (!prf_rec1) { printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_GROUP", prf_rec2->group, "", "", ""); + "APSS_GROUP", prf_rec2->group, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_CHASSIS", prf_rec2->chassis, "", "", ""); + "APSS_CHASSIS", prf_rec2->chassis, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_BLADE", prf_rec2->blade, "", "", ""); + "APSS_BLADE", prf_rec2->blade, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_NODE", prf_rec2->node, "", "", ""); + "APSS_NODE", prf_rec2->node, "", "", ""); } else { if (prf_rec1->group != prf_rec2->group) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_GROUP", prf_rec1->group, "", "", ""); + "APSS_GROUP", prf_rec1->group, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_GROUP", prf_rec2->group, "", "", ""); + "APSS_GROUP", prf_rec2->group, "", "", ""); } if (prf_rec1->chassis != prf_rec2->chassis) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_CHASSIS", prf_rec1->chassis, "", "", ""); + "APSS_CHASSIS", prf_rec1->chassis, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_CHASSIS", prf_rec2->chassis, "", "", ""); + "APSS_CHASSIS", prf_rec2->chassis, "", "", ""); } if (prf_rec1->blade != prf_rec2->blade) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_BLADE", prf_rec1->blade, "", "", ""); + "APSS_BLADE", prf_rec1->blade, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_BLADE", prf_rec2->blade, "", "", ""); + "APSS_BLADE", prf_rec2->blade, "", "", ""); } if (prf_rec1->node != prf_rec2->node) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_NODE", prf_rec1->node, "", "", ""); + "APSS_NODE", prf_rec1->node, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_NODE", prf_rec2->node, "", "", ""); + "APSS_NODE", prf_rec2->node, "", "", ""); } } int i; /* router tile record */ - for(i = 0; i < apss_NUM_INDICES; i++) + for(i = 0; i < APSS_NUM_INDICES; i++) { if (!prf_rec2) { printf("- "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, apss_counter_names[i], prf_rec1->counters[i], "", "", ""); @@ -458,7 +458,7 @@ static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, else if (!prf_rec1) { printf("+ "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, apss_counter_names[i], prf_rec2->counters[i], "", "", ""); @@ -466,13 +466,13 @@ static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, else if (prf_rec1->counters[i] != prf_rec2->counters[i]) { printf("- "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, apss_counter_names[i], prf_rec1->counters[i], "", "", ""); printf("+ "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, apss_counter_names[i], prf_rec2->counters[i], "", "", ""); From eb71117a01ce42f1ab068de288f2d24d65d83218 Mon Sep 17 00:00:00 2001 From: Sudheer Chunduri Date: Fri, 11 Feb 2022 12:41:19 -0500 Subject: [PATCH 08/12] APSS: updates to include the slot number in the node coordinates The node name format for OLCF Crusher is different from ALCF Polaris Only one slot in a chassis was populated and the topology spread with 10 groups on Polaris, where as Crusher nodes are populated in just 2 groups. Hence, the slots in the chassis are populated on Crusher. This necessitated change in the node coordinates. --- apss/darshan-apss-log-format.h | 2 + apss/lib/darshan-apss-utils.h | 17 ++- apss/lib/darshan-apss.c | 245 ++++++++++++++++++------------ apss/util/darshan-apss-logutils.c | 58 ++++++- 4 files changed, 212 insertions(+), 110 deletions(-) diff --git a/apss/darshan-apss-log-format.h b/apss/darshan-apss-log-format.h index de9680e..8e12eb2 100644 --- a/apss/darshan-apss-log-format.h +++ b/apss/darshan-apss-log-format.h @@ -99,6 +99,7 @@ struct darshan_apss_perf_record struct darshan_base_record base_rec; int64_t group; int64_t chassis; + int64_t slot; int64_t blade; int64_t node; uint64_t counters[APSS_NUM_INDICES]; @@ -110,6 +111,7 @@ struct darshan_apss_header_record int64_t magic; int64_t nblades; int64_t nchassis; + int64_t nslots; int64_t ngroups; uint64_t appid; }; diff --git a/apss/lib/darshan-apss-utils.h b/apss/lib/darshan-apss-utils.h index 598716b..f4bc7fd 100644 --- a/apss/lib/darshan-apss-utils.h +++ b/apss/lib/darshan-apss-utils.h @@ -2,7 +2,7 @@ #define __APSS_UTILS_H__ #include - +extern char crusher_nodelist[192][14]; static void search_hwinfo(const char * mstr, char *mode) { FILE *f; @@ -33,21 +33,28 @@ static void search_hwinfo(const char * mstr, char *mode) return; } -static int sstopo_get_mycoords(int *rack, int *chassis, int *blade, int *node) +static int sstopo_get_mycoords(int* group, int* chassis, int* slot, int* blade, int* node) { char hostname[HOST_NAME_MAX + 1]; gethostname(hostname, HOST_NAME_MAX + 1); char a, b, c, d, e; int layer_temp; + int rack; /* format example: c1-0c1s2n1 c3-0c2s15n3 */ /* format example: x3012c0s13b0n0 */ - sscanf(hostname, + sscanf(crusher_nodelist[atoi(hostname+7)], "%c%d%c%d%c%d%c%d%c%d", - &a, rack, &b, &layer_temp, &c, chassis, &d, blade, &e, node); + &a, &rack, &b, chassis, &c, slot, &d, blade, &e, node); + if(rack == 2601){ + *group = 0; + } + else{ + *group = 1; + } #ifdef DEBUG - fprintf(stderr, "coords = (%d,%d, %d, %d,%d,%d,%d) \n", layer_rank, blade_rank, node_rank, *rack, *chassis, *blade, *node); + fprintf(stderr, "coords = (%d,%d, %d, %d,%d,%d,%d) \n", layer_rank, blade_rank, node_rank, *group, *chassis, *slot, *blade, *node); #endif return 0; diff --git a/apss/lib/darshan-apss.c b/apss/lib/darshan-apss.c index bb45703..f98bfbe 100644 --- a/apss/lib/darshan-apss.c +++ b/apss/lib/darshan-apss.c @@ -6,7 +6,12 @@ #define _XOPEN_SOURCE 500 #define _GNU_SOURCE -#define csJOBID_ENV_STR "PALS_APP_ID" +//#ifdef SLURM_JOBID +#define APP_ID "SLURM_TASK_PID" +//#endif +//#ifdef PBS_JOBID +//#define APP_ID "PALS_APP_ID +//#endif #include "darshan-runtime-config.h" #include @@ -22,6 +27,7 @@ #include "darshan-apss-log-format.h" #include "darshan-apss-utils.h" +#include "crusher_nodelist.h" /* * PAPI_events are defined by the Aries counters listed in the log header. @@ -36,8 +42,9 @@ static char* PAPI_events[] = #undef Z #define MAX_GROUPS (128) -#define MAX_CHASSIS (MAX_GROUPS*16) -#define MAX_BLADES (MAX_CHASSIS*16) +#define MAX_CHASSIS (MAX_GROUPS*8) +#define MAX_SLOTS (MAX_CHASSIS*8) +#define MAX_BLADES (MAX_SLOTS*2) /* * @@ -60,6 +67,7 @@ struct apss_runtime int PAPI_event_count; int group; int chassis; + int slot; int blade; int node; int perf_record_marked; @@ -77,10 +85,10 @@ void apss_runtime_initialize(void); /* forward declaration for shutdown function needed to interface with darshan-core */ //#ifdef HAVE_MPI static void apss_mpi_redux( - void *buffer, - MPI_Comm mod_comm, - darshan_record_id *shared_recs, - int shared_rec_count); + void *buffer, + MPI_Comm mod_comm, + darshan_record_id *shared_recs, + int shared_rec_count); //#endif static void apss_output( void **buffer, @@ -103,11 +111,27 @@ static void initialize_counters (void) PAPI_library_init(PAPI_VER_CURRENT); apss_runtime->PAPI_event_set = PAPI_NULL; PAPI_create_eventset(&apss_runtime->PAPI_event_set); +#if 0 + /* Determine the number of NIC devices - read the number of dirs in /sys/class/cxi/ */ + int num_nics = 0; + struct dirent *dp; + DIR *fdir; + if ((fdir = opendir("/sys/class/cxi")) == NULL) { + fprintf(stderr, "listdir: can't open %s\n", "/sys/class/cxi"); + return 1; + } + while ((dp = readdir(fdir)) != NULL) { + if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, "..")) + continue; /* skip self and parent */ + num_nics++; + } + closedir(fdir); +#endif /* start with first PAPI counter */ for (i = CQ_CQ_OXE_NUM_STALLS; - strcmp(PAPI_events[i], "APSS_NUM_INDICES") != 0; - i++) + strcmp(PAPI_events[i], "APSS_NUM_INDICES") != 0; + i++) { PAPI_event_name_to_code(PAPI_events[i], &code); PAPI_add_event(apss_runtime->PAPI_event_set, code); @@ -133,14 +157,15 @@ static void finalize_counters (void) * Function which updates all the counter data */ static void capture(struct darshan_apss_perf_record *rec, - darshan_record_id rec_id) + darshan_record_id rec_id) { PAPI_stop(apss_runtime->PAPI_event_set, - (long long*) &rec->counters[CQ_CQ_OXE_NUM_STALLS]); + (long long*) &rec->counters[CQ_CQ_OXE_NUM_STALLS]); PAPI_reset(apss_runtime->PAPI_event_set); rec->group = apss_runtime->group; rec->chassis = apss_runtime->chassis; + rec->slot = apss_runtime->slot; rec->blade = apss_runtime->blade; rec->node = apss_runtime->node; rec->base_rec.id = rec_id; @@ -152,18 +177,17 @@ static void capture(struct darshan_apss_perf_record *rec, void apss_runtime_initialize() { size_t apss_buf_size; - //char rtr_rec_name[128]; - + char rtr_rec_name[128]; darshan_module_funcs mod_funcs = { -//#ifdef HAVE_MPI + //#ifdef HAVE_MPI .mod_redux_func = &apss_mpi_redux, -//#endif + //#endif .mod_output_func = &apss_output, .mod_cleanup_func = &apss_cleanup - }; + }; APSS_LOCK(); - + /* don't do anything if already initialized */ if(apss_runtime) @@ -174,15 +198,15 @@ void apss_runtime_initialize() apss_buf_size = sizeof(struct darshan_apss_header_record) + - sizeof(struct darshan_apss_perf_record); + sizeof(struct darshan_apss_perf_record); /* register the APSS module with the darshan-core component */ darshan_core_register_module( - DARSHAN_APSS_MOD, - mod_funcs, - &apss_buf_size, - &my_rank, - NULL); + DARSHAN_APSS_MOD, + mod_funcs, + &apss_buf_size, + &my_rank, + NULL); /* initialize module's global state */ @@ -201,42 +225,41 @@ void apss_runtime_initialize() /* register the apss file record with darshan-core */ apss_runtime->header_record = darshan_core_register_record( - apss_runtime->header_id, - //NULL, - "darshan-apss-header", - DARSHAN_APSS_MOD, - sizeof(struct darshan_apss_header_record), - NULL); + apss_runtime->header_id, + //NULL, + "darshan-apss-header", + DARSHAN_APSS_MOD, + sizeof(struct darshan_apss_header_record), + NULL); if(!(apss_runtime->header_record)) { darshan_core_unregister_module(DARSHAN_APSS_MOD); free(apss_runtime); apss_runtime = NULL; APSS_UNLOCK(); - return; + return; } apss_runtime->header_record->base_rec.id = apss_runtime->header_id; apss_runtime->header_record->base_rec.rank = my_rank; apss_runtime->header_record->magic = APSS_MAGIC; } - sstopo_get_mycoords(&apss_runtime->group, - &apss_runtime->chassis, - &apss_runtime->blade, - &apss_runtime->node); - - //sprintf(rtr_rec_name, "darshan-apss-rtr-%d-%d-%d", - // apss_runtime->group, apss_runtime->chassis, apss_runtime->blade); + &apss_runtime->chassis, + &apss_runtime->slot, + &apss_runtime->blade, + &apss_runtime->node); + sprintf(rtr_rec_name, "darshan-apss-rtr-%d-%d-%d-%d", + apss_runtime->group, apss_runtime->chassis, apss_runtime->slot, apss_runtime->blade); //apss_runtime->rtr_id = darshan_core_gen_record_id(rtr_rec_name); apss_runtime->rtr_id = darshan_core_gen_record_id("APSS"); apss_runtime->perf_record = darshan_core_register_record( - apss_runtime->rtr_id, - //NULL, - "APSS", // we want the record for each rank to be treated as shared records so that mpi_redux can operate on - //rtr_rec_name, - DARSHAN_APSS_MOD, - sizeof(struct darshan_apss_perf_record), - NULL); + apss_runtime->rtr_id, + //NULL, + "APSS", // we want the record for each rank to be treated as shared records so that mpi_redux can operate on + //rtr_rec_name, + DARSHAN_APSS_MOD, + sizeof(struct darshan_apss_perf_record), + NULL); if(!(apss_runtime->perf_record)) { darshan_core_unregister_module(DARSHAN_APSS_MOD); @@ -245,7 +268,6 @@ void apss_runtime_initialize() APSS_UNLOCK(); return; } - initialize_counters(); APSS_UNLOCK(); @@ -259,16 +281,17 @@ void apss_runtime_initialize() /* Pass data for the apss module back to darshan-core to log to file. */ //#ifdef HAVE_MPI static void apss_mpi_redux( - void *apss_buf, - MPI_Comm mod_comm, - darshan_record_id *shared_recs, - int shared_rec_count) + void *apss_buf, + MPI_Comm mod_comm, + darshan_record_id *shared_recs, + int shared_rec_count) { int i; int color; int router_rank; int router_count; int chassis_count; + int slot_count; int group_count; unsigned int *bitvec; unsigned int bitlen; @@ -287,20 +310,21 @@ static void apss_mpi_redux( bitlen = sizeof(unsigned int) * (MAX_BLADES/bitcnt); bitsiz = bitlen / sizeof(unsigned int); bitvec = malloc(bitlen); - + /* collect perf counters */ capture(apss_runtime->perf_record, apss_runtime->rtr_id); if (my_rank == 0) { - apss_runtime->header_record->appid = atoi((char*)getenv( csJOBID_ENV_STR )); + apss_runtime->header_record->appid = atoi((char*)getenv( APP_ID )); } -#if 0 +#if 1 /* count network dimensions */ if (bitvec) { int idx; unsigned int uchassis; + unsigned int uslot; unsigned int ublade; /* group */ @@ -308,46 +332,61 @@ static void apss_mpi_redux( idx = apss_runtime->group / bitcnt; bitvec[idx] |= (1 << apss_runtime->group % bitcnt); PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), - bitvec, - bitsiz, - MPI_INT, - MPI_BOR, - 0, - MPI_COMM_WORLD); + bitvec, + bitsiz, + MPI_INT, + MPI_BOR, + 0, + MPI_COMM_WORLD); group_count = count_bits(bitvec, bitsiz); /* chassis */ memset(bitvec, 0, bitlen); - uchassis = apss_runtime->group * 6 + apss_runtime->chassis; + uchassis = apss_runtime->group * 8 + apss_runtime->chassis; idx = uchassis / bitcnt; bitvec[idx] |= (1 << uchassis % bitcnt); PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), - bitvec, - bitsiz, - MPI_INT, - MPI_BOR, - 0, - MPI_COMM_WORLD); + bitvec, + bitsiz, + MPI_INT, + MPI_BOR, + 0, + MPI_COMM_WORLD); chassis_count = count_bits(bitvec, bitsiz); + /* slot */ + memset(bitvec, 0, bitlen); + uslot = uchassis * 8 + apss_runtime->slot; + idx = uslot / bitcnt; + bitvec[idx] |= (1 << uslot % bitcnt); + PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), + bitvec, + bitsiz, + MPI_INT, + MPI_BOR, + 0, + MPI_COMM_WORLD); + slot_count = count_bits(bitvec, bitsiz); + /* blade */ memset(bitvec, 0, bitlen); - ublade = uchassis * 16 + apss_runtime->blade; + ublade = uchassis * 8 + uslot * 8 + apss_runtime->blade; idx = ublade / bitcnt; bitvec[idx] |= (1 << ublade % bitcnt); PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), - bitvec, - bitsiz, - MPI_INT, - MPI_BOR, - 0, - MPI_COMM_WORLD); + bitvec, + bitsiz, + MPI_INT, + MPI_BOR, + 0, + MPI_COMM_WORLD); router_count = count_bits(bitvec, bitsiz); if (my_rank == 0) { apss_runtime->header_record->nblades = router_count; apss_runtime->header_record->nchassis = chassis_count; + apss_runtime->header_record->nslots = slot_count; apss_runtime->header_record->ngroups = group_count; } free(bitvec); @@ -355,63 +394,69 @@ static void apss_mpi_redux( else { apss_runtime->header_record->nblades = 0; + apss_runtime->header_record->nslots = 0; apss_runtime->header_record->nchassis = 0; apss_runtime->header_record->ngroups = 0; } - + /* * reduce data * - * aggregate data from processes which share the same blade and avg. + * aggregate data from processes which share the same node and avg. + * create a communicator with all the ranks that share the same node * */ - color = (apss_runtime->group << (4+3)) + \ - (apss_runtime->chassis << 4) + \ - apss_runtime->blade; + color = (apss_runtime->group << (7+3)) + \ + (apss_runtime->chassis << (3+4)) + \ + (apss_runtime->slot << (3+1)) + \ + (apss_runtime->blade << 1) + apss_runtime->node; +#endif +#if 1 PMPI_Comm_split(MPI_COMM_WORLD, color, my_rank, &router_comm); PMPI_Comm_rank(router_comm, &router_rank); PMPI_Comm_size(router_comm, &router_count); - - PMPI_Reduce((router_rank?apss_runtime->perf_record->counters:MPI_IN_PLACE), - apss_runtime->perf_record->counters, - APSS_NUM_INDICES, - MPI_LONG_LONG_INT, - MPI_SUM, - 0, - router_comm); - + /* + PMPI_Reduce((router_rank?apss_runtime->perf_record->counters:MPI_IN_PLACE), + apss_runtime->perf_record->counters, + APSS_NUM_INDICES, + MPI_LONG_LONG_INT, + MPI_SUM, + 0, + router_comm); + */ if (router_rank == 0) { - for (i = 0; i < APSS_NUM_INDICES; i++) - { - apss_runtime->perf_record->counters[i] /= router_count; - } - apss_runtime->perf_record_marked = -1; + /* + for (i = 0; i < APSS_NUM_INDICES; i++) + { + apss_runtime->perf_record->counters[i] /= router_count; + } + */ + apss_runtime->perf_record_marked = -1; } PMPI_Comm_free(&router_comm); - - APSS_UNLOCK(); #endif + APSS_UNLOCK(); return; } //#endif static void apss_output( - void **apss_buf, - int *apss_buf_sz) + void **apss_buf, + int *apss_buf_sz) { APSS_LOCK(); assert(apss_runtime); *apss_buf_sz = 0; - + if (my_rank == 0) { *apss_buf_sz += sizeof(*apss_runtime->header_record); } - + if (apss_runtime->perf_record_marked == -1) - { - *apss_buf_sz += sizeof( *apss_runtime->perf_record); - } + { + *apss_buf_sz += sizeof( *apss_runtime->perf_record); + } APSS_UNLOCK(); return; diff --git a/apss/util/darshan-apss-logutils.c b/apss/util/darshan-apss-logutils.c index 1291328..9f9115b 100644 --- a/apss/util/darshan-apss-logutils.c +++ b/apss/util/darshan-apss-logutils.c @@ -115,6 +115,7 @@ static int darshan_log_get_apss_rec(darshan_fd fd, void** buf_p) DARSHAN_BSWAP64(&(hdr_rec->base_rec.rank)); DARSHAN_BSWAP64(&(hdr_rec->magic)); DARSHAN_BSWAP64(&(hdr_rec->nblades)); + DARSHAN_BSWAP64(&(hdr_rec->nslots)); DARSHAN_BSWAP64(&(hdr_rec->nchassis)); DARSHAN_BSWAP64(&(hdr_rec->ngroups)); DARSHAN_BSWAP64(&(hdr_rec->appid)); @@ -126,6 +127,7 @@ static int darshan_log_get_apss_rec(darshan_fd fd, void** buf_p) DARSHAN_BSWAP64(&(prf_rec->base_rec.rank)); DARSHAN_BSWAP64(&(prf_rec->group)); DARSHAN_BSWAP64(&(prf_rec->chassis)); + DARSHAN_BSWAP64(&(prf_rec->slot)); DARSHAN_BSWAP64(&(prf_rec->blade)); DARSHAN_BSWAP64(&(prf_rec->node)); for (i = 0; i < APSS_NUM_INDICES; i++) @@ -192,6 +194,9 @@ static void darshan_log_print_apss_rec(void *rec, char *file_name, DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec->base_rec.rank, hdr_rec->base_rec.id, "APSS_CHASSIS", hdr_rec->nchassis, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + hdr_rec->base_rec.rank, hdr_rec->base_rec.id, + "APSS_SLOTS", hdr_rec->nslots, "", "", ""); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec->base_rec.rank, hdr_rec->base_rec.id, "APSS_BLADES", hdr_rec->nblades, "", "", ""); @@ -210,6 +215,9 @@ static void darshan_log_print_apss_rec(void *rec, char *file_name, DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec->base_rec.rank, prf_rec->base_rec.id, "APSS_CHASSIS", prf_rec->chassis, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + prf_rec->base_rec.rank, prf_rec->base_rec.id, + "APSS_SLOT", prf_rec->slot, "", "", ""); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec->base_rec.rank, prf_rec->base_rec.id, "APSS_BLADE", prf_rec->blade, "", "", ""); @@ -237,17 +245,19 @@ static void darshan_log_print_apss_description(int ver) printf("# APSS_GROUPS: total number of groups\n"); printf("# APSS_CHASSIS: total number of chassis\n"); printf("# APSS_BLADES: total number of blades\n"); + printf("# APSS_SLOTS: total number of slots\n"); printf("# per-router statistics for the apss module:\n"); printf("# APSS_GROUP: group this router is on\n"); printf("# APSS_CHASSIS: chassis this router is on\n"); + printf("# APSS_SLOT: slot in the chassis this router is on\n"); printf("# APSS_BLADE: blade this router is on\n"); printf("# APSS_NODE: node connected to this router\n"); printf("# APSS_AR_RTR_* port counters for the 40 router-router ports\n"); - printf("# APSS_AR_RTR_x_y_INQ_PRF_INCOMING_FLIT_VC[0-7]: flits on VCs of x y tile\n"); - printf("# APSS_AR_RTR_x_y_INQ_PRF_ROWBUS_STALL_CNT: stalls on x y tile\n"); - printf("# APSS_AR_RTR_PT_* port counters for the 8 router-nic ports\n"); - printf("# APSS_AR_RTR_PT_x_y_INQ_PRF_INCOMING_FLIT_VC[0,4]: flits on VCs of x y tile\n"); - printf("# APSS_AR_RTR_PT_x_y_INQ_PRF_REQ_ROWBUS_STALL_CNT: stalls on x y tile\n"); + //printf("# APSS_AR_RTR_x_y_INQ_PRF_INCOMING_FLIT_VC[0-7]: flits on VCs of x y tile\n"); + //printf("# APSS_AR_RTR_x_y_INQ_PRF_ROWBUS_STALL_CNT: stalls on x y tile\n"); + //printf("# APSS_AR_RTR_PT_* port counters for the 8 router-nic ports\n"); + //printf("# APSS_AR_RTR_PT_x_y_INQ_PRF_INCOMING_FLIT_VC[0,4]: flits on VCs of x y tile\n"); + //printf("# APSS_AR_RTR_PT_x_y_INQ_PRF_REQ_ROWBUS_STALL_CNT: stalls on x y tile\n"); return; } @@ -279,6 +289,10 @@ static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, "APSS_CHASSIS", hdr_rec1->nchassis, "", "", ""); printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APSS_SLOTS", hdr_rec1->nslots, "", "", ""); + printf("- "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, "APSS_BLADES", hdr_rec1->nblades, "", "", ""); @@ -298,6 +312,10 @@ static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, "APSS_CHASSIS", hdr_rec2->nchassis, "", "", ""); printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APSS_SLOTS", hdr_rec2->nslots, "", "", ""); + printf("+ "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, "APSS_BLADES", hdr_rec2->nblades, "", "", ""); @@ -330,6 +348,17 @@ static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, "APSS_CHASSIS", hdr_rec2->nchassis, "", "", ""); } + if (hdr_rec1->nslots != hdr_rec2->nslots) + { + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APSS_SLOTS", hdr_rec1->nslots, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APSS_SLOTS", hdr_rec2->nslots, "", "", ""); + } if (hdr_rec1->nblades != hdr_rec2->nblades) { printf("- "); @@ -369,6 +398,10 @@ static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, prf_rec1->base_rec.rank, prf_rec1->base_rec.id, "APSS_CHASSIS", prf_rec1->chassis, "", "", ""); printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APSS_SLOT", prf_rec1->slot, "", "", ""); + printf("- "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, "APSS_BLADE", prf_rec1->blade, "", "", ""); @@ -388,6 +421,10 @@ static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, prf_rec2->base_rec.rank, prf_rec2->base_rec.id, "APSS_CHASSIS", prf_rec2->chassis, "", "", ""); printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APSS_SLOT", prf_rec2->slot, "", "", ""); + printf("+ "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, "APSS_BLADE", prf_rec2->blade, "", "", ""); @@ -419,6 +456,17 @@ static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, prf_rec2->base_rec.rank, prf_rec2->base_rec.id, "APSS_CHASSIS", prf_rec2->chassis, "", "", ""); } + if (prf_rec1->slot != prf_rec2->slot) + { + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APSS_SLOT", prf_rec1->slot, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APSS_SLOT", prf_rec2->slot, "", "", ""); + } if (prf_rec1->blade != prf_rec2->blade) { printf("- "); From 800f4f64e84694a6cd653095b1b2613455783d9e Mon Sep 17 00:00:00 2001 From: Sudheer Chunduri Date: Fri, 11 Feb 2022 12:45:32 -0500 Subject: [PATCH 09/12] APSS: nodelist for crusher The node names on Crusher use a custom notation, hence a mapping from the custom node names to HPE specific node names is required. HPE specific code names can be used to decipher the topology details. --- apss/lib/crusher_nodelist.h | 193 ++++++++++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 apss/lib/crusher_nodelist.h diff --git a/apss/lib/crusher_nodelist.h b/apss/lib/crusher_nodelist.h new file mode 100644 index 0000000..c4ccf34 --- /dev/null +++ b/apss/lib/crusher_nodelist.h @@ -0,0 +1,193 @@ +char crusher_nodelist[192][14] = { +"x2601c0s0b0n0", +"x2601c0s0b1n0", +"x2601c0s1b0n0", +"x2601c0s1b1n0", +"x2601c0s2b0n0", +"x2601c0s2b1n0", +"x2601c0s3b0n0", +"x2601c0s3b1n0", +"x2601c0s4b0n0", +"x2601c0s4b1n0", +"x2601c0s5b0n0", +"x2601c0s5b1n0", +"x2601c0s6b0n0", +"x2601c0s6b1n0", +"x2601c0s7b0n0", +"x2601c0s7b1n0", +"x2601c1s0b0n0", +"x2601c1s0b1n0", +"x2601c1s1b0n0", +"x2601c1s1b1n0", +"x2601c1s2b0n0", +"x2601c1s2b1n0", +"x2601c1s3b0n0", +"x2601c1s3b1n0", +"x2601c1s4b0n0", +"x2601c1s4b1n0", +"x2601c1s5b0n0", +"x2601c1s5b1n0", +"x2601c1s6b0n0", +"x2601c1s6b1n0", +"x2601c1s7b0n0", +"x2601c1s7b1n0", +"x2601c2s0b0n0", +"x2601c2s0b1n0", +"x2601c2s1b0n0", +"x2601c2s1b1n0", +"x2601c2s2b0n0", +"x2601c2s2b1n0", +"x2601c2s3b0n0", +"x2601c2s3b1n0", +"x2601c2s4b0n0", +"x2601c2s4b1n0", +"x2601c2s5b0n0", +"x2601c2s5b1n0", +"x2601c2s6b0n0", +"x2601c2s6b1n0", +"x2601c2s7b0n0", +"x2601c2s7b1n0", +"x2601c3s0b0n0", +"x2601c3s0b1n0", +"x2601c3s1b0n0", +"x2601c3s1b1n0", +"x2601c3s2b0n0", +"x2601c3s2b1n0", +"x2601c3s3b0n0", +"x2601c3s3b1n0", +"x2601c3s4b0n0", +"x2601c3s4b1n0", +"x2601c3s5b0n0", +"x2601c3s5b1n0", +"x2601c3s6b0n0", +"x2601c3s6b1n0", +"x2601c3s7b0n0", +"x2601c3s7b1n0", +"x2601c4s0b0n0", +"x2601c4s0b1n0", +"x2601c4s1b0n0", +"x2601c4s1b1n0", +"x2601c4s2b0n0", +"x2601c4s2b1n0", +"x2601c4s3b0n0", +"x2601c4s3b1n0", +"x2601c4s4b0n0", +"x2601c4s4b1n0", +"x2601c4s5b0n0", +"x2601c4s5b1n0", +"x2601c4s6b0n0", +"x2601c4s6b1n0", +"x2601c4s7b0n0", +"x2601c4s7b1n0", +"x2601c5s0b0n0", +"x2601c5s0b1n0", +"x2601c5s1b0n0", +"x2601c5s1b1n0", +"x2601c5s2b0n0", +"x2601c5s2b1n0", +"x2601c5s3b0n0", +"x2601c5s3b1n0", +"x2601c5s4b0n0", +"x2601c5s4b1n0", +"x2601c5s5b0n0", +"x2601c5s5b1n0", +"x2601c5s6b0n0", +"x2601c5s6b1n0", +"x2601c5s7b0n0", +"x2601c5s7b1n0", +"x2601c6s0b0n0", +"x2601c6s0b1n0", +"x2601c6s1b0n0", +"x2601c6s1b1n0", +"x2601c6s2b0n0", +"x2601c6s2b1n0", +"x2601c6s3b0n0", +"x2601c6s3b1n0", +"x2601c6s4b0n0", +"x2601c6s4b1n0", +"x2601c6s5b0n0", +"x2601c6s5b1n0", +"x2601c6s6b0n0", +"x2601c6s6b1n0", +"x2601c6s7b0n0", +"x2601c6s7b1n0", +"x2601c7s0b0n0", +"x2601c7s0b1n0", +"x2601c7s1b0n0", +"x2601c7s1b1n0", +"x2601c7s2b0n0", +"x2601c7s2b1n0", +"x2601c7s3b0n0", +"x2601c7s3b1n0", +"x2601c7s4b0n0", +"x2601c7s4b1n0", +"x2601c7s5b0n0", +"x2601c7s5b1n0", +"x2601c7s6b0n0", +"x2601c7s6b1n0", +"x2601c7s7b0n0", +"x2601c7s7b1n0", +"x2602c0s0b0n0", +"x2602c0s0b1n0", +"x2602c0s1b0n0", +"x2602c0s1b1n0", +"x2602c0s2b0n0", +"x2602c0s2b1n0", +"x2602c0s3b0n0", +"x2602c0s3b1n0", +"x2602c0s4b0n0", +"x2602c0s4b1n0", +"x2602c0s5b0n0", +"x2602c0s5b1n0", +"x2602c0s6b0n0", +"x2602c0s6b1n0", +"x2602c0s7b0n0", +"x2602c0s7b1n0", +"x2602c1s0b0n0", +"x2602c1s0b1n0", +"x2602c1s1b0n0", +"x2602c1s1b1n0", +"x2602c1s2b0n0", +"x2602c1s2b1n0", +"x2602c1s3b0n0", +"x2602c1s3b1n0", +"x2602c1s4b0n0", +"x2602c1s4b1n0", +"x2602c1s5b0n0", +"x2602c1s5b1n0", +"x2602c1s6b0n0", +"x2602c1s6b1n0", +"x2602c1s7b0n0", +"x2602c1s7b1n0", +"x2602c2s0b0n0", +"x2602c2s0b1n0", +"x2602c2s1b0n0", +"x2602c2s1b1n0", +"x2602c2s2b0n0", +"x2602c2s2b1n0", +"x2602c2s3b0n0", +"x2602c2s3b1n0", +"x2602c2s4b0n0", +"x2602c2s4b1n0", +"x2602c2s5b0n0", +"x2602c2s5b1n0", +"x2602c2s6b0n0", +"x2602c2s6b1n0", +"x2602c2s7b0n0", +"x2602c2s7b1n0", +"x2602c3s0b0n0", +"x2602c3s0b1n0", +"x2602c3s1b0n0", +"x2602c3s1b1n0", +"x2602c3s2b0n0", +"x2602c3s2b1n0", +"x2602c3s3b0n0", +"x2602c3s3b1n0", +"x2602c3s4b0n0", +"x2602c3s4b1n0", +"x2602c3s5b0n0", +"x2602c3s5b1n0", +"x2602c3s6b0n0", +"x2602c3s6b1n0", +"x2602c3s7b0n0", +"x2602c3s7b1n0"}; From 1a03b1a19ca1c03651dcc2bfe8af741955832970 Mon Sep 17 00:00:00 2001 From: Sudheer Chunduri Date: Fri, 11 Feb 2022 13:26:10 -0500 Subject: [PATCH 10/12] apcxi: rename the module for slingshot NIC module Rename the HPE slingshot Cassini nic performance counter module as apcxi. --- apcxi/Makefile.darshan | 20 + {apss => apcxi}/darshan-apss-log-format.h | 28 +- {apss => apcxi}/lib/crusher_nodelist.h | 0 {apss => apcxi}/lib/darshan-apss-utils.h | 4 +- {apss => apcxi}/lib/darshan-apss.c | 266 +++++++------- apcxi/util/Makefile.darshan | 27 ++ {apss => apcxi}/util/apss-analysis.py | 14 +- {apss => apcxi}/util/apss-backend.py | 14 +- {apss => apcxi}/util/darshan-apss-logutils.c | 366 +++++++++---------- apcxi/util/darshan-apss-logutils.h | 15 + apcxi/util/print-apss-size.c | 12 + apss/Makefile.darshan | 20 - apss/util/Makefile.darshan | 27 -- apss/util/darshan-apss-logutils.h | 15 - apss/util/print-apss-size.c | 12 - 15 files changed, 420 insertions(+), 420 deletions(-) create mode 100644 apcxi/Makefile.darshan rename {apss => apcxi}/darshan-apss-log-format.h (83%) rename {apss => apcxi}/lib/crusher_nodelist.h (100%) rename {apss => apcxi}/lib/darshan-apss-utils.h (96%) rename {apss => apcxi}/lib/darshan-apss.c (52%) create mode 100644 apcxi/util/Makefile.darshan rename {apss => apcxi}/util/apss-analysis.py (76%) rename {apss => apcxi}/util/apss-backend.py (85%) rename {apss => apcxi}/util/darshan-apss-logutils.c (57%) create mode 100644 apcxi/util/darshan-apss-logutils.h create mode 100644 apcxi/util/print-apss-size.c delete mode 100644 apss/Makefile.darshan delete mode 100644 apss/util/Makefile.darshan delete mode 100644 apss/util/darshan-apss-logutils.h delete mode 100644 apss/util/print-apss-size.c diff --git a/apcxi/Makefile.darshan b/apcxi/Makefile.darshan new file mode 100644 index 0000000..aeb9981 --- /dev/null +++ b/apcxi/Makefile.darshan @@ -0,0 +1,20 @@ +# +# AutoPerf Make rules for Darshan +# +DARSHAN_STATIC_MOD_OBJS += lib/darshan-apcxi.o +DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-apcxi.po + +VPATH += :$(srcdir)/../modules/autoperf/apcxi +CFLAGS += \ + -DDARSHAN_USE_APCXI \ + -I$(srcdir)/../modules/autoperf/apcxi + +CFLAGS_SHARED += \ + -DDARSHAN_USE_APCXI \ + -I$(srcdir)/../modules/autoperf/apcxi + +lib/darshan-apcxi.o: lib/darshan-apcxi.c lib/darshan-apcxi-utils.h darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) darshan-apcxi-log-format.h | lib + $(CC) $(CFLAGS) -c $< -o $@ + +lib/darshan-apcxi.po: lib/darshan-apcxi.c lib/darshan-apcxi-utils.h darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) darshan-apcxi-log-format.h | lib + $(CC) $(CFLAGS_SHARED) -c $< -o $@ diff --git a/apss/darshan-apss-log-format.h b/apcxi/darshan-apss-log-format.h similarity index 83% rename from apss/darshan-apss-log-format.h rename to apcxi/darshan-apss-log-format.h index 8e12eb2..e54c25e 100644 --- a/apss/darshan-apss-log-format.h +++ b/apcxi/darshan-apss-log-format.h @@ -4,13 +4,13 @@ * */ -#ifndef __APSS_LOG_FORMAT_H -#define __APSS_LOG_FORMAT_H +#ifndef __APCXI_LOG_FORMAT_H +#define __APCXI_LOG_FORMAT_H /* current AutoPerf Cray XC log format version */ -#define APSS_VER 1 +#define APCXI_VER 1 -#define APSS_MAGIC ('A'*0x100000000000000+\ +#define APCXI_MAGIC ('A'*0x100000000000000+\ 'U'*0x1000000000000+\ 'T'*0x10000000000+\ 'O'*0x100000000+\ @@ -19,7 +19,7 @@ 'R'*0x100+\ 'F'*0x1) -#define APSS_PERF_COUNTERS \ +#define APCXI_PERF_COUNTERS \ /* PAPI counters */\ X(CQ_CQ_OXE_NUM_STALLS) \ X(CQ_CQ_OXE_NUM_FLITS) \ @@ -72,21 +72,21 @@ X(HNI_TX_OK_4096_TO_8191) \ X(HNI_TX_OK_8192_TO_MAX) \ /* end of counters */\ - Z(APSS_NUM_INDICES) + Z(APCXI_NUM_INDICES) #define X(a) a, #define Z(a) a -/* integer counters for the "APSS" example module */ -enum darshan_apss_perf_indices +/* integer counters for the "APCXI" example module */ +enum darshan_apcxi_perf_indices { - APSS_PERF_COUNTERS + APCXI_PERF_COUNTERS }; #undef Z #undef X -/* the darshan_apss_router_record structure encompasses the data/counters +/* the darshan_apcxi_router_record structure encompasses the data/counters * which would actually be logged to file by Darshan for the AP HPE Slingshot * module. This example implementation logs the following data for each * record: @@ -94,7 +94,7 @@ enum darshan_apss_perf_indices * - integer I/O counters * - floating point I/O counters */ -struct darshan_apss_perf_record +struct darshan_apcxi_perf_record { struct darshan_base_record base_rec; int64_t group; @@ -102,10 +102,10 @@ struct darshan_apss_perf_record int64_t slot; int64_t blade; int64_t node; - uint64_t counters[APSS_NUM_INDICES]; + uint64_t counters[APCXI_NUM_INDICES]; }; -struct darshan_apss_header_record +struct darshan_apcxi_header_record { struct darshan_base_record base_rec; int64_t magic; @@ -116,4 +116,4 @@ struct darshan_apss_header_record uint64_t appid; }; -#endif /* __APSS_LOG_FORMAT_H */ +#endif /* __APCXI_LOG_FORMAT_H */ diff --git a/apss/lib/crusher_nodelist.h b/apcxi/lib/crusher_nodelist.h similarity index 100% rename from apss/lib/crusher_nodelist.h rename to apcxi/lib/crusher_nodelist.h diff --git a/apss/lib/darshan-apss-utils.h b/apcxi/lib/darshan-apss-utils.h similarity index 96% rename from apss/lib/darshan-apss-utils.h rename to apcxi/lib/darshan-apss-utils.h index f4bc7fd..b82394c 100644 --- a/apss/lib/darshan-apss-utils.h +++ b/apcxi/lib/darshan-apss-utils.h @@ -1,5 +1,5 @@ -#ifndef __APSS_UTILS_H__ -#define __APSS_UTILS_H__ +#ifndef __APCXI_UTILS_H__ +#define __APCXI_UTILS_H__ #include extern char crusher_nodelist[192][14]; diff --git a/apss/lib/darshan-apss.c b/apcxi/lib/darshan-apss.c similarity index 52% rename from apss/lib/darshan-apss.c rename to apcxi/lib/darshan-apss.c index f98bfbe..caca68b 100644 --- a/apss/lib/darshan-apss.c +++ b/apcxi/lib/darshan-apss.c @@ -24,9 +24,9 @@ #include "uthash.h" #include "darshan.h" #include "darshan-dynamic.h" -#include "darshan-apss-log-format.h" +#include "darshan-apcxi-log-format.h" -#include "darshan-apss-utils.h" +#include "darshan-apcxi-utils.h" #include "crusher_nodelist.h" /* @@ -36,7 +36,7 @@ #define Z(a) #a static char* PAPI_events[] = { - APSS_PERF_COUNTERS + APCXI_PERF_COUNTERS }; #undef X #undef Z @@ -57,10 +57,10 @@ static char* PAPI_events[] = /* * Global runtime struct for tracking data needed at runtime */ -struct apss_runtime +struct apcxi_runtime { - struct darshan_apss_header_record *header_record; - struct darshan_apss_perf_record *perf_record; + struct darshan_apcxi_header_record *header_record; + struct darshan_apcxi_perf_record *perf_record; darshan_record_id header_id; darshan_record_id rtr_id; int PAPI_event_set; @@ -73,32 +73,32 @@ struct apss_runtime int perf_record_marked; }; -static struct apss_runtime *apss_runtime = NULL; -static pthread_mutex_t apss_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; +static struct apcxi_runtime *apcxi_runtime = NULL; +static pthread_mutex_t apcxi_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; /* my_rank indicates the MPI rank of this process */ static int my_rank = -1; -/* internal helper functions for the APSS module */ -void apss_runtime_initialize(void); +/* internal helper functions for the APCXI module */ +void apcxi_runtime_initialize(void); /* forward declaration for shutdown function needed to interface with darshan-core */ //#ifdef HAVE_MPI -static void apss_mpi_redux( +static void apcxi_mpi_redux( void *buffer, MPI_Comm mod_comm, darshan_record_id *shared_recs, int shared_rec_count); //#endif -static void apss_output( +static void apcxi_output( void **buffer, int *size); -static void apss_cleanup( +static void apcxi_cleanup( void); -/* macros for obtaining/releasing the APSS module lock */ -#define APSS_LOCK() pthread_mutex_lock(&apss_runtime_mutex) -#define APSS_UNLOCK() pthread_mutex_unlock(&apss_runtime_mutex) +/* macros for obtaining/releasing the APCXI module lock */ +#define APCXI_LOCK() pthread_mutex_lock(&apcxi_runtime_mutex) +#define APCXI_UNLOCK() pthread_mutex_unlock(&apcxi_runtime_mutex) /* * Initialize counters using PAPI @@ -109,8 +109,8 @@ static void initialize_counters (void) int code = 0; PAPI_library_init(PAPI_VER_CURRENT); - apss_runtime->PAPI_event_set = PAPI_NULL; - PAPI_create_eventset(&apss_runtime->PAPI_event_set); + apcxi_runtime->PAPI_event_set = PAPI_NULL; + PAPI_create_eventset(&apcxi_runtime->PAPI_event_set); #if 0 /* Determine the number of NIC devices - read the number of dirs in /sys/class/cxi/ */ int num_nics = 0; @@ -130,24 +130,24 @@ static void initialize_counters (void) #endif /* start with first PAPI counter */ for (i = CQ_CQ_OXE_NUM_STALLS; - strcmp(PAPI_events[i], "APSS_NUM_INDICES") != 0; + strcmp(PAPI_events[i], "APCXI_NUM_INDICES") != 0; i++) { PAPI_event_name_to_code(PAPI_events[i], &code); - PAPI_add_event(apss_runtime->PAPI_event_set, code); + PAPI_add_event(apcxi_runtime->PAPI_event_set, code); } - apss_runtime->PAPI_event_count = i; + apcxi_runtime->PAPI_event_count = i; - PAPI_start(apss_runtime->PAPI_event_set); + PAPI_start(apcxi_runtime->PAPI_event_set); return; } static void finalize_counters (void) { - PAPI_cleanup_eventset(apss_runtime->PAPI_event_set); - PAPI_destroy_eventset(&apss_runtime->PAPI_event_set); + PAPI_cleanup_eventset(apcxi_runtime->PAPI_event_set); + PAPI_destroy_eventset(&apcxi_runtime->PAPI_event_set); PAPI_shutdown(); return; @@ -156,120 +156,120 @@ static void finalize_counters (void) /* * Function which updates all the counter data */ -static void capture(struct darshan_apss_perf_record *rec, +static void capture(struct darshan_apcxi_perf_record *rec, darshan_record_id rec_id) { - PAPI_stop(apss_runtime->PAPI_event_set, + PAPI_stop(apcxi_runtime->PAPI_event_set, (long long*) &rec->counters[CQ_CQ_OXE_NUM_STALLS]); - PAPI_reset(apss_runtime->PAPI_event_set); + PAPI_reset(apcxi_runtime->PAPI_event_set); - rec->group = apss_runtime->group; - rec->chassis = apss_runtime->chassis; - rec->slot = apss_runtime->slot; - rec->blade = apss_runtime->blade; - rec->node = apss_runtime->node; + rec->group = apcxi_runtime->group; + rec->chassis = apcxi_runtime->chassis; + rec->slot = apcxi_runtime->slot; + rec->blade = apcxi_runtime->blade; + rec->node = apcxi_runtime->node; rec->base_rec.id = rec_id; rec->base_rec.rank = my_rank; return; } -void apss_runtime_initialize() +void apcxi_runtime_initialize() { - size_t apss_buf_size; + size_t apcxi_buf_size; char rtr_rec_name[128]; darshan_module_funcs mod_funcs = { //#ifdef HAVE_MPI - .mod_redux_func = &apss_mpi_redux, + .mod_redux_func = &apcxi_mpi_redux, //#endif - .mod_output_func = &apss_output, - .mod_cleanup_func = &apss_cleanup + .mod_output_func = &apcxi_output, + .mod_cleanup_func = &apcxi_cleanup }; - APSS_LOCK(); + APCXI_LOCK(); /* don't do anything if already initialized */ - if(apss_runtime) + if(apcxi_runtime) { - APSS_UNLOCK(); + APCXI_UNLOCK(); return; } - apss_buf_size = sizeof(struct darshan_apss_header_record) + - sizeof(struct darshan_apss_perf_record); + apcxi_buf_size = sizeof(struct darshan_apcxi_header_record) + + sizeof(struct darshan_apcxi_perf_record); - /* register the APSS module with the darshan-core component */ + /* register the APCXI module with the darshan-core component */ darshan_core_register_module( - DARSHAN_APSS_MOD, + DARSHAN_APCXI_MOD, mod_funcs, - &apss_buf_size, + &apcxi_buf_size, &my_rank, NULL); /* initialize module's global state */ - apss_runtime = malloc(sizeof(*apss_runtime)); - if(!apss_runtime) + apcxi_runtime = malloc(sizeof(*apcxi_runtime)); + if(!apcxi_runtime) { - darshan_core_unregister_module(DARSHAN_APSS_MOD); - APSS_UNLOCK(); + darshan_core_unregister_module(DARSHAN_APCXI_MOD); + APCXI_UNLOCK(); return; } - memset(apss_runtime, 0, sizeof(*apss_runtime)); + memset(apcxi_runtime, 0, sizeof(*apcxi_runtime)); if (my_rank == 0) { - apss_runtime->header_id = darshan_core_gen_record_id("darshan-apss-header"); + apcxi_runtime->header_id = darshan_core_gen_record_id("darshan-apcxi-header"); - /* register the apss file record with darshan-core */ - apss_runtime->header_record = darshan_core_register_record( - apss_runtime->header_id, + /* register the apcxi file record with darshan-core */ + apcxi_runtime->header_record = darshan_core_register_record( + apcxi_runtime->header_id, //NULL, - "darshan-apss-header", - DARSHAN_APSS_MOD, - sizeof(struct darshan_apss_header_record), + "darshan-apcxi-header", + DARSHAN_APCXI_MOD, + sizeof(struct darshan_apcxi_header_record), NULL); - if(!(apss_runtime->header_record)) + if(!(apcxi_runtime->header_record)) { - darshan_core_unregister_module(DARSHAN_APSS_MOD); - free(apss_runtime); - apss_runtime = NULL; - APSS_UNLOCK(); + darshan_core_unregister_module(DARSHAN_APCXI_MOD); + free(apcxi_runtime); + apcxi_runtime = NULL; + APCXI_UNLOCK(); return; } - apss_runtime->header_record->base_rec.id = apss_runtime->header_id; - apss_runtime->header_record->base_rec.rank = my_rank; - apss_runtime->header_record->magic = APSS_MAGIC; + apcxi_runtime->header_record->base_rec.id = apcxi_runtime->header_id; + apcxi_runtime->header_record->base_rec.rank = my_rank; + apcxi_runtime->header_record->magic = APCXI_MAGIC; } - sstopo_get_mycoords(&apss_runtime->group, - &apss_runtime->chassis, - &apss_runtime->slot, - &apss_runtime->blade, - &apss_runtime->node); - sprintf(rtr_rec_name, "darshan-apss-rtr-%d-%d-%d-%d", - apss_runtime->group, apss_runtime->chassis, apss_runtime->slot, apss_runtime->blade); - //apss_runtime->rtr_id = darshan_core_gen_record_id(rtr_rec_name); - apss_runtime->rtr_id = darshan_core_gen_record_id("APSS"); - apss_runtime->perf_record = darshan_core_register_record( - apss_runtime->rtr_id, + sstopo_get_mycoords(&apcxi_runtime->group, + &apcxi_runtime->chassis, + &apcxi_runtime->slot, + &apcxi_runtime->blade, + &apcxi_runtime->node); + sprintf(rtr_rec_name, "darshan-apcxi-rtr-%d-%d-%d-%d", + apcxi_runtime->group, apcxi_runtime->chassis, apcxi_runtime->slot, apcxi_runtime->blade); + //apcxi_runtime->rtr_id = darshan_core_gen_record_id(rtr_rec_name); + apcxi_runtime->rtr_id = darshan_core_gen_record_id("APCXI"); + apcxi_runtime->perf_record = darshan_core_register_record( + apcxi_runtime->rtr_id, //NULL, - "APSS", // we want the record for each rank to be treated as shared records so that mpi_redux can operate on + "APCXI", // we want the record for each rank to be treated as shared records so that mpi_redux can operate on //rtr_rec_name, - DARSHAN_APSS_MOD, - sizeof(struct darshan_apss_perf_record), + DARSHAN_APCXI_MOD, + sizeof(struct darshan_apcxi_perf_record), NULL); - if(!(apss_runtime->perf_record)) + if(!(apcxi_runtime->perf_record)) { - darshan_core_unregister_module(DARSHAN_APSS_MOD); - free(apss_runtime); - apss_runtime = NULL; - APSS_UNLOCK(); + darshan_core_unregister_module(DARSHAN_APCXI_MOD); + free(apcxi_runtime); + apcxi_runtime = NULL; + APCXI_UNLOCK(); return; } initialize_counters(); - APSS_UNLOCK(); + APCXI_UNLOCK(); return; } @@ -278,10 +278,10 @@ void apss_runtime_initialize() * shutdown function exported by this module for coordinating with darshan-core * ********************************************************************************/ -/* Pass data for the apss module back to darshan-core to log to file. */ +/* Pass data for the apcxi module back to darshan-core to log to file. */ //#ifdef HAVE_MPI -static void apss_mpi_redux( - void *apss_buf, +static void apcxi_mpi_redux( + void *apcxi_buf, MPI_Comm mod_comm, darshan_record_id *shared_recs, int shared_rec_count) @@ -299,10 +299,10 @@ static void apss_mpi_redux( unsigned int bitsiz; MPI_Comm router_comm; - APSS_LOCK(); - if (!apss_runtime) + APCXI_LOCK(); + if (!apcxi_runtime) { - APSS_UNLOCK(); + APCXI_UNLOCK(); return; } @@ -312,11 +312,11 @@ static void apss_mpi_redux( bitvec = malloc(bitlen); /* collect perf counters */ - capture(apss_runtime->perf_record, apss_runtime->rtr_id); + capture(apcxi_runtime->perf_record, apcxi_runtime->rtr_id); if (my_rank == 0) { - apss_runtime->header_record->appid = atoi((char*)getenv( APP_ID )); + apcxi_runtime->header_record->appid = atoi((char*)getenv( APP_ID )); } #if 1 /* count network dimensions */ @@ -329,8 +329,8 @@ static void apss_mpi_redux( /* group */ memset(bitvec, 0, bitlen); - idx = apss_runtime->group / bitcnt; - bitvec[idx] |= (1 << apss_runtime->group % bitcnt); + idx = apcxi_runtime->group / bitcnt; + bitvec[idx] |= (1 << apcxi_runtime->group % bitcnt); PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), bitvec, bitsiz, @@ -342,7 +342,7 @@ static void apss_mpi_redux( /* chassis */ memset(bitvec, 0, bitlen); - uchassis = apss_runtime->group * 8 + apss_runtime->chassis; + uchassis = apcxi_runtime->group * 8 + apcxi_runtime->chassis; idx = uchassis / bitcnt; bitvec[idx] |= (1 << uchassis % bitcnt); PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), @@ -356,7 +356,7 @@ static void apss_mpi_redux( /* slot */ memset(bitvec, 0, bitlen); - uslot = uchassis * 8 + apss_runtime->slot; + uslot = uchassis * 8 + apcxi_runtime->slot; idx = uslot / bitcnt; bitvec[idx] |= (1 << uslot % bitcnt); PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), @@ -370,7 +370,7 @@ static void apss_mpi_redux( /* blade */ memset(bitvec, 0, bitlen); - ublade = uchassis * 8 + uslot * 8 + apss_runtime->blade; + ublade = uchassis * 8 + uslot * 8 + apcxi_runtime->blade; idx = ublade / bitcnt; bitvec[idx] |= (1 << ublade % bitcnt); PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), @@ -384,19 +384,19 @@ static void apss_mpi_redux( if (my_rank == 0) { - apss_runtime->header_record->nblades = router_count; - apss_runtime->header_record->nchassis = chassis_count; - apss_runtime->header_record->nslots = slot_count; - apss_runtime->header_record->ngroups = group_count; + apcxi_runtime->header_record->nblades = router_count; + apcxi_runtime->header_record->nchassis = chassis_count; + apcxi_runtime->header_record->nslots = slot_count; + apcxi_runtime->header_record->ngroups = group_count; } free(bitvec); } else { - apss_runtime->header_record->nblades = 0; - apss_runtime->header_record->nslots = 0; - apss_runtime->header_record->nchassis = 0; - apss_runtime->header_record->ngroups = 0; + apcxi_runtime->header_record->nblades = 0; + apcxi_runtime->header_record->nslots = 0; + apcxi_runtime->header_record->nchassis = 0; + apcxi_runtime->header_record->ngroups = 0; } /* @@ -406,19 +406,19 @@ static void apss_mpi_redux( * create a communicator with all the ranks that share the same node * */ - color = (apss_runtime->group << (7+3)) + \ - (apss_runtime->chassis << (3+4)) + \ - (apss_runtime->slot << (3+1)) + \ - (apss_runtime->blade << 1) + apss_runtime->node; + color = (apcxi_runtime->group << (7+3)) + \ + (apcxi_runtime->chassis << (3+4)) + \ + (apcxi_runtime->slot << (3+1)) + \ + (apcxi_runtime->blade << 1) + apcxi_runtime->node; #endif #if 1 PMPI_Comm_split(MPI_COMM_WORLD, color, my_rank, &router_comm); PMPI_Comm_rank(router_comm, &router_rank); PMPI_Comm_size(router_comm, &router_count); /* - PMPI_Reduce((router_rank?apss_runtime->perf_record->counters:MPI_IN_PLACE), - apss_runtime->perf_record->counters, - APSS_NUM_INDICES, + PMPI_Reduce((router_rank?apcxi_runtime->perf_record->counters:MPI_IN_PLACE), + apcxi_runtime->perf_record->counters, + APCXI_NUM_INDICES, MPI_LONG_LONG_INT, MPI_SUM, 0, @@ -427,49 +427,49 @@ static void apss_mpi_redux( if (router_rank == 0) { /* - for (i = 0; i < APSS_NUM_INDICES; i++) + for (i = 0; i < APCXI_NUM_INDICES; i++) { - apss_runtime->perf_record->counters[i] /= router_count; + apcxi_runtime->perf_record->counters[i] /= router_count; } */ - apss_runtime->perf_record_marked = -1; + apcxi_runtime->perf_record_marked = -1; } PMPI_Comm_free(&router_comm); #endif - APSS_UNLOCK(); + APCXI_UNLOCK(); return; } //#endif -static void apss_output( - void **apss_buf, - int *apss_buf_sz) +static void apcxi_output( + void **apcxi_buf, + int *apcxi_buf_sz) { - APSS_LOCK(); - assert(apss_runtime); - *apss_buf_sz = 0; + APCXI_LOCK(); + assert(apcxi_runtime); + *apcxi_buf_sz = 0; if (my_rank == 0) { - *apss_buf_sz += sizeof(*apss_runtime->header_record); + *apcxi_buf_sz += sizeof(*apcxi_runtime->header_record); } - if (apss_runtime->perf_record_marked == -1) + if (apcxi_runtime->perf_record_marked == -1) { - *apss_buf_sz += sizeof( *apss_runtime->perf_record); + *apcxi_buf_sz += sizeof( *apcxi_runtime->perf_record); } - APSS_UNLOCK(); + APCXI_UNLOCK(); return; } -static void apss_cleanup() +static void apcxi_cleanup() { - APSS_LOCK(); - assert(apss_runtime); + APCXI_LOCK(); + assert(apcxi_runtime); finalize_counters(); - free(apss_runtime); - apss_runtime = NULL; - APSS_UNLOCK(); + free(apcxi_runtime); + apcxi_runtime = NULL; + APCXI_UNLOCK(); return; } diff --git a/apcxi/util/Makefile.darshan b/apcxi/util/Makefile.darshan new file mode 100644 index 0000000..3953b88 --- /dev/null +++ b/apcxi/util/Makefile.darshan @@ -0,0 +1,27 @@ +# +# AutoPerf Make rules for Darshan +# + +VPATH += :$(srcdir)/../modules/autoperf/apcxi/util + +DARSHAN_MOD_LOG_FORMATS += $(srcdir)/../modules/autoperf/apcxi/darshan-apcxi-log-format.h +DARSHAN_MOD_LOGUTIL_HEADERS += darshan-apcxi-logutils.h +DARSHAN_STATIC_MOD_OBJS += darshan-apcxi-logutils.o +DARSHAN_DYNAMIC_MOD_OBJS += darshan-apcxi-logutils.po + +CFLAGS += \ + -DDARSHAN_USE_apcxi \ + -I$(srcdir)/../modules/autoperf/apcxi \ + -I$(srcdir)/../modules/autoperf/apcxi/util + +CFLAGS_SHARED += \ + -DDARSHAN_USE_apcxi \ + -I$(srcdir)/../modules/autoperf/apcxi \ + -I$(srcdir)/../modules/autoperf/apcxi/util + +darshan-apcxi-logutils.o: darshan-apcxi-logutils.c darshan-logutils.h darshan-apcxi-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../modules/autoperf/apcxi/darshan-apcxi-log-format.h | uthash-1.9.2 + $(CC) $(CFLAGS) -c $< -o $@ + +darshan-apcxi-logutils.po: darshan-apcxi-logutils.c darshan-logutils.h darshan-apcxi-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../modules/autoperf/apcxi/darshan-apcxi-log-format.h | uthash-1.9.2 + $(CC) $(CFLAGS_SHARED) -c $< -o $@ + diff --git a/apss/util/apss-analysis.py b/apcxi/util/apss-analysis.py similarity index 76% rename from apss/util/apss-analysis.py rename to apcxi/util/apss-analysis.py index 854562e..e02eda2 100755 --- a/apss/util/apss-analysis.py +++ b/apcxi/util/apss-analysis.py @@ -1,9 +1,9 @@ #!/usr/bin/env python -# DarshanUtils for Python for processing APSS records +# DarshanUtils for Python for processing APCXI records # This script gives an overview of features provided by the Python bindings for DarshanUtils. -# By default all APSS module records, metadata, and the name records are loaded when opening a Darshan log: +# By default all APCXI module records, metadata, and the name records are loaded when opening a Darshan log: import argparse import darshan @@ -48,19 +48,19 @@ def main(): report = darshan.DarshanReport(args.logname[0], read_all=False) report.info() - if "APSS" not in report.modules: + if "APCXI" not in report.modules: print("This log does not contain AutoPerf XC data") return - r = report.mod_read_all_apss_records("APSS") + r = report.mod_read_all_apcxi_records("APCXI") report.update_name_records() report.info() - #pdf = matplotlib.backends.backend_pdf.PdfPages("apss_output.pdf") + #pdf = matplotlib.backends.backend_pdf.PdfPages("apcxi_output.pdf") - header_rec = report.records["APSS"][0] + header_rec = report.records["APCXI"][0] - for rec in report.records["APSS"][1:]: + for rec in report.records["APCXI"][1:]: # skip the first record which is header record print(rec) diff --git a/apss/util/apss-backend.py b/apcxi/util/apss-backend.py similarity index 85% rename from apss/util/apss-backend.py rename to apcxi/util/apss-backend.py index d6dbc2f..a18bf83 100644 --- a/apss/util/apss-backend.py +++ b/apcxi/util/apss-backend.py @@ -4,9 +4,9 @@ import numpy as np import darshan.backend.cffi_backend -# apss structure defs +# apcxi structure defs structdefs = ''' -struct darshan_apss_perf_record +struct darshan_apcxi_perf_record { struct darshan_base_record base_rec; int64_t group; @@ -15,7 +15,7 @@ int64_t node; uint64_t counters[10]; }; -struct darshan_apss_header_record +struct darshan_apcxi_header_record { struct darshan_base_record base_rec; int64_t magic; @@ -25,16 +25,16 @@ uint64_t appid; }; -extern char *apss_counter_names[]; +extern char *apcxi_counter_names[]; ''' -def get_apss_defs(): +def get_apcxi_defs(): return structdefs # load header record -def log_get_apss_record(log, mod_name, structname, dtype='dict'): +def log_get_apcxi_record(log, mod_name, structname, dtype='dict'): from darshan.backend.cffi_backend import ffi, libdutil, log_get_modules, counter_names, _structdefs modules = log_get_modules(log) @@ -47,7 +47,7 @@ def log_get_apss_record(log, mod_name, structname, dtype='dict'): if r < 1: return None - if mod_type == 'struct darshan_apss_header_record **': + if mod_type == 'struct darshan_apcxi_header_record **': hdr = ffi.cast(mod_type, buf) rec['id'] = hdr[0].base_rec.id rec['rank'] = hdr[0].base_rec.rank diff --git a/apss/util/darshan-apss-logutils.c b/apcxi/util/darshan-apss-logutils.c similarity index 57% rename from apss/util/darshan-apss-logutils.c rename to apcxi/util/darshan-apss-logutils.c index 9f9115b..9fc80b3 100644 --- a/apss/util/darshan-apss-logutils.c +++ b/apcxi/util/darshan-apss-logutils.c @@ -18,62 +18,62 @@ #include #include "darshan-logutils.h" -#include "darshan-apss-log-format.h" +#include "darshan-apcxi-log-format.h" -/* counter name strings for the apss module */ +/* counter name strings for the apcxi module */ #define Y(a) #a, -#define X(a) Y(APSS_ ## a) +#define X(a) Y(APCXI_ ## a) #define Z(a) #a -char *apss_counter_names[] = { - APSS_PERF_COUNTERS +char *apcxi_counter_names[] = { + APCXI_PERF_COUNTERS }; #undef Y #undef X #undef Z -static int darshan_log_get_apss_rec(darshan_fd fd, void** buf_p); -static int darshan_log_put_apss_rec(darshan_fd fd, void* buf); -static void darshan_log_print_apss_rec(void *file_rec, +static int darshan_log_get_apcxi_rec(darshan_fd fd, void** buf_p); +static int darshan_log_put_apcxi_rec(darshan_fd fd, void* buf); +static void darshan_log_print_apcxi_rec(void *file_rec, char *file_name, char *mnt_pt, char *fs_type); -static void darshan_log_print_apss_description(int ver); -static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, +static void darshan_log_print_apcxi_description(int ver); +static void darshan_log_print_apcxi_rec_diff(void *file_rec1, char *file_name1, void *file_rec2, char *file_name2); -struct darshan_mod_logutil_funcs apss_logutils = +struct darshan_mod_logutil_funcs apcxi_logutils = { - .log_get_record = &darshan_log_get_apss_rec, - .log_put_record = &darshan_log_put_apss_rec, - .log_print_record = &darshan_log_print_apss_rec, - .log_print_description = &darshan_log_print_apss_description, - .log_print_diff = &darshan_log_print_apss_rec_diff, + .log_get_record = &darshan_log_get_apcxi_rec, + .log_put_record = &darshan_log_put_apcxi_rec, + .log_print_record = &darshan_log_print_apcxi_rec, + .log_print_description = &darshan_log_print_apcxi_description, + .log_print_diff = &darshan_log_print_apcxi_rec_diff, .log_agg_records = NULL }; -static int darshan_log_get_apss_rec(darshan_fd fd, void** buf_p) +static int darshan_log_get_apcxi_rec(darshan_fd fd, void** buf_p) { - struct darshan_apss_header_record *hdr_rec; - struct darshan_apss_perf_record *prf_rec; + struct darshan_apcxi_header_record *hdr_rec; + struct darshan_apcxi_perf_record *prf_rec; int rec_len; char *buffer; int i; int ret = -1; static int first_rec = 1; - if(fd->mod_map[DARSHAN_APSS_MOD].len == 0) + if(fd->mod_map[DARSHAN_APCXI_MOD].len == 0) return(0); - if(fd->mod_ver[DARSHAN_APSS_MOD] == 0 || - fd->mod_ver[DARSHAN_APSS_MOD] > APSS_VER) + if(fd->mod_ver[DARSHAN_APCXI_MOD] == 0 || + fd->mod_ver[DARSHAN_APCXI_MOD] > APCXI_VER) { - fprintf(stderr, "Error: Invalid APSS module version number (got %d)\n", - fd->mod_ver[DARSHAN_APSS_MOD]); + fprintf(stderr, "Error: Invalid APCXI module version number (got %d)\n", + fd->mod_ver[DARSHAN_APCXI_MOD]); return(-1); } if (!*buf_p) { /* assume this is the largest possible record size */ - buffer = malloc(sizeof(struct darshan_apss_perf_record)); + buffer = malloc(sizeof(struct darshan_apcxi_perf_record)); if (!buffer) { return(-1); @@ -84,32 +84,32 @@ static int darshan_log_get_apss_rec(darshan_fd fd, void** buf_p) buffer = *buf_p; } - if (fd->mod_ver[DARSHAN_APSS_MOD] < APSS_VER) + if (fd->mod_ver[DARSHAN_APCXI_MOD] < APCXI_VER) { /* perform conversion as needed */ } /* v1, current version */ - if (fd->mod_ver[DARSHAN_APSS_MOD] == APSS_VER) + if (fd->mod_ver[DARSHAN_APCXI_MOD] == APCXI_VER) { if (first_rec) { - rec_len = sizeof(struct darshan_apss_header_record); + rec_len = sizeof(struct darshan_apcxi_header_record); first_rec = 0; } else - rec_len = sizeof(struct darshan_apss_perf_record); + rec_len = sizeof(struct darshan_apcxi_perf_record); - ret = darshan_log_get_mod(fd, DARSHAN_APSS_MOD, buffer, rec_len); + ret = darshan_log_get_mod(fd, DARSHAN_APCXI_MOD, buffer, rec_len); } if (ret == rec_len) { if(fd->swap_flag) { - if (rec_len == sizeof(struct darshan_apss_header_record)) + if (rec_len == sizeof(struct darshan_apcxi_header_record)) { - hdr_rec = (struct darshan_apss_header_record*)buffer; + hdr_rec = (struct darshan_apcxi_header_record*)buffer; /* swap bytes if necessary */ DARSHAN_BSWAP64(&(hdr_rec->base_rec.id)); DARSHAN_BSWAP64(&(hdr_rec->base_rec.rank)); @@ -122,7 +122,7 @@ static int darshan_log_get_apss_rec(darshan_fd fd, void** buf_p) } else { - prf_rec = (struct darshan_apss_perf_record*)buffer; + prf_rec = (struct darshan_apcxi_perf_record*)buffer; DARSHAN_BSWAP64(&(prf_rec->base_rec.id)); DARSHAN_BSWAP64(&(prf_rec->base_rec.rank)); DARSHAN_BSWAP64(&(prf_rec->group)); @@ -130,7 +130,7 @@ static int darshan_log_get_apss_rec(darshan_fd fd, void** buf_p) DARSHAN_BSWAP64(&(prf_rec->slot)); DARSHAN_BSWAP64(&(prf_rec->blade)); DARSHAN_BSWAP64(&(prf_rec->node)); - for (i = 0; i < APSS_NUM_INDICES; i++) + for (i = 0; i < APCXI_NUM_INDICES; i++) { DARSHAN_BSWAP64(&prf_rec->counters[i]); } @@ -155,7 +155,7 @@ static int darshan_log_get_apss_rec(darshan_fd fd, void** buf_p) } } -static int darshan_log_put_apss_rec(darshan_fd fd, void* buf) +static int darshan_log_put_apcxi_rec(darshan_fd fd, void* buf) { int ret; int rec_len; @@ -163,74 +163,74 @@ static int darshan_log_put_apss_rec(darshan_fd fd, void* buf) if (first_rec) { - rec_len = sizeof(struct darshan_apss_header_record); + rec_len = sizeof(struct darshan_apcxi_header_record); first_rec = 0; } else - rec_len = sizeof(struct darshan_apss_perf_record); + rec_len = sizeof(struct darshan_apcxi_perf_record); - ret = darshan_log_put_mod(fd, DARSHAN_APSS_MOD, buf, - rec_len, APSS_VER); + ret = darshan_log_put_mod(fd, DARSHAN_APCXI_MOD, buf, + rec_len, APCXI_VER); if(ret < 0) return(-1); return(0); } -static void darshan_log_print_apss_rec(void *rec, char *file_name, +static void darshan_log_print_apcxi_rec(void *rec, char *file_name, char *mnt_pt, char *fs_type) { int i; static int first_rec = 1; - struct darshan_apss_header_record *hdr_rec; - struct darshan_apss_perf_record *prf_rec; + struct darshan_apcxi_header_record *hdr_rec; + struct darshan_apcxi_perf_record *prf_rec; if (first_rec) { hdr_rec = rec; - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "APSS_GROUPS", hdr_rec->ngroups, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + "APCXI_GROUPS", hdr_rec->ngroups, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "APSS_CHASSIS", hdr_rec->nchassis, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + "APCXI_CHASSIS", hdr_rec->nchassis, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "APSS_SLOTS", hdr_rec->nslots, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + "APCXI_SLOTS", hdr_rec->nslots, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "APSS_BLADES", hdr_rec->nblades, "", "", ""); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + "APCXI_BLADES", hdr_rec->nblades, "", "", ""); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "APSS_APPLICATION_ID", hdr_rec->appid, "", "", ""); + "APCXI_APPLICATION_ID", hdr_rec->appid, "", "", ""); first_rec = 0; } else { prf_rec = rec; - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec->base_rec.rank, prf_rec->base_rec.id, - "APSS_GROUP", prf_rec->group, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + "APCXI_GROUP", prf_rec->group, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec->base_rec.rank, prf_rec->base_rec.id, - "APSS_CHASSIS", prf_rec->chassis, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + "APCXI_CHASSIS", prf_rec->chassis, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec->base_rec.rank, prf_rec->base_rec.id, - "APSS_SLOT", prf_rec->slot, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + "APCXI_SLOT", prf_rec->slot, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec->base_rec.rank, prf_rec->base_rec.id, - "APSS_BLADE", prf_rec->blade, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + "APCXI_BLADE", prf_rec->blade, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec->base_rec.rank, prf_rec->base_rec.id, - "APSS_NODE", prf_rec->node, "", "", ""); + "APCXI_NODE", prf_rec->node, "", "", ""); - for(i = 0; i < APSS_NUM_INDICES; i++) + for(i = 0; i < APCXI_NUM_INDICES; i++) { - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec->base_rec.rank, prf_rec->base_rec.id, - apss_counter_names[i], prf_rec->counters[i], + apcxi_counter_names[i], prf_rec->counters[i], "", "", ""); } } @@ -238,150 +238,150 @@ static void darshan_log_print_apss_rec(void *rec, char *file_name, return; } -static void darshan_log_print_apss_description(int ver) +static void darshan_log_print_apcxi_description(int ver) { - printf("\n# description of apss counters:\n"); - printf("# global summary stats for the apss module:\n"); - printf("# APSS_GROUPS: total number of groups\n"); - printf("# APSS_CHASSIS: total number of chassis\n"); - printf("# APSS_BLADES: total number of blades\n"); - printf("# APSS_SLOTS: total number of slots\n"); - printf("# per-router statistics for the apss module:\n"); - printf("# APSS_GROUP: group this router is on\n"); - printf("# APSS_CHASSIS: chassis this router is on\n"); - printf("# APSS_SLOT: slot in the chassis this router is on\n"); - printf("# APSS_BLADE: blade this router is on\n"); - printf("# APSS_NODE: node connected to this router\n"); - printf("# APSS_AR_RTR_* port counters for the 40 router-router ports\n"); - //printf("# APSS_AR_RTR_x_y_INQ_PRF_INCOMING_FLIT_VC[0-7]: flits on VCs of x y tile\n"); - //printf("# APSS_AR_RTR_x_y_INQ_PRF_ROWBUS_STALL_CNT: stalls on x y tile\n"); - //printf("# APSS_AR_RTR_PT_* port counters for the 8 router-nic ports\n"); - //printf("# APSS_AR_RTR_PT_x_y_INQ_PRF_INCOMING_FLIT_VC[0,4]: flits on VCs of x y tile\n"); - //printf("# APSS_AR_RTR_PT_x_y_INQ_PRF_REQ_ROWBUS_STALL_CNT: stalls on x y tile\n"); + printf("\n# description of apcxi counters:\n"); + printf("# global summary stats for the apcxi module:\n"); + printf("# APCXI_GROUPS: total number of groups\n"); + printf("# APCXI_CHASSIS: total number of chassis\n"); + printf("# APCXI_BLADES: total number of blades\n"); + printf("# APCXI_SLOTS: total number of slots\n"); + printf("# per-router statistics for the apcxi module:\n"); + printf("# APCXI_GROUP: group this router is on\n"); + printf("# APCXI_CHASSIS: chassis this router is on\n"); + printf("# APCXI_SLOT: slot in the chassis this router is on\n"); + printf("# APCXI_BLADE: blade this router is on\n"); + printf("# APCXI_NODE: node connected to this router\n"); + printf("# APCXI_AR_RTR_* port counters for the 40 router-router ports\n"); + //printf("# APCXI_AR_RTR_x_y_INQ_PRF_INCOMING_FLIT_VC[0-7]: flits on VCs of x y tile\n"); + //printf("# APCXI_AR_RTR_x_y_INQ_PRF_ROWBUS_STALL_CNT: stalls on x y tile\n"); + //printf("# APCXI_AR_RTR_PT_* port counters for the 8 router-nic ports\n"); + //printf("# APCXI_AR_RTR_PT_x_y_INQ_PRF_INCOMING_FLIT_VC[0,4]: flits on VCs of x y tile\n"); + //printf("# APCXI_AR_RTR_PT_x_y_INQ_PRF_REQ_ROWBUS_STALL_CNT: stalls on x y tile\n"); return; } -static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, +static void darshan_log_print_apcxi_rec_diff(void *file_rec1, char *file_name1, void *file_rec2, char *file_name2) { - struct darshan_apss_header_record *hdr_rec1; - struct darshan_apss_header_record *hdr_rec2; - struct darshan_apss_perf_record *prf_rec1; - struct darshan_apss_perf_record *prf_rec2; + struct darshan_apcxi_header_record *hdr_rec1; + struct darshan_apcxi_header_record *hdr_rec2; + struct darshan_apcxi_perf_record *prf_rec1; + struct darshan_apcxi_perf_record *prf_rec2; - hdr_rec1 = (struct darshan_apss_header_record*) file_rec1; - hdr_rec2 = (struct darshan_apss_header_record*) file_rec2; - prf_rec1 = (struct darshan_apss_perf_record*) file_rec1; - prf_rec2 = (struct darshan_apss_perf_record*) file_rec2; + hdr_rec1 = (struct darshan_apcxi_header_record*) file_rec1; + hdr_rec2 = (struct darshan_apcxi_header_record*) file_rec2; + prf_rec1 = (struct darshan_apcxi_perf_record*) file_rec1; + prf_rec2 = (struct darshan_apcxi_perf_record*) file_rec2; - if (hdr_rec1->magic == APSS_MAGIC) + if (hdr_rec1->magic == APCXI_MAGIC) { /* this is the header record */ if (!hdr_rec2) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APSS_GROUPS", hdr_rec1->ngroups, "", "", ""); + "APCXI_GROUPS", hdr_rec1->ngroups, "", "", ""); printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APSS_CHASSIS", hdr_rec1->nchassis, "", "", ""); + "APCXI_CHASSIS", hdr_rec1->nchassis, "", "", ""); printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APSS_SLOTS", hdr_rec1->nslots, "", "", ""); + "APCXI_SLOTS", hdr_rec1->nslots, "", "", ""); printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APSS_BLADES", hdr_rec1->nblades, "", "", ""); + "APCXI_BLADES", hdr_rec1->nblades, "", "", ""); printf("- "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APSS_APPLICATION_ID", hdr_rec1->appid, "", "", ""); + "APCXI_APPLICATION_ID", hdr_rec1->appid, "", "", ""); } else if (!hdr_rec1) { printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APSS_GROUPS", hdr_rec2->ngroups, "", "", ""); + "APCXI_GROUPS", hdr_rec2->ngroups, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APSS_CHASSIS", hdr_rec2->nchassis, "", "", ""); + "APCXI_CHASSIS", hdr_rec2->nchassis, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APSS_SLOTS", hdr_rec2->nslots, "", "", ""); + "APCXI_SLOTS", hdr_rec2->nslots, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APSS_BLADES", hdr_rec2->nblades, "", "", ""); + "APCXI_BLADES", hdr_rec2->nblades, "", "", ""); printf("+ "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APSS_APPLICATION_ID", hdr_rec2->appid, "", "", ""); + "APCXI_APPLICATION_ID", hdr_rec2->appid, "", "", ""); } else { if (hdr_rec1->ngroups != hdr_rec2->ngroups) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APSS_GROUPS", hdr_rec1->ngroups, "", "", ""); + "APCXI_GROUPS", hdr_rec1->ngroups, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APSS_GROUPS", hdr_rec2->ngroups, "", "", ""); + "APCXI_GROUPS", hdr_rec2->ngroups, "", "", ""); } if (hdr_rec1->nchassis != hdr_rec2->nchassis) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APSS_CHASSIS", hdr_rec1->nchassis, "", "", ""); + "APCXI_CHASSIS", hdr_rec1->nchassis, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APSS_CHASSIS", hdr_rec2->nchassis, "", "", ""); + "APCXI_CHASSIS", hdr_rec2->nchassis, "", "", ""); } if (hdr_rec1->nslots != hdr_rec2->nslots) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APSS_SLOTS", hdr_rec1->nslots, "", "", ""); + "APCXI_SLOTS", hdr_rec1->nslots, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APSS_SLOTS", hdr_rec2->nslots, "", "", ""); + "APCXI_SLOTS", hdr_rec2->nslots, "", "", ""); } if (hdr_rec1->nblades != hdr_rec2->nblades) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APSS_BLADES", hdr_rec1->nblades, "", "", ""); + "APCXI_BLADES", hdr_rec1->nblades, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APSS_BLADES", hdr_rec2->nblades, "", "", ""); + "APCXI_BLADES", hdr_rec2->nblades, "", "", ""); } if (hdr_rec1->appid != hdr_rec2->appid) { printf("- "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APSS_APPLICATION_ID", hdr_rec1->appid, "", "", ""); + "APCXI_APPLICATION_ID", hdr_rec1->appid, "", "", ""); printf("+ "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APSS_APPLICATION_ID", hdr_rec2->appid, "", "", ""); + "APCXI_APPLICATION_ID", hdr_rec2->appid, "", "", ""); } } } @@ -390,139 +390,139 @@ static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, if (!prf_rec2) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "APSS_GROUP", prf_rec1->group, "", "", ""); + "APCXI_GROUP", prf_rec1->group, "", "", ""); printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "APSS_CHASSIS", prf_rec1->chassis, "", "", ""); + "APCXI_CHASSIS", prf_rec1->chassis, "", "", ""); printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "APSS_SLOT", prf_rec1->slot, "", "", ""); + "APCXI_SLOT", prf_rec1->slot, "", "", ""); printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "APSS_BLADE", prf_rec1->blade, "", "", ""); + "APCXI_BLADE", prf_rec1->blade, "", "", ""); printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "APSS_NODE", prf_rec1->node, "", "", ""); + "APCXI_NODE", prf_rec1->node, "", "", ""); } else if (!prf_rec1) { printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "APSS_GROUP", prf_rec2->group, "", "", ""); + "APCXI_GROUP", prf_rec2->group, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "APSS_CHASSIS", prf_rec2->chassis, "", "", ""); + "APCXI_CHASSIS", prf_rec2->chassis, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "APSS_SLOT", prf_rec2->slot, "", "", ""); + "APCXI_SLOT", prf_rec2->slot, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "APSS_BLADE", prf_rec2->blade, "", "", ""); + "APCXI_BLADE", prf_rec2->blade, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "APSS_NODE", prf_rec2->node, "", "", ""); + "APCXI_NODE", prf_rec2->node, "", "", ""); } else { if (prf_rec1->group != prf_rec2->group) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "APSS_GROUP", prf_rec1->group, "", "", ""); + "APCXI_GROUP", prf_rec1->group, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "APSS_GROUP", prf_rec2->group, "", "", ""); + "APCXI_GROUP", prf_rec2->group, "", "", ""); } if (prf_rec1->chassis != prf_rec2->chassis) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "APSS_CHASSIS", prf_rec1->chassis, "", "", ""); + "APCXI_CHASSIS", prf_rec1->chassis, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "APSS_CHASSIS", prf_rec2->chassis, "", "", ""); + "APCXI_CHASSIS", prf_rec2->chassis, "", "", ""); } if (prf_rec1->slot != prf_rec2->slot) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "APSS_SLOT", prf_rec1->slot, "", "", ""); + "APCXI_SLOT", prf_rec1->slot, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "APSS_SLOT", prf_rec2->slot, "", "", ""); + "APCXI_SLOT", prf_rec2->slot, "", "", ""); } if (prf_rec1->blade != prf_rec2->blade) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "APSS_BLADE", prf_rec1->blade, "", "", ""); + "APCXI_BLADE", prf_rec1->blade, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "APSS_BLADE", prf_rec2->blade, "", "", ""); + "APCXI_BLADE", prf_rec2->blade, "", "", ""); } if (prf_rec1->node != prf_rec2->node) { printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "APSS_NODE", prf_rec1->node, "", "", ""); + "APCXI_NODE", prf_rec1->node, "", "", ""); printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "APSS_NODE", prf_rec2->node, "", "", ""); + "APCXI_NODE", prf_rec2->node, "", "", ""); } } int i; /* router tile record */ - for(i = 0; i < APSS_NUM_INDICES; i++) + for(i = 0; i < APCXI_NUM_INDICES; i++) { if (!prf_rec2) { printf("- "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - apss_counter_names[i], prf_rec1->counters[i], + apcxi_counter_names[i], prf_rec1->counters[i], "", "", ""); } else if (!prf_rec1) { printf("+ "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - apss_counter_names[i], prf_rec2->counters[i], + apcxi_counter_names[i], prf_rec2->counters[i], "", "", ""); } else if (prf_rec1->counters[i] != prf_rec2->counters[i]) { printf("- "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - apss_counter_names[i], prf_rec1->counters[i], + apcxi_counter_names[i], prf_rec1->counters[i], "", "", ""); printf("+ "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APSS_MOD], + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - apss_counter_names[i], prf_rec2->counters[i], + apcxi_counter_names[i], prf_rec2->counters[i], "", "", ""); } } diff --git a/apcxi/util/darshan-apss-logutils.h b/apcxi/util/darshan-apss-logutils.h new file mode 100644 index 0000000..1c0b256 --- /dev/null +++ b/apcxi/util/darshan-apss-logutils.h @@ -0,0 +1,15 @@ +/* + * Copyright (C) 2018 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#ifndef __APCXI_LOG_UTILS_H +#define __APCXI_LOG_UTILS_H + +extern char *apcxi_counter_names[]; + +extern struct darshan_mod_logutil_funcs apcxi_logutils; + +#endif + diff --git a/apcxi/util/print-apss-size.c b/apcxi/util/print-apss-size.c new file mode 100644 index 0000000..80cd4af --- /dev/null +++ b/apcxi/util/print-apss-size.c @@ -0,0 +1,12 @@ +#include +#include "darshan-log-format.h" +#include "darshan-apcxi-log-format.h" + +int main (int argc, char **argv) +{ + printf ("APCXI_NUM_INDICES = %d\n", APCXI_NUM_INDICES); + printf ("sizeof darshan_apcxi_header_record = %d\n", sizeof(struct darshan_apcxi_header_record)); + printf ("sizeof darshan_apcxi_perf_record = %d\n", sizeof(struct darshan_apcxi_perf_record)); + + return 0; +} diff --git a/apss/Makefile.darshan b/apss/Makefile.darshan deleted file mode 100644 index bec287a..0000000 --- a/apss/Makefile.darshan +++ /dev/null @@ -1,20 +0,0 @@ -# -# AutoPerf Make rules for Darshan -# -DARSHAN_STATIC_MOD_OBJS += lib/darshan-apss.o -DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-apss.po - -VPATH += :$(srcdir)/../modules/autoperf/apss -CFLAGS += \ - -DDARSHAN_USE_APSS \ - -I$(srcdir)/../modules/autoperf/apss - -CFLAGS_SHARED += \ - -DDARSHAN_USE_APSS \ - -I$(srcdir)/../modules/autoperf/apss - -lib/darshan-apss.o: lib/darshan-apss.c lib/darshan-apss-utils.h darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) darshan-apss-log-format.h | lib - $(CC) $(CFLAGS) -c $< -o $@ - -lib/darshan-apss.po: lib/darshan-apss.c lib/darshan-apss-utils.h darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) darshan-apss-log-format.h | lib - $(CC) $(CFLAGS_SHARED) -c $< -o $@ diff --git a/apss/util/Makefile.darshan b/apss/util/Makefile.darshan deleted file mode 100644 index 1a9784b..0000000 --- a/apss/util/Makefile.darshan +++ /dev/null @@ -1,27 +0,0 @@ -# -# AutoPerf Make rules for Darshan -# - -VPATH += :$(srcdir)/../modules/autoperf/apss/util - -DARSHAN_MOD_LOG_FORMATS += $(srcdir)/../modules/autoperf/apss/darshan-apss-log-format.h -DARSHAN_MOD_LOGUTIL_HEADERS += darshan-apss-logutils.h -DARSHAN_STATIC_MOD_OBJS += darshan-apss-logutils.o -DARSHAN_DYNAMIC_MOD_OBJS += darshan-apss-logutils.po - -CFLAGS += \ - -DDARSHAN_USE_apss \ - -I$(srcdir)/../modules/autoperf/apss \ - -I$(srcdir)/../modules/autoperf/apss/util - -CFLAGS_SHARED += \ - -DDARSHAN_USE_apss \ - -I$(srcdir)/../modules/autoperf/apss \ - -I$(srcdir)/../modules/autoperf/apss/util - -darshan-apss-logutils.o: darshan-apss-logutils.c darshan-logutils.h darshan-apss-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../modules/autoperf/apss/darshan-apss-log-format.h | uthash-1.9.2 - $(CC) $(CFLAGS) -c $< -o $@ - -darshan-apss-logutils.po: darshan-apss-logutils.c darshan-logutils.h darshan-apss-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../modules/autoperf/apss/darshan-apss-log-format.h | uthash-1.9.2 - $(CC) $(CFLAGS_SHARED) -c $< -o $@ - diff --git a/apss/util/darshan-apss-logutils.h b/apss/util/darshan-apss-logutils.h deleted file mode 100644 index 6d12e17..0000000 --- a/apss/util/darshan-apss-logutils.h +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright (C) 2018 University of Chicago. - * See COPYRIGHT notice in top-level directory. - * - */ - -#ifndef __apss_LOG_UTILS_H -#define __apss_LOG_UTILS_H - -extern char *apss_counter_names[]; - -extern struct darshan_mod_logutil_funcs apss_logutils; - -#endif - diff --git a/apss/util/print-apss-size.c b/apss/util/print-apss-size.c deleted file mode 100644 index a60244f..0000000 --- a/apss/util/print-apss-size.c +++ /dev/null @@ -1,12 +0,0 @@ -#include -#include "darshan-log-format.h" -#include "darshan-apss-log-format.h" - -int main (int argc, char **argv) -{ - printf ("APSS_NUM_INDICES = %d\n", APSS_NUM_INDICES); - printf ("sizeof darshan_apss_header_record = %d\n", sizeof(struct darshan_apss_header_record)); - printf ("sizeof darshan_apss_perf_record = %d\n", sizeof(struct darshan_apss_perf_record)); - - return 0; -} From 6eaec26bc695ac10c26539dbfed549448e2d8a2b Mon Sep 17 00:00:00 2001 From: Sudheer Chunduri Date: Fri, 11 Feb 2022 14:46:18 -0500 Subject: [PATCH 11/12] apcxi: change the file names to apcxi --- apcxi/{darshan-apss-log-format.h => darshan-apcxi-log-format.h} | 0 apcxi/lib/{darshan-apss-utils.h => darshan-apcxi-utils.h} | 0 apcxi/lib/{darshan-apss.c => darshan-apcxi.c} | 0 apcxi/util/{apss-analysis.py => apcxi-analysis.py} | 0 apcxi/util/{apss-backend.py => apcxi-backend.py} | 0 apcxi/util/{darshan-apss-logutils.c => darshan-apcxi-logutils.c} | 0 apcxi/util/{darshan-apss-logutils.h => darshan-apcxi-logutils.h} | 0 apcxi/util/{print-apss-size.c => print-apcxi-size.c} | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename apcxi/{darshan-apss-log-format.h => darshan-apcxi-log-format.h} (100%) rename apcxi/lib/{darshan-apss-utils.h => darshan-apcxi-utils.h} (100%) rename apcxi/lib/{darshan-apss.c => darshan-apcxi.c} (100%) rename apcxi/util/{apss-analysis.py => apcxi-analysis.py} (100%) rename apcxi/util/{apss-backend.py => apcxi-backend.py} (100%) rename apcxi/util/{darshan-apss-logutils.c => darshan-apcxi-logutils.c} (100%) rename apcxi/util/{darshan-apss-logutils.h => darshan-apcxi-logutils.h} (100%) rename apcxi/util/{print-apss-size.c => print-apcxi-size.c} (100%) diff --git a/apcxi/darshan-apss-log-format.h b/apcxi/darshan-apcxi-log-format.h similarity index 100% rename from apcxi/darshan-apss-log-format.h rename to apcxi/darshan-apcxi-log-format.h diff --git a/apcxi/lib/darshan-apss-utils.h b/apcxi/lib/darshan-apcxi-utils.h similarity index 100% rename from apcxi/lib/darshan-apss-utils.h rename to apcxi/lib/darshan-apcxi-utils.h diff --git a/apcxi/lib/darshan-apss.c b/apcxi/lib/darshan-apcxi.c similarity index 100% rename from apcxi/lib/darshan-apss.c rename to apcxi/lib/darshan-apcxi.c diff --git a/apcxi/util/apss-analysis.py b/apcxi/util/apcxi-analysis.py similarity index 100% rename from apcxi/util/apss-analysis.py rename to apcxi/util/apcxi-analysis.py diff --git a/apcxi/util/apss-backend.py b/apcxi/util/apcxi-backend.py similarity index 100% rename from apcxi/util/apss-backend.py rename to apcxi/util/apcxi-backend.py diff --git a/apcxi/util/darshan-apss-logutils.c b/apcxi/util/darshan-apcxi-logutils.c similarity index 100% rename from apcxi/util/darshan-apss-logutils.c rename to apcxi/util/darshan-apcxi-logutils.c diff --git a/apcxi/util/darshan-apss-logutils.h b/apcxi/util/darshan-apcxi-logutils.h similarity index 100% rename from apcxi/util/darshan-apss-logutils.h rename to apcxi/util/darshan-apcxi-logutils.h diff --git a/apcxi/util/print-apss-size.c b/apcxi/util/print-apcxi-size.c similarity index 100% rename from apcxi/util/print-apss-size.c rename to apcxi/util/print-apcxi-size.c From d3926a38fa9eaed52e560ffd959ae6da0bf583d3 Mon Sep 17 00:00:00 2001 From: Sudheer Chunduri Date: Mon, 14 Feb 2022 20:02:39 -0500 Subject: [PATCH 12/12] apcxi: autoperf cxi network performance counter module using sysfs file system reads The cassini nic performance counter module reads the nic performance counter module by reading from the sys file system First, the number of available nics on the compute node are determined by reading the number of directories in /sys/class/cxi/ The topology is discovered using the xnodename node name format on the HPE EX systems Only one rank/process per compute node is used to read and report the nic performance counters Each monitoring rank reads the counters predefined and configurable list of counters in apcxi/darshan-apcxi-log-format.h file. The counters from each of the available nics are read and reported. --- apcxi/darshan-apcxi-log-format.h | 16 +- apcxi/lib/darshan-apcxi-utils.h | 10 + apcxi/lib/darshan-apcxi.c | 156 +++++++++---- apcxi/util/darshan-apcxi-logutils.c | 337 +++++++++++++++++----------- apcxi/util/print-apcxi-size.c | 4 +- apxc/util/darshan-apxc-logutils.c | 2 +- 6 files changed, 345 insertions(+), 180 deletions(-) diff --git a/apcxi/darshan-apcxi-log-format.h b/apcxi/darshan-apcxi-log-format.h index e54c25e..3000b39 100644 --- a/apcxi/darshan-apcxi-log-format.h +++ b/apcxi/darshan-apcxi-log-format.h @@ -91,8 +91,12 @@ enum darshan_apcxi_perf_indices * module. This example implementation logs the following data for each * record: * - a darshan_base_record structure, which contains the record id & rank - * - integer I/O counters - * - floating point I/O counters + * - the number of the performance counters (see above) monitored is static + * - the number of nic devices on a compute node for which the counters are + * - recorded is not static and is dynamic + * - in order for the record to be contiguous the counters is declared as + * - counters[1][NUM_COUNTERS], the actual record size is calculated based on + * - number of nics */ struct darshan_apcxi_perf_record { @@ -102,7 +106,8 @@ struct darshan_apcxi_perf_record int64_t slot; int64_t blade; int64_t node; - uint64_t counters[APCXI_NUM_INDICES]; + int64_t num_nics; + uint64_t counters[1][APCXI_NUM_INDICES]; }; struct darshan_apcxi_header_record @@ -116,4 +121,9 @@ struct darshan_apcxi_header_record uint64_t appid; }; +/* + * helper function to calculate the size of a APCXI record + */ +#define APCXI_PERF_RECORD_SIZE( num_nics ) (sizeof(struct darshan_apcxi_perf_record) + (sizeof(uint64_t) * APCXI_NUM_INDICES * (num_nics - 1))) + #endif /* __APCXI_LOG_FORMAT_H */ diff --git a/apcxi/lib/darshan-apcxi-utils.h b/apcxi/lib/darshan-apcxi-utils.h index b82394c..b1b85fd 100644 --- a/apcxi/lib/darshan-apcxi-utils.h +++ b/apcxi/lib/darshan-apcxi-utils.h @@ -2,6 +2,7 @@ #define __APCXI_UTILS_H__ #include +#include extern char crusher_nodelist[192][14]; static void search_hwinfo(const char * mstr, char *mode) { @@ -72,4 +73,13 @@ static unsigned int count_bits(unsigned int *bitvec, int cnt) return count; } +void strlwr(char *str) +{ + char *p = str; + while (*p!='\0') { + *p = tolower((char)*p); + p++; + } +} + #endif diff --git a/apcxi/lib/darshan-apcxi.c b/apcxi/lib/darshan-apcxi.c index caca68b..e013625 100644 --- a/apcxi/lib/darshan-apcxi.c +++ b/apcxi/lib/darshan-apcxi.c @@ -6,6 +6,12 @@ #define _XOPEN_SOURCE 500 #define _GNU_SOURCE + +/* + * TODO: A job id given by the job scheduler uniquely identifies a job at a system level, + * similarly, the application launcher id uniquely identifies an application + * execution on the system. + */ //#ifdef SLURM_JOBID #define APP_ID "SLURM_TASK_PID" //#endif @@ -18,8 +24,8 @@ #include #include #include +#include #include -#include #include "uthash.h" #include "darshan.h" @@ -28,19 +34,22 @@ #include "darshan-apcxi-utils.h" #include "crusher_nodelist.h" +#define MAX_COUNTER_NAME_LENGTH 100 /* - * PAPI_events are defined by the Aries counters listed in the log header. + * apcxi_events are defined by the Cassini counters listed in the log header. */ #define X(a) #a, #define Z(a) #a -static char* PAPI_events[] = +static char* apcxi_events[] = { APCXI_PERF_COUNTERS }; #undef X #undef Z +char ***apcxi_filenames; + #define MAX_GROUPS (128) #define MAX_CHASSIS (MAX_GROUPS*8) #define MAX_SLOTS (MAX_CHASSIS*8) @@ -63,13 +72,13 @@ struct apcxi_runtime struct darshan_apcxi_perf_record *perf_record; darshan_record_id header_id; darshan_record_id rtr_id; - int PAPI_event_set; - int PAPI_event_count; + int num_counters; int group; int chassis; int slot; int blade; int node; + int num_nics; int perf_record_marked; }; @@ -101,55 +110,80 @@ static void apcxi_cleanup( #define APCXI_UNLOCK() pthread_mutex_unlock(&apcxi_runtime_mutex) /* - * Initialize counters using PAPI + * Initialize counters + * determine the number of nic devices on the node + * build the list of files to read from the sysfs telemetry */ static void initialize_counters (void) { int i; int code = 0; - PAPI_library_init(PAPI_VER_CURRENT); - apcxi_runtime->PAPI_event_set = PAPI_NULL; - PAPI_create_eventset(&apcxi_runtime->PAPI_event_set); -#if 0 - /* Determine the number of NIC devices - read the number of dirs in /sys/class/cxi/ */ - int num_nics = 0; - struct dirent *dp; - DIR *fdir; - if ((fdir = opendir("/sys/class/cxi")) == NULL) { - fprintf(stderr, "listdir: can't open %s\n", "/sys/class/cxi"); - return 1; + + /* start with first apcxi counter */ + i = CQ_CQ_OXE_NUM_STALLS; + while(strcmp(apcxi_events[i], "APCXI_NUM_INDICES") != 0){ + i++; } - while ((dp = readdir(fdir)) != NULL) { - if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, "..")) - continue; /* skip self and parent */ - num_nics++; + apcxi_runtime->num_counters = i; + + // allocate memory space for the sys fs file names + // for each of the available nics (num_nics) + apcxi_filenames = (char***) malloc(sizeof(char**) * apcxi_runtime->num_nics); + for(int i=0; inum_nics; i++){ + apcxi_filenames[i] = (char**) malloc(sizeof(char*) * apcxi_runtime->num_counters); + for(int j=0; jnum_counters; j++){ + apcxi_filenames[i][j] = (char*) malloc(sizeof(char) * MAX_COUNTER_NAME_LENGTH); + } } - closedir(fdir); -#endif - /* start with first PAPI counter */ - for (i = CQ_CQ_OXE_NUM_STALLS; - strcmp(PAPI_events[i], "APCXI_NUM_INDICES") != 0; - i++) + char line[MAX_COUNTER_NAME_LENGTH]; + for(int i=0; inum_counters; i++) { - PAPI_event_name_to_code(PAPI_events[i], &code); - PAPI_add_event(apcxi_runtime->PAPI_event_set, code); + size_t ln = strlen(apcxi_events[i]); + strcpy(line, apcxi_events[i]); + strlwr(line); + line[ln] = '\0'; + for(int j=0; jnum_nics; j++){ + // /sys/class/cxi/cxi[0..3] + char str[MAX_COUNTER_NAME_LENGTH] = ""; + char buf[5]; + snprintf(buf, sizeof(buf), "%d", j); + strcat(str, "/sys/class/cxi/cxi"); + strcat(str, buf); + strcat(str, "/device/telemetry/"); + strcat(str, line); + strcpy(apcxi_filenames[j][i],str); + } } - apcxi_runtime->PAPI_event_count = i; - - PAPI_start(apcxi_runtime->PAPI_event_set); - + FILE *fff; + for(int i=0; inum_nics; i++){ + for(int j=0; jnum_counters; j++){ + fff = fopen(apcxi_filenames[i][j], "r"); + if (fff == NULL) { + fprintf(stderr,"\tError opening %s!\n",apcxi_filenames[i][j]); + } + else { + uint64_t b, c; + fscanf(fff,"%lu@%lu.%lu",&apcxi_runtime->perf_record->counters[i][j], &b, &c); + fclose(fff); + } + } + } return; } static void finalize_counters (void) { - PAPI_cleanup_eventset(apcxi_runtime->PAPI_event_set); - PAPI_destroy_eventset(&apcxi_runtime->PAPI_event_set); - PAPI_shutdown(); - + // free apcxi_filenames + for(int i=0; inum_nics; i++){ + for(int j=0; jnum_counters; j++){ + free(apcxi_filenames[i][j]); + } + free(apcxi_filenames[i]); + } + free(apcxi_filenames); return; } @@ -159,15 +193,28 @@ static void finalize_counters (void) static void capture(struct darshan_apcxi_perf_record *rec, darshan_record_id rec_id) { - PAPI_stop(apcxi_runtime->PAPI_event_set, - (long long*) &rec->counters[CQ_CQ_OXE_NUM_STALLS]); - PAPI_reset(apcxi_runtime->PAPI_event_set); + FILE *fff; + for(int i=0; inum_nics; i++){ + for(int j=0; jnum_counters; j++){ + fff = fopen(apcxi_filenames[i][j], "r"); + if (fff == NULL) { + fprintf(stderr,"\tError opening %s!\n",apcxi_filenames[i][j]); + } + else { + uint64_t a, b, c; + fscanf(fff,"%lu@%lu.%lu", &a, &b, &c); + apcxi_runtime->perf_record->counters[i][j] = a - apcxi_runtime->perf_record->counters[i][j]; + fclose(fff); + } + } + } rec->group = apcxi_runtime->group; rec->chassis = apcxi_runtime->chassis; rec->slot = apcxi_runtime->slot; rec->blade = apcxi_runtime->blade; rec->node = apcxi_runtime->node; + rec->num_nics = apcxi_runtime->num_nics; rec->base_rec.id = rec_id; rec->base_rec.rank = my_rank; @@ -188,6 +235,24 @@ void apcxi_runtime_initialize() APCXI_LOCK(); + /* + * Determine the number of available NIC devices on the compute node + * read the number of dirs in /sys/class/cxi/ + */ + int num_nics = 0; + struct dirent *dp; + DIR *fdir; + if ((fdir = opendir("/sys/class/cxi")) == NULL) { + fprintf(stderr, "listdir: can't open %s\n", "/sys/class/cxi"); + return; + } + while ((dp = readdir(fdir)) != NULL) { + if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, "..")) + continue; /* skip self and parent */ + num_nics++; + } + closedir(fdir); + /* don't do anything if already initialized */ if(apcxi_runtime) @@ -198,7 +263,7 @@ void apcxi_runtime_initialize() apcxi_buf_size = sizeof(struct darshan_apcxi_header_record) + - sizeof(struct darshan_apcxi_perf_record); + APCXI_PERF_RECORD_SIZE(num_nics); /* register the APCXI module with the darshan-core component */ darshan_core_register_module( @@ -208,7 +273,6 @@ void apcxi_runtime_initialize() &my_rank, NULL); - /* initialize module's global state */ apcxi_runtime = malloc(sizeof(*apcxi_runtime)); if(!apcxi_runtime) @@ -218,6 +282,7 @@ void apcxi_runtime_initialize() return; } memset(apcxi_runtime, 0, sizeof(*apcxi_runtime)); + apcxi_runtime->num_nics = num_nics; if (my_rank == 0) { @@ -258,7 +323,8 @@ void apcxi_runtime_initialize() "APCXI", // we want the record for each rank to be treated as shared records so that mpi_redux can operate on //rtr_rec_name, DARSHAN_APCXI_MOD, - sizeof(struct darshan_apcxi_perf_record), + APCXI_PERF_RECORD_SIZE(num_nics), + //sizeof(struct darshan_apcxi_perf_record), NULL); if(!(apcxi_runtime->perf_record)) { @@ -268,6 +334,7 @@ void apcxi_runtime_initialize() APCXI_UNLOCK(); return; } + apcxi_runtime->perf_record->num_nics = num_nics; initialize_counters(); APCXI_UNLOCK(); @@ -455,7 +522,8 @@ static void apcxi_output( if (apcxi_runtime->perf_record_marked == -1) { - *apcxi_buf_sz += sizeof( *apcxi_runtime->perf_record); + //*apcxi_buf_sz += sizeof( *apcxi_runtime->perf_record); + *apcxi_buf_sz += APCXI_PERF_RECORD_SIZE(apcxi_runtime->num_nics); } APCXI_UNLOCK(); diff --git a/apcxi/util/darshan-apcxi-logutils.c b/apcxi/util/darshan-apcxi-logutils.c index 9fc80b3..db77869 100644 --- a/apcxi/util/darshan-apcxi-logutils.c +++ b/apcxi/util/darshan-apcxi-logutils.c @@ -34,10 +34,10 @@ char *apcxi_counter_names[] = { static int darshan_log_get_apcxi_rec(darshan_fd fd, void** buf_p); static int darshan_log_put_apcxi_rec(darshan_fd fd, void* buf); static void darshan_log_print_apcxi_rec(void *file_rec, - char *file_name, char *mnt_pt, char *fs_type); + char *file_name, char *mnt_pt, char *fs_type); static void darshan_log_print_apcxi_description(int ver); static void darshan_log_print_apcxi_rec_diff(void *file_rec1, char *file_name1, - void *file_rec2, char *file_name2); + void *file_rec2, char *file_name2); struct darshan_mod_logutil_funcs apcxi_logutils = { @@ -51,10 +51,12 @@ struct darshan_mod_logutil_funcs apcxi_logutils = static int darshan_log_get_apcxi_rec(darshan_fd fd, void** buf_p) { - struct darshan_apcxi_header_record *hdr_rec; + struct darshan_apcxi_header_record *hdr_rec; //= *((struct darshan_apcxi_header_record **)buf_p);; + struct darshan_apcxi_perf_record *tmp_rec; //= *((struct darshan_apcxi_perf_record **)buf_p); struct darshan_apcxi_perf_record *prf_rec; + int rec_len; - char *buffer; + char *buffer = NULL; int i; int ret = -1; static int first_rec = 1; @@ -63,10 +65,10 @@ static int darshan_log_get_apcxi_rec(darshan_fd fd, void** buf_p) return(0); if(fd->mod_ver[DARSHAN_APCXI_MOD] == 0 || - fd->mod_ver[DARSHAN_APCXI_MOD] > APCXI_VER) + fd->mod_ver[DARSHAN_APCXI_MOD] > APCXI_VER) { fprintf(stderr, "Error: Invalid APCXI module version number (got %d)\n", - fd->mod_ver[DARSHAN_APCXI_MOD]); + fd->mod_ver[DARSHAN_APCXI_MOD]); return(-1); } @@ -97,11 +99,16 @@ static int darshan_log_get_apcxi_rec(darshan_fd fd, void** buf_p) rec_len = sizeof(struct darshan_apcxi_header_record); first_rec = 0; } - else + else{ + /* retrieve the fixed-size portion of the record */ rec_len = sizeof(struct darshan_apcxi_perf_record); - + } ret = darshan_log_get_mod(fd, DARSHAN_APCXI_MOD, buffer, rec_len); } + if(ret < 0){ + return(-1); + } + if (ret == rec_len) { @@ -122,37 +129,77 @@ static int darshan_log_get_apcxi_rec(darshan_fd fd, void** buf_p) } else { - prf_rec = (struct darshan_apcxi_perf_record*)buffer; - DARSHAN_BSWAP64(&(prf_rec->base_rec.id)); - DARSHAN_BSWAP64(&(prf_rec->base_rec.rank)); - DARSHAN_BSWAP64(&(prf_rec->group)); - DARSHAN_BSWAP64(&(prf_rec->chassis)); - DARSHAN_BSWAP64(&(prf_rec->slot)); - DARSHAN_BSWAP64(&(prf_rec->blade)); - DARSHAN_BSWAP64(&(prf_rec->node)); + tmp_rec = (struct darshan_apcxi_perf_record*)buffer; + DARSHAN_BSWAP64(&(tmp_rec->base_rec.id)); + DARSHAN_BSWAP64(&(tmp_rec->base_rec.rank)); + DARSHAN_BSWAP64(&(tmp_rec->group)); + DARSHAN_BSWAP64(&(tmp_rec->chassis)); + DARSHAN_BSWAP64(&(tmp_rec->slot)); + DARSHAN_BSWAP64(&(tmp_rec->blade)); + DARSHAN_BSWAP64(&(tmp_rec->node)); + DARSHAN_BSWAP64(&(tmp_rec->num_nics)); for (i = 0; i < APCXI_NUM_INDICES; i++) { - DARSHAN_BSWAP64(&prf_rec->counters[i]); + DARSHAN_BSWAP64(&tmp_rec->counters[0][i]); + } + } + } + + if (rec_len != sizeof(struct darshan_apcxi_header_record)){ + tmp_rec = (struct darshan_apcxi_perf_record*)buffer; + if(*buf_p == NULL){ + prf_rec = malloc(APCXI_PERF_RECORD_SIZE(tmp_rec->num_nics)); + if(!prf_rec) + return(-1); + memcpy(prf_rec, tmp_rec, sizeof(struct darshan_apcxi_perf_record)); + } + else{ + prf_rec = *buf_p; + } + + /* now read the rest of the record */ + if(prf_rec->num_nics > 1){ + ret = darshan_log_get_mod(fd, DARSHAN_APCXI_MOD, + (void*)(&(prf_rec->counters[1][0])), + ((prf_rec->num_nics-1) * sizeof(uint64_t) * APCXI_NUM_INDICES) + ); + + if(ret < ((prf_rec->num_nics-1) * sizeof(uint64_t) * APCXI_NUM_INDICES)){ + ret = -1; } + else{ + ret = 1; + /* swap bytes if necessary */ + if ( fd->swap_flag ) { + for(int i=1; inum_nics; i++){ + for (int j = 0; j < APCXI_NUM_INDICES; j++){ + DARSHAN_BSWAP64(&tmp_rec->counters[i][j]); + } + } + } + } + } + if(*buf_p == NULL){ + *buf_p = prf_rec; } } - *buf_p = buffer; - return(1); + ret = 1; } else if (ret < 0) { - //*buf_p = NULL; - // if (buffer) free(buffer); + //*buf_p = NULL; + // if (buffer) free(buffer); if (!*buf_p) free(buffer); return(-1); } else { - // *buf_p = NULL; - // if (buffer) free(buffer); + // *buf_p = NULL; + // if (buffer) free(buffer); if (!*buf_p) free(buffer); return(0); } + return ret; } static int darshan_log_put_apcxi_rec(darshan_fd fd, void* buf) @@ -168,9 +215,9 @@ static int darshan_log_put_apcxi_rec(darshan_fd fd, void* buf) } else rec_len = sizeof(struct darshan_apcxi_perf_record); - + ret = darshan_log_put_mod(fd, DARSHAN_APCXI_MOD, buf, - rec_len, APCXI_VER); + rec_len, APCXI_VER); if(ret < 0) return(-1); @@ -178,61 +225,68 @@ static int darshan_log_put_apcxi_rec(darshan_fd fd, void* buf) } static void darshan_log_print_apcxi_rec(void *rec, char *file_name, - char *mnt_pt, char *fs_type) + char *mnt_pt, char *fs_type) { int i; static int first_rec = 1; struct darshan_apcxi_header_record *hdr_rec; struct darshan_apcxi_perf_record *prf_rec; - + if (first_rec) { hdr_rec = rec; DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "APCXI_GROUPS", hdr_rec->ngroups, "", "", ""); + hdr_rec->base_rec.rank, hdr_rec->base_rec.id, + "APCXI_GROUPS", hdr_rec->ngroups, "", "", ""); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "APCXI_CHASSIS", hdr_rec->nchassis, "", "", ""); + hdr_rec->base_rec.rank, hdr_rec->base_rec.id, + "APCXI_CHASSIS", hdr_rec->nchassis, "", "", ""); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "APCXI_SLOTS", hdr_rec->nslots, "", "", ""); + hdr_rec->base_rec.rank, hdr_rec->base_rec.id, + "APCXI_SLOTS", hdr_rec->nslots, "", "", ""); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "APCXI_BLADES", hdr_rec->nblades, "", "", ""); + hdr_rec->base_rec.rank, hdr_rec->base_rec.id, + "APCXI_BLADES", hdr_rec->nblades, "", "", ""); DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "APCXI_APPLICATION_ID", hdr_rec->appid, "", "", ""); + hdr_rec->base_rec.rank, hdr_rec->base_rec.id, + "APCXI_APPLICATION_ID", hdr_rec->appid, "", "", ""); first_rec = 0; } else { prf_rec = rec; - + + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec->base_rec.rank, prf_rec->base_rec.id, + "APCXI_GROUP", prf_rec->group, "", "", ""); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - prf_rec->base_rec.rank, prf_rec->base_rec.id, - "APCXI_GROUP", prf_rec->group, "", "", ""); + prf_rec->base_rec.rank, prf_rec->base_rec.id, + "APCXI_CHASSIS", prf_rec->chassis, "", "", ""); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - prf_rec->base_rec.rank, prf_rec->base_rec.id, - "APCXI_CHASSIS", prf_rec->chassis, "", "", ""); + prf_rec->base_rec.rank, prf_rec->base_rec.id, + "APCXI_SLOT", prf_rec->slot, "", "", ""); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - prf_rec->base_rec.rank, prf_rec->base_rec.id, - "APCXI_SLOT", prf_rec->slot, "", "", ""); + prf_rec->base_rec.rank, prf_rec->base_rec.id, + "APCXI_BLADE", prf_rec->blade, "", "", ""); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - prf_rec->base_rec.rank, prf_rec->base_rec.id, - "APCXI_BLADE", prf_rec->blade, "", "", ""); + prf_rec->base_rec.rank, prf_rec->base_rec.id, + "APCXI_NODE", prf_rec->node, "", "", ""); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - prf_rec->base_rec.rank, prf_rec->base_rec.id, - "APCXI_NODE", prf_rec->node, "", "", ""); - - for(i = 0; i < APCXI_NUM_INDICES; i++) - { - - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec->base_rec.rank, prf_rec->base_rec.id, - apcxi_counter_names[i], prf_rec->counters[i], - "", "", ""); + "APCXI_NUM_NICS", prf_rec->num_nics, "", "", ""); + + for(i = 0; i < prf_rec->num_nics; i++){ + DARSHAN_I_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec->base_rec.rank, prf_rec->base_rec.id, + "APCXI_NIC_ID", i, "", "", ""); + for(int j = 0; j < APCXI_NUM_INDICES; j++){ + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec->base_rec.rank, prf_rec->base_rec.id, + apcxi_counter_names[j], prf_rec->counters[i][j], + "", "", ""); + } } + } return; @@ -247,11 +301,12 @@ static void darshan_log_print_apcxi_description(int ver) printf("# APCXI_BLADES: total number of blades\n"); printf("# APCXI_SLOTS: total number of slots\n"); printf("# per-router statistics for the apcxi module:\n"); - printf("# APCXI_GROUP: group this router is on\n"); - printf("# APCXI_CHASSIS: chassis this router is on\n"); - printf("# APCXI_SLOT: slot in the chassis this router is on\n"); - printf("# APCXI_BLADE: blade this router is on\n"); - printf("# APCXI_NODE: node connected to this router\n"); + printf("# APCXI_GROUP: group this node is on\n"); + printf("# APCXI_CHASSIS: chassis this node is on\n"); + printf("# APCXI_SLOT: slot in the chassis this node is on\n"); + printf("# APCXI_BLADE: blade this node is on\n"); + printf("# APCXI_NODE: node index in the blade\n"); + printf("# APCXI_NUM_NICS: number of nics connected to this node\n"); printf("# APCXI_AR_RTR_* port counters for the 40 router-router ports\n"); //printf("# APCXI_AR_RTR_x_y_INQ_PRF_INCOMING_FLIT_VC[0-7]: flits on VCs of x y tile\n"); //printf("# APCXI_AR_RTR_x_y_INQ_PRF_ROWBUS_STALL_CNT: stalls on x y tile\n"); @@ -263,7 +318,7 @@ static void darshan_log_print_apcxi_description(int ver) } static void darshan_log_print_apcxi_rec_diff(void *file_rec1, char *file_name1, - void *file_rec2, char *file_name2) + void *file_rec2, char *file_name2) { struct darshan_apcxi_header_record *hdr_rec1; struct darshan_apcxi_header_record *hdr_rec2; @@ -282,47 +337,47 @@ static void darshan_log_print_apcxi_rec_diff(void *file_rec1, char *file_name1, { printf("- "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APCXI_GROUPS", hdr_rec1->ngroups, "", "", ""); + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_GROUPS", hdr_rec1->ngroups, "", "", ""); printf("- "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APCXI_CHASSIS", hdr_rec1->nchassis, "", "", ""); + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_CHASSIS", hdr_rec1->nchassis, "", "", ""); printf("- "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APCXI_SLOTS", hdr_rec1->nslots, "", "", ""); + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_SLOTS", hdr_rec1->nslots, "", "", ""); printf("- "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APCXI_BLADES", hdr_rec1->nblades, "", "", ""); + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_BLADES", hdr_rec1->nblades, "", "", ""); printf("- "); DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APCXI_APPLICATION_ID", hdr_rec1->appid, "", "", ""); + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_APPLICATION_ID", hdr_rec1->appid, "", "", ""); } else if (!hdr_rec1) { printf("+ "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APCXI_GROUPS", hdr_rec2->ngroups, "", "", ""); + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_GROUPS", hdr_rec2->ngroups, "", "", ""); printf("+ "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APCXI_CHASSIS", hdr_rec2->nchassis, "", "", ""); + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_CHASSIS", hdr_rec2->nchassis, "", "", ""); printf("+ "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APCXI_SLOTS", hdr_rec2->nslots, "", "", ""); + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_SLOTS", hdr_rec2->nslots, "", "", ""); printf("+ "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APCXI_BLADES", hdr_rec2->nblades, "", "", ""); + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_BLADES", hdr_rec2->nblades, "", "", ""); printf("+ "); DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APCXI_APPLICATION_ID", hdr_rec2->appid, "", "", ""); + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_APPLICATION_ID", hdr_rec2->appid, "", "", ""); } else { @@ -330,58 +385,58 @@ static void darshan_log_print_apcxi_rec_diff(void *file_rec1, char *file_name1, { printf("- "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APCXI_GROUPS", hdr_rec1->ngroups, "", "", ""); + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_GROUPS", hdr_rec1->ngroups, "", "", ""); printf("+ "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APCXI_GROUPS", hdr_rec2->ngroups, "", "", ""); + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_GROUPS", hdr_rec2->ngroups, "", "", ""); } if (hdr_rec1->nchassis != hdr_rec2->nchassis) { printf("- "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APCXI_CHASSIS", hdr_rec1->nchassis, "", "", ""); + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_CHASSIS", hdr_rec1->nchassis, "", "", ""); printf("+ "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APCXI_CHASSIS", hdr_rec2->nchassis, "", "", ""); + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_CHASSIS", hdr_rec2->nchassis, "", "", ""); } if (hdr_rec1->nslots != hdr_rec2->nslots) { printf("- "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APCXI_SLOTS", hdr_rec1->nslots, "", "", ""); + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_SLOTS", hdr_rec1->nslots, "", "", ""); printf("+ "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APCXI_SLOTS", hdr_rec2->nslots, "", "", ""); + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_SLOTS", hdr_rec2->nslots, "", "", ""); } if (hdr_rec1->nblades != hdr_rec2->nblades) { printf("- "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APCXI_BLADES", hdr_rec1->nblades, "", "", ""); + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_BLADES", hdr_rec1->nblades, "", "", ""); printf("+ "); DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APCXI_BLADES", hdr_rec2->nblades, "", "", ""); + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_BLADES", hdr_rec2->nblades, "", "", ""); } if (hdr_rec1->appid != hdr_rec2->appid) { printf("- "); DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "APCXI_APPLICATION_ID", hdr_rec1->appid, "", "", ""); + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_APPLICATION_ID", hdr_rec1->appid, "", "", ""); printf("+ "); DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "APCXI_APPLICATION_ID", hdr_rec2->appid, "", "", ""); + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_APPLICATION_ID", hdr_rec2->appid, "", "", ""); } } } @@ -409,6 +464,10 @@ static void darshan_log_print_apcxi_rec_diff(void *file_rec1, char *file_name1, DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec1->base_rec.rank, prf_rec1->base_rec.id, "APCXI_NODE", prf_rec1->node, "", "", ""); + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APCXI_NUM_NICS", prf_rec1->num_nics, "", "", ""); } else if (!prf_rec1) { @@ -432,6 +491,10 @@ static void darshan_log_print_apcxi_rec_diff(void *file_rec1, char *file_name1, DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], prf_rec2->base_rec.rank, prf_rec2->base_rec.id, "APCXI_NODE", prf_rec2->node, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APCXI_NUM_NICS", prf_rec2->num_nics, "", "", ""); } else { if (prf_rec1->group != prf_rec2->group) @@ -489,41 +552,55 @@ static void darshan_log_print_apcxi_rec_diff(void *file_rec1, char *file_name1, prf_rec2->base_rec.rank, prf_rec2->base_rec.id, "APCXI_NODE", prf_rec2->node, "", "", ""); } - } - - int i; - /* router tile record */ - for(i = 0; i < APCXI_NUM_INDICES; i++) - { - if (!prf_rec2) + if (prf_rec1->num_nics != prf_rec2->num_nics) { printf("- "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - apcxi_counter_names[i], prf_rec1->counters[i], - "", "", ""); - } - else if (!prf_rec1) - { + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APCXI_NUM_NICS", prf_rec1->num_nics, "", "", ""); printf("+ "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - apcxi_counter_names[i], prf_rec2->counters[i], - "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APCXI_NUM_NICS", prf_rec2->num_nics, "", "", ""); } - else if (prf_rec1->counters[i] != prf_rec2->counters[i]) + } + + int i, j; + /* counters from all the available nics */ + for(i = 0; i < prf_rec1->num_nics; i++) + { + for(j = 0; j < APCXI_NUM_INDICES; j++) { - printf("- "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - apcxi_counter_names[i], prf_rec1->counters[i], - "", "", ""); + if (!prf_rec2) + { + printf("- "); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + apcxi_counter_names[j], prf_rec1->counters[i][j], + "", "", ""); + } + else if (!prf_rec1) + { + printf("+ "); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + apcxi_counter_names[j], prf_rec2->counters[i][j], + "", "", ""); + } + else if (prf_rec1->counters[j] != prf_rec2->counters[j]) + { + printf("- "); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + apcxi_counter_names[j], prf_rec1->counters[i][j], + "", "", ""); - printf("+ "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], - prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - apcxi_counter_names[i], prf_rec2->counters[i], - "", "", ""); + printf("+ "); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + apcxi_counter_names[j], prf_rec2->counters[i][j], + "", "", ""); + } } } } diff --git a/apcxi/util/print-apcxi-size.c b/apcxi/util/print-apcxi-size.c index 80cd4af..b99cdfd 100644 --- a/apcxi/util/print-apcxi-size.c +++ b/apcxi/util/print-apcxi-size.c @@ -5,8 +5,8 @@ int main (int argc, char **argv) { printf ("APCXI_NUM_INDICES = %d\n", APCXI_NUM_INDICES); - printf ("sizeof darshan_apcxi_header_record = %d\n", sizeof(struct darshan_apcxi_header_record)); - printf ("sizeof darshan_apcxi_perf_record = %d\n", sizeof(struct darshan_apcxi_perf_record)); + printf ("sizeof darshan_apcxi_header_record = %lu\n", sizeof(struct darshan_apcxi_header_record)); + printf ("sizeof darshan_apcxi_perf_record = %lu\n", sizeof(struct darshan_apcxi_perf_record)); return 0; } diff --git a/apxc/util/darshan-apxc-logutils.c b/apxc/util/darshan-apxc-logutils.c index 87bd65d..606647d 100644 --- a/apxc/util/darshan-apxc-logutils.c +++ b/apxc/util/darshan-apxc-logutils.c @@ -139,8 +139,8 @@ static int darshan_log_get_apxc_rec(darshan_fd fd, void** buf_p) DARSHAN_BSWAP64(&prf_rec->counters[i]); } } + *buf_p = buffer; } - *buf_p = buffer; return(1); } else if (ret < 0)