diff --git a/apcxi/Makefile.darshan b/apcxi/Makefile.darshan new file mode 100644 index 0000000..aeb9981 --- /dev/null +++ b/apcxi/Makefile.darshan @@ -0,0 +1,20 @@ +# +# AutoPerf Make rules for Darshan +# +DARSHAN_STATIC_MOD_OBJS += lib/darshan-apcxi.o +DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-apcxi.po + +VPATH += :$(srcdir)/../modules/autoperf/apcxi +CFLAGS += \ + -DDARSHAN_USE_APCXI \ + -I$(srcdir)/../modules/autoperf/apcxi + +CFLAGS_SHARED += \ + -DDARSHAN_USE_APCXI \ + -I$(srcdir)/../modules/autoperf/apcxi + +lib/darshan-apcxi.o: lib/darshan-apcxi.c lib/darshan-apcxi-utils.h darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) darshan-apcxi-log-format.h | lib + $(CC) $(CFLAGS) -c $< -o $@ + +lib/darshan-apcxi.po: lib/darshan-apcxi.c lib/darshan-apcxi-utils.h darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) darshan-apcxi-log-format.h | lib + $(CC) $(CFLAGS_SHARED) -c $< -o $@ diff --git a/apcxi/darshan-apcxi-log-format.h b/apcxi/darshan-apcxi-log-format.h new file mode 100644 index 0000000..3000b39 --- /dev/null +++ b/apcxi/darshan-apcxi-log-format.h @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2017 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#ifndef __APCXI_LOG_FORMAT_H +#define __APCXI_LOG_FORMAT_H + +/* current AutoPerf Cray XC log format version */ +#define APCXI_VER 1 + +#define APCXI_MAGIC ('A'*0x100000000000000+\ + 'U'*0x1000000000000+\ + 'T'*0x10000000000+\ + 'O'*0x100000000+\ + 'P'*0x1000000+\ + 'E'*0x10000+\ + 'R'*0x100+\ + 'F'*0x1) + +#define APCXI_PERF_COUNTERS \ + /* PAPI counters */\ + X(CQ_CQ_OXE_NUM_STALLS) \ + X(CQ_CQ_OXE_NUM_FLITS) \ + X(CQ_CYCLES_BLOCKED_0) \ + X(CQ_CYCLES_BLOCKED_1) \ + X(CQ_CYCLES_BLOCKED_2) \ + X(CQ_CYCLES_BLOCKED_3) \ + X(HNI_RX_PAUSED_0) \ + X(HNI_RX_PAUSED_1) \ + X(HNI_RX_PAUSED_2) \ + X(HNI_RX_PAUSED_3) \ + X(HNI_RX_PAUSED_4) \ + X(HNI_RX_PAUSED_5) \ + X(HNI_RX_PAUSED_6) \ + X(HNI_RX_PAUSED_7) \ + X(HNI_TX_PAUSED_0) \ + X(HNI_TX_PAUSED_1) \ + X(HNI_TX_PAUSED_2) \ + X(HNI_TX_PAUSED_3) \ + X(HNI_TX_PAUSED_4) \ + X(HNI_TX_PAUSED_5) \ + X(HNI_TX_PAUSED_6) \ + X(HNI_TX_PAUSED_7) \ + X(HNI_PKTS_SENT_BY_TC_0) \ + X(HNI_PKTS_SENT_BY_TC_1) \ + X(HNI_PKTS_SENT_BY_TC_2) \ + X(HNI_PKTS_SENT_BY_TC_3) \ + X(HNI_PKTS_SENT_BY_TC_4) \ + X(HNI_PKTS_SENT_BY_TC_5) \ + X(HNI_PKTS_SENT_BY_TC_6) \ + X(HNI_PKTS_SENT_BY_TC_7) \ + X(HNI_PKTS_RECV_BY_TC_0) \ + X(HNI_PKTS_RECV_BY_TC_1) \ + X(HNI_PKTS_RECV_BY_TC_2) \ + X(HNI_PKTS_RECV_BY_TC_3) \ + X(HNI_PKTS_RECV_BY_TC_4) \ + X(HNI_PKTS_RECV_BY_TC_5) \ + X(HNI_PKTS_RECV_BY_TC_6) \ + X(HNI_PKTS_RECV_BY_TC_7) \ + X(HNI_TX_OK_27) \ + X(HNI_TX_OK_35) \ + X(HNI_TX_OK_36_TO_63) \ + X(HNI_TX_OK_64) \ + X(HNI_TX_OK_65_TO_127) \ + X(HNI_TX_OK_128_TO_255) \ + X(HNI_TX_OK_256_TO_511) \ + X(HNI_TX_OK_512_TO_1023) \ + X(HNI_TX_OK_1024_TO_2047) \ + X(HNI_TX_OK_2048_TO_4095) \ + X(HNI_TX_OK_4096_TO_8191) \ + X(HNI_TX_OK_8192_TO_MAX) \ + /* end of counters */\ + Z(APCXI_NUM_INDICES) + + +#define X(a) a, +#define Z(a) a +/* integer counters for the "APCXI" example module */ +enum darshan_apcxi_perf_indices +{ + APCXI_PERF_COUNTERS +}; + +#undef Z +#undef X + +/* the darshan_apcxi_router_record structure encompasses the data/counters + * which would actually be logged to file by Darshan for the AP HPE Slingshot + * module. This example implementation logs the following data for each + * record: + * - a darshan_base_record structure, which contains the record id & rank + * - the number of the performance counters (see above) monitored is static + * - the number of nic devices on a compute node for which the counters are + * - recorded is not static and is dynamic + * - in order for the record to be contiguous the counters is declared as + * - counters[1][NUM_COUNTERS], the actual record size is calculated based on + * - number of nics + */ +struct darshan_apcxi_perf_record +{ + struct darshan_base_record base_rec; + int64_t group; + int64_t chassis; + int64_t slot; + int64_t blade; + int64_t node; + int64_t num_nics; + uint64_t counters[1][APCXI_NUM_INDICES]; +}; + +struct darshan_apcxi_header_record +{ + struct darshan_base_record base_rec; + int64_t magic; + int64_t nblades; + int64_t nchassis; + int64_t nslots; + int64_t ngroups; + uint64_t appid; +}; + +/* + * helper function to calculate the size of a APCXI record + */ +#define APCXI_PERF_RECORD_SIZE( num_nics ) (sizeof(struct darshan_apcxi_perf_record) + (sizeof(uint64_t) * APCXI_NUM_INDICES * (num_nics - 1))) + +#endif /* __APCXI_LOG_FORMAT_H */ diff --git a/apcxi/lib/crusher_nodelist.h b/apcxi/lib/crusher_nodelist.h new file mode 100644 index 0000000..c4ccf34 --- /dev/null +++ b/apcxi/lib/crusher_nodelist.h @@ -0,0 +1,193 @@ +char crusher_nodelist[192][14] = { +"x2601c0s0b0n0", +"x2601c0s0b1n0", +"x2601c0s1b0n0", +"x2601c0s1b1n0", +"x2601c0s2b0n0", +"x2601c0s2b1n0", +"x2601c0s3b0n0", +"x2601c0s3b1n0", +"x2601c0s4b0n0", +"x2601c0s4b1n0", +"x2601c0s5b0n0", +"x2601c0s5b1n0", +"x2601c0s6b0n0", +"x2601c0s6b1n0", +"x2601c0s7b0n0", +"x2601c0s7b1n0", +"x2601c1s0b0n0", +"x2601c1s0b1n0", +"x2601c1s1b0n0", +"x2601c1s1b1n0", +"x2601c1s2b0n0", +"x2601c1s2b1n0", +"x2601c1s3b0n0", +"x2601c1s3b1n0", +"x2601c1s4b0n0", +"x2601c1s4b1n0", +"x2601c1s5b0n0", +"x2601c1s5b1n0", +"x2601c1s6b0n0", +"x2601c1s6b1n0", +"x2601c1s7b0n0", +"x2601c1s7b1n0", +"x2601c2s0b0n0", +"x2601c2s0b1n0", +"x2601c2s1b0n0", +"x2601c2s1b1n0", +"x2601c2s2b0n0", +"x2601c2s2b1n0", +"x2601c2s3b0n0", +"x2601c2s3b1n0", +"x2601c2s4b0n0", +"x2601c2s4b1n0", +"x2601c2s5b0n0", +"x2601c2s5b1n0", +"x2601c2s6b0n0", +"x2601c2s6b1n0", +"x2601c2s7b0n0", +"x2601c2s7b1n0", +"x2601c3s0b0n0", +"x2601c3s0b1n0", +"x2601c3s1b0n0", +"x2601c3s1b1n0", +"x2601c3s2b0n0", +"x2601c3s2b1n0", +"x2601c3s3b0n0", +"x2601c3s3b1n0", +"x2601c3s4b0n0", +"x2601c3s4b1n0", +"x2601c3s5b0n0", +"x2601c3s5b1n0", +"x2601c3s6b0n0", +"x2601c3s6b1n0", +"x2601c3s7b0n0", +"x2601c3s7b1n0", +"x2601c4s0b0n0", +"x2601c4s0b1n0", +"x2601c4s1b0n0", +"x2601c4s1b1n0", +"x2601c4s2b0n0", +"x2601c4s2b1n0", +"x2601c4s3b0n0", +"x2601c4s3b1n0", +"x2601c4s4b0n0", +"x2601c4s4b1n0", +"x2601c4s5b0n0", +"x2601c4s5b1n0", +"x2601c4s6b0n0", +"x2601c4s6b1n0", +"x2601c4s7b0n0", +"x2601c4s7b1n0", +"x2601c5s0b0n0", +"x2601c5s0b1n0", +"x2601c5s1b0n0", +"x2601c5s1b1n0", +"x2601c5s2b0n0", +"x2601c5s2b1n0", +"x2601c5s3b0n0", +"x2601c5s3b1n0", +"x2601c5s4b0n0", +"x2601c5s4b1n0", +"x2601c5s5b0n0", +"x2601c5s5b1n0", +"x2601c5s6b0n0", +"x2601c5s6b1n0", +"x2601c5s7b0n0", +"x2601c5s7b1n0", +"x2601c6s0b0n0", +"x2601c6s0b1n0", +"x2601c6s1b0n0", +"x2601c6s1b1n0", +"x2601c6s2b0n0", +"x2601c6s2b1n0", +"x2601c6s3b0n0", +"x2601c6s3b1n0", +"x2601c6s4b0n0", +"x2601c6s4b1n0", +"x2601c6s5b0n0", +"x2601c6s5b1n0", +"x2601c6s6b0n0", +"x2601c6s6b1n0", +"x2601c6s7b0n0", +"x2601c6s7b1n0", +"x2601c7s0b0n0", +"x2601c7s0b1n0", +"x2601c7s1b0n0", +"x2601c7s1b1n0", +"x2601c7s2b0n0", +"x2601c7s2b1n0", +"x2601c7s3b0n0", +"x2601c7s3b1n0", +"x2601c7s4b0n0", +"x2601c7s4b1n0", +"x2601c7s5b0n0", +"x2601c7s5b1n0", +"x2601c7s6b0n0", +"x2601c7s6b1n0", +"x2601c7s7b0n0", +"x2601c7s7b1n0", +"x2602c0s0b0n0", +"x2602c0s0b1n0", +"x2602c0s1b0n0", +"x2602c0s1b1n0", +"x2602c0s2b0n0", +"x2602c0s2b1n0", +"x2602c0s3b0n0", +"x2602c0s3b1n0", +"x2602c0s4b0n0", +"x2602c0s4b1n0", +"x2602c0s5b0n0", +"x2602c0s5b1n0", +"x2602c0s6b0n0", +"x2602c0s6b1n0", +"x2602c0s7b0n0", +"x2602c0s7b1n0", +"x2602c1s0b0n0", +"x2602c1s0b1n0", +"x2602c1s1b0n0", +"x2602c1s1b1n0", +"x2602c1s2b0n0", +"x2602c1s2b1n0", +"x2602c1s3b0n0", +"x2602c1s3b1n0", +"x2602c1s4b0n0", +"x2602c1s4b1n0", +"x2602c1s5b0n0", +"x2602c1s5b1n0", +"x2602c1s6b0n0", +"x2602c1s6b1n0", +"x2602c1s7b0n0", +"x2602c1s7b1n0", +"x2602c2s0b0n0", +"x2602c2s0b1n0", +"x2602c2s1b0n0", +"x2602c2s1b1n0", +"x2602c2s2b0n0", +"x2602c2s2b1n0", +"x2602c2s3b0n0", +"x2602c2s3b1n0", +"x2602c2s4b0n0", +"x2602c2s4b1n0", +"x2602c2s5b0n0", +"x2602c2s5b1n0", +"x2602c2s6b0n0", +"x2602c2s6b1n0", +"x2602c2s7b0n0", +"x2602c2s7b1n0", +"x2602c3s0b0n0", +"x2602c3s0b1n0", +"x2602c3s1b0n0", +"x2602c3s1b1n0", +"x2602c3s2b0n0", +"x2602c3s2b1n0", +"x2602c3s3b0n0", +"x2602c3s3b1n0", +"x2602c3s4b0n0", +"x2602c3s4b1n0", +"x2602c3s5b0n0", +"x2602c3s5b1n0", +"x2602c3s6b0n0", +"x2602c3s6b1n0", +"x2602c3s7b0n0", +"x2602c3s7b1n0"}; diff --git a/apss/lib/darshan-apss-utils.h b/apcxi/lib/darshan-apcxi-utils.h similarity index 66% rename from apss/lib/darshan-apss-utils.h rename to apcxi/lib/darshan-apcxi-utils.h index 8ce7378..b1b85fd 100644 --- a/apss/lib/darshan-apss-utils.h +++ b/apcxi/lib/darshan-apcxi-utils.h @@ -1,8 +1,9 @@ -#ifndef __APSS_UTILS_H__ -#define __APSS_UTILS_H__ +#ifndef __APCXI_UTILS_H__ +#define __APCXI_UTILS_H__ #include - +#include +extern char crusher_nodelist[192][14]; static void search_hwinfo(const char * mstr, char *mode) { FILE *f; @@ -33,29 +34,30 @@ static void search_hwinfo(const char * mstr, char *mode) return; } -static int sstopo_get_mycoords(int *rack, int *chassis, int *blade, int *node) +static int sstopo_get_mycoords(int* group, int* chassis, int* slot, int* blade, int* node) { char hostname[HOST_NAME_MAX + 1]; gethostname(hostname, HOST_NAME_MAX + 1); char a, b, c, d, e; int layer_temp; - int rack, chassis, blade, anode; + int rack; /* format example: c1-0c1s2n1 c3-0c2s15n3 */ /* format example: x3012c0s13b0n0 */ - sscanf(hostname, + sscanf(crusher_nodelist[atoi(hostname+7)], "%c%d%c%d%c%d%c%d%c%d", - &a, &rack, &b, &layer_temp, &c, &chassis, &d, &blade, &e, &anode); + &a, &rack, &b, chassis, &c, slot, &d, blade, &e, node); + if(rack == 2601){ + *group = 0; + } + else{ + *group = 1; + } #ifdef DEBUG - fprintf(stderr, "coords = (%d,%d, %d, %d,%d,%d,%d) \n", layer_rank, blade_rank, node_rank, rack, chassis, blade, anode); + fprintf(stderr, "coords = (%d,%d, %d, %d,%d,%d,%d) \n", layer_rank, blade_rank, node_rank, *group, *chassis, *slot, *blade, *node); #endif - rack = rack; - chassis = chassis; - blade = blade; - node = anode; - return 0; } @@ -71,4 +73,13 @@ static unsigned int count_bits(unsigned int *bitvec, int cnt) return count; } +void strlwr(char *str) +{ + char *p = str; + while (*p!='\0') { + *p = tolower((char)*p); + p++; + } +} + #endif diff --git a/apcxi/lib/darshan-apcxi.c b/apcxi/lib/darshan-apcxi.c new file mode 100644 index 0000000..e013625 --- /dev/null +++ b/apcxi/lib/darshan-apcxi.c @@ -0,0 +1,551 @@ +/* + * Copyright (C) 2017 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#define _XOPEN_SOURCE 500 +#define _GNU_SOURCE + +/* + * TODO: A job id given by the job scheduler uniquely identifies a job at a system level, + * similarly, the application launcher id uniquely identifies an application + * execution on the system. + */ +//#ifdef SLURM_JOBID +#define APP_ID "SLURM_TASK_PID" +//#endif +//#ifdef PBS_JOBID +//#define APP_ID "PALS_APP_ID +//#endif + +#include "darshan-runtime-config.h" +#include +#include +#include +#include +#include +#include + +#include "uthash.h" +#include "darshan.h" +#include "darshan-dynamic.h" +#include "darshan-apcxi-log-format.h" + +#include "darshan-apcxi-utils.h" +#include "crusher_nodelist.h" +#define MAX_COUNTER_NAME_LENGTH 100 + +/* + * apcxi_events are defined by the Cassini counters listed in the log header. + */ +#define X(a) #a, +#define Z(a) #a +static char* apcxi_events[] = +{ + APCXI_PERF_COUNTERS +}; +#undef X +#undef Z + +char ***apcxi_filenames; + +#define MAX_GROUPS (128) +#define MAX_CHASSIS (MAX_GROUPS*8) +#define MAX_SLOTS (MAX_CHASSIS*8) +#define MAX_BLADES (MAX_SLOTS*2) + +/* + * + * + * This module does not intercept any system calls. It just pulls data + * from the personality struct at initialization. + */ + + +/* + * Global runtime struct for tracking data needed at runtime + */ +struct apcxi_runtime +{ + struct darshan_apcxi_header_record *header_record; + struct darshan_apcxi_perf_record *perf_record; + darshan_record_id header_id; + darshan_record_id rtr_id; + int num_counters; + int group; + int chassis; + int slot; + int blade; + int node; + int num_nics; + int perf_record_marked; +}; + +static struct apcxi_runtime *apcxi_runtime = NULL; +static pthread_mutex_t apcxi_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; + +/* my_rank indicates the MPI rank of this process */ +static int my_rank = -1; + +/* internal helper functions for the APCXI module */ +void apcxi_runtime_initialize(void); + +/* forward declaration for shutdown function needed to interface with darshan-core */ +//#ifdef HAVE_MPI +static void apcxi_mpi_redux( + void *buffer, + MPI_Comm mod_comm, + darshan_record_id *shared_recs, + int shared_rec_count); +//#endif +static void apcxi_output( + void **buffer, + int *size); +static void apcxi_cleanup( + void); + +/* macros for obtaining/releasing the APCXI module lock */ +#define APCXI_LOCK() pthread_mutex_lock(&apcxi_runtime_mutex) +#define APCXI_UNLOCK() pthread_mutex_unlock(&apcxi_runtime_mutex) + +/* + * Initialize counters + * determine the number of nic devices on the node + * build the list of files to read from the sysfs telemetry + */ +static void initialize_counters (void) +{ + int i; + int code = 0; + + + /* start with first apcxi counter */ + i = CQ_CQ_OXE_NUM_STALLS; + while(strcmp(apcxi_events[i], "APCXI_NUM_INDICES") != 0){ + i++; + } + apcxi_runtime->num_counters = i; + + // allocate memory space for the sys fs file names + // for each of the available nics (num_nics) + apcxi_filenames = (char***) malloc(sizeof(char**) * apcxi_runtime->num_nics); + for(int i=0; inum_nics; i++){ + apcxi_filenames[i] = (char**) malloc(sizeof(char*) * apcxi_runtime->num_counters); + for(int j=0; jnum_counters; j++){ + apcxi_filenames[i][j] = (char*) malloc(sizeof(char) * MAX_COUNTER_NAME_LENGTH); + } + } + + char line[MAX_COUNTER_NAME_LENGTH]; + for(int i=0; inum_counters; i++) + { + size_t ln = strlen(apcxi_events[i]); + strcpy(line, apcxi_events[i]); + strlwr(line); + line[ln] = '\0'; + for(int j=0; jnum_nics; j++){ + // /sys/class/cxi/cxi[0..3] + char str[MAX_COUNTER_NAME_LENGTH] = ""; + char buf[5]; + snprintf(buf, sizeof(buf), "%d", j); + strcat(str, "/sys/class/cxi/cxi"); + strcat(str, buf); + strcat(str, "/device/telemetry/"); + strcat(str, line); + strcpy(apcxi_filenames[j][i],str); + } + } + + FILE *fff; + for(int i=0; inum_nics; i++){ + for(int j=0; jnum_counters; j++){ + fff = fopen(apcxi_filenames[i][j], "r"); + if (fff == NULL) { + fprintf(stderr,"\tError opening %s!\n",apcxi_filenames[i][j]); + } + else { + uint64_t b, c; + fscanf(fff,"%lu@%lu.%lu",&apcxi_runtime->perf_record->counters[i][j], &b, &c); + fclose(fff); + } + } + } + return; +} + +static void finalize_counters (void) +{ + // free apcxi_filenames + for(int i=0; inum_nics; i++){ + for(int j=0; jnum_counters; j++){ + free(apcxi_filenames[i][j]); + } + free(apcxi_filenames[i]); + } + free(apcxi_filenames); + return; +} + +/* + * Function which updates all the counter data + */ +static void capture(struct darshan_apcxi_perf_record *rec, + darshan_record_id rec_id) +{ + FILE *fff; + for(int i=0; inum_nics; i++){ + for(int j=0; jnum_counters; j++){ + fff = fopen(apcxi_filenames[i][j], "r"); + if (fff == NULL) { + fprintf(stderr,"\tError opening %s!\n",apcxi_filenames[i][j]); + } + else { + uint64_t a, b, c; + fscanf(fff,"%lu@%lu.%lu", &a, &b, &c); + apcxi_runtime->perf_record->counters[i][j] = a - apcxi_runtime->perf_record->counters[i][j]; + fclose(fff); + } + } + } + + rec->group = apcxi_runtime->group; + rec->chassis = apcxi_runtime->chassis; + rec->slot = apcxi_runtime->slot; + rec->blade = apcxi_runtime->blade; + rec->node = apcxi_runtime->node; + rec->num_nics = apcxi_runtime->num_nics; + rec->base_rec.id = rec_id; + rec->base_rec.rank = my_rank; + + return; +} + +void apcxi_runtime_initialize() +{ + size_t apcxi_buf_size; + char rtr_rec_name[128]; + darshan_module_funcs mod_funcs = { + //#ifdef HAVE_MPI + .mod_redux_func = &apcxi_mpi_redux, + //#endif + .mod_output_func = &apcxi_output, + .mod_cleanup_func = &apcxi_cleanup + }; + + APCXI_LOCK(); + + /* + * Determine the number of available NIC devices on the compute node + * read the number of dirs in /sys/class/cxi/ + */ + int num_nics = 0; + struct dirent *dp; + DIR *fdir; + if ((fdir = opendir("/sys/class/cxi")) == NULL) { + fprintf(stderr, "listdir: can't open %s\n", "/sys/class/cxi"); + return; + } + while ((dp = readdir(fdir)) != NULL) { + if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, "..")) + continue; /* skip self and parent */ + num_nics++; + } + closedir(fdir); + + + /* don't do anything if already initialized */ + if(apcxi_runtime) + { + APCXI_UNLOCK(); + return; + } + + + apcxi_buf_size = sizeof(struct darshan_apcxi_header_record) + + APCXI_PERF_RECORD_SIZE(num_nics); + + /* register the APCXI module with the darshan-core component */ + darshan_core_register_module( + DARSHAN_APCXI_MOD, + mod_funcs, + &apcxi_buf_size, + &my_rank, + NULL); + + /* initialize module's global state */ + apcxi_runtime = malloc(sizeof(*apcxi_runtime)); + if(!apcxi_runtime) + { + darshan_core_unregister_module(DARSHAN_APCXI_MOD); + APCXI_UNLOCK(); + return; + } + memset(apcxi_runtime, 0, sizeof(*apcxi_runtime)); + apcxi_runtime->num_nics = num_nics; + + if (my_rank == 0) + { + apcxi_runtime->header_id = darshan_core_gen_record_id("darshan-apcxi-header"); + + /* register the apcxi file record with darshan-core */ + apcxi_runtime->header_record = darshan_core_register_record( + apcxi_runtime->header_id, + //NULL, + "darshan-apcxi-header", + DARSHAN_APCXI_MOD, + sizeof(struct darshan_apcxi_header_record), + NULL); + if(!(apcxi_runtime->header_record)) + { + darshan_core_unregister_module(DARSHAN_APCXI_MOD); + free(apcxi_runtime); + apcxi_runtime = NULL; + APCXI_UNLOCK(); + return; + } + apcxi_runtime->header_record->base_rec.id = apcxi_runtime->header_id; + apcxi_runtime->header_record->base_rec.rank = my_rank; + apcxi_runtime->header_record->magic = APCXI_MAGIC; + } + sstopo_get_mycoords(&apcxi_runtime->group, + &apcxi_runtime->chassis, + &apcxi_runtime->slot, + &apcxi_runtime->blade, + &apcxi_runtime->node); + sprintf(rtr_rec_name, "darshan-apcxi-rtr-%d-%d-%d-%d", + apcxi_runtime->group, apcxi_runtime->chassis, apcxi_runtime->slot, apcxi_runtime->blade); + //apcxi_runtime->rtr_id = darshan_core_gen_record_id(rtr_rec_name); + apcxi_runtime->rtr_id = darshan_core_gen_record_id("APCXI"); + apcxi_runtime->perf_record = darshan_core_register_record( + apcxi_runtime->rtr_id, + //NULL, + "APCXI", // we want the record for each rank to be treated as shared records so that mpi_redux can operate on + //rtr_rec_name, + DARSHAN_APCXI_MOD, + APCXI_PERF_RECORD_SIZE(num_nics), + //sizeof(struct darshan_apcxi_perf_record), + NULL); + if(!(apcxi_runtime->perf_record)) + { + darshan_core_unregister_module(DARSHAN_APCXI_MOD); + free(apcxi_runtime); + apcxi_runtime = NULL; + APCXI_UNLOCK(); + return; + } + apcxi_runtime->perf_record->num_nics = num_nics; + initialize_counters(); + APCXI_UNLOCK(); + + return; +} + +/******************************************************************************** + * shutdown function exported by this module for coordinating with darshan-core * + ********************************************************************************/ + +/* Pass data for the apcxi module back to darshan-core to log to file. */ +//#ifdef HAVE_MPI +static void apcxi_mpi_redux( + void *apcxi_buf, + MPI_Comm mod_comm, + darshan_record_id *shared_recs, + int shared_rec_count) +{ + int i; + int color; + int router_rank; + int router_count; + int chassis_count; + int slot_count; + int group_count; + unsigned int *bitvec; + unsigned int bitlen; + unsigned int bitcnt; + unsigned int bitsiz; + MPI_Comm router_comm; + + APCXI_LOCK(); + if (!apcxi_runtime) + { + APCXI_UNLOCK(); + return; + } + + bitcnt = sizeof(unsigned int) * 8; + bitlen = sizeof(unsigned int) * (MAX_BLADES/bitcnt); + bitsiz = bitlen / sizeof(unsigned int); + bitvec = malloc(bitlen); + + /* collect perf counters */ + capture(apcxi_runtime->perf_record, apcxi_runtime->rtr_id); + + if (my_rank == 0) + { + apcxi_runtime->header_record->appid = atoi((char*)getenv( APP_ID )); + } +#if 1 + /* count network dimensions */ + if (bitvec) + { + int idx; + unsigned int uchassis; + unsigned int uslot; + unsigned int ublade; + + /* group */ + memset(bitvec, 0, bitlen); + idx = apcxi_runtime->group / bitcnt; + bitvec[idx] |= (1 << apcxi_runtime->group % bitcnt); + PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), + bitvec, + bitsiz, + MPI_INT, + MPI_BOR, + 0, + MPI_COMM_WORLD); + group_count = count_bits(bitvec, bitsiz); + + /* chassis */ + memset(bitvec, 0, bitlen); + uchassis = apcxi_runtime->group * 8 + apcxi_runtime->chassis; + idx = uchassis / bitcnt; + bitvec[idx] |= (1 << uchassis % bitcnt); + PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), + bitvec, + bitsiz, + MPI_INT, + MPI_BOR, + 0, + MPI_COMM_WORLD); + chassis_count = count_bits(bitvec, bitsiz); + + /* slot */ + memset(bitvec, 0, bitlen); + uslot = uchassis * 8 + apcxi_runtime->slot; + idx = uslot / bitcnt; + bitvec[idx] |= (1 << uslot % bitcnt); + PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), + bitvec, + bitsiz, + MPI_INT, + MPI_BOR, + 0, + MPI_COMM_WORLD); + slot_count = count_bits(bitvec, bitsiz); + + /* blade */ + memset(bitvec, 0, bitlen); + ublade = uchassis * 8 + uslot * 8 + apcxi_runtime->blade; + idx = ublade / bitcnt; + bitvec[idx] |= (1 << ublade % bitcnt); + PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), + bitvec, + bitsiz, + MPI_INT, + MPI_BOR, + 0, + MPI_COMM_WORLD); + router_count = count_bits(bitvec, bitsiz); + + if (my_rank == 0) + { + apcxi_runtime->header_record->nblades = router_count; + apcxi_runtime->header_record->nchassis = chassis_count; + apcxi_runtime->header_record->nslots = slot_count; + apcxi_runtime->header_record->ngroups = group_count; + } + free(bitvec); + } + else + { + apcxi_runtime->header_record->nblades = 0; + apcxi_runtime->header_record->nslots = 0; + apcxi_runtime->header_record->nchassis = 0; + apcxi_runtime->header_record->ngroups = 0; + } + + /* + * reduce data + * + * aggregate data from processes which share the same node and avg. + * create a communicator with all the ranks that share the same node + * + */ + color = (apcxi_runtime->group << (7+3)) + \ + (apcxi_runtime->chassis << (3+4)) + \ + (apcxi_runtime->slot << (3+1)) + \ + (apcxi_runtime->blade << 1) + apcxi_runtime->node; +#endif +#if 1 + PMPI_Comm_split(MPI_COMM_WORLD, color, my_rank, &router_comm); + PMPI_Comm_rank(router_comm, &router_rank); + PMPI_Comm_size(router_comm, &router_count); + /* + PMPI_Reduce((router_rank?apcxi_runtime->perf_record->counters:MPI_IN_PLACE), + apcxi_runtime->perf_record->counters, + APCXI_NUM_INDICES, + MPI_LONG_LONG_INT, + MPI_SUM, + 0, + router_comm); + */ + if (router_rank == 0) + { + /* + for (i = 0; i < APCXI_NUM_INDICES; i++) + { + apcxi_runtime->perf_record->counters[i] /= router_count; + } + */ + apcxi_runtime->perf_record_marked = -1; + } + PMPI_Comm_free(&router_comm); +#endif + APCXI_UNLOCK(); + return; +} + +//#endif +static void apcxi_output( + void **apcxi_buf, + int *apcxi_buf_sz) +{ + APCXI_LOCK(); + assert(apcxi_runtime); + *apcxi_buf_sz = 0; + + if (my_rank == 0) { + *apcxi_buf_sz += sizeof(*apcxi_runtime->header_record); + } + + if (apcxi_runtime->perf_record_marked == -1) + { + //*apcxi_buf_sz += sizeof( *apcxi_runtime->perf_record); + *apcxi_buf_sz += APCXI_PERF_RECORD_SIZE(apcxi_runtime->num_nics); + } + + APCXI_UNLOCK(); + return; +} + +static void apcxi_cleanup() +{ + APCXI_LOCK(); + assert(apcxi_runtime); + finalize_counters(); + free(apcxi_runtime); + apcxi_runtime = NULL; + APCXI_UNLOCK(); + return; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * End: + * + * vim: ts=8 sts=4 sw=4 expandtab + */ diff --git a/apcxi/util/Makefile.darshan b/apcxi/util/Makefile.darshan new file mode 100644 index 0000000..3953b88 --- /dev/null +++ b/apcxi/util/Makefile.darshan @@ -0,0 +1,27 @@ +# +# AutoPerf Make rules for Darshan +# + +VPATH += :$(srcdir)/../modules/autoperf/apcxi/util + +DARSHAN_MOD_LOG_FORMATS += $(srcdir)/../modules/autoperf/apcxi/darshan-apcxi-log-format.h +DARSHAN_MOD_LOGUTIL_HEADERS += darshan-apcxi-logutils.h +DARSHAN_STATIC_MOD_OBJS += darshan-apcxi-logutils.o +DARSHAN_DYNAMIC_MOD_OBJS += darshan-apcxi-logutils.po + +CFLAGS += \ + -DDARSHAN_USE_apcxi \ + -I$(srcdir)/../modules/autoperf/apcxi \ + -I$(srcdir)/../modules/autoperf/apcxi/util + +CFLAGS_SHARED += \ + -DDARSHAN_USE_apcxi \ + -I$(srcdir)/../modules/autoperf/apcxi \ + -I$(srcdir)/../modules/autoperf/apcxi/util + +darshan-apcxi-logutils.o: darshan-apcxi-logutils.c darshan-logutils.h darshan-apcxi-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../modules/autoperf/apcxi/darshan-apcxi-log-format.h | uthash-1.9.2 + $(CC) $(CFLAGS) -c $< -o $@ + +darshan-apcxi-logutils.po: darshan-apcxi-logutils.c darshan-logutils.h darshan-apcxi-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../modules/autoperf/apcxi/darshan-apcxi-log-format.h | uthash-1.9.2 + $(CC) $(CFLAGS_SHARED) -c $< -o $@ + diff --git a/apss/util/apss-analysis.py b/apcxi/util/apcxi-analysis.py similarity index 76% rename from apss/util/apss-analysis.py rename to apcxi/util/apcxi-analysis.py index 854562e..e02eda2 100755 --- a/apss/util/apss-analysis.py +++ b/apcxi/util/apcxi-analysis.py @@ -1,9 +1,9 @@ #!/usr/bin/env python -# DarshanUtils for Python for processing APSS records +# DarshanUtils for Python for processing APCXI records # This script gives an overview of features provided by the Python bindings for DarshanUtils. -# By default all APSS module records, metadata, and the name records are loaded when opening a Darshan log: +# By default all APCXI module records, metadata, and the name records are loaded when opening a Darshan log: import argparse import darshan @@ -48,19 +48,19 @@ def main(): report = darshan.DarshanReport(args.logname[0], read_all=False) report.info() - if "APSS" not in report.modules: + if "APCXI" not in report.modules: print("This log does not contain AutoPerf XC data") return - r = report.mod_read_all_apss_records("APSS") + r = report.mod_read_all_apcxi_records("APCXI") report.update_name_records() report.info() - #pdf = matplotlib.backends.backend_pdf.PdfPages("apss_output.pdf") + #pdf = matplotlib.backends.backend_pdf.PdfPages("apcxi_output.pdf") - header_rec = report.records["APSS"][0] + header_rec = report.records["APCXI"][0] - for rec in report.records["APSS"][1:]: + for rec in report.records["APCXI"][1:]: # skip the first record which is header record print(rec) diff --git a/apss/util/apss-backend.py b/apcxi/util/apcxi-backend.py similarity index 85% rename from apss/util/apss-backend.py rename to apcxi/util/apcxi-backend.py index d6dbc2f..a18bf83 100644 --- a/apss/util/apss-backend.py +++ b/apcxi/util/apcxi-backend.py @@ -4,9 +4,9 @@ import numpy as np import darshan.backend.cffi_backend -# apss structure defs +# apcxi structure defs structdefs = ''' -struct darshan_apss_perf_record +struct darshan_apcxi_perf_record { struct darshan_base_record base_rec; int64_t group; @@ -15,7 +15,7 @@ int64_t node; uint64_t counters[10]; }; -struct darshan_apss_header_record +struct darshan_apcxi_header_record { struct darshan_base_record base_rec; int64_t magic; @@ -25,16 +25,16 @@ uint64_t appid; }; -extern char *apss_counter_names[]; +extern char *apcxi_counter_names[]; ''' -def get_apss_defs(): +def get_apcxi_defs(): return structdefs # load header record -def log_get_apss_record(log, mod_name, structname, dtype='dict'): +def log_get_apcxi_record(log, mod_name, structname, dtype='dict'): from darshan.backend.cffi_backend import ffi, libdutil, log_get_modules, counter_names, _structdefs modules = log_get_modules(log) @@ -47,7 +47,7 @@ def log_get_apss_record(log, mod_name, structname, dtype='dict'): if r < 1: return None - if mod_type == 'struct darshan_apss_header_record **': + if mod_type == 'struct darshan_apcxi_header_record **': hdr = ffi.cast(mod_type, buf) rec['id'] = hdr[0].base_rec.id rec['rank'] = hdr[0].base_rec.rank diff --git a/apcxi/util/darshan-apcxi-logutils.c b/apcxi/util/darshan-apcxi-logutils.c new file mode 100644 index 0000000..db77869 --- /dev/null +++ b/apcxi/util/darshan-apcxi-logutils.c @@ -0,0 +1,618 @@ +/* + * Copyright (C) 2018 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#define _GNU_SOURCE +#include "darshan-util-config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "darshan-logutils.h" +#include "darshan-apcxi-log-format.h" + +/* counter name strings for the apcxi module */ +#define Y(a) #a, +#define X(a) Y(APCXI_ ## a) +#define Z(a) #a +char *apcxi_counter_names[] = { + APCXI_PERF_COUNTERS +}; +#undef Y +#undef X +#undef Z + +static int darshan_log_get_apcxi_rec(darshan_fd fd, void** buf_p); +static int darshan_log_put_apcxi_rec(darshan_fd fd, void* buf); +static void darshan_log_print_apcxi_rec(void *file_rec, + char *file_name, char *mnt_pt, char *fs_type); +static void darshan_log_print_apcxi_description(int ver); +static void darshan_log_print_apcxi_rec_diff(void *file_rec1, char *file_name1, + void *file_rec2, char *file_name2); + +struct darshan_mod_logutil_funcs apcxi_logutils = +{ + .log_get_record = &darshan_log_get_apcxi_rec, + .log_put_record = &darshan_log_put_apcxi_rec, + .log_print_record = &darshan_log_print_apcxi_rec, + .log_print_description = &darshan_log_print_apcxi_description, + .log_print_diff = &darshan_log_print_apcxi_rec_diff, + .log_agg_records = NULL +}; + +static int darshan_log_get_apcxi_rec(darshan_fd fd, void** buf_p) +{ + struct darshan_apcxi_header_record *hdr_rec; //= *((struct darshan_apcxi_header_record **)buf_p);; + struct darshan_apcxi_perf_record *tmp_rec; //= *((struct darshan_apcxi_perf_record **)buf_p); + struct darshan_apcxi_perf_record *prf_rec; + + int rec_len; + char *buffer = NULL; + int i; + int ret = -1; + static int first_rec = 1; + + if(fd->mod_map[DARSHAN_APCXI_MOD].len == 0) + return(0); + + if(fd->mod_ver[DARSHAN_APCXI_MOD] == 0 || + fd->mod_ver[DARSHAN_APCXI_MOD] > APCXI_VER) + { + fprintf(stderr, "Error: Invalid APCXI module version number (got %d)\n", + fd->mod_ver[DARSHAN_APCXI_MOD]); + return(-1); + } + + if (!*buf_p) + { + /* assume this is the largest possible record size */ + buffer = malloc(sizeof(struct darshan_apcxi_perf_record)); + if (!buffer) + { + return(-1); + } + } + else + { + buffer = *buf_p; + } + + if (fd->mod_ver[DARSHAN_APCXI_MOD] < APCXI_VER) + { + /* perform conversion as needed */ + } + + /* v1, current version */ + if (fd->mod_ver[DARSHAN_APCXI_MOD] == APCXI_VER) + { + if (first_rec) + { + rec_len = sizeof(struct darshan_apcxi_header_record); + first_rec = 0; + } + else{ + /* retrieve the fixed-size portion of the record */ + rec_len = sizeof(struct darshan_apcxi_perf_record); + } + ret = darshan_log_get_mod(fd, DARSHAN_APCXI_MOD, buffer, rec_len); + } + if(ret < 0){ + return(-1); + } + + + if (ret == rec_len) + { + if(fd->swap_flag) + { + if (rec_len == sizeof(struct darshan_apcxi_header_record)) + { + hdr_rec = (struct darshan_apcxi_header_record*)buffer; + /* swap bytes if necessary */ + DARSHAN_BSWAP64(&(hdr_rec->base_rec.id)); + DARSHAN_BSWAP64(&(hdr_rec->base_rec.rank)); + DARSHAN_BSWAP64(&(hdr_rec->magic)); + DARSHAN_BSWAP64(&(hdr_rec->nblades)); + DARSHAN_BSWAP64(&(hdr_rec->nslots)); + DARSHAN_BSWAP64(&(hdr_rec->nchassis)); + DARSHAN_BSWAP64(&(hdr_rec->ngroups)); + DARSHAN_BSWAP64(&(hdr_rec->appid)); + } + else + { + tmp_rec = (struct darshan_apcxi_perf_record*)buffer; + DARSHAN_BSWAP64(&(tmp_rec->base_rec.id)); + DARSHAN_BSWAP64(&(tmp_rec->base_rec.rank)); + DARSHAN_BSWAP64(&(tmp_rec->group)); + DARSHAN_BSWAP64(&(tmp_rec->chassis)); + DARSHAN_BSWAP64(&(tmp_rec->slot)); + DARSHAN_BSWAP64(&(tmp_rec->blade)); + DARSHAN_BSWAP64(&(tmp_rec->node)); + DARSHAN_BSWAP64(&(tmp_rec->num_nics)); + for (i = 0; i < APCXI_NUM_INDICES; i++) + { + DARSHAN_BSWAP64(&tmp_rec->counters[0][i]); + } + } + } + + if (rec_len != sizeof(struct darshan_apcxi_header_record)){ + tmp_rec = (struct darshan_apcxi_perf_record*)buffer; + if(*buf_p == NULL){ + prf_rec = malloc(APCXI_PERF_RECORD_SIZE(tmp_rec->num_nics)); + if(!prf_rec) + return(-1); + memcpy(prf_rec, tmp_rec, sizeof(struct darshan_apcxi_perf_record)); + } + else{ + prf_rec = *buf_p; + } + + /* now read the rest of the record */ + if(prf_rec->num_nics > 1){ + ret = darshan_log_get_mod(fd, DARSHAN_APCXI_MOD, + (void*)(&(prf_rec->counters[1][0])), + ((prf_rec->num_nics-1) * sizeof(uint64_t) * APCXI_NUM_INDICES) + ); + + if(ret < ((prf_rec->num_nics-1) * sizeof(uint64_t) * APCXI_NUM_INDICES)){ + ret = -1; + } + else{ + ret = 1; + /* swap bytes if necessary */ + if ( fd->swap_flag ) { + for(int i=1; inum_nics; i++){ + for (int j = 0; j < APCXI_NUM_INDICES; j++){ + DARSHAN_BSWAP64(&tmp_rec->counters[i][j]); + } + } + } + } + } + if(*buf_p == NULL){ + *buf_p = prf_rec; + } + } + ret = 1; + } + else if (ret < 0) + { + //*buf_p = NULL; + // if (buffer) free(buffer); + if (!*buf_p) free(buffer); + return(-1); + } + else + { + // *buf_p = NULL; + // if (buffer) free(buffer); + if (!*buf_p) free(buffer); + return(0); + } + return ret; +} + +static int darshan_log_put_apcxi_rec(darshan_fd fd, void* buf) +{ + int ret; + int rec_len; + static int first_rec = 1; + + if (first_rec) + { + rec_len = sizeof(struct darshan_apcxi_header_record); + first_rec = 0; + } + else + rec_len = sizeof(struct darshan_apcxi_perf_record); + + ret = darshan_log_put_mod(fd, DARSHAN_APCXI_MOD, buf, + rec_len, APCXI_VER); + if(ret < 0) + return(-1); + + return(0); +} + +static void darshan_log_print_apcxi_rec(void *rec, char *file_name, + char *mnt_pt, char *fs_type) +{ + int i; + static int first_rec = 1; + struct darshan_apcxi_header_record *hdr_rec; + struct darshan_apcxi_perf_record *prf_rec; + + if (first_rec) + { + hdr_rec = rec; + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec->base_rec.rank, hdr_rec->base_rec.id, + "APCXI_GROUPS", hdr_rec->ngroups, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec->base_rec.rank, hdr_rec->base_rec.id, + "APCXI_CHASSIS", hdr_rec->nchassis, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec->base_rec.rank, hdr_rec->base_rec.id, + "APCXI_SLOTS", hdr_rec->nslots, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec->base_rec.rank, hdr_rec->base_rec.id, + "APCXI_BLADES", hdr_rec->nblades, "", "", ""); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec->base_rec.rank, hdr_rec->base_rec.id, + "APCXI_APPLICATION_ID", hdr_rec->appid, "", "", ""); + first_rec = 0; + } + else + { + prf_rec = rec; + + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec->base_rec.rank, prf_rec->base_rec.id, + "APCXI_GROUP", prf_rec->group, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec->base_rec.rank, prf_rec->base_rec.id, + "APCXI_CHASSIS", prf_rec->chassis, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec->base_rec.rank, prf_rec->base_rec.id, + "APCXI_SLOT", prf_rec->slot, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec->base_rec.rank, prf_rec->base_rec.id, + "APCXI_BLADE", prf_rec->blade, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec->base_rec.rank, prf_rec->base_rec.id, + "APCXI_NODE", prf_rec->node, "", "", ""); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec->base_rec.rank, prf_rec->base_rec.id, + "APCXI_NUM_NICS", prf_rec->num_nics, "", "", ""); + + for(i = 0; i < prf_rec->num_nics; i++){ + DARSHAN_I_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec->base_rec.rank, prf_rec->base_rec.id, + "APCXI_NIC_ID", i, "", "", ""); + for(int j = 0; j < APCXI_NUM_INDICES; j++){ + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec->base_rec.rank, prf_rec->base_rec.id, + apcxi_counter_names[j], prf_rec->counters[i][j], + "", "", ""); + } + } + + } + + return; +} + +static void darshan_log_print_apcxi_description(int ver) +{ + printf("\n# description of apcxi counters:\n"); + printf("# global summary stats for the apcxi module:\n"); + printf("# APCXI_GROUPS: total number of groups\n"); + printf("# APCXI_CHASSIS: total number of chassis\n"); + printf("# APCXI_BLADES: total number of blades\n"); + printf("# APCXI_SLOTS: total number of slots\n"); + printf("# per-router statistics for the apcxi module:\n"); + printf("# APCXI_GROUP: group this node is on\n"); + printf("# APCXI_CHASSIS: chassis this node is on\n"); + printf("# APCXI_SLOT: slot in the chassis this node is on\n"); + printf("# APCXI_BLADE: blade this node is on\n"); + printf("# APCXI_NODE: node index in the blade\n"); + printf("# APCXI_NUM_NICS: number of nics connected to this node\n"); + printf("# APCXI_AR_RTR_* port counters for the 40 router-router ports\n"); + //printf("# APCXI_AR_RTR_x_y_INQ_PRF_INCOMING_FLIT_VC[0-7]: flits on VCs of x y tile\n"); + //printf("# APCXI_AR_RTR_x_y_INQ_PRF_ROWBUS_STALL_CNT: stalls on x y tile\n"); + //printf("# APCXI_AR_RTR_PT_* port counters for the 8 router-nic ports\n"); + //printf("# APCXI_AR_RTR_PT_x_y_INQ_PRF_INCOMING_FLIT_VC[0,4]: flits on VCs of x y tile\n"); + //printf("# APCXI_AR_RTR_PT_x_y_INQ_PRF_REQ_ROWBUS_STALL_CNT: stalls on x y tile\n"); + + return; +} + +static void darshan_log_print_apcxi_rec_diff(void *file_rec1, char *file_name1, + void *file_rec2, char *file_name2) +{ + struct darshan_apcxi_header_record *hdr_rec1; + struct darshan_apcxi_header_record *hdr_rec2; + struct darshan_apcxi_perf_record *prf_rec1; + struct darshan_apcxi_perf_record *prf_rec2; + + hdr_rec1 = (struct darshan_apcxi_header_record*) file_rec1; + hdr_rec2 = (struct darshan_apcxi_header_record*) file_rec2; + prf_rec1 = (struct darshan_apcxi_perf_record*) file_rec1; + prf_rec2 = (struct darshan_apcxi_perf_record*) file_rec2; + + if (hdr_rec1->magic == APCXI_MAGIC) + { + /* this is the header record */ + if (!hdr_rec2) + { + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_GROUPS", hdr_rec1->ngroups, "", "", ""); + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_CHASSIS", hdr_rec1->nchassis, "", "", ""); + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_SLOTS", hdr_rec1->nslots, "", "", ""); + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_BLADES", hdr_rec1->nblades, "", "", ""); + printf("- "); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_APPLICATION_ID", hdr_rec1->appid, "", "", ""); + } + else if (!hdr_rec1) + { + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_GROUPS", hdr_rec2->ngroups, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_CHASSIS", hdr_rec2->nchassis, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_SLOTS", hdr_rec2->nslots, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_BLADES", hdr_rec2->nblades, "", "", ""); + printf("+ "); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_APPLICATION_ID", hdr_rec2->appid, "", "", ""); + } + else + { + if (hdr_rec1->ngroups != hdr_rec2->ngroups) + { + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_GROUPS", hdr_rec1->ngroups, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_GROUPS", hdr_rec2->ngroups, "", "", ""); + } + if (hdr_rec1->nchassis != hdr_rec2->nchassis) + { + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_CHASSIS", hdr_rec1->nchassis, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_CHASSIS", hdr_rec2->nchassis, "", "", ""); + } + if (hdr_rec1->nslots != hdr_rec2->nslots) + { + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_SLOTS", hdr_rec1->nslots, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_SLOTS", hdr_rec2->nslots, "", "", ""); + } + if (hdr_rec1->nblades != hdr_rec2->nblades) + { + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_BLADES", hdr_rec1->nblades, "", "", ""); + + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_BLADES", hdr_rec2->nblades, "", "", ""); + } + if (hdr_rec1->appid != hdr_rec2->appid) + { + printf("- "); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, + "APCXI_APPLICATION_ID", hdr_rec1->appid, "", "", ""); + + printf("+ "); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, + "APCXI_APPLICATION_ID", hdr_rec2->appid, "", "", ""); + } + } + } + else + { + if (!prf_rec2) + { + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APCXI_GROUP", prf_rec1->group, "", "", ""); + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APCXI_CHASSIS", prf_rec1->chassis, "", "", ""); + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APCXI_SLOT", prf_rec1->slot, "", "", ""); + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APCXI_BLADE", prf_rec1->blade, "", "", ""); + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APCXI_NODE", prf_rec1->node, "", "", ""); + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APCXI_NUM_NICS", prf_rec1->num_nics, "", "", ""); + } + else if (!prf_rec1) + { + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APCXI_GROUP", prf_rec2->group, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APCXI_CHASSIS", prf_rec2->chassis, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APCXI_SLOT", prf_rec2->slot, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APCXI_BLADE", prf_rec2->blade, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APCXI_NODE", prf_rec2->node, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APCXI_NUM_NICS", prf_rec2->num_nics, "", "", ""); + } + else { + if (prf_rec1->group != prf_rec2->group) + { + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APCXI_GROUP", prf_rec1->group, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APCXI_GROUP", prf_rec2->group, "", "", ""); + } + if (prf_rec1->chassis != prf_rec2->chassis) + { + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APCXI_CHASSIS", prf_rec1->chassis, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APCXI_CHASSIS", prf_rec2->chassis, "", "", ""); + } + if (prf_rec1->slot != prf_rec2->slot) + { + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APCXI_SLOT", prf_rec1->slot, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APCXI_SLOT", prf_rec2->slot, "", "", ""); + } + if (prf_rec1->blade != prf_rec2->blade) + { + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APCXI_BLADE", prf_rec1->blade, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APCXI_BLADE", prf_rec2->blade, "", "", ""); + } + if (prf_rec1->node != prf_rec2->node) + { + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APCXI_NODE", prf_rec1->node, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APCXI_NODE", prf_rec2->node, "", "", ""); + } + if (prf_rec1->num_nics != prf_rec2->num_nics) + { + printf("- "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + "APCXI_NUM_NICS", prf_rec1->num_nics, "", "", ""); + printf("+ "); + DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + "APCXI_NUM_NICS", prf_rec2->num_nics, "", "", ""); + } + } + + int i, j; + /* counters from all the available nics */ + for(i = 0; i < prf_rec1->num_nics; i++) + { + for(j = 0; j < APCXI_NUM_INDICES; j++) + { + if (!prf_rec2) + { + printf("- "); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + apcxi_counter_names[j], prf_rec1->counters[i][j], + "", "", ""); + } + else if (!prf_rec1) + { + printf("+ "); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + apcxi_counter_names[j], prf_rec2->counters[i][j], + "", "", ""); + } + else if (prf_rec1->counters[j] != prf_rec2->counters[j]) + { + printf("- "); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec1->base_rec.rank, prf_rec1->base_rec.id, + apcxi_counter_names[j], prf_rec1->counters[i][j], + "", "", ""); + + printf("+ "); + DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_APCXI_MOD], + prf_rec2->base_rec.rank, prf_rec2->base_rec.id, + apcxi_counter_names[j], prf_rec2->counters[i][j], + "", "", ""); + } + } + } + } + + return; +} + +/* + * Local variables: + * c-indent-level: 4 + * c-basic-offset: 4 + * End: + * + * vim: ts=8 sts=4 sw=4 expandtab + */ diff --git a/apcxi/util/darshan-apcxi-logutils.h b/apcxi/util/darshan-apcxi-logutils.h new file mode 100644 index 0000000..1c0b256 --- /dev/null +++ b/apcxi/util/darshan-apcxi-logutils.h @@ -0,0 +1,15 @@ +/* + * Copyright (C) 2018 University of Chicago. + * See COPYRIGHT notice in top-level directory. + * + */ + +#ifndef __APCXI_LOG_UTILS_H +#define __APCXI_LOG_UTILS_H + +extern char *apcxi_counter_names[]; + +extern struct darshan_mod_logutil_funcs apcxi_logutils; + +#endif + diff --git a/apcxi/util/print-apcxi-size.c b/apcxi/util/print-apcxi-size.c new file mode 100644 index 0000000..b99cdfd --- /dev/null +++ b/apcxi/util/print-apcxi-size.c @@ -0,0 +1,12 @@ +#include +#include "darshan-log-format.h" +#include "darshan-apcxi-log-format.h" + +int main (int argc, char **argv) +{ + printf ("APCXI_NUM_INDICES = %d\n", APCXI_NUM_INDICES); + printf ("sizeof darshan_apcxi_header_record = %lu\n", sizeof(struct darshan_apcxi_header_record)); + printf ("sizeof darshan_apcxi_perf_record = %lu\n", sizeof(struct darshan_apcxi_perf_record)); + + return 0; +} diff --git a/apss/Makefile.darshan b/apss/Makefile.darshan deleted file mode 100644 index bec287a..0000000 --- a/apss/Makefile.darshan +++ /dev/null @@ -1,20 +0,0 @@ -# -# AutoPerf Make rules for Darshan -# -DARSHAN_STATIC_MOD_OBJS += lib/darshan-apss.o -DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-apss.po - -VPATH += :$(srcdir)/../modules/autoperf/apss -CFLAGS += \ - -DDARSHAN_USE_APSS \ - -I$(srcdir)/../modules/autoperf/apss - -CFLAGS_SHARED += \ - -DDARSHAN_USE_APSS \ - -I$(srcdir)/../modules/autoperf/apss - -lib/darshan-apss.o: lib/darshan-apss.c lib/darshan-apss-utils.h darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) darshan-apss-log-format.h | lib - $(CC) $(CFLAGS) -c $< -o $@ - -lib/darshan-apss.po: lib/darshan-apss.c lib/darshan-apss-utils.h darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) darshan-apss-log-format.h | lib - $(CC) $(CFLAGS_SHARED) -c $< -o $@ diff --git a/apss/darshan-apss-log-format.h b/apss/darshan-apss-log-format.h deleted file mode 100644 index 123a22f..0000000 --- a/apss/darshan-apss-log-format.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (C) 2017 University of Chicago. - * See COPYRIGHT notice in top-level directory. - * - */ - -#ifndef __APSS_LOG_FORMAT_H -#define __APSS_LOG_FORMAT_H - -/* current AutoPerf Cray XC log format version */ -#define APSS_VER 1 - -#define APSS_MAGIC ('A'*0x100000000000000+\ - 'U'*0x1000000000000+\ - 'T'*0x10000000000+\ - 'O'*0x100000000+\ - 'P'*0x1000000+\ - 'E'*0x10000+\ - 'R'*0x100+\ - 'F'*0x1) - -#define APSS_PERF_COUNTERS \ - /* PAPI counters */\ - X(CQ_CQ_OXE_NUM_STALLS) \ - X(CQ_CQ_OXE_NUM_FLITS) \ - X(CQ_CYCLES_BLOCKED_0) \ - X(CQ_CYCLES_BLOCKED_1) \ - X(CQ_CYCLES_BLOCKED_2) \ - X(CQ_CYCLES_BLOCKED_3) \ - /* end of counters */\ - Z(APSS_NUM_INDICES) - - -#define X(a) a, -#define Z(a) a -/* integer counters for the "APSS" example module */ -enum darshan_apss_perf_indices -{ - APSS_PERF_COUNTERS -}; - -#undef Z -#undef X - -/* the darshan_apss_router_record structure encompasses the data/counters - * which would actually be logged to file by Darshan for the AP HPE Slingshot - * module. This example implementation logs the following data for each - * record: - * - a darshan_base_record structure, which contains the record id & rank - * - integer I/O counters - * - floating point I/O counters - */ -struct darshan_apss_perf_record -{ - struct darshan_base_record base_rec; - int64_t group; - int64_t chassis; - int64_t blade; - int64_t node; - uint64_t counters[APSS_NUM_INDICES]; -}; - -struct darshan_apss_header_record -{ - struct darshan_base_record base_rec; - int64_t magic; - int64_t nblades; - int64_t nchassis; - int64_t ngroups; - uint64_t appid; -}; - -#endif /* __APSS_LOG_FORMAT_H */ diff --git a/apss/lib/darshan-apss.c b/apss/lib/darshan-apss.c deleted file mode 100644 index 459ce9e..0000000 --- a/apss/lib/darshan-apss.c +++ /dev/null @@ -1,438 +0,0 @@ -/* - * Copyright (C) 2017 University of Chicago. - * See COPYRIGHT notice in top-level directory. - * - */ - -#define _XOPEN_SOURCE 500 -#define _GNU_SOURCE -#define csJOBID_ENV_STR "PALS_APP_ID" - -#include "darshan-runtime-config.h" -#include -#include -#include -#include -#include -#include - -#include "uthash.h" -#include "darshan.h" -#include "darshan-dynamic.h" -#include "darshan-apss-log-format.h" - -#include "darshan-apss-utils.h" - -/* - * PAPI_events are defined by the Aries counters listed in the log header. - */ -#define X(a) #a, -#define Z(a) #a -static char* PAPI_events[] = -{ - APSS_PERF_COUNTERS -}; -#undef X -#undef Z - -#define MAX_GROUPS (128) -#define MAX_CHASSIS (MAX_GROUPS*6) -#define MAX_BLADES (MAX_CHASSIS*16) - -/* - * - * - * This module does not intercept any system calls. It just pulls data - * from the personality struct at initialization. - */ - - -/* - * Global runtime struct for tracking data needed at runtime - */ -struct apss_runtime -{ - struct darshan_apss_header_record *header_record; - struct darshan_apss_perf_record *perf_record; - darshan_record_id header_id; - darshan_record_id rtr_id; - int PAPI_event_set; - int PAPI_event_count; - int group; - int chassis; - int blade; - int node; - int perf_record_marked; -}; - -static struct apss_runtime *apss_runtime = NULL; -static pthread_mutex_t apss_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP; - -/* my_rank indicates the MPI rank of this process */ -static int my_rank = -1; - -/* internal helper functions for the APSS module */ -void apss_runtime_initialize(void); - -/* forward declaration for shutdown function needed to interface with darshan-core */ -//#ifdef HAVE_MPI -static void apss_mpi_redux( - void *buffer, - MPI_Comm mod_comm, - darshan_record_id *shared_recs, - int shared_rec_count); -//#endif -static void apss_output( - void **buffer, - int *size); -static void apss_cleanup( - void); - -/* macros for obtaining/releasing the APSS module lock */ -#define APSS_LOCK() pthread_mutex_lock(&apss_runtime_mutex) -#define APSS_UNLOCK() pthread_mutex_unlock(&apss_runtime_mutex) - -/* - * Initialize counters using PAPI - */ -static void initialize_counters (void) -{ - int i; - int code = 0; - - PAPI_library_init(PAPI_VER_CURRENT); - apss_runtime->PAPI_event_set = PAPI_NULL; - PAPI_create_eventset(&apss_runtime->PAPI_event_set); - - /* start with first PAPI counter */ - for (i = AR_RTR_0_0_INQ_PRF_INCOMING_FLIT_VC0; - strcmp(PAPI_events[i], "APSS_NUM_INDICES") != 0; - i++) - { - PAPI_event_name_to_code(PAPI_events[i], &code); - PAPI_add_event(apss_runtime->PAPI_event_set, code); - } - - apss_runtime->PAPI_event_count = i; - - PAPI_start(apss_runtime->PAPI_event_set); - - return; -} - -static void finalize_counters (void) -{ - PAPI_cleanup_eventset(apss_runtime->PAPI_event_set); - PAPI_destroy_eventset(&apss_runtime->PAPI_event_set); - PAPI_shutdown(); - - return; -} - -/* - * Function which updates all the counter data - */ -static void capture(struct darshan_apss_perf_record *rec, - darshan_record_id rec_id) -{ - PAPI_stop(apss_runtime->PAPI_event_set, - (long long*) &rec->counters[AR_RTR_0_0_INQ_PRF_INCOMING_FLIT_VC0]); - PAPI_reset(apss_runtime->PAPI_event_set); - - rec->group = apss_runtime->group; - rec->chassis = apss_runtime->chassis; - rec->blade = apss_runtime->blade; - rec->node = apss_runtime->node; - rec->base_rec.id = rec_id; - rec->base_rec.rank = my_rank; - - return; -} - -void apss_runtime_initialize() -{ - size_t apss_buf_size; - char rtr_rec_name[128]; - - darshan_module_funcs mod_funcs = { -//#ifdef HAVE_MPI - .mod_redux_func = &apss_mpi_redux, -//#endif - .mod_output_func = &apss_output, - .mod_cleanup_func = &apss_cleanup - }; - - APSS_LOCK(); - - - /* don't do anything if already initialized */ - if(apss_runtime) - { - APSS_UNLOCK(); - return; - } - - - apss_buf_size = sizeof(struct darshan_apss_header_record) + - sizeof(struct darshan_apss_perf_record); - - /* register the APSS module with the darshan-core component */ - darshan_core_register_module( - DARSHAN_APSS_MOD, - mod_funcs, - &apss_buf_size, - &my_rank, - NULL); - - - /* initialize module's global state */ - apss_runtime = malloc(sizeof(*apss_runtime)); - if(!apss_runtime) - { - darshan_core_unregister_module(DARSHAN_APSS_MOD); - APSS_UNLOCK(); - return; - } - memset(apss_runtime, 0, sizeof(*apss_runtime)); - - if (my_rank == 0) - { - apss_runtime->header_id = darshan_core_gen_record_id("darshan-apss-header"); - - /* register the apss file record with darshan-core */ - apss_runtime->header_record = darshan_core_register_record( - apss_runtime->header_id, - //NULL, - "darshan-apss-header", - DARSHAN_APSS_MOD, - sizeof(struct darshan_apss_header_record), - NULL); - if(!(apss_runtime->header_record)) - { - darshan_core_unregister_module(DARSHAN_APSS_MOD); - free(apss_runtime); - apss_runtime = NULL; - APSS_UNLOCK(); - return; - } - apss_runtime->header_record->base_rec.id = apss_runtime->header_id; - apss_runtime->header_record->base_rec.rank = my_rank; - apss_runtime->header_record->magic = APSS_MAGIC; - } - - get_xc_coords(&apss_runtime->group, - &apss_runtime->chassis, - &apss_runtime->blade, - &apss_runtime->node); - - sprintf(rtr_rec_name, "darshan-apss-rtr-%d-%d-%d", - apss_runtime->group, apss_runtime->chassis, apss_runtime->blade); - //apss_runtime->rtr_id = darshan_core_gen_record_id(rtr_rec_name); - apss_runtime->rtr_id = darshan_core_gen_record_id("APSS"); - apss_runtime->perf_record = darshan_core_register_record( - apss_runtime->rtr_id, - //NULL, - "APSS", // we want the record for each rank to be treated as shared records so that mpi_redux can operate on - //rtr_rec_name, - DARSHAN_APSS_MOD, - sizeof(struct darshan_apss_perf_record), - NULL); - if(!(apss_runtime->perf_record)) - { - darshan_core_unregister_module(DARSHAN_APSS_MOD); - free(apss_runtime); - apss_runtime = NULL; - APSS_UNLOCK(); - return; - } - - initialize_counters(); - APSS_UNLOCK(); - - return; -} - -/******************************************************************************** - * shutdown function exported by this module for coordinating with darshan-core * - ********************************************************************************/ - -/* Pass data for the apss module back to darshan-core to log to file. */ -//#ifdef HAVE_MPI -static void apss_mpi_redux( - void *apss_buf, - MPI_Comm mod_comm, - darshan_record_id *shared_recs, - int shared_rec_count) -{ - int i; - int color; - int router_rank; - int router_count; - int chassis_count; - int group_count; - unsigned int *bitvec; - unsigned int bitlen; - unsigned int bitcnt; - unsigned int bitsiz; - MPI_Comm router_comm; - - APSS_LOCK(); - if (!apss_runtime) - { - APSS_UNLOCK(); - return; - } - - bitcnt = sizeof(unsigned int) * 8; - bitlen = sizeof(unsigned int) * (MAX_BLADES/bitcnt); - bitsiz = bitlen / sizeof(unsigned int); - bitvec = malloc(bitlen); - - /* collect perf counters */ - capture(apss_runtime->perf_record, apss_runtime->rtr_id); - - if (my_rank == 0) - { - apss_runtime->header_record->appid = atoi((char*)getenv( csJOBID_ENV_STR )); - } - - /* count network dimensions */ - if (bitvec) - { - int idx; - unsigned int uchassis; - unsigned int ublade; - - /* group */ - memset(bitvec, 0, bitlen); - idx = apss_runtime->group / bitcnt; - bitvec[idx] |= (1 << apss_runtime->group % bitcnt); - PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), - bitvec, - bitsiz, - MPI_INT, - MPI_BOR, - 0, - MPI_COMM_WORLD); - group_count = count_bits(bitvec, bitsiz); - - /* chassis */ - memset(bitvec, 0, bitlen); - uchassis = apss_runtime->group * 6 + apss_runtime->chassis; - idx = uchassis / bitcnt; - bitvec[idx] |= (1 << uchassis % bitcnt); - PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), - bitvec, - bitsiz, - MPI_INT, - MPI_BOR, - 0, - MPI_COMM_WORLD); - chassis_count = count_bits(bitvec, bitsiz); - - /* blade */ - memset(bitvec, 0, bitlen); - ublade = uchassis * 16 + apss_runtime->blade; - idx = ublade / bitcnt; - bitvec[idx] |= (1 << ublade % bitcnt); - PMPI_Reduce((my_rank ? bitvec : MPI_IN_PLACE), - bitvec, - bitsiz, - MPI_INT, - MPI_BOR, - 0, - MPI_COMM_WORLD); - router_count = count_bits(bitvec, bitsiz); - - if (my_rank == 0) - { - apss_runtime->header_record->nblades = router_count; - apss_runtime->header_record->nchassis = chassis_count; - apss_runtime->header_record->ngroups = group_count; - } - free(bitvec); - } - else - { - apss_runtime->header_record->nblades = 0; - apss_runtime->header_record->nchassis = 0; - apss_runtime->header_record->ngroups = 0; - } - - /* - * reduce data - * - * aggregate data from processes which share the same blade and avg. - * - */ - color = (apss_runtime->group << (4+3)) + \ - (apss_runtime->chassis << 4) + \ - apss_runtime->blade; - PMPI_Comm_split(MPI_COMM_WORLD, color, my_rank, &router_comm); - PMPI_Comm_rank(router_comm, &router_rank); - PMPI_Comm_size(router_comm, &router_count); - - PMPI_Reduce((router_rank?apss_runtime->perf_record->counters:MPI_IN_PLACE), - apss_runtime->perf_record->counters, - APSS_NUM_INDICES, - MPI_LONG_LONG_INT, - MPI_SUM, - 0, - router_comm); - - if (router_rank == 0) - { - for (i = 0; i < APSS_NUM_INDICES; i++) - { - apss_runtime->perf_record->counters[i] /= router_count; - } - apss_runtime->perf_record_marked = -1; - } - PMPI_Comm_free(&router_comm); - - APSS_UNLOCK(); - - return; -} - -//#endif -static void apss_output( - void **apss_buf, - int *apss_buf_sz) -{ - APSS_LOCK(); - assert(apss_runtime); - *apss_buf_sz = 0; - - if (my_rank == 0) { - *apss_buf_sz += sizeof(*apss_runtime->header_record); - } - - if (apss_runtime->perf_record_marked == -1) - { - *apss_buf_sz += sizeof( *apss_runtime->perf_record); - } - - APSS_UNLOCK(); - return; -} - -static void apss_cleanup() -{ - APSS_LOCK(); - assert(apss_runtime); - finalize_counters(); - free(apss_runtime); - apss_runtime = NULL; - APSS_UNLOCK(); - return; -} - -/* - * Local variables: - * c-indent-level: 4 - * c-basic-offset: 4 - * End: - * - * vim: ts=8 sts=4 sw=4 expandtab - */ diff --git a/apss/util/Makefile.darshan b/apss/util/Makefile.darshan deleted file mode 100644 index 1a9784b..0000000 --- a/apss/util/Makefile.darshan +++ /dev/null @@ -1,27 +0,0 @@ -# -# AutoPerf Make rules for Darshan -# - -VPATH += :$(srcdir)/../modules/autoperf/apss/util - -DARSHAN_MOD_LOG_FORMATS += $(srcdir)/../modules/autoperf/apss/darshan-apss-log-format.h -DARSHAN_MOD_LOGUTIL_HEADERS += darshan-apss-logutils.h -DARSHAN_STATIC_MOD_OBJS += darshan-apss-logutils.o -DARSHAN_DYNAMIC_MOD_OBJS += darshan-apss-logutils.po - -CFLAGS += \ - -DDARSHAN_USE_apss \ - -I$(srcdir)/../modules/autoperf/apss \ - -I$(srcdir)/../modules/autoperf/apss/util - -CFLAGS_SHARED += \ - -DDARSHAN_USE_apss \ - -I$(srcdir)/../modules/autoperf/apss \ - -I$(srcdir)/../modules/autoperf/apss/util - -darshan-apss-logutils.o: darshan-apss-logutils.c darshan-logutils.h darshan-apss-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../modules/autoperf/apss/darshan-apss-log-format.h | uthash-1.9.2 - $(CC) $(CFLAGS) -c $< -o $@ - -darshan-apss-logutils.po: darshan-apss-logutils.c darshan-logutils.h darshan-apss-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../modules/autoperf/apss/darshan-apss-log-format.h | uthash-1.9.2 - $(CC) $(CFLAGS_SHARED) -c $< -o $@ - diff --git a/apss/util/darshan-apss-logutils.c b/apss/util/darshan-apss-logutils.c deleted file mode 100644 index 2de463f..0000000 --- a/apss/util/darshan-apss-logutils.c +++ /dev/null @@ -1,493 +0,0 @@ -/* - * Copyright (C) 2018 University of Chicago. - * See COPYRIGHT notice in top-level directory. - * - */ - -#define _GNU_SOURCE -#include "darshan-util-config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "darshan-logutils.h" -#include "darshan-apss-log-format.h" - -/* counter name strings for the apss module */ -#define Y(a) #a, -#define X(a) Y(apss_ ## a) -#define Z(a) #a -char *apss_counter_names[] = { - apss_PERF_COUNTERS -}; -#undef Y -#undef X -#undef Z - -static int darshan_log_get_apss_rec(darshan_fd fd, void** buf_p); -static int darshan_log_put_apss_rec(darshan_fd fd, void* buf); -static void darshan_log_print_apss_rec(void *file_rec, - char *file_name, char *mnt_pt, char *fs_type); -static void darshan_log_print_apss_description(int ver); -static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, - void *file_rec2, char *file_name2); - -struct darshan_mod_logutil_funcs apss_logutils = -{ - .log_get_record = &darshan_log_get_apss_rec, - .log_put_record = &darshan_log_put_apss_rec, - .log_print_record = &darshan_log_print_apss_rec, - .log_print_description = &darshan_log_print_apss_description, - .log_print_diff = &darshan_log_print_apss_rec_diff, - .log_agg_records = NULL -}; - -static int darshan_log_get_apss_rec(darshan_fd fd, void** buf_p) -{ - struct darshan_apss_header_record *hdr_rec; - struct darshan_apss_perf_record *prf_rec; - int rec_len; - char *buffer; - int i; - int ret = -1; - static int first_rec = 1; - - if(fd->mod_map[DARSHAN_apss_MOD].len == 0) - return(0); - - if(fd->mod_ver[DARSHAN_apss_MOD] == 0 || - fd->mod_ver[DARSHAN_apss_MOD] > apss_VER) - { - fprintf(stderr, "Error: Invalid apss module version number (got %d)\n", - fd->mod_ver[DARSHAN_apss_MOD]); - return(-1); - } - - if (!*buf_p) - { - /* assume this is the largest possible record size */ - buffer = malloc(sizeof(struct darshan_apss_perf_record)); - if (!buffer) - { - return(-1); - } - } - else - { - buffer = *buf_p; - } - - if (fd->mod_ver[DARSHAN_apss_MOD] < apss_VER) - { - /* perform conversion as needed */ - } - - /* v1, current version */ - if (fd->mod_ver[DARSHAN_apss_MOD] == apss_VER) - { - if (first_rec) - { - rec_len = sizeof(struct darshan_apss_header_record); - first_rec = 0; - } - else - rec_len = sizeof(struct darshan_apss_perf_record); - - ret = darshan_log_get_mod(fd, DARSHAN_apss_MOD, buffer, rec_len); - } - - if (ret == rec_len) - { - if(fd->swap_flag) - { - if (rec_len == sizeof(struct darshan_apss_header_record)) - { - hdr_rec = (struct darshan_apss_header_record*)buffer; - /* swap bytes if necessary */ - DARSHAN_BSWAP64(&(hdr_rec->base_rec.id)); - DARSHAN_BSWAP64(&(hdr_rec->base_rec.rank)); - DARSHAN_BSWAP64(&(hdr_rec->magic)); - DARSHAN_BSWAP64(&(hdr_rec->nblades)); - DARSHAN_BSWAP64(&(hdr_rec->nchassis)); - DARSHAN_BSWAP64(&(hdr_rec->ngroups)); - DARSHAN_BSWAP64(&(hdr_rec->appid)); - } - else - { - prf_rec = (struct darshan_apss_perf_record*)buffer; - DARSHAN_BSWAP64(&(prf_rec->base_rec.id)); - DARSHAN_BSWAP64(&(prf_rec->base_rec.rank)); - DARSHAN_BSWAP64(&(prf_rec->group)); - DARSHAN_BSWAP64(&(prf_rec->chassis)); - DARSHAN_BSWAP64(&(prf_rec->blade)); - DARSHAN_BSWAP64(&(prf_rec->node)); - for (i = 0; i < apss_NUM_INDICES; i++) - { - DARSHAN_BSWAP64(&prf_rec->counters[i]); - } - } - } - *buf_p = buffer; - return(1); - } - else if (ret < 0) - { - //*buf_p = NULL; - // if (buffer) free(buffer); - if (!*buf_p) free(buffer); - return(-1); - } - else - { - // *buf_p = NULL; - // if (buffer) free(buffer); - if (!*buf_p) free(buffer); - return(0); - } -} - -static int darshan_log_put_apss_rec(darshan_fd fd, void* buf) -{ - int ret; - int rec_len; - static int first_rec = 1; - - if (first_rec) - { - rec_len = sizeof(struct darshan_apss_header_record); - first_rec = 0; - } - else - rec_len = sizeof(struct darshan_apss_perf_record); - - ret = darshan_log_put_mod(fd, DARSHAN_apss_MOD, buf, - rec_len, apss_VER); - if(ret < 0) - return(-1); - - return(0); -} - -static void darshan_log_print_apss_rec(void *rec, char *file_name, - char *mnt_pt, char *fs_type) -{ - int i; - static int first_rec = 1; - struct darshan_apss_header_record *hdr_rec; - struct darshan_apss_perf_record *prf_rec; - - if (first_rec) - { - hdr_rec = rec; - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "apss_GROUPS", hdr_rec->ngroups, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "apss_CHASSIS", hdr_rec->nchassis, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "apss_BLADES", hdr_rec->nblades, "", "", ""); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec->base_rec.rank, hdr_rec->base_rec.id, - "apss_APPLICATION_ID", hdr_rec->appid, "", "", ""); - first_rec = 0; - } - else - { - prf_rec = rec; - - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec->base_rec.rank, prf_rec->base_rec.id, - "apss_GROUP", prf_rec->group, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec->base_rec.rank, prf_rec->base_rec.id, - "apss_CHASSIS", prf_rec->chassis, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec->base_rec.rank, prf_rec->base_rec.id, - "apss_BLADE", prf_rec->blade, "", "", ""); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec->base_rec.rank, prf_rec->base_rec.id, - "apss_NODE", prf_rec->node, "", "", ""); - - for(i = 0; i < apss_NUM_INDICES; i++) - { - - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec->base_rec.rank, prf_rec->base_rec.id, - apss_counter_names[i], prf_rec->counters[i], - "", "", ""); - } - } - - return; -} - -static void darshan_log_print_apss_description(int ver) -{ - printf("\n# description of apss counters:\n"); - printf("# global summary stats for the apss module:\n"); - printf("# apss_GROUPS: total number of groups\n"); - printf("# apss_CHASSIS: total number of chassis\n"); - printf("# apss_BLADES: total number of blades\n"); - printf("# per-router statistics for the apss module:\n"); - printf("# apss_GROUP: group this router is on\n"); - printf("# apss_CHASSIS: chassis this router is on\n"); - printf("# apss_BLADE: blade this router is on\n"); - printf("# apss_NODE: node connected to this router\n"); - printf("# apss_AR_RTR_* port counters for the 40 router-router ports\n"); - printf("# apss_AR_RTR_x_y_INQ_PRF_INCOMING_FLIT_VC[0-7]: flits on VCs of x y tile\n"); - printf("# apss_AR_RTR_x_y_INQ_PRF_ROWBUS_STALL_CNT: stalls on x y tile\n"); - printf("# apss_AR_RTR_PT_* port counters for the 8 router-nic ports\n"); - printf("# apss_AR_RTR_PT_x_y_INQ_PRF_INCOMING_FLIT_VC[0,4]: flits on VCs of x y tile\n"); - printf("# apss_AR_RTR_PT_x_y_INQ_PRF_REQ_ROWBUS_STALL_CNT: stalls on x y tile\n"); - - return; -} - -static void darshan_log_print_apss_rec_diff(void *file_rec1, char *file_name1, - void *file_rec2, char *file_name2) -{ - struct darshan_apss_header_record *hdr_rec1; - struct darshan_apss_header_record *hdr_rec2; - struct darshan_apss_perf_record *prf_rec1; - struct darshan_apss_perf_record *prf_rec2; - - hdr_rec1 = (struct darshan_apss_header_record*) file_rec1; - hdr_rec2 = (struct darshan_apss_header_record*) file_rec2; - prf_rec1 = (struct darshan_apss_perf_record*) file_rec1; - prf_rec2 = (struct darshan_apss_perf_record*) file_rec2; - - if (hdr_rec1->magic == apss_MAGIC) - { - /* this is the header record */ - if (!hdr_rec2) - { - printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_GROUPS", hdr_rec1->ngroups, "", "", ""); - printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_CHASSIS", hdr_rec1->nchassis, "", "", ""); - printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_BLADES", hdr_rec1->nblades, "", "", ""); - printf("- "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_APPLICATION_ID", hdr_rec1->appid, "", "", ""); - } - else if (!hdr_rec1) - { - printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_GROUPS", hdr_rec2->ngroups, "", "", ""); - printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_CHASSIS", hdr_rec2->nchassis, "", "", ""); - printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_BLADES", hdr_rec2->nblades, "", "", ""); - printf("+ "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_APPLICATION_ID", hdr_rec2->appid, "", "", ""); - } - else - { - if (hdr_rec1->ngroups != hdr_rec2->ngroups) - { - printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_GROUPS", hdr_rec1->ngroups, "", "", ""); - printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_GROUPS", hdr_rec2->ngroups, "", "", ""); - } - if (hdr_rec1->nchassis != hdr_rec2->nchassis) - { - printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_CHASSIS", hdr_rec1->nchassis, "", "", ""); - printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_CHASSIS", hdr_rec2->nchassis, "", "", ""); - } - if (hdr_rec1->nblades != hdr_rec2->nblades) - { - printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_BLADES", hdr_rec1->nblades, "", "", ""); - - printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_BLADES", hdr_rec2->nblades, "", "", ""); - } - if (hdr_rec1->appid != hdr_rec2->appid) - { - printf("- "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec1->base_rec.rank, hdr_rec1->base_rec.id, - "apss_APPLICATION_ID", hdr_rec1->appid, "", "", ""); - - printf("+ "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - hdr_rec2->base_rec.rank, hdr_rec2->base_rec.id, - "apss_APPLICATION_ID", hdr_rec2->appid, "", "", ""); - } - } - } - else - { - if (!prf_rec2) - { - printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_GROUP", prf_rec1->group, "", "", ""); - printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_CHASSIS", prf_rec1->chassis, "", "", ""); - printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_BLADE", prf_rec1->blade, "", "", ""); - printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_NODE", prf_rec1->node, "", "", ""); - } - else if (!prf_rec1) - { - printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_GROUP", prf_rec2->group, "", "", ""); - printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_CHASSIS", prf_rec2->chassis, "", "", ""); - printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_BLADE", prf_rec2->blade, "", "", ""); - printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_NODE", prf_rec2->node, "", "", ""); - } - else { - if (prf_rec1->group != prf_rec2->group) - { - printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_GROUP", prf_rec1->group, "", "", ""); - printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_GROUP", prf_rec2->group, "", "", ""); - } - if (prf_rec1->chassis != prf_rec2->chassis) - { - printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_CHASSIS", prf_rec1->chassis, "", "", ""); - printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_CHASSIS", prf_rec2->chassis, "", "", ""); - } - if (prf_rec1->blade != prf_rec2->blade) - { - printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_BLADE", prf_rec1->blade, "", "", ""); - printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_BLADE", prf_rec2->blade, "", "", ""); - } - if (prf_rec1->node != prf_rec2->node) - { - printf("- "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - "apss_NODE", prf_rec1->node, "", "", ""); - printf("+ "); - DARSHAN_D_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - "apss_NODE", prf_rec2->node, "", "", ""); - } - } - - int i; - /* router tile record */ - for(i = 0; i < apss_NUM_INDICES; i++) - { - if (!prf_rec2) - { - printf("- "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - apss_counter_names[i], prf_rec1->counters[i], - "", "", ""); - } - else if (!prf_rec1) - { - printf("+ "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - apss_counter_names[i], prf_rec2->counters[i], - "", "", ""); - } - else if (prf_rec1->counters[i] != prf_rec2->counters[i]) - { - printf("- "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec1->base_rec.rank, prf_rec1->base_rec.id, - apss_counter_names[i], prf_rec1->counters[i], - "", "", ""); - - printf("+ "); - DARSHAN_U_COUNTER_PRINT(darshan_module_names[DARSHAN_apss_MOD], - prf_rec2->base_rec.rank, prf_rec2->base_rec.id, - apss_counter_names[i], prf_rec2->counters[i], - "", "", ""); - } - } - } - - return; -} - -/* - * Local variables: - * c-indent-level: 4 - * c-basic-offset: 4 - * End: - * - * vim: ts=8 sts=4 sw=4 expandtab - */ diff --git a/apss/util/darshan-apss-logutils.h b/apss/util/darshan-apss-logutils.h deleted file mode 100644 index 6d12e17..0000000 --- a/apss/util/darshan-apss-logutils.h +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright (C) 2018 University of Chicago. - * See COPYRIGHT notice in top-level directory. - * - */ - -#ifndef __apss_LOG_UTILS_H -#define __apss_LOG_UTILS_H - -extern char *apss_counter_names[]; - -extern struct darshan_mod_logutil_funcs apss_logutils; - -#endif - diff --git a/apss/util/print-apss-size.c b/apss/util/print-apss-size.c deleted file mode 100644 index a60244f..0000000 --- a/apss/util/print-apss-size.c +++ /dev/null @@ -1,12 +0,0 @@ -#include -#include "darshan-log-format.h" -#include "darshan-apss-log-format.h" - -int main (int argc, char **argv) -{ - printf ("APSS_NUM_INDICES = %d\n", APSS_NUM_INDICES); - printf ("sizeof darshan_apss_header_record = %d\n", sizeof(struct darshan_apss_header_record)); - printf ("sizeof darshan_apss_perf_record = %d\n", sizeof(struct darshan_apss_perf_record)); - - return 0; -} diff --git a/apxc/util/darshan-apxc-logutils.c b/apxc/util/darshan-apxc-logutils.c index 87bd65d..606647d 100644 --- a/apxc/util/darshan-apxc-logutils.c +++ b/apxc/util/darshan-apxc-logutils.c @@ -139,8 +139,8 @@ static int darshan_log_get_apxc_rec(darshan_fd fd, void** buf_p) DARSHAN_BSWAP64(&prf_rec->counters[i]); } } + *buf_p = buffer; } - *buf_p = buffer; return(1); } else if (ret < 0)