From 478dbbeefb903a8ecb287c6f602b075b6420decf Mon Sep 17 00:00:00 2001 From: Andre Holzner Date: Fri, 10 Nov 2017 14:39:08 +0100 Subject: [PATCH 01/10] initial commit: integration test for F3DataRetriever --- .../datasource/F3DataRetrieverIT.java | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java diff --git a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java new file mode 100644 index 00000000..26e51c23 --- /dev/null +++ b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java @@ -0,0 +1,82 @@ +package rcms.utilities.daqaggregator.datasource; + +import java.io.IOException; +import org.apache.log4j.Logger; +import static org.junit.Assert.assertNotNull; +import org.junit.BeforeClass; +import org.junit.Test; +import rcms.utilities.daqaggregator.Application; +import rcms.utilities.daqaggregator.Settings; +import rcms.utilities.daqaggregator.data.BUSummary; +import rcms.utilities.daqaggregator.data.DAQ; +import rcms.utilities.daqaggregator.data.HltInfo; + +/** + * Integration test for F3 retrieval: contacts the actual F3 server + * and checks that the values can be retrieved + * + */ +public class F3DataRetrieverIT { + private static final Logger logger = Logger.getLogger(F3DataRetriever.class); + + private static F3DataRetriever f3dataRetriever; + private static DAQ daq; + + /** retrieve the information only once and check it in the individual test cases */ + @BeforeClass + public static void makeRetriever() { + + Application.initialize("DAQAggregator.properties"); + + String hltUrl = Application.get().getProp(Settings.F3_HLT_URL); + String diskUrl = Application.get().getProp(Settings.F3_DISK_URL); + String crashesUrl = Application.get().getProp(Settings.F3_CRASHES_URL); + f3dataRetriever = new F3DataRetriever(new Connector(false), hltUrl, diskUrl,crashesUrl); + + daq = new DAQ(); + + // TODO: use MappingManager.map() instead + daq.setBuSummary(new BUSummary()); + daq.setHltInfo(new HltInfo()); + + f3dataRetriever.dispatch(daq); + } + + /** checks that the F3DataRetriever could retrieve non-null values */ + @Test + public void diskInfoDirectTest() throws IOException { + + // retrieve data from the F3mon serverse + F3DataRetriever.DiskInfo di = f3dataRetriever.getDiskInfo(); + + logger.info("disk output occupancy fraction is " + di.getOutputOccupancyFraction()); + logger.info("disk total output is " + di.getOutputTotal()); + logger.info("ramdisk occupancy fraction is " + di.getRamdiskOccupancyFraction()); + logger.info("ramdisk total is " + di.getRamdiskTotal()); + + assertNotNull(di.getOutputOccupancyFraction()); + assertNotNull(di.getOutputTotal()); + assertNotNull(di.getRamdiskOccupancyFraction()); + assertNotNull(di.getRamdiskTotal()); + + } + + /** checks that some values were dispatched to the DAQ object. Note that + it looks like not all quantities in F3DataRetriever.DiskInfo are + put into the DAQ object */ + @Test + public void diskInfoDispatchTest() { + + BUSummary buSummary = daq.getBuSummary(); + assertNotNull(buSummary); + + assertNotNull(buSummary.getOutputDiskTotal()); + assertNotNull(buSummary.getOutputDiskUsage()); + + // F3DataRetriever.DiskInfo.ramdiskOccupancyFraction and + // F3DataRetriever.DiskInfo.ramdiskTotal are not mapped into the DAQ + // object + + } + +} From b7e0714ab24efe760c671bf442790ad2b20992a4 Mon Sep 17 00:00:00 2001 From: Andre Holzner Date: Fri, 10 Nov 2017 16:09:04 +0100 Subject: [PATCH 02/10] added a new method getLatestRun() to class F3DataRetrieverIT --- .../datasource/F3DataRetrieverIT.java | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java index 26e51c23..7803347d 100644 --- a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java +++ b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java @@ -1,11 +1,18 @@ package rcms.utilities.daqaggregator.datasource; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; +import java.util.List; +import java.util.NoSuchElementException; +import org.apache.commons.lang3.tuple.Pair; import org.apache.log4j.Logger; import static org.junit.Assert.assertNotNull; import org.junit.BeforeClass; import org.junit.Test; import rcms.utilities.daqaggregator.Application; +import rcms.utilities.daqaggregator.DAQException; +import rcms.utilities.daqaggregator.DAQExceptionCode; import rcms.utilities.daqaggregator.Settings; import rcms.utilities.daqaggregator.data.BUSummary; import rcms.utilities.daqaggregator.data.DAQ; @@ -78,5 +85,42 @@ public void diskInfoDispatchTest() { // object } + + /** fetches the latest run number from F3mon */ + private static Integer getLatestRun() throws IOException { + + // get the base URL + String lastRunUrl = Application.get().getProp(Settings.F3_LAST_RUN_URL); + + if (lastRunUrl == null) { + throw new DAQException(DAQExceptionCode.MissingProperty, "property " + Settings.F3_LAST_RUN_URL.getKey() + " is not set"); + } + + Connector connector = new Connector(false); + + Pair> a = connector.retrieveLines(lastRunUrl + "?setup=cdaq"); + + List result = a.getRight(); + + long count = result.size(); + if (count == 1) { + JsonNode resultJson = new ObjectMapper().readValue(result.get(0), JsonNode.class); + + try { + + return resultJson.get("number").asInt(); + + } catch (NoSuchElementException e) { + logger.warn("Cannot retrieve latest run number (no such element) from response: " + result.get(0)); + return null; + } catch (NullPointerException e) { + logger.warn("Cannot retrieve latest run number from response: " + result.get(0)); + return null; + } + } else { + logger.warn("Expected 1 node as a response but was " + count); + return null; + } + } } From 8b540abb95d54fbbda0ab81474732a7346fc0975 Mon Sep 17 00:00:00 2001 From: Andre Holzner Date: Fri, 10 Nov 2017 16:09:54 +0100 Subject: [PATCH 03/10] changed name of logger in class F3DataRetrieverIT (to have the correct file name with the line number in the log messages) --- .../utilities/daqaggregator/datasource/F3DataRetrieverIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java index 7803347d..d77b2b1f 100644 --- a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java +++ b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java @@ -24,7 +24,7 @@ * */ public class F3DataRetrieverIT { - private static final Logger logger = Logger.getLogger(F3DataRetriever.class); + private static final Logger logger = Logger.getLogger(F3DataRetrieverIT.class); private static F3DataRetriever f3dataRetriever; private static DAQ daq; From 51e184e8550f6c8941d6b54869f40fc9fa33f9aa Mon Sep 17 00:00:00 2001 From: Andre Holzner Date: Fri, 10 Nov 2017 16:11:00 +0100 Subject: [PATCH 04/10] added a new Settings parameter mapped to f3.lastrun.url needed by new method F3DataRetrieverIT.getLatestRun() --- DAQAggregator_example.properties | 1 + src/main/java/rcms/utilities/daqaggregator/Settings.java | 1 + 2 files changed, 2 insertions(+) diff --git a/DAQAggregator_example.properties b/DAQAggregator_example.properties index b79d59a2..7abd79d6 100644 --- a/DAQAggregator_example.properties +++ b/DAQAggregator_example.properties @@ -66,6 +66,7 @@ f3.disk.url = http://es-cdaq.cms/sc/php/summarydisks.php f3.crashes.url = http://es-cdaq.cms/sc/php/resource_status.php f3.cpuload.url = http://cmsdaqfff/prod/sc/php/cpuusage.php f3.cpuload.type = 20% htcor(2x-x*x) +f3.lastrun.url = http://es-cdaq.cms/sc/php/brun.php # # Flag indicating if flashlist discovery should be based on static catalog, by default based on dynamic diff --git a/src/main/java/rcms/utilities/daqaggregator/Settings.java b/src/main/java/rcms/utilities/daqaggregator/Settings.java index 32e26f8e..d16ff6e0 100644 --- a/src/main/java/rcms/utilities/daqaggregator/Settings.java +++ b/src/main/java/rcms/utilities/daqaggregator/Settings.java @@ -24,6 +24,7 @@ public enum Settings { F3_CRASHES_URL("f3.crashes.url"), F3_CPU_LOAD_URL("f3.cpuload.url"), F3_CPU_LOAD_TYPE("f3.cpuload.type"), + F3_LAST_RUN_URL("f3.lastrun.url"), // settings for monitoring MONITOR_SETUPNAME("monitor.setupName"), From ac00dea9685dc6a1c1527c3f1efaf17efa44dbe1 Mon Sep 17 00:00:00 2001 From: Andre Holzner Date: Fri, 10 Nov 2017 16:13:35 +0100 Subject: [PATCH 05/10] now retrieving the latest run number in F3DataRetrieverIT to be able to get non-empty HLT rates --- .../daqaggregator/datasource/F3DataRetrieverIT.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java index d77b2b1f..07e4629b 100644 --- a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java +++ b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java @@ -21,17 +21,17 @@ /** * Integration test for F3 retrieval: contacts the actual F3 server * and checks that the values can be retrieved - * */ public class F3DataRetrieverIT { private static final Logger logger = Logger.getLogger(F3DataRetrieverIT.class); private static F3DataRetriever f3dataRetriever; private static DAQ daq; + private static Integer runNumber; /** retrieve the information only once and check it in the individual test cases */ @BeforeClass - public static void makeRetriever() { + public static void makeRetriever() throws IOException { Application.initialize("DAQAggregator.properties"); @@ -46,6 +46,15 @@ public static void makeRetriever() { daq.setBuSummary(new BUSummary()); daq.setHltInfo(new HltInfo()); + // find a recent/current run number first + // (the web applications for the hlt info will not return anything + // if the run url parameter is not specified) + runNumber = getLatestRun(); + logger.info("found latest run number " + runNumber); + + // needed for F3DataRetriever.dispatch() + daq.setRunNumber(runNumber); + f3dataRetriever.dispatch(daq); } From af40ab635f006a9ac6b832c18f72b91d0df66500 Mon Sep 17 00:00:00 2001 From: Andre Holzner Date: Fri, 10 Nov 2017 16:18:11 +0100 Subject: [PATCH 06/10] added a new method hltInfoDirectTest() to class F3DataRetrieverIT --- .../datasource/F3DataRetrieverIT.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java index 07e4629b..ed9c8205 100644 --- a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java +++ b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java @@ -131,5 +131,33 @@ private static Integer getLatestRun() throws IOException { return null; } } + + + /** + * Test for HLT rates and bandwidths. + * + * Note that dispatching the HLT rates will only dispatch the physics stream + * rate/bandwidth to the daq object which does not not exist during + * interfills. We therefore do not implement test of the dispatching mechanism + * for the HLT rate and bandwidth quantities. + */ + @Test + public void hltInfoDirectTest() throws IOException { + + F3DataRetriever.HLToutputInfo hltInfo = f3dataRetriever.getHLToutputInfo(runNumber); + + // "PhysicsMuons" is there both for LHC fills and interfill runs + // but we have seen cases of zero rates during interfill + // use "HLTRates" instead + String streamName = "HLTRates"; + + Double eventRate = hltInfo.getEventRate(streamName); + logger.info("HLT event rate for stream " + streamName + " is " + eventRate); + assertNotNull(eventRate); + + Double bandwidth = hltInfo.getBandwidth(streamName); + logger.info("HLT bandwidth for stream " + streamName + " is " + bandwidth); + assertNotNull(bandwidth); + } } From e3e2226f07d4d5f3938d99c033ebe90bc1fc4fd5 Mon Sep 17 00:00:00 2001 From: Andre Holzner Date: Fri, 10 Nov 2017 16:30:03 +0100 Subject: [PATCH 07/10] added tests for number of crashes in F3DataRetrieverIT --- .../datasource/F3DataRetrieverIT.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java index ed9c8205..3453cc85 100644 --- a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java +++ b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java @@ -159,5 +159,28 @@ public void hltInfoDirectTest() throws IOException { logger.info("HLT bandwidth for stream " + streamName + " is " + bandwidth); assertNotNull(bandwidth); } + + /** + * Test for number of HLT crashes + */ + @Test + public void crashesDirectTest() throws IOException { + + Integer crashes = f3dataRetriever.getCrashes(); + + logger.info("HLT number of crashes is " + crashes); + assertNotNull(crashes); + } + + /** + * test of dispatching of number of HLT crashes + */ + @Test + public void crashesDispatchTest() { + + Integer crashes = daq.getHltInfo().getCrashes(); + assertNotNull(crashes); + + } } From 7a3e886f80ebb3504aeb4c8f65570d9731e0d5eb Mon Sep 17 00:00:00 2001 From: Andre Holzner Date: Mon, 13 Nov 2017 13:52:13 +0100 Subject: [PATCH 08/10] added missing arguments to constructor of F3DataRetriever after rebasing on latest dev --- .../daqaggregator/datasource/F3DataRetrieverIT.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java index 3453cc85..944a75f2 100644 --- a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java +++ b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java @@ -17,6 +17,7 @@ import rcms.utilities.daqaggregator.data.BUSummary; import rcms.utilities.daqaggregator.data.DAQ; import rcms.utilities.daqaggregator.data.HltInfo; +import rcms.utilities.daqaggregator.datasource.F3DataRetriever.CpuLoadType; /** * Integration test for F3 retrieval: contacts the actual F3 server @@ -38,7 +39,11 @@ public static void makeRetriever() throws IOException { String hltUrl = Application.get().getProp(Settings.F3_HLT_URL); String diskUrl = Application.get().getProp(Settings.F3_DISK_URL); String crashesUrl = Application.get().getProp(Settings.F3_CRASHES_URL); - f3dataRetriever = new F3DataRetriever(new Connector(false), hltUrl, diskUrl,crashesUrl); + String cpuLoadUrl = Application.get().getProp(Settings.F3_CPU_LOAD_URL); + String cpuLoadType = Application.get().getProp(Settings.F3_CPU_LOAD_TYPE); + + f3dataRetriever = new F3DataRetriever(new Connector(false), hltUrl, diskUrl,crashesUrl, + cpuLoadUrl, CpuLoadType.getByKey(cpuLoadType)); daq = new DAQ(); From 06f581215c44c36d1e264cda2edd617a248502ad Mon Sep 17 00:00:00 2001 From: Andre Holzner Date: Mon, 13 Nov 2017 14:02:05 +0100 Subject: [PATCH 09/10] added a method cpuLoadDirectTest() to class F3DataRetrieverIT --- .../daqaggregator/datasource/F3DataRetrieverIT.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java index 944a75f2..8db0a04e 100644 --- a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java +++ b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java @@ -188,4 +188,14 @@ public void crashesDispatchTest() { } + @Test + public void cpuLoadDirectTest() { + + Float cpuLoad = f3dataRetriever.getCpuLoad(); + + logger.info("HLT CPU load is " + cpuLoad); + assertNotNull(cpuLoad); + + } + } From 2e81a86cf560fe355077653c2c2c476324c019fa Mon Sep 17 00:00:00 2001 From: Andre Holzner Date: Mon, 13 Nov 2017 14:03:13 +0100 Subject: [PATCH 10/10] added a method cpuLoadDispatchTest() to class F3DataRetrieverIT --- .../daqaggregator/datasource/F3DataRetrieverIT.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java index 8db0a04e..cebf9593 100644 --- a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java +++ b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverIT.java @@ -197,5 +197,13 @@ public void cpuLoadDirectTest() { assertNotNull(cpuLoad); } - + + /** checks that a cpu load was dispatched to the DAQ object */ + @Test + public void cpuLoadDispatchTest() { + + Float cpuLoad = daq.getHltInfo().getCpuLoad(); + assertNotNull(cpuLoad); + + } }