diff --git a/DAQAggregator_example.properties b/DAQAggregator_example.properties index 36d24fea..d841640c 100644 --- a/DAQAggregator_example.properties +++ b/DAQAggregator_example.properties @@ -59,6 +59,13 @@ socksproxy.enableproxy = true socksproy.host = localhost socksproxy.port = 1080 + +f3.enabled = true +f3.htl.url = http://es-cdaq.cms/sc/php/stream_summary_last.php +f3.disk.url = http://es-cdaq.cms/sc/php/summarydisks.php +f3.crashes.url = http://es-cdaq.cms/sc/php/resource_status.php + + # # Flag indicating if flashlist discovery should be based on static catalog, by default based on dynamic # diff --git a/pom.xml b/pom.xml index ae9ac7bd..26fc524d 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 ch.cern DAQAggregator - 1.17.6 + 1.17.7 libs-release-local diff --git a/src/main/java/rcms/utilities/daqaggregator/DAQAggregator.java b/src/main/java/rcms/utilities/daqaggregator/DAQAggregator.java index 9f1ac63b..d804e0c2 100644 --- a/src/main/java/rcms/utilities/daqaggregator/DAQAggregator.java +++ b/src/main/java/rcms/utilities/daqaggregator/DAQAggregator.java @@ -69,7 +69,7 @@ public static void main(String[] args) { } /* - * Persist mode from properties file + * Persist mode from properties file */ PersistMode persistMode = PersistMode.decode(Application.get().getProp(Settings.PERSISTENCE_MODE)); logger.info("Persist mode:" + persistMode); @@ -282,12 +282,13 @@ public static Pair initialize(RunMode runMode) boolean f3Enabled = Boolean.parseBoolean(Application.get().getProp(Settings.F3_ENABLED)); String hltUrl = Application.get().getProp(Settings.F3_HLT_URL); String diskUrl = Application.get().getProp(Settings.F3_DISK_URL); + String crashesUrl = Application.get().getProp(Settings.F3_CRASHES_URL); F3DataRetriever f3DataRetriever = null; - if (f3Enabled && !"".equals(hltUrl) && !"".equals(diskUrl)) { + if (f3Enabled && !"".equals(hltUrl) && !"".equals(diskUrl) && !"".equals(crashesUrl)) { logger.info("F3 monitoring is enabled and set to following urls: " + hltUrl + ", " + diskUrl); - f3DataRetriever = new F3DataRetriever(new Connector(false), hltUrl, diskUrl); + f3DataRetriever = new F3DataRetriever(new Connector(false), hltUrl, diskUrl,crashesUrl); } else if (f3Enabled) { throw new DAQException(DAQExceptionCode.MissingProperty, "Specify url for F3 data retrieval. Required: " + Settings.F3_DISK_URL.getKey() + ", " + Settings.F3_HLT_URL.getKey()); } else { diff --git a/src/main/java/rcms/utilities/daqaggregator/Settings.java b/src/main/java/rcms/utilities/daqaggregator/Settings.java index a0d4fb6b..f9acfc08 100644 --- a/src/main/java/rcms/utilities/daqaggregator/Settings.java +++ b/src/main/java/rcms/utilities/daqaggregator/Settings.java @@ -21,6 +21,7 @@ public enum Settings { F3_ENABLED("f3.enabled", true), F3_HLT_URL("f3.htl.url"), F3_DISK_URL("f3.disk.url"), + F3_CRASHES_URL("f3.crashes.url"), // settings for monitoring MONITOR_SETUPNAME("monitor.setupName"), diff --git a/src/main/java/rcms/utilities/daqaggregator/data/DAQ.java b/src/main/java/rcms/utilities/daqaggregator/data/DAQ.java index 0688110d..b9e0ce5d 100644 --- a/src/main/java/rcms/utilities/daqaggregator/data/DAQ.java +++ b/src/main/java/rcms/utilities/daqaggregator/data/DAQ.java @@ -64,6 +64,8 @@ public class DAQ implements FlashlistUpdatable { private TCDSGlobalInfo tcdsGlobalInfo; + private HltInfo hltInfo; + /** * HLT stream physics output rate in Hz (events per second) */ @@ -548,6 +550,15 @@ public void setDaqAggregatorProducer(String daqAggregatorProducer) { this.daqAggregatorProducer = daqAggregatorProducer; } + public HltInfo getHltInfo() { + return hltInfo; + } + + public void setHltInfo(HltInfo hltInfo) { + this.hltInfo = hltInfo; + } + + @Override public String toString() { return "DAQ [sessionId=" + sessionId + ", dpsetPath=" + dpsetPath + ", runNumber=" + runNumber + ", runStart=" diff --git a/src/main/java/rcms/utilities/daqaggregator/data/HltInfo.java b/src/main/java/rcms/utilities/daqaggregator/data/HltInfo.java new file mode 100644 index 00000000..ee40db35 --- /dev/null +++ b/src/main/java/rcms/utilities/daqaggregator/data/HltInfo.java @@ -0,0 +1,14 @@ +package rcms.utilities.daqaggregator.data; + +public class HltInfo { + + private Integer crashes; + + public Integer getCrashes() { + return crashes; + } + + public void setCrashes(Integer crashes) { + this.crashes = crashes; + } +} diff --git a/src/main/java/rcms/utilities/daqaggregator/datasource/F3DataRetriever.java b/src/main/java/rcms/utilities/daqaggregator/datasource/F3DataRetriever.java index ea0b7572..3eba4f9e 100644 --- a/src/main/java/rcms/utilities/daqaggregator/datasource/F3DataRetriever.java +++ b/src/main/java/rcms/utilities/daqaggregator/datasource/F3DataRetriever.java @@ -31,12 +31,14 @@ public class F3DataRetriever { private final ObjectMapper mapper; private final String hltUrl; private final String diskUrl; + private final String crashUrl; - public F3DataRetriever(Connector connector, String hltUrl, String diskUrl) { + public F3DataRetriever(Connector connector, String hltUrl, String diskUrl, String crashUrl) { this.mapper = new ObjectMapper(); this.connector = connector; this.hltUrl = hltUrl; this.diskUrl = diskUrl; + this.crashUrl = crashUrl; } /** @@ -45,15 +47,17 @@ public F3DataRetriever(Connector connector, String hltUrl, String diskUrl) { * @param args */ public static void main(String[] args) { - F3DataRetriever f3dr = new F3DataRetriever(new Connector(false), "http://es-cdaq.cms/sc/php/stream_summary_last.php", "http://es-cdaq.cms/sc/php/summarydisks.php"); + F3DataRetriever f3dr = new F3DataRetriever(new Connector(false), "http://es-cdaq.cms/sc/php/stream_summary_last.php", "http://es-cdaq.cms/sc/php/summarydisks.php", "http://es-cdaq.cms/sc/php/resource_status.php"); try { Application.initialize("DAQAggregator.properties"); ProxyManager.get().startProxy(); DiskInfo d = f3dr.getDiskInfo(); Double h = f3dr.getHLToutputInfo(288498).getEventRate(PHYSICS_STREAM_NAME); + Integer c = f3dr.getCrashes(); - logger.info(d); - logger.info(h); + logger.info("Disk info: " + d); + logger.info("Hlt output: " + h); + logger.info("Crashes: " + c); } catch (JsonMappingException e) { e.printStackTrace(); } catch (IOException e) { @@ -61,6 +65,110 @@ public static void main(String[] args) { } } + public void dispatch(DAQ daq) { + + long start = System.currentTimeMillis(); + + boolean diskSuccessful = dispatchDisk(daq); + boolean hltSuccessful = dispatchHLT(daq); + boolean crashSuccessful = dispatchCrashes(daq); + + long end = System.currentTimeMillis(); + + if (diskSuccessful && hltSuccessful && crashSuccessful) + logger.info("F3 data successfully retrieved and mapped in: " + (end - start) + "ms"); + else { + logger.warn("Problem retrieving F3 data [disk successful,hlt successful]=[" + diskSuccessful + "," + + hltSuccessful + "," +crashSuccessful+ "]"); + } + } + + /** + * @return true if retrieving HLT output information from F3mon + * was successful, false otherwise + */ + protected boolean dispatchHLT(DAQ daq) { + + try { + HLToutputInfo hltInfo = getHLToutputInfo(daq.getRunNumber()); + + Double hltOutputRate = hltInfo.getEventRate(PHYSICS_STREAM_NAME); + Double hltOutputBW = hltInfo.getBandwidth(PHYSICS_STREAM_NAME); + + daq.setHltRate(hltOutputRate); + daq.setHltBandwidth(hltOutputBW); + + return hltOutputRate != null; + + } catch (JsonMappingException e) { + logger.warn("Could not retrieve F3 HLT rate, json mapping exception: ", e); + } catch (IOException e) { + logger.warn("Could not retrieve F3 HLT rate, IO exception: ", e); + } + return false; + + } + + protected boolean dispatchCrashes(DAQ daq) { + Integer crashes = getCrashes(); + daq.getHltInfo().setCrashes(crashes); + if (crashes != null) + return true; + else + return false; + } + + protected Integer getCrashes() { + Pair> a = null; + try { + a = connector.retrieveLines(crashUrl + "?setup=cdaq"); + + List result = a.getRight(); + + long count = result.size(); + if (count == 1) { + JsonNode resultJson = mapper.readValue(result.get(0), JsonNode.class); + + try { + Integer crashes = resultJson.get("crashes").asInt(); + return crashes; + + } catch (NoSuchElementException e) { + logger.warn("Cannot retrieve HLT crashes (no such element) from response: " + result.get(0)); + } catch (NullPointerException e) { + logger.warn("Cannot retrieve HLT crashes from response: " + result.get(0)); + } + } + + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + + protected boolean dispatchDisk(DAQ daq) { + + try { + DiskInfo d = getDiskInfo(); + if (d != null) { + daq.getBuSummary().setOutputDiskTotal(d.getOutputTotal()); + daq.getBuSummary().setOutputDiskUsage(d.getOutputOccupancyFraction()); + return true; + } else { + daq.getBuSummary().setOutputDiskTotal(null); + daq.getBuSummary().setOutputDiskUsage(null); + } + + } catch (JsonMappingException e) { + logger.warn("Could not retrieve F3 disk info, json mapping exception: ", e); + } catch (IOException e) { + logger.warn("Could not retrieve F3 disk info, IO exception: ", e); + } + return false; + + } + /** * @param events if true, fills event rates otherwise fills bandwidths */ @@ -112,7 +220,7 @@ private void fillHLTInfo(int runNumber, HLToutputInfo hltOutputInfo, boolean eve } } - public HLToutputInfo getHLToutputInfo(int runNumber) throws IOException { + protected HLToutputInfo getHLToutputInfo(int runNumber) throws IOException { HLToutputInfo info = new HLToutputInfo(); // fill event rates @@ -132,7 +240,7 @@ public HLToutputInfo getHLToutputInfo(int runNumber) throws IOException { * @throws IOException * @throws JsonMappingException */ - public DiskInfo getDiskInfo() throws IOException, JsonMappingException { + protected DiskInfo getDiskInfo() throws IOException, JsonMappingException { Pair> a = connector.retrieveLines(diskUrl); List result = a.getRight(); @@ -162,70 +270,6 @@ public DiskInfo getDiskInfo() throws IOException, JsonMappingException { } - public void dispatch(DAQ daq) { - - long start = System.currentTimeMillis(); - - boolean diskSuccessful = dispatchDisk(daq); - boolean hltSuccessful = dispatchHLT(daq); - - long end = System.currentTimeMillis(); - - if (diskSuccessful && hltSuccessful) - logger.info("F3 data successfully retrieved and mapped in: " + (end - start) + "ms"); - else { - logger.warn("Problem retrieving F3 data [disk successful,hlt successful]=[" + diskSuccessful + "," - + hltSuccessful + "]"); - } - } - - /** - * @return true if retrieving HLT output information from F3mon - * was successful, false otherwise - */ - public boolean dispatchHLT(DAQ daq) { - - try { - HLToutputInfo hltInfo = getHLToutputInfo(daq.getRunNumber()); - - Double hltOutputRate = hltInfo.getEventRate(PHYSICS_STREAM_NAME); - Double hltOutputBW = hltInfo.getBandwidth(PHYSICS_STREAM_NAME); - - daq.setHltRate(hltOutputRate); - daq.setHltBandwidth(hltOutputBW); - - return hltOutputRate != null; - - } catch (JsonMappingException e) { - logger.warn("Could not retrieve F3 HLT rate, json mapping exception: ", e); - } catch (IOException e) { - logger.warn("Could not retrieve F3 HLT rate, IO exception: ", e); - } - return false; - - } - - public boolean dispatchDisk(DAQ daq) { - - try { - DiskInfo d = getDiskInfo(); - if (d != null) { - daq.getBuSummary().setOutputDiskTotal(d.getOutputTotal()); - daq.getBuSummary().setOutputDiskUsage(d.getOutputOccupancyFraction()); - return true; - } else { - daq.getBuSummary().setOutputDiskTotal(null); - daq.getBuSummary().setOutputDiskUsage(null); - } - - } catch (JsonMappingException e) { - logger.warn("Could not retrieve F3 disk info, json mapping exception: ", e); - } catch (IOException e) { - logger.warn("Could not retrieve F3 disk info, IO exception: ", e); - } - return false; - - } public class DiskInfo { private Double ramdiskOccupancyFraction; diff --git a/src/main/java/rcms/utilities/daqaggregator/mappers/ObjectMapper.java b/src/main/java/rcms/utilities/daqaggregator/mappers/ObjectMapper.java index b3dad637..eda009ef 100644 --- a/src/main/java/rcms/utilities/daqaggregator/mappers/ObjectMapper.java +++ b/src/main/java/rcms/utilities/daqaggregator/mappers/ObjectMapper.java @@ -14,25 +14,8 @@ import org.apache.log4j.Logger; +import rcms.utilities.daqaggregator.data.*; import rcms.utilities.hwcfg.HardwareConfigurationException; -import rcms.utilities.daqaggregator.data.BU; -import rcms.utilities.daqaggregator.data.BUSummary; -import rcms.utilities.daqaggregator.data.DAQ; -import rcms.utilities.daqaggregator.data.FED; -import rcms.utilities.daqaggregator.data.FEDBuilder; -import rcms.utilities.daqaggregator.data.FEDBuilderSummary; -import rcms.utilities.daqaggregator.data.FMM; -import rcms.utilities.daqaggregator.data.FMMApplication; -import rcms.utilities.daqaggregator.data.FRL; -import rcms.utilities.daqaggregator.data.FRLPc; -import rcms.utilities.daqaggregator.data.FRLType; -import rcms.utilities.daqaggregator.data.GlobalTTSState; -import rcms.utilities.daqaggregator.data.FMMType; -import rcms.utilities.daqaggregator.data.RU; -import rcms.utilities.daqaggregator.data.SubFEDBuilder; -import rcms.utilities.daqaggregator.data.SubSystem; -import rcms.utilities.daqaggregator.data.TCDSGlobalInfo; -import rcms.utilities.daqaggregator.data.TTCPartition; import rcms.utilities.daqaggregator.datasource.TCDSFMInfoRetriever; import rcms.utilities.hwcfg.HardwareConfigurationException; import rcms.utilities.hwcfg.dp.DAQPartition; @@ -217,6 +200,7 @@ public void mapAllObjects(DAQPartition daqPartition) { daq.setFrls(new ArrayList<>(frls.values())); daq.setFeds(new ArrayList<>(feds.values())); daq.setSubFEDBuilders(new ArrayList<>(subFedBuilders.values())); + daq.setHltInfo(new HltInfo()); logger.info("Retrieval summary " + this.toString()); logger.info("Subsystem summary " + subSystems.values()); diff --git a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverTest.java b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverTest.java index baf1f3bc..c62fe1f3 100644 --- a/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverTest.java +++ b/src/test/java/rcms/utilities/daqaggregator/datasource/F3DataRetrieverTest.java @@ -23,17 +23,12 @@ public class F3DataRetrieverTest { private static final Logger logger = Logger.getLogger(F3DataRetriever.class); - @BeforeClass - public static void enableProxy() { - Application.initialize("DAQAggregator.properties"); - ProxyManager.get().startProxy(); - } @Test public void diskInfoTest() throws IOException { logger.info("Test connection"); String fakeResponse = "{\"ramdisk_occ\":0,\"output_occ\":0.012734683851866,\"ramdisk_tot\":16370688,\"output_tot\":278240437}"; - F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),null,null); + F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),null,null,null); DiskInfo di = f3dataRetriever.getDiskInfo(); Assert.assertEquals(new Double(0.012734683851866), di.getOutputOccupancyFraction()); Assert.assertEquals(new Integer(278240437), di.getOutputTotal()); @@ -44,18 +39,25 @@ public void diskInfoTest() throws IOException { @Test public void testHlt() throws IOException { String fakeResponse = "{\"219\":{\"ALCALUMIPIXELS\":47.190047190047,\"ALCAPHISYM\":0,\"Calibration\":79.365079365079,\"DQM\":6.8640068640069,\"DQMCalibration\":7.9365079365079,\"DQMEventDisplay\":18.618618618619,\"DQMHistograms\":612.44101244101,\"EcalCalibration\":79.365079365079,\"Error\":0,\"ExpressCosmics\":19.004719004719,\"HLTRates\":612.44101244101,\"L1Rates\":612.44101244101,\"NanoDST\":5.7915057915058,\"Physics\":71.600171600172,\"RPCMON\":27.627627627628}}"; - F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),"",""); + F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),"","",null); Assert.assertEquals(new Double(71.600171600172d), f3dataRetriever.getHLToutputInfo(0).getEventRate(F3DataRetriever.PHYSICS_STREAM_NAME)); } @Test public void testHltOutputBandwidth() throws IOException { String fakeResponse = "{\"58\":{\"ALCALUMIPIXELS\":127814.84341484,\"ALCAPHISYM\":0,\"Calibration\":2844043.7580438,\"DQM\":64910.424710425,\"DQMCalibration\":290670.87087087,\"DQMEventDisplay\":0,\"DQMHistograms\":0,\"EcalCalibration\":118622.05062205,\"Error\":0,\"ExpressCosmics\":181467.43886744,\"HLTRates\":41108.193908194,\"L1Rates\":254211.66881167,\"NanoDST\":4938.4384384384,\"Physics\":439103.56070356,\"RPCMON\":0}}"; - F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),null,null); + F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),null,null,null); Assert.assertEquals(new Double(439103.56070356d), f3dataRetriever.getHLToutputInfo(0).getBandwidth(F3DataRetriever.PHYSICS_STREAM_NAME)); } - + @Test + public void testHltCrashes() throws IOException { + String fakeResponse = "{\"last_run\":306155,\"num_BUs_with_last_run\":73,\"quarantinedRes\":0,\"crashedRes\":0,\"crashedOrQuarantinedRes\":0,\"crashes\":123,\"activeRes\":50728,\"activeResOldRuns\":0}"; + F3DataRetriever f3dataRetriever = new F3DataRetriever(new ConnectorFake(fakeResponse),null,null,null); + Assert.assertEquals(new Integer(123), f3dataRetriever.getCrashes()); + } + + public class ConnectorFake extends Connector { private final String response;