Add HDDEMO database

invisible_cities/conftest.py

@@ -30,7 +30,7 @@
 pmp_dfs  = namedtuple('pmp_dfs' , 's1 s2 si, s1pmt, s2pmt')
 pmp_data = namedtuple('pmp_data', 's1 s2 si')
 mcs_data = namedtuple('mcs_data', 'pmap hdst')
-db_data  = namedtuple('db_data', 'detector npmts nsipms feboxes nfreqs')
+db_data  = namedtuple('db_data', 'detector npmts nsipms feboxes nfreqs nfibers', defaults=(0,))
 
 
 @pytest.fixture(scope = 'session')
@@ -540,9 +540,10 @@ def dbflex100():
 @pytest.fixture(scope='session',
                 params=[db_data('demopp' ,  3,  256, 3, 79),
                         db_data('new'    , 12, 1792, 3, 79),
-                        db_data('next100', 60, 3584, 0, 0),
-                        db_data('flex100', 60, 3093, 0, 0)],
-               ids=["demo", "new", "next100", "flex100"])
+                        db_data('next100', 60, 3584, 0,  0),
+                        db_data('flex100', 60, 3093, 0,  0),
+                        db_data('hddemo' ,  7,  828, 0,  0, 36)],
+               ids=["demo", "new", "next100", "flex100", "hddemo"])
 def db(request):
     return request.param
 

invisible_cities/database/download.py

@@ -70,11 +70,12 @@ def loadDB(dbname : str, tables : list):
         copy_all_rows(conn_sqlite, cursor_sqlite, cursor_mysql, table)
 
 
-dbnames        = ('NEWDB', 'DEMOPPDB', 'NEXT100DB', 'Flex100DB')
+dbnames        = ('NEWDB', 'DEMOPPDB', 'NEXT100DB', 'Flex100DB', 'HDDEMODB')
 common_tables  = ('DetectorGeo','PmtBlr','ChannelGain','ChannelMapping','ChannelMask',
                   'PmtNoiseRms','ChannelPosition','SipmBaseline', 'SipmNoisePDF',
                   'PMTFEMapping', 'PMTFELowFrequencyNoise')
-extended       = dict(NEXT100DB = ("Activity", "Efficiency"))
+extended       = dict(NEXT100DB = ("Activity", "Efficiency"),
+                      HDDEMODB = ("ChannelAmplification",))
 
 table_dict = dict.fromkeys(dbnames, common_tables)
 for dbname, extra in extended.items():

invisible_cities/database/download_test.py

@@ -9,7 +9,7 @@
 from . import download as db
 
 @mark.skip(reason='server timeouts cause too many spurious test failures')
-@mark.parametrize('dbname', 'DEMOPPDB NEWDB NEXT100DB Flex100DB'.split())
+@mark.parametrize('dbname', 'DEMOPPDB NEWDB NEXT100DB Flex100DB HDDEMODB'.split())
 def test_create_table_sqlite(dbname, output_tmpdir):
     dbfile = os.path.join(output_tmpdir, 'db.sqlite3')
 

invisible_cities/database/load_db.py

@@ -11,6 +11,7 @@ class DetDB:
     demopp  = os.environ['ICTDIR'] + '/invisible_cities/database/localdb.DEMOPPDB.sqlite3'
     next100 = os.environ['ICTDIR'] + '/invisible_cities/database/localdb.NEXT100DB.sqlite3'
     flex100 = os.environ['ICTDIR'] + '/invisible_cities/database/localdb.Flex100DB.sqlite3'
+    hddemo  = os.environ['ICTDIR'] + '/invisible_cities/database/localdb.HDDEMODB.sqlite3'
 
 def tmap(*args):
     return tuple(map(*args))
@@ -68,7 +69,7 @@ def DataSiPM(db_file, run_number=1e5):
 ON pos.SensorID = map.SensorID LEFT JOIN
 (select * from ChannelMask where MinRun <= {0} and {0} <= MaxRun) as msk
 ON pos.SensorID = msk.SensorID
-where pos.SensorID > 100
+where pos.SensorID >= 1000
 and pos.MinRun <= {0} and {0} <= pos.MaxRun
 and gain.MinRun <= {0} and {0} <= gain.MaxRun
 and map.MinRun <= {0} and {0} <= map.MaxRun
@@ -82,6 +83,34 @@ def DataSiPM(db_file, run_number=1e5):
 
     return data
 
+@lru_cache(maxsize=10)
+def DataFiber(db_file, run_number=1e5):
+    if run_number == 0:
+        run_number = runNumberForMC
+
+    conn = sqlite3.connect(get_db(db_file))
+
+    sql='''select pos.SensorID, map.ElecID "ChannelID",
+case when msk.SensorID is NULL then 1 else 0 end "Active",
+X, Y,
+gain.Centroid "adc_to_pes", gain.ErrorCentroid "adc_to_pes_err", gain.Sigma, gain.ErrorSigma,
+amp.Centroid "amplification", amp.ErrorCentroid "amplification_err", amp.Sigma "amp_Sigma", amp.ErrorSigma "amp_ErrorSigma"
+from ChannelPosition as pos
+INNER JOIN ChannelGain as gain ON pos.SensorID = gain.SensorID
+INNER JOIN ChannelAmplification as amp ON pos.SensorID = amp.SensorID
+INNER JOIN ChannelMapping as map ON pos.SensorID = map.SensorID LEFT JOIN
+(select * from ChannelMask where MinRun <= {0} and {0} <= MaxRun) as msk
+ON pos.SensorID = msk.SensorID
+where pos.Label = 'Fiber'
+and pos.MinRun <= {0} and {0} <= pos.MaxRun
+and gain.MinRun <= {0} and {0} <= gain.MaxRun
+and amp.MinRun <= {0} and {0} <= amp.MaxRun
+and map.MinRun <= {0} and {0} <= map.MaxRun
+order by pos.SensorID'''.format(abs(run_number))
+    data = pd.read_sql_query(sql, conn)
+    conn.close()
+    return data
+
 @lru_cache(maxsize=10)
 def DetectorGeo(db_file):
     conn = sqlite3.connect(get_db(db_file))

invisible_cities/database/load_db_test.py

@@ -41,6 +41,18 @@ def test_sipm_pd(db):
     assert sipms.shape[0] == db.nsipms
 
 
+def test_fiber_pd(db):
+    """Check that we retrieve the correct number of Fibers."""
+    if db.nfibers == 0:
+        pytest.skip("No fibers in this detector")
+    fibers = DB.DataFiber(db.detector)
+    columns = ['SensorID', 'ChannelID', 'Active', 'X', 'Y',
+               'adc_to_pes', 'adc_to_pes_err', 'Sigma', 'ErrorSigma',
+               'amplification', 'amplification_err', 'amp_Sigma', 'amp_ErrorSigma']
+    assert columns == list(fibers)
+    assert fibers.shape[0] == db.nfibers
+
+
 def test_SiPMNoise(db):
     """Check we have noise for all SiPMs and energy of each bin."""
     noise, energy, baseline = DB.SiPMNoise(db.detector)
@@ -64,6 +76,8 @@ def test_DetectorGeometry(dbnew):
 def test_mc_runs_equal_data_runs(db):
     assert (DB.DataPMT (db.detector, -3550).values == DB.DataPMT (db.detector, 3550).values).all()
     assert (DB.DataSiPM(db.detector, -3550).values == DB.DataSiPM(db.detector, 3550).values).all()
+    if db.nfibers > 0:
+        assert (DB.DataFiber(db.detector, -3550).values == DB.DataFiber(db.detector, 3550).values).all()
 
 
 @fixture(scope='module')
@@ -151,6 +165,8 @@ def test_frontend_mapping(db):
         pytest.skip("NEXT100 not implemented yet")
     if db.detector == "flex100":
         pytest.skip("Flex100 not implemented yet")
+    if db.detector == "hddemo":
+        pytest.skip("hddemo PMT FE tables not populated")
 
     run_number = 6000
     fe_mapping, _ = DB.PMTLowFrequencyNoise(db.detector, run_number)
@@ -169,6 +185,8 @@ def test_pmt_noise_frequencies(db):
         pytest.skip("NEXT100 not implemented yet")
     if db.detector == "flex100":
         pytest.skip("Flex100 not implemented yet")
+    if db.detector == "hddemo":
+        pytest.skip("hddemo PMT FE tables not populated")
 
     run_number = 5000
     _, frequencies = DB.PMTLowFrequencyNoise(db.detector, run_number)