diff --git a/carta/cpp/CartaLib/CartaLib.pro b/carta/cpp/CartaLib/CartaLib.pro index 086db5ad..0fe33200 100755 --- a/carta/cpp/CartaLib/CartaLib.pro +++ b/carta/cpp/CartaLib/CartaLib.pro @@ -47,6 +47,8 @@ SOURCES += \ Algorithms/LineCombiner.cpp \ IImageRenderService.cpp \ IRemoteVGView.cpp \ + IPCache.cpp \ + Hooks/GetPersistentCache.cpp \ Hooks/GetProfileExtractor.cpp \ Regions/IRegion.cpp \ InputEvents.cpp \ @@ -55,8 +57,7 @@ SOURCES += \ Regions/CoordinateSystemFormatter.cpp \ Regions/Ellipse.cpp \ Regions/Point.cpp \ - Regions/Rectangle.cpp \ - IPCache.cpp + Regions/Rectangle.cpp HEADERS += \ CartaLib.h\ @@ -108,12 +109,14 @@ HEADERS += \ IImageRenderService.h \ Hooks/GetImageRenderService.h \ IRemoteVGView.h \ + RegionInfo.h \ Hooks/GetProfileExtractor.h \ Regions/IRegion.h \ InputEvents.h \ Regions/ICoordSystem.h \ Hooks/CoordSystemHook.h \ Regions/CoordinateSystemFormatter.h \ + Hooks/GetPersistentCache.h \ Regions/Ellipse.h \ Regions/Point.h \ Regions/Rectangle.h \ diff --git a/carta/cpp/CartaLib/Hooks/GetPersistentCache.cpp b/carta/cpp/CartaLib/Hooks/GetPersistentCache.cpp new file mode 100644 index 00000000..48ba5c52 --- /dev/null +++ b/carta/cpp/CartaLib/Hooks/GetPersistentCache.cpp @@ -0,0 +1,14 @@ +/** + * + **/ + +#include "GetPersistentCache.h" + +namespace Carta +{ +namespace Lib +{ +namespace Hooks +{ } +} +} diff --git a/carta/cpp/CartaLib/Hooks/GetPersistentCache.h b/carta/cpp/CartaLib/Hooks/GetPersistentCache.h new file mode 100644 index 00000000..b1ac8202 --- /dev/null +++ b/carta/cpp/CartaLib/Hooks/GetPersistentCache.h @@ -0,0 +1,57 @@ +/** + * Defines a hook for obtaining a persistent cache object. + * + **/ + +#pragma once + +#include "CartaLib/CartaLib.h" +#include "CartaLib/IPlugin.h" +#include "CartaLib/IPCache.h" + +namespace Carta +{ +namespace Lib +{ +namespace Hooks +{ +/// \brief Hook for loading a plugin of an unknown type +/// +class GetPersistentCache : public BaseHook +{ + CARTA_HOOK_BOILER1( GetPersistentCache ); + +public: + + /// Result of the hook is an instance of IPCache object. + /// This hook should only be ever called once, since the persistent cache + /// should be treated as a singleton. The reason I am not implementing this + /// as a singleton is because in the future, we may addd a mechanism (see below) + /// to be able to create multiple independent caches. + typedef IPCache::SharedPtr ResultType; + + /// input parameters are: + /// nothing + /// All necessary parameters are retrieved from carta.config. + /// + /// In the future we could make this contain overrides of some sort, in case + /// some code would like to create additional independent caches... + struct Params { + Params( ) + { + } + }; + + /// standard constructor (could be probably a macro) + GetPersistentCache( Params * pptr ) : BaseHook( staticId ), paramsPtr( pptr ) + { + // force instantiation of templates + CARTA_ASSERT( is < Me > () ); + } + + ResultType result; + Params * paramsPtr = nullptr; +}; +} +} +} diff --git a/carta/cpp/CartaLib/Hooks/HookIDs.h b/carta/cpp/CartaLib/Hooks/HookIDs.h index 256d3109..fb4af389 100644 --- a/carta/cpp/CartaLib/Hooks/HookIDs.h +++ b/carta/cpp/CartaLib/Hooks/HookIDs.h @@ -31,7 +31,7 @@ enum class UniqueHookIDs { ProfileHook_ID, Fit1DHook_ID, ImageStatisticsHook_ID, - + GetPersistentCache_ID, GetProfileExtractor_ID, /// region related stuff, still to be considered experimental diff --git a/carta/cpp/CartaLib/IPCache.h b/carta/cpp/CartaLib/IPCache.h index a12856cd..edc30b6e 100644 --- a/carta/cpp/CartaLib/IPCache.h +++ b/carta/cpp/CartaLib/IPCache.h @@ -1,9 +1,8 @@ -/** - * - **/ +/// IPCache is a set of APIs to access persistent cache. #pragma once +#include "CartaLib/CartaLib.h" #include #include #include @@ -15,6 +14,8 @@ namespace Lib { class IPCache { + CLASS_BOILERPLATE( IPCache ); + public: /// return maximum storage in bytes @@ -34,8 +35,8 @@ class IPCache deleteAll() = 0; /// read a value of an entry - /// if entry does not exist, val will be null - virtual void + /// if entry does not exist, false is returned + virtual bool readEntry( const QByteArray & key, QByteArray & val ) = 0; diff --git a/carta/cpp/CartaLib/IPlugin.h b/carta/cpp/CartaLib/IPlugin.h index 66e0dd23..41dfe757 100644 --- a/carta/cpp/CartaLib/IPlugin.h +++ b/carta/cpp/CartaLib/IPlugin.h @@ -104,6 +104,9 @@ class IPlugin /// called immediately after the plugin was loaded /// TODO: should be pure virtual, don't be lazy! + /// + /// This is different from the Initialize hook, which is delivered after + /// all plugins have been loaded and after core is started. virtual void initialize( const InitInfo & initInfo ) { diff --git a/carta/cpp/core/Algorithms/cacheUtils.h b/carta/cpp/core/Algorithms/cacheUtils.h new file mode 100644 index 00000000..a01b3639 --- /dev/null +++ b/carta/cpp/core/Algorithms/cacheUtils.h @@ -0,0 +1,119 @@ +/** + * Functions for serializing and deserializing keys and values to be used with the disk cache + **/ + +#pragma once + +/** + * Int --> byte array + **/ +QByteArray i2qb( const int & d) { + QByteArray ba; + ba.append( (const char *)( & d), sizeof( int)); + return ba; +} + +/** + * Byte array --> int + **/ +int qb2i( const QByteArray & ba) { + if( ba.size() != sizeof(int)) { + throw std::runtime_error("Could not unpack QByteArray into int: size is incorrect."); + } + return * ((const int *) (ba.constData())); +} + +/** + * Double --> byte array + **/ +QByteArray d2qb( const double & d) { + QByteArray ba; + ba.append( (const char *)( & d), sizeof( double)); + return ba; +} + +/** + * Byte array --> double + **/ +double qb2d( const QByteArray & ba) { + if( ba.size() != sizeof(double)) { + throw std::runtime_error("Could not unpack QByteArray into double: size is incorrect."); + } + return * ((const double *) (ba.constData())); +} + +/** + * Vector of doubles --> byte array + **/ +QByteArray vd2qb( const std::vector & vd) { + QByteArray ba; + for( const double & d : vd) { + ba.append( (const char *)( & d), sizeof( double)); + } + return ba; +} + +/** + * Byte array --> vector of doubles + **/ +std::vector qb2vd( const QByteArray & ba) { + std::vector vd; + if( ba.size() % sizeof(double) != 0) { + throw std::runtime_error("Could not unpack QByteArray into std::vector: size is incorrect."); + } + const char * cptr = ba.constData(); + for( int i = 0 ; i < ba.size() ; i += sizeof(double)) { + vd.push_back( * ((const double *) (cptr + i))); + } + return vd; +} + +/** + * Vector of integers --> byte array + **/ +QByteArray vi2qb( const std::vector & vi) { + QByteArray ba; + for( const int & i : vi) { + ba.append( (const char *)( & i), sizeof( int)); + } + return ba; +} + +/** + * Byte array --> vector of integers + **/ +std::vector qb2vi( const QByteArray & ba) { + std::vector vi; + if( ba.size() % sizeof(int) != 0) { + throw std::runtime_error("Could not unpack QByteArray into std::vector: size is incorrect."); + } + const char * cptr = ba.constData(); + for( int i = 0 ; i < ba.size() ; i += sizeof(int)) { + vi.push_back( * ((const int *) (cptr + i))); + } + return vi; +} + +/** + * Pair of int, double --> byte array + **/ +QByteArray id2qb( const std::pair & id) { + QByteArray ba; + ba.append( (const char *)( & id.first), sizeof( int)); + ba.append( (const char *)( & id.second), sizeof( double)); + return ba; +} + + +/** + * Byte array --> pair of int, double + **/ +std::pair qb2id( const QByteArray & ba) { + if( ba.size() != (sizeof(double) + sizeof(int))) { + throw std::runtime_error("Could not unpack QByteArray into std::pair: size is incorrect."); + } + const char * cptr = ba.constData(); + int int_val( * ((const int *) (cptr))); + double double_val( * ((const double *) (cptr + sizeof(int)))); + return std::make_pair(int_val, double_val); +} diff --git a/carta/cpp/core/Data/Image/DataSource.cpp b/carta/cpp/core/Data/Image/DataSource.cpp index 74bfefc1..5513b7a8 100755 --- a/carta/cpp/core/Data/Image/DataSource.cpp +++ b/carta/cpp/core/Data/Image/DataSource.cpp @@ -8,9 +8,12 @@ #include "Data/Util.h" #include "Data/Colormap/TransformsData.h" #include "CartaLib/Hooks/LoadAstroImage.h" +#include "CartaLib/Hooks/GetPersistentCache.h" #include "CartaLib/PixelPipeline/CustomizablePixelPipeline.h" +#include "CartaLib/IPCache.h" #include "../../ImageRenderService.h" #include "../../Algorithms/quantileAlgorithms.h" +#include "../../Algorithms/cacheUtils.h" #include #include #include @@ -55,6 +58,17 @@ DataSource::DataSource() : m_pixelPipeline-> setColormap( std::make_shared < Carta::Core::GrayColormap > () ); m_pixelPipeline-> setMinMax( 0, 1 ); m_renderService-> setPixelPipeline( m_pixelPipeline, m_pixelPipeline-> cacheId()); + + // initialize disk cache + auto res = Globals::instance()-> pluginManager() + -> prepare < Carta::Lib::Hooks::GetPersistentCache > ().first(); + if ( res.isNull() || ! res.val() ) { + qWarning( "Could not find a disk cache plugin." ); + m_diskCache = nullptr; + } + else { + m_diskCache = res.val(); + } } @@ -333,7 +347,7 @@ std::shared_ptr DataSource::_getRender return m_renderService; } - +// TODO: create another function which only looks for the intensity. Most calling functions don't need the location. std::vector > DataSource::_getIntensityCache( int frameLow, int frameHigh, const std::vector& percentiles ){ //See if it is in the cached percentiles first. @@ -342,15 +356,44 @@ std::vector > DataSource::_getIntensityCache( int frameLow //Find all the intensities we can in the cache. int foundCount = 0; for ( int i = 0; i < percentileCount; i++ ){ + std::pair val = m_cachedPercentiles.getIntensity( frameLow, frameHigh, percentiles[i]); if ( val.first>= 0 ){ + qDebug() << "++++++++ found location and intensity in memory cache"; intensities[i] = val; foundCount++; + } else if (m_diskCache) { + // disk cache + // Look for the location first + QString locationKey = QString("%1/%2/%3/%4/location").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); + QByteArray locationVal; + bool locationInCache = m_diskCache->readEntry(locationKey.toUtf8(), locationVal); + + qDebug() << "++++++++ location key is" << locationKey.toUtf8(); + + if (locationInCache) { + QString intensityKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); + QByteArray intensityVal; + bool intensityInCache = m_diskCache->readEntry(intensityKey.toUtf8(), intensityVal); + + qDebug() << "++++++++ intensity key is" << intensityKey.toUtf8(); + + if (intensityInCache) { + qDebug() << "++++++++ found location and intensity in disk cache"; + intensities[i] = std::make_pair(qb2i(locationVal), qb2d(intensityVal)); + foundCount++; + // put them in the memory cache + m_cachedPercentiles.put( frameLow, frameHigh, intensities[i].first, percentiles[i], intensities[i].second ); + } + } } + + qDebug() << "++++++++ For percentile" << percentiles[i] << "intensity is" << intensities[i].second << "and location is" << intensities[i].first; } //Not all percentiles were in the cache. We are going to have to look some up. if ( foundCount < percentileCount ){ + qDebug() << "++++++++ Calculating some values"; std::vector > allValues; int spectralIndex = Util::getAxisIndex( m_image, AxisInfo::KnownType::SPECTRAL ); @@ -408,7 +451,19 @@ std::vector > DataSource::_getIntensityCache( int frameLow intensities[i].first += frameLow; } + // put calculated values in both the memory cache and the disk cache + m_cachedPercentiles.put( frameLow, frameHigh, intensities[i].first, percentiles[i], intensities[i].second ); + + if (m_diskCache) { + QString locationKey = QString("%1/%2/%3/%4/location").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); + QString intensityKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); + + m_diskCache->setEntry(locationKey.toUtf8(), i2qb(intensities[i].first), 0); + m_diskCache->setEntry(intensityKey.toUtf8(), d2qb(intensities[i].second), 0); + } + + qDebug() << "++++++++ For percentile" << percentiles[i] << "intensity is" << intensities[i].second << "and location is" << intensities[i].first; } } } @@ -895,18 +950,51 @@ void DataSource::_setGamma( double gamma ){ void DataSource::_updateClips( std::shared_ptr& view, double minClipPercentile, double maxClipPercentile, const std::vector& frames ){ - std::vector mFrames = _fitFramesToImage( frames ); + std::vector mFrames = _fitFramesToImage( frames ); int quantileIndex = _getQuantileCacheIndex( mFrames ); std::vector clips = m_quantileCache[ quantileIndex].m_clips; if ( clips.size() < 2 || - m_quantileCache[quantileIndex].m_minPercentile != minClipPercentile || - m_quantileCache[quantileIndex].m_maxPercentile != maxClipPercentile ) { - Carta::Lib::NdArray::Double doubleView( view.get(), false ); - clips = Carta::Core::Algorithms::quantiles2pixels( - doubleView, { minClipPercentile, maxClipPercentile }); - m_quantileCache[quantileIndex].m_clips = clips; - m_quantileCache[quantileIndex].m_minPercentile = minClipPercentile; - m_quantileCache[quantileIndex].m_maxPercentile = maxClipPercentile; + m_quantileCache[quantileIndex].m_minPercentile != minClipPercentile || + m_quantileCache[quantileIndex].m_maxPercentile != maxClipPercentile ) { + + bool minClipInCache(0); + bool maxClipInCache(0); + + // TODO: check if these are the right frame values and percentile values + QString minClipKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frames[0]).arg(frames.back()).arg(minClipPercentile); + QString maxClipKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frames[0]).arg(frames.back()).arg(maxClipPercentile); + + qDebug() << "++++++++ minClipKey" << minClipKey.toUtf8() << "maxClipKey" << maxClipKey.toUtf8(); + + QByteArray minClipVal; + QByteArray maxClipVal; + + if (m_diskCache) { + minClipInCache = m_diskCache->readEntry(minClipKey.toUtf8(), minClipVal); + maxClipInCache = m_diskCache->readEntry(maxClipKey.toUtf8(), maxClipVal); + } + + if (minClipInCache && maxClipInCache) { + clips.clear(); + clips.push_back(qb2d(minClipVal)); + clips.push_back(qb2d(maxClipVal)); + qDebug() << "++++++++ got clips from cache"; + } else { + Carta::Lib::NdArray::Double doubleView( view.get(), false ); + clips = Carta::Core::Algorithms::quantiles2pixels(doubleView, { minClipPercentile, maxClipPercentile }); + + if (m_diskCache) { + m_diskCache->setEntry( minClipKey.toUtf8(), d2qb(clips[0]), 0); + m_diskCache->setEntry( maxClipKey.toUtf8(), d2qb(clips[1]), 0); + qDebug() << "++++++++ calculated clips and put in cache"; + } + } + + qDebug() << "++++++++ clips are" << clips[0] << "and" << clips[1]; + + m_quantileCache[quantileIndex].m_clips = clips; + m_quantileCache[quantileIndex].m_minPercentile = minClipPercentile; + m_quantileCache[quantileIndex].m_maxPercentile = maxClipPercentile; } m_pixelPipeline-> setMinMax( clips[0], clips[1] ); m_renderService-> setPixelPipeline( m_pixelPipeline, m_pixelPipeline-> cacheId()); diff --git a/carta/cpp/core/Data/Image/DataSource.h b/carta/cpp/core/Data/Image/DataSource.h index c697db6e..948d33d3 100755 --- a/carta/cpp/core/Data/Image/DataSource.h +++ b/carta/cpp/core/Data/Image/DataSource.h @@ -26,6 +26,7 @@ class ImageInterface; namespace NdArray { class RawViewInterface; } +class IPCache; } @@ -502,7 +503,10 @@ class DataSource : public QObject { ///pixel pipeline std::shared_ptr m_pixelPipeline; - + + // disk cache + std::shared_ptr m_diskCache; + //Indices of the display axes. int m_axisIndexX; int m_axisIndexY; diff --git a/carta/cpp/core/MainConfig.cpp b/carta/cpp/core/MainConfig.cpp index a1706770..591f1ab1 100644 --- a/carta/cpp/core/MainConfig.cpp +++ b/carta/cpp/core/MainConfig.cpp @@ -170,6 +170,11 @@ int ParsedInfo::toInt( const QJsonValue& jsonValue, QString& errorMsg ){ return val; } +QJsonObject::iterator ParsedInfo::insert(const QString &key, const QJsonValue &value) { + return m_json.insert(key, value); +} + + } // namespace MainConfig diff --git a/carta/cpp/core/MainConfig.h b/carta/cpp/core/MainConfig.h index 880b2887..156499e3 100644 --- a/carta/cpp/core/MainConfig.h +++ b/carta/cpp/core/MainConfig.h @@ -78,7 +78,12 @@ class ParsedInfo { * error doing the conversion. */ static int toInt( const QJsonValue& jsonValue, QString& errorMsg ); - + + + /** + * Allows modification of the JSON object by tests. + */ + QJsonObject::iterator insert(const QString &key, const QJsonValue &value); protected: diff --git a/carta/cpp/core/PluginManager.cpp b/carta/cpp/core/PluginManager.cpp index 526ce8f7..b67a9d82 100644 --- a/carta/cpp/core/PluginManager.cpp +++ b/carta/cpp/core/PluginManager.cpp @@ -171,6 +171,11 @@ PluginManager::loadPlugins() initInfo.pluginPath = pInfo.dirPath; auto json = Globals::instance()-> mainConfig()-> json(); initInfo.json = json["plugins"].toObject()[pInfo.json.name].toObject(); + if( 0) { + QJsonDocument doc( initInfo.json); + qDebug() << " name:" << pInfo.json.name; + qDebug() << " json:" << doc.toJson(); + } pInfo.rawPlugin->initialize( initInfo ); // find out what hooks this plugin wants to listen to diff --git a/carta/cpp/cpp.pro b/carta/cpp/cpp.pro index 22e23faf..a822a053 100644 --- a/carta/cpp/cpp.pro +++ b/carta/cpp/cpp.pro @@ -9,6 +9,7 @@ SUBDIRS = \ desktop \ plugins \ Tests \ + testCache \ testRegion isEmpty(NOSERVER) { @@ -21,6 +22,8 @@ desktop.depends = core server.depends = core testRegion.depends = core plugins.depends = core +testRegion.depends = core +testCache.depends = core isEmpty(NOSERVER) { Tests.depends = core desktop server plugins } diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp new file mode 100644 index 00000000..58ea2f73 --- /dev/null +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp @@ -0,0 +1,186 @@ +#include "PCacheLevelDB.h" +#include "CartaLib/Hooks/GetPersistentCache.h" +#include +#include +#include +#include "leveldb/db.h" + +typedef Carta::Lib::Hooks::GetPersistentCache GetPersistentCacheHook; + +/// +/// Implementation of IPCache using LevelDB +/// +class LevelDBPCache : public Carta::Lib::IPCache +{ +public: + + virtual uint64_t + maxStorage() override + { + return 1; + } + + virtual uint64_t + usedStorage() override + { + return 1; + } + + virtual uint64_t + nEntries() override + { + return 1; + } + + virtual void + deleteAll() override + { + if ( ! p_db ) { + return; + } + + leveldb::Iterator* it = p_db->NewIterator(p_readOptions); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + auto status = p_db->Delete(p_writeOptions, it->key()); + if ( ! status.ok() ) { + qWarning() << "query delete failed:" << status.ToString().c_str(); + } + } + } // deleteAll + + virtual bool + readEntry( const QByteArray & key, QByteArray & val ) override + { + + if ( ! p_db ) { + return false; + } + std::string tmpVal; + auto status = p_db-> Get( p_readOptions, key.constData(), & tmpVal ); + if ( ! status.ok() ) { + //qWarning() << "query read failed:" << status.ToString().c_str(); + return false; + } + val = QByteArray( tmpVal.data(), tmpVal.size()); + return true; + } // readEntry + + virtual void + setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) override + { + if( ! p_db) return; + auto status = p_db-> Put( p_writeOptions, + leveldb::Slice( key.constData(), key.size()), + leveldb::Slice( val.constData(), val.size())); + if ( ! status.ok() ) { + qWarning() << "query insert failed:" << status.ToString().c_str(); + } + + Q_UNUSED( priority); + // \todo we'll have to embed priority into the value, e.g. first 8 bytes? + } // setEntry + + static + Carta::Lib::IPCache::SharedPtr + getCacheSingleton( QString dirPath) + { + if ( m_cachePtr ) { + qCritical() << "PCacheLevelDBPlugin::Calling GetPersistentCacheHook multiple times!!!"; + } + else { + m_cachePtr.reset( new LevelDBPCache( dirPath) ); + } + return m_cachePtr; + } + + //~LevelDBPCache() + //{ + // I don't think a destructor is required; leaving this as a placeholder for now + //} + +private: + + LevelDBPCache( QString dirPath) + { + p_readOptions = leveldb::ReadOptions(); + p_writeOptions = leveldb::WriteOptions(); + p_writeOptions.sync = false; + + // Set up database connection information and open database + leveldb::DB * db; + leveldb::Options options; + + options.create_if_missing = true; + + leveldb::Status status = leveldb::DB::Open( options, dirPath.toStdString(), & db ); + + if ( false == status.ok() ) { + qDebug() << "Unable to open/create database '" << dirPath.toStdString().c_str() << "':" << status.ToString().c_str(); + return; + } + + p_db.reset( db ); + } + + std::unique_ptr< leveldb::DB > p_db; + leveldb::ReadOptions p_readOptions; + leveldb::WriteOptions p_writeOptions; + static Carta::Lib::IPCache::SharedPtr m_cachePtr; // = nullptr; +}; + +Carta::Lib::IPCache::SharedPtr LevelDBPCache::m_cachePtr = nullptr; + +PCacheLevelDBPlugin::PCacheLevelDBPlugin( QObject * parent ) : + QObject( parent ) +{ } + +bool +PCacheLevelDBPlugin::handleHook( BaseHook & hookData ) +{ + // we only handle one hook: get the cache object + if ( hookData.is < GetPersistentCacheHook > () ) { + // decode hook data + GetPersistentCacheHook & hook = static_cast < GetPersistentCacheHook & > ( hookData ); + + // if no dbdir was specified, refuse to work :) + if( m_dbPath.isNull()) { + hook.result.reset(); + return false; + } + + // try to create the database + hook.result = LevelDBPCache::getCacheSingleton( m_dbPath); + + // return true if result is not null + return hook.result != nullptr; + } + + qWarning() << "PCacheLevelDBPlugin: Sorrry, dont' know how to handle this hook"; + return false; +} // handleHook + +void +PCacheLevelDBPlugin::initialize( const IPlugin::InitInfo & initInfo ) +{ + qDebug() << "PCacheLevelDBPlugin::initialized"; + QJsonDocument doc( initInfo.json ); + qDebug() << doc.toJson(); + + // extract the location of the database from carta.config + m_dbPath = initInfo.json.value( "dbPath").toString(); + if( m_dbPath.isNull()) { + qCritical() << "No dbPath specified for PCacheLevelDB plugin!!!"; + } + else { + // convert this to absolute path just in case + m_dbPath = QDir(m_dbPath).absolutePath(); + } +} + +std::vector < HookId > +PCacheLevelDBPlugin::getInitialHookList() +{ + return { + GetPersistentCacheHook::staticId + }; +} diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h new file mode 100644 index 00000000..ca2c8f5b --- /dev/null +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h @@ -0,0 +1,31 @@ +/// Implements plugin for persistent cache, using QT's sqlite3 database driver + +/// This plugin can read image formats that Qt supports. + +#pragma once + +#include "CartaLib/IPlugin.h" +#include +#include + +class PCacheLevelDBPlugin : public QObject, public IPlugin +{ + Q_OBJECT + Q_PLUGIN_METADATA( IID "org.cartaviewer.IPlugin" ) + Q_INTERFACES( IPlugin ) + +public : + PCacheLevelDBPlugin( QObject * parent = 0 ); + virtual bool + handleHook( BaseHook & hookData ) override; + + virtual std::vector < HookId > + getInitialHookList() override; + + virtual void + initialize( const InitInfo & initInfo ) override; + +private: + + QString m_dbPath; +}; diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.pro b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.pro new file mode 100644 index 00000000..1ea12f24 --- /dev/null +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.pro @@ -0,0 +1,34 @@ +! include(../../common.pri) { + error( "Could not find the common.pri file!" ) +} + +INCLUDEPATH += $$PROJECT_ROOT +DEPENDPATH += $$PROJECT_ROOT + +QT += core gui + +TARGET = plugin +TEMPLATE = lib +CONFIG += plugin + +SOURCES += \ + PCacheLevelDB.cpp + +HEADERS += \ + PCacheLevelDB.h + +LIBS += -lleveldb + +OTHER_FILES += \ + plugin.json + +# copy json to build directory +#MYFILES = $$files($${PWD}/files/*.*) +MYFILES = plugin.json +copy_files.name = copy large files +copy_files.input = MYFILES +# change datafiles to a directory you want to put the files to +copy_files.output = $${OUT_PWD}/${QMAKE_FILE_BASE}${QMAKE_FILE_EXT} +copy_files.commands = ${COPY_FILE} ${QMAKE_FILE_IN} ${QMAKE_FILE_OUT} +copy_files.CONFIG += no_link target_predeps +QMAKE_EXTRA_COMPILERS += copy_files diff --git a/carta/cpp/plugins/PCacheLevelDB/plugin.json b/carta/cpp/plugins/PCacheLevelDB/plugin.json new file mode 100644 index 00000000..4e51b32d --- /dev/null +++ b/carta/cpp/plugins/PCacheLevelDB/plugin.json @@ -0,0 +1,9 @@ +{ + "api" : "1", + "name" : "PCacheLevelDB", + "version" : "1", + "type" : "C++", + "description": "Implements persistent cache using LevelDB.", + "about" : "Part of CARTA. Written by Pavol and Adrianna.", + "depends" : [ ] +} diff --git a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp new file mode 100644 index 00000000..8fb68b54 --- /dev/null +++ b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp @@ -0,0 +1,201 @@ +#include "PCacheSqlite3.h" +#include "CartaLib/Hooks/GetPersistentCache.h" +#include +#include +#include + +typedef Carta::Lib::Hooks::GetPersistentCache GetPersistentCacheHook; + +/// +/// Implementation of IPCache using sqlite +/// +class SqLitePCache : public Carta::Lib::IPCache +{ +public: + + virtual uint64_t + maxStorage() override + { + return 1; + } + + virtual uint64_t + usedStorage() override + { + return 1; + } + + virtual uint64_t + nEntries() override + { + return 1; + } + + virtual void + deleteAll() override + { + if ( ! m_db.isOpen() ) { + return; + } + QSqlQuery query( m_db ); + query.prepare( "DELETE FROM db;" ); + + if ( ! query.exec() ) { + qWarning() << "query delete failed"; + } + } // deleteAll + + virtual bool + readEntry( const QByteArray & key, QByteArray & val ) override + { + if ( ! m_db.isOpen() ) { + return false; + } + QSqlQuery query( m_db ); + query.prepare( "SELECT val FROM db WHERE key = :key" ); + query.bindValue( ":key", key ); + + // qDebug() << "select query" << query.lastQuery(); + if ( ! query.exec() ) { + qWarning() << "query read failed"; + return false; + } + if ( query.next() ) { + val = query.value( 0 ).toByteArray(); + return true; + } + return false; + } // readEntry + + virtual void + setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) override + { + if ( ! m_db.isOpen() ) { + return; + } + + Q_UNUSED( priority ); + QSqlQuery query( m_db ); + query.prepare( "INSERT OR REPLACE INTO db (key, val) VALUES ( :key, :val)" ); + + // query.prepare( "INSERT INTO db (key, val) VALUES ( :key, :val)" ); + query.bindValue( ":key", key ); + query.bindValue( ":val", val ); + + // qDebug() << "insert query" << query.lastQuery(); + if ( ! query.exec() ) { + qWarning() << "query insert failed" + << query.lastError().text(); + } + } // setEntry + + static + Carta::Lib::IPCache::SharedPtr + getCacheSingleton( QString dirPath) + { + if ( m_cachePtr ) { + qCritical() << "PCacheSQlite3Plugin::Calling GetPersistentCacheHook multiple times!!!"; + } + else { + // stupid c++ won't allow this + // m_cachePtr = std::make_shared < SqLitePCache > (); + m_cachePtr.reset( new SqLitePCache( dirPath) ); + } + return m_cachePtr; + } + + ~SqLitePCache() + { + qDebug() << "Destroying SqLitePCache"; + qDebug() << "Possible segmentation fault below, it's a bug and we'll try to fix it."; + m_db.close(); + qDebug() << "Ok, no segmentation fault occurred, whew :)"; + } + +private: + + SqLitePCache( QString dirPath) + { + m_db = QSqlDatabase::addDatabase( "QSQLITE" ); + + m_db.setDatabaseName( dirPath ); + bool ok = m_db.open(); + if ( ! ok ) { + qCritical() << "Could not open sqlite database"; + qCritical() << " - at location:" + dirPath; + } + + QSqlQuery query( m_db ); + + query.prepare( "create table db (key text primary key, " + "val blob)" ); + + // qDebug() << "create query" << query.lastQuery(); + if ( ! query.exec() ) { + qCritical() << "query create table failed" + << query.lastError().text(); + } + } + +private: + + QSqlDatabase m_db; + static Carta::Lib::IPCache::SharedPtr m_cachePtr; // = nullptr; +}; + +Carta::Lib::IPCache::SharedPtr SqLitePCache::m_cachePtr = nullptr; + +PCacheSQlite3Plugin::PCacheSQlite3Plugin( QObject * parent ) : + QObject( parent ) +{ } + +bool +PCacheSQlite3Plugin::handleHook( BaseHook & hookData ) +{ + // we only handle one hook: get the cache object + if ( hookData.is < GetPersistentCacheHook > () ) { + // decode hook data + GetPersistentCacheHook & hook = static_cast < GetPersistentCacheHook & > ( hookData ); + + // if no dbdir was specified, refuse to work :) + if( m_dbPath.isNull()) { + hook.result.reset(); + return false; + } + + // try to create the database + hook.result = SqLitePCache::getCacheSingleton( m_dbPath); + + // return true if result is not null + return hook.result != nullptr; + } + + qWarning() << "PCacheSQlite3Plugin: Sorrry, dont' know how to handle this hook"; + return false; +} // handleHook + +void +PCacheSQlite3Plugin::initialize( const IPlugin::InitInfo & initInfo ) +{ + qDebug() << "PCacheSQlite3Plugin::initialized"; + QJsonDocument doc( initInfo.json ); + qDebug() << doc.toJson(); + + // extract the location of the database from carta.config + m_dbPath = initInfo.json.value( "dbPath").toString(); + if( m_dbPath.isNull()) { + qCritical() << "No dbPath specified for PCacheSqlite3 plugin!!!"; + } + else { + // convert this to absolute path just in case + m_dbPath = QDir(m_dbPath).absolutePath(); + } +} + +std::vector < HookId > +PCacheSQlite3Plugin::getInitialHookList() +{ + return { + GetPersistentCacheHook::staticId + }; +} diff --git a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h new file mode 100644 index 00000000..aa4051d5 --- /dev/null +++ b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h @@ -0,0 +1,31 @@ +/// Implements plugin for persistent cache, using QT's sqlite3 database driver + +/// This plugin can read image formats that Qt supports. + +#pragma once + +#include "CartaLib/IPlugin.h" +#include +#include + +class PCacheSQlite3Plugin : public QObject, public IPlugin +{ + Q_OBJECT + Q_PLUGIN_METADATA( IID "org.cartaviewer.IPlugin" ) + Q_INTERFACES( IPlugin ) + +public : + PCacheSQlite3Plugin( QObject * parent = 0 ); + virtual bool + handleHook( BaseHook & hookData ) override; + + virtual std::vector < HookId > + getInitialHookList() override; + + virtual void + initialize( const InitInfo & initInfo ) override; + +private: + + QString m_dbPath; +}; diff --git a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.pro b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.pro new file mode 100644 index 00000000..821bcf8c --- /dev/null +++ b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.pro @@ -0,0 +1,32 @@ +! include(../../common.pri) { + error( "Could not find the common.pri file!" ) +} + +INCLUDEPATH += $$PROJECT_ROOT +DEPENDPATH += $$PROJECT_ROOT + +QT += core gui sql + +TARGET = plugin +TEMPLATE = lib +CONFIG += plugin + +SOURCES += \ + PCacheSqlite3.cpp + +HEADERS += \ + PCacheSqlite3.h + +OTHER_FILES += \ + plugin.json + +# copy json to build directory +#MYFILES = $$files($${PWD}/files/*.*) +MYFILES = plugin.json +copy_files.name = copy large files +copy_files.input = MYFILES +# change datafiles to a directory you want to put the files to +copy_files.output = $${OUT_PWD}/${QMAKE_FILE_BASE}${QMAKE_FILE_EXT} +copy_files.commands = ${COPY_FILE} ${QMAKE_FILE_IN} ${QMAKE_FILE_OUT} +copy_files.CONFIG += no_link target_predeps +QMAKE_EXTRA_COMPILERS += copy_files diff --git a/carta/cpp/plugins/PCacheSqlite3/plugin.json b/carta/cpp/plugins/PCacheSqlite3/plugin.json new file mode 100644 index 00000000..73db97b8 --- /dev/null +++ b/carta/cpp/plugins/PCacheSqlite3/plugin.json @@ -0,0 +1,9 @@ +{ + "api" : "1", + "name" : "PCacheSqlite3", + "version" : "1", + "type" : "C++", + "description": "Implements persistent cache using QT's sqlite3 driver.", + "about" : "Part of CARTA. Written by Pavol", + "depends" : [ ] +} diff --git a/carta/cpp/plugins/plugins.pro b/carta/cpp/plugins/plugins.pro index a51d3211..d0163f11 100644 --- a/carta/cpp/plugins/plugins.pro +++ b/carta/cpp/plugins/plugins.pro @@ -25,6 +25,9 @@ SUBDIRS += ColormapsPy SUBDIRS += CyberSKA SUBDIRS += DevIntegration +SUBDIRS += PCacheSqlite3 +SUBDIRS += PCacheLevelDB + # adrianna's render plugin SUBDIRS += hpcImgRender diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp new file mode 100644 index 00000000..a51be0fc --- /dev/null +++ b/carta/cpp/testCache/main.cpp @@ -0,0 +1,254 @@ +#include "CartaLib/Hooks/Initialize.h" +#include "CartaLib/Hooks/GetPersistentCache.h" +#include "core/MyQApp.h" +#include "core/CmdLine.h" +#include "core/MainConfig.h" +#include "core/Globals.h" +#include "core/Algorithms/cacheUtils.h" +#include "CartaLib/IPCache.h" +#include +#include +#include + +namespace tCache +{ +std::shared_ptr < Carta::Lib::IPCache > pcache; + +int imWidth = 20; +int imHeight = 20; +int imDepth = 16000; + +double +generateData( int x, int y, int z ) +{ + return ( x + 0.1 ) * ( y - 0.1 ) / ( z * x + 0.1 ); +} + +static std::vector < double > +genProfile( int x, int y ) +{ + std::vector < double > arr( imDepth ); + for ( int z = 0 ; z < imDepth ; z++ ) { + arr[z] = generateData( x, y, z ); + } + return arr; +} + +static std::vector < double > +readProfile( int x, int y ) +{ + QString key = QString( "%1/%2" ).arg( x ).arg( y ); + QByteArray val; + if ( ! pcache-> readEntry( key.toUtf8(), val ) ) { + return { }; + } + if ( val.size() != int (sizeof( double ) * imDepth) ) { + qWarning() << "Cache size mismatch"; + return { }; + } + + return qb2vd( val ); +} // readProfile + +static void +populateCache() +{ + // figure out which row we have already finished generating + + int startx = 0; + + QByteArray buff; + if ( pcache-> readEntry( "lastx", buff ) ) { + QString str = buff; + bool ok; + int x = str.toInt( & ok ); + if ( ok ) { + startx = x; + } + } + + qDebug() << "Starting from x" << startx; + for ( int x = startx ; x < imWidth ; x++ ) { + qDebug() << "Writing x" << x; + for ( int y = 0 ; y < imHeight ; y++ ) { + std::vector < double > arr = genProfile( x, y ); + + // write the entry + QString keyString = QString( "%1/%2" ).arg( x ).arg( y ); + pcache-> setEntry( keyString.toUtf8(), vd2qb( arr ), 0 ); + } + pcache-> setEntry( "lastx", QString::number( x + 1 ).toUtf8(), 0 ); + } +} // populateCache + +static void +readCache() +{ + QTime t; + t.restart(); + qDebug() << "Sequential read..."; + for ( int x = 0 ; x < imWidth ; x++ ) { + qDebug() << "Testing x" << x; + for ( int y = 0 ; y < imHeight ; y++ ) { + std::vector < double > arr = genProfile( x, y ); + QString keyString = QString( "%1/%2" ).arg( x ).arg( y ); + + QByteArray ba; + if ( ! pcache-> readEntry( keyString.toUtf8(), ba ) ) { + qCritical() << "Failed to read" << x << y; + continue; + } + std::vector < double > arr2 = qb2vd( ba ); + if ( arr != arr2 ) { + qCritical() << "Failed to match" << x << y; + } + + } + qDebug() << " " << ( x + 1 ) * imHeight * 1000.0 / t.elapsed() << "pix/s"; + } +} // readCache + +static void +testCache() +{ + + { + pcache-> setEntry( "hello", "world", 0 ); + QByteArray val; + if ( pcache-> readEntry( "hello", val ) ) { + qDebug() << "hello=" << val; + } + else { + qDebug() << "hello not found"; + } + } + + // populate cache + populateCache(); + + // test cache by reading it + readCache(); + + QByteArray key = "hello"; + QByteArray val; + if ( pcache-> readEntry( key, val ) ) { + qDebug() << "db already had value" << val; + } + else { + qDebug() << "db did not have value"; + } + pcache-> setEntry( key, "hola", 0 ); +} // testCache + +static int +coreMainCPP( QString platformString, int argc, char * * argv ) +{ + + MyQApp qapp( argc, argv ); + + QString appName = "carta-" + platformString; +#ifndef QT_NO_DEBUG_OUTPUT + appName += "-verbose"; +#endif + if ( CARTA_RUNTIME_CHECKS ) { + appName += "-runtimeChecks"; + } + MyQApp::setApplicationName( appName ); + + qDebug() << "Starting" << qapp.applicationName() << qapp.applicationVersion(); + + // alias globals + auto & globals = * Globals::instance(); + + // parse command line arguments & environment variables + // ==================================================== + auto cmdLineInfo = CmdLine::parse( MyQApp::arguments() ); + globals.setCmdLineInfo( & cmdLineInfo ); + + // load the config file + // ==================== + QString configFilePath = cmdLineInfo.configFilePath(); + MainConfig::ParsedInfo mainConfig = MainConfig::parse( configFilePath ); + + // Re-enable all plugins so that all pcache plugins are tested + mainConfig.insert("disabledPlugins", QJsonValue(QJsonArray())); + + // Use test database files + // Hideous, but apparently the only way to modify nested QJSONObject values. + + QJsonObject plugins = mainConfig.json()["plugins"].toObject(); + QJsonObject sqlite3Config = plugins["PCacheSqlite3"].toObject(); + QJsonObject leveldbConfig = plugins["PCacheLevelDB"].toObject(); + sqlite3Config.insert("dbPath", "/tmp/pcache.sqlite.test"); + leveldbConfig.insert("dbPath", "/tmp/pcache.leveldb.test"); + plugins.insert("PCacheSqlite3", QJsonValue(sqlite3Config)); + plugins.insert("PCacheLevelDB", QJsonValue(leveldbConfig)); + mainConfig.insert("plugins", QJsonValue(plugins)); + + globals.setMainConfig( & mainConfig ); + qDebug() << "plugin directories:\n - " + mainConfig.pluginDirectories().join( "\n - " ); + + // initialize plugin manager + // ========================= + globals.setPluginManager( std::make_shared < PluginManager > () ); + auto pm = globals.pluginManager(); + + // tell plugin manager where to find plugins + pm-> setPluginSearchPaths( globals.mainConfig()->pluginDirectories() ); + + // find and load plugins + pm-> loadPlugins(); + + qDebug() << "Loading plugins..."; + auto infoList = pm-> getInfoList(); + qDebug() << "List of loaded plugins: [" << infoList.size() << "]"; + for ( const auto & entry : infoList ) { + qDebug() << " path:" << entry.json.name; + } + + // send an initialize hook to all plugins, because some may rely on it + pm-> prepare < Carta::Lib::Hooks::Initialize > ().executeAll(); + + + // make a lambda to set the value of pcache and call the tests + auto lam = [=] ( const Carta::Lib::Hooks::GetPersistentCache::ResultType &res ) { + pcache = res; + pcache->deleteAll(); + testCache(); + }; + + // call the lambda on every pcache plugin + auto pcacheRes = pm-> prepare< Carta::Lib::Hooks::GetPersistentCache >(); + pcacheRes.forEach(lam); + + // give QT control +// int res = qapp.exec(); + int res = 0; + + // if we get here, it means we are quitting... + qDebug() << "Exiting"; + return res; +} // coreMainCPP +} + +int +main( int argc, char * * argv ) +{ + try { + return tCache::coreMainCPP( "tRegion", argc, argv ); + } + catch ( const char * err ) { + qCritical() << "Exception(char*):" << err; + } + catch ( const std::string & err ) { + qCritical() << "Exception(std::string &):" << err.c_str(); + } + catch ( const QString & err ) { + qCritical() << "Exception(QString &):" << err; + } + catch ( ... ) { + qCritical() << "Exception(unknown type)!"; + } + qFatal( "%s", "...caught in main()" ); + return - 1; +} // main diff --git a/carta/cpp/testCache/testCache.pro b/carta/cpp/testCache/testCache.pro new file mode 100644 index 00000000..3452554e --- /dev/null +++ b/carta/cpp/testCache/testCache.pro @@ -0,0 +1,31 @@ +! include(../common.pri) { + error( "Could not find the common.pri file!" ) +} + +QT += webkitwidgets network widgets xml + +HEADERS += + +SOURCES += \ + main.cpp + +RESOURCES = + +unix: LIBS += -L$$OUT_PWD/../core/ -lcore +unix: LIBS += -L$$OUT_PWD/../CartaLib/ -lCartaLib +DEPENDPATH += $$PROJECT_ROOT/core +DEPENDPATH += $$PROJECT_ROOT/CartaLib + +QMAKE_LFLAGS += '-Wl,-rpath,\'\$$ORIGIN/../CartaLib:\$$ORIGIN/../core\'' + +QWT_ROOT = $$absolute_path("../../../ThirdParty/qwt") +unix:macx { + QMAKE_LFLAGS += '-F$$QWT_ROOT/lib' + LIBS +=-framework qwt + PRE_TARGETDEPS += $$OUT_PWD/../core/libcore.dylib +} +else{ + QMAKE_LFLAGS += '-Wl,-rpath,\'$$QWT_ROOT/lib\'' + LIBS +=-L$$QWT_ROOT/lib -lqwt + PRE_TARGETDEPS += $$OUT_PWD/../core/libcore.so +}