From 4b910475231f5a8115bfd26b4e1beba52c51f850 Mon Sep 17 00:00:00 2001 From: Pavol Federl Date: Thu, 11 Aug 2016 13:47:41 -0600 Subject: [PATCH 01/43] added very basic persistant cache API --- carta/cpp/CartaLib/CartaLib.pro | 6 ++-- carta/cpp/CartaLib/IPCache.cpp | 6 ++++ carta/cpp/CartaLib/IPCache.h | 49 +++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 carta/cpp/CartaLib/IPCache.cpp create mode 100644 carta/cpp/CartaLib/IPCache.h diff --git a/carta/cpp/CartaLib/CartaLib.pro b/carta/cpp/CartaLib/CartaLib.pro index 645224cc..b8afa6a3 100755 --- a/carta/cpp/CartaLib/CartaLib.pro +++ b/carta/cpp/CartaLib/CartaLib.pro @@ -47,7 +47,8 @@ SOURCES += \ Algorithms/LineCombiner.cpp \ IImageRenderService.cpp \ IRemoteVGView.cpp \ - RegionInfo.cpp + RegionInfo.cpp \ + IPCache.cpp HEADERS += \ CartaLib.h\ @@ -99,7 +100,8 @@ HEADERS += \ IImageRenderService.h \ Hooks/GetImageRenderService.h \ IRemoteVGView.h \ - RegionInfo.h + RegionInfo.h \ + IPCache.h unix { target.path = /usr/lib diff --git a/carta/cpp/CartaLib/IPCache.cpp b/carta/cpp/CartaLib/IPCache.cpp new file mode 100644 index 00000000..07d1d75c --- /dev/null +++ b/carta/cpp/CartaLib/IPCache.cpp @@ -0,0 +1,6 @@ +/** + * + **/ + +#include "IPCache.h" + diff --git a/carta/cpp/CartaLib/IPCache.h b/carta/cpp/CartaLib/IPCache.h new file mode 100644 index 00000000..a12856cd --- /dev/null +++ b/carta/cpp/CartaLib/IPCache.h @@ -0,0 +1,49 @@ +/** + * + **/ + +#pragma once + +#include +#include +#include +#include + +namespace Carta +{ +namespace Lib +{ +class IPCache +{ +public: + + /// return maximum storage in bytes + virtual uint64_t + maxStorage() = 0; + + /// return used storage in bytes + virtual uint64_t + usedStorage() = 0; + + //// return number of entries + virtual uint64_t + nEntries() = 0; + + /// remove all entries + virtual void + deleteAll() = 0; + + /// read a value of an entry + /// if entry does not exist, val will be null + virtual void + readEntry( const QByteArray & key, + QByteArray & val ) = 0; + + /// set a value of an entry + virtual void + setEntry( const QByteArray & key, + const QByteArray & val, + int64_t priority ) = 0; +}; +} +} From e6afff8afb5de86a23ae5b9d31e6e79df4425b8b Mon Sep 17 00:00:00 2001 From: Pavol Federl Date: Tue, 23 Aug 2016 12:17:02 -0600 Subject: [PATCH 02/43] basic cache performance testing using leveldb --- carta/cpp/CartaLib/IPCache.h | 4 +- carta/cpp/cpp.pro | 5 +- carta/cpp/testCache/LevelDbIPCache.cpp | 62 ++++++ carta/cpp/testCache/LevelDbIPCache.h | 50 +++++ carta/cpp/testCache/main.cpp | 275 +++++++++++++++++++++++++ carta/cpp/testCache/testCache.pro | 39 ++++ 6 files changed, 432 insertions(+), 3 deletions(-) create mode 100644 carta/cpp/testCache/LevelDbIPCache.cpp create mode 100644 carta/cpp/testCache/LevelDbIPCache.h create mode 100644 carta/cpp/testCache/main.cpp create mode 100644 carta/cpp/testCache/testCache.pro diff --git a/carta/cpp/CartaLib/IPCache.h b/carta/cpp/CartaLib/IPCache.h index a12856cd..37d34807 100644 --- a/carta/cpp/CartaLib/IPCache.h +++ b/carta/cpp/CartaLib/IPCache.h @@ -34,8 +34,8 @@ class IPCache deleteAll() = 0; /// read a value of an entry - /// if entry does not exist, val will be null - virtual void + /// if entry does not exist, false is returned + virtual bool readEntry( const QByteArray & key, QByteArray & val ) = 0; diff --git a/carta/cpp/cpp.pro b/carta/cpp/cpp.pro index 356ba6bc..a6b7507b 100644 --- a/carta/cpp/cpp.pro +++ b/carta/cpp/cpp.pro @@ -8,7 +8,8 @@ SUBDIRS = \ core \ desktop \ plugins \ - Tests + Tests \ + testCache isEmpty(NOSERVER) { SUBDIRS +=server @@ -19,6 +20,8 @@ core.depends = CartaLib desktop.depends = core server.depends = core plugins.depends = core +testRegion.depends = core +testCache.depends = core isEmpty(NOSERVER) { Tests.depends = core desktop server plugins } diff --git a/carta/cpp/testCache/LevelDbIPCache.cpp b/carta/cpp/testCache/LevelDbIPCache.cpp new file mode 100644 index 00000000..ff43e19e --- /dev/null +++ b/carta/cpp/testCache/LevelDbIPCache.cpp @@ -0,0 +1,62 @@ +/** + * + **/ + +#include "LevelDbIPCache.h" +#include + +namespace tCache +{ +LevelDbIPCache::LevelDbIPCache() +{ + p_readOptions = leveldb::ReadOptions(); + p_writeOptions = leveldb::WriteOptions(); + p_writeOptions.sync = false; + + + // Set up database connection information and open database + leveldb::DB * db; + leveldb::Options options; +// qDebug() << "write buffer size=" << options.write_buffer_size; +// options.write_buffer_size *= 256; + options.create_if_missing = true; + + leveldb::Status status = leveldb::DB::Open( options, "/scratch/testLevelDb.leveldb", & db ); + + if ( false == status.ok() ) { + qDebug() << "Unable to open/create test database './testdb'" << endl; + qDebug() << status.ToString().c_str() << endl; + return; + } + + p_db.reset( db ); +} + +bool +LevelDbIPCache::readEntry( const QByteArray & key, QByteArray & val ) +{ + if ( ! p_db ) { + return false; + } + std::string tmpVal; + auto status = p_db-> Get( p_readOptions, key.constData(), & tmpVal ); + if ( ! status.ok() ) { + return false; + } + val = QByteArray( tmpVal.data(), tmpVal.size()); +// val.setRawData( tmpVal.data(), tmpVal.size()); + return true; +} + +void +LevelDbIPCache::setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) +{ + if( ! p_db) return; + p_db-> Put( p_writeOptions, + leveldb::Slice( key.constData(), key.size()), + leveldb::Slice( val.constData(), val.size())); + + Q_UNUSED( priority); + /// \todo we'll have to embed priority into the value, e.g. first 8 bytes? +} +} diff --git a/carta/cpp/testCache/LevelDbIPCache.h b/carta/cpp/testCache/LevelDbIPCache.h new file mode 100644 index 00000000..7940a14e --- /dev/null +++ b/carta/cpp/testCache/LevelDbIPCache.h @@ -0,0 +1,50 @@ +/** + * + **/ + +#pragma once + +#include "CartaLib/IPCache.h" +#include "leveldb/db.h" + +namespace tCache +{ + + +class LevelDbIPCache : public Carta::Lib::IPCache +{ +public: + +public: + + LevelDbIPCache(); + virtual uint64_t maxStorage() override + { + return 1; + } + virtual uint64_t usedStorage() override + { + return 1; + } + virtual uint64_t nEntries() override + { + return 1; + } + virtual void deleteAll() override + { + // not implemented + } + virtual bool readEntry(const QByteArray & key, QByteArray & val) override; + virtual void setEntry(const QByteArray & key, const QByteArray & val, int64_t priority) override; + +private: + + std::unique_ptr< leveldb::DB > p_db; + leveldb::ReadOptions p_readOptions; + leveldb::WriteOptions p_writeOptions; + +}; + + +} + diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp new file mode 100644 index 00000000..b452b964 --- /dev/null +++ b/carta/cpp/testCache/main.cpp @@ -0,0 +1,275 @@ +//#include "core/Viewer.h" +#include "core/MyQApp.h" +#include "core/CmdLine.h" +#include "core/MainConfig.h" +#include "core/Globals.h" +#include "CartaLib/Hooks/LoadAstroImage.h" +#include "CartaLib/Hooks/Initialize.h" +#include "LevelDbIPCache.h" +#include +#include + +namespace tCache +{ +std::unique_ptr < Carta::Lib::IPCache > pcache; +int imWidth = 20; +int imHeight = 8000; +int imDepth = 16000; + +double +generateData( int x, int y, int z ) +{ + return ( x + 0.1 ) * ( y - 0.1 ) / ( z * x + 0.1 ); +} + +static std::vector < double > +genProfile( int x, int y ) +{ + std::vector < double > arr( imDepth ); + for ( int z = 0 ; z < imDepth ; z++ ) { + arr[z] = generateData( x, y, z ); + } + return arr; +} + +QByteArray vd2qb( const std::vector & vd) { + QByteArray ba; + for( const double & d : vd) { + ba.append( (const char *)( & d), sizeof( double)); + } + return ba; +} + +std::vector qb2vd( const QByteArray & ba) { + std::vector vd; + if( ba.size() % sizeof(double) != 0) { + return vd; + } + const char * cptr = ba.constData(); + for( int i = 0 ; i < ba.size() ; i += sizeof(double)) { + vd.push_back( * ((const double *) (cptr + i))); + } + return vd; +} + +static std::vector < double > +readProfile( int x, int y) { + + QString key = QString( "%1/%2").arg(x).arg(y); + QByteArray val; + if( ! pcache-> readEntry( key.toUtf8(), val)) + { + return {}; + } + if( val.size() != int(sizeof(double) * imDepth)) { + qWarning() << "Cache size mismatch"; + return {}; + } +// std::vector res(imWidth); +// const double * dptr = reinterpret_cast (val.constData()); +// for( int z = 0 ; z < imDepth ; z ++ ) { +// res[z] = * dptr; +// dptr ++; +// } + + return qb2vd( val); +} + +static void +populateCache() +{ + // figure out which row we have already finished generating + + int startx = 0; + + QByteArray buff; + if ( pcache-> readEntry( "lastx", buff ) ) { + QString str = buff; + bool ok; + int x = str.toInt( & ok ); + if ( ok ) { + startx = x; + } + } + + qDebug() << "Starting from x" << startx; + for ( int x = startx ; x < imWidth ; x++ ) { + qDebug() << "Writing x" << x; + for ( int y = 0 ; y < imHeight ; y++ ) { + std::vector < double > arr = genProfile( x, y ); + + // write the entry + QString keyString = QString( "%1/%2" ).arg( x ).arg( y ); + +// QByteArray ba = QByteArray::fromRawData( +// reinterpret_cast < char * > ( & arr[0] ), +// sizeof( double ) * imDepth ); +// pcache-> setEntry( +// keyString.toUtf8(), ba, 0 ); + + pcache-> setEntry( keyString.toUtf8(), vd2qb(arr), 0); + } + pcache-> setEntry( "lastx", QString::number( x + 1 ).toUtf8(), 0 ); + } +} // populateCache + +static void +readCache() +{ + QTime t; + t.restart(); + qDebug() << "Sequential read..."; + for ( int x = 0 ; x < imWidth ; x++ ) { + qDebug() << "Testing x" << x; + for ( int y = 0 ; y < imHeight ; y++ ) { + std::vector < double > arr = genProfile( x, y ); + QString keyString = QString( "%1/%2" ).arg( x ).arg( y ); + + QByteArray ba; + if( ! pcache-> readEntry( keyString.toUtf8(), ba)) { + qCritical() << "Failed to read" << x << y; + } + std::vector< double> arr2 = qb2vd(ba); + if( arr != arr2) { + qCritical() << "Failed to match" << x << y; + + } + + /* + + QByteArray ba = QByteArray::fromRawData( + reinterpret_cast < char * > ( & arr[0] ), + sizeof( double ) * imDepth ); + + QString keyString = QString( "%1/%2" ).arg( x ).arg( y ); + QByteArray ba2; + if( ! pcache-> readEntry( keyString.toUtf8(), ba2)) { + qCritical() << "Failed to read" << x << y; + } + if( ba != ba2) { + qCritical() << "Failed to match" << x << y << ba.size() << ba2.size(); + qDebug() << * (const double *) (ba.constData()); + qDebug() << * (const double *) (ba2.constData()); + } + */ + } + qDebug() << " " << (x+1) * imHeight * 1000.0 / t.elapsed() << "pix/s"; + } + +} + +static void +testCache() +{ + pcache.reset( + new LevelDbIPCache() ); + + // populate cache + populateCache(); + + // test cache by reading it + readCache(); + + QByteArray key = "hello"; + QByteArray val; + if ( pcache-> readEntry( key, val ) ) { + qDebug() << "db already had value" << val; + } + else { + qDebug() << "db did not have value"; + } + pcache-> setEntry( key, "hola", 0 ); +} // testCache + +static int +coreMainCPP( QString platformString, int argc, char * * argv ) +{ + // + // initialize Qt + // + // don't require a window manager even though we're a QGuiApplication +// qputenv("QT_QPA_PLATFORM", QByteArrayLiteral("minimal")); + + MyQApp qapp( argc, argv ); + + QString appName = "carta-" + platformString; +#ifndef QT_NO_DEBUG_OUTPUT + appName += "-verbose"; +#endif + if ( CARTA_RUNTIME_CHECKS ) { + appName += "-runtimeChecks"; + } + MyQApp::setApplicationName( appName ); + + qDebug() << "Starting" << qapp.applicationName() << qapp.applicationVersion(); + + // alias globals + auto & globals = * Globals::instance(); + + // parse command line arguments & environment variables + // ==================================================== + auto cmdLineInfo = CmdLine::parse( MyQApp::arguments() ); + globals.setCmdLineInfo( & cmdLineInfo ); + + if ( cmdLineInfo.fileList().size() < 2 ) { + qFatal( "Need 2 files" ); + } + + // load the config file + // ==================== + QString configFilePath = cmdLineInfo.configFilePath(); + MainConfig::ParsedInfo mainConfig = MainConfig::parse( configFilePath ); + globals.setMainConfig( & mainConfig ); + qDebug() << "plugin directories:\n - " + mainConfig.pluginDirectories().join( "\n - " ); + + // initialize plugin manager + // ========================= + globals.setPluginManager( std::make_shared < PluginManager > () ); + auto pm = globals.pluginManager(); + + // tell plugin manager where to find plugins + pm-> setPluginSearchPaths( globals.mainConfig()->pluginDirectories() ); + + // find and load plugins + pm-> loadPlugins(); + + qDebug() << "Loading plugins..."; + auto infoList = pm-> getInfoList(); + qDebug() << "List of loaded plugins: [" << infoList.size() << "]"; + for ( const auto & entry : infoList ) { + qDebug() << " path:" << entry.json.name; + } + + testCache(); + + // give QT control +// int res = qapp.exec(); + int res = 0; + + // if we get here, it means we are quitting... + qDebug() << "Exiting"; + return res; +} // coreMainCPP +} + +int +main( int argc, char * * argv ) +{ + try { + return tCache::coreMainCPP( "tRegion", argc, argv ); + } + catch ( const char * err ) { + qCritical() << "Exception(char*):" << err; + } + catch ( const std::string & err ) { + qCritical() << "Exception(std::string &):" << err.c_str(); + } + catch ( const QString & err ) { + qCritical() << "Exception(QString &):" << err; + } + catch ( ... ) { + qCritical() << "Exception(unknown type)!"; + } + qFatal( "%s", "...caught in main()" ); + return - 1; +} // main diff --git a/carta/cpp/testCache/testCache.pro b/carta/cpp/testCache/testCache.pro new file mode 100644 index 00000000..025950b6 --- /dev/null +++ b/carta/cpp/testCache/testCache.pro @@ -0,0 +1,39 @@ +! include(../common.pri) { + error( "Could not find the common.pri file!" ) +} + +QT += webkitwidgets network widgets xml + +HEADERS += \ + LevelDbIPCache.h + +SOURCES += \ + main.cpp \ + LevelDbIPCache.cpp + +RESOURCES = + +unix: LIBS += -L$$OUT_PWD/../core/ -lcore +unix: LIBS += -L$$OUT_PWD/../CartaLib/ -lCartaLib +DEPENDPATH += $$PROJECT_ROOT/core +DEPENDPATH += $$PROJECT_ROOT/CartaLib + +QMAKE_LFLAGS += '-Wl,-rpath,\'\$$ORIGIN/../CartaLib:\$$ORIGIN/../core\'' + +QWT_ROOT = $$absolute_path("../../../ThirdParty/qwt") +unix:macx { + QMAKE_LFLAGS += '-F$$QWT_ROOT/lib' + LIBS +=-framework qwt + PRE_TARGETDEPS += $$OUT_PWD/../core/libcore.dylib +} +else{ + QMAKE_LFLAGS += '-Wl,-rpath,\'$$QWT_ROOT/lib\'' + LIBS +=-L$$QWT_ROOT/lib -lqwt + PRE_TARGETDEPS += $$OUT_PWD/../core/libcore.so +} + + +LEVELDBDIR=/home/pfederl/Downloads/build/leveldb-1.19 +#unix: LIBS += -L$$LEVELDBDIR/out-shared -lleveldb +unix: LIBS += -L$$LEVELDBDIR/out-static -lleveldb +INCLUDEPATH += $$LEVELDBDIR/include From 7f926ff17dc9a87728f9db61a063f5a3b5fb94db Mon Sep 17 00:00:00 2001 From: Pavol Federl Date: Sun, 6 Nov 2016 20:06:10 -0700 Subject: [PATCH 03/43] Added persistent cache plugin using Qt's sqlite. --- carta/cpp/CartaLib/CartaLib.pro | 15 +- .../cpp/CartaLib/Hooks/GetPersistantCache.cpp | 14 ++ carta/cpp/CartaLib/Hooks/GetPersistantCache.h | 57 +++++ carta/cpp/CartaLib/Hooks/HookIDs.h | 2 +- carta/cpp/CartaLib/IPCache.h | 7 +- carta/cpp/CartaLib/IPlugin.h | 3 + carta/cpp/core/PluginManager.cpp | 5 + .../plugins/PCacheSqlite3/PCacheSqlite3.cpp | 196 ++++++++++++++++++ .../cpp/plugins/PCacheSqlite3/PCacheSqlite3.h | 31 +++ .../plugins/PCacheSqlite3/PCacheSqlite3.pro | 32 +++ carta/cpp/plugins/PCacheSqlite3/plugin.json | 9 + carta/cpp/plugins/plugins.pro | 2 + carta/cpp/testCache/LevelDbIPCache.cpp | 2 +- carta/cpp/testCache/SqLitePCache.cpp | 65 ++++++ carta/cpp/testCache/SqLitePCache.h | 51 +++++ carta/cpp/testCache/main.cpp | 107 ++++++---- carta/cpp/testCache/testCache.pro | 8 +- 17 files changed, 557 insertions(+), 49 deletions(-) create mode 100644 carta/cpp/CartaLib/Hooks/GetPersistantCache.cpp create mode 100644 carta/cpp/CartaLib/Hooks/GetPersistantCache.h create mode 100644 carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp create mode 100644 carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h create mode 100644 carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.pro create mode 100644 carta/cpp/plugins/PCacheSqlite3/plugin.json create mode 100644 carta/cpp/testCache/SqLitePCache.cpp create mode 100644 carta/cpp/testCache/SqLitePCache.h diff --git a/carta/cpp/CartaLib/CartaLib.pro b/carta/cpp/CartaLib/CartaLib.pro index b8afa6a3..ff28cb9a 100755 --- a/carta/cpp/CartaLib/CartaLib.pro +++ b/carta/cpp/CartaLib/CartaLib.pro @@ -47,8 +47,8 @@ SOURCES += \ Algorithms/LineCombiner.cpp \ IImageRenderService.cpp \ IRemoteVGView.cpp \ - RegionInfo.cpp \ - IPCache.cpp + IPCache.cpp \ + Hooks/GetPersistantCache.cpp HEADERS += \ CartaLib.h\ @@ -100,8 +100,19 @@ HEADERS += \ IImageRenderService.h \ Hooks/GetImageRenderService.h \ IRemoteVGView.h \ +<<<<<<< e6afff8afb5de86a23ae5b9d31e6e79df4425b8b RegionInfo.h \ IPCache.h +======= + Hooks/GetProfileExtractor.h \ + Regions/IRegion.h \ + InputEvents.h \ + Regions/ICoordSystem.h \ + Hooks/CoordSystemHook.h \ + Regions/CoordinateSystemFormatter.h \ + IPCache.h \ + Hooks/GetPersistantCache.h +>>>>>>> Added persistent cache plugin using Qt's sqlite. unix { target.path = /usr/lib diff --git a/carta/cpp/CartaLib/Hooks/GetPersistantCache.cpp b/carta/cpp/CartaLib/Hooks/GetPersistantCache.cpp new file mode 100644 index 00000000..440270bf --- /dev/null +++ b/carta/cpp/CartaLib/Hooks/GetPersistantCache.cpp @@ -0,0 +1,14 @@ +/** + * + **/ + +#include "GetPersistantCache.h" + +namespace Carta +{ +namespace Lib +{ +namespace Hooks +{ } +} +} diff --git a/carta/cpp/CartaLib/Hooks/GetPersistantCache.h b/carta/cpp/CartaLib/Hooks/GetPersistantCache.h new file mode 100644 index 00000000..cd6c90b1 --- /dev/null +++ b/carta/cpp/CartaLib/Hooks/GetPersistantCache.h @@ -0,0 +1,57 @@ +/** + * Defines a hook for obtaining a persistant cache object. + * + **/ + +#pragma once + +#include "CartaLib/CartaLib.h" +#include "CartaLib/IPlugin.h" +#include "CartaLib/IPCache.h" + +namespace Carta +{ +namespace Lib +{ +namespace Hooks +{ +/// \brief Hook for loading a plugin of an unknown type +/// +class GetPersistantCache : public BaseHook +{ + CARTA_HOOK_BOILER1( GetPersistantCache ); + +public: + + /// Result of the hook is an instance of IPCache object. + /// This hook should only be ever called once, since the persistent cache + /// should be treated as a singleton. The reason I am not implementing this + /// as a singleton is because in the future, we may addd a mechanism (see below) + /// to be able to create multiple independent caches. + typedef IPCache::SharedPtr ResultType; + + /// input parameters are: + /// nothing + /// All necessary parameters are retrieved from carta.config. + /// + /// In the future we could make this contain overrides of some sort, in case + /// some code would like to create additional independent caches... + struct Params { + Params( ) + { + } + }; + + /// standard constructor (could be probably a macro) + GetPersistantCache( Params * pptr ) : BaseHook( staticId ), paramsPtr( pptr ) + { + // force instantiation of templates + CARTA_ASSERT( is < Me > () ); + } + + ResultType result; + Params * paramsPtr = nullptr; +}; +} +} +} diff --git a/carta/cpp/CartaLib/Hooks/HookIDs.h b/carta/cpp/CartaLib/Hooks/HookIDs.h index 46cd7ad1..fce384f7 100644 --- a/carta/cpp/CartaLib/Hooks/HookIDs.h +++ b/carta/cpp/CartaLib/Hooks/HookIDs.h @@ -31,7 +31,7 @@ enum class UniqueHookIDs { ProfileHook_ID, Fit1DHook_ID, ImageStatisticsHook_ID, - + GetPersistantCache_ID, /// experimental, soon to be removed: PreRender_ID, diff --git a/carta/cpp/CartaLib/IPCache.h b/carta/cpp/CartaLib/IPCache.h index 37d34807..59d46173 100644 --- a/carta/cpp/CartaLib/IPCache.h +++ b/carta/cpp/CartaLib/IPCache.h @@ -1,9 +1,8 @@ -/** - * - **/ +/// IPCache is a set of APIs to access persistant cache. #pragma once +#include "CartaLib/CartaLib.h" #include #include #include @@ -15,6 +14,8 @@ namespace Lib { class IPCache { + CLASS_BOILERPLATE( IPCache ); + public: /// return maximum storage in bytes diff --git a/carta/cpp/CartaLib/IPlugin.h b/carta/cpp/CartaLib/IPlugin.h index 66e0dd23..41dfe757 100644 --- a/carta/cpp/CartaLib/IPlugin.h +++ b/carta/cpp/CartaLib/IPlugin.h @@ -104,6 +104,9 @@ class IPlugin /// called immediately after the plugin was loaded /// TODO: should be pure virtual, don't be lazy! + /// + /// This is different from the Initialize hook, which is delivered after + /// all plugins have been loaded and after core is started. virtual void initialize( const InitInfo & initInfo ) { diff --git a/carta/cpp/core/PluginManager.cpp b/carta/cpp/core/PluginManager.cpp index e4b13ba6..43310468 100644 --- a/carta/cpp/core/PluginManager.cpp +++ b/carta/cpp/core/PluginManager.cpp @@ -171,6 +171,11 @@ PluginManager::loadPlugins() initInfo.pluginPath = pInfo.dirPath; auto json = Globals::instance()-> mainConfig()-> json(); initInfo.json = json["plugins"].toObject()[pInfo.json.name].toObject(); + if( 0) { + QJsonDocument doc( initInfo.json); + qDebug() << " name:" << pInfo.json.name; + qDebug() << " json:" << doc.toJson(); + } pInfo.rawPlugin->initialize( initInfo ); // find out what hooks this plugin wants to listen to diff --git a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp new file mode 100644 index 00000000..1816a5b9 --- /dev/null +++ b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp @@ -0,0 +1,196 @@ +#include "PCacheSqlite3.h" +#include "CartaLib/Hooks/GetPersistantCache.h" +#include +#include +#include + +typedef Carta::Lib::Hooks::GetPersistantCache GetPersistantCacheHook; + +/// +/// Implementation of IPCache using sqlite +/// +class SqLitePCache : public Carta::Lib::IPCache +{ +public: + + virtual uint64_t + maxStorage() override + { + return 1; + } + + virtual uint64_t + usedStorage() override + { + return 1; + } + + virtual uint64_t + nEntries() override + { + return 1; + } + + virtual void + deleteAll() override + { + // not implemented + } + + virtual bool + readEntry( const QByteArray & key, QByteArray & val ) override + { + if ( ! m_db.isOpen() ) { + return false; + } + QSqlQuery query( m_db ); + query.prepare( "SELECT val FROM db WHERE key = :key" ); + query.bindValue( ":key", key ); + + // qDebug() << "select query" << query.lastQuery(); + if ( ! query.exec() ) { + qWarning() << "query read failed"; + return false; + } + if ( query.next() ) { + val = query.value( 0 ).toByteArray(); + return true; + } + return false; + } // readEntry + + virtual void + setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) override + { + if ( ! m_db.isOpen() ) { + return; + } + + Q_UNUSED( priority ); + QSqlQuery query( m_db ); + query.prepare( "INSERT OR REPLACE INTO db (key, val) VALUES ( :key, :val)" ); + + // query.prepare( "INSERT INTO db (key, val) VALUES ( :key, :val)" ); + query.bindValue( ":key", key ); + query.bindValue( ":val", val ); + + // qDebug() << "insert query" << query.lastQuery(); + if ( ! query.exec() ) { + qWarning() << "query insert failed" + << query.lastError().text(); + } + } // setEntry + + static + Carta::Lib::IPCache::SharedPtr + getCacheSingleton( QString dirPath) + { + if ( m_cachePtr ) { + qCritical() << "PCacheSQlite3Plugin::Calling GetPersistantCacheHook multiple times!!!"; + } + else { + // stupid c++ won't allow this + // m_cachePtr = std::make_shared < SqLitePCache > (); + m_cachePtr.reset( new SqLitePCache( dirPath) ); + } + return m_cachePtr; + } + + ~SqLitePCache() + { + qDebug() << "Destroying SqLitePCache"; + qDebug() << "Possible segmentation fault below, it's a bug and we'll try to fix it."; + m_db.close(); + qDebug() << "Ok, no segmentation fault occurred, whew :)"; + } + +private: + + SqLitePCache( QString dirPath) + { + m_db = QSqlDatabase::addDatabase( "QSQLITE" ); + + QString fname = dirPath + "/pcache.sqlite"; + + // m_db.setDatabaseName( "/scratchSD/test.sqlite" ); + m_db.setDatabaseName( fname ); + bool ok = m_db.open(); + if ( ! ok ) { + qCritical() << "Could not open sqlite database"; + qCritical() << " - at location:" + fname; + } + + QSqlQuery query( m_db ); + + query.prepare( "create table db (key text primary key, " + "val blob)" ); + + // qDebug() << "create query" << query.lastQuery(); + if ( ! query.exec() ) { + qCritical() << "query create table failed" + << query.lastError().text(); + } + } + +private: + + QSqlDatabase m_db; + static Carta::Lib::IPCache::SharedPtr m_cachePtr; // = nullptr; +}; + +Carta::Lib::IPCache::SharedPtr SqLitePCache::m_cachePtr = nullptr; + +PCacheSQlite3Plugin::PCacheSQlite3Plugin( QObject * parent ) : + QObject( parent ) +{ } + +bool +PCacheSQlite3Plugin::handleHook( BaseHook & hookData ) +{ + // we only handle one hook: get the cache object + if ( hookData.is < GetPersistantCacheHook > () ) { + // decode hook data + GetPersistantCacheHook & hook = static_cast < GetPersistantCacheHook & > ( hookData ); + + // if no dbdir was specified, refuse to work :) + if( m_dbDir.isNull()) { + hook.result.reset(); + return false; + } + + // try to create the database + hook.result = SqLitePCache::getCacheSingleton( m_dbDir); + + // return true if result is not null + return hook.result != nullptr; + } + + qWarning() << "PCacheSQlite3Plugin: Sorrry, dont' know how to handle this hook"; + return false; +} // handleHook + +void +PCacheSQlite3Plugin::initialize( const IPlugin::InitInfo & initInfo ) +{ + qDebug() << "PCacheSQlite3Plugin::initialized"; + QJsonDocument doc( initInfo.json ); + qDebug() << doc.toJson(); + + // extract the location of the database from carta.config + m_dbDir = initInfo.json.value( "dbDir").toString(); + if( m_dbDir.isNull()) { + qCritical() << "No dbDir specified for PCacheSqlite3 plugin!!!"; + } + else { + // convert this to absolute path just in case + m_dbDir = QDir(m_dbDir).absolutePath(); + } +} + +std::vector < HookId > +PCacheSQlite3Plugin::getInitialHookList() +{ + return { + GetPersistantCacheHook::staticId + }; +} diff --git a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h new file mode 100644 index 00000000..be099ec4 --- /dev/null +++ b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h @@ -0,0 +1,31 @@ +/// Implements plugin for persistant cache, using QT's sqlite3 database driver + +/// This plugin can read image formats that Qt supports. + +#pragma once + +#include "CartaLib/IPlugin.h" +#include +#include + +class PCacheSQlite3Plugin : public QObject, public IPlugin +{ + Q_OBJECT + Q_PLUGIN_METADATA( IID "org.cartaviewer.IPlugin" ) + Q_INTERFACES( IPlugin ) + +public : + PCacheSQlite3Plugin( QObject * parent = 0 ); + virtual bool + handleHook( BaseHook & hookData ) override; + + virtual std::vector < HookId > + getInitialHookList() override; + + virtual void + initialize( const InitInfo & initInfo ) override; + +private: + + QString m_dbDir; +}; diff --git a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.pro b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.pro new file mode 100644 index 00000000..821bcf8c --- /dev/null +++ b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.pro @@ -0,0 +1,32 @@ +! include(../../common.pri) { + error( "Could not find the common.pri file!" ) +} + +INCLUDEPATH += $$PROJECT_ROOT +DEPENDPATH += $$PROJECT_ROOT + +QT += core gui sql + +TARGET = plugin +TEMPLATE = lib +CONFIG += plugin + +SOURCES += \ + PCacheSqlite3.cpp + +HEADERS += \ + PCacheSqlite3.h + +OTHER_FILES += \ + plugin.json + +# copy json to build directory +#MYFILES = $$files($${PWD}/files/*.*) +MYFILES = plugin.json +copy_files.name = copy large files +copy_files.input = MYFILES +# change datafiles to a directory you want to put the files to +copy_files.output = $${OUT_PWD}/${QMAKE_FILE_BASE}${QMAKE_FILE_EXT} +copy_files.commands = ${COPY_FILE} ${QMAKE_FILE_IN} ${QMAKE_FILE_OUT} +copy_files.CONFIG += no_link target_predeps +QMAKE_EXTRA_COMPILERS += copy_files diff --git a/carta/cpp/plugins/PCacheSqlite3/plugin.json b/carta/cpp/plugins/PCacheSqlite3/plugin.json new file mode 100644 index 00000000..32585247 --- /dev/null +++ b/carta/cpp/plugins/PCacheSqlite3/plugin.json @@ -0,0 +1,9 @@ +{ + "api" : "1", + "name" : "PCacheSqlite3", + "version" : "1", + "type" : "C++", + "description": "Implements persistant cache using QT's sqlite3 driver.", + "about" : "Part of CARTA. Written by Pavol", + "depends" : [ ] +} diff --git a/carta/cpp/plugins/plugins.pro b/carta/cpp/plugins/plugins.pro index a51d3211..654e0227 100644 --- a/carta/cpp/plugins/plugins.pro +++ b/carta/cpp/plugins/plugins.pro @@ -25,6 +25,8 @@ SUBDIRS += ColormapsPy SUBDIRS += CyberSKA SUBDIRS += DevIntegration +SUBDIRS += PCacheSqlite3 + # adrianna's render plugin SUBDIRS += hpcImgRender diff --git a/carta/cpp/testCache/LevelDbIPCache.cpp b/carta/cpp/testCache/LevelDbIPCache.cpp index ff43e19e..c1ef3dd0 100644 --- a/carta/cpp/testCache/LevelDbIPCache.cpp +++ b/carta/cpp/testCache/LevelDbIPCache.cpp @@ -21,7 +21,7 @@ LevelDbIPCache::LevelDbIPCache() // options.write_buffer_size *= 256; options.create_if_missing = true; - leveldb::Status status = leveldb::DB::Open( options, "/scratch/testLevelDb.leveldb", & db ); + leveldb::Status status = leveldb::DB::Open( options, "/scratchSD/testLevelDb.leveldb", & db ); if ( false == status.ok() ) { qDebug() << "Unable to open/create test database './testdb'" << endl; diff --git a/carta/cpp/testCache/SqLitePCache.cpp b/carta/cpp/testCache/SqLitePCache.cpp new file mode 100644 index 00000000..f0c84b15 --- /dev/null +++ b/carta/cpp/testCache/SqLitePCache.cpp @@ -0,0 +1,65 @@ +/** + * + **/ + +#include "SqLitePCache.h" +#include +#include + +namespace tCache +{ +SqLitePCache::SqLitePCache() +{ + m_db = QSqlDatabase::addDatabase( "QSQLITE" ); +// m_db.setDatabaseName( "/scratchSD/test.sqlite" ); + m_db.setDatabaseName( "/scratch/test.sqlite" ); + bool ok = m_db.open(); + if ( ! ok ) { + qCritical() << "Could not open sqlite database"; + } + + QSqlQuery query(m_db); + + query.prepare( "create table db (key text primary key, " + "val blob)" ); +// qDebug() << "create query" << query.lastQuery(); + if( ! query.exec()) { + qCritical() << "query create table failed" + << query.lastError().text(); + } +} + +bool +SqLitePCache::readEntry( const QByteArray & key, QByteArray & val ) +{ + QSqlQuery query(m_db); + query.prepare( "SELECT val FROM db WHERE key = :key" ); + query.bindValue( ":key", key ); +// qDebug() << "select query" << query.lastQuery(); + if ( ! query.exec() ) { + qWarning() << "query read failed"; + return false; + } + if ( query.next() ) { + val = query.value( 0 ).toByteArray(); + return true; + } + return false; +} + +void +SqLitePCache::setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) +{ + QSqlQuery query(m_db); + query.prepare( "INSERT OR REPLACE INTO db (key, val) VALUES ( :key, :val)" ); +// query.prepare( "INSERT INTO db (key, val) VALUES ( :key, :val)" ); + query.bindValue( ":key", key ); + query.bindValue( ":val", val ); +// qDebug() << "insert query" << query.lastQuery(); + if( ! query.exec()) { + qWarning() << "query insert failed" + << query.lastError().text(); + + } +} +} diff --git a/carta/cpp/testCache/SqLitePCache.h b/carta/cpp/testCache/SqLitePCache.h new file mode 100644 index 00000000..0658b469 --- /dev/null +++ b/carta/cpp/testCache/SqLitePCache.h @@ -0,0 +1,51 @@ +/** + * + **/ + +#pragma once + +#include "CartaLib/IPCache.h" + +#include +namespace tCache +{ +class SqLitePCache : public Carta::Lib::IPCache +{ +public: + + SqLitePCache(); + + virtual uint64_t + maxStorage() override + { + return 1; + } + + virtual uint64_t + usedStorage() override + { + return 1; + } + + virtual uint64_t + nEntries() override + { + return 1; + } + + virtual void + deleteAll() override + { + // not implemented + } + + virtual bool + readEntry( const QByteArray & key, QByteArray & val ) override; + + virtual void + setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) override; + +private: + QSqlDatabase m_db; +}; +} diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index b452b964..20d21b6b 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -1,19 +1,25 @@ //#include "core/Viewer.h" +#include "CartaLib/Hooks/Initialize.h" +#include "CartaLib/Hooks/GetPersistantCache.h" #include "core/MyQApp.h" #include "core/CmdLine.h" #include "core/MainConfig.h" #include "core/Globals.h" -#include "CartaLib/Hooks/LoadAstroImage.h" -#include "CartaLib/Hooks/Initialize.h" #include "LevelDbIPCache.h" +#include "SqLitePCache.h" #include #include namespace tCache { -std::unique_ptr < Carta::Lib::IPCache > pcache; +std::shared_ptr < Carta::Lib::IPCache > pcache; + +//int imWidth = 20; +//int imHeight = 8000; +//int imDepth = 16000; + int imWidth = 20; -int imHeight = 8000; +int imHeight = 20; int imDepth = 16000; double @@ -32,39 +38,43 @@ genProfile( int x, int y ) return arr; } -QByteArray vd2qb( const std::vector & vd) { +QByteArray +vd2qb( const std::vector < double > & vd ) +{ QByteArray ba; - for( const double & d : vd) { - ba.append( (const char *)( & d), sizeof( double)); + for ( const double & d : vd ) { + ba.append( (const char *) ( & d ), sizeof( double ) ); } return ba; } -std::vector qb2vd( const QByteArray & ba) { - std::vector vd; - if( ba.size() % sizeof(double) != 0) { +std::vector < double > +qb2vd( const QByteArray & ba ) +{ + std::vector < double > vd; + if ( ba.size() % sizeof( double ) != 0 ) { return vd; } const char * cptr = ba.constData(); - for( int i = 0 ; i < ba.size() ; i += sizeof(double)) { - vd.push_back( * ((const double *) (cptr + i))); + for ( int i = 0 ; i < ba.size() ; i += sizeof( double ) ) { + vd.push_back( * ( (const double *) ( cptr + i ) ) ); } return vd; } static std::vector < double > -readProfile( int x, int y) { - - QString key = QString( "%1/%2").arg(x).arg(y); +readProfile( int x, int y ) +{ + QString key = QString( "%1/%2" ).arg( x ).arg( y ); QByteArray val; - if( ! pcache-> readEntry( key.toUtf8(), val)) - { - return {}; + if ( ! pcache-> readEntry( key.toUtf8(), val ) ) { + return { }; } - if( val.size() != int(sizeof(double) * imDepth)) { + if ( val.size() != int (sizeof( double ) * imDepth) ) { qWarning() << "Cache size mismatch"; - return {}; + return { }; } + // std::vector res(imWidth); // const double * dptr = reinterpret_cast (val.constData()); // for( int z = 0 ; z < imDepth ; z ++ ) { @@ -72,8 +82,8 @@ readProfile( int x, int y) { // dptr ++; // } - return qb2vd( val); -} + return qb2vd( val ); +} // readProfile static void populateCache() @@ -100,14 +110,7 @@ populateCache() // write the entry QString keyString = QString( "%1/%2" ).arg( x ).arg( y ); - -// QByteArray ba = QByteArray::fromRawData( -// reinterpret_cast < char * > ( & arr[0] ), -// sizeof( double ) * imDepth ); -// pcache-> setEntry( -// keyString.toUtf8(), ba, 0 ); - - pcache-> setEntry( keyString.toUtf8(), vd2qb(arr), 0); + pcache-> setEntry( keyString.toUtf8(), vd2qb( arr ), 0 ); } pcache-> setEntry( "lastx", QString::number( x + 1 ).toUtf8(), 0 ); } @@ -126,13 +129,13 @@ readCache() QString keyString = QString( "%1/%2" ).arg( x ).arg( y ); QByteArray ba; - if( ! pcache-> readEntry( keyString.toUtf8(), ba)) { + if ( ! pcache-> readEntry( keyString.toUtf8(), ba ) ) { qCritical() << "Failed to read" << x << y; + continue; } - std::vector< double> arr2 = qb2vd(ba); - if( arr != arr2) { + std::vector < double > arr2 = qb2vd( ba ); + if ( arr != arr2 ) { qCritical() << "Failed to match" << x << y; - } /* @@ -153,16 +156,29 @@ readCache() } */ } - qDebug() << " " << (x+1) * imHeight * 1000.0 / t.elapsed() << "pix/s"; + qDebug() << " " << ( x + 1 ) * imHeight * 1000.0 / t.elapsed() << "pix/s"; } - -} +} // readCache static void testCache() { - pcache.reset( - new LevelDbIPCache() ); +// pcache.reset( +// new LevelDbIPCache() ); + +// pcache.reset( +// new SqLitePCache() ); + + { + pcache-> setEntry( "hello", "world", 0 ); + QByteArray val; + if ( pcache-> readEntry( "hello", val ) ) { + qDebug() << "hello=" << val; + } + else { + qDebug() << "hello not found"; + } + } // populate cache populateCache(); @@ -240,6 +256,19 @@ coreMainCPP( QString platformString, int argc, char * * argv ) qDebug() << " path:" << entry.json.name; } + // send an initialize hook to all plugins, because some may rely on it + pm-> prepare < Carta::Lib::Hooks::Initialize > ().executeAll(); + + // let's get pcache object + auto pcacheRes = pm-> prepare< Carta::Lib::Hooks::GetPersistantCache >().first(); + if( pcacheRes.isNull()) { + qWarning() << "Could not initialize persistent cache."; + return -1; + } + else { + pcache = pcacheRes.val(); + } + testCache(); // give QT control diff --git a/carta/cpp/testCache/testCache.pro b/carta/cpp/testCache/testCache.pro index 025950b6..5a9629e0 100644 --- a/carta/cpp/testCache/testCache.pro +++ b/carta/cpp/testCache/testCache.pro @@ -2,14 +2,16 @@ error( "Could not find the common.pri file!" ) } -QT += webkitwidgets network widgets xml +QT += webkitwidgets network widgets xml sql HEADERS += \ - LevelDbIPCache.h + LevelDbIPCache.h \ + SqLitePCache.h SOURCES += \ main.cpp \ - LevelDbIPCache.cpp + LevelDbIPCache.cpp \ + SqLitePCache.cpp RESOURCES = From 0ab0ea2a583fdcd758c6d3a2585ac97cb9ef1178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Sun, 20 Nov 2016 13:08:09 +0200 Subject: [PATCH 04/43] basic usage of Pavol's sqlite cache; needs testing --- carta/cpp/core/Algorithms/cacheUtils.h | 119 +++++++++++++++++++++++ carta/cpp/core/Data/Image/DataSource.cpp | 78 ++++++++++++--- carta/cpp/core/Data/Image/DataSource.h | 5 +- 3 files changed, 187 insertions(+), 15 deletions(-) create mode 100644 carta/cpp/core/Algorithms/cacheUtils.h diff --git a/carta/cpp/core/Algorithms/cacheUtils.h b/carta/cpp/core/Algorithms/cacheUtils.h new file mode 100644 index 00000000..7eb8156c --- /dev/null +++ b/carta/cpp/core/Algorithms/cacheUtils.h @@ -0,0 +1,119 @@ +/** + * Functions for serializing and deserializing keys and values to be used with the disk cache + **/ + +#pragma once + +/** + * Int --> byte array + **/ +QByteArray i2qb( const int & d) { + QByteArray ba; + ba.append( (const char *)( & d), sizeof( int)); + return ba; +} + +/** + * Byte array --> int + **/ +int qb2i( const QByteArray & ba) { + if( ba.size() != sizeof(int)) { + throw std::runtime_error("Could not unpack QByteArray into int: size is incorrect."); + } + return * ((const int *) (ba.constData())); +} + +/** + * Double --> byte array + **/ +QByteArray d2qb( const double & d) { + QByteArray ba; + ba.append( (const char *)( & d), sizeof( double)); + return ba; +} + +/** + * Byte array --> double + **/ +double qb2d( const QByteArray & ba) { + if( ba.size() != sizeof(double)) { + throw std::runtime_error("Could not unpack QByteArray into double: size is incorrect."); + } + return * ((const double *) (ba.constData())); +} + +/** + * Vector of doubles --> byte array + **/ +QByteArray vd2qb( const std::vector & vd) { + QByteArray ba; + for( const double & d : vd) { + ba.append( (const char *)( & d), sizeof( double)); + } + return ba; +} + +/** + * Byte array --> vector of doubles + **/ +std::vector qb2vd( const QByteArray & ba) { + std::vector vd; + if( ba.size() % sizeof(double) != 0) { + throw std::runtime_error("Could not unpack QByteArray into std::vector: size is incorrect."); + } + const char * cptr = ba.constData(); + for( int i = 0 ; i < ba.size() ; i += sizeof(double)) { + vd.push_back( * ((const double *) (cptr + i))); + } + return vd; +} + +/** + * Vector of integers --> byte array + **/ +QByteArray vi2qb( const std::vector & vd) { + QByteArray ba; + for( const int & i : vi) { + ba.append( (const char *)( & i), sizeof( int)); + } + return ba; +} + +/** + * Byte array --> vector of integers + **/ +std::vector qb2vi( const QByteArray & ba) { + std::vector vi; + if( ba.size() % sizeof(int) != 0) { + throw std::runtime_error("Could not unpack QByteArray into std::vector: size is incorrect."); + } + const char * cptr = ba.constData(); + for( int i = 0 ; i < ba.size() ; i += sizeof(int)) { + vi.push_back( * ((const int *) (cptr + i))); + } + return vi; +} + +/** + * Pair of int, double --> byte array + **/ +QByteArray id2qb( const std::pair & id) { + QByteArray ba; + ba.append( (const char *)( & id.first), sizeof( int)); + ba.append( (const char *)( & id.second), sizeof( double)); + return ba; +} + + +/** + * Byte array --> pair of int, double + **/ +std::pair qb2id( const QByteArray & ba) { + if( ba.size() != (sizeof(double) + sizeof(int))) { + throw std::runtime_error("Could not unpack QByteArray into std::pair: size is incorrect."); + } + const char * cptr = ba.constData(); + int int_val( * ((const int *) (cptr))); + double double_val( * ((const double *) (cptr + sizeof(int)))); + return std::make_pair(int_val, double_val); +} diff --git a/carta/cpp/core/Data/Image/DataSource.cpp b/carta/cpp/core/Data/Image/DataSource.cpp index 74bfefc1..2387511e 100755 --- a/carta/cpp/core/Data/Image/DataSource.cpp +++ b/carta/cpp/core/Data/Image/DataSource.cpp @@ -11,6 +11,7 @@ #include "CartaLib/PixelPipeline/CustomizablePixelPipeline.h" #include "../../ImageRenderService.h" #include "../../Algorithms/quantileAlgorithms.h" +#include "../../Algorithms/cacheUtils.h" #include #include #include @@ -55,6 +56,19 @@ DataSource::DataSource() : m_pixelPipeline-> setColormap( std::make_shared < Carta::Core::GrayColormap > () ); m_pixelPipeline-> setMinMax( 0, 1 ); m_renderService-> setPixelPipeline( m_pixelPipeline, m_pixelPipeline-> cacheId()); + + // initialize disk cache + auto res = Globals::instance()-> pluginManager() + -> prepare < Carta::Lib::Hooks::GetPersistantCache > ().first(); + if ( res.isNull() || ! res.val() ) { + qWarning( "Could not find a disk cache plugin." ); + m_diskCache = nullptr; + // TODO: convert the existing in-memory cache to a default in-memory cache to use if no plugin is found. + // Do we want to have memory *and* disk cache? Check performance. + } + else { + m_diskCache = res.val(); + } } @@ -333,7 +347,7 @@ std::shared_ptr DataSource::_getRender return m_renderService; } - +// TODO: create another function which only looks for the intensity. Most calling functions don't need the location. std::vector > DataSource::_getIntensityCache( int frameLow, int frameHigh, const std::vector& percentiles ){ //See if it is in the cached percentiles first. @@ -342,11 +356,28 @@ std::vector > DataSource::_getIntensityCache( int frameLow //Find all the intensities we can in the cache. int foundCount = 0; for ( int i = 0; i < percentileCount; i++ ){ - std::pair val = m_cachedPercentiles.getIntensity( frameLow, frameHigh, percentiles[i]); - if ( val.first>= 0 ){ - intensities[i] = val; - foundCount++; + + // Look for the location first + QString locationKey = QString("%1/%2/%3/%4/location").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); + QByteArray locationVal; + bool locationInCache = m_diskCache.readEntry(locationKey.toUtf8(), locationVal); + + if (locationInCache) { + QString intensityKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); + QByteArray intensityVal; + bool intensityInCache = m_diskCache.readEntry(intensityKey.toUtf8(), intensityVal); + + if (intensityInCache) { + intensities[i] = std::make_pair(qb2i(locationVal), qb2d(intensityVal)); + foundCount++; + } } + + //std::pair val = m_cachedPercentiles.getIntensity( frameLow, frameHigh, percentiles[i]); + //if ( val.first>= 0 ){ + //intensities[i] = val; + //foundCount++; + //} } //Not all percentiles were in the cache. We are going to have to look some up. @@ -895,18 +926,37 @@ void DataSource::_setGamma( double gamma ){ void DataSource::_updateClips( std::shared_ptr& view, double minClipPercentile, double maxClipPercentile, const std::vector& frames ){ - std::vector mFrames = _fitFramesToImage( frames ); + std::vector mFrames = _fitFramesToImage( frames ); int quantileIndex = _getQuantileCacheIndex( mFrames ); std::vector clips = m_quantileCache[ quantileIndex].m_clips; if ( clips.size() < 2 || - m_quantileCache[quantileIndex].m_minPercentile != minClipPercentile || - m_quantileCache[quantileIndex].m_maxPercentile != maxClipPercentile ) { - Carta::Lib::NdArray::Double doubleView( view.get(), false ); - clips = Carta::Core::Algorithms::quantiles2pixels( - doubleView, { minClipPercentile, maxClipPercentile }); - m_quantileCache[quantileIndex].m_clips = clips; - m_quantileCache[quantileIndex].m_minPercentile = minClipPercentile; - m_quantileCache[quantileIndex].m_maxPercentile = maxClipPercentile; + m_quantileCache[quantileIndex].m_minPercentile != minClipPercentile || + m_quantileCache[quantileIndex].m_maxPercentile != maxClipPercentile ) { + + // TODO: check if these are the right frame values and percentile values + QString minClipKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frames[0]).arg(frames.back().c).arg(minClipPercentile); + QString maxClipKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frames[0]).arg(frames.back().c).arg(maxClipPercentile); + + QByteArray minClipVal; + QByteArray maxClipVal; + + bool minClipInCache = m_diskCache.readEntry(minClipKey.toUtf8(), minClipVal); + bool maxClipInCache = m_diskCache.readEntry(maxClipKey.toUtf8(), maxClipVal); + + if (minClipInCache && maxClipInCache) { + clips.clear(); + clips.push_back(qb2d(minClipVal)); + clips.push_back(qb2d(maxClipVal)); + } else { + Carta::Lib::NdArray::Double doubleView( view.get(), false ); + clips = Carta::Core::Algorithms::quantiles2pixels(doubleView, { minClipPercentile, maxClipPercentile }); + m_diskCache.setEntry( minClipKey.toUtf8(), d2qb(clips[0]), 0); + m_diskCache.setEntry( maxClipKey.toUtf8(), d2qb(clips[1]), 0); + } + + m_quantileCache[quantileIndex].m_clips = clips; + m_quantileCache[quantileIndex].m_minPercentile = minClipPercentile; + m_quantileCache[quantileIndex].m_maxPercentile = maxClipPercentile; } m_pixelPipeline-> setMinMax( clips[0], clips[1] ); m_renderService-> setPixelPipeline( m_pixelPipeline, m_pixelPipeline-> cacheId()); diff --git a/carta/cpp/core/Data/Image/DataSource.h b/carta/cpp/core/Data/Image/DataSource.h index c697db6e..22581c41 100755 --- a/carta/cpp/core/Data/Image/DataSource.h +++ b/carta/cpp/core/Data/Image/DataSource.h @@ -502,7 +502,10 @@ class DataSource : public QObject { ///pixel pipeline std::shared_ptr m_pixelPipeline; - + + // disk cache + std::shared_ptr m_diskCache; + //Indices of the display axes. int m_axisIndexX; int m_axisIndexY; From f776eeadff9ef15fd8720ee288f126211b3f2fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Sun, 20 Nov 2016 13:24:50 +0200 Subject: [PATCH 05/43] added forgotten insertion into disk cache; put back memory cache in _getIntensityCache; added debugging --- carta/cpp/core/Data/Image/DataSource.cpp | 59 +++++++++++++++++------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/carta/cpp/core/Data/Image/DataSource.cpp b/carta/cpp/core/Data/Image/DataSource.cpp index 2387511e..035b3a39 100755 --- a/carta/cpp/core/Data/Image/DataSource.cpp +++ b/carta/cpp/core/Data/Image/DataSource.cpp @@ -357,31 +357,43 @@ std::vector > DataSource::_getIntensityCache( int frameLow int foundCount = 0; for ( int i = 0; i < percentileCount; i++ ){ - // Look for the location first - QString locationKey = QString("%1/%2/%3/%4/location").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); - QByteArray locationVal; - bool locationInCache = m_diskCache.readEntry(locationKey.toUtf8(), locationVal); - - if (locationInCache) { - QString intensityKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); - QByteArray intensityVal; - bool intensityInCache = m_diskCache.readEntry(intensityKey.toUtf8(), intensityVal); + std::pair val = m_cachedPercentiles.getIntensity( frameLow, frameHigh, percentiles[i]); + if ( val.first>= 0 ){ + qDebug() << "++++++++ found location and intensity in memory cache"; + intensities[i] = val; + foundCount++; + } else { + // disk cache + // Look for the location first + QString locationKey = QString("%1/%2/%3/%4/location").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); + QByteArray locationVal; + bool locationInCache = m_diskCache.readEntry(locationKey.toUtf8(), locationVal); + + qDebug() << "++++++++ location key is" << locationKey; - if (intensityInCache) { - intensities[i] = std::make_pair(qb2i(locationVal), qb2d(intensityVal)); - foundCount++; + if (locationInCache) { + QString intensityKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); + QByteArray intensityVal; + bool intensityInCache = m_diskCache.readEntry(intensityKey.toUtf8(), intensityVal); + + qDebug() << "++++++++ intensity key is" << intensityKey; + + if (intensityInCache) { + qDebug() << "++++++++ found location and intensity in disk cache"; + intensities[i] = std::make_pair(qb2i(locationVal), qb2d(intensityVal)); + foundCount++; + // put them in the memory cache + m_cachedPercentiles.put( frameLow, frameHigh, intensities[i].first, percentiles[i], intensities[i].second ); + } } } - //std::pair val = m_cachedPercentiles.getIntensity( frameLow, frameHigh, percentiles[i]); - //if ( val.first>= 0 ){ - //intensities[i] = val; - //foundCount++; - //} + qDebug() << "++++++++ For percentile" << percentiles[i] << "intensity is" << intensities[i].second << "and location is" << intensities[i].first; } //Not all percentiles were in the cache. We are going to have to look some up. if ( foundCount < percentileCount ){ + qDebug() << "++++++++ Calculating some values"; std::vector > allValues; int spectralIndex = Util::getAxisIndex( m_image, AxisInfo::KnownType::SPECTRAL ); @@ -439,7 +451,14 @@ std::vector > DataSource::_getIntensityCache( int frameLow intensities[i].first += frameLow; } + // put calculated values in both the memory cache and the disk cache + m_cachedPercentiles.put( frameLow, frameHigh, intensities[i].first, percentiles[i], intensities[i].second ); + + m_diskCache.setEntry(locationKey.toUtf8(), i2qb(intensities[i].first), 0); + m_diskCache.setEntry(intensityKey.toUtf8(), d2qb(intensities[i].second), 0); + + qDebug() << "++++++++ For percentile" << percentiles[i] << "intensity is" << intensities[i].second << "and location is" << intensities[i].first; } } } @@ -937,6 +956,8 @@ void DataSource::_updateClips( std::shared_ptr Date: Sun, 20 Nov 2016 15:05:04 +0200 Subject: [PATCH 06/43] somehow this manual merge got screwed up --- carta/cpp/CartaLib/CartaLib.pro | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/carta/cpp/CartaLib/CartaLib.pro b/carta/cpp/CartaLib/CartaLib.pro index de0df7b4..5825d822 100755 --- a/carta/cpp/CartaLib/CartaLib.pro +++ b/carta/cpp/CartaLib/CartaLib.pro @@ -109,29 +109,18 @@ HEADERS += \ IImageRenderService.h \ Hooks/GetImageRenderService.h \ IRemoteVGView.h \ -<<<<<<< HEAD -<<<<<<< e6afff8afb5de86a23ae5b9d31e6e79df4425b8b RegionInfo.h \ - IPCache.h -======= -======= ->>>>>>> upstream/develop Hooks/GetProfileExtractor.h \ Regions/IRegion.h \ InputEvents.h \ Regions/ICoordSystem.h \ Hooks/CoordSystemHook.h \ Regions/CoordinateSystemFormatter.h \ -<<<<<<< HEAD - IPCache.h \ Hooks/GetPersistantCache.h ->>>>>>> Added persistent cache plugin using Qt's sqlite. -======= Regions/Ellipse.h \ Regions/Point.h \ Regions/Rectangle.h \ IPCache.h ->>>>>>> upstream/develop unix { target.path = /usr/lib From ca49962fa04ac7440f2934c9767c7f6e844db782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Sun, 20 Nov 2016 15:06:52 +0200 Subject: [PATCH 07/43] missing linebreak escapes --- carta/cpp/CartaLib/CartaLib.pro | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/carta/cpp/CartaLib/CartaLib.pro b/carta/cpp/CartaLib/CartaLib.pro index 5825d822..62dbc081 100755 --- a/carta/cpp/CartaLib/CartaLib.pro +++ b/carta/cpp/CartaLib/CartaLib.pro @@ -48,7 +48,7 @@ SOURCES += \ IImageRenderService.cpp \ IRemoteVGView.cpp \ IPCache.cpp \ - Hooks/GetPersistantCache.cpp + Hooks/GetPersistantCache.cpp \ Hooks/GetProfileExtractor.cpp \ Regions/IRegion.cpp \ InputEvents.cpp \ @@ -116,7 +116,7 @@ HEADERS += \ Regions/ICoordSystem.h \ Hooks/CoordSystemHook.h \ Regions/CoordinateSystemFormatter.h \ - Hooks/GetPersistantCache.h + Hooks/GetPersistantCache.h \ Regions/Ellipse.h \ Regions/Point.h \ Regions/Rectangle.h \ From 909821e854abcd4bba5a7309bb3ec4e027df8cd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Sun, 20 Nov 2016 15:49:30 +0200 Subject: [PATCH 08/43] missing declaration and header --- carta/cpp/core/Data/Image/DataSource.cpp | 1 + carta/cpp/core/Data/Image/DataSource.h | 1 + 2 files changed, 2 insertions(+) diff --git a/carta/cpp/core/Data/Image/DataSource.cpp b/carta/cpp/core/Data/Image/DataSource.cpp index 035b3a39..49f25607 100755 --- a/carta/cpp/core/Data/Image/DataSource.cpp +++ b/carta/cpp/core/Data/Image/DataSource.cpp @@ -9,6 +9,7 @@ #include "Data/Colormap/TransformsData.h" #include "CartaLib/Hooks/LoadAstroImage.h" #include "CartaLib/PixelPipeline/CustomizablePixelPipeline.h" +#include "CartaLib/IPCache.h" #include "../../ImageRenderService.h" #include "../../Algorithms/quantileAlgorithms.h" #include "../../Algorithms/cacheUtils.h" diff --git a/carta/cpp/core/Data/Image/DataSource.h b/carta/cpp/core/Data/Image/DataSource.h index 22581c41..948d33d3 100755 --- a/carta/cpp/core/Data/Image/DataSource.h +++ b/carta/cpp/core/Data/Image/DataSource.h @@ -26,6 +26,7 @@ class ImageInterface; namespace NdArray { class RawViewInterface; } +class IPCache; } From b01220e363f2904671718a522da762a08cbf32b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Sun, 20 Nov 2016 16:00:21 +0200 Subject: [PATCH 09/43] m_diskCache is a pointer --- carta/cpp/core/Data/Image/DataSource.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/carta/cpp/core/Data/Image/DataSource.cpp b/carta/cpp/core/Data/Image/DataSource.cpp index 49f25607..d238e582 100755 --- a/carta/cpp/core/Data/Image/DataSource.cpp +++ b/carta/cpp/core/Data/Image/DataSource.cpp @@ -9,7 +9,7 @@ #include "Data/Colormap/TransformsData.h" #include "CartaLib/Hooks/LoadAstroImage.h" #include "CartaLib/PixelPipeline/CustomizablePixelPipeline.h" -#include "CartaLib/IPCache.h" +#include "CartaLib/IPCacheff.h" #include "../../ImageRenderService.h" #include "../../Algorithms/quantileAlgorithms.h" #include "../../Algorithms/cacheUtils.h" @@ -368,14 +368,14 @@ std::vector > DataSource::_getIntensityCache( int frameLow // Look for the location first QString locationKey = QString("%1/%2/%3/%4/location").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); QByteArray locationVal; - bool locationInCache = m_diskCache.readEntry(locationKey.toUtf8(), locationVal); + bool locationInCache = m_diskCache->readEntry(locationKey.toUtf8(), locationVal); qDebug() << "++++++++ location key is" << locationKey; if (locationInCache) { QString intensityKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); QByteArray intensityVal; - bool intensityInCache = m_diskCache.readEntry(intensityKey.toUtf8(), intensityVal); + bool intensityInCache = m_diskCache->readEntry(intensityKey.toUtf8(), intensityVal); qDebug() << "++++++++ intensity key is" << intensityKey; @@ -456,8 +456,8 @@ std::vector > DataSource::_getIntensityCache( int frameLow m_cachedPercentiles.put( frameLow, frameHigh, intensities[i].first, percentiles[i], intensities[i].second ); - m_diskCache.setEntry(locationKey.toUtf8(), i2qb(intensities[i].first), 0); - m_diskCache.setEntry(intensityKey.toUtf8(), d2qb(intensities[i].second), 0); + m_diskCache->setEntry(locationKey.toUtf8(), i2qb(intensities[i].first), 0); + m_diskCache->setEntry(intensityKey.toUtf8(), d2qb(intensities[i].second), 0); qDebug() << "++++++++ For percentile" << percentiles[i] << "intensity is" << intensities[i].second << "and location is" << intensities[i].first; } @@ -962,8 +962,8 @@ void DataSource::_updateClips( std::shared_ptrreadEntry(minClipKey.toUtf8(), minClipVal); + bool maxClipInCache = m_diskCache->readEntry(maxClipKey.toUtf8(), maxClipVal); if (minClipInCache && maxClipInCache) { clips.clear(); @@ -973,8 +973,8 @@ void DataSource::_updateClips( std::shared_ptrsetEntry( minClipKey.toUtf8(), d2qb(clips[0]), 0); + m_diskCache->setEntry( maxClipKey.toUtf8(), d2qb(clips[1]), 0); qDebug() << "++++++++ calculated clips and put in cache"; } From c80570e714603fc953644eae9e52d6e7bc6f52ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Sun, 20 Nov 2016 16:01:25 +0200 Subject: [PATCH 10/43] revert cat typing --- carta/cpp/core/Data/Image/DataSource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carta/cpp/core/Data/Image/DataSource.cpp b/carta/cpp/core/Data/Image/DataSource.cpp index d238e582..85eabf2d 100755 --- a/carta/cpp/core/Data/Image/DataSource.cpp +++ b/carta/cpp/core/Data/Image/DataSource.cpp @@ -9,7 +9,7 @@ #include "Data/Colormap/TransformsData.h" #include "CartaLib/Hooks/LoadAstroImage.h" #include "CartaLib/PixelPipeline/CustomizablePixelPipeline.h" -#include "CartaLib/IPCacheff.h" +#include "CartaLib/IPCache.h" #include "../../ImageRenderService.h" #include "../../Algorithms/quantileAlgorithms.h" #include "../../Algorithms/cacheUtils.h" From 6f5c13f4afb83d5fcf565bc8545d70afacf105d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Sun, 20 Nov 2016 16:02:55 +0200 Subject: [PATCH 11/43] fixed c&p error --- carta/cpp/core/Algorithms/cacheUtils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carta/cpp/core/Algorithms/cacheUtils.h b/carta/cpp/core/Algorithms/cacheUtils.h index 7eb8156c..a01b3639 100644 --- a/carta/cpp/core/Algorithms/cacheUtils.h +++ b/carta/cpp/core/Algorithms/cacheUtils.h @@ -71,7 +71,7 @@ std::vector qb2vd( const QByteArray & ba) { /** * Vector of integers --> byte array **/ -QByteArray vi2qb( const std::vector & vd) { +QByteArray vi2qb( const std::vector & vi) { QByteArray ba; for( const int & i : vi) { ba.append( (const char *)( & i), sizeof( int)); From a60696e60374fea637d6c62fc962132d31126b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Sun, 20 Nov 2016 16:06:07 +0200 Subject: [PATCH 12/43] missing header? --- carta/cpp/core/Data/Image/DataSource.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/carta/cpp/core/Data/Image/DataSource.cpp b/carta/cpp/core/Data/Image/DataSource.cpp index 85eabf2d..0b79b643 100755 --- a/carta/cpp/core/Data/Image/DataSource.cpp +++ b/carta/cpp/core/Data/Image/DataSource.cpp @@ -8,6 +8,7 @@ #include "Data/Util.h" #include "Data/Colormap/TransformsData.h" #include "CartaLib/Hooks/LoadAstroImage.h" +#include "CartaLib/Hooks/GetPersistantCache.h" #include "CartaLib/PixelPipeline/CustomizablePixelPipeline.h" #include "CartaLib/IPCache.h" #include "../../ImageRenderService.h" From 8be4b3d829d5fd77aff12cb35e6807a5f96882bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Sun, 20 Nov 2016 17:06:56 +0200 Subject: [PATCH 13/43] re-create key variables --- carta/cpp/core/Data/Image/DataSource.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/carta/cpp/core/Data/Image/DataSource.cpp b/carta/cpp/core/Data/Image/DataSource.cpp index 0b79b643..882dfda5 100755 --- a/carta/cpp/core/Data/Image/DataSource.cpp +++ b/carta/cpp/core/Data/Image/DataSource.cpp @@ -457,6 +457,9 @@ std::vector > DataSource::_getIntensityCache( int frameLow m_cachedPercentiles.put( frameLow, frameHigh, intensities[i].first, percentiles[i], intensities[i].second ); + QString locationKey = QString("%1/%2/%3/%4/location").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); + QString intensityKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); + m_diskCache->setEntry(locationKey.toUtf8(), i2qb(intensities[i].first), 0); m_diskCache->setEntry(intensityKey.toUtf8(), d2qb(intensities[i].second), 0); From ea449486737cc750f740368c2fc81c1a35827bff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Sun, 20 Nov 2016 17:17:01 +0200 Subject: [PATCH 14/43] misunderstanding how .back() works --- carta/cpp/core/Data/Image/DataSource.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/carta/cpp/core/Data/Image/DataSource.cpp b/carta/cpp/core/Data/Image/DataSource.cpp index 882dfda5..f6661ff0 100755 --- a/carta/cpp/core/Data/Image/DataSource.cpp +++ b/carta/cpp/core/Data/Image/DataSource.cpp @@ -958,8 +958,8 @@ void DataSource::_updateClips( std::shared_ptr Date: Sun, 20 Nov 2016 17:21:13 +0200 Subject: [PATCH 15/43] can't print QString --- carta/cpp/core/Data/Image/DataSource.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/carta/cpp/core/Data/Image/DataSource.cpp b/carta/cpp/core/Data/Image/DataSource.cpp index f6661ff0..3061ebb4 100755 --- a/carta/cpp/core/Data/Image/DataSource.cpp +++ b/carta/cpp/core/Data/Image/DataSource.cpp @@ -371,14 +371,14 @@ std::vector > DataSource::_getIntensityCache( int frameLow QByteArray locationVal; bool locationInCache = m_diskCache->readEntry(locationKey.toUtf8(), locationVal); - qDebug() << "++++++++ location key is" << locationKey; + qDebug() << "++++++++ location key is" << locationKey.toUtf8(); if (locationInCache) { QString intensityKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); QByteArray intensityVal; bool intensityInCache = m_diskCache->readEntry(intensityKey.toUtf8(), intensityVal); - qDebug() << "++++++++ intensity key is" << intensityKey; + qDebug() << "++++++++ intensity key is" << intensityKey.toUtf8(); if (intensityInCache) { qDebug() << "++++++++ found location and intensity in disk cache"; @@ -961,7 +961,7 @@ void DataSource::_updateClips( std::shared_ptr Date: Sun, 20 Nov 2016 17:29:19 +0200 Subject: [PATCH 16/43] qDebug, not QDebug --- carta/cpp/core/Data/Image/DataSource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carta/cpp/core/Data/Image/DataSource.cpp b/carta/cpp/core/Data/Image/DataSource.cpp index 3061ebb4..c492d033 100755 --- a/carta/cpp/core/Data/Image/DataSource.cpp +++ b/carta/cpp/core/Data/Image/DataSource.cpp @@ -961,7 +961,7 @@ void DataSource::_updateClips( std::shared_ptr Date: Tue, 29 Nov 2016 00:16:57 +0200 Subject: [PATCH 17/43] converted Pavol's LevelDB persistent cache implementation to a plugin --- .../plugins/PCacheLevelDB/PCacheLevelDB.cpp | 175 ++++++++++++++++++ .../cpp/plugins/PCacheLevelDB/PCacheLevelDB.h | 31 ++++ .../plugins/PCacheLevelDB/PCacheLevelDB.pro | 34 ++++ carta/cpp/plugins/PCacheLevelDB/plugin.json | 9 + 4 files changed, 249 insertions(+) create mode 100644 carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp create mode 100644 carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h create mode 100644 carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.pro create mode 100644 carta/cpp/plugins/PCacheLevelDB/plugin.json diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp new file mode 100644 index 00000000..9638409d --- /dev/null +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp @@ -0,0 +1,175 @@ +#include "PCacheLevelDB.h" +#include "CartaLib/Hooks/GetPersistantCache.h" +#include +#include +#include "leveldb/db.h" + +typedef Carta::Lib::Hooks::GetPersistantCache GetPersistantCacheHook; + +/// +/// Implementation of IPCache using LevelDB +/// +class LevelDBPCache : public Carta::Lib::IPCache +{ +public: + + virtual uint64_t + maxStorage() override + { + return 1; + } + + virtual uint64_t + usedStorage() override + { + return 1; + } + + virtual uint64_t + nEntries() override + { + return 1; + } + + virtual void + deleteAll() override + { + // not implemented + } + + virtual bool + readEntry( const QByteArray & key, QByteArray & val ) override + { + + if ( ! p_db ) { + return false; + } + std::string tmpVal; + auto status = p_db-> Get( p_readOptions, key.constData(), & tmpVal ); + if ( ! status.ok() ) { + qWarning() << "query read failed:" << status.ToString().c_str(); + return false; + } + val = QByteArray( tmpVal.data(), tmpVal.size()); + return true; + } // readEntry + + virtual void + setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) override + { + if( ! p_db) return; + auto status = p_db-> Put( p_writeOptions, + leveldb::Slice( key.constData(), key.size()), + leveldb::Slice( val.constData(), val.size())); + if ( ! status.ok() ) { + qWarning() << "query insert failed:" << status.ToString().c_str(); + } + + Q_UNUSED( priority); + // \todo we'll have to embed priority into the value, e.g. first 8 bytes? + } // setEntry + + static + Carta::Lib::IPCache::SharedPtr + getCacheSingleton( QString dirPath) + { + if ( m_cachePtr ) { + qCritical() << "PCacheLevelDBPlugin::Calling GetPersistantCacheHook multiple times!!!"; + } + else { + m_cachePtr.reset( new LevelDBPCache( dirPath) ); + } + return m_cachePtr; + } + + //~LevelDBPCache() + //{ + // I don't think a destructor is required; leaving this as a placeholder for now + //} + +private: + + LevelDBPCache( QString dirPath) + { + p_readOptions = leveldb::ReadOptions(); + p_writeOptions = leveldb::WriteOptions(); + p_writeOptions.sync = false; + + // Set up database connection information and open database + leveldb::DB * db; + leveldb::Options options; + + options.create_if_missing = true; + + leveldb::Status status = leveldb::DB::Open( options, dirPath.toStdString(), & db ); + + if ( false == status.ok() ) { + qDebug() << "Unable to open/create database '" << dirPath << "':" << status.ToString().c_str(); + return; + } + + p_db.reset( db ); + } + + std::unique_ptr< leveldb::DB > p_db; + leveldb::ReadOptions p_readOptions; + leveldb::WriteOptions p_writeOptions; + static Carta::Lib::IPCache::SharedPtr m_cachePtr; // = nullptr; +}; + +Carta::Lib::IPCache::SharedPtr LevelDBPCache::m_cachePtr = nullptr; + +PCacheLevelDBPlugin::PCacheLevelDBPlugin( QObject * parent ) : + QObject( parent ) +{ } + +bool +PCacheLevelDBPlugin::handleHook( BaseHook & hookData ) +{ + // we only handle one hook: get the cache object + if ( hookData.is < GetPersistantCacheHook > () ) { + // decode hook data + GetPersistantCacheHook & hook = static_cast < GetPersistantCacheHook & > ( hookData ); + + // if no dbdir was specified, refuse to work :) + if( m_dbDir.isNull()) { + hook.result.reset(); + return false; + } + + // try to create the database + hook.result = LevelDBPCache::getCacheSingleton( m_dbDir); + + // return true if result is not null + return hook.result != nullptr; + } + + qWarning() << "PCacheLevelDBPlugin: Sorrry, dont' know how to handle this hook"; + return false; +} // handleHook + +void +PCacheLevelDBPlugin::initialize( const IPlugin::InitInfo & initInfo ) +{ + qDebug() << "PCacheLevelDBPlugin::initialized"; + QJsonDocument doc( initInfo.json ); + qDebug() << doc.toJson(); + + // extract the location of the database from carta.config + m_dbDir = initInfo.json.value( "dbDir").toString(); + if( m_dbDir.isNull()) { + qCritical() << "No dbDir specified for PCacheSqlite3 plugin!!!"; + } + else { + // convert this to absolute path just in case + m_dbDir = QDir(m_dbDir).absolutePath(); + } +} + +std::vector < HookId > +PCacheLevelDBPlugin::getInitialHookList() +{ + return { + GetPersistantCacheHook::staticId + }; +} diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h new file mode 100644 index 00000000..12b87f61 --- /dev/null +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h @@ -0,0 +1,31 @@ +/// Implements plugin for persistant cache, using QT's sqlite3 database driver + +/// This plugin can read image formats that Qt supports. + +#pragma once + +#include "CartaLib/IPlugin.h" +#include +#include + +class PCacheLevelDBPlugin : public QObject, public IPlugin +{ + Q_OBJECT + Q_PLUGIN_METADATA( IID "org.cartaviewer.IPlugin" ) + Q_INTERFACES( IPlugin ) + +public : + PCacheLevelDBPlugin( QObject * parent = 0 ); + virtual bool + handleHook( BaseHook & hookData ) override; + + virtual std::vector < HookId > + getInitialHookList() override; + + virtual void + initialize( const InitInfo & initInfo ) override; + +private: + + QString m_dbDir; +}; diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.pro b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.pro new file mode 100644 index 00000000..1ea12f24 --- /dev/null +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.pro @@ -0,0 +1,34 @@ +! include(../../common.pri) { + error( "Could not find the common.pri file!" ) +} + +INCLUDEPATH += $$PROJECT_ROOT +DEPENDPATH += $$PROJECT_ROOT + +QT += core gui + +TARGET = plugin +TEMPLATE = lib +CONFIG += plugin + +SOURCES += \ + PCacheLevelDB.cpp + +HEADERS += \ + PCacheLevelDB.h + +LIBS += -lleveldb + +OTHER_FILES += \ + plugin.json + +# copy json to build directory +#MYFILES = $$files($${PWD}/files/*.*) +MYFILES = plugin.json +copy_files.name = copy large files +copy_files.input = MYFILES +# change datafiles to a directory you want to put the files to +copy_files.output = $${OUT_PWD}/${QMAKE_FILE_BASE}${QMAKE_FILE_EXT} +copy_files.commands = ${COPY_FILE} ${QMAKE_FILE_IN} ${QMAKE_FILE_OUT} +copy_files.CONFIG += no_link target_predeps +QMAKE_EXTRA_COMPILERS += copy_files diff --git a/carta/cpp/plugins/PCacheLevelDB/plugin.json b/carta/cpp/plugins/PCacheLevelDB/plugin.json new file mode 100644 index 00000000..994b973b --- /dev/null +++ b/carta/cpp/plugins/PCacheLevelDB/plugin.json @@ -0,0 +1,9 @@ +{ + "api" : "1", + "name" : "PCacheLevelDB", + "version" : "1", + "type" : "C++", + "description": "Implements persistant cache using LevelDB.", + "about" : "Part of CARTA. Written by Pavol and Adrianna.", + "depends" : [ ] +} From dfd6775d2bfeacd4d17c1faa3c2b9ef2b767bcf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Tue, 29 Nov 2016 14:58:01 +0200 Subject: [PATCH 18/43] added new plugin --- carta/cpp/plugins/plugins.pro | 1 + 1 file changed, 1 insertion(+) diff --git a/carta/cpp/plugins/plugins.pro b/carta/cpp/plugins/plugins.pro index 654e0227..d0163f11 100644 --- a/carta/cpp/plugins/plugins.pro +++ b/carta/cpp/plugins/plugins.pro @@ -26,6 +26,7 @@ SUBDIRS += CyberSKA SUBDIRS += DevIntegration SUBDIRS += PCacheSqlite3 +SUBDIRS += PCacheLevelDB # adrianna's render plugin From 7ebfad1e01b02ab963b717af7ebecf2f80c43b04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Tue, 29 Nov 2016 15:17:19 +0200 Subject: [PATCH 19/43] missign header? --- carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp index 9638409d..11ade0dd 100644 --- a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp @@ -2,6 +2,7 @@ #include "CartaLib/Hooks/GetPersistantCache.h" #include #include +#include #include "leveldb/db.h" typedef Carta::Lib::Hooks::GetPersistantCache GetPersistantCacheHook; From f544684704741e2781ba9ff4185baf88ba4e44af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Tue, 29 Nov 2016 15:23:33 +0200 Subject: [PATCH 20/43] This spelling mistake was driving me crazy. --- carta/cpp/CartaLib/CartaLib.pro | 4 ++-- ...GetPersistantCache.cpp => GetPersistentCache.cpp} | 2 +- .../{GetPersistantCache.h => GetPersistentCache.h} | 8 ++++---- carta/cpp/CartaLib/Hooks/HookIDs.h | 2 +- carta/cpp/CartaLib/IPCache.h | 2 +- carta/cpp/core/Data/Image/DataSource.cpp | 4 ++-- carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp | 12 ++++++------ carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h | 2 +- carta/cpp/plugins/PCacheLevelDB/plugin.json | 2 +- carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp | 12 ++++++------ carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h | 2 +- carta/cpp/plugins/PCacheSqlite3/plugin.json | 2 +- carta/cpp/testCache/main.cpp | 4 ++-- 13 files changed, 29 insertions(+), 29 deletions(-) rename carta/cpp/CartaLib/Hooks/{GetPersistantCache.cpp => GetPersistentCache.cpp} (69%) rename carta/cpp/CartaLib/Hooks/{GetPersistantCache.h => GetPersistentCache.h} (85%) diff --git a/carta/cpp/CartaLib/CartaLib.pro b/carta/cpp/CartaLib/CartaLib.pro index 62dbc081..0fe33200 100755 --- a/carta/cpp/CartaLib/CartaLib.pro +++ b/carta/cpp/CartaLib/CartaLib.pro @@ -48,7 +48,7 @@ SOURCES += \ IImageRenderService.cpp \ IRemoteVGView.cpp \ IPCache.cpp \ - Hooks/GetPersistantCache.cpp \ + Hooks/GetPersistentCache.cpp \ Hooks/GetProfileExtractor.cpp \ Regions/IRegion.cpp \ InputEvents.cpp \ @@ -116,7 +116,7 @@ HEADERS += \ Regions/ICoordSystem.h \ Hooks/CoordSystemHook.h \ Regions/CoordinateSystemFormatter.h \ - Hooks/GetPersistantCache.h \ + Hooks/GetPersistentCache.h \ Regions/Ellipse.h \ Regions/Point.h \ Regions/Rectangle.h \ diff --git a/carta/cpp/CartaLib/Hooks/GetPersistantCache.cpp b/carta/cpp/CartaLib/Hooks/GetPersistentCache.cpp similarity index 69% rename from carta/cpp/CartaLib/Hooks/GetPersistantCache.cpp rename to carta/cpp/CartaLib/Hooks/GetPersistentCache.cpp index 440270bf..48ba5c52 100644 --- a/carta/cpp/CartaLib/Hooks/GetPersistantCache.cpp +++ b/carta/cpp/CartaLib/Hooks/GetPersistentCache.cpp @@ -2,7 +2,7 @@ * **/ -#include "GetPersistantCache.h" +#include "GetPersistentCache.h" namespace Carta { diff --git a/carta/cpp/CartaLib/Hooks/GetPersistantCache.h b/carta/cpp/CartaLib/Hooks/GetPersistentCache.h similarity index 85% rename from carta/cpp/CartaLib/Hooks/GetPersistantCache.h rename to carta/cpp/CartaLib/Hooks/GetPersistentCache.h index cd6c90b1..b1ac8202 100644 --- a/carta/cpp/CartaLib/Hooks/GetPersistantCache.h +++ b/carta/cpp/CartaLib/Hooks/GetPersistentCache.h @@ -1,5 +1,5 @@ /** - * Defines a hook for obtaining a persistant cache object. + * Defines a hook for obtaining a persistent cache object. * **/ @@ -17,9 +17,9 @@ namespace Hooks { /// \brief Hook for loading a plugin of an unknown type /// -class GetPersistantCache : public BaseHook +class GetPersistentCache : public BaseHook { - CARTA_HOOK_BOILER1( GetPersistantCache ); + CARTA_HOOK_BOILER1( GetPersistentCache ); public: @@ -43,7 +43,7 @@ class GetPersistantCache : public BaseHook }; /// standard constructor (could be probably a macro) - GetPersistantCache( Params * pptr ) : BaseHook( staticId ), paramsPtr( pptr ) + GetPersistentCache( Params * pptr ) : BaseHook( staticId ), paramsPtr( pptr ) { // force instantiation of templates CARTA_ASSERT( is < Me > () ); diff --git a/carta/cpp/CartaLib/Hooks/HookIDs.h b/carta/cpp/CartaLib/Hooks/HookIDs.h index 31c58fd4..fb4af389 100644 --- a/carta/cpp/CartaLib/Hooks/HookIDs.h +++ b/carta/cpp/CartaLib/Hooks/HookIDs.h @@ -31,7 +31,7 @@ enum class UniqueHookIDs { ProfileHook_ID, Fit1DHook_ID, ImageStatisticsHook_ID, - GetPersistantCache_ID, + GetPersistentCache_ID, GetProfileExtractor_ID, /// region related stuff, still to be considered experimental diff --git a/carta/cpp/CartaLib/IPCache.h b/carta/cpp/CartaLib/IPCache.h index 59d46173..edc30b6e 100644 --- a/carta/cpp/CartaLib/IPCache.h +++ b/carta/cpp/CartaLib/IPCache.h @@ -1,4 +1,4 @@ -/// IPCache is a set of APIs to access persistant cache. +/// IPCache is a set of APIs to access persistent cache. #pragma once diff --git a/carta/cpp/core/Data/Image/DataSource.cpp b/carta/cpp/core/Data/Image/DataSource.cpp index c492d033..fa7c2966 100755 --- a/carta/cpp/core/Data/Image/DataSource.cpp +++ b/carta/cpp/core/Data/Image/DataSource.cpp @@ -8,7 +8,7 @@ #include "Data/Util.h" #include "Data/Colormap/TransformsData.h" #include "CartaLib/Hooks/LoadAstroImage.h" -#include "CartaLib/Hooks/GetPersistantCache.h" +#include "CartaLib/Hooks/GetPersistentCache.h" #include "CartaLib/PixelPipeline/CustomizablePixelPipeline.h" #include "CartaLib/IPCache.h" #include "../../ImageRenderService.h" @@ -61,7 +61,7 @@ DataSource::DataSource() : // initialize disk cache auto res = Globals::instance()-> pluginManager() - -> prepare < Carta::Lib::Hooks::GetPersistantCache > ().first(); + -> prepare < Carta::Lib::Hooks::GetPersistentCache > ().first(); if ( res.isNull() || ! res.val() ) { qWarning( "Could not find a disk cache plugin." ); m_diskCache = nullptr; diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp index 11ade0dd..036bcf51 100644 --- a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp @@ -1,11 +1,11 @@ #include "PCacheLevelDB.h" -#include "CartaLib/Hooks/GetPersistantCache.h" +#include "CartaLib/Hooks/GetPersistentCache.h" #include #include #include #include "leveldb/db.h" -typedef Carta::Lib::Hooks::GetPersistantCache GetPersistantCacheHook; +typedef Carta::Lib::Hooks::GetPersistentCache GetPersistentCacheHook; /// /// Implementation of IPCache using LevelDB @@ -75,7 +75,7 @@ class LevelDBPCache : public Carta::Lib::IPCache getCacheSingleton( QString dirPath) { if ( m_cachePtr ) { - qCritical() << "PCacheLevelDBPlugin::Calling GetPersistantCacheHook multiple times!!!"; + qCritical() << "PCacheLevelDBPlugin::Calling GetPersistentCacheHook multiple times!!!"; } else { m_cachePtr.reset( new LevelDBPCache( dirPath) ); @@ -128,9 +128,9 @@ bool PCacheLevelDBPlugin::handleHook( BaseHook & hookData ) { // we only handle one hook: get the cache object - if ( hookData.is < GetPersistantCacheHook > () ) { + if ( hookData.is < GetPersistentCacheHook > () ) { // decode hook data - GetPersistantCacheHook & hook = static_cast < GetPersistantCacheHook & > ( hookData ); + GetPersistentCacheHook & hook = static_cast < GetPersistentCacheHook & > ( hookData ); // if no dbdir was specified, refuse to work :) if( m_dbDir.isNull()) { @@ -171,6 +171,6 @@ std::vector < HookId > PCacheLevelDBPlugin::getInitialHookList() { return { - GetPersistantCacheHook::staticId + GetPersistentCacheHook::staticId }; } diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h index 12b87f61..f78067aa 100644 --- a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h @@ -1,4 +1,4 @@ -/// Implements plugin for persistant cache, using QT's sqlite3 database driver +/// Implements plugin for persistent cache, using QT's sqlite3 database driver /// This plugin can read image formats that Qt supports. diff --git a/carta/cpp/plugins/PCacheLevelDB/plugin.json b/carta/cpp/plugins/PCacheLevelDB/plugin.json index 994b973b..4e51b32d 100644 --- a/carta/cpp/plugins/PCacheLevelDB/plugin.json +++ b/carta/cpp/plugins/PCacheLevelDB/plugin.json @@ -3,7 +3,7 @@ "name" : "PCacheLevelDB", "version" : "1", "type" : "C++", - "description": "Implements persistant cache using LevelDB.", + "description": "Implements persistent cache using LevelDB.", "about" : "Part of CARTA. Written by Pavol and Adrianna.", "depends" : [ ] } diff --git a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp index 1816a5b9..c6746faf 100644 --- a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp +++ b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp @@ -1,10 +1,10 @@ #include "PCacheSqlite3.h" -#include "CartaLib/Hooks/GetPersistantCache.h" +#include "CartaLib/Hooks/GetPersistentCache.h" #include #include #include -typedef Carta::Lib::Hooks::GetPersistantCache GetPersistantCacheHook; +typedef Carta::Lib::Hooks::GetPersistentCache GetPersistentCacheHook; /// /// Implementation of IPCache using sqlite @@ -86,7 +86,7 @@ class SqLitePCache : public Carta::Lib::IPCache getCacheSingleton( QString dirPath) { if ( m_cachePtr ) { - qCritical() << "PCacheSQlite3Plugin::Calling GetPersistantCacheHook multiple times!!!"; + qCritical() << "PCacheSQlite3Plugin::Calling GetPersistentCacheHook multiple times!!!"; } else { // stupid c++ won't allow this @@ -148,9 +148,9 @@ bool PCacheSQlite3Plugin::handleHook( BaseHook & hookData ) { // we only handle one hook: get the cache object - if ( hookData.is < GetPersistantCacheHook > () ) { + if ( hookData.is < GetPersistentCacheHook > () ) { // decode hook data - GetPersistantCacheHook & hook = static_cast < GetPersistantCacheHook & > ( hookData ); + GetPersistentCacheHook & hook = static_cast < GetPersistentCacheHook & > ( hookData ); // if no dbdir was specified, refuse to work :) if( m_dbDir.isNull()) { @@ -191,6 +191,6 @@ std::vector < HookId > PCacheSQlite3Plugin::getInitialHookList() { return { - GetPersistantCacheHook::staticId + GetPersistentCacheHook::staticId }; } diff --git a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h index be099ec4..a744de02 100644 --- a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h +++ b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h @@ -1,4 +1,4 @@ -/// Implements plugin for persistant cache, using QT's sqlite3 database driver +/// Implements plugin for persistent cache, using QT's sqlite3 database driver /// This plugin can read image formats that Qt supports. diff --git a/carta/cpp/plugins/PCacheSqlite3/plugin.json b/carta/cpp/plugins/PCacheSqlite3/plugin.json index 32585247..73db97b8 100644 --- a/carta/cpp/plugins/PCacheSqlite3/plugin.json +++ b/carta/cpp/plugins/PCacheSqlite3/plugin.json @@ -3,7 +3,7 @@ "name" : "PCacheSqlite3", "version" : "1", "type" : "C++", - "description": "Implements persistant cache using QT's sqlite3 driver.", + "description": "Implements persistent cache using QT's sqlite3 driver.", "about" : "Part of CARTA. Written by Pavol", "depends" : [ ] } diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index 20d21b6b..8c1d53ec 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -1,6 +1,6 @@ //#include "core/Viewer.h" #include "CartaLib/Hooks/Initialize.h" -#include "CartaLib/Hooks/GetPersistantCache.h" +#include "CartaLib/Hooks/GetPersistentCache.h" #include "core/MyQApp.h" #include "core/CmdLine.h" #include "core/MainConfig.h" @@ -260,7 +260,7 @@ coreMainCPP( QString platformString, int argc, char * * argv ) pm-> prepare < Carta::Lib::Hooks::Initialize > ().executeAll(); // let's get pcache object - auto pcacheRes = pm-> prepare< Carta::Lib::Hooks::GetPersistantCache >().first(); + auto pcacheRes = pm-> prepare< Carta::Lib::Hooks::GetPersistentCache >().first(); if( pcacheRes.isNull()) { qWarning() << "Could not initialize persistent cache."; return -1; From ae244decfb4184ab4774d5cfb9ab0dd460a03ea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Tue, 29 Nov 2016 15:42:43 +0200 Subject: [PATCH 21/43] wrong header --- carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp index 036bcf51..3713fe16 100644 --- a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp @@ -2,7 +2,7 @@ #include "CartaLib/Hooks/GetPersistentCache.h" #include #include -#include +#include #include "leveldb/db.h" typedef Carta::Lib::Hooks::GetPersistentCache GetPersistentCacheHook; From 17ef885deca3edc8a1664722a69f4cb87ba4bce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Tue, 29 Nov 2016 16:03:33 +0200 Subject: [PATCH 22/43] creating database file incorrectly --- carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp index 3713fe16..6d5562ae 100644 --- a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp @@ -101,11 +101,13 @@ class LevelDBPCache : public Carta::Lib::IPCache leveldb::Options options; options.create_if_missing = true; + + QString fname = dirPath + "/pcache.leveldb"; - leveldb::Status status = leveldb::DB::Open( options, dirPath.toStdString(), & db ); + leveldb::Status status = leveldb::DB::Open( options, fname.toStdString(), & db ); if ( false == status.ok() ) { - qDebug() << "Unable to open/create database '" << dirPath << "':" << status.ToString().c_str(); + qDebug() << "Unable to open/create database '" << fname.toStdString().c_str() << "':" << status.ToString().c_str(); return; } From 3645189e452dad4ef04dd561f9d20e6801eb6354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Tue, 29 Nov 2016 16:19:14 +0200 Subject: [PATCH 23/43] We don't actually want to warn on failed reads; that happens whenever the key doesn't exist --- carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp index 6d5562ae..48019e92 100644 --- a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp @@ -48,7 +48,7 @@ class LevelDBPCache : public Carta::Lib::IPCache std::string tmpVal; auto status = p_db-> Get( p_readOptions, key.constData(), & tmpVal ); if ( ! status.ok() ) { - qWarning() << "query read failed:" << status.ToString().c_str(); + //qWarning() << "query read failed:" << status.ToString().c_str(); return false; } val = QByteArray( tmpVal.data(), tmpVal.size()); From 5d098a8f0a83b1d79650e26ac0860f619afbb0ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Mon, 5 Dec 2016 19:21:43 +0200 Subject: [PATCH 24/43] Cleaning up cache test. Commented out duplicate implementation; trying to call test on all cache plugins. --- carta/cpp/testCache/LevelDbIPCache.cpp | 124 ++++++++++++------------- carta/cpp/testCache/LevelDbIPCache.h | 74 +++++++-------- carta/cpp/testCache/SqLitePCache.cpp | 116 +++++++++++------------ carta/cpp/testCache/SqLitePCache.h | 102 ++++++++++---------- carta/cpp/testCache/main.cpp | 44 +++++---- carta/cpp/testCache/testCache.pro | 22 +++-- 6 files changed, 248 insertions(+), 234 deletions(-) diff --git a/carta/cpp/testCache/LevelDbIPCache.cpp b/carta/cpp/testCache/LevelDbIPCache.cpp index c1ef3dd0..1d79e0ab 100644 --- a/carta/cpp/testCache/LevelDbIPCache.cpp +++ b/carta/cpp/testCache/LevelDbIPCache.cpp @@ -1,62 +1,62 @@ -/** - * - **/ - -#include "LevelDbIPCache.h" -#include - -namespace tCache -{ -LevelDbIPCache::LevelDbIPCache() -{ - p_readOptions = leveldb::ReadOptions(); - p_writeOptions = leveldb::WriteOptions(); - p_writeOptions.sync = false; - - - // Set up database connection information and open database - leveldb::DB * db; - leveldb::Options options; -// qDebug() << "write buffer size=" << options.write_buffer_size; -// options.write_buffer_size *= 256; - options.create_if_missing = true; - - leveldb::Status status = leveldb::DB::Open( options, "/scratchSD/testLevelDb.leveldb", & db ); - - if ( false == status.ok() ) { - qDebug() << "Unable to open/create test database './testdb'" << endl; - qDebug() << status.ToString().c_str() << endl; - return; - } - - p_db.reset( db ); -} - -bool -LevelDbIPCache::readEntry( const QByteArray & key, QByteArray & val ) -{ - if ( ! p_db ) { - return false; - } - std::string tmpVal; - auto status = p_db-> Get( p_readOptions, key.constData(), & tmpVal ); - if ( ! status.ok() ) { - return false; - } - val = QByteArray( tmpVal.data(), tmpVal.size()); -// val.setRawData( tmpVal.data(), tmpVal.size()); - return true; -} - -void -LevelDbIPCache::setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) -{ - if( ! p_db) return; - p_db-> Put( p_writeOptions, - leveldb::Slice( key.constData(), key.size()), - leveldb::Slice( val.constData(), val.size())); - - Q_UNUSED( priority); - /// \todo we'll have to embed priority into the value, e.g. first 8 bytes? -} -} +///** + //* + //**/ + +//#include "LevelDbIPCache.h" +//#include + +//namespace tCache +//{ +//LevelDbIPCache::LevelDbIPCache() +//{ + //p_readOptions = leveldb::ReadOptions(); + //p_writeOptions = leveldb::WriteOptions(); + //p_writeOptions.sync = false; + + + //// Set up database connection information and open database + //leveldb::DB * db; + //leveldb::Options options; +//// qDebug() << "write buffer size=" << options.write_buffer_size; +//// options.write_buffer_size *= 256; + //options.create_if_missing = true; + + //leveldb::Status status = leveldb::DB::Open( options, "/scratchSD/testLevelDb.leveldb", & db ); + + //if ( false == status.ok() ) { + //qDebug() << "Unable to open/create test database './testdb'" << endl; + //qDebug() << status.ToString().c_str() << endl; + //return; + //} + + //p_db.reset( db ); +//} + +//bool +//LevelDbIPCache::readEntry( const QByteArray & key, QByteArray & val ) +//{ + //if ( ! p_db ) { + //return false; + //} + //std::string tmpVal; + //auto status = p_db-> Get( p_readOptions, key.constData(), & tmpVal ); + //if ( ! status.ok() ) { + //return false; + //} + //val = QByteArray( tmpVal.data(), tmpVal.size()); +//// val.setRawData( tmpVal.data(), tmpVal.size()); + //return true; +//} + +//void +//LevelDbIPCache::setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) +//{ + //if( ! p_db) return; + //p_db-> Put( p_writeOptions, + //leveldb::Slice( key.constData(), key.size()), + //leveldb::Slice( val.constData(), val.size())); + + //Q_UNUSED( priority); + ///// \todo we'll have to embed priority into the value, e.g. first 8 bytes? +//} +//} diff --git a/carta/cpp/testCache/LevelDbIPCache.h b/carta/cpp/testCache/LevelDbIPCache.h index 7940a14e..400dfbb0 100644 --- a/carta/cpp/testCache/LevelDbIPCache.h +++ b/carta/cpp/testCache/LevelDbIPCache.h @@ -1,50 +1,50 @@ -/** - * - **/ +///** + //* + //**/ -#pragma once +//#pragma once -#include "CartaLib/IPCache.h" -#include "leveldb/db.h" +//#include "CartaLib/IPCache.h" +//#include "leveldb/db.h" -namespace tCache -{ +//namespace tCache +//{ -class LevelDbIPCache : public Carta::Lib::IPCache -{ -public: +//class LevelDbIPCache : public Carta::Lib::IPCache +//{ +//public: -public: +//public: - LevelDbIPCache(); - virtual uint64_t maxStorage() override - { - return 1; - } - virtual uint64_t usedStorage() override - { - return 1; - } - virtual uint64_t nEntries() override - { - return 1; - } - virtual void deleteAll() override - { - // not implemented - } - virtual bool readEntry(const QByteArray & key, QByteArray & val) override; - virtual void setEntry(const QByteArray & key, const QByteArray & val, int64_t priority) override; + //LevelDbIPCache(); + //virtual uint64_t maxStorage() override + //{ + //return 1; + //} + //virtual uint64_t usedStorage() override + //{ + //return 1; + //} + //virtual uint64_t nEntries() override + //{ + //return 1; + //} + //virtual void deleteAll() override + //{ + //// not implemented + //} + //virtual bool readEntry(const QByteArray & key, QByteArray & val) override; + //virtual void setEntry(const QByteArray & key, const QByteArray & val, int64_t priority) override; -private: +//private: - std::unique_ptr< leveldb::DB > p_db; - leveldb::ReadOptions p_readOptions; - leveldb::WriteOptions p_writeOptions; + //std::unique_ptr< leveldb::DB > p_db; + //leveldb::ReadOptions p_readOptions; + //leveldb::WriteOptions p_writeOptions; -}; +//}; -} +//} diff --git a/carta/cpp/testCache/SqLitePCache.cpp b/carta/cpp/testCache/SqLitePCache.cpp index f0c84b15..00881540 100644 --- a/carta/cpp/testCache/SqLitePCache.cpp +++ b/carta/cpp/testCache/SqLitePCache.cpp @@ -1,65 +1,65 @@ -/** - * - **/ +///** + //* + //**/ -#include "SqLitePCache.h" -#include -#include +//#include "SqLitePCache.h" +//#include +//#include -namespace tCache -{ -SqLitePCache::SqLitePCache() -{ - m_db = QSqlDatabase::addDatabase( "QSQLITE" ); -// m_db.setDatabaseName( "/scratchSD/test.sqlite" ); - m_db.setDatabaseName( "/scratch/test.sqlite" ); - bool ok = m_db.open(); - if ( ! ok ) { - qCritical() << "Could not open sqlite database"; - } +//namespace tCache +//{ +//SqLitePCache::SqLitePCache() +//{ + //m_db = QSqlDatabase::addDatabase( "QSQLITE" ); +//// m_db.setDatabaseName( "/scratchSD/test.sqlite" ); + //m_db.setDatabaseName( "/scratch/test.sqlite" ); + //bool ok = m_db.open(); + //if ( ! ok ) { + //qCritical() << "Could not open sqlite database"; + //} - QSqlQuery query(m_db); + //QSqlQuery query(m_db); - query.prepare( "create table db (key text primary key, " - "val blob)" ); -// qDebug() << "create query" << query.lastQuery(); - if( ! query.exec()) { - qCritical() << "query create table failed" - << query.lastError().text(); - } -} + //query.prepare( "create table db (key text primary key, " + //"val blob)" ); +//// qDebug() << "create query" << query.lastQuery(); + //if( ! query.exec()) { + //qCritical() << "query create table failed" + //<< query.lastError().text(); + //} +//} -bool -SqLitePCache::readEntry( const QByteArray & key, QByteArray & val ) -{ - QSqlQuery query(m_db); - query.prepare( "SELECT val FROM db WHERE key = :key" ); - query.bindValue( ":key", key ); -// qDebug() << "select query" << query.lastQuery(); - if ( ! query.exec() ) { - qWarning() << "query read failed"; - return false; - } - if ( query.next() ) { - val = query.value( 0 ).toByteArray(); - return true; - } - return false; -} +//bool +//SqLitePCache::readEntry( const QByteArray & key, QByteArray & val ) +//{ + //QSqlQuery query(m_db); + //query.prepare( "SELECT val FROM db WHERE key = :key" ); + //query.bindValue( ":key", key ); +//// qDebug() << "select query" << query.lastQuery(); + //if ( ! query.exec() ) { + //qWarning() << "query read failed"; + //return false; + //} + //if ( query.next() ) { + //val = query.value( 0 ).toByteArray(); + //return true; + //} + //return false; +//} -void -SqLitePCache::setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) -{ - QSqlQuery query(m_db); - query.prepare( "INSERT OR REPLACE INTO db (key, val) VALUES ( :key, :val)" ); -// query.prepare( "INSERT INTO db (key, val) VALUES ( :key, :val)" ); - query.bindValue( ":key", key ); - query.bindValue( ":val", val ); -// qDebug() << "insert query" << query.lastQuery(); - if( ! query.exec()) { - qWarning() << "query insert failed" - << query.lastError().text(); +//void +//SqLitePCache::setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) +//{ + //QSqlQuery query(m_db); + //query.prepare( "INSERT OR REPLACE INTO db (key, val) VALUES ( :key, :val)" ); +//// query.prepare( "INSERT INTO db (key, val) VALUES ( :key, :val)" ); + //query.bindValue( ":key", key ); + //query.bindValue( ":val", val ); +//// qDebug() << "insert query" << query.lastQuery(); + //if( ! query.exec()) { + //qWarning() << "query insert failed" + //<< query.lastError().text(); - } -} -} + //} +//} +//} diff --git a/carta/cpp/testCache/SqLitePCache.h b/carta/cpp/testCache/SqLitePCache.h index 0658b469..ad232360 100644 --- a/carta/cpp/testCache/SqLitePCache.h +++ b/carta/cpp/testCache/SqLitePCache.h @@ -1,51 +1,51 @@ -/** - * - **/ - -#pragma once - -#include "CartaLib/IPCache.h" - -#include -namespace tCache -{ -class SqLitePCache : public Carta::Lib::IPCache -{ -public: - - SqLitePCache(); - - virtual uint64_t - maxStorage() override - { - return 1; - } - - virtual uint64_t - usedStorage() override - { - return 1; - } - - virtual uint64_t - nEntries() override - { - return 1; - } - - virtual void - deleteAll() override - { - // not implemented - } - - virtual bool - readEntry( const QByteArray & key, QByteArray & val ) override; - - virtual void - setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) override; - -private: - QSqlDatabase m_db; -}; -} +///** + //* + //**/ + +//#pragma once + +//#include "CartaLib/IPCache.h" + +//#include +//namespace tCache +//{ +//class SqLitePCache : public Carta::Lib::IPCache +//{ +//public: + + //SqLitePCache(); + + //virtual uint64_t + //maxStorage() override + //{ + //return 1; + //} + + //virtual uint64_t + //usedStorage() override + //{ + //return 1; + //} + + //virtual uint64_t + //nEntries() override + //{ + //return 1; + //} + + //virtual void + //deleteAll() override + //{ + //// not implemented + //} + + //virtual bool + //readEntry( const QByteArray & key, QByteArray & val ) override; + + //virtual void + //setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) override; + +//private: + //QSqlDatabase m_db; +//}; +//} diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index 8c1d53ec..043e17f9 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -5,8 +5,9 @@ #include "core/CmdLine.h" #include "core/MainConfig.h" #include "core/Globals.h" -#include "LevelDbIPCache.h" -#include "SqLitePCache.h" +#include "CartaLib/IPCache.h" +//#include "LevelDbIPCache.h" +//#include "SqLitePCache.h" #include #include @@ -227,9 +228,9 @@ coreMainCPP( QString platformString, int argc, char * * argv ) auto cmdLineInfo = CmdLine::parse( MyQApp::arguments() ); globals.setCmdLineInfo( & cmdLineInfo ); - if ( cmdLineInfo.fileList().size() < 2 ) { - qFatal( "Need 2 files" ); - } + //if ( cmdLineInfo.fileList().size() < 2 ) { + //qFatal( "Need 2 files" ); + //} // load the config file // ==================== @@ -258,18 +259,29 @@ coreMainCPP( QString platformString, int argc, char * * argv ) // send an initialize hook to all plugins, because some may rely on it pm-> prepare < Carta::Lib::Hooks::Initialize > ().executeAll(); - - // let's get pcache object - auto pcacheRes = pm-> prepare< Carta::Lib::Hooks::GetPersistentCache >().first(); - if( pcacheRes.isNull()) { - qWarning() << "Could not initialize persistent cache."; - return -1; + + + // make a lambda to set the value of pcache and call the tests + auto lam = [=] ( const Carta::Lib::Hooks::GetPersistentCache::ResultType &res ) { + pcache = res.val(); + testCache(); } - else { - pcache = pcacheRes.val(); - } - - testCache(); + + // call the lambda on every pcache plugin + auto pcacheRes = pm-> prepare< Carta::Lib::Hooks::GetPersistentCache >(); + pcacheRes.forEach(lam); + + //// let's get pcache object + //auto pcacheRes = pm-> prepare< Carta::Lib::Hooks::GetPersistentCache >().first(); + //if( pcacheRes.isNull()) { + //qWarning() << "Could not initialize persistent cache."; + //return -1; + //} + //else { + //pcache = pcacheRes.val(); + //} + + //testCache(); // give QT control // int res = qapp.exec(); diff --git a/carta/cpp/testCache/testCache.pro b/carta/cpp/testCache/testCache.pro index 5a9629e0..6a2bd2b2 100644 --- a/carta/cpp/testCache/testCache.pro +++ b/carta/cpp/testCache/testCache.pro @@ -2,16 +2,18 @@ error( "Could not find the common.pri file!" ) } -QT += webkitwidgets network widgets xml sql +#QT += webkitwidgets network widgets xml sql +QT += webkitwidgets network widgets xml -HEADERS += \ - LevelDbIPCache.h \ - SqLitePCache.h +#HEADERS += \ +# LevelDbIPCache.h \ +# SqLitePCache.h SOURCES += \ - main.cpp \ - LevelDbIPCache.cpp \ - SqLitePCache.cpp + main.cpp +# main.cpp \ +# LevelDbIPCache.cpp \ +# SqLitePCache.cpp RESOURCES = @@ -35,7 +37,7 @@ else{ } -LEVELDBDIR=/home/pfederl/Downloads/build/leveldb-1.19 +#LEVELDBDIR=/home/pfederl/Downloads/build/leveldb-1.19 #unix: LIBS += -L$$LEVELDBDIR/out-shared -lleveldb -unix: LIBS += -L$$LEVELDBDIR/out-static -lleveldb -INCLUDEPATH += $$LEVELDBDIR/include +#unix: LIBS += -L$$LEVELDBDIR/out-static -lleveldb +#INCLUDEPATH += $$LEVELDBDIR/include From 274c71deef2151a1272af10195c3ba6fa3bcd969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Mon, 5 Dec 2016 23:39:22 +0200 Subject: [PATCH 25/43] trying to use forEach correctly --- carta/cpp/testCache/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index 043e17f9..1cc2e7cd 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -263,7 +263,7 @@ coreMainCPP( QString platformString, int argc, char * * argv ) // make a lambda to set the value of pcache and call the tests auto lam = [=] ( const Carta::Lib::Hooks::GetPersistentCache::ResultType &res ) { - pcache = res.val(); + pcache = res; testCache(); } From 6e73e65886bb2ea7e8c5d5d9a998294db676a311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Mon, 5 Dec 2016 23:45:18 +0200 Subject: [PATCH 26/43] semicolon after lambda --- carta/cpp/testCache/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index 1cc2e7cd..65c7c2c8 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -265,7 +265,7 @@ coreMainCPP( QString platformString, int argc, char * * argv ) auto lam = [=] ( const Carta::Lib::Hooks::GetPersistentCache::ResultType &res ) { pcache = res; testCache(); - } + }; // call the lambda on every pcache plugin auto pcacheRes = pm-> prepare< Carta::Lib::Hooks::GetPersistentCache >(); From c673a1625efdd3e0ae1c38dcb85a0679a4f9a5e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Mon, 5 Dec 2016 23:55:00 +0200 Subject: [PATCH 27/43] removed commented-out cruft --- carta/cpp/testCache/LevelDbIPCache.cpp | 62 ------------------------ carta/cpp/testCache/LevelDbIPCache.h | 50 -------------------- carta/cpp/testCache/SqLitePCache.cpp | 65 -------------------------- carta/cpp/testCache/SqLitePCache.h | 51 -------------------- carta/cpp/testCache/main.cpp | 57 ---------------------- carta/cpp/testCache/testCache.pro | 14 +----- 6 files changed, 1 insertion(+), 298 deletions(-) delete mode 100644 carta/cpp/testCache/LevelDbIPCache.cpp delete mode 100644 carta/cpp/testCache/LevelDbIPCache.h delete mode 100644 carta/cpp/testCache/SqLitePCache.cpp delete mode 100644 carta/cpp/testCache/SqLitePCache.h diff --git a/carta/cpp/testCache/LevelDbIPCache.cpp b/carta/cpp/testCache/LevelDbIPCache.cpp deleted file mode 100644 index 1d79e0ab..00000000 --- a/carta/cpp/testCache/LevelDbIPCache.cpp +++ /dev/null @@ -1,62 +0,0 @@ -///** - //* - //**/ - -//#include "LevelDbIPCache.h" -//#include - -//namespace tCache -//{ -//LevelDbIPCache::LevelDbIPCache() -//{ - //p_readOptions = leveldb::ReadOptions(); - //p_writeOptions = leveldb::WriteOptions(); - //p_writeOptions.sync = false; - - - //// Set up database connection information and open database - //leveldb::DB * db; - //leveldb::Options options; -//// qDebug() << "write buffer size=" << options.write_buffer_size; -//// options.write_buffer_size *= 256; - //options.create_if_missing = true; - - //leveldb::Status status = leveldb::DB::Open( options, "/scratchSD/testLevelDb.leveldb", & db ); - - //if ( false == status.ok() ) { - //qDebug() << "Unable to open/create test database './testdb'" << endl; - //qDebug() << status.ToString().c_str() << endl; - //return; - //} - - //p_db.reset( db ); -//} - -//bool -//LevelDbIPCache::readEntry( const QByteArray & key, QByteArray & val ) -//{ - //if ( ! p_db ) { - //return false; - //} - //std::string tmpVal; - //auto status = p_db-> Get( p_readOptions, key.constData(), & tmpVal ); - //if ( ! status.ok() ) { - //return false; - //} - //val = QByteArray( tmpVal.data(), tmpVal.size()); -//// val.setRawData( tmpVal.data(), tmpVal.size()); - //return true; -//} - -//void -//LevelDbIPCache::setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) -//{ - //if( ! p_db) return; - //p_db-> Put( p_writeOptions, - //leveldb::Slice( key.constData(), key.size()), - //leveldb::Slice( val.constData(), val.size())); - - //Q_UNUSED( priority); - ///// \todo we'll have to embed priority into the value, e.g. first 8 bytes? -//} -//} diff --git a/carta/cpp/testCache/LevelDbIPCache.h b/carta/cpp/testCache/LevelDbIPCache.h deleted file mode 100644 index 400dfbb0..00000000 --- a/carta/cpp/testCache/LevelDbIPCache.h +++ /dev/null @@ -1,50 +0,0 @@ -///** - //* - //**/ - -//#pragma once - -//#include "CartaLib/IPCache.h" -//#include "leveldb/db.h" - -//namespace tCache -//{ - - -//class LevelDbIPCache : public Carta::Lib::IPCache -//{ -//public: - -//public: - - //LevelDbIPCache(); - //virtual uint64_t maxStorage() override - //{ - //return 1; - //} - //virtual uint64_t usedStorage() override - //{ - //return 1; - //} - //virtual uint64_t nEntries() override - //{ - //return 1; - //} - //virtual void deleteAll() override - //{ - //// not implemented - //} - //virtual bool readEntry(const QByteArray & key, QByteArray & val) override; - //virtual void setEntry(const QByteArray & key, const QByteArray & val, int64_t priority) override; - -//private: - - //std::unique_ptr< leveldb::DB > p_db; - //leveldb::ReadOptions p_readOptions; - //leveldb::WriteOptions p_writeOptions; - -//}; - - -//} - diff --git a/carta/cpp/testCache/SqLitePCache.cpp b/carta/cpp/testCache/SqLitePCache.cpp deleted file mode 100644 index 00881540..00000000 --- a/carta/cpp/testCache/SqLitePCache.cpp +++ /dev/null @@ -1,65 +0,0 @@ -///** - //* - //**/ - -//#include "SqLitePCache.h" -//#include -//#include - -//namespace tCache -//{ -//SqLitePCache::SqLitePCache() -//{ - //m_db = QSqlDatabase::addDatabase( "QSQLITE" ); -//// m_db.setDatabaseName( "/scratchSD/test.sqlite" ); - //m_db.setDatabaseName( "/scratch/test.sqlite" ); - //bool ok = m_db.open(); - //if ( ! ok ) { - //qCritical() << "Could not open sqlite database"; - //} - - //QSqlQuery query(m_db); - - //query.prepare( "create table db (key text primary key, " - //"val blob)" ); -//// qDebug() << "create query" << query.lastQuery(); - //if( ! query.exec()) { - //qCritical() << "query create table failed" - //<< query.lastError().text(); - //} -//} - -//bool -//SqLitePCache::readEntry( const QByteArray & key, QByteArray & val ) -//{ - //QSqlQuery query(m_db); - //query.prepare( "SELECT val FROM db WHERE key = :key" ); - //query.bindValue( ":key", key ); -//// qDebug() << "select query" << query.lastQuery(); - //if ( ! query.exec() ) { - //qWarning() << "query read failed"; - //return false; - //} - //if ( query.next() ) { - //val = query.value( 0 ).toByteArray(); - //return true; - //} - //return false; -//} - -//void -//SqLitePCache::setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) -//{ - //QSqlQuery query(m_db); - //query.prepare( "INSERT OR REPLACE INTO db (key, val) VALUES ( :key, :val)" ); -//// query.prepare( "INSERT INTO db (key, val) VALUES ( :key, :val)" ); - //query.bindValue( ":key", key ); - //query.bindValue( ":val", val ); -//// qDebug() << "insert query" << query.lastQuery(); - //if( ! query.exec()) { - //qWarning() << "query insert failed" - //<< query.lastError().text(); - - //} -//} -//} diff --git a/carta/cpp/testCache/SqLitePCache.h b/carta/cpp/testCache/SqLitePCache.h deleted file mode 100644 index ad232360..00000000 --- a/carta/cpp/testCache/SqLitePCache.h +++ /dev/null @@ -1,51 +0,0 @@ -///** - //* - //**/ - -//#pragma once - -//#include "CartaLib/IPCache.h" - -//#include -//namespace tCache -//{ -//class SqLitePCache : public Carta::Lib::IPCache -//{ -//public: - - //SqLitePCache(); - - //virtual uint64_t - //maxStorage() override - //{ - //return 1; - //} - - //virtual uint64_t - //usedStorage() override - //{ - //return 1; - //} - - //virtual uint64_t - //nEntries() override - //{ - //return 1; - //} - - //virtual void - //deleteAll() override - //{ - //// not implemented - //} - - //virtual bool - //readEntry( const QByteArray & key, QByteArray & val ) override; - - //virtual void - //setEntry( const QByteArray & key, const QByteArray & val, int64_t priority ) override; - -//private: - //QSqlDatabase m_db; -//}; -//} diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index 65c7c2c8..25ef4e9d 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -1,4 +1,3 @@ -//#include "core/Viewer.h" #include "CartaLib/Hooks/Initialize.h" #include "CartaLib/Hooks/GetPersistentCache.h" #include "core/MyQApp.h" @@ -6,8 +5,6 @@ #include "core/MainConfig.h" #include "core/Globals.h" #include "CartaLib/IPCache.h" -//#include "LevelDbIPCache.h" -//#include "SqLitePCache.h" #include #include @@ -15,10 +12,6 @@ namespace tCache { std::shared_ptr < Carta::Lib::IPCache > pcache; -//int imWidth = 20; -//int imHeight = 8000; -//int imDepth = 16000; - int imWidth = 20; int imHeight = 20; int imDepth = 16000; @@ -76,13 +69,6 @@ readProfile( int x, int y ) return { }; } -// std::vector res(imWidth); -// const double * dptr = reinterpret_cast (val.constData()); -// for( int z = 0 ; z < imDepth ; z ++ ) { -// res[z] = * dptr; -// dptr ++; -// } - return qb2vd( val ); } // readProfile @@ -139,23 +125,6 @@ readCache() qCritical() << "Failed to match" << x << y; } - /* - - QByteArray ba = QByteArray::fromRawData( - reinterpret_cast < char * > ( & arr[0] ), - sizeof( double ) * imDepth ); - - QString keyString = QString( "%1/%2" ).arg( x ).arg( y ); - QByteArray ba2; - if( ! pcache-> readEntry( keyString.toUtf8(), ba2)) { - qCritical() << "Failed to read" << x << y; - } - if( ba != ba2) { - qCritical() << "Failed to match" << x << y << ba.size() << ba2.size(); - qDebug() << * (const double *) (ba.constData()); - qDebug() << * (const double *) (ba2.constData()); - } - */ } qDebug() << " " << ( x + 1 ) * imHeight * 1000.0 / t.elapsed() << "pix/s"; } @@ -164,11 +133,6 @@ readCache() static void testCache() { -// pcache.reset( -// new LevelDbIPCache() ); - -// pcache.reset( -// new SqLitePCache() ); { pcache-> setEntry( "hello", "world", 0 ); @@ -201,11 +165,6 @@ testCache() static int coreMainCPP( QString platformString, int argc, char * * argv ) { - // - // initialize Qt - // - // don't require a window manager even though we're a QGuiApplication -// qputenv("QT_QPA_PLATFORM", QByteArrayLiteral("minimal")); MyQApp qapp( argc, argv ); @@ -228,10 +187,6 @@ coreMainCPP( QString platformString, int argc, char * * argv ) auto cmdLineInfo = CmdLine::parse( MyQApp::arguments() ); globals.setCmdLineInfo( & cmdLineInfo ); - //if ( cmdLineInfo.fileList().size() < 2 ) { - //qFatal( "Need 2 files" ); - //} - // load the config file // ==================== QString configFilePath = cmdLineInfo.configFilePath(); @@ -271,18 +226,6 @@ coreMainCPP( QString platformString, int argc, char * * argv ) auto pcacheRes = pm-> prepare< Carta::Lib::Hooks::GetPersistentCache >(); pcacheRes.forEach(lam); - //// let's get pcache object - //auto pcacheRes = pm-> prepare< Carta::Lib::Hooks::GetPersistentCache >().first(); - //if( pcacheRes.isNull()) { - //qWarning() << "Could not initialize persistent cache."; - //return -1; - //} - //else { - //pcache = pcacheRes.val(); - //} - - //testCache(); - // give QT control // int res = qapp.exec(); int res = 0; diff --git a/carta/cpp/testCache/testCache.pro b/carta/cpp/testCache/testCache.pro index 6a2bd2b2..32eff5b9 100644 --- a/carta/cpp/testCache/testCache.pro +++ b/carta/cpp/testCache/testCache.pro @@ -2,18 +2,12 @@ error( "Could not find the common.pri file!" ) } -#QT += webkitwidgets network widgets xml sql QT += webkitwidgets network widgets xml -#HEADERS += \ -# LevelDbIPCache.h \ -# SqLitePCache.h +HEADERS += SOURCES += \ main.cpp -# main.cpp \ -# LevelDbIPCache.cpp \ -# SqLitePCache.cpp RESOURCES = @@ -35,9 +29,3 @@ else{ LIBS +=-L$$QWT_ROOT/lib -lqwt PRE_TARGETDEPS += $$OUT_PWD/../core/libcore.so } - - -#LEVELDBDIR=/home/pfederl/Downloads/build/leveldb-1.19 -#unix: LIBS += -L$$LEVELDBDIR/out-shared -lleveldb -#unix: LIBS += -L$$LEVELDBDIR/out-static -lleveldb -#INCLUDEPATH += $$LEVELDBDIR/include From 3bb1e4241fc2a161ef30f5e57348a9a520e0dd17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Tue, 6 Dec 2016 00:08:32 +0200 Subject: [PATCH 28/43] removed duplicate implementation of packing / unpacking functions --- carta/cpp/testCache/main.cpp | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index 25ef4e9d..3f4b6a22 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -4,6 +4,7 @@ #include "core/CmdLine.h" #include "core/MainConfig.h" #include "core/Globals.h" +#include "core/Algorithms/cacheUtils.h" #include "CartaLib/IPCache.h" #include #include @@ -32,30 +33,6 @@ genProfile( int x, int y ) return arr; } -QByteArray -vd2qb( const std::vector < double > & vd ) -{ - QByteArray ba; - for ( const double & d : vd ) { - ba.append( (const char *) ( & d ), sizeof( double ) ); - } - return ba; -} - -std::vector < double > -qb2vd( const QByteArray & ba ) -{ - std::vector < double > vd; - if ( ba.size() % sizeof( double ) != 0 ) { - return vd; - } - const char * cptr = ba.constData(); - for ( int i = 0 ; i < ba.size() ; i += sizeof( double ) ) { - vd.push_back( * ( (const double *) ( cptr + i ) ) ); - } - return vd; -} - static std::vector < double > readProfile( int x, int y ) { From a099477509ae972e1d2c230f174137ac860c1bf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Wed, 7 Dec 2016 13:54:17 +0200 Subject: [PATCH 29/43] Change plugin config to specify the full path to the pcache file / directory, not just the parent directory, to make it easier to specify a different location for a test database. --- .../plugins/PCacheLevelDB/PCacheLevelDB.cpp | 18 ++++++++---------- .../cpp/plugins/PCacheLevelDB/PCacheLevelDB.h | 2 +- .../plugins/PCacheSqlite3/PCacheSqlite3.cpp | 19 ++++++++----------- .../cpp/plugins/PCacheSqlite3/PCacheSqlite3.h | 2 +- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp index 48019e92..0adda2af 100644 --- a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp @@ -102,12 +102,10 @@ class LevelDBPCache : public Carta::Lib::IPCache options.create_if_missing = true; - QString fname = dirPath + "/pcache.leveldb"; - - leveldb::Status status = leveldb::DB::Open( options, fname.toStdString(), & db ); + leveldb::Status status = leveldb::DB::Open( options, dirPath.toStdString(), & db ); if ( false == status.ok() ) { - qDebug() << "Unable to open/create database '" << fname.toStdString().c_str() << "':" << status.ToString().c_str(); + qDebug() << "Unable to open/create database '" << dirPath.toStdString().c_str() << "':" << status.ToString().c_str(); return; } @@ -135,13 +133,13 @@ PCacheLevelDBPlugin::handleHook( BaseHook & hookData ) GetPersistentCacheHook & hook = static_cast < GetPersistentCacheHook & > ( hookData ); // if no dbdir was specified, refuse to work :) - if( m_dbDir.isNull()) { + if( m_dbPath.isNull()) { hook.result.reset(); return false; } // try to create the database - hook.result = LevelDBPCache::getCacheSingleton( m_dbDir); + hook.result = LevelDBPCache::getCacheSingleton( m_dbPath); // return true if result is not null return hook.result != nullptr; @@ -159,13 +157,13 @@ PCacheLevelDBPlugin::initialize( const IPlugin::InitInfo & initInfo ) qDebug() << doc.toJson(); // extract the location of the database from carta.config - m_dbDir = initInfo.json.value( "dbDir").toString(); - if( m_dbDir.isNull()) { - qCritical() << "No dbDir specified for PCacheSqlite3 plugin!!!"; + m_dbPath = initInfo.json.value( "dbPath").toString(); + if( m_dbPath.isNull()) { + qCritical() << "No dbPath specified for PCacheSqlite3 plugin!!!"; } else { // convert this to absolute path just in case - m_dbDir = QDir(m_dbDir).absolutePath(); + m_dbPath = QDir(m_dbPath).absolutePath(); } } diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h index f78067aa..ca2c8f5b 100644 --- a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.h @@ -27,5 +27,5 @@ public : private: - QString m_dbDir; + QString m_dbPath; }; diff --git a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp index c6746faf..5a9e1e55 100644 --- a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp +++ b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp @@ -110,14 +110,11 @@ class SqLitePCache : public Carta::Lib::IPCache { m_db = QSqlDatabase::addDatabase( "QSQLITE" ); - QString fname = dirPath + "/pcache.sqlite"; - - // m_db.setDatabaseName( "/scratchSD/test.sqlite" ); - m_db.setDatabaseName( fname ); + m_db.setDatabaseName( dirPath ); bool ok = m_db.open(); if ( ! ok ) { qCritical() << "Could not open sqlite database"; - qCritical() << " - at location:" + fname; + qCritical() << " - at location:" + dirPath; } QSqlQuery query( m_db ); @@ -153,13 +150,13 @@ PCacheSQlite3Plugin::handleHook( BaseHook & hookData ) GetPersistentCacheHook & hook = static_cast < GetPersistentCacheHook & > ( hookData ); // if no dbdir was specified, refuse to work :) - if( m_dbDir.isNull()) { + if( m_dbPath.isNull()) { hook.result.reset(); return false; } // try to create the database - hook.result = SqLitePCache::getCacheSingleton( m_dbDir); + hook.result = SqLitePCache::getCacheSingleton( m_dbPath); // return true if result is not null return hook.result != nullptr; @@ -177,13 +174,13 @@ PCacheSQlite3Plugin::initialize( const IPlugin::InitInfo & initInfo ) qDebug() << doc.toJson(); // extract the location of the database from carta.config - m_dbDir = initInfo.json.value( "dbDir").toString(); - if( m_dbDir.isNull()) { - qCritical() << "No dbDir specified for PCacheSqlite3 plugin!!!"; + m_dbPath = initInfo.json.value( "dbPath").toString(); + if( m_dbPath.isNull()) { + qCritical() << "No dbPath specified for PCacheSqlite3 plugin!!!"; } else { // convert this to absolute path just in case - m_dbDir = QDir(m_dbDir).absolutePath(); + m_dbPath = QDir(m_dbPath).absolutePath(); } } diff --git a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h index a744de02..aa4051d5 100644 --- a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h +++ b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.h @@ -27,5 +27,5 @@ public : private: - QString m_dbDir; + QString m_dbPath; }; From 3ba9bbe91803155045ee7a49192c0d08e4a4617b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Wed, 7 Dec 2016 15:05:50 +0200 Subject: [PATCH 30/43] fixed c&p error in error message --- carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp index 0adda2af..7c5cdaa9 100644 --- a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp @@ -159,7 +159,7 @@ PCacheLevelDBPlugin::initialize( const IPlugin::InitInfo & initInfo ) // extract the location of the database from carta.config m_dbPath = initInfo.json.value( "dbPath").toString(); if( m_dbPath.isNull()) { - qCritical() << "No dbPath specified for PCacheSqlite3 plugin!!!"; + qCritical() << "No dbPath specified for PCacheLevelDB plugin!!!"; } else { // convert this to absolute path just in case From 7878924275470b5ac705e1f496b84534cd618524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Wed, 7 Dec 2016 15:34:06 +0200 Subject: [PATCH 31/43] attempting to add function to config object class to allow config to be overridden by tests --- carta/cpp/core/MainConfig.cpp | 5 +++++ carta/cpp/core/MainConfig.h | 7 ++++++- carta/cpp/testCache/main.cpp | 5 +++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/carta/cpp/core/MainConfig.cpp b/carta/cpp/core/MainConfig.cpp index a1706770..177665b1 100644 --- a/carta/cpp/core/MainConfig.cpp +++ b/carta/cpp/core/MainConfig.cpp @@ -170,6 +170,11 @@ int ParsedInfo::toInt( const QJsonValue& jsonValue, QString& errorMsg ){ return val; } +iterator ParsedInfo::insert(const QString &key, const QJsonValue &value) { + return m_json.insert(key, value); +} + + } // namespace MainConfig diff --git a/carta/cpp/core/MainConfig.h b/carta/cpp/core/MainConfig.h index 880b2887..364d5b1d 100644 --- a/carta/cpp/core/MainConfig.h +++ b/carta/cpp/core/MainConfig.h @@ -78,7 +78,12 @@ class ParsedInfo { * error doing the conversion. */ static int toInt( const QJsonValue& jsonValue, QString& errorMsg ); - + + + /** + * Allows modification of the JSON object by tests. + */ + iterator insert(const QString &key, const QJsonValue &value); protected: diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index 3f4b6a22..60118c17 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -168,6 +168,11 @@ coreMainCPP( QString platformString, int argc, char * * argv ) // ==================== QString configFilePath = cmdLineInfo.configFilePath(); MainConfig::ParsedInfo mainConfig = MainConfig::parse( configFilePath ); + + qDebug() << "++++++++++++++++++ what's the config?" << mainConfig.json(); + mainConfig.insert("disabledPlugins", QJsonValue(QJsonArray())); + qDebug() << "++++++++++++++++++ what's the config?" << mainConfig.json(); + globals.setMainConfig( & mainConfig ); qDebug() << "plugin directories:\n - " + mainConfig.pluginDirectories().join( "\n - " ); From 6ff4ab81918ace118dd5fb815d037ef9d88a4c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Wed, 7 Dec 2016 16:04:27 +0200 Subject: [PATCH 32/43] correct return type --- carta/cpp/core/MainConfig.cpp | 2 +- carta/cpp/core/MainConfig.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/carta/cpp/core/MainConfig.cpp b/carta/cpp/core/MainConfig.cpp index 177665b1..591f1ab1 100644 --- a/carta/cpp/core/MainConfig.cpp +++ b/carta/cpp/core/MainConfig.cpp @@ -170,7 +170,7 @@ int ParsedInfo::toInt( const QJsonValue& jsonValue, QString& errorMsg ){ return val; } -iterator ParsedInfo::insert(const QString &key, const QJsonValue &value) { +QJsonObject::iterator ParsedInfo::insert(const QString &key, const QJsonValue &value) { return m_json.insert(key, value); } diff --git a/carta/cpp/core/MainConfig.h b/carta/cpp/core/MainConfig.h index 364d5b1d..156499e3 100644 --- a/carta/cpp/core/MainConfig.h +++ b/carta/cpp/core/MainConfig.h @@ -83,7 +83,7 @@ class ParsedInfo { /** * Allows modification of the JSON object by tests. */ - iterator insert(const QString &key, const QJsonValue &value); + QJsonObject::iterator insert(const QString &key, const QJsonValue &value); protected: From a906ca656d873edb1612264b0e20a2927739f656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Wed, 7 Dec 2016 16:06:22 +0200 Subject: [PATCH 33/43] missing header --- carta/cpp/testCache/main.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index 60118c17..2a6248d5 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -8,6 +8,7 @@ #include "CartaLib/IPCache.h" #include #include +#include namespace tCache { From b9706804991a8c0d66f3a9503653fbf634f8e367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Wed, 7 Dec 2016 16:36:29 +0200 Subject: [PATCH 34/43] use test database files in test; delete afterwards --- carta/cpp/testCache/main.cpp | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index 2a6248d5..54c80561 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace tCache { @@ -170,9 +171,20 @@ coreMainCPP( QString platformString, int argc, char * * argv ) QString configFilePath = cmdLineInfo.configFilePath(); MainConfig::ParsedInfo mainConfig = MainConfig::parse( configFilePath ); - qDebug() << "++++++++++++++++++ what's the config?" << mainConfig.json(); + // Re-enable all plugins so that all pcache plugins are tested mainConfig.insert("disabledPlugins", QJsonValue(QJsonArray())); - qDebug() << "++++++++++++++++++ what's the config?" << mainConfig.json(); + + // Use test database files + // Hideous, but apparently the only way to modify nested QJSONObject values. + + QJsonObject plugins = mainConfig.json()["plugins"].toObject(); + QJsonObject sqlite3Config = plugins["PCacheSqlite3"].toObject(); + QJsonObject leveldbConfig = plugins["PCacheLevelDB"].toObject(); + sqlite3Config.insert("dbPath", "/tmp/pcache.sqlite.test"); + leveldbConfig.insert("dbPath", "/tmp/pcache.leveldb.test"); + plugins.insert("PCacheSqlite3", QJsonValue(sqlite3Config)); + plugins.insert("PCacheLevelDB", QJsonValue(leveldbConfig)); + mainConfig.insert("plugins", QJsonValue(plugins)); globals.setMainConfig( & mainConfig ); qDebug() << "plugin directories:\n - " + mainConfig.pluginDirectories().join( "\n - " ); @@ -212,6 +224,10 @@ coreMainCPP( QString platformString, int argc, char * * argv ) // give QT control // int res = qapp.exec(); int res = 0; + + // Delete the test database files afterwards + std::remove("/tmp/pcache.sqlite.test"); + std::remove("/tmp/pcache.leveldb.test"); // if we get here, it means we are quitting... qDebug() << "Exiting"; From 78316ba35b769095b7a42f42b9544541e54ecfe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Wed, 7 Dec 2016 16:53:13 +0200 Subject: [PATCH 35/43] leveldb cache file is a directory --- carta/cpp/testCache/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index 54c80561..41d23b84 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -227,7 +227,7 @@ coreMainCPP( QString platformString, int argc, char * * argv ) // Delete the test database files afterwards std::remove("/tmp/pcache.sqlite.test"); - std::remove("/tmp/pcache.leveldb.test"); + std::remove_all("/tmp/pcache.leveldb.test"); // if we get here, it means we are quitting... qDebug() << "Exiting"; From eadc3dffd8732e64a455fa34967543e992430052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Wed, 7 Dec 2016 17:22:05 +0200 Subject: [PATCH 36/43] revert last commit; this function is in Boost and C++17 --- carta/cpp/testCache/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index 41d23b84..54c80561 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -227,7 +227,7 @@ coreMainCPP( QString platformString, int argc, char * * argv ) // Delete the test database files afterwards std::remove("/tmp/pcache.sqlite.test"); - std::remove_all("/tmp/pcache.leveldb.test"); + std::remove("/tmp/pcache.leveldb.test"); // if we get here, it means we are quitting... qDebug() << "Exiting"; From 0efb609d88b5fa68ef55b07106e5ca020dc68620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Wed, 7 Dec 2016 17:37:20 +0200 Subject: [PATCH 37/43] try to delete the contents of the leveldb dir using leveldb's function --- carta/cpp/testCache/main.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index 54c80561..87d4e38d 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -10,6 +10,7 @@ #include #include #include +#include "leveldb/db.h" namespace tCache { @@ -227,6 +228,7 @@ coreMainCPP( QString platformString, int argc, char * * argv ) // Delete the test database files afterwards std::remove("/tmp/pcache.sqlite.test"); + leveldb::DestroyDB("/tmp/pcache.leveldb.test"); std::remove("/tmp/pcache.leveldb.test"); // if we get here, it means we are quitting... From 9b44336aade81f247995fef6e9580edfb3c05697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Wed, 7 Dec 2016 17:40:50 +0200 Subject: [PATCH 38/43] missing parameter --- carta/cpp/testCache/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index 87d4e38d..add5fe43 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -228,7 +228,7 @@ coreMainCPP( QString platformString, int argc, char * * argv ) // Delete the test database files afterwards std::remove("/tmp/pcache.sqlite.test"); - leveldb::DestroyDB("/tmp/pcache.leveldb.test"); + leveldb::DestroyDB("/tmp/pcache.leveldb.test", leveldb::Options()); std::remove("/tmp/pcache.leveldb.test"); // if we get here, it means we are quitting... From dd27811896f6da326781d35bff7393c2b8568769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Wed, 7 Dec 2016 17:45:57 +0200 Subject: [PATCH 39/43] now we have to link to leveldb from the test --- carta/cpp/testCache/testCache.pro | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/carta/cpp/testCache/testCache.pro b/carta/cpp/testCache/testCache.pro index 32eff5b9..c9032c5c 100644 --- a/carta/cpp/testCache/testCache.pro +++ b/carta/cpp/testCache/testCache.pro @@ -13,6 +13,7 @@ RESOURCES = unix: LIBS += -L$$OUT_PWD/../core/ -lcore unix: LIBS += -L$$OUT_PWD/../CartaLib/ -lCartaLib +LIBS += -lleveldb DEPENDPATH += $$PROJECT_ROOT/core DEPENDPATH += $$PROJECT_ROOT/CartaLib @@ -26,6 +27,6 @@ unix:macx { } else{ QMAKE_LFLAGS += '-Wl,-rpath,\'$$QWT_ROOT/lib\'' - LIBS +=-L$$QWT_ROOT/lib -lqwt + LIBS +=-L$$QWT_ROOT/lib -lqwt PRE_TARGETDEPS += $$OUT_PWD/../core/libcore.so } From 2038d03662fba8426b9cc76f4129c64cb3c3bd2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Wed, 7 Dec 2016 18:20:49 +0200 Subject: [PATCH 40/43] Delete all entries instead of deleting db files, which was being done before plugin shutdown. If files are to be deleted, this should be done by whatever is calling the test executables. --- carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp | 14 ++++++++++++-- carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp | 12 ++++++++++-- carta/cpp/testCache/main.cpp | 8 +------- carta/cpp/testCache/testCache.pro | 1 - 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp index 7c5cdaa9..58ea2f73 100644 --- a/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp +++ b/carta/cpp/plugins/PCacheLevelDB/PCacheLevelDB.cpp @@ -35,8 +35,18 @@ class LevelDBPCache : public Carta::Lib::IPCache virtual void deleteAll() override { - // not implemented - } + if ( ! p_db ) { + return; + } + + leveldb::Iterator* it = p_db->NewIterator(p_readOptions); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + auto status = p_db->Delete(p_writeOptions, it->key()); + if ( ! status.ok() ) { + qWarning() << "query delete failed:" << status.ToString().c_str(); + } + } + } // deleteAll virtual bool readEntry( const QByteArray & key, QByteArray & val ) override diff --git a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp index 5a9e1e55..8fb68b54 100644 --- a/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp +++ b/carta/cpp/plugins/PCacheSqlite3/PCacheSqlite3.cpp @@ -34,8 +34,16 @@ class SqLitePCache : public Carta::Lib::IPCache virtual void deleteAll() override { - // not implemented - } + if ( ! m_db.isOpen() ) { + return; + } + QSqlQuery query( m_db ); + query.prepare( "DELETE FROM db;" ); + + if ( ! query.exec() ) { + qWarning() << "query delete failed"; + } + } // deleteAll virtual bool readEntry( const QByteArray & key, QByteArray & val ) override diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index add5fe43..4642f5be 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -9,8 +9,6 @@ #include #include #include -#include -#include "leveldb/db.h" namespace tCache { @@ -216,6 +214,7 @@ coreMainCPP( QString platformString, int argc, char * * argv ) auto lam = [=] ( const Carta::Lib::Hooks::GetPersistentCache::ResultType &res ) { pcache = res; testCache(); + pcache->deleteAll(); }; // call the lambda on every pcache plugin @@ -225,11 +224,6 @@ coreMainCPP( QString platformString, int argc, char * * argv ) // give QT control // int res = qapp.exec(); int res = 0; - - // Delete the test database files afterwards - std::remove("/tmp/pcache.sqlite.test"); - leveldb::DestroyDB("/tmp/pcache.leveldb.test", leveldb::Options()); - std::remove("/tmp/pcache.leveldb.test"); // if we get here, it means we are quitting... qDebug() << "Exiting"; diff --git a/carta/cpp/testCache/testCache.pro b/carta/cpp/testCache/testCache.pro index c9032c5c..3452554e 100644 --- a/carta/cpp/testCache/testCache.pro +++ b/carta/cpp/testCache/testCache.pro @@ -13,7 +13,6 @@ RESOURCES = unix: LIBS += -L$$OUT_PWD/../core/ -lcore unix: LIBS += -L$$OUT_PWD/../CartaLib/ -lCartaLib -LIBS += -lleveldb DEPENDPATH += $$PROJECT_ROOT/core DEPENDPATH += $$PROJECT_ROOT/CartaLib From ff4759f028a54f20e2350cfd74d53cbf5710ee37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Wed, 7 Dec 2016 18:34:42 +0200 Subject: [PATCH 41/43] in the test, delete all data before the test, not after, in case it wasn't cleaned up after a previous failed test --- carta/cpp/testCache/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carta/cpp/testCache/main.cpp b/carta/cpp/testCache/main.cpp index 4642f5be..a51be0fc 100644 --- a/carta/cpp/testCache/main.cpp +++ b/carta/cpp/testCache/main.cpp @@ -213,8 +213,8 @@ coreMainCPP( QString platformString, int argc, char * * argv ) // make a lambda to set the value of pcache and call the tests auto lam = [=] ( const Carta::Lib::Hooks::GetPersistentCache::ResultType &res ) { pcache = res; - testCache(); pcache->deleteAll(); + testCache(); }; // call the lambda on every pcache plugin From 79c2699f8930adf0f8809ca916c84af24cd7b300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Thu, 19 Jan 2017 09:24:25 +0200 Subject: [PATCH 42/43] check if disk cache exists before using it, otherwise ignore it --- carta/cpp/core/Data/Image/DataSource.cpp | 52 ++++++++++++++---------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/carta/cpp/core/Data/Image/DataSource.cpp b/carta/cpp/core/Data/Image/DataSource.cpp index fa7c2966..726877ad 100755 --- a/carta/cpp/core/Data/Image/DataSource.cpp +++ b/carta/cpp/core/Data/Image/DataSource.cpp @@ -65,8 +65,6 @@ DataSource::DataSource() : if ( res.isNull() || ! res.val() ) { qWarning( "Could not find a disk cache plugin." ); m_diskCache = nullptr; - // TODO: convert the existing in-memory cache to a default in-memory cache to use if no plugin is found. - // Do we want to have memory *and* disk cache? Check performance. } else { m_diskCache = res.val(); @@ -364,7 +362,7 @@ std::vector > DataSource::_getIntensityCache( int frameLow qDebug() << "++++++++ found location and intensity in memory cache"; intensities[i] = val; foundCount++; - } else { + } else if (m_diskCache) { // disk cache // Look for the location first QString locationKey = QString("%1/%2/%3/%4/location").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); @@ -457,11 +455,13 @@ std::vector > DataSource::_getIntensityCache( int frameLow m_cachedPercentiles.put( frameLow, frameHigh, intensities[i].first, percentiles[i], intensities[i].second ); - QString locationKey = QString("%1/%2/%3/%4/location").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); - QString intensityKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); + if (m_diskCache) { + QString locationKey = QString("%1/%2/%3/%4/location").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); + QString intensityKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frameLow).arg(frameHigh).arg(percentiles[i]); - m_diskCache->setEntry(locationKey.toUtf8(), i2qb(intensities[i].first), 0); - m_diskCache->setEntry(intensityKey.toUtf8(), d2qb(intensities[i].second), 0); + m_diskCache->setEntry(locationKey.toUtf8(), i2qb(intensities[i].first), 0); + m_diskCache->setEntry(intensityKey.toUtf8(), d2qb(intensities[i].second), 0); + } qDebug() << "++++++++ For percentile" << percentiles[i] << "intensity is" << intensities[i].second << "and location is" << intensities[i].first; } @@ -956,18 +956,23 @@ void DataSource::_updateClips( std::shared_ptrreadEntry(minClipKey.toUtf8(), minClipVal); - bool maxClipInCache = m_diskCache->readEntry(maxClipKey.toUtf8(), maxClipVal); + + bool minClipInCache(0); + bool maxClipInCache(0); + + if (m_diskCache) { + // TODO: check if these are the right frame values and percentile values + QString minClipKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frames[0]).arg(frames.back()).arg(minClipPercentile); + QString maxClipKey = QString("%1/%2/%3/%4/intensity").arg(m_fileName).arg(frames[0]).arg(frames.back()).arg(maxClipPercentile); + + qDebug() << "++++++++ minClipKey" << minClipKey.toUtf8() << "maxClipKey" << maxClipKey.toUtf8(); + + QByteArray minClipVal; + QByteArray maxClipVal; + + minClipInCache = m_diskCache->readEntry(minClipKey.toUtf8(), minClipVal); + maxClipInCache = m_diskCache->readEntry(maxClipKey.toUtf8(), maxClipVal); + } if (minClipInCache && maxClipInCache) { clips.clear(); @@ -977,9 +982,12 @@ void DataSource::_updateClips( std::shared_ptrsetEntry( minClipKey.toUtf8(), d2qb(clips[0]), 0); - m_diskCache->setEntry( maxClipKey.toUtf8(), d2qb(clips[1]), 0); - qDebug() << "++++++++ calculated clips and put in cache"; + + if (m_diskCache) { + m_diskCache->setEntry( minClipKey.toUtf8(), d2qb(clips[0]), 0); + m_diskCache->setEntry( maxClipKey.toUtf8(), d2qb(clips[1]), 0); + qDebug() << "++++++++ calculated clips and put in cache"; + } } qDebug() << "++++++++ clips are" << clips[0] << "and" << clips[1]; From aa2a23f319757f4269dc0fe84be4470ceb178565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrianna=20Pi=C5=84ska?= Date: Thu, 19 Jan 2017 09:27:39 +0200 Subject: [PATCH 43/43] fixed variable scope --- carta/cpp/core/Data/Image/DataSource.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/carta/cpp/core/Data/Image/DataSource.cpp b/carta/cpp/core/Data/Image/DataSource.cpp index 726877ad..5513b7a8 100755 --- a/carta/cpp/core/Data/Image/DataSource.cpp +++ b/carta/cpp/core/Data/Image/DataSource.cpp @@ -960,16 +960,16 @@ void DataSource::_updateClips( std::shared_ptrreadEntry(minClipKey.toUtf8(), minClipVal); maxClipInCache = m_diskCache->readEntry(maxClipKey.toUtf8(), maxClipVal); }